From 896e2a2f6d74a09633c36dbae26d7362a978f891 Mon Sep 17 00:00:00 2001 From: "Jeffrey C. Ollie" Date: Sun, 13 Aug 2023 17:25:49 -0500 Subject: [PATCH] actually use healthchecks --- flake.nix | 215 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 137 insertions(+), 78 deletions(-) diff --git a/flake.nix b/flake.nix index 06420d5..ac24f8a 100644 --- a/flake.nix +++ b/flake.nix @@ -230,31 +230,27 @@ }; default = { }; }; + healthcheck = lib.options.mkOption { + type = lib.types.submodule { + options = { + enable = lib.options.mkEnableOption "use healthchecks"; + fullBackupPingURL = lib.options.mkOption { + type = lib.types.str; + }; + differentialBackupPingURL = lib.options.mkOption { + type = lib.types.str; + }; + incrementalBackupPingURL = lib.options.mkOption { + type = lib.types.str; + }; + }; + }; + }; }; + default = { }; }; - default = { }; - }; - healthcheck = lib.options.mkOption { - type = lib.types.submodule { - options = { - enable = lib.options.mkEnableOption "use healthcheck"; - api_url = lib.options.mkOption { - type = lib.types.str; - }; - api_key = lib.options.mkOption { - type = lib.types.str; - }; - timeout = lib.options.mkOption { - type = lib.types.int; - default = 86400; - }; - grace = lib.options.mkOption { - type = lib.types.int; - default = 14400; - }; - }; - }; + default = { }; }; @@ -627,6 +623,7 @@ ] else ""; + curl = "${pkgs.curl}/bin/curl --silent --show-error --max-time 10 --retry 5"; in { description = "PostgreSQL Server"; @@ -643,7 +640,15 @@ ); path = [ postgresql - ]; + ] ++ ( + if cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary") + then + [ + pgbackrest + ] + else + [ ] + ); preStart = if (!cfg.replication.enable || cfg.replication.role == "primary") @@ -815,25 +820,43 @@ } ); - systemd.services.postgresql-backup-full = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) { - description = "PostgreSQL Full Backup"; - requires = [ "postgresql.service" ]; - script = '' - while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null - do - sleep 0.1 - done + systemd.services.postgresql-backup-full = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) ( + let + hcStart = + if cfg.backup.healthchecks.enable then + "${curl} ${cfg.backup.healthchecks.fullBackupPingURL}/start" + else + ""; + hcStop = + if cfg.backup.healthchecks.enable then + "${curl} ${cfg.backup.healthchecks.fullBackupPingURL}" + else + ""; + in + { + description = "PostgreSQL Full Backup"; + requires = [ "postgresql.service" ]; + script = '' + ${hcStart} - ${pgbackrest}/bin/pgbackrest --type=full --start-fast --stop-auto --delta backup - ''; - environment = pgbackrestEnvironment; - serviceConfig = { - Type = "oneshot"; - User = "postgres"; - Group = "postgres"; - TimeoutSec = 3600; - }; - }; + while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null + do + sleep 0.1 + done + + ${pgbackrest}/bin/pgbackrest --type=full --start-fast --stop-auto --delta backup + + ${hcStop} + ''; + environment = pgbackrestEnvironment; + serviceConfig = { + Type = "oneshot"; + User = "postgres"; + Group = "postgres"; + TimeoutSec = 3600; + }; + } + ); systemd.timers.postgresql-backup-full = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) { description = "PostgreSQL Full Backup"; @@ -844,25 +867,43 @@ wantedBy = [ "multi-user.target" ]; }; - systemd.services.postgresql-backup-diff = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) { - description = "PostgreSQL Differential Backup"; - requires = [ "postgresql.service" ]; - script = '' - while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null - do - sleep 0.1 - done + systemd.services.postgresql-backup-diff = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) ( + let + hcStart = + if cfg.backup.healthchecks.enable then + "${curl} ${cfg.backup.healthchecks.differentialBackupPingURL}/start" + else + ""; + hcStop = + if cfg.backup.healthchecks.enable then + "${curl} ${cfg.backup.healthchecks.differentialBackupPingURL}" + else + ""; + in + { + description = "PostgreSQL Differential Backup"; + requires = [ "postgresql.service" ]; + script = '' + ${hcStart} - ${pgbackrest}/bin/pgbackrest --type=diff --start-fast --stop-auto --delta backup - ''; - environment = pgbackrestEnvironment; - serviceConfig = { - Type = "oneshot"; - User = "postgres"; - Group = "postgres"; - TimeoutSec = 3600; - }; - }; + while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null + do + sleep 0.1 + done + + ${pgbackrest}/bin/pgbackrest --type=diff --start-fast --stop-auto --delta backup + + ${hcStop} + ''; + environment = pgbackrestEnvironment; + serviceConfig = { + Type = "oneshot"; + User = "postgres"; + Group = "postgres"; + TimeoutSec = 3600; + }; + } + ); systemd.timers.postgresql-backup-diff = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) { description = "PostgreSQL Differential Backup"; @@ -873,30 +914,48 @@ wantedBy = [ "multi-user.target" ]; }; - systemd.services.postgresql-backup-incr = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) { - description = "PostgreSQL Incremental Backup"; - requires = [ "postgresql.service" ]; - script = '' - while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null - do - sleep 0.1 - done + systemd.services.postgresql-backup-incr = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) ( + let + hcStart = + if cfg.backup.healthchecks.enable then + "${curl} ${cfg.backup.healthchecks.incrementalBackupPingURL}/start" + else + ""; + hcStop = + if cfg.backup.healthchecks.enable then + "${curl} ${cfg.backup.healthchecks.incrementalBackupPingURL}" + else + ""; + in + { + description = "PostgreSQL Incremental Backup"; + requires = [ "postgresql.service" ]; + script = '' + ${hcStart} - ${pgbackrest}/bin/pgbackrest --type=incr --start-fast --stop-auto --delta backup - ''; - environment = pgbackrestEnvironment; - serviceConfig = { - Type = "oneshot"; - User = "postgres"; - Group = "postgres"; - TimeoutSec = 3600; - }; - }; + while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null + do + sleep 0.1 + done + + ${pgbackrest}/bin/pgbackrest --type=incr --start-fast --stop-auto --delta backup + + ${hsStop} + ''; + environment = pgbackrestEnvironment; + serviceConfig = { + Type = "oneshot"; + User = "postgres"; + Group = "postgres"; + TimeoutSec = 3600; + }; + } + ); systemd.timers.postgresql-backup-incr = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) { description = "PostgreSQL Incremental Backup"; timerConfig = { - OnCalendar = "*-*-* 06,10,14,18,22:00:00"; + OnCalendar = "*-*-* 00,06,10,14,18,22:00:00"; RandomizedDelaySec = "5m"; }; wantedBy = [ "multi-user.target" ];