actually use healthchecks

This commit is contained in:
Jeffrey C. Ollie 2023-08-13 17:25:49 -05:00
parent cb25aef9d7
commit 896e2a2f6d
Signed by: jeff
GPG key ID: 6F86035A6D97044E

215
flake.nix
View file

@ -230,31 +230,27 @@
}; };
default = { }; default = { };
}; };
healthcheck = lib.options.mkOption {
type = lib.types.submodule {
options = {
enable = lib.options.mkEnableOption "use healthchecks";
fullBackupPingURL = lib.options.mkOption {
type = lib.types.str;
};
differentialBackupPingURL = lib.options.mkOption {
type = lib.types.str;
};
incrementalBackupPingURL = lib.options.mkOption {
type = lib.types.str;
};
};
};
};
}; };
default = { };
}; };
default = { };
};
healthcheck = lib.options.mkOption {
type = lib.types.submodule {
options = {
enable = lib.options.mkEnableOption "use healthcheck";
api_url = lib.options.mkOption {
type = lib.types.str;
};
api_key = lib.options.mkOption {
type = lib.types.str;
};
timeout = lib.options.mkOption {
type = lib.types.int;
default = 86400;
};
grace = lib.options.mkOption {
type = lib.types.int;
default = 14400;
};
};
};
default = { }; default = { };
}; };
@ -627,6 +623,7 @@
] ]
else else
""; "";
curl = "${pkgs.curl}/bin/curl --silent --show-error --max-time 10 --retry 5";
in in
{ {
description = "PostgreSQL Server"; description = "PostgreSQL Server";
@ -643,7 +640,15 @@
); );
path = [ path = [
postgresql postgresql
]; ] ++ (
if cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")
then
[
pgbackrest
]
else
[ ]
);
preStart = preStart =
if (!cfg.replication.enable || cfg.replication.role == "primary") if (!cfg.replication.enable || cfg.replication.role == "primary")
@ -815,25 +820,43 @@
} }
); );
systemd.services.postgresql-backup-full = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) { systemd.services.postgresql-backup-full = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) (
description = "PostgreSQL Full Backup"; let
requires = [ "postgresql.service" ]; hcStart =
script = '' if cfg.backup.healthchecks.enable then
while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null "${curl} ${cfg.backup.healthchecks.fullBackupPingURL}/start"
do else
sleep 0.1 "";
done hcStop =
if cfg.backup.healthchecks.enable then
"${curl} ${cfg.backup.healthchecks.fullBackupPingURL}"
else
"";
in
{
description = "PostgreSQL Full Backup";
requires = [ "postgresql.service" ];
script = ''
${hcStart}
${pgbackrest}/bin/pgbackrest --type=full --start-fast --stop-auto --delta backup while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null
''; do
environment = pgbackrestEnvironment; sleep 0.1
serviceConfig = { done
Type = "oneshot";
User = "postgres"; ${pgbackrest}/bin/pgbackrest --type=full --start-fast --stop-auto --delta backup
Group = "postgres";
TimeoutSec = 3600; ${hcStop}
}; '';
}; environment = pgbackrestEnvironment;
serviceConfig = {
Type = "oneshot";
User = "postgres";
Group = "postgres";
TimeoutSec = 3600;
};
}
);
systemd.timers.postgresql-backup-full = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) { systemd.timers.postgresql-backup-full = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) {
description = "PostgreSQL Full Backup"; description = "PostgreSQL Full Backup";
@ -844,25 +867,43 @@
wantedBy = [ "multi-user.target" ]; wantedBy = [ "multi-user.target" ];
}; };
systemd.services.postgresql-backup-diff = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) { systemd.services.postgresql-backup-diff = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) (
description = "PostgreSQL Differential Backup"; let
requires = [ "postgresql.service" ]; hcStart =
script = '' if cfg.backup.healthchecks.enable then
while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null "${curl} ${cfg.backup.healthchecks.differentialBackupPingURL}/start"
do else
sleep 0.1 "";
done hcStop =
if cfg.backup.healthchecks.enable then
"${curl} ${cfg.backup.healthchecks.differentialBackupPingURL}"
else
"";
in
{
description = "PostgreSQL Differential Backup";
requires = [ "postgresql.service" ];
script = ''
${hcStart}
${pgbackrest}/bin/pgbackrest --type=diff --start-fast --stop-auto --delta backup while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null
''; do
environment = pgbackrestEnvironment; sleep 0.1
serviceConfig = { done
Type = "oneshot";
User = "postgres"; ${pgbackrest}/bin/pgbackrest --type=diff --start-fast --stop-auto --delta backup
Group = "postgres";
TimeoutSec = 3600; ${hcStop}
}; '';
}; environment = pgbackrestEnvironment;
serviceConfig = {
Type = "oneshot";
User = "postgres";
Group = "postgres";
TimeoutSec = 3600;
};
}
);
systemd.timers.postgresql-backup-diff = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) { systemd.timers.postgresql-backup-diff = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) {
description = "PostgreSQL Differential Backup"; description = "PostgreSQL Differential Backup";
@ -873,30 +914,48 @@
wantedBy = [ "multi-user.target" ]; wantedBy = [ "multi-user.target" ];
}; };
systemd.services.postgresql-backup-incr = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) { systemd.services.postgresql-backup-incr = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) (
description = "PostgreSQL Incremental Backup"; let
requires = [ "postgresql.service" ]; hcStart =
script = '' if cfg.backup.healthchecks.enable then
while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null "${curl} ${cfg.backup.healthchecks.incrementalBackupPingURL}/start"
do else
sleep 0.1 "";
done hcStop =
if cfg.backup.healthchecks.enable then
"${curl} ${cfg.backup.healthchecks.incrementalBackupPingURL}"
else
"";
in
{
description = "PostgreSQL Incremental Backup";
requires = [ "postgresql.service" ];
script = ''
${hcStart}
${pgbackrest}/bin/pgbackrest --type=incr --start-fast --stop-auto --delta backup while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null
''; do
environment = pgbackrestEnvironment; sleep 0.1
serviceConfig = { done
Type = "oneshot";
User = "postgres"; ${pgbackrest}/bin/pgbackrest --type=incr --start-fast --stop-auto --delta backup
Group = "postgres";
TimeoutSec = 3600; ${hsStop}
}; '';
}; environment = pgbackrestEnvironment;
serviceConfig = {
Type = "oneshot";
User = "postgres";
Group = "postgres";
TimeoutSec = 3600;
};
}
);
systemd.timers.postgresql-backup-incr = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) { systemd.timers.postgresql-backup-incr = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) {
description = "PostgreSQL Incremental Backup"; description = "PostgreSQL Incremental Backup";
timerConfig = { timerConfig = {
OnCalendar = "*-*-* 06,10,14,18,22:00:00"; OnCalendar = "*-*-* 00,06,10,14,18,22:00:00";
RandomizedDelaySec = "5m"; RandomizedDelaySec = "5m";
}; };
wantedBy = [ "multi-user.target" ]; wantedBy = [ "multi-user.target" ];