actually use healthchecks

This commit is contained in:
Jeffrey C. Ollie 2023-08-13 17:25:49 -05:00
parent cb25aef9d7
commit 896e2a2f6d
Signed by: jeff
GPG key ID: 6F86035A6D97044E

215
flake.nix
View file

@ -230,31 +230,27 @@
};
default = { };
};
healthcheck = lib.options.mkOption {
type = lib.types.submodule {
options = {
enable = lib.options.mkEnableOption "use healthchecks";
fullBackupPingURL = lib.options.mkOption {
type = lib.types.str;
};
differentialBackupPingURL = lib.options.mkOption {
type = lib.types.str;
};
incrementalBackupPingURL = lib.options.mkOption {
type = lib.types.str;
};
};
};
};
};
default = { };
};
default = { };
};
healthcheck = lib.options.mkOption {
type = lib.types.submodule {
options = {
enable = lib.options.mkEnableOption "use healthcheck";
api_url = lib.options.mkOption {
type = lib.types.str;
};
api_key = lib.options.mkOption {
type = lib.types.str;
};
timeout = lib.options.mkOption {
type = lib.types.int;
default = 86400;
};
grace = lib.options.mkOption {
type = lib.types.int;
default = 14400;
};
};
};
default = { };
};
@ -627,6 +623,7 @@
]
else
"";
curl = "${pkgs.curl}/bin/curl --silent --show-error --max-time 10 --retry 5";
in
{
description = "PostgreSQL Server";
@ -643,7 +640,15 @@
);
path = [
postgresql
];
] ++ (
if cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")
then
[
pgbackrest
]
else
[ ]
);
preStart =
if (!cfg.replication.enable || cfg.replication.role == "primary")
@ -815,25 +820,43 @@
}
);
systemd.services.postgresql-backup-full = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) {
description = "PostgreSQL Full Backup";
requires = [ "postgresql.service" ];
script = ''
while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null
do
sleep 0.1
done
systemd.services.postgresql-backup-full = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) (
let
hcStart =
if cfg.backup.healthchecks.enable then
"${curl} ${cfg.backup.healthchecks.fullBackupPingURL}/start"
else
"";
hcStop =
if cfg.backup.healthchecks.enable then
"${curl} ${cfg.backup.healthchecks.fullBackupPingURL}"
else
"";
in
{
description = "PostgreSQL Full Backup";
requires = [ "postgresql.service" ];
script = ''
${hcStart}
${pgbackrest}/bin/pgbackrest --type=full --start-fast --stop-auto --delta backup
'';
environment = pgbackrestEnvironment;
serviceConfig = {
Type = "oneshot";
User = "postgres";
Group = "postgres";
TimeoutSec = 3600;
};
};
while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null
do
sleep 0.1
done
${pgbackrest}/bin/pgbackrest --type=full --start-fast --stop-auto --delta backup
${hcStop}
'';
environment = pgbackrestEnvironment;
serviceConfig = {
Type = "oneshot";
User = "postgres";
Group = "postgres";
TimeoutSec = 3600;
};
}
);
systemd.timers.postgresql-backup-full = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) {
description = "PostgreSQL Full Backup";
@ -844,25 +867,43 @@
wantedBy = [ "multi-user.target" ];
};
systemd.services.postgresql-backup-diff = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) {
description = "PostgreSQL Differential Backup";
requires = [ "postgresql.service" ];
script = ''
while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null
do
sleep 0.1
done
systemd.services.postgresql-backup-diff = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) (
let
hcStart =
if cfg.backup.healthchecks.enable then
"${curl} ${cfg.backup.healthchecks.differentialBackupPingURL}/start"
else
"";
hcStop =
if cfg.backup.healthchecks.enable then
"${curl} ${cfg.backup.healthchecks.differentialBackupPingURL}"
else
"";
in
{
description = "PostgreSQL Differential Backup";
requires = [ "postgresql.service" ];
script = ''
${hcStart}
${pgbackrest}/bin/pgbackrest --type=diff --start-fast --stop-auto --delta backup
'';
environment = pgbackrestEnvironment;
serviceConfig = {
Type = "oneshot";
User = "postgres";
Group = "postgres";
TimeoutSec = 3600;
};
};
while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null
do
sleep 0.1
done
${pgbackrest}/bin/pgbackrest --type=diff --start-fast --stop-auto --delta backup
${hcStop}
'';
environment = pgbackrestEnvironment;
serviceConfig = {
Type = "oneshot";
User = "postgres";
Group = "postgres";
TimeoutSec = 3600;
};
}
);
systemd.timers.postgresql-backup-diff = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) {
description = "PostgreSQL Differential Backup";
@ -873,30 +914,48 @@
wantedBy = [ "multi-user.target" ];
};
systemd.services.postgresql-backup-incr = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) {
description = "PostgreSQL Incremental Backup";
requires = [ "postgresql.service" ];
script = ''
while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null
do
sleep 0.1
done
systemd.services.postgresql-backup-incr = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) (
let
hcStart =
if cfg.backup.healthchecks.enable then
"${curl} ${cfg.backup.healthchecks.incrementalBackupPingURL}/start"
else
"";
hcStop =
if cfg.backup.healthchecks.enable then
"${curl} ${cfg.backup.healthchecks.incrementalBackupPingURL}"
else
"";
in
{
description = "PostgreSQL Incremental Backup";
requires = [ "postgresql.service" ];
script = ''
${hcStart}
${pgbackrest}/bin/pgbackrest --type=incr --start-fast --stop-auto --delta backup
'';
environment = pgbackrestEnvironment;
serviceConfig = {
Type = "oneshot";
User = "postgres";
Group = "postgres";
TimeoutSec = 3600;
};
};
while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null
do
sleep 0.1
done
${pgbackrest}/bin/pgbackrest --type=incr --start-fast --stop-auto --delta backup
${hsStop}
'';
environment = pgbackrestEnvironment;
serviceConfig = {
Type = "oneshot";
User = "postgres";
Group = "postgres";
TimeoutSec = 3600;
};
}
);
systemd.timers.postgresql-backup-incr = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) {
description = "PostgreSQL Incremental Backup";
timerConfig = {
OnCalendar = "*-*-* 06,10,14,18,22:00:00";
OnCalendar = "*-*-* 00,06,10,14,18,22:00:00";
RandomizedDelaySec = "5m";
};
wantedBy = [ "multi-user.target" ];