From 9d3a72de1f9b4ab44dd2bbdf661ce9638e177a30 Mon Sep 17 00:00:00 2001 From: "Jeffrey C. Ollie" Date: Sun, 13 Aug 2023 11:02:38 -0500 Subject: [PATCH] move passwords/keys to files so that they can be encrypted --- flake.nix | 294 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 196 insertions(+), 98 deletions(-) diff --git a/flake.nix b/flake.nix index 0ace9c4..e52aa76 100644 --- a/flake.nix +++ b/flake.nix @@ -155,14 +155,16 @@ cipher = lib.options.mkOption { type = lib.types.submodule { options = { - password = lib.options.mkOption { - type = lib.types.str; + passwordFile = lib.options.mkOption { + type = lib.types.path; + description = "Path to file containing encryption password."; }; type = lib.options.mkOption { type = lib.types.enum [ "aes-256-cbc" ]; default = "aes-256-cbc"; + description = "Type of encryption to use."; }; }; }; @@ -176,15 +178,36 @@ "azure" "b2" ]; + description = "The type of storage used for backups."; + }; + azure = lib.options.mkOption { + type = lib.types.submodule { + options = { + accountName = lib.options.mkOption { + type = lib.types.str; + }; + accountKeyFile = lib.options.mkOption { + type = lib.types.path; + }; + container = lib.options.mkOption { + type = lib.types.str; + }; + path = lib.options.mkOption { + type = lib.types.str; + }; + }; + }; + default = { }; + description = "Options for connecting to Azure Blob storage"; }; b2 = lib.options.mkOption { type = lib.types.submodule { options = { - account_id = lib.options.mkOption { + accountId = lib.options.mkOption { type = lib.types.str; }; - account_key = lib.options.mkOption { - type = lib.types.str; + accountKeyFile = lib.options.mkOption { + type = lib.types.path; }; endpoint = lib.options.mkOption { type = lib.types.str; @@ -201,25 +224,7 @@ }; }; default = { }; - }; - azure = lib.options.mkOption { - type = lib.types.submodule { - options = { - account_name = lib.options.mkOption { - type = lib.types.str; - }; - account_key = lib.options.mkOption { - type = lib.types.str; - }; - container = lib.options.mkOption { - type = lib.types.str; - }; - path = lib.options.mkOption { - type = lib.types.str; - }; - }; - }; - default = { }; + description = "Options for connection to BackBlaze B2"; }; }; }; @@ -262,6 +267,7 @@ }; password = lib.options.mkOption { type = lib.types.addCheck lib.types.str (x: lib.hasPrefix "SCRAM-SHA-256$" x); + description = "Encrypted password for the database account."; }; }; } @@ -326,66 +332,48 @@ ] ); - rcloneConfig = pkgs.writeTextFile { - name = "rclone.conf"; - text = { - "b2" = '' - [b2] - type = b2 - account = ${cfg.backup.storage.b2.account_id} - key = ${cfg.backup.storage.b2.account_key} - ''; - "azure" = '' - [azure] - type = azureblob - account = ${cfg.backup.storage.azure.account_name} - key = ${cfg.backup.storage.azure.account_key} - ''; - }.${cfg.backup.storage.type}; - }; + # rcloneConfig = pkgs.writeTextFile { + # name = "rclone.conf"; + # text = { + # "b2" = '' + # [b2] + # type = b2 + # account = ${cfg.backup.storage.b2.accountId} + # key = ${cfg.backup.storage.b2.accountKey} + # ''; + # "azure" = '' + # [azure] + # type = azureblob + # account = ${cfg.backup.storage.azure.accountName} + # key = ${cfg.backup.storage.azure.accountKey} + # ''; + # }.${cfg.backup.storage.type}; + # }; + + rcloneEnvironment = { + RCLONE_CONFIG = "/dev/null"; + } // ( + { + "azure" = { + RCLONE_CONFIG_AZURE_TYPE = "azureblob"; + RCLONE_CONFIG_AZURE_ACCOUNT = cfg.backup.storage.azure.accountName; + }; + "b2" = { + RCLONE_CONFIG_B2_TYPE = "b2"; + RCLONE_CONFIG_B2_ACCOUNT = cfg.backup.storage.b2.accountId; + }; + }.${cfg.backup.storage.type} + ); + rcloneEnvironmentFiles = { + "azure" = { + RCLONE_CONFIG_AZURE_KEY = cfg.backup.storage.azure.accountKeyFile; + }; + "b2" = { + RCLONE_CONFIG_B2_KEY = cfg.backup.storage.b2.accountKeyFile; + }; + }.${cfg.backup.storage.type}; rclone = lib.mkIf (cfg.backup.enable) ( - pkgs.writeShellScriptBin "rclone" '' - exec ${pkgs.rclone}/bin/rclone --config ${rcloneConfig} "''$@" - '' - ); - - rootDir = "/var/lib/postgresql"; - dataDir = "${rootDir}/15"; - - pgbackrestEnvironment = - { - PGBACKREST_LOG_LEVEL_CONSOLE = cfg.backup.log.console; - PGBACKREST_LOG_LEVEL_FILE = cfg.backup.log.file; - PGBACKREST_LOG_LEVEL_STDERR = cfg.backup.log.stderr; - PGBACKREST_PG1_PATH = dataDir; - PGBACKREST_PROCESS_MAX = "${builtins.toString cfg.backup.processMax}"; - PGBACKREST_REPO1_CIPHER_PASS = cfg.backup.cipher.password; - PGBACKREST_REPO1_CIPHER_TYPE = cfg.backup.cipher.type; - PGBACKREST_REPO1_RETENTION_FULL = "14"; - PGBACKREST_REPO1_RETENTION_FULL_TYPE = "time"; - PGBACKREST_STANZA = "${config.networking.hostName}.${config.networking.domain}"; - } // ( - { - "azure" = { - PGBACKREST_REPO1_AZURE_ACCOUNT = cfg.backup.storage.azure.account_name; - PGBACKREST_REPO1_AZURE_CONTAINER = cfg.backup.storage.azure.container; - PGBACKREST_REPO1_AZURE_KEY = cfg.backup.storage.azure.account_key; - PGBACKREST_REPO1_PATH = cfg.backup.storage.azure.path; - PGBACKREST_REPO1_TYPE = "azure"; - }; - "b2" = { - PGBACKREST_REPO1_PATH = cfg.backup.storage.b2.path; - PGBACKREST_REPO1_S3_BUCKET = cfg.backup.storage.b2.bucket; - PGBACKREST_REPO1_S3_ENDPOINT = cfg.backup.storage.b2.endpoint; - PGBACKREST_REPO1_S3_KEY = cfg.backup.storage.b2.account_id; - PGBACKREST_REPO1_S3_KEY_SECRET = cfg.backup.storage.b2.account_key; - PGBACKREST_REPO1_S3_REGION = cfg.backup.storage.b2.region; - PGBACKREST_REPO1_TYPE = "s3"; - }; - }.${cfg.backup.storage.type} - ); - pgbackrest = lib.mkIf (cfg.backup.enable) ( let environment = lib.concatStringsSep "\n" ( lib.mapAttrsToList @@ -393,16 +381,120 @@ n: v: ''export ${n}="${builtins.toString v}"'' ) - pgbackrestEnvironment + rcloneEnvironment + ); + environmentFiles = lib.concatStringsSep "\n" ( + lib.mapAttrsToList + ( + n: v: + ''export ${n}=''$(<${v})'' + ) + rcloneEnvironmentFiles ); in - pkgs.writeShellScriptBin "pgbackrest" '' + pkgs.writeShellScriptBin "rclone" '' ${environment} - exec ${pkgs.pgbackrest}/bin/pgbackrest "''$@" + ${environmentFiles} + exec ${pkgs.rclone}/bin/rclone "''$@" '' ); + + rootDir = "/var/lib/postgresql"; + dataDir = "${rootDir}/15"; + + pgbackrestEnvironment = { + PGBACKREST_LOG_LEVEL_CONSOLE = cfg.backup.log.console; + PGBACKREST_LOG_LEVEL_FILE = cfg.backup.log.file; + PGBACKREST_LOG_LEVEL_STDERR = cfg.backup.log.stderr; + PGBACKREST_PG1_PATH = dataDir; + PGBACKREST_PROCESS_MAX = "${builtins.toString cfg.backup.processMax}"; + PGBACKREST_REPO1_CIPHER_TYPE = cfg.backup.cipher.type; + PGBACKREST_REPO1_RETENTION_FULL = "14"; + PGBACKREST_REPO1_RETENTION_FULL_TYPE = "time"; + PGBACKREST_STANZA = "${config.networking.hostName}.${config.networking.domain}"; + } // ( + { + "azure" = { + PGBACKREST_REPO1_AZURE_ACCOUNT = cfg.backup.storage.azure.accountName; + PGBACKREST_REPO1_AZURE_CONTAINER = cfg.backup.storage.azure.container; + PGBACKREST_REPO1_PATH = cfg.backup.storage.azure.path; + PGBACKREST_REPO1_TYPE = "azure"; + }; + "b2" = { + PGBACKREST_REPO1_PATH = cfg.backup.storage.b2.path; + PGBACKREST_REPO1_S3_BUCKET = cfg.backup.storage.b2.bucket; + PGBACKREST_REPO1_S3_ENDPOINT = cfg.backup.storage.b2.endpoint; + PGBACKREST_REPO1_S3_KEY = cfg.backup.storage.b2.accountId; + PGBACKREST_REPO1_S3_REGION = cfg.backup.storage.b2.region; + PGBACKREST_REPO1_TYPE = "s3"; + }; + }.${cfg.backup.storage.type} + ); + + pgbackrestEnvironmentFiles = { + PGBACKREST_REPO1_CIPHER_PASS = cfg.backup.cipher.passwordFile; + } // { + "azure" = { + PGBACKREST_REPO1_AZURE_KEY = cfg.backup.storage.azure.accountKeyFile; + }; + "b2" = { + PGBACKREST_REPO1_S3_KEY_SECRET = cfg.backup.storage.b2.accountKeyFile; + }; + }.${cfg.backup.storage.type}; + + pgbackrest = + if (cfg.backup.enable) + then + ( + let + environment = lib.concatStringsSep "\n" ( + lib.mapAttrsToList + ( + n: v: + ''export ${n}="${builtins.toString v}"'' + ) + pgbackrestEnvironment + ); + environmentFiles = lib.concatStringsSep "\n" ( + lib.mapAttrsToList + ( + n: v: + ''export ${n}=''$(<${v})'' + ) + pgbackrestEnvironmentFiles + ); + in + pkgs.writeShellScriptBin "pgbackrest" '' + ${environment} + ${environmentFiles} + exec ${pkgs.pgbackrest}/bin/pgbackrest "''$@" + '' + ) + else + { }; + # pgbackrestnu = lib.mkIf (cfg.backup.enable) ( + # let + # environment = writeText "pgbackrest-environment.json" (builtins.toJSON pgbackrestEnvironment); + # in + # pkgs.writeScriptBin "pgbackrest" '' + # #!${pkgs.nushell}/bin/nu + + # def main [...args: string] { + # load-env (open ${data}) + + # exec ${pkgs.pgbackrest}/bin/pgbackrest $args + # } + # '' + # ); in lib.mkIf cfg.enable { + assertions = [ + # { + # assertion = !(cfg.backup.storage.b2.accountKey != null && cfg.backup.storage.b2.accountKeyFile != null); + # message = "cannot set both accountKey and accountKeyFile on B2 storage"; + # } + ]; + environment.systemPackages = [ postgresql pgbackrest @@ -447,7 +539,7 @@ default root postgres default postgres postgres ''; - archiveCommand = "${pkgs.pgbackrest}/bin/pgbackrest archive-push %p"; + archiveCommand = "${pgbackrest}/bin/pgbackrest archive-push %p"; settings = { bgwriter_flush_after = "512kB"; checkpoint_flush_after = "256kB"; @@ -506,16 +598,28 @@ if (cfg.backup.enable) then '' - init=''$(${pkgs.pgbackrest}/bin/pgbackrest info --output=json | ${pkgs.jq}/bin/jq '.[0].status.code == 0') + init=''$(${pgbackrest}/bin/pgbackrest info --output=json | ${pkgs.jq}/bin/jq '.[0].status.code == 0') if [ "$init" != "true" ] then ${postgresql}/bin/pg_ctl -o "-c ssl=off -c listen_addresses=''' -p 55432" -w start - ${pkgs.pgbackrest}/bin/pgbackrest --pg1-port=55432 stanza-create + ${pgbackrest}/bin/pgbackrest --pg1-port=55432 stanza-create ${postgresql}/bin/pg_ctl -m fast -w stop fi '' else ""; + pgpass = + if (cfg.replication.enable && cfg.replication.role == "replica") + then + lib.concatStringsSep ":" [ + cfg.replication.primary.hostname + (builtins.toString cfg.replication.primary.port) + "replication" + cfg.replication.username + ''''$(<${cfg.replication.passwordFile}'' + ] + else + ""; in { description = "PostgreSQL Server"; @@ -524,11 +628,6 @@ environment = { PGDATA = dataDir; } // ( - if (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) - then - pgbackrestEnvironment - else { } - ) // ( if (cfg.replication.enable && cfg.replication.role == "replica") then { PGPASSFILE = "${rootDir}/.pgpass"; @@ -563,8 +662,7 @@ '' else '' - read -r password < ${cfg.replication.passwordFile} - (umask 077; echo "${cfg.replication.primary.hostname}:${builtins.toString cfg.replication.primary.port}:replication:${cfg.replication.username}:''${password}" > ${rootDir}/.pgpass) + (umask 077; echo "${pgpass}" > ${rootDir}/.pgpass) chmod 0600 ${rootDir}/.pgpass if [ ! -s "${dataDir}/PG_VERSION" ] @@ -720,7 +818,7 @@ sleep 0.1 done - ${pkgs.pgbackrest}/bin/pgbackrest --type=full --start-fast --stop-auto --delta backup + ${pgbackrest}/bin/pgbackrest --type=full --start-fast --stop-auto --delta backup ''; environment = pgbackrestEnvironment; serviceConfig = { @@ -749,7 +847,7 @@ sleep 0.1 done - ${pkgs.pgbackrest}/bin/pgbackrest --type=diff --start-fast --stop-auto --delta backup + ${pgbackrest}/bin/pgbackrest --type=diff --start-fast --stop-auto --delta backup ''; environment = pgbackrestEnvironment; serviceConfig = { @@ -778,7 +876,7 @@ sleep 0.1 done - ${pkgs.pgbackrest}/bin/pgbackrest --type=incr --start-fast --stop-auto --delta backup + ${pgbackrest}/bin/pgbackrest --type=incr --start-fast --stop-auto --delta backup ''; environment = pgbackrestEnvironment; serviceConfig = {