jeff/nixos-postgresql
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add pgvecto.rs extension

Browse source
This commit is contained in:
Jeffrey C. Ollie 2023-12-16 14:51:38 -06:00
parent 960a8bac72
commit 86df433c30
Signed by: jeff
GPG key ID: 6F86035A6D97044E
2 changed files with 996 additions and 957 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
flake.lock
View file

@ -5,11 +5,11 @@
"systems": "systems"
},
"locked": {
"lastModified": 1681202837,
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
"lastModified": 1701680307,
"narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
"rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
"type": "github"
},
"original": {
@ -20,11 +20,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1684215771,
"narHash": "sha256-fsum28z+g18yreNa1Y7MPo9dtps5h1VkHfZbYQ+YPbk=",
"lastModified": 1702312524,
"narHash": "sha256-gkZJRDBUCpTPBvQk25G0B7vfbpEYM5s5OZqghkjZsnE=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "963006aab35e3e8ebbf6052b6bf4ea712fdd3c28",
"rev": "a9bf124c46ef298113270b1f84a164865987a91c",
"type": "github"
},
"original": {

329
flake.nix
View file

@ -9,19 +9,22 @@
url = "github:numtide/flake-utils";
};
};
outputs = { self, nixpkgs, flake-utils, ... }@inputs:
outputs = {
self,
nixpkgs,
flake-utils,
...
} @ inputs:
(
flake-utils.lib.eachDefaultSystem
(system:
let
(
system: let
pkgs = import nixpkgs {
inherit system;
};
in
{
in {
packages = {
scram-sha-256 =
let
scram-sha-256 = let
pname = "scram-sha-256";
version = "1.0.1";
in
@ -41,16 +44,58 @@
license = licenses.mit;
};
};
pgvecto-rs = let
pname = "pgvecto.rs";
version = "0.1.11";
hashes = {
"15" = "sha256-IVx/LgRnGyvBRYvrrJatd7yboWEoSYSJogLaH5N/wPA=";
};
major = pkgs.lib.versions.major pkgs.postgresql_15.version;
in
pkgs.stdenv.mkDerivation {
inherit pname version;
buildInputs = [pkgs.dpkg];
src = pkgs.fetchurl {
url = "https://github.com/tensorchord/pgvecto.rs/releases/download/v${version}/vectors-pg${major}-v${version}-x86_64-unknown-linux-gnu.deb";
hash = hashes."${major}";
};
dontUnpack = true;
dontBuild = true;
dontStrip = true;
installPhase = ''
mkdir -p $out
dpkg -x $src $out
install -D -t $out/lib $out/usr/lib/postgresql/${major}/lib/*.so
install -D -t $out/share/postgresql/extension $out/usr/share/postgresql/${major}/extension/*.sql
install -D -t $out/share/postgresql/extension $out/usr/share/postgresql/${major}/extension/*.control
rm -rf $out/usr
'';
meta = {
description = "Scalable Vector database plugin for Postgres, written in Rust, specifically designed for LLM";
homepage = "https://github.com/tensorchord/pgvecto.rs";
license = pkgs.lib.licenses.asl20;
platforms = pkgs.postgresql.meta.platforms;
};
};
};
}
)
) // {
)
// {
nixosModules = {
postgresql = { config, lib, pkgs, ... }:
let
postgresql = {
config,
lib,
pkgs,
...
}: let
cfg = config.jcollie.postgresql;
in
{
in {
options = {
jcollie.postgresql = lib.options.mkOption {
type = lib.types.submodule {
@ -76,7 +121,7 @@
};
};
};
default = { };
default = {};
};
replication = lib.options.mkOption {
type = lib.types.submodule {
@ -108,7 +153,7 @@
};
};
};
default = { };
default = {};
};
};
};
@ -123,8 +168,7 @@
};
log = lib.options.mkOption {
type = lib.types.submodule {
options =
let
options = let
logLevelType = lib.types.enum [
"off"
"error"
@ -134,8 +178,7 @@
"debug"
"trace"
];
in
{
in {
console = lib.options.mkOption {
type = logLevelType;
default = "detail";
@ -150,7 +193,7 @@
};
};
};
default = { };
default = {};
};
cipher = lib.options.mkOption {
type = lib.types.submodule {
@ -168,7 +211,7 @@
};
};
};
default = { };
default = {};
};
storage = lib.options.mkOption {
type = lib.types.submodule {
@ -197,7 +240,7 @@
};
};
};
default = { };
default = {};
description = "Options for connecting to Azure Blob storage";
};
b2 = lib.options.mkOption {
@ -223,12 +266,12 @@
};
};
};
default = { };
default = {};
description = "Options for connection to BackBlaze B2";
};
};
};
default = { };
default = {};
};
healthcheck = lib.options.mkOption {
type = lib.types.submodule {
@ -245,11 +288,11 @@
};
};
};
default = { };
default = {};
};
};
};
default = { };
default = {};
};
users = lib.options.mkOption {
@ -266,7 +309,7 @@
};
}
);
default = [ ];
default = [];
};
databases = lib.options.mkOption {
@ -281,10 +324,10 @@
};
extensions = lib.options.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
default = [];
};
template = lib.options.mkOption {
type = lib.types.enum [ "template0" "template1" ];
type = lib.types.enum ["template0" "template1"];
default = "template1";
};
encoding = lib.options.mkOption {
@ -302,23 +345,25 @@
};
}
);
default = [ ];
default = [];
};
};
};
default = { };
default = {};
};
};
config =
let
config = let
toStr = value:
if true == value then "yes"
else if false == value then "no"
else if lib.isString value then "'${lib.replaceStrings ["'"] ["''"] value}'"
if true == value
then "yes"
else if false == value
then "no"
else if lib.isString value
then "'${lib.replaceStrings ["'"] ["''"] value}'"
else toString value;
escapeShell = value: lib.replaceStrings [ "$" ] [ "\\$" ] value;
escapeShell = value: lib.replaceStrings ["$"] ["\\$"] value;
postgresql = pkgs.postgresql_15.withPackages (
plugins: [
@ -344,9 +389,11 @@
# }.${cfg.backup.storage.type};
# };
rcloneEnvironment = {
rcloneEnvironment =
{
RCLONE_CONFIG = "/dev/null";
} // (
}
// (
{
"azure" = {
RCLONE_CONFIG_AZURE_TYPE = "azureblob";
@ -356,32 +403,33 @@
RCLONE_CONFIG_B2_TYPE = "b2";
RCLONE_CONFIG_B2_ACCOUNT = cfg.backup.storage.b2.accountId;
};
}.${cfg.backup.storage.type}
}
.${cfg.backup.storage.type}
);
rcloneEnvironmentFiles = {
rcloneEnvironmentFiles =
{
"azure" = {
RCLONE_CONFIG_AZURE_KEY = cfg.backup.storage.azure.accountKeyFile;
};
"b2" = {
RCLONE_CONFIG_B2_KEY = cfg.backup.storage.b2.accountKeyFile;
};
}.${cfg.backup.storage.type};
}
.${cfg.backup.storage.type};
rclone = lib.mkIf (cfg.backup.enable) (
let
environment = lib.concatStringsSep "\n" (
lib.mapAttrsToList
(
n: v:
''export ${n}="${builtins.toString v}"''
n: v: ''export ${n}="${builtins.toString v}"''
)
rcloneEnvironment
);
environmentFiles = lib.concatStringsSep "\n" (
lib.mapAttrsToList
(
n: v:
''export ${n}=''$(<${v})''
n: v: ''export ${n}=''$(<${v})''
)
rcloneEnvironmentFiles
);
@ -396,7 +444,8 @@
rootDir = "/var/lib/postgresql";
dataDir = "${rootDir}/15";
pgbackrestEnvironment = {
pgbackrestEnvironment =
{
PGBACKREST_LOG_LEVEL_CONSOLE = cfg.backup.log.console;
PGBACKREST_LOG_LEVEL_FILE = cfg.backup.log.file;
PGBACKREST_LOG_LEVEL_STDERR = cfg.backup.log.stderr;
@ -406,7 +455,8 @@
PGBACKREST_REPO1_RETENTION_FULL = "14";
PGBACKREST_REPO1_RETENTION_FULL_TYPE = "time";
PGBACKREST_STANZA = "${config.networking.hostName}.${config.networking.domain}";
} // (
}
// (
{
"azure" = {
PGBACKREST_REPO1_AZURE_ACCOUNT = cfg.backup.storage.azure.accountName;
@ -422,19 +472,23 @@
PGBACKREST_REPO1_S3_REGION = cfg.backup.storage.b2.region;
PGBACKREST_REPO1_TYPE = "s3";
};
}.${cfg.backup.storage.type}
}
.${cfg.backup.storage.type}
);
pgbackrestEnvironmentFiles = {
pgbackrestEnvironmentFiles =
{
PGBACKREST_REPO1_CIPHER_PASS = cfg.backup.cipher.passwordFile;
} // {
}
// {
"azure" = {
PGBACKREST_REPO1_AZURE_KEY = cfg.backup.storage.azure.accountKeyFile;
};
"b2" = {
PGBACKREST_REPO1_S3_KEY_SECRET = cfg.backup.storage.b2.accountKeyFile;
};
}.${cfg.backup.storage.type};
}
.${cfg.backup.storage.type};
pgbackrest =
if (cfg.backup.enable)
@ -444,16 +498,14 @@
environment = lib.concatStringsSep "\n" (
lib.mapAttrsToList
(
n: v:
''export ${n}="${builtins.toString v}"''
n: v: ''export ${n}="${builtins.toString v}"''
)
pgbackrestEnvironment
);
environmentFiles = lib.concatStringsSep "\n" (
lib.mapAttrsToList
(
n: v:
''export ${n}=''$(<${v})''
n: v: ''export ${n}=''$(<${v})''
)
pgbackrestEnvironmentFiles
);
@ -464,8 +516,7 @@
exec ${pkgs.pgbackrest}/bin/pgbackrest "''$@"
''
)
else
{ };
else {};
# pgbackrestnu = lib.mkIf (cfg.backup.enable) (
# let
# environment = writeText "pgbackrest-environment.json" (builtins.toJSON pgbackrestEnvironment);
@ -490,18 +541,18 @@
# }
];
environment.systemPackages = [
environment.systemPackages =
[
postgresql
self.packages.${pkgs.system}.scram-sha-256
] ++ (
]
++ (
if cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")
then
[
then [
pgbackrest
rclone
]
else
[ ]
else []
);
# networking.firewall.allowedTCPPorts = [ cfg.port ];
@ -527,8 +578,7 @@
"d ${dataDir} 0700 postgres postgres -"
];
systemd.services.postgresql =
let
systemd.services.postgresql = let
hbaFile = pkgs.writeTextDir "pg_hba.conf" ''
local all all ident map=default
hostnossl all all all reject
@ -541,16 +591,20 @@
''
default root postgres
default postgres postgres
'' + (lib.strings.concatStringsSep "\n"
''
+ (
lib.strings.concatStringsSep "\n"
(
map
(user: "default root ${user.username}")
cfg.users
)
) + "\n"
)
+ "\n"
);
archiveCommand = "${pgbackrest}/bin/pgbackrest archive-push %p";
settings = {
settings =
{
bgwriter_flush_after = "512kB";
checkpoint_flush_after = "256kB";
data_directory = dataDir;
@ -585,29 +639,27 @@
unix_socket_directories = "/run/postgresql";
wal_level = "replica";
wal_log_hints = "on";
} // (
}
// (
if (cfg.backup.enable)
then {
archive_command = archiveCommand;
archive_mode = "on";
}
else
{ }
else {}
);
configFile = pkgs.writeTextDir "postgresql.conf" (
lib.concatStrings (
lib.mapAttrsToList
(
n: v:
"${n} = ${toStr v}\n"
n: v: "${n} = ${toStr v}\n"
)
settings
)
);
backupSetup =
if (cfg.backup.enable)
then
''
then ''
init=''$(${pgbackrest}/bin/pgbackrest info --output=json | ${pkgs.jq}/bin/jq '.[0].status.code == 0')
if [ "$init" != "true" ]
then
@ -616,8 +668,7 @@
${postgresql}/bin/pg_ctl -m fast -w stop
fi
''
else
"";
else "";
pgpass =
if (cfg.replication.enable && cfg.replication.role == "replica")
then
@ -628,39 +679,37 @@
cfg.replication.username
''''$(<${cfg.replication.passwordFile})''
]
else
"";
in
{
else "";
in {
description = "PostgreSQL Server";
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
environment = {
wantedBy = ["multi-user.target"];
after = ["network.target"];
environment =
{
PGDATA = dataDir;
} // (
}
// (
if (cfg.replication.enable && cfg.replication.role == "replica")
then {
PGPASSFILE = "${rootDir}/.pgpass";
}
else { }
else {}
);
path = [
postgresql
] ++ (
if cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")
then
path =
[
postgresql
]
++ (
if cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")
then [
pgbackrest
]
else
[ ]
else []
);
preStart =
if (!cfg.replication.enable || cfg.replication.role == "primary")
then
''
then ''
if [ ! -s "${dataDir}/PG_VERSION" ]
then
${postgresql}/bin/initdb \
@ -678,8 +727,7 @@
${backupSetup}
''
else
''
else ''
(umask 077; echo "${pgpass}" > ${rootDir}/.pgpass)
chmod 0600 ${rootDir}/.pgpass
@ -727,8 +775,8 @@
systemd.services.postgresql-setup = lib.mkIf (!cfg.replication.enable || cfg.replication.role == "primary") (
let
replicationSetup =
if (cfg.replication.enable && cfg.replication.role == "primary") then
''
if (cfg.replication.enable && cfg.replication.role == "primary")
then ''
if $PSQL --command "SELECT 1 FROM pg_roles WHERE rolname='${cfg.replication.username}';" | grep -q 1
then
echo "alter replication user ${cfg.replication.username}"
@ -741,12 +789,12 @@
fi
''
else "";
userSetup = lib.strings.concatStringsSep "\n"
userSetup =
lib.strings.concatStringsSep "\n"
(
map
(
user:
''
user: ''
if $PSQL --command "SELECT 1 FROM pg_roles WHERE rolname='${user.username}';" | grep -q 1
then
echo "alter user ${user.username}"
@ -765,7 +813,8 @@
databaseSetup = lib.strings.concatStringsSep "\n" (
map
(
database: ''
database:
''
if ! ( echo "SELECT 1 FROM pg_database WHERE datname=:'name';" | $PSQL --variable name="${database.name}" | grep -q 1 )
then
echo "create database ${database.name}"
@ -776,8 +825,8 @@
echo "GRANT ALL PRIVILEGES ON SCHEMA public TO :owner;" | $PSQL --dbname "${database.name}" --variable name="${database.name}" --variable owner="${database.owner}"
echo "grant priviliges on database ${database.name} to user ${database.owner}"
echo "GRANT ALL PRIVILEGES ON DATABASE :name TO :owner;" | $PSQL --dbname "${database.name}" --variable name="${database.name}" --variable owner="${database.owner}"
'' +
(
''
+ (
lib.strings.concatStringsSep "\n" (
map
(
@ -795,10 +844,9 @@
)
cfg.databases
);
in
{
in {
description = "PostgreSQL User/Database Setup";
after = [ "postgresql.service" ];
after = ["postgresql.service"];
# bindsTo = [ "postgresql.service" ];
script = ''
while ! ${postgresql}/bin/psql -d postgres -c "" 2> /dev/null
@ -830,19 +878,16 @@
systemd.services.postgresql-backup-full = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) (
let
hcStart =
if cfg.backup.healthcheck.enable then
"${curl} ${cfg.backup.healthcheck.fullBackupPingURL}/start"
else
"";
if cfg.backup.healthcheck.enable
then "${curl} ${cfg.backup.healthcheck.fullBackupPingURL}/start"
else "";
hcStop =
if cfg.backup.healthcheck.enable then
"${curl} ${cfg.backup.healthcheck.fullBackupPingURL}"
else
"";
in
{
if cfg.backup.healthcheck.enable
then "${curl} ${cfg.backup.healthcheck.fullBackupPingURL}"
else "";
in {
description = "PostgreSQL Full Backup";
requires = [ "postgresql.service" ];
requires = ["postgresql.service"];
script = ''
${hcStart}
@ -871,25 +916,22 @@
OnCalendar = "Sun *-*-* 02:00:00";
RandomizedDelaySec = "5m";
};
wantedBy = [ "multi-user.target" ];
wantedBy = ["multi-user.target"];
};
systemd.services.postgresql-backup-diff = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) (
let
hcStart =
if cfg.backup.healthcheck.enable then
"${curl} ${cfg.backup.healthcheck.differentialBackupPingURL}/start"
else
"";
if cfg.backup.healthcheck.enable
then "${curl} ${cfg.backup.healthcheck.differentialBackupPingURL}/start"
else "";
hcStop =
if cfg.backup.healthcheck.enable then
"${curl} ${cfg.backup.healthcheck.differentialBackupPingURL}"
else
"";
in
{
if cfg.backup.healthcheck.enable
then "${curl} ${cfg.backup.healthcheck.differentialBackupPingURL}"
else "";
in {
description = "PostgreSQL Differential Backup";
requires = [ "postgresql.service" ];
requires = ["postgresql.service"];
script = ''
${hcStart}
@ -918,25 +960,22 @@
OnCalendar = "Mon,Tue,Wed,Thu,Fri,Sat *-*-* 02:00:00";
RandomizedDelaySec = "5m";
};
wantedBy = [ "multi-user.target" ];
wantedBy = ["multi-user.target"];
};
systemd.services.postgresql-backup-incr = lib.mkIf (cfg.backup.enable && (!cfg.replication.enable || cfg.replication.role == "primary")) (
let
hcStart =
if cfg.backup.healthcheck.enable then
"${curl} ${cfg.backup.healthcheck.incrementalBackupPingURL}/start"
else
"";
if cfg.backup.healthcheck.enable
then "${curl} ${cfg.backup.healthcheck.incrementalBackupPingURL}/start"
else "";
hcStop =
if cfg.backup.healthcheck.enable then
"${curl} ${cfg.backup.healthcheck.incrementalBackupPingURL}"
else
"";
in
{
if cfg.backup.healthcheck.enable
then "${curl} ${cfg.backup.healthcheck.incrementalBackupPingURL}"
else "";
in {
description = "PostgreSQL Incremental Backup";
requires = [ "postgresql.service" ];
requires = ["postgresql.service"];
script = ''
${hcStart}
@ -965,7 +1004,7 @@
OnCalendar = "*-*-* 00,06,10,14,18,22:00:00";
RandomizedDelaySec = "5m";
};
wantedBy = [ "multi-user.target" ];
wantedBy = ["multi-user.target"];
};
services.prometheus.exporters.postgres = {