update
This commit is contained in:
parent
57f260d095
commit
c4bc0294ef
GPG key ID:
6F86035A6D97044E
1 changed files with 217 additions and 245 deletions
462
flake.nix
462
flake.nix
|
|
@ -280,6 +280,19 @@
|
|||
};
|
||||
};
|
||||
};
|
||||
etcd = mkOption {
|
||||
type = submodule {
|
||||
options = {
|
||||
certPath = mkOption {
|
||||
type = path;
|
||||
};
|
||||
keyPath = mkOption {
|
||||
type = path;
|
||||
};
|
||||
};
|
||||
};
|
||||
default = { };
|
||||
};
|
||||
service_subnet = mkOption {
|
||||
type = str;
|
||||
};
|
||||
|
|
@ -294,6 +307,9 @@
|
|||
role = mkOption {
|
||||
type = types.enum [ "etcd" "master+etcd" "master" "worker" ];
|
||||
};
|
||||
domain = mkOption {
|
||||
type = str;
|
||||
};
|
||||
endpoint = mkOption {
|
||||
type = submodule {
|
||||
options = {
|
||||
|
|
@ -352,197 +368,123 @@
|
|||
kubeadm = (self.packages.${pkgs.system}.kubeadm cfg.package_versions.kubeadm);
|
||||
kubectl = (self.packages.${pkgs.system}.kubectl cfg.package_versions.kubectl);
|
||||
kubelet = (self.packages.${pkgs.system}.kubelet cfg.package_versions.kubelet);
|
||||
|
||||
ca-config-json = pkgs.writeTextFile "ca-config-json" (
|
||||
builtins.toJSON {
|
||||
signing = {
|
||||
default = {
|
||||
expiry = "87600h";
|
||||
};
|
||||
profiles = {
|
||||
server = {
|
||||
expiry = "87600h";
|
||||
usages = [
|
||||
"signing"
|
||||
"key encipherment"
|
||||
"server auth"
|
||||
"client auth"
|
||||
];
|
||||
};
|
||||
client = {
|
||||
expiry = "87600h";
|
||||
usages = [
|
||||
"signing"
|
||||
"key encipherment"
|
||||
"client auth"
|
||||
];
|
||||
};
|
||||
peer = {
|
||||
expiry = "87600h";
|
||||
usages = [
|
||||
"signing"
|
||||
"key encipherment"
|
||||
"server auth"
|
||||
"client auth"
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
apiserver-etcd-client-csr-json = pkgs.writeTextFile "apiserver-etcd-client-csr-json" (
|
||||
builtins.toJSON {
|
||||
CN = "kube-apiserver-etcd-client";
|
||||
names = [
|
||||
{
|
||||
O = "system:masters";
|
||||
}
|
||||
];
|
||||
key = {
|
||||
algo = "rsa";
|
||||
size = 2048;
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
healthcheck-client-csr-json = pkgs.writeTextFile "healthcheck-client-csr.json" (
|
||||
builtins.toJSON {
|
||||
CN = "kube-etcd-healthcheck-client";
|
||||
names = [
|
||||
{
|
||||
O = "system:masters";
|
||||
}
|
||||
];
|
||||
key = {
|
||||
algo = "rsa";
|
||||
size = 2048;
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
server-csr-json = pkgs.writeTextFile "server-csr.json" (
|
||||
builtins.toJSON {
|
||||
CN = "${name}.${cfg.hosts.${name}.domain}";
|
||||
hosts = [
|
||||
"localhost"
|
||||
"${name}.${cfg.hosts.${name}.domain}"
|
||||
"127.0.0.1"
|
||||
"0:0:0:0:0:0:0:1"
|
||||
"${cfg.hosts.${name}.endpoint.address}"
|
||||
"${cfg.hosts.${name}.wireguard_subnet.address}"
|
||||
];
|
||||
key = {
|
||||
algo = "rsa";
|
||||
size = 2048;
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
etcd-ca-csr-json = pkgs.writeTextFile "etcd-ca-csr-json" (
|
||||
builtins.toJSON {
|
||||
CN = "etcd-ca";
|
||||
key = {
|
||||
algo = "rsa";
|
||||
size = 2048;
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
peer-csr-json = pkgs.writeTextFile "peer-csr.json" (
|
||||
builtins.toJSON {
|
||||
CN = "${name}.${cfg.hosts.${name}.domain}";
|
||||
hosts = [
|
||||
"localhost"
|
||||
"${name}.${cfg.hosts.${name}.domain}"
|
||||
"127.0.0.1"
|
||||
"0:0:0:0:0:0:0:1"
|
||||
"${cfg.hosts.${name}.endpoint.address}"
|
||||
"${cfg.hosts.${name}.wireguard_subnet.address}"
|
||||
];
|
||||
key = {
|
||||
algo = "rsa";
|
||||
size = 2048;
|
||||
};
|
||||
}
|
||||
);
|
||||
|
||||
in
|
||||
{
|
||||
|
||||
deployment.keys."etcd-ca.pem" = lib.mkIf (cfg.hosts.${name}.role == "etcd" || cfg.hosts.${name}.role == "master") {
|
||||
keyFile = ./etcd-ca.pem;
|
||||
destDir = (if cfg.hosts.${name}.role == "etcd" then "/var/lib/etcd" else "/etc/kubernetes/pki");
|
||||
user = (if cfg.hosts.${name}.role == "etcd" then "etcd" else "root");
|
||||
group = (if cfg.hosts.${name}.role == "etcd" then "etcd" else "root");
|
||||
permissions = "0644";
|
||||
uploadAt = "pre-activation";
|
||||
};
|
||||
|
||||
deployment.keys."etcd-ca-key.pem" = lib.mkIf (cfg.hosts.${name}.role == "etcd" || cfg.hosts.${name}.role == "master") {
|
||||
keyFile = ./etcd-ca-key.pem;
|
||||
destDir = (if cfg.hosts.${name}.role == "etcd" then "/var/lib/etcd" else "/etc/kubernetes/pki");
|
||||
user = (if cfg.hosts.${name}.role == "etcd" then "etcd" else "root");
|
||||
group = (if cfg.hosts.${name}.role == "etcd" then "etcd" else "root");
|
||||
permissions = "0400";
|
||||
uploadAt = "pre-activation";
|
||||
};
|
||||
|
||||
deployment.keys."ca-config.json" =
|
||||
let
|
||||
config = builtins.toJSON {
|
||||
signing = {
|
||||
default = {
|
||||
expiry = "87600h";
|
||||
};
|
||||
profiles = {
|
||||
server = {
|
||||
expiry = "87600h";
|
||||
usages = [
|
||||
"signing"
|
||||
"key encipherment"
|
||||
"server auth"
|
||||
"client auth"
|
||||
];
|
||||
};
|
||||
client = {
|
||||
expiry = "87600h";
|
||||
usages = [
|
||||
"signing"
|
||||
"key encipherment"
|
||||
"client auth"
|
||||
];
|
||||
};
|
||||
peer = {
|
||||
expiry = "87600h";
|
||||
usages = [
|
||||
"signing"
|
||||
"key encipherment"
|
||||
"server auth"
|
||||
"client auth"
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
lib.mkIf (cfg.hosts.${name}.role == "etcd" || cfg.hosts.${name}.role == "master") {
|
||||
text = config;
|
||||
destDir = (if cfg.hosts.${name}.role == "etcd" then "/var/lib/etcd" else "/etc/kubernetes/pki");
|
||||
user = (if cfg.hosts.${name}.role == "etcd" then "etcd" else "root");
|
||||
group = (if cfg.hosts.${name}.role == "etcd" then "etcd" else "root");
|
||||
permissions = "0400";
|
||||
uploadAt = "pre-activation";
|
||||
};
|
||||
|
||||
deployment.keys."etcd-ca-csr.json" =
|
||||
let
|
||||
config = builtins.toJSON {
|
||||
CN = "etcd-ca";
|
||||
key = {
|
||||
algo = "rsa";
|
||||
size = 2048;
|
||||
};
|
||||
};
|
||||
in
|
||||
lib.mkIf (cfg.hosts.${name}.role == "etcd") {
|
||||
text = config;
|
||||
destDir = "/var/lib/etcd";
|
||||
user = "etcd";
|
||||
group = "etcd";
|
||||
permissions = "0400";
|
||||
uploadAt = "pre-activation";
|
||||
};
|
||||
|
||||
deployment.keys."peer-csr.json" =
|
||||
let
|
||||
config = builtins.toJSON {
|
||||
CN = "${name}.dmacc.net";
|
||||
hosts = [
|
||||
"localhost"
|
||||
"${name}.dmacc.net"
|
||||
"127.0.0.1"
|
||||
"0:0:0:0:0:0:0:1"
|
||||
"${cfg.hosts.${name}.endpoint.address}"
|
||||
"${cfg.hosts.${name}.wireguard_subnet.address}"
|
||||
];
|
||||
key = {
|
||||
algo = "rsa";
|
||||
size = 2048;
|
||||
};
|
||||
};
|
||||
in
|
||||
lib.mkIf (cfg.hosts.${name}.role == "etcd") {
|
||||
text = config;
|
||||
destDir = "/var/lib/etcd";
|
||||
user = "etcd";
|
||||
group = "etcd";
|
||||
permissions = "0400";
|
||||
uploadAt = "pre-activation";
|
||||
};
|
||||
|
||||
deployment.keys."server-csr.json" =
|
||||
let
|
||||
config = builtins.toJSON {
|
||||
CN = "${name}.dmacc.net";
|
||||
hosts = [
|
||||
"localhost"
|
||||
"${name}.dmacc.net"
|
||||
"127.0.0.1"
|
||||
"0:0:0:0:0:0:0:1"
|
||||
"${cfg.hosts.${name}.endpoint.address}"
|
||||
"${cfg.hosts.${name}.wireguard_subnet.address}"
|
||||
];
|
||||
key = {
|
||||
algo = "rsa";
|
||||
size = 2048;
|
||||
};
|
||||
};
|
||||
in
|
||||
lib.mkIf (cfg.hosts.${name}.role == "etcd") {
|
||||
text = config;
|
||||
destDir = "/var/lib/etcd";
|
||||
user = "etcd";
|
||||
group = "etcd";
|
||||
permissions = "0400";
|
||||
uploadAt = "pre-activation";
|
||||
};
|
||||
|
||||
|
||||
deployment.keys."healthcheck-client-csr.json" =
|
||||
let
|
||||
config = builtins.toJSON {
|
||||
CN = "kube-etcd-healthcheck-client";
|
||||
names = [
|
||||
{
|
||||
O = "system:masters";
|
||||
}
|
||||
];
|
||||
key = {
|
||||
algo = "rsa";
|
||||
size = 2048;
|
||||
};
|
||||
};
|
||||
in
|
||||
lib.mkIf (cfg.hosts.${name}.role == "etcd") {
|
||||
text = config;
|
||||
destDir = "/var/lib/etcd";
|
||||
name = "healthcheck-client-csr.json";
|
||||
user = "etcd";
|
||||
group = "etcd";
|
||||
permissions = "0400";
|
||||
uploadAt = "pre-activation";
|
||||
};
|
||||
|
||||
deployment.keys."apiserver-etcd-client-csr.json" =
|
||||
let
|
||||
config = builtins.toJSON {
|
||||
CN = "kube-apiserver-etcd-client";
|
||||
names = [
|
||||
{
|
||||
O = "system:masters";
|
||||
}
|
||||
];
|
||||
key = {
|
||||
algo = "rsa";
|
||||
size = 2048;
|
||||
};
|
||||
};
|
||||
in
|
||||
lib.mkIf (cfg.hosts.${name}.role == "master") {
|
||||
text = config;
|
||||
destDir = "/etc/kubernetes/pki";
|
||||
user = "root";
|
||||
group = "root";
|
||||
permissions = "0400";
|
||||
uploadAt = "pre-activation";
|
||||
};
|
||||
|
||||
deployment.keys."private_key" = {
|
||||
text = cfg.hosts.${name}.private_key;
|
||||
destDir = "/etc/wireguard";
|
||||
|
|
@ -593,19 +535,22 @@
|
|||
|
||||
environment.systemPackages =
|
||||
let
|
||||
endpoints = map
|
||||
(
|
||||
n: "https://${cfg.hosts.${n}.endpoint.address}:2379"
|
||||
)
|
||||
(
|
||||
filter (n: cfg.hosts.${n}.role == "etcd" || cfg.hosts.${n}.role == "master+etcd") (attrNames cfg.hosts)
|
||||
);
|
||||
endpoints = concatStringsSep "," (
|
||||
map
|
||||
(
|
||||
n: "https://${cfg.hosts.${n}.wireguard_subnet.address}:2379"
|
||||
)
|
||||
(
|
||||
filter (n: cfg.hosts.${n}.role == "etcd" || cfg.hosts.${n}.role == "master+etcd") (attrNames cfg.hosts)
|
||||
)
|
||||
);
|
||||
etcdctl = pkgs.writeShellScriptBin "etcdctl" ''
|
||||
export ETCDCTL_API=3
|
||||
export ETCDCTL_CERT=/etc/kubernetes/pki/etcd/healthcheck-client.crt
|
||||
export ETCDCTL_KEY=/etc/kubernetes/pki/etcd/healthcheck-client.key
|
||||
export ETCDCTL_CACERT=/etc/kubernetes/pki/etcd/ca.crt
|
||||
export ETCDCTL_ENDPOINTS=${concatStringsSep "," endpoints}
|
||||
#export ETCDCTL_ENDPOINTS=${endpoints}
|
||||
export ETCDCTL_ENDPOINTS=https://${cfg.hosts.${name}.wireguard_subnet.address}:2379
|
||||
exec ${pkgs.etcd_3_5}/bin/etcdctl "$@"
|
||||
'';
|
||||
in
|
||||
|
|
@ -783,6 +728,27 @@
|
|||
);
|
||||
};
|
||||
|
||||
systemd.services.etcd.preStart =
|
||||
if
|
||||
(cfg.hosts.${name}.role == "etcd")
|
||||
then
|
||||
''
|
||||
cd /var/lib/etcd
|
||||
if [ ! -f server.pem ]
|
||||
then
|
||||
cat ${server-csr-json} | ${pkgs.cfssl}/bin/cfssl gencert -ca=${cfg.etcd.certPath} -ca-key=${cfg.etcd.keyPath} -config=${ca-config-json} -profile=server - | ${pkgs.cfssl}/bin/cfssljson -bare server
|
||||
fi
|
||||
if [ ! -f peer.pem ]
|
||||
then
|
||||
cat ${peer-csr-json} | ${pkgs.cfssl}/bin/cfssl gencert -ca=${cfg.etcd.certPath} -ca-key=${cfg.etcd.keyPath} -config=${ca-config-json} -profile=peer - | ${pkgs.cfssl}/bin/cfssljson -bare peer
|
||||
fi
|
||||
if [ ! -f healthcheck-client.pem ]
|
||||
then
|
||||
cat ${healthcheck-client-csr-json} | ${pkgs.cfssl}/bin/cfssl gencert -ca=${cfg.etcd.certPath} -ca-key=${cfg.etcd.keyPath} -config=${ca-config-json} -profile=client - | ${pkgs.cfssl}/bin/cfssljson -bare healthcheck-client
|
||||
fi
|
||||
''
|
||||
else "";
|
||||
|
||||
services.etcd = lib.mkIf (cfg.hosts.${name}.role == "etcd") {
|
||||
enable = true;
|
||||
name = "${name}.ocjtech.us";
|
||||
|
|
@ -794,7 +760,7 @@
|
|||
];
|
||||
initialCluster = map
|
||||
(
|
||||
n: "${n}.ocjtech.us=https://${cfg.hosts.${n}.wireguard_subnet.address}:2380"
|
||||
n: "${n}.${cfg.hosts.${n}.domain}=https://${cfg.hosts.${n}.wireguard_subnet.address}:2380"
|
||||
)
|
||||
(
|
||||
filter (n: cfg.hosts.${n}.role == "etcd") (attrNames cfg.hosts)
|
||||
|
|
@ -810,10 +776,12 @@
|
|||
clientCertAuth = true;
|
||||
certFile = "/var/lib/etcd/server.pem";
|
||||
keyFile = "/var/lib/etcd/server-key.pem";
|
||||
trustedCaFile = "/var/lib/etcd/etcd-ca.pem";
|
||||
trustedCaFile = cfg.etcd.certPath;
|
||||
# trustedCaFile = "/var/lib/etcd/etcd-ca.pem";
|
||||
peerCertFile = "/var/lib/etcd/peer.pem";
|
||||
peerKeyFile = "/var/lib/etcd/peer-key.pem";
|
||||
peerTrustedCaFile = "/var/lib/etcd/etcd-ca.pem";
|
||||
peerTrustedCaFile = cfg.etcd.certPath;
|
||||
# peerTrustedCaFile = "/var/lib/etcd/etcd-ca.pem";
|
||||
extraConf = {
|
||||
SNAPSHOT_COUNT = "10000";
|
||||
EXPERIMENTAL_INITIAL_CORRUPT_CHECK = "true";
|
||||
|
|
@ -822,53 +790,57 @@
|
|||
};
|
||||
|
||||
|
||||
systemd.services."kubelet" = lib.mkIf (cfg.hosts.${name}.role != "etcd") {
|
||||
enable = true;
|
||||
description = "kubelet";
|
||||
path = [
|
||||
pkgs.ceph-client
|
||||
pkgs.ethtool
|
||||
pkgs.iproute2
|
||||
pkgs.iptables
|
||||
pkgs.kmod
|
||||
pkgs.socat
|
||||
pkgs.thin-provisioning-tools
|
||||
pkgs.util-linux
|
||||
];
|
||||
unitConfig = {
|
||||
StartLimitIntervalSec = 0;
|
||||
After = [ "network-online.target" ];
|
||||
Wants = [ "network-online.target" ];
|
||||
};
|
||||
preStart = lib.mkIf (cfg.hosts.${name}.role == "master") ''
|
||||
mkdir -p /etc/kubernetes/pki
|
||||
cd /etc/kubernetes/pki
|
||||
if [ ! -f apiserver-etcd-client.pem ]
|
||||
then
|
||||
cat apiserver-etcd-client-csr.json | ${pkgs.cfssl}/bin/cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -profile=client - | ${pkgs.cfssl}/bin/cfssljson -bare apiserver-etcd-client
|
||||
cp apiserver-etcd-client.pem apiserver-etcd-client.crt
|
||||
cp apiserver-etcd-client-key.pem apiserver-etcd-client.key
|
||||
fi
|
||||
'';
|
||||
serviceConfig = {
|
||||
Slice = "kubernetes.slice";
|
||||
CPUAccounting = true;
|
||||
MemoryAccounting = true;
|
||||
Type = "simple";
|
||||
Environment = [
|
||||
"KUBELET_KUBECONFIG_ARGS=\"--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\""
|
||||
"KUBELET_CONFIG_ARGS=\"--config=/var/lib/kubelet/config.yaml\""
|
||||
systemd.services."kubelet" =
|
||||
let
|
||||
|
||||
in
|
||||
lib.mkIf (cfg.hosts.${name}.role != "etcd") {
|
||||
enable = true;
|
||||
description = "kubelet";
|
||||
path = [
|
||||
pkgs.ceph-client
|
||||
pkgs.ethtool
|
||||
pkgs.iproute2
|
||||
pkgs.iptables
|
||||
pkgs.kmod
|
||||
pkgs.socat
|
||||
pkgs.thin-provisioning-tools
|
||||
pkgs.util-linux
|
||||
];
|
||||
EnvironmentFile = [
|
||||
"-/var/lib/kubelet/kubeadm-flags.env"
|
||||
"-/etc/sysconfig/kubelet"
|
||||
];
|
||||
ExecStart = "${kubelet}/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS";
|
||||
Restart = "always";
|
||||
RestartSec = "10s";
|
||||
unitConfig = {
|
||||
StartLimitIntervalSec = 0;
|
||||
After = [ "network-online.target" ];
|
||||
Wants = [ "network-online.target" ];
|
||||
};
|
||||
preStart = lib.mkIf (cfg.hosts.${name}.role == "master") ''
|
||||
mkdir -p /etc/kubernetes/pki
|
||||
cd /etc/kubernetes/pki
|
||||
if [ ! -f apiserver-etcd-client.crt ]
|
||||
then
|
||||
cat ${apiserver-etcd-client-csr-json} | ${pkgs.cfssl}/bin/cfssl gencert -ca=${cfg.etcd.certPath} -ca-key=${cfg.etcd.keyPath} -config=${ca-config-json} -profile=client - | ${pkgs.cfssl}/bin/cfssljson -bare apiserver-etcd-client
|
||||
mv apiserver-etcd-client.pem apiserver-etcd-client.crt
|
||||
mv apiserver-etcd-client-key.pem apiserver-etcd-client.key
|
||||
fi
|
||||
'';
|
||||
serviceConfig = {
|
||||
Slice = "kubernetes.slice";
|
||||
CPUAccounting = true;
|
||||
MemoryAccounting = true;
|
||||
Type = "simple";
|
||||
Environment = [
|
||||
"KUBELET_KUBECONFIG_ARGS=\"--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf\""
|
||||
"KUBELET_CONFIG_ARGS=\"--config=/var/lib/kubelet/config.yaml\""
|
||||
];
|
||||
EnvironmentFile = [
|
||||
"-/var/lib/kubelet/kubeadm-flags.env"
|
||||
"-/etc/sysconfig/kubelet"
|
||||
];
|
||||
ExecStart = "${kubelet}/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS";
|
||||
Restart = "always";
|
||||
RestartSec = "10s";
|
||||
};
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
};
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue