From 4f41fd746a70b21557a8f1220f0dc5b71dc459c6 Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Wed, 29 Nov 2023 10:02:50 +0100 Subject: [PATCH 1/6] enable k3s cluster add simple kubenix script --- cluster/README.md | 3 ++ cluster/flake.lock | 99 ++++++++++++++++++++++++++++++++++++++++++++++ cluster/flake.nix | 22 +++++++++++ configuration.nix | 6 ++- flake.nix | 5 ++- 5 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 cluster/README.md create mode 100644 cluster/flake.lock create mode 100644 cluster/flake.nix diff --git a/cluster/README.md b/cluster/README.md new file mode 100644 index 0000000..85cc5c8 --- /dev/null +++ b/cluster/README.md @@ -0,0 +1,3 @@ +``` +nix --extra-experimental-features nix-command --extra-experimental-features flakes run .#kubenix +``` diff --git a/cluster/flake.lock b/cluster/flake.lock new file mode 100644 index 0000000..7bc00f8 --- /dev/null +++ b/cluster/flake.lock @@ -0,0 +1,99 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "kubenix": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": "nixpkgs", + "systems": "systems", + "treefmt": "treefmt" + }, + "locked": { + "lastModified": 1700116223, + "narHash": "sha256-Pld/UXlBcIDnQMY0JkDzChJkbof/zEcRkaiXtzvArEE=", + "owner": "hall", + "repo": "kubenix", + "rev": "e4d036576436b9983216584a89388af3da995043", + "type": "github" + }, + "original": { + "owner": "hall", + "repo": "kubenix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1686488075, + "narHash": "sha256-2otSBt2hbeD+5yY25NF3RhWx7l5SDt1aeU3cJ/9My4M=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9401a0c780b49faf6c28adf55764f230301d0dce", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "kubenix": "kubenix" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "id": "systems", + "type": "indirect" + } + }, + "treefmt": { + "inputs": { + "nixpkgs": [ + "kubenix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1688026376, + "narHash": "sha256-qJmkr9BWDpqblk4E9/rCsAEl39y2n4Ycw6KRopvpUcY=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "df3f32b0cc253dfc7009b7317e8f0e7ccd70b1cf", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/cluster/flake.nix b/cluster/flake.nix new file mode 100644 index 0000000..1b3f6c6 --- /dev/null +++ b/cluster/flake.nix @@ -0,0 +1,22 @@ +{ + inputs.kubenix.url = "github:hall/kubenix"; + outputs = { self, kubenix, ... }: + let + system = "x86_64-linux"; + in { + kubenix = kubenix.packages.${system}.default.override { + module = {kubenix, ...}: { + imports = [kubenix.modules.k8s]; + kubernetes = { + kubeconfig = "/etc/rancher/k3s/k3s.yaml"; + version = "1.24"; + }; + kubenix.project = "yeet"; + + kubernetes.resources.pods.web1.spec.containers.nginx.image = "nginx"; + kubernetes.resources.pods.web2.spec.containers.nginx.image = "nginx"; + }; + }; + }; +} + diff --git a/configuration.nix b/configuration.nix index 950b605..2dfc05a 100644 --- a/configuration.nix +++ b/configuration.nix @@ -109,6 +109,7 @@ dig tree file + k3s ]; networking = { @@ -169,10 +170,11 @@ }; }; - virtualisation.libvirtd.enable = true; - hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; age.identityPaths = [ "/root/age_ed25519" ]; + + services.k3s.enable = true; + services.k3s.role = "server"; } diff --git a/flake.nix b/flake.nix index aa1690b..bc3ce12 100644 --- a/flake.nix +++ b/flake.nix @@ -5,17 +5,20 @@ nixpkgs.url = "github:nixos/nixpkgs/nixos-23.05"; nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; deploy-rs.url = "github:serokell/deploy-rs"; + disko = { url = "github:nix-community/disko"; inputs.nixpkgs.follows = "nixpkgs"; }; + agenix = { url = "github:ryantm/agenix"; inputs.nixpkgs.follows = "nixpkgs"; }; }; - outputs = { self, nixpkgs, deploy-rs, disko, agenix, nixpkgs-unstable, ... }: + outputs = + { self, nixpkgs, deploy-rs, disko, agenix, nixpkgs-unstable, ... }: let system = "x86_64-linux"; pkgs = nixpkgs.legacyPackages.${system}; From 1c0e4794a84e426ae07caefacbdd78fbc6002197 Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Thu, 14 Dec 2023 21:42:58 +0100 Subject: [PATCH 2/6] change k3s data dir to external disk add additional SAN to k3s certificates update README with k8s certificate instructions open port for kubectl --- README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ configuration.nix | 5 +++++ flake.nix | 2 ++ nftables.conf | 1 + 4 files changed, 52 insertions(+) diff --git a/README.md b/README.md index c8e0376..bddf974 100644 --- a/README.md +++ b/README.md @@ -25,3 +25,47 @@ Additionally, it deploys an age identity, which is later used for decrypting sec ## Deployment Deployment can simply be done as follows: `deploy` + +## Creating an admin certificate for k3s + +Create the admin's private key: +``` +openssl genpkey -algorithm ed25519 -out -key.pem +``` + +Create a CSR for the admin: +``` +openssl req -new -key -key.pem -out .csr -subj "/CN=" +``` + +Create a Kubernetes CSR object on the cluster: +``` +k3s kubectl create -f - <-csr +spec: + request: $(cat .csr | base64 | tr -d '\n') + expirationSeconds: 307584000 # 10 years + signerName: kubernetes.io/kube-apiserver-client + usages: + - digital signature + - key encipherment + - client auth +EOF +``` + +Approve and sign the admin's CSR: +``` +k3s kubectl certificate approve -csr +``` + +Extract the resulting signed certificate from the CSR object: +``` +k3s kubectl get csr -csr -o jsonpath='{.status.certificate}' | base64 --decode > .crt +``` + +## TODO + +1. Manage the bootstrap k3s clusterrolebinding with kubenix: `k3s kubectl create clusterrolebinding pim-cluster-admin --user=pim --clusterrole=cluster-admin`. diff --git a/configuration.nix b/configuration.nix index 2dfc05a..2b1387f 100644 --- a/configuration.nix +++ b/configuration.nix @@ -177,4 +177,9 @@ services.k3s.enable = true; services.k3s.role = "server"; + # Temporary fix: by default the full hostname of the server (jefke.hyp) is not included into the Subject Alternative Name of certificates of the server. + # We can hardcode this as a CLI flag to k3s. + services.k3s.extraFlags = "--tls-san jefke.hyp --data-dir /mnt/data/k3s"; + + virtualisation.libvirtd.enable = true; } diff --git a/flake.nix b/flake.nix index bc3ce12..ce3af39 100644 --- a/flake.nix +++ b/flake.nix @@ -24,6 +24,7 @@ pkgs = nixpkgs.legacyPackages.${system}; pkgs-unstable = nixpkgs-unstable.legacyPackages.${system}; machines = import ./machines; + # TODO: Maybe use mergeAttrLists mkNixosSystems = systemDef: nixpkgs.lib.foldlAttrs (acc: name: machine: acc // { @@ -44,6 +45,7 @@ pkgs-unstable.deploy-rs pkgs.openssl pkgs.postgresql_15 + pkgs.kubectl ]; }; diff --git a/nftables.conf b/nftables.conf index 46dd6cb..10d456f 100644 --- a/nftables.conf +++ b/nftables.conf @@ -15,6 +15,7 @@ table inet nixos-fw { chain input-allow { tcp dport 22 accept tcp dport 5432 accept comment "PostgreSQL server" + tcp dport 6443 accept comment "k3s" icmp type echo-request accept comment "allow ping" icmpv6 type != { nd-redirect, 139 } accept comment "Accept all ICMPv6 messages except redirects and node information queries (type 139). See RFC 4890, section 4.4." ip6 daddr fe80::/64 udp dport 546 accept comment "DHCPv6 client" From 0071dbfee58af8e63f8449d73e8fe78c8e8973e9 Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Fri, 15 Dec 2023 14:34:33 +0100 Subject: [PATCH 3/6] bootstrap admin clusterrolebinding --- README.md | 4 ---- configuration.nix | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bddf974..26fbe01 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,3 @@ Extract the resulting signed certificate from the CSR object: ``` k3s kubectl get csr -csr -o jsonpath='{.status.certificate}' | base64 --decode > .crt ``` - -## TODO - -1. Manage the bootstrap k3s clusterrolebinding with kubenix: `k3s kubectl create clusterrolebinding pim-cluster-admin --user=pim --clusterrole=cluster-admin`. diff --git a/configuration.nix b/configuration.nix index 2b1387f..2738b75 100644 --- a/configuration.nix +++ b/configuration.nix @@ -182,4 +182,28 @@ services.k3s.extraFlags = "--tls-san jefke.hyp --data-dir /mnt/data/k3s"; virtualisation.libvirtd.enable = true; + + system.activationScripts.k3s-bootstrap.text = + let + k3sBootstrapFile = pkgs.writeTextFile { + name = "k3s-bootstrap"; + text = '' + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: pim-cluster-admin + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - apiGroup: rbac.authorization.k8s.io + kind: User + name: pim + ''; + }; + in + '' + ln -sf ${k3sBootstrapFile} /mnt/data/k3s/server/manifests/k3s-bootstrap.yaml + ''; } From 052e3d7b632436f1c405c4bb3e213dbcedadf87a Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Fri, 15 Dec 2023 14:55:48 +0100 Subject: [PATCH 4/6] create custom module for k3s configuration --- configuration.nix | 31 ------------------------- machines/default.nix | 2 ++ modules/custom/default.nix | 2 +- modules/custom/k3s-bootstrap.yaml | 12 ++++++++++ modules/custom/k3s.nix | 33 +++++++++++++++++++++++++++ modules/custom/terraform-database.nix | 20 ++++++++-------- 6 files changed, 59 insertions(+), 41 deletions(-) create mode 100644 modules/custom/k3s-bootstrap.yaml create mode 100644 modules/custom/k3s.nix diff --git a/configuration.nix b/configuration.nix index 2738b75..0148767 100644 --- a/configuration.nix +++ b/configuration.nix @@ -109,7 +109,6 @@ dig tree file - k3s ]; networking = { @@ -175,35 +174,5 @@ age.identityPaths = [ "/root/age_ed25519" ]; - services.k3s.enable = true; - services.k3s.role = "server"; - # Temporary fix: by default the full hostname of the server (jefke.hyp) is not included into the Subject Alternative Name of certificates of the server. - # We can hardcode this as a CLI flag to k3s. - services.k3s.extraFlags = "--tls-san jefke.hyp --data-dir /mnt/data/k3s"; - virtualisation.libvirtd.enable = true; - - system.activationScripts.k3s-bootstrap.text = - let - k3sBootstrapFile = pkgs.writeTextFile { - name = "k3s-bootstrap"; - text = '' - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: pim-cluster-admin - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin - subjects: - - apiGroup: rbac.authorization.k8s.io - kind: User - name: pim - ''; - }; - in - '' - ln -sf ${k3sBootstrapFile} /mnt/data/k3s/server/manifests/k3s-bootstrap.yaml - ''; } diff --git a/machines/default.nix b/machines/default.nix index fad2e3d..a942b42 100644 --- a/machines/default.nix +++ b/machines/default.nix @@ -13,6 +13,8 @@ }; terraformDatabase.enable = true; + + k3s.enable = true; }; }; }; diff --git a/modules/custom/default.nix b/modules/custom/default.nix index ceeaefa..6e7528d 100644 --- a/modules/custom/default.nix +++ b/modules/custom/default.nix @@ -1,3 +1,3 @@ { - imports = [ ./terraform-database.nix ./data-disk.nix ./ssh-certificates.nix ]; + imports = [ ./terraform-database.nix ./data-disk.nix ./ssh-certificates.nix ./k3s.nix ]; } diff --git a/modules/custom/k3s-bootstrap.yaml b/modules/custom/k3s-bootstrap.yaml new file mode 100644 index 0000000..efcc5e2 --- /dev/null +++ b/modules/custom/k3s-bootstrap.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pim-cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: pim diff --git a/modules/custom/k3s.nix b/modules/custom/k3s.nix new file mode 100644 index 0000000..0d5ae49 --- /dev/null +++ b/modules/custom/k3s.nix @@ -0,0 +1,33 @@ +{ pkgs, lib, config, ... }: +let cfg = config.custom.k3s; +in { + options = { + custom = { + k3s.enable = lib.mkOption { + default = false; + type = lib.types.bool; + description = '' + Whether to start k3s with custom configuration. + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + environment.systemPackages = [ pkgs.k3s ]; + services.k3s.enable = true; + services.k3s.role = "server"; + # Temporary fix: by default the full hostname of the server (jefke.hyp) is not included into the Subject Alternative Name of certificates of the server. + # We can hardcode this as a CLI flag to k3s. + services.k3s.extraFlags = "--tls-san jefke.hyp --data-dir /mnt/data/k3s"; + + # TODO: use kubenix for this. + system.activationScripts.k3s-bootstrap.text = + let + k3sBootstrapFile = pkgs.writeText "k3s-bootstrap" (builtins.readFile ./k3s-bootstrap.yaml); + in + '' + ln -sf ${k3sBootstrapFile} /mnt/data/k3s/server/manifests/k3s-bootstrap.yaml + ''; + }; +} diff --git a/modules/custom/terraform-database.nix b/modules/custom/terraform-database.nix index bdad8a7..0625311 100644 --- a/modules/custom/terraform-database.nix +++ b/modules/custom/terraform-database.nix @@ -24,15 +24,17 @@ in { authentication = '' hostssl terraformstates terraform all cert ''; - settings = let - serverCert = builtins.toFile "postgresql_server.crt" - (builtins.readFile ../../postgresql_server.crt); - in { - ssl = true; - ssl_cert_file = serverCert; - ssl_key_file = config.age.secrets."postgresql_server.key".path; - ssl_ca_file = serverCert; - }; + settings = + let + serverCert = builtins.toFile "postgresql_server.crt" + (builtins.readFile ../../postgresql_server.crt); + in + { + ssl = true; + ssl_cert_file = serverCert; + ssl_key_file = config.age.secrets."postgresql_server.key".path; + ssl_ca_file = serverCert; + }; ensureUsers = [{ name = "terraform"; ensurePermissions = { "DATABASE terraformstates" = "ALL PRIVILEGES"; }; From b6a37eabbd78fb8395d0f763abe2dd3541f6df12 Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Fri, 15 Dec 2023 15:11:14 +0100 Subject: [PATCH 5/6] parameterize fqdn for k3s SAN --- configuration.nix | 1 + flake.nix | 22 ++++++++++++++-------- modules/custom/k3s.nix | 6 +++--- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/configuration.nix b/configuration.nix index 0148767..30e566f 100644 --- a/configuration.nix +++ b/configuration.nix @@ -112,6 +112,7 @@ ]; networking = { + domain = "hyp"; firewall.enable = false; useDHCP = false; diff --git a/flake.nix b/flake.nix index ce3af39..8385b32 100644 --- a/flake.nix +++ b/flake.nix @@ -26,15 +26,20 @@ machines = import ./machines; # TODO: Maybe use mergeAttrLists mkNixosSystems = systemDef: - nixpkgs.lib.foldlAttrs (acc: name: machine: - acc // { - "${name}" = nixpkgs.lib.nixosSystem (systemDef machine); - }) { } machines; + nixpkgs.lib.foldlAttrs + (acc: name: machine: + acc // { + "${name}" = nixpkgs.lib.nixosSystem (systemDef machine); + }) + { } + machines; mkDeployNodes = nodeDef: nixpkgs.lib.foldlAttrs - (acc: name: machine: acc // { "${name}" = nodeDef machine; }) { } - machines; - in { + (acc: name: machine: acc // { "${name}" = nodeDef machine; }) + { } + machines; + in + { devShells.${system}.default = pkgs.mkShell { packages = [ pkgs.libsecret @@ -76,6 +81,7 @@ }; checks = builtins.mapAttrs - (system: deployLib: deployLib.deployChecks self.deploy) deploy-rs.lib; + (system: deployLib: deployLib.deployChecks self.deploy) + deploy-rs.lib; }; } diff --git a/modules/custom/k3s.nix b/modules/custom/k3s.nix index 0d5ae49..f851be6 100644 --- a/modules/custom/k3s.nix +++ b/modules/custom/k3s.nix @@ -17,10 +17,10 @@ in { environment.systemPackages = [ pkgs.k3s ]; services.k3s.enable = true; services.k3s.role = "server"; - # Temporary fix: by default the full hostname of the server (jefke.hyp) is not included into the Subject Alternative Name of certificates of the server. - # We can hardcode this as a CLI flag to k3s. - services.k3s.extraFlags = "--tls-san jefke.hyp --data-dir /mnt/data/k3s"; + # TODO: parameterize data disk mount point. + services.k3s.extraFlags = "--tls-san ${config.networking.fqdn} --data-dir /mnt/data/k3s"; + # TODO: parameterize data disk mount point. # TODO: use kubenix for this. system.activationScripts.k3s-bootstrap.text = let From 38fce7d2b44eab7fa76b188754b39f2f4e189ace Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Fri, 15 Dec 2023 15:20:28 +0100 Subject: [PATCH 6/6] parameterize data disk mount point --- modules/custom/data-disk.nix | 32 +++++++++++++++++++++------ modules/custom/k3s.nix | 6 ++--- modules/custom/terraform-database.nix | 2 +- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/modules/custom/data-disk.nix b/modules/custom/data-disk.nix index 4e2d485..99fa46d 100644 --- a/modules/custom/data-disk.nix +++ b/modules/custom/data-disk.nix @@ -3,17 +3,35 @@ let cfg = config.custom.dataDisk; in { options = { custom = { - dataDisk.enable = lib.mkOption { - default = false; - type = lib.types.bool; - description = '' - Whether to automatically mount /dev/sda1 on /mnt/data - ''; + dataDisk = { + enable = lib.mkOption { + default = false; + type = lib.types.bool; + description = '' + Whether to automatically mount a disk to be used as a data disk. + ''; + }; + + mountPoint = lib.mkOption { + default = "/mnt/data"; + type = lib.types.str; + description = '' + Mount point of the data disk (if enabled). + ''; + }; + + devicePath = lib.mkOption { + default = "/dev/sda1"; + type = lib.types.str; + description = '' + Path of the device to be used as a data disk. + ''; + }; }; }; }; config = lib.mkIf cfg.enable { - fileSystems."/mnt/data" = { device = "/dev/sda1"; }; + fileSystems.${cfg.mountPoint} = { device = cfg.devicePath; }; }; } diff --git a/modules/custom/k3s.nix b/modules/custom/k3s.nix index f851be6..e37ff0a 100644 --- a/modules/custom/k3s.nix +++ b/modules/custom/k3s.nix @@ -17,17 +17,15 @@ in { environment.systemPackages = [ pkgs.k3s ]; services.k3s.enable = true; services.k3s.role = "server"; - # TODO: parameterize data disk mount point. - services.k3s.extraFlags = "--tls-san ${config.networking.fqdn} --data-dir /mnt/data/k3s"; + services.k3s.extraFlags = "--tls-san ${config.networking.fqdn} --data-dir ${config.custom.dataDisk.mountPoint}/k3s"; - # TODO: parameterize data disk mount point. # TODO: use kubenix for this. system.activationScripts.k3s-bootstrap.text = let k3sBootstrapFile = pkgs.writeText "k3s-bootstrap" (builtins.readFile ./k3s-bootstrap.yaml); in '' - ln -sf ${k3sBootstrapFile} /mnt/data/k3s/server/manifests/k3s-bootstrap.yaml + ln -sf ${k3sBootstrapFile} ${config.custom.dataDisk.mountPoint}/k3s/server/manifests/k3s-bootstrap.yaml ''; }; } diff --git a/modules/custom/terraform-database.nix b/modules/custom/terraform-database.nix index 0625311..01b7bb5 100644 --- a/modules/custom/terraform-database.nix +++ b/modules/custom/terraform-database.nix @@ -20,7 +20,7 @@ in { package = pkgs.postgresql_15; enableTCPIP = true; dataDir = lib.mkIf config.custom.dataDisk.enable - "/mnt/data/postgresql/${config.services.postgresql.package.psqlSchema}"; + "${config.custom.dataDisk.mountPoint}/postgresql/${config.services.postgresql.package.psqlSchema}"; authentication = '' hostssl terraformstates terraform all cert '';