Merge branch 'kubernetes'

README.md

@@ -25,3 +25,43 @@ Additionally, it deploys an age identity, which is later used for decrypting sec
 ## Deployment
 
 Deployment can simply be done as follows: `deploy`
+
+## Creating an admin certificate for k3s
+
+Create the admin's private key:
+```
+openssl genpkey -algorithm ed25519 -out <username>-key.pem
+```
+
+Create a CSR for the admin:
+```
+openssl req -new -key <username>-key.pem -out <username>.csr -subj "/CN=<username>"
+```
+
+Create a Kubernetes CSR object on the cluster:
+```
+k3s kubectl create -f - <<EOF
+apiVersion: certificates.k8s.io/v1
+kind: CertificateSigningRequest
+metadata:
+  name: <username>-csr
+spec:
+  request: $(cat <username>.csr | base64 | tr -d '\n')
+  expirationSeconds: 307584000 # 10 years
+  signerName: kubernetes.io/kube-apiserver-client
+  usages:
+    - digital signature
+    - key encipherment
+    - client auth
+EOF
+```
+
+Approve and sign the admin's CSR:
+```
+k3s kubectl certificate approve <username>-csr
+```
+
+Extract the resulting signed certificate from the CSR object:
+```
+k3s kubectl get csr <username>-csr -o jsonpath='{.status.certificate}' | base64 --decode > <username>.crt
+```

cluster/README.md

@@ -0,0 +1,3 @@
+```
+nix --extra-experimental-features nix-command --extra-experimental-features flakes  run .#kubenix
+```

cluster/flake.lock

@@ -0,0 +1,99 @@
+{
+  "nodes": {
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "kubenix": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "nixpkgs": "nixpkgs",
+        "systems": "systems",
+        "treefmt": "treefmt"
+      },
+      "locked": {
+        "lastModified": 1700116223,
+        "narHash": "sha256-Pld/UXlBcIDnQMY0JkDzChJkbof/zEcRkaiXtzvArEE=",
+        "owner": "hall",
+        "repo": "kubenix",
+        "rev": "e4d036576436b9983216584a89388af3da995043",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hall",
+        "repo": "kubenix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1686488075,
+        "narHash": "sha256-2otSBt2hbeD+5yY25NF3RhWx7l5SDt1aeU3cJ/9My4M=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "9401a0c780b49faf6c28adf55764f230301d0dce",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "kubenix": "kubenix"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "id": "systems",
+        "type": "indirect"
+      }
+    },
+    "treefmt": {
+      "inputs": {
+        "nixpkgs": [
+          "kubenix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1688026376,
+        "narHash": "sha256-qJmkr9BWDpqblk4E9/rCsAEl39y2n4Ycw6KRopvpUcY=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "df3f32b0cc253dfc7009b7317e8f0e7ccd70b1cf",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

cluster/flake.nix

@@ -0,0 +1,22 @@
+{
+  inputs.kubenix.url = "github:hall/kubenix";
+  outputs = { self, kubenix, ... }:
+    let
+      system = "x86_64-linux";
+    in {
+      kubenix = kubenix.packages.${system}.default.override {
+        module = {kubenix, ...}: {
+          imports = [kubenix.modules.k8s];
+          kubernetes = {
+            kubeconfig = "/etc/rancher/k3s/k3s.yaml";
+            version = "1.24";
+          };
+	  kubenix.project = "yeet";
+
+          kubernetes.resources.pods.web1.spec.containers.nginx.image = "nginx";
+          kubernetes.resources.pods.web2.spec.containers.nginx.image = "nginx";
+        };
+      };
+    };
+}
+

configuration.nix

@@ -112,6 +112,7 @@
   ];
 
   networking = {
+    domain = "hyp";
     firewall.enable = false;
     useDHCP = false;
 
@@ -169,10 +170,10 @@
     };
   };
 
-  virtualisation.libvirtd.enable = true;
-
   hardware.cpu.intel.updateMicrocode =
     lib.mkDefault config.hardware.enableRedistributableFirmware;
 
   age.identityPaths = [ "/root/age_ed25519" ];
+
+  virtualisation.libvirtd.enable = true;
 }

flake.nix

@@ -5,32 +5,41 @@
     nixpkgs.url = "github:nixos/nixpkgs/nixos-23.05";
     nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     deploy-rs.url = "github:serokell/deploy-rs";
+
     disko = {
       url = "github:nix-community/disko";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+
     agenix = {
       url = "github:ryantm/agenix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
   };
 
-  outputs = { self, nixpkgs, deploy-rs, disko, agenix, nixpkgs-unstable, ... }:
+  outputs =
+    { self, nixpkgs, deploy-rs, disko, agenix, nixpkgs-unstable, ... }:
     let
       system = "x86_64-linux";
       pkgs = nixpkgs.legacyPackages.${system};
       pkgs-unstable = nixpkgs-unstable.legacyPackages.${system};
       machines = import ./machines;
+      # TODO: Maybe use mergeAttrLists
       mkNixosSystems = systemDef:
-        nixpkgs.lib.foldlAttrs (acc: name: machine:
+        nixpkgs.lib.foldlAttrs
-          acc // {
+          (acc: name: machine:
-            "${name}" = nixpkgs.lib.nixosSystem (systemDef machine);
+            acc // {
-          }) { } machines;
+              "${name}" = nixpkgs.lib.nixosSystem (systemDef machine);
+            })
+          { }
+          machines;
       mkDeployNodes = nodeDef:
         nixpkgs.lib.foldlAttrs
-        (acc: name: machine: acc // { "${name}" = nodeDef machine; }) { }
+          (acc: name: machine: acc // { "${name}" = nodeDef machine; })
-        machines;
+          { }
-    in {
+          machines;
+    in
+    {
       devShells.${system}.default = pkgs.mkShell {
         packages = [
           pkgs.libsecret
@@ -43,6 +52,7 @@
           pkgs.postgresql_15
           pkgs-unstable.opentofu
           pkgs.cdrtools
+          pkgs.kubectl
         ];
       };
 
@@ -73,6 +83,7 @@
       };
 
       checks = builtins.mapAttrs
-        (system: deployLib: deployLib.deployChecks self.deploy) deploy-rs.lib;
+        (system: deployLib: deployLib.deployChecks self.deploy)
+        deploy-rs.lib;
     };
 }

machines/default.nix

@@ -14,6 +14,8 @@
         };
 
         terraformDatabase.enable = true;
+
+        k3s.enable = true;
       };
 
       disko.devices = {

modules/custom/data-disk.nix

@@ -3,17 +3,35 @@ let cfg = config.custom.dataDisk;
 in {
   options = {
     custom = {
-      dataDisk.enable = lib.mkOption {
+      dataDisk = {
-        default = false;
+        enable = lib.mkOption {
-        type = lib.types.bool;
+          default = false;
-        description = ''
+          type = lib.types.bool;
-          Whether to automatically mount /dev/sda1 on /mnt/data
+          description = ''
-        '';
+            Whether to automatically mount a disk to be used as a data disk.
+          '';
+        };
+
+        mountPoint = lib.mkOption {
+          default = "/mnt/data";
+          type = lib.types.str;
+          description = ''
+            Mount point of the data disk (if enabled).
+          '';
+        };
+
+        devicePath = lib.mkOption {
+          default = "/dev/sda1";
+          type = lib.types.str;
+          description = ''
+            Path of the device to be used as a data disk.
+          '';
+        };
       };
     };
   };
 
   config = lib.mkIf cfg.enable {
-    fileSystems."/mnt/data" = { device = "/dev/sda1"; };
+    fileSystems.${cfg.mountPoint} = { device = cfg.devicePath; };
   };
 }

modules/custom/default.nix

@@ -1,3 +1,3 @@
 {
-  imports = [ ./terraform-database.nix ./data-disk.nix ./ssh-certificates.nix ];
+  imports = [ ./terraform-database.nix ./data-disk.nix ./ssh-certificates.nix ./k3s.nix ];
 }

modules/custom/k3s-bootstrap.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: pim-cluster-admin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: pim

modules/custom/k3s.nix

@@ -0,0 +1,31 @@
+{ pkgs, lib, config, ... }:
+let cfg = config.custom.k3s;
+in {
+  options = {
+    custom = {
+      k3s.enable = lib.mkOption {
+        default = false;
+        type = lib.types.bool;
+        description = ''
+          Whether to start k3s with custom configuration.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ pkgs.k3s ];
+    services.k3s.enable = true;
+    services.k3s.role = "server";
+    services.k3s.extraFlags = "--tls-san ${config.networking.fqdn} --data-dir ${config.custom.dataDisk.mountPoint}/k3s";
+
+    # TODO: use kubenix for this.
+    system.activationScripts.k3s-bootstrap.text =
+      let
+        k3sBootstrapFile = pkgs.writeText "k3s-bootstrap" (builtins.readFile ./k3s-bootstrap.yaml);
+      in
+      ''
+        ln -sf ${k3sBootstrapFile} ${config.custom.dataDisk.mountPoint}/k3s/server/manifests/k3s-bootstrap.yaml
+      '';
+  };
+}

modules/custom/terraform-database.nix

@@ -20,19 +20,21 @@ in {
       package = pkgs.postgresql_15;
       enableTCPIP = true;
       dataDir = lib.mkIf config.custom.dataDisk.enable
-        "/mnt/data/postgresql/${config.services.postgresql.package.psqlSchema}";
+        "${config.custom.dataDisk.mountPoint}/postgresql/${config.services.postgresql.package.psqlSchema}";
       authentication = ''
         hostssl terraformstates terraform all cert
       '';
-      settings = let
+      settings =
-        serverCert = builtins.toFile "postgresql_server.crt"
+        let
-          (builtins.readFile ../../postgresql_server.crt);
+          serverCert = builtins.toFile "postgresql_server.crt"
-      in {
+            (builtins.readFile ../../postgresql_server.crt);
-        ssl = true;
+        in
-        ssl_cert_file = serverCert;
+        {
-        ssl_key_file = config.age.secrets."postgresql_server.key".path;
+          ssl = true;
-        ssl_ca_file = serverCert;
+          ssl_cert_file = serverCert;
-      };
+          ssl_key_file = config.age.secrets."postgresql_server.key".path;
+          ssl_ca_file = serverCert;
+        };
       ensureUsers = [{
         name = "terraform";
         ensurePermissions = { "DATABASE terraformstates" = "ALL PRIVILEGES"; };

nftables.conf

@@ -15,6 +15,7 @@ table inet nixos-fw {
 	chain input-allow {
 		tcp dport 22 accept
 		tcp dport 5432 accept comment "PostgreSQL server"
+		tcp dport 6443 accept comment "k3s"
 		icmp type echo-request accept comment "allow ping"
 		icmpv6 type != { nd-redirect, 139 } accept comment "Accept all ICMPv6 messages except redirects and node information queries (type 139).  See RFC 4890, section 4.4."
 		ip6 daddr fe80::/64 udp dport 546 accept comment "DHCPv6 client"