add support for multi-node k3s cluster

machines/atlas.nix

@@ -7,6 +7,11 @@
         osDisk = "/dev/sda";
         dataPartition = "/dev/nvme0n1p1";
       };
+
+      k3s = {
+        enable = true;
+        serverAddr = "https://jefke.dmz:6443";
+      };
     };
   };
 }

nixos-modules/k3s/default.nix

@@ -1,12 +1,23 @@
 { pkgs, lib, config, kubenix, ... }:
 let cfg = config.lab.k3s;
 in {
-  options.lab.k3s.enable = lib.mkOption {
+  options.lab.k3s = {
-    default = false;
+    enable = lib.mkOption {
-    type = lib.types.bool;
+      default = false;
-    description = ''
+      type = lib.types.bool;
-      Whether to start k3s with custom configuration.
+      description = ''
-    '';
+        Whether to run k3s on this server.
+      '';
+    };
+
+    serverAddr = lib.mkOption {
+      default = null;
+      type = with lib.types; nullOr str;
+      description = ''
+        Address of the server whose cluster this server should join.
+        Leaving this empty will make the server initialize the cluster.
+      '';
+    };
   };
 
   config = lib.mkIf cfg.enable {
@@ -20,18 +31,26 @@ in {
     services.k3s = {
       enable = true;
       role = "server";
+      tokenFile = config.age.secrets.k3s-server-token.path;
       extraFlags = "--tls-san ${config.networking.fqdn} --disable servicelb";
+      clusterInit = cfg.serverAddr == null;
+      serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr;
     };
 
-    system.activationScripts.k3s-bootstrap.text =
+    system = lib.mkIf (cfg.serverAddr == null) {
-      let
+      activationScripts.k3s-bootstrap.text = (
-        k3sBootstrapFile = (kubenix.evalModules.x86_64-linux {
+        let
-          module = import ./bootstrap.nix;
+          k3sBootstrapFile = (kubenix.evalModules.x86_64-linux {
-        }).config.kubernetes.result;
+            module = import ./bootstrap.nix;
-      in
+          }).config.kubernetes.result;
-      ''
+        in
-        mkdir -p /var/lib/rancher/k3s/server/manifests
+        ''
-        ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json
+          mkdir -p /var/lib/rancher/k3s/server/manifests
-      '';
+          ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json
+        ''
+      );
+    };
+
+    age.secrets.k3s-server-token.file = ../../secrets/k3s-server-token.age;
   };
 }

secrets/k3s-server-token.age

@@ -0,0 +1,15 @@
+age-encryption.org/v1
+-> ssh-ed25519 UwNSRQ /B3zuCTP4RhYNPfmErYcFxkL4PrUWs92Q0KGTFTe33g
+ar6/o3O1AQFYHBbvs7U9wm5JBXG8suk29Ul56uC39Ok
+-> ssh-ed25519 JJ7S4A hJpjR4TFVOHCASfRosTa0oQSr4Q2HjD54Pv1LLY8u1Y
+ughx4kBl8IwoEnrpC1Q1P1VZVDxb7BwX32F5JULBz78
+-> ssh-ed25519 aqswPA Kyen24puaGTH9Qx11QtZrJrpIiRLh3GR89u8DOxHhTQ
+n+RSyHbWLLA6YxWwtsBkwxZePCGZtd0k1DTlXy0rOt8
+-> ssh-ed25519 LAPUww 9WvReHxes3jeagSidtztlb06gEKzWbXaSm/wxdcVWGc
+4hOVE30jlFUjzXZngJMlyOvW4rK6kAFTZgceyw49DsE
+-> ssh-ed25519 vBZj5g Iy2k/NumAyRy2lgv8NFVd7PW1kAgY/HtUAA0DpbY/Xw
+jfNr7QiXqTE/jfEOZFEhct7qfKbLYxIAnzPupIfxnnY
+-> ssh-ed25519 QP0PgA dFlkBqcgmXd7GnpoI1X4ezDDYuqKtSG8VbUB08As2k8
++KlOiHi+vi0RntHTbdOWzp2lRWdd4SpTU/4dCs51qBU
+--- BapxmCnFven9QR0bZDuYWk+lM/2U4AVWQYZsGKRI/W0
+°ëDÓF¢y{¥Ýjñƒ2Ñö<C391>h4þ<34>ôrŽyʼ9¦Å…²åo‘"VJˆN§ÈÛ3ÓOÍ¡´€a	s°ö0ùïÁ

secrets/secrets.nix

@@ -11,6 +11,7 @@ let
     "database_passwords.env.age"
     "borg_passphrase.age"
     "borgbase.pem.age"
+    "k3s-server-token.age"
   ];
 
   machinePublicKeys = [