From 03e816ff21626f99d91a795ee7c63dea80fa8f40 Mon Sep 17 00:00:00 2001
From: Pim Kunis <pim@kunis.nl>
Date: Sun, 11 Feb 2024 14:18:11 +0100
Subject: [PATCH] create VM running k3s

---
 nixos/machines/default.nix    | 24 +++++++++++++++++++++++-
 nixos/modules/k3s/default.nix | 12 ++++++++----
 nixos/physical.nix            |  4 ++--
 3 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/nixos/machines/default.nix b/nixos/machines/default.nix
index 4257725..7d5786d 100644
--- a/nixos/machines/default.nix
+++ b/nixos/machines/default.nix
@@ -56,7 +56,6 @@
     };
   };
 
-  # TODO: make leases persistent!
   hermes = {
     type = "virtual";
     hypervisorName = "lewis";
@@ -67,6 +66,7 @@
 
         vm = {
           # TODO: would be cool to create a check that a mac address is only ever assigned to one VM.
+          # TODO: idea: what if we generated these IDs by hashing the host name and reducing that to the amount of hosts possible?
           id = 7;
           staticNetworking = true;
           staticIPv4 = config.lab.networking.dmz.ipv4.services;
@@ -125,4 +125,26 @@
       };
     };
   };
+
+  k3s = {
+    type = "virtual";
+    hypervisorName = "atlas";
+
+    nixosModule = {
+      microvm.balloonMem = 7680;
+
+      lab = {
+        k3s.enable = true;
+
+        vm = {
+          id = 4;
+
+          shares = [{
+            name = "k3s";
+            mountPoint = "/var/lib/rancher/k3s";
+          }];
+        };
+      };
+    };
+  };
 }
diff --git a/nixos/modules/k3s/default.nix b/nixos/modules/k3s/default.nix
index 4dbafd9..67a35c0 100644
--- a/nixos/modules/k3s/default.nix
+++ b/nixos/modules/k3s/default.nix
@@ -10,13 +10,17 @@ in {
   };
 
   config = lib.mkIf cfg.enable {
-    environment.systemPackages = [ pkgs.k3s ];
-    networking.firewall.allowedTCPPorts = [ 6443 ];
+    environment.systemPackages = with pkgs; [ k3s ];
+
+    networking = {
+      nftables.enable = lib.mkForce false;
+      firewall.enable = lib.mkForce false;
+    };
 
     services.k3s = {
       enable = true;
       role = "server";
-      extraFlags = "--tls-san ${config.networking.fqdn} --data-dir ${config.lab.dataDisk.mountPoint}/k3s";
+      extraFlags = "--tls-san ${config.networking.fqdn} --snapshotter native";
     };
 
     system.activationScripts.k3s-bootstrap.text =
@@ -26,7 +30,7 @@ in {
         }).config.kubernetes.result;
       in
       ''
-        ln -sf ${k3sBootstrapFile} ${config.lab.dataDisk.mountPoint}/k3s/server/manifests/k3s-bootstrap.json
+        ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json
       '';
   };
 }
diff --git a/nixos/physical.nix b/nixos/physical.nix
index 72bdd94..ba5641b 100644
--- a/nixos/physical.nix
+++ b/nixos/physical.nix
@@ -1,4 +1,4 @@
-{ pkgs, config, lib, modulesPath, microvm, disko, agenix, machines, dns, ... }: {
+{ pkgs, config, lib, modulesPath, microvm, disko, agenix, machines, dns, kubenix, ... }: {
   imports = [
     (modulesPath + "/installer/scan/not-detected.nix")
     microvm.nixosModules.host
@@ -62,7 +62,7 @@
         (name: vm:
           {
             # TODO Simplify?
-            specialArgs = { inherit agenix disko pkgs lib microvm dns; machine = vm; hypervisorConfig = config; };
+            specialArgs = { inherit agenix disko pkgs lib microvm dns kubenix; machine = vm; hypervisorConfig = config; };
             config.imports = [
               ./.
               { networking.hostName = name; }