Resolve cyberchef and radicale images using nix-snapshotter

Increase inotify max user instances to 256
Disable tailscale by default

configuration.nix

@@ -91,6 +91,7 @@
     boot = lib.mkIf (! machine.isRaspberryPi) {
       kernelModules = [ "kvm-intel" ];
       extraModulePackages = [ ];
+      kernel.sysctl."fs.inotify.max_user_instances" = 256;
 
       initrd = {
         kernelModules = [ ];

container-images.nix

@@ -1,2 +1,2 @@
-{ cyberchef = { cyberchef = { finalImageName = "mpepping/cyberchef"; finalImageTag = "latest"; imageDigest = "sha256:5044c72dd8070fb6e0595e720fc4440bf6168493b2433db06a1c966406398ba2"; imageName = "mpepping/cyberchef"; sha256 = "177yjfbz0ijc8lfqfr50fhqqmjk72373c0igyrxv3wwg0pyrgpv4"; }; }; }
+{ cyberchef = { finalImageName = "mpepping/cyberchef"; finalImageTag = "latest"; imageDigest = "sha256:2c89d08580395b932c92d708041c2a702dc8fa899fcc1677901c2dc881bed789"; imageName = "mpepping/cyberchef"; sha256 = "1frlvv6lyghf99pa37l48r7j2wvh7mcb9x99fvf0ba2zhq2xfsy4"; }; radicale = { finalImageName = "tomsquest/docker-radicale"; finalImageTag = "3.2.2.0"; imageDigest = "sha256:af050e02c4a3f7385a09595dd2a1424db6831aa9f24404095c6d2244d1c94138"; imageName = "tomsquest/docker-radicale"; sha256 = "07mxn6iqm0fhb06dwmxdhqnw3c8yi2dm2jpb2n80dxzmvavrv0lk"; }; }
 

flake-parts/scripts/default.nix

@@ -30,12 +30,16 @@ in
     let
       images = {
         cyberchef = {
-          cyberchef = {
+          name = "mpepping/cyberchef";
-            image-name = "mpepping/cyberchef";
+          tag = "latest";
-            image-tag = "latest";
+        };
-          };
+
+        radicale = {
+          name = "tomsquest/docker-radicale";
+          tag = "3.2.2.0";
         };
       };
+
       imagesJSON = builtins.toFile "images.json" (builtins.toJSON images);
     in
     pkgs.writers.writePython3Bin "prefetch-container-images"
@@ -55,31 +59,28 @@ in
       with open(images_file_name, 'r') as file:
           data = json.load(file)
 
-      for project_name, images in data.items():
+      for image_name, image in data.items():
-          print(f"Prefetching images for project {project_name}", file=sys.stderr)
+          name = image["name"]
+          tag = image["tag"]
 
-          for image_name, image in images.items():
+          print(f"Prefetching image {name}:{tag}", file=sys.stderr)
-              name = image["image-name"]
-              tag = image["image-tag"]
 
-              print(f"Prefetching image {name}:{tag}", file=sys.stderr)
+          prefetch_args = [
+              prefetch_docker_cmd,
+              "--os", "linux",
+              "--arch", "amd64",
+              "--image-name", name,
+              "--image-tag", tag,
+              "--json",
+              "--quiet"
+          ]
+          result = subprocess.run(prefetch_args,
+                                  check=True,
+                                  capture_output=True,
+                                  text=True)
 
-              prefetch_args = [
+          prefetch_data = json.loads(result.stdout)
-                  prefetch_docker_cmd,
+          results[image_name] = prefetch_data
-                  "--os", "linux",
-                  "--arch", "amd64",
-                  "--image-name", name,
-                  "--image-tag", tag,
-                  "--json",
-                  "--quiet"
-              ]
-              result = subprocess.run(prefetch_args,
-                                      check=True,
-                                      capture_output=True,
-                                      text=True)
-
-              prefetch_data = json.loads(result.stdout)
-              results[project_name][image_name] = prefetch_data
 
       with tempfile.NamedTemporaryFile(mode='w+', suffix='.json') as temp_file:
           json.dump(results, temp_file, indent=4)

kubenix-modules/cyberchef.nix

@@ -9,6 +9,7 @@
 
         spec.containers.cyberchef = {
           image = myLib.globals.images.cyberchef;
+          imagePullPolicy = "Always";
           ports.web.containerPort = 8000;
         };
       };

machines/atlas.nix

@@ -5,6 +5,7 @@
 
     nixosModule.lab = {
       storage.profile = "kubernetes";
+      tailscale.enable = true;
 
       k3s = {
         enable = true;

machines/jefke.nix

@@ -5,6 +5,7 @@
 
     nixosModule.lab = {
       storage.profile = "kubernetes";
+      tailscale.enable = true;
 
       k3s = {
         enable = true;

machines/lewis.nix

@@ -11,6 +11,7 @@
         storage.profile = "kubernetes";
         backups.enable = true;
         data-sharing.enable = true;
+        tailscale.enable = true;
 
         k3s = {
           enable = true;

machines/warwick.nix

@@ -7,7 +7,11 @@
       lab = {
         storage.profile = "pi";
         monitoring.server.enable = true;
-        tailscale.advertiseExitNode = true;
+
+        tailscale = {
+          advertiseExitNode = true;
+          enable = true;
+        };
       };
     };
   };

my-lib/globals.nix

@@ -44,7 +44,7 @@
     nextcloudPostgres = "postgres:15";
     inbucket = "inbucket/inbucket:edge";
     syncthing = "lscr.io/linuxserver/syncthing:1.27.10";
-    radicale = "tomsquest/docker-radicale:3.2.2.0";
+    radicale = "nix:0/var/docker_images/radicale.tar";
     ntfy = "binwiederhier/ntfy:v2.11.0";
     forgejo = "codeberg.org/forgejo/forgejo:8.0.1";
     pihole = "pihole/pihole:2024.07.0";
@@ -53,7 +53,7 @@
     immichRedis = "docker.io/redis:6.2-alpine@sha256:e3b17ba9479deec4b7d1eeec1548a253acc5374d68d3b27937fcfe4df8d18c7e";
     immichPostgres = "docker.io/tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:90724186f0a3517cf6914295b5ab410db9ce23190a2d9d0b9dd6463e3fa298f0";
     kitchenowl = "tombursch/kitchenowl:v0.5.2";
-    cyberchef = "mpepping/cyberchef";
+    cyberchef = "nix:0/var/docker_images/cyberchef.tar";
     freshrss = "freshrss/freshrss:1.24.2";
     bind9 = "ubuntu/bind9:9.18-22.04_beta";
     dnsmasq = "dockurr/dnsmasq:2.90";

nixos-modules/k3s/default.nix

@@ -1,22 +1,6 @@
-{ inputs, pkgs, lib, config, ... }:
+{ self, inputs, pkgs, lib, config, ... }:
 let
   cfg = config.lab.k3s;
-
-  k3s-cni-plugins = pkgs.buildEnv {
-    name = "k3s-cni-plugins";
-    paths = with pkgs; [
-      cni-plugins
-      cni-plugin-flannel
-    ];
-  };
-
-  image = pkgs.nix-snapshotter.buildImage {
-    name = "redis";
-    resolvedByNix = true;
-    config = {
-      entrypoint = [ "${pkgs.redis}/bin/redis-server" ];
-    };
-  };
 in
 {
   options.lab.k3s = {
@@ -78,30 +62,40 @@ in
           address = "/run/nix-snapshotter/nix-snapshotter.sock";
         };
 
-        plugins = {
+        plugins =
-          "io.containerd.grpc.v1.cri" = {
+          let
-            stream_server_address = "127.0.0.1";
+            k3s-cni-plugins = pkgs.buildEnv {
-            stream_server_port = "10010";
+              name = "k3s-cni-plugins";
-            enable_selinux = false;
+              paths = with pkgs; [
-            enable_unprivileged_ports = true;
+                cni-plugins
-            enable_unprivileged_icmp = true;
+                cni-plugin-flannel
-            disable_apparmor = true;
+              ];
-            disable_cgroup = true;
-            restrict_oom_score_adj = true;
-            sandbox_image = "rancher/mirrored-pause:3.6";
-            containerd.snapshotter = "nix";
-
-            cni = {
-              conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/";
-              bin_dir = "${k3s-cni-plugins}/bin";
             };
-          };
+          in
+          {
+            "io.containerd.grpc.v1.cri" = {
+              stream_server_address = "127.0.0.1";
+              stream_server_port = "10010";
+              enable_selinux = false;
+              enable_unprivileged_ports = true;
+              enable_unprivileged_icmp = true;
+              disable_apparmor = true;
+              disable_cgroup = true;
+              restrict_oom_score_adj = true;
+              sandbox_image = "rancher/mirrored-pause:3.6";
+              containerd.snapshotter = "nix";
 
-          "io.containerd.transfer.v1.local".unpack_config = [{
+              cni = {
-            platform = "linux/amd64";
+                conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/";
-            snapshotter = "nix";
+                bin_dir = "${k3s-cni-plugins}/bin";
-          }];
+              };
-        };
+            };
+
+            "io.containerd.transfer.v1.local".unpack_config = [{
+              platform = "linux/amd64";
+              snapshotter = "nix";
+            }];
+          };
       };
     };
 
@@ -145,9 +139,9 @@ in
       "L+ /usr/local/bin - - - - /run/current-system/sw/bin/"
     ];
 
-    system = lib.mkIf (cfg.role == "server") {
+    system.activationScripts = {
-      activationScripts = {
+      k3s-bootstrap = lib.mkIf (cfg.role == "server") {
-        k3s-bootstrap.text = (
+        text = (
           let
             k3sBootstrapFile = (inputs.kubenix.evalModules.x86_64-linux {
               module = import ./bootstrap.nix;
@@ -158,8 +152,10 @@ in
             ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json
           ''
         );
+      };
 
-        k3s-certs.text = ''
+      k3s-certs = lib.mkIf (cfg.role == "server") {
+        text = ''
           mkdir -p /var/lib/rancher/k3s/server/tls/etcd
           cp -f ${./k3s-ca/server-ca.crt} /var/lib/rancher/k3s/server/tls/server-ca.crt
           cp -f ${./k3s-ca/client-ca.crt} /var/lib/rancher/k3s/server/tls/client-ca.crt
@@ -167,11 +163,47 @@ in
           cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt
           cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt
         '';
-
-        nix-snapshotter-image = ''
-          ln -sf ${image} /root/image.tar
-        '';
       };
+
+      docker-images.text =
+        let
+          imageLinkDir = "/var/docker_images";
+          imageDefs = import "${self}/container-images.nix";
+
+          setupCommands = [
+            "rm -rf ${imageLinkDir}"
+            "mkdir -p ${imageLinkDir}"
+          ];
+
+          getDockerImageConfig = dockerImage:
+            let
+              configJson = pkgs.runCommand "config.json"
+                {
+                  nativeBuildInputs = [ pkgs.skopeo pkgs.jq ];
+                }
+                ''
+                  skopeo --tmpdir $TMPDIR --insecure-policy inspect docker-archive:${dockerImage} --config | jq '.config' > $out
+                '';
+            in
+            builtins.fromJSON (builtins.readFile configJson);
+
+          imageDefToLinkCommand = name: imageDef:
+            let
+              dockerImage = pkgs.dockerTools.pullImage imageDef;
+              nixSnapshotterImage = pkgs.nix-snapshotter.buildImage {
+                inherit name;
+                resolvedByNix = true;
+                fromImage = dockerImage;
+                config = getDockerImageConfig dockerImage;
+              };
+              imageLinkPath = "${imageLinkDir}/${name}.tar";
+            in
+            "ln -sf ${nixSnapshotterImage} ${imageLinkPath}";
+
+          linkCommandList = lib.attrsets.mapAttrsToList imageDefToLinkCommand imageDefs;
+          commandList = setupCommands ++ linkCommandList;
+        in
+        builtins.concatStringsSep "\n" commandList;
     };
 
     sops.secrets =

nixos-modules/tailscale.nix

@@ -4,13 +4,17 @@ let
 in
 {
   options = {
-    lab.tailscale.advertiseExitNode = lib.mkOption {
+    lab.tailscale = {
-      type = lib.types.bool;
+      enable = lib.mkEnableOption "tailscale";
-      default = false;
+
+      advertiseExitNode = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+      };
     };
   };
 
-  config = {
+  config = lib.mkIf cfg.enable {
     services.tailscale = {
       enable = true;
       authKeyFile = config.sops.secrets."tailscale/authKey".path;
@@ -25,5 +29,7 @@ in
     };
 
     sops.secrets."tailscale/authKey" = { };
+
+    systemd.network.wait-online.ignoredInterfaces = [ "tailscale0" ];
   };
 }