Use nix-snapshotter as k3s' snapshotter and image service

Replace k3s' embedded containerd with Nix-managed one
Deploy test k8s with image from Nix store

configuration.nix

@@ -4,6 +4,7 @@
     machine.nixosModule
     inputs.disko.nixosModules.disko
     inputs.sops-nix.nixosModules.sops
+    inputs.nix-snapshotter.nixosModules.nix-snapshotter
   ] ++ lib.lists.optional (machine.isRaspberryPi) inputs.nixos-hardware.nixosModules.raspberry-pi-4;
 
   config = {

flake.lock

@@ -115,6 +115,43 @@
         "type": "github"
       }
     },
+    "flake-compat_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nix-snapshotter",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1704152458,
+        "narHash": "sha256-DS+dGw7SKygIWf9w4eNBUZsK+4Ug27NwEWmn2tnbycg=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "88a2cd8166694ba0b6cb374700799cec53aef527",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "locked": {
         "lastModified": 1614513358,
@@ -277,6 +314,28 @@
         "type": "github"
       }
     },
+    "nix-snapshotter": {
+      "inputs": {
+        "flake-compat": "flake-compat_3",
+        "flake-parts": "flake-parts",
+        "nixpkgs": [
+          "nixpkgs-unstable"
+        ]
+      },
+      "locked": {
+        "lastModified": 1715438114,
+        "narHash": "sha256-btb702TXuhDg0D6tW0dCOy4+II9Wl6BJ0LvpT+O9wrs=",
+        "owner": "pdtpartners",
+        "repo": "nix-snapshotter",
+        "rev": "7b251c9356bc7bb383ebeedcd0045b3ae431bff7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pdtpartners",
+        "repo": "nix-snapshotter",
+        "type": "github"
+      }
+    },
     "nixhelm": {
       "inputs": {
         "flake-utils": "flake-utils_3",
@@ -413,6 +472,7 @@
         "dns": "dns",
         "flake-utils": "flake-utils_2",
         "kubenix": "kubenix",
+        "nix-snapshotter": "nix-snapshotter",
         "nixhelm": "nixhelm",
         "nixos-hardware": "nixos-hardware",
         "nixpkgs": "nixpkgs_2",

flake.nix

@@ -37,6 +37,11 @@
       url = "github:Mic92/sops-nix";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+
+    nix-snapshotter = {
+      url = "github:pdtpartners/nix-snapshotter";
+      inputs.nixpkgs.follows = "nixpkgs-unstable";
+    };
   };
 
   outputs =

kubenix-modules/all.nix

@@ -24,6 +24,26 @@ let
   ];
 in
 {
+  kubernetes.resources.pods.testje.spec = {
+    containers.redis = {
+      image = "nix:0/nix/store/5nmh9qawhbwinzxidafjlfw68wfkh0pj-nix-image-redis.tar";
+      args = [ "--protected-mode" "no" ];
+
+      ports = [{
+        name = "redis";
+        containerPort = 6379;
+      }];
+    };
+
+    affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{
+      matchExpressions = [{
+        key = "kubernetes.io/hostname";
+        operator = "In";
+        values = [ "atlas" ];
+      }];
+    }];
+  };
+
   imports = [
     ./base.nix
     ./longhorn.nix

nixos-modules/k3s/default.nix

@@ -1,6 +1,24 @@
 { inputs, pkgs, lib, config, ... }:
-let cfg = config.lab.k3s;
+let
-in {
+  cfg = config.lab.k3s;
+
+  k3s-cni-plugins = pkgs.buildEnv {
+    name = "k3s-cni-plugins";
+    paths = with pkgs; [
+      cni-plugins
+      cni-plugin-flannel
+    ];
+  };
+
+  image = pkgs.nix-snapshotter.buildImage {
+    name = "redis";
+    resolvedByNix = true;
+    config = {
+      entrypoint = [ "${pkgs.redis}/bin/redis-server" ];
+    };
+  };
+in
+{
   options.lab.k3s = {
     enable = lib.mkOption {
       default = false;
@@ -48,16 +66,66 @@ in {
       firewall.enable = lib.mkForce false;
     };
 
+    virtualisation.containerd = {
+      enable = true;
+
+      settings = {
+        version = 2;
+
+        proxy_plugins.nix = {
+          type = "snapshot";
+          address = "/run/nix-snapshotter/nix-snapshotter.sock";
+        };
+
+        plugins = {
+          "io.containerd.grpc.v1.cri" = {
+            stream_server_address = "127.0.0.1";
+            stream_server_port = "10010";
+            enable_selinux = false;
+            enable_unprivileged_ports = true;
+            enable_unprivileged_icmp = true;
+            disable_apparmor = true;
+            disable_cgroup = true;
+            restrict_oom_score_adj = true;
+            sandbox_image = "rancher/mirrored-pause:3.6";
+            containerd.snapshotter = "nix";
+
+            cni = {
+              conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/";
+              bin_dir = "${k3s-cni-plugins}/bin";
+            };
+          };
+
+          "io.containerd.transfer.v1.local".unpack_config = [{
+            platform = "linux/amd64";
+            snapshotter = "nix";
+          }];
+        };
+      };
+    };
+
     services = {
+      nix-snapshotter.enable = true;
+
       k3s =
         let
-          serverFlags = "--tls-san ${config.networking.fqdn} --disable servicelb --cluster-cidr=10.42.0.0/16,2001:cafe:42::/56 --service-cidr=10.43.0.0/16,2001:cafe:43::/112";
+          serverFlagList = [
+            "--image-service-endpoint=unix:///run/nix-snapshotter/nix-snapshotter.sock"
+            "--snapshotter=overlayfs"
+            "--container-runtime-endpoint=unix:///run/containerd/containerd.sock"
+            "--tls-san=${config.networking.fqdn}"
+            "--disable=servicelb"
+            "--cluster-cidr=10.42.0.0/16,2001:cafe:42::/56"
+            "--service-cidr=10.43.0.0/16,2001:cafe:43::/112"
+          ];
+
+          serverFlags = builtins.concatStringsSep " " serverFlagList;
         in
         {
           enable = true;
           role = cfg.role;
           tokenFile = config.sops.secrets."k3s/serverToken".path;
-          extraFlags = lib.mkIf (cfg.role == "server") serverFlags;
+          extraFlags = lib.mkIf (cfg.role == "server") (lib.mkForce serverFlags);
           clusterInit = cfg.clusterInit;
           serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr;
         };
@@ -98,6 +166,10 @@ in {
           cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt
           cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt
         '';
+
+        nix-snapshotter-image = ''
+          ln -sf ${image} /root/image.tar
+        '';
       };
     };