From e7d75fbb21f8a599f9bc2543a22ad067c0898b12 Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Mon, 24 Jun 2024 23:31:06 +0200 Subject: [PATCH] Use nix-snapshotter as k3s' snapshotter and image service Replace k3s' embedded containerd with Nix-managed one Deploy test k8s with image from Nix store --- configuration.nix | 1 + flake.lock | 60 ++++++++++++++++++++++++++ flake.nix | 5 +++ kubenix-modules/all.nix | 20 +++++++++ nixos-modules/k3s/default.nix | 80 +++++++++++++++++++++++++++++++++-- 5 files changed, 162 insertions(+), 4 deletions(-) diff --git a/configuration.nix b/configuration.nix index d5072f3..e53a6f7 100644 --- a/configuration.nix +++ b/configuration.nix @@ -4,6 +4,7 @@ machine.nixosModule inputs.disko.nixosModules.disko inputs.sops-nix.nixosModules.sops + inputs.nix-snapshotter.nixosModules.nix-snapshotter ] ++ lib.lists.optional (machine.isRaspberryPi) inputs.nixos-hardware.nixosModules.raspberry-pi-4; config = { diff --git a/flake.lock b/flake.lock index 591cb37..68221e4 100644 --- a/flake.lock +++ b/flake.lock @@ -115,6 +115,43 @@ "type": "github" } }, + "flake-compat_3": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "nix-snapshotter", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1704152458, + "narHash": "sha256-DS+dGw7SKygIWf9w4eNBUZsK+4Ug27NwEWmn2tnbycg=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "88a2cd8166694ba0b6cb374700799cec53aef527", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "locked": { "lastModified": 1614513358, @@ -277,6 +314,28 @@ "type": "github" } }, + "nix-snapshotter": { + "inputs": { + "flake-compat": "flake-compat_3", + "flake-parts": "flake-parts", + "nixpkgs": [ + "nixpkgs-unstable" + ] + }, + "locked": { + "lastModified": 1715438114, + "narHash": "sha256-btb702TXuhDg0D6tW0dCOy4+II9Wl6BJ0LvpT+O9wrs=", + "owner": "pdtpartners", + "repo": "nix-snapshotter", + "rev": "7b251c9356bc7bb383ebeedcd0045b3ae431bff7", + "type": "github" + }, + "original": { + "owner": "pdtpartners", + "repo": "nix-snapshotter", + "type": "github" + } + }, "nixhelm": { "inputs": { "flake-utils": "flake-utils_3", @@ -413,6 +472,7 @@ "dns": "dns", "flake-utils": "flake-utils_2", "kubenix": "kubenix", + "nix-snapshotter": "nix-snapshotter", "nixhelm": "nixhelm", "nixos-hardware": "nixos-hardware", "nixpkgs": "nixpkgs_2", diff --git a/flake.nix b/flake.nix index 022af5d..23f85a8 100644 --- a/flake.nix +++ b/flake.nix @@ -37,6 +37,11 @@ url = "github:Mic92/sops-nix"; inputs.nixpkgs.follows = "nixpkgs"; }; + + nix-snapshotter = { + url = "github:pdtpartners/nix-snapshotter"; + inputs.nixpkgs.follows = "nixpkgs-unstable"; + }; }; outputs = diff --git a/kubenix-modules/all.nix b/kubenix-modules/all.nix index eb470c4..fe91bc5 100644 --- a/kubenix-modules/all.nix +++ b/kubenix-modules/all.nix @@ -24,6 +24,26 @@ let ]; in { + kubernetes.resources.pods.testje.spec = { + containers.redis = { + image = "nix:0/nix/store/5nmh9qawhbwinzxidafjlfw68wfkh0pj-nix-image-redis.tar"; + args = [ "--protected-mode" "no" ]; + + ports = [{ + name = "redis"; + containerPort = 6379; + }]; + }; + + affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms = [{ + matchExpressions = [{ + key = "kubernetes.io/hostname"; + operator = "In"; + values = [ "atlas" ]; + }]; + }]; + }; + imports = [ ./base.nix ./longhorn.nix diff --git a/nixos-modules/k3s/default.nix b/nixos-modules/k3s/default.nix index 91ed160..4c902d9 100644 --- a/nixos-modules/k3s/default.nix +++ b/nixos-modules/k3s/default.nix @@ -1,6 +1,24 @@ { inputs, pkgs, lib, config, ... }: -let cfg = config.lab.k3s; -in { +let + cfg = config.lab.k3s; + + k3s-cni-plugins = pkgs.buildEnv { + name = "k3s-cni-plugins"; + paths = with pkgs; [ + cni-plugins + cni-plugin-flannel + ]; + }; + + image = pkgs.nix-snapshotter.buildImage { + name = "redis"; + resolvedByNix = true; + config = { + entrypoint = [ "${pkgs.redis}/bin/redis-server" ]; + }; + }; +in +{ options.lab.k3s = { enable = lib.mkOption { default = false; @@ -48,16 +66,66 @@ in { firewall.enable = lib.mkForce false; }; + virtualisation.containerd = { + enable = true; + + settings = { + version = 2; + + proxy_plugins.nix = { + type = "snapshot"; + address = "/run/nix-snapshotter/nix-snapshotter.sock"; + }; + + plugins = { + "io.containerd.grpc.v1.cri" = { + stream_server_address = "127.0.0.1"; + stream_server_port = "10010"; + enable_selinux = false; + enable_unprivileged_ports = true; + enable_unprivileged_icmp = true; + disable_apparmor = true; + disable_cgroup = true; + restrict_oom_score_adj = true; + sandbox_image = "rancher/mirrored-pause:3.6"; + containerd.snapshotter = "nix"; + + cni = { + conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/"; + bin_dir = "${k3s-cni-plugins}/bin"; + }; + }; + + "io.containerd.transfer.v1.local".unpack_config = [{ + platform = "linux/amd64"; + snapshotter = "nix"; + }]; + }; + }; + }; + services = { + nix-snapshotter.enable = true; + k3s = let - serverFlags = "--tls-san ${config.networking.fqdn} --disable servicelb --cluster-cidr=10.42.0.0/16,2001:cafe:42::/56 --service-cidr=10.43.0.0/16,2001:cafe:43::/112"; + serverFlagList = [ + "--image-service-endpoint=unix:///run/nix-snapshotter/nix-snapshotter.sock" + "--snapshotter=overlayfs" + "--container-runtime-endpoint=unix:///run/containerd/containerd.sock" + "--tls-san=${config.networking.fqdn}" + "--disable=servicelb" + "--cluster-cidr=10.42.0.0/16,2001:cafe:42::/56" + "--service-cidr=10.43.0.0/16,2001:cafe:43::/112" + ]; + + serverFlags = builtins.concatStringsSep " " serverFlagList; in { enable = true; role = cfg.role; tokenFile = config.sops.secrets."k3s/serverToken".path; - extraFlags = lib.mkIf (cfg.role == "server") serverFlags; + extraFlags = lib.mkIf (cfg.role == "server") (lib.mkForce serverFlags); clusterInit = cfg.clusterInit; serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr; }; @@ -98,6 +166,10 @@ in { cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt ''; + + nix-snapshotter-image = '' + ln -sf ${image} /root/image.tar + ''; }; };