From e9899c0d0f875cf17cc2bd50d360c524eb0fe4db Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Sun, 25 Aug 2024 17:04:31 +0200 Subject: [PATCH] Resolve cyberchef and radicale images using nix-snapshotter Increase inotify max user instances to 256 Disable tailscale by default --- configuration.nix | 1 + container-images.nix | 2 +- flake-parts/scripts/default.nix | 53 +++++++------- kubenix-modules/cyberchef.nix | 1 + machines/atlas.nix | 1 + machines/jefke.nix | 1 + machines/lewis.nix | 1 + machines/warwick.nix | 6 +- my-lib/globals.nix | 4 +- nixos-modules/k3s/default.nix | 126 ++++++++++++++++++++------------ nixos-modules/tailscale.nix | 14 +++- 11 files changed, 129 insertions(+), 81 deletions(-) diff --git a/configuration.nix b/configuration.nix index 3186878..72a46e9 100644 --- a/configuration.nix +++ b/configuration.nix @@ -91,6 +91,7 @@ boot = lib.mkIf (! machine.isRaspberryPi) { kernelModules = [ "kvm-intel" ]; extraModulePackages = [ ]; + kernel.sysctl."fs.inotify.max_user_instances" = 256; initrd = { kernelModules = [ ]; diff --git a/container-images.nix b/container-images.nix index 037defa..bc9d6a2 100644 --- a/container-images.nix +++ b/container-images.nix @@ -1,2 +1,2 @@ -{ cyberchef = { cyberchef = { finalImageName = "mpepping/cyberchef"; finalImageTag = "latest"; imageDigest = "sha256:5044c72dd8070fb6e0595e720fc4440bf6168493b2433db06a1c966406398ba2"; imageName = "mpepping/cyberchef"; sha256 = "177yjfbz0ijc8lfqfr50fhqqmjk72373c0igyrxv3wwg0pyrgpv4"; }; }; } +{ cyberchef = { finalImageName = "mpepping/cyberchef"; finalImageTag = "latest"; imageDigest = "sha256:2c89d08580395b932c92d708041c2a702dc8fa899fcc1677901c2dc881bed789"; imageName = "mpepping/cyberchef"; sha256 = "1frlvv6lyghf99pa37l48r7j2wvh7mcb9x99fvf0ba2zhq2xfsy4"; }; radicale = { finalImageName = "tomsquest/docker-radicale"; finalImageTag = "3.2.2.0"; imageDigest = "sha256:af050e02c4a3f7385a09595dd2a1424db6831aa9f24404095c6d2244d1c94138"; imageName = "tomsquest/docker-radicale"; sha256 = "07mxn6iqm0fhb06dwmxdhqnw3c8yi2dm2jpb2n80dxzmvavrv0lk"; }; } diff --git a/flake-parts/scripts/default.nix b/flake-parts/scripts/default.nix index 451e575..80d8ebd 100644 --- a/flake-parts/scripts/default.nix +++ b/flake-parts/scripts/default.nix @@ -30,12 +30,16 @@ in let images = { cyberchef = { - cyberchef = { - image-name = "mpepping/cyberchef"; - image-tag = "latest"; - }; + name = "mpepping/cyberchef"; + tag = "latest"; + }; + + radicale = { + name = "tomsquest/docker-radicale"; + tag = "3.2.2.0"; }; }; + imagesJSON = builtins.toFile "images.json" (builtins.toJSON images); in pkgs.writers.writePython3Bin "prefetch-container-images" @@ -55,31 +59,28 @@ in with open(images_file_name, 'r') as file: data = json.load(file) - for project_name, images in data.items(): - print(f"Prefetching images for project {project_name}", file=sys.stderr) + for image_name, image in data.items(): + name = image["name"] + tag = image["tag"] - for image_name, image in images.items(): - name = image["image-name"] - tag = image["image-tag"] + print(f"Prefetching image {name}:{tag}", file=sys.stderr) - print(f"Prefetching image {name}:{tag}", file=sys.stderr) + prefetch_args = [ + prefetch_docker_cmd, + "--os", "linux", + "--arch", "amd64", + "--image-name", name, + "--image-tag", tag, + "--json", + "--quiet" + ] + result = subprocess.run(prefetch_args, + check=True, + capture_output=True, + text=True) - prefetch_args = [ - prefetch_docker_cmd, - "--os", "linux", - "--arch", "amd64", - "--image-name", name, - "--image-tag", tag, - "--json", - "--quiet" - ] - result = subprocess.run(prefetch_args, - check=True, - capture_output=True, - text=True) - - prefetch_data = json.loads(result.stdout) - results[project_name][image_name] = prefetch_data + prefetch_data = json.loads(result.stdout) + results[image_name] = prefetch_data with tempfile.NamedTemporaryFile(mode='w+', suffix='.json') as temp_file: json.dump(results, temp_file, indent=4) diff --git a/kubenix-modules/cyberchef.nix b/kubenix-modules/cyberchef.nix index d7ab260..44bbfef 100644 --- a/kubenix-modules/cyberchef.nix +++ b/kubenix-modules/cyberchef.nix @@ -9,6 +9,7 @@ spec.containers.cyberchef = { image = myLib.globals.images.cyberchef; + imagePullPolicy = "Always"; ports.web.containerPort = 8000; }; }; diff --git a/machines/atlas.nix b/machines/atlas.nix index a9866d4..a6bf86f 100644 --- a/machines/atlas.nix +++ b/machines/atlas.nix @@ -5,6 +5,7 @@ nixosModule.lab = { storage.profile = "kubernetes"; + tailscale.enable = true; k3s = { enable = true; diff --git a/machines/jefke.nix b/machines/jefke.nix index 362b268..096be8c 100644 --- a/machines/jefke.nix +++ b/machines/jefke.nix @@ -5,6 +5,7 @@ nixosModule.lab = { storage.profile = "kubernetes"; + tailscale.enable = true; k3s = { enable = true; diff --git a/machines/lewis.nix b/machines/lewis.nix index 6999c81..5350142 100644 --- a/machines/lewis.nix +++ b/machines/lewis.nix @@ -11,6 +11,7 @@ storage.profile = "kubernetes"; backups.enable = true; data-sharing.enable = true; + tailscale.enable = true; k3s = { enable = true; diff --git a/machines/warwick.nix b/machines/warwick.nix index 6160f02..f000881 100644 --- a/machines/warwick.nix +++ b/machines/warwick.nix @@ -7,7 +7,11 @@ lab = { storage.profile = "pi"; monitoring.server.enable = true; - tailscale.advertiseExitNode = true; + + tailscale = { + advertiseExitNode = true; + enable = true; + }; }; }; }; diff --git a/my-lib/globals.nix b/my-lib/globals.nix index 5daa29e..a5a9808 100644 --- a/my-lib/globals.nix +++ b/my-lib/globals.nix @@ -44,7 +44,7 @@ nextcloudPostgres = "postgres:15"; inbucket = "inbucket/inbucket:edge"; syncthing = "lscr.io/linuxserver/syncthing:1.27.10"; - radicale = "tomsquest/docker-radicale:3.2.2.0"; + radicale = "nix:0/var/docker_images/radicale.tar"; ntfy = "binwiederhier/ntfy:v2.11.0"; forgejo = "codeberg.org/forgejo/forgejo:8.0.1"; pihole = "pihole/pihole:2024.07.0"; @@ -53,7 +53,7 @@ immichRedis = "docker.io/redis:6.2-alpine@sha256:e3b17ba9479deec4b7d1eeec1548a253acc5374d68d3b27937fcfe4df8d18c7e"; immichPostgres = "docker.io/tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:90724186f0a3517cf6914295b5ab410db9ce23190a2d9d0b9dd6463e3fa298f0"; kitchenowl = "tombursch/kitchenowl:v0.5.2"; - cyberchef = "mpepping/cyberchef"; + cyberchef = "nix:0/var/docker_images/cyberchef.tar"; freshrss = "freshrss/freshrss:1.24.2"; bind9 = "ubuntu/bind9:9.18-22.04_beta"; dnsmasq = "dockurr/dnsmasq:2.90"; diff --git a/nixos-modules/k3s/default.nix b/nixos-modules/k3s/default.nix index d47f182..52def8c 100644 --- a/nixos-modules/k3s/default.nix +++ b/nixos-modules/k3s/default.nix @@ -1,22 +1,6 @@ -{ inputs, pkgs, lib, config, ... }: +{ self, inputs, pkgs, lib, config, ... }: let cfg = config.lab.k3s; - - k3s-cni-plugins = pkgs.buildEnv { - name = "k3s-cni-plugins"; - paths = with pkgs; [ - cni-plugins - cni-plugin-flannel - ]; - }; - - image = pkgs.nix-snapshotter.buildImage { - name = "redis"; - resolvedByNix = true; - config = { - entrypoint = [ "${pkgs.redis}/bin/redis-server" ]; - }; - }; in { options.lab.k3s = { @@ -78,30 +62,40 @@ in address = "/run/nix-snapshotter/nix-snapshotter.sock"; }; - plugins = { - "io.containerd.grpc.v1.cri" = { - stream_server_address = "127.0.0.1"; - stream_server_port = "10010"; - enable_selinux = false; - enable_unprivileged_ports = true; - enable_unprivileged_icmp = true; - disable_apparmor = true; - disable_cgroup = true; - restrict_oom_score_adj = true; - sandbox_image = "rancher/mirrored-pause:3.6"; - containerd.snapshotter = "nix"; - - cni = { - conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/"; - bin_dir = "${k3s-cni-plugins}/bin"; + plugins = + let + k3s-cni-plugins = pkgs.buildEnv { + name = "k3s-cni-plugins"; + paths = with pkgs; [ + cni-plugins + cni-plugin-flannel + ]; }; - }; + in + { + "io.containerd.grpc.v1.cri" = { + stream_server_address = "127.0.0.1"; + stream_server_port = "10010"; + enable_selinux = false; + enable_unprivileged_ports = true; + enable_unprivileged_icmp = true; + disable_apparmor = true; + disable_cgroup = true; + restrict_oom_score_adj = true; + sandbox_image = "rancher/mirrored-pause:3.6"; + containerd.snapshotter = "nix"; - "io.containerd.transfer.v1.local".unpack_config = [{ - platform = "linux/amd64"; - snapshotter = "nix"; - }]; - }; + cni = { + conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/"; + bin_dir = "${k3s-cni-plugins}/bin"; + }; + }; + + "io.containerd.transfer.v1.local".unpack_config = [{ + platform = "linux/amd64"; + snapshotter = "nix"; + }]; + }; }; }; @@ -145,9 +139,9 @@ in "L+ /usr/local/bin - - - - /run/current-system/sw/bin/" ]; - system = lib.mkIf (cfg.role == "server") { - activationScripts = { - k3s-bootstrap.text = ( + system.activationScripts = { + k3s-bootstrap = lib.mkIf (cfg.role == "server") { + text = ( let k3sBootstrapFile = (inputs.kubenix.evalModules.x86_64-linux { module = import ./bootstrap.nix; @@ -158,8 +152,10 @@ in ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json '' ); + }; - k3s-certs.text = '' + k3s-certs = lib.mkIf (cfg.role == "server") { + text = '' mkdir -p /var/lib/rancher/k3s/server/tls/etcd cp -f ${./k3s-ca/server-ca.crt} /var/lib/rancher/k3s/server/tls/server-ca.crt cp -f ${./k3s-ca/client-ca.crt} /var/lib/rancher/k3s/server/tls/client-ca.crt @@ -167,11 +163,47 @@ in cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt ''; - - nix-snapshotter-image = '' - ln -sf ${image} /root/image.tar - ''; }; + + docker-images.text = + let + imageLinkDir = "/var/docker_images"; + imageDefs = import "${self}/container-images.nix"; + + setupCommands = [ + "rm -rf ${imageLinkDir}" + "mkdir -p ${imageLinkDir}" + ]; + + getDockerImageConfig = dockerImage: + let + configJson = pkgs.runCommand "config.json" + { + nativeBuildInputs = [ pkgs.skopeo pkgs.jq ]; + } + '' + skopeo --tmpdir $TMPDIR --insecure-policy inspect docker-archive:${dockerImage} --config | jq '.config' > $out + ''; + in + builtins.fromJSON (builtins.readFile configJson); + + imageDefToLinkCommand = name: imageDef: + let + dockerImage = pkgs.dockerTools.pullImage imageDef; + nixSnapshotterImage = pkgs.nix-snapshotter.buildImage { + inherit name; + resolvedByNix = true; + fromImage = dockerImage; + config = getDockerImageConfig dockerImage; + }; + imageLinkPath = "${imageLinkDir}/${name}.tar"; + in + "ln -sf ${nixSnapshotterImage} ${imageLinkPath}"; + + linkCommandList = lib.attrsets.mapAttrsToList imageDefToLinkCommand imageDefs; + commandList = setupCommands ++ linkCommandList; + in + builtins.concatStringsSep "\n" commandList; }; sops.secrets = diff --git a/nixos-modules/tailscale.nix b/nixos-modules/tailscale.nix index 8cfbfb3..d50408e 100644 --- a/nixos-modules/tailscale.nix +++ b/nixos-modules/tailscale.nix @@ -4,13 +4,17 @@ let in { options = { - lab.tailscale.advertiseExitNode = lib.mkOption { - type = lib.types.bool; - default = false; + lab.tailscale = { + enable = lib.mkEnableOption "tailscale"; + + advertiseExitNode = lib.mkOption { + type = lib.types.bool; + default = false; + }; }; }; - config = { + config = lib.mkIf cfg.enable { services.tailscale = { enable = true; authKeyFile = config.sops.secrets."tailscale/authKey".path; @@ -25,5 +29,7 @@ in }; sops.secrets."tailscale/authKey" = { }; + + systemd.network.wait-online.ignoredInterfaces = [ "tailscale0" ]; }; }