{ self, inputs, pkgs, lib, config, ... }: let cfg = config.lab.k3s; k3s-cni-plugins = pkgs.buildEnv { name = "k3s-cni-plugins"; paths = with pkgs; [ cni-plugins cni-plugin-flannel ]; }; image = pkgs.nix-snapshotter.buildImage { name = "redis"; resolvedByNix = true; config = { entrypoint = [ "${pkgs.redis}/bin/redis-server" ]; }; }; in { options.lab.k3s = { enable = lib.mkOption { default = false; type = lib.types.bool; description = '' Whether to run k3s on this server. ''; }; role = lib.mkOption { default = "server"; type = lib.types.str; description = '' Whether to run k3s as a server or an agent. ''; }; clusterInit = lib.mkOption { default = false; type = lib.types.bool; description = '' Whether this node should initialize the K8s cluster. ''; }; serverAddr = lib.mkOption { default = null; type = with lib.types; nullOr str; description = '' Address of the server whose cluster this server should join. Leaving this empty will make the server initialize the cluster. ''; }; }; config = lib.mkIf cfg.enable { environment.systemPackages = with pkgs; [ k3s openiscsi # Required for Longhorn nfs-utils # Required for Longhorn ]; networking = { nftables.enable = lib.mkForce false; firewall.enable = lib.mkForce false; }; virtualisation.containerd = { enable = true; settings = { version = 2; proxy_plugins.nix = { type = "snapshot"; address = "/run/nix-snapshotter/nix-snapshotter.sock"; }; plugins = { "io.containerd.grpc.v1.cri" = { stream_server_address = "127.0.0.1"; stream_server_port = "10010"; enable_selinux = false; enable_unprivileged_ports = true; enable_unprivileged_icmp = true; disable_apparmor = true; disable_cgroup = true; restrict_oom_score_adj = true; sandbox_image = "rancher/mirrored-pause:3.6"; containerd.snapshotter = "nix"; cni = { conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/"; bin_dir = "${k3s-cni-plugins}/bin"; }; }; "io.containerd.transfer.v1.local".unpack_config = [{ platform = "linux/amd64"; snapshotter = "nix"; }]; }; }; }; services = { nix-snapshotter.enable = true; k3s = let serverFlagList = [ "--image-service-endpoint=unix:///run/nix-snapshotter/nix-snapshotter.sock" "--snapshotter=overlayfs" "--container-runtime-endpoint=unix:///run/containerd/containerd.sock" "--tls-san=${config.networking.fqdn}" "--disable=servicelb" "--cluster-cidr=10.42.0.0/16,2001:cafe:42::/56" "--service-cidr=10.43.0.0/16,2001:cafe:43::/112" ]; serverFlags = builtins.concatStringsSep " " serverFlagList; in { enable = true; role = cfg.role; tokenFile = config.sops.secrets."k3s/serverToken".path; extraFlags = lib.mkIf (cfg.role == "server") (lib.mkForce serverFlags); clusterInit = cfg.clusterInit; serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr; }; # Required for Longhorn openiscsi = { enable = true; name = "iqn.2016-04.com.open-iscsi:${config.networking.fqdn}"; }; }; # HACK: Symlink binaries to /usr/local/bin such that Longhorn can find them # when they use nsenter. # https://github.com/longhorn/longhorn/issues/2166#issuecomment-1740179416 systemd.tmpfiles.rules = [ "L+ /usr/local/bin - - - - /run/current-system/sw/bin/" ]; system = lib.mkIf (cfg.role == "server") { activationScripts = { k3s-bootstrap.text = ( let k3sBootstrapFile = (inputs.kubenix.evalModules.x86_64-linux { module = import ./bootstrap.nix; }).config.kubernetes.result; in '' mkdir -p /var/lib/rancher/k3s/server/manifests ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json '' ); k3s-certs.text = '' mkdir -p /var/lib/rancher/k3s/server/tls/etcd cp -f ${./k3s-ca/server-ca.crt} /var/lib/rancher/k3s/server/tls/server-ca.crt cp -f ${./k3s-ca/client-ca.crt} /var/lib/rancher/k3s/server/tls/client-ca.crt cp -f ${./k3s-ca/request-header-ca.crt} /var/lib/rancher/k3s/server/tls/request-header-ca.crt cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt ''; docker-images = let pulledImages = (import "${self}/container-images/pulled-images.nix") pkgs lib; basePath = "/var/docker_images"; linesForImage = projectName: imageName: pulledImage: let projectPath = "${basePath}/${projectName}"; in '' mkdir -p ${projectPath} ln -sf ${pulledImage} ${projectPath}/${imageName}.tar ''; linesForProject = projectName: project: let lines = lib.attrsets.mapAttrsToList (linesForImage projectName) project; in builtins.concatStringsSep "\n" lines; generateLines = projects: let lines = lib.attrsets.mapAttrsToList linesForProject projects; in builtins.concatStringsSep "\n" lines; in generateLines pulledImages; }; }; sops.secrets = let keyPathBase = "/var/lib/rancher/k3s/server/tls"; in { "k3s/serverToken" = { }; "k3s/keys/clientCAKey".path = "${keyPathBase}/client-ca.key"; "k3s/keys/requestHeaderCAKey".path = "${keyPathBase}/request-header-ca.key"; "k3s/keys/serverCAKey".path = "${keyPathBase}/server-ca.key"; "k3s/keys/serviceKey".path = "${keyPathBase}/service.key"; "k3s/keys/etcd/peerCAKey".path = "${keyPathBase}/etcd/peer-ca.key"; "k3s/keys/etcd/serverCAKey".path = "${keyPathBase}/etcd/server-ca.key"; }; }; }