{ self, inputs, pkgs, lib, config, ... }: let cfg = config.lab.k3s; in { options.lab.k3s = { enable = lib.mkOption { default = false; type = lib.types.bool; description = '' Whether to run k3s on this server. ''; }; role = lib.mkOption { default = "server"; type = lib.types.str; description = '' Whether to run k3s as a server or an agent. ''; }; clusterInit = lib.mkOption { default = false; type = lib.types.bool; description = '' Whether this node should initialize the K8s cluster. ''; }; serverAddr = lib.mkOption { default = null; type = with lib.types; nullOr str; description = '' Address of the server whose cluster this server should join. Leaving this empty will make the server initialize the cluster. ''; }; }; config = lib.mkIf cfg.enable { environment.systemPackages = with pkgs; [ k3s openiscsi # Required for Longhorn nfs-utils # Required for Longhorn ]; # TODO!!!!! networking = { nftables.enable = lib.mkForce false; firewall.enable = lib.mkForce false; }; virtualisation.containerd = { enable = true; settings = { version = 2; proxy_plugins.nix = { type = "snapshot"; address = "/run/nix-snapshotter/nix-snapshotter.sock"; }; plugins = let k3s-cni-plugins = pkgs.buildEnv { name = "k3s-cni-plugins"; paths = with pkgs; [ cni-plugins cni-plugin-flannel ]; }; in { "io.containerd.grpc.v1.cri" = { stream_server_address = "127.0.0.1"; stream_server_port = "10010"; enable_selinux = false; enable_unprivileged_ports = true; enable_unprivileged_icmp = true; disable_apparmor = true; disable_cgroup = true; restrict_oom_score_adj = true; sandbox_image = "rancher/mirrored-pause:3.6"; containerd.snapshotter = "nix"; cni = { conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/"; bin_dir = "${k3s-cni-plugins}/bin"; }; }; "io.containerd.transfer.v1.local".unpack_config = [{ platform = "linux/amd64"; snapshotter = "nix"; }]; }; }; }; services = { nix-snapshotter.enable = true; k3s = let serverFlagList = [ "--image-service-endpoint=unix:///run/nix-snapshotter/nix-snapshotter.sock" "--snapshotter=overlayfs" "--container-runtime-endpoint=unix:///run/containerd/containerd.sock" "--tls-san=${config.networking.fqdn}" "--disable=servicelb" "--cluster-cidr=10.42.0.0/16,2001:cafe:42::/56" "--service-cidr=10.43.0.0/16,2001:cafe:43::/112" ]; serverFlags = builtins.concatStringsSep " " serverFlagList; in { enable = true; role = cfg.role; tokenFile = config.sops.secrets."k3s/serverToken".path; extraFlags = lib.mkIf (cfg.role == "server") (lib.mkForce serverFlags); clusterInit = cfg.clusterInit; serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr; }; # Required for Longhorn openiscsi = { enable = true; name = "iqn.2016-04.com.open-iscsi:${config.networking.fqdn}"; }; }; # HACK: Symlink binaries to /usr/local/bin such that Longhorn can find them # when they use nsenter. # https://github.com/longhorn/longhorn/issues/2166#issuecomment-1740179416 systemd.tmpfiles.rules = [ "L+ /usr/local/bin - - - - /run/current-system/sw/bin/" ]; system.activationScripts = { k3s-bootstrap = lib.mkIf (cfg.role == "server") { text = ( let k3sBootstrapFile = (inputs.kubenix.evalModules.x86_64-linux { module = import ./bootstrap.nix; }).config.kubernetes.result; in '' mkdir -p /var/lib/rancher/k3s/server/manifests ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json '' ); }; k3s-certs = lib.mkIf (cfg.role == "server") { text = '' mkdir -p /var/lib/rancher/k3s/server/tls/etcd cp -f ${./k3s-ca/server-ca.crt} /var/lib/rancher/k3s/server/tls/server-ca.crt cp -f ${./k3s-ca/client-ca.crt} /var/lib/rancher/k3s/server/tls/client-ca.crt cp -f ${./k3s-ca/request-header-ca.crt} /var/lib/rancher/k3s/server/tls/request-header-ca.crt cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt ''; }; docker-images.text = let imageDefs = import "${self}/container-images.nix"; setupCommands = [ "rm -rf ${self.globals.imageDir}" "mkdir -p ${self.globals.imageDir}" ]; getDockerImageConfig = dockerImage: let configJson = pkgs.runCommand "config.json" { nativeBuildInputs = [ pkgs.skopeo pkgs.jq ]; } '' skopeo --tmpdir $TMPDIR --insecure-policy inspect docker-archive:${dockerImage} --config | jq '.config' > $out ''; in builtins.fromJSON (builtins.readFile configJson); imageDefToLinkCommand = name: imageDef: let dockerImage = pkgs.dockerTools.pullImage imageDef; nixSnapshotterImage = pkgs.nix-snapshotter.buildImage { inherit name; resolvedByNix = true; fromImage = dockerImage; config = getDockerImageConfig dockerImage; }; imageLinkPath = "${self.globals.imageDir}/${name}.tar"; in "ln -sf ${nixSnapshotterImage} ${imageLinkPath}"; linkCommandList = lib.attrsets.mapAttrsToList imageDefToLinkCommand imageDefs; # TODO: Creating Docker images like this seems to *explode* in size. # Doing this for every image we currently have is infeasible. # I should investigate why the size increases like that. commandList = setupCommands; # ++ linkCommandList; in builtins.concatStringsSep "\n" commandList; }; sops.secrets = let keyPathBase = "/var/lib/rancher/k3s/server/tls"; in { "k3s/serverToken" = { }; "k3s/keys/clientCAKey".path = "${keyPathBase}/client-ca.key"; "k3s/keys/requestHeaderCAKey".path = "${keyPathBase}/request-header-ca.key"; "k3s/keys/serverCAKey".path = "${keyPathBase}/server-ca.key"; "k3s/keys/serviceKey".path = "${keyPathBase}/service.key"; "k3s/keys/etcd/peerCAKey".path = "${keyPathBase}/etcd/peer-ca.key"; "k3s/keys/etcd/serverCAKey".path = "${keyPathBase}/etcd/server-ca.key"; }; }; }