{ pkgs, lib, config, kubenix, ... }: let cfg = config.lab.k3s; in { options.lab.k3s = { enable = lib.mkOption { default = false; type = lib.types.bool; description = '' Whether to run k3s on this server. ''; }; role = lib.mkOption { default = "server"; type = lib.types.str; description = '' Whether to run k3s as a server or an agent. ''; }; clusterInit = lib.mkOption { default = false; type = lib.types.bool; description = '' Whether this node should initialize the K8s cluster. ''; }; serverAddr = lib.mkOption { default = null; type = with lib.types; nullOr str; description = '' Address of the server whose cluster this server should join. Leaving this empty will make the server initialize the cluster. ''; }; }; config = lib.mkIf cfg.enable { environment.systemPackages = with pkgs; [ k3s openiscsi # Required for Longhorn nfs-utils # Required for Longhorn ]; networking = { nftables.enable = lib.mkForce false; firewall.enable = lib.mkForce false; }; services = { k3s = let serverFlags = "--tls-san ${config.networking.fqdn} --disable servicelb --cluster-cidr=10.42.0.0/16,2001:cafe:42::/56 --service-cidr=10.43.0.0/16,2001:cafe:43::/112"; in { enable = true; role = cfg.role; tokenFile = config.age.secrets.k3s-server-token.path; extraFlags = lib.mkIf (cfg.role == "server") serverFlags; clusterInit = cfg.clusterInit; serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr; }; # Required for Longhorn openiscsi = { enable = true; name = "iqn.2016-04.com.open-iscsi:${config.networking.fqdn}"; }; }; # HACK: Symlink binaries to /usr/local/bin such that Longhorn can find them # when they use nsenter. # https://github.com/longhorn/longhorn/issues/2166#issuecomment-1740179416 systemd.tmpfiles.rules = [ "L+ /usr/local/bin - - - - /run/current-system/sw/bin/" ]; system = lib.mkIf (cfg.role == "server") { activationScripts = { k3s-bootstrap.text = ( let k3sBootstrapFile = (kubenix.evalModules.x86_64-linux { module = import ./bootstrap.nix; }).config.kubernetes.result; in '' mkdir -p /var/lib/rancher/k3s/server/manifests ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json '' ); k3s-certs.text = '' mkdir -p /var/lib/rancher/k3s/server/tls/etcd cp -f ${./k3s-ca/server-ca.crt} /var/lib/rancher/k3s/server/tls/server-ca.crt cp -f ${./k3s-ca/client-ca.crt} /var/lib/rancher/k3s/server/tls/client-ca.crt cp -f ${./k3s-ca/request-header-ca.crt} /var/lib/rancher/k3s/server/tls/request-header-ca.crt cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt ''; }; }; age.secrets = { k3s-server-token.file = ../../secrets/k3s-server-token.age; k3s-server-ca-key = lib.mkIf (cfg.role == "server") { file = ../../secrets/k3s-ca/server-ca.key.age; path = "/var/lib/rancher/k3s/server/tls/server-ca.key"; }; k3s-client-ca-key = lib.mkIf (cfg.role == "server") { file = ../../secrets/k3s-ca/client-ca.key.age; path = "/var/lib/rancher/k3s/server/tls/client-ca.key"; }; k3s-request-header-ca-key = lib.mkIf (cfg.role == "server") { file = ../../secrets/k3s-ca/request-header-ca.key.age; path = "/var/lib/rancher/k3s/server/tls/request-header-ca.key"; }; k3s-etcd-peer-ca-key = lib.mkIf (cfg.role == "server") { file = ../../secrets/k3s-ca/etcd/peer-ca.key.age; path = "/var/lib/rancher/k3s/server/tls/etcd/peer-ca.key"; }; k3s-etcd-server-ca-key = lib.mkIf (cfg.role == "server") { file = ../../secrets/k3s-ca/etcd/server-ca.key.age; path = "/var/lib/rancher/k3s/server/tls/etcd/server-ca.key"; }; k3s-service-key = lib.mkIf (cfg.role == "server") { file = ../../secrets/k3s-ca/service.key.age; path = "/var/lib/rancher/k3s/server/tls/service.key"; }; }; }; }