nixos-servers/modules/k3s/default.nix

{ self, inputs, pkgs, lib, config, globals, ... }:
let
  cfg = config.lab.k3s;
in
{
  options.lab.k3s = {
    enable = lib.mkOption {
      default = false;
      type = lib.types.bool;
      description = ''
        Whether to run k3s on this server.
      '';
    };

    role = lib.mkOption {
      default = "server";
      type = lib.types.str;
      description = ''
        Whether to run k3s as a server or an agent.
      '';
    };

    clusterInit = lib.mkOption {
      default = false;
      type = lib.types.bool;
      description = ''
        Whether this node should initialize the K8s cluster.
      '';
    };

    serverAddr = lib.mkOption {
      default = null;
      type = with lib.types; nullOr str;
      description = ''
        Address of the server whose cluster this server should join.
        Leaving this empty will make the server initialize the cluster.
      '';
    };
  };

  config = lib.mkIf cfg.enable {
    environment.systemPackages = with pkgs; [
      k3s
      openiscsi # Required for Longhorn
      nfs-utils # Required for Longhorn
    ];

    # TODO!!!!!
    networking = {
      nftables.enable = lib.mkForce false;
      firewall.enable = lib.mkForce false;
    };

    virtualisation.containerd = {
      enable = true;

      settings = {
        version = 2;

        proxy_plugins.nix = {
          type = "snapshot";
          address = "/run/nix-snapshotter/nix-snapshotter.sock";
        };

        plugins =
          let
            k3s-cni-plugins = pkgs.buildEnv {
              name = "k3s-cni-plugins";
              paths = with pkgs; [
                cni-plugins
                cni-plugin-flannel
              ];
            };
          in
          {
            "io.containerd.grpc.v1.cri" = {
              stream_server_address = "127.0.0.1";
              stream_server_port = "10010";
              enable_selinux = false;
              enable_unprivileged_ports = true;
              enable_unprivileged_icmp = true;
              disable_apparmor = true;
              disable_cgroup = true;
              restrict_oom_score_adj = true;
              sandbox_image = "rancher/mirrored-pause:3.6";
              containerd.snapshotter = "nix";

              cni = {
                conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d/";
                bin_dir = "${k3s-cni-plugins}/bin";
              };
            };

            "io.containerd.transfer.v1.local".unpack_config = [{
              platform = "linux/amd64";
              snapshotter = "nix";
            }];
          };
      };
    };

    services = {
      nix-snapshotter.enable = true;

      k3s =
        let
          serverFlagList = [
            "--image-service-endpoint=unix:///run/nix-snapshotter/nix-snapshotter.sock"
            "--snapshotter=overlayfs"
            "--container-runtime-endpoint=unix:///run/containerd/containerd.sock"
            "--tls-san=${config.networking.fqdn}"
            "--disable=servicelb"
            "--cluster-cidr=10.42.0.0/16,2001:cafe:42::/56"
            "--service-cidr=10.43.0.0/16,2001:cafe:43::/112"
          ];

          serverFlags = builtins.concatStringsSep " " serverFlagList;
        in
        {
          enable = true;
          role = cfg.role;
          tokenFile = config.sops.secrets."k3s/serverToken".path;
          extraFlags = lib.mkIf (cfg.role == "server") (lib.mkForce serverFlags);
          clusterInit = cfg.clusterInit;
          serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr;
        };

      # Required for Longhorn
      openiscsi = {
        enable = true;
        name = "iqn.2016-04.com.open-iscsi:${config.networking.fqdn}";
      };
    };

    # HACK: Symlink binaries to /usr/local/bin such that Longhorn can find them
    # when they use nsenter.
    # https://github.com/longhorn/longhorn/issues/2166#issuecomment-1740179416
    systemd.tmpfiles.rules = [
      "L+ /usr/local/bin - - - - /run/current-system/sw/bin/"
    ];

    system.activationScripts = {
      k3s-bootstrap = lib.mkIf (cfg.role == "server") {
        text = (
          let
            k3sBootstrapFile = (inputs.kubenix.evalModules.x86_64-linux {
              module = import ./bootstrap.nix;
            }).config.kubernetes.result;
          in
          ''
            mkdir -p /var/lib/rancher/k3s/server/manifests
            ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json
          ''
        );
      };

      k3s-certs = lib.mkIf (cfg.role == "server") {
        text = ''
          mkdir -p /var/lib/rancher/k3s/server/tls/etcd
          cp -f ${./k3s-ca/server-ca.crt} /var/lib/rancher/k3s/server/tls/server-ca.crt
          cp -f ${./k3s-ca/client-ca.crt} /var/lib/rancher/k3s/server/tls/client-ca.crt
          cp -f ${./k3s-ca/request-header-ca.crt} /var/lib/rancher/k3s/server/tls/request-header-ca.crt
          cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt
          cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt
        '';
      };

      nixng = lib.mkIf (cfg.role == "server")
        (
          let
            dnsmasqStream = (import ./dnsmasq.nix {
              inherit (inputs) nixpkgs nixng;
              inherit (inputs.nixng) nglib;
              inherit (self) globals;
            }).config.system.build.ociImage.stream;

            dnsmasqImage = pkgs.stdenv.mkDerivation {
              name = "dnsmasq.tar";
              src = dnsmasqStream;
              dontUnpack = true;
              buildPhase = ''
                $src > $out
              '';
            };
          in
          {
            text = ''
              ln -sf ${dnsmasqImage} /root/dnsmasq.tar
            '';
          }
        );

      docker-images.text =
        let
          imageDefs = import "${self}/container-images.nix";

          setupCommands = [
            "rm -rf ${self.globals.imageDir}"
            "mkdir -p ${self.globals.imageDir}"
          ];

          getDockerImageConfig = dockerImage:
            let
              configJson = pkgs.runCommand "config.json"
                {
                  nativeBuildInputs = [ pkgs.skopeo pkgs.jq ];
                }
                ''
                  skopeo --tmpdir $TMPDIR --insecure-policy inspect docker-archive:${dockerImage} --config | jq '.config' > $out
                '';
            in
            builtins.fromJSON (builtins.readFile configJson);

          imageDefToLinkCommand = name: imageDef:
            let
              dockerImage = pkgs.dockerTools.pullImage imageDef;
              nixSnapshotterImage = pkgs.nix-snapshotter.buildImage {
                inherit name;
                resolvedByNix = true;
                fromImage = dockerImage;
                config = getDockerImageConfig dockerImage;
              };
              imageLinkPath = "${self.globals.imageDir}/${name}.tar";
            in
            "ln -sf ${nixSnapshotterImage} ${imageLinkPath}";

          linkCommandList = lib.attrsets.mapAttrsToList imageDefToLinkCommand imageDefs;
          # TODO: Creating Docker images like this seems to *explode* in size.
          # Doing this for every image we currently have is infeasible.
          # I should investigate why the size increases like that.
          commandList = setupCommands; # ++ linkCommandList;
        in
        builtins.concatStringsSep "\n" commandList;
    };

    sops.secrets =
      let
        keyPathBase = "/var/lib/rancher/k3s/server/tls";
      in
      {
        "k3s/serverToken" = { };
        "k3s/keys/clientCAKey".path = "${keyPathBase}/client-ca.key";
        "k3s/keys/requestHeaderCAKey".path = "${keyPathBase}/request-header-ca.key";
        "k3s/keys/serverCAKey".path = "${keyPathBase}/server-ca.key";
        "k3s/keys/serviceKey".path = "${keyPathBase}/service.key";
        "k3s/keys/etcd/peerCAKey".path = "${keyPathBase}/etcd/peer-ca.key";
        "k3s/keys/etcd/serverCAKey".path = "${keyPathBase}/etcd/server-ca.key";
      };
  };
}