nixos-servers/nixos-modules/k3s/default.nix

{ pkgs, lib, config, kubenix, ... }:
let cfg = config.lab.k3s;
in {
  options.lab.k3s = {
    enable = lib.mkOption {
      default = false;
      type = lib.types.bool;
      description = ''
        Whether to run k3s on this server.
      '';
    };

    role = lib.mkOption {
      default = "server";
      type = lib.types.str;
      description = ''
        Whether to run k3s as a server or an agent.
      '';
    };

    clusterInit = lib.mkOption {
      default = false;
      type = lib.types.bool;
      description = ''
        Whether this node should initialize the K8s cluster.
      '';
    };

    serverAddr = lib.mkOption {
      default = null;
      type = with lib.types; nullOr str;
      description = ''
        Address of the server whose cluster this server should join.
        Leaving this empty will make the server initialize the cluster.
      '';
    };
  };

  config = lib.mkIf cfg.enable {
    environment.systemPackages = with pkgs; [
      k3s
      openiscsi # Required for Longhorn
      nfs-utils # Required for Longhorn
      jq # Required for Longhorn
    ];

    networking = {
      nftables.enable = lib.mkForce false;
      firewall.enable = lib.mkForce false;
    };

    services = {
      k3s =
        let
          serverFlags = "--tls-san ${config.networking.fqdn} --disable servicelb --cluster-cidr=10.42.0.0/16,2001:cafe:42::/56 --service-cidr=10.43.0.0/16,2001:cafe:43::/112";
        in
        {
          enable = true;
          role = cfg.role;
          tokenFile = config.age.secrets.k3s-server-token.path;
          extraFlags = lib.mkIf (cfg.role == "server") serverFlags;
          clusterInit = cfg.clusterInit;
          serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr;
        };

      # Required for Longhorn
      openiscsi = {
        enable = true;
        name = "iqn.2016-04.com.open-iscsi:${config.networking.fqdn}";
      };
    };

    # HACK: Symlink binaries to /usr/local/bin such that Longhorn can find them
    # when they use nsenter.
    # https://github.com/longhorn/longhorn/issues/2166#issuecomment-1740179416
    systemd.tmpfiles.rules = [
      "L+ /usr/local/bin - - - - /run/current-system/sw/bin/"
    ];

    system = lib.mkIf (cfg.role == "server") {
      activationScripts = {
        k3s-bootstrap.text = (
          let
            k3sBootstrapFile = (kubenix.evalModules.x86_64-linux {
              module = import ./bootstrap.nix;
            }).config.kubernetes.result;
          in
          ''
            mkdir -p /var/lib/rancher/k3s/server/manifests
            ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json
          ''
        );

        k3s-certs.text = ''
          mkdir -p /var/lib/rancher/k3s/server/tls/etcd
          cp -f ${./k3s-ca/server-ca.crt} /var/lib/rancher/k3s/server/tls/server-ca.crt
          cp -f ${./k3s-ca/client-ca.crt} /var/lib/rancher/k3s/server/tls/client-ca.crt
          cp -f ${./k3s-ca/request-header-ca.crt} /var/lib/rancher/k3s/server/tls/request-header-ca.crt
          cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt
          cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt
        '';
      };
    };

    age.secrets = {
      k3s-server-token.file = ../../secrets/k3s-server-token.age;

      k3s-server-ca-key = lib.mkIf (cfg.role == "server") {
        file = ../../secrets/k3s-ca/server-ca.key.age;
        path = "/var/lib/rancher/k3s/server/tls/server-ca.key";
      };

      k3s-client-ca-key = lib.mkIf (cfg.role == "server") {
        file = ../../secrets/k3s-ca/client-ca.key.age;
        path = "/var/lib/rancher/k3s/server/tls/client-ca.key";
      };

      k3s-request-header-ca-key = lib.mkIf (cfg.role == "server") {
        file = ../../secrets/k3s-ca/request-header-ca.key.age;
        path = "/var/lib/rancher/k3s/server/tls/request-header-ca.key";
      };

      k3s-etcd-peer-ca-key = lib.mkIf (cfg.role == "server") {
        file = ../../secrets/k3s-ca/etcd/peer-ca.key.age;
        path = "/var/lib/rancher/k3s/server/tls/etcd/peer-ca.key";
      };

      k3s-etcd-server-ca-key = lib.mkIf (cfg.role == "server") {
        file = ../../secrets/k3s-ca/etcd/server-ca.key.age;
        path = "/var/lib/rancher/k3s/server/tls/etcd/server-ca.key";
      };

      k3s-service-key = lib.mkIf (cfg.role == "server") {
        file = ../../secrets/k3s-ca/service.key.age;
        path = "/var/lib/rancher/k3s/server/tls/service.key";
      };
    };
  };
}