diff --git a/docs/k3s.md b/docs/k3s.md new file mode 100644 index 0000000..76d97a1 --- /dev/null +++ b/docs/k3s.md @@ -0,0 +1,41 @@ +# Kubernetes + +## Creating an admin certificate for k3s + +Create the admin's private key: +``` +openssl genpkey -algorithm ed25519 -out -key.pem +``` + +Create a CSR for the admin: +``` +openssl req -new -key -key.pem -out .csr -subj "/CN=" +``` + +Create a Kubernetes CSR object on the cluster: +``` +k3s kubectl create -f - <-csr +spec: + request: $(cat .csr | base64 | tr -d '\n') + expirationSeconds: 307584000 # 10 years + signerName: kubernetes.io/kube-apiserver-client + usages: + - digital signature + - key encipherment + - client auth +EOF +``` + +Approve and sign the admin's CSR: +``` +k3s kubectl certificate approve -csr +``` + +Extract the resulting signed certificate from the CSR object: +``` +k3s kubectl get csr -csr -o jsonpath='{.status.certificate}' | base64 --decode > .crt +``` diff --git a/flake.lock b/flake.lock index b6917b3..1531b2a 100644 --- a/flake.lock +++ b/flake.lock @@ -122,6 +122,22 @@ "type": "github" } }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "locked": { "lastModified": 1614513358, @@ -157,7 +173,7 @@ }, "flake-utils_3": { "inputs": { - "systems": "systems_4" + "systems": "systems_5" }, "locked": { "lastModified": 1701680307, @@ -194,6 +210,29 @@ "type": "github" } }, + "kubenix": { + "inputs": { + "flake-compat": "flake-compat_2", + "nixpkgs": [ + "nixpkgs" + ], + "systems": "systems_4", + "treefmt": "treefmt" + }, + "locked": { + "lastModified": 1705801181, + "narHash": "sha256-vH+n5qMnwFCx3LMON2hQMi9PjMpmTraGYXe1czJTfAg=", + "owner": "hall", + "repo": "kubenix", + "rev": "76b8053b27b062b11f0c9b495050cc55606ac9dc", + "type": "github" + }, + "original": { + "owner": "hall", + "repo": "kubenix", + "type": "github" + } + }, "microvm": { "inputs": { "flake-utils": "flake-utils_3", @@ -250,11 +289,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1709499969, - "narHash": "sha256-PXi7pnvg+uYY5oUm8Vgw4pXZaGUGjGIOZ4gQ/yGVoPo=", + "lastModified": 1710789445, + "narHash": "sha256-MDljQvOU5jX4S5MUGJ3izSfzaeIWQ1phoc3MarCyUes=", "owner": "pizzapim", "repo": "nixpkgs", - "rev": "c74dae81760b4e48633133e689358ff4a120eabe", + "rev": "9235a7f0f9f6f9970cf46345bfb23ad84fa2a5a7", "type": "github" }, "original": { @@ -287,6 +326,7 @@ "disko": "disko", "dns": "dns", "flake-utils": "flake-utils_2", + "kubenix": "kubenix", "microvm": "microvm", "nixos-hardware": "nixos-hardware", "nixpkgs": "nixpkgs_2", @@ -364,8 +404,43 @@ "type": "github" }, "original": { + "id": "systems", + "type": "indirect" + } + }, + "systems_5": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", "owner": "nix-systems", "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "treefmt": { + "inputs": { + "nixpkgs": [ + "kubenix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1688026376, + "narHash": "sha256-qJmkr9BWDpqblk4E9/rCsAEl39y2n4Ycw6KRopvpUcY=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "df3f32b0cc253dfc7009b7317e8f0e7ccd70b1cf", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", "type": "github" } }, diff --git a/flake.nix b/flake.nix index c20fa29..572b81b 100644 --- a/flake.nix +++ b/flake.nix @@ -27,6 +27,11 @@ url = "github:astro/microvm.nix"; inputs.nixpkgs.follows = "nixpkgs"; }; + + kubenix = { + url = "github:hall/kubenix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = diff --git a/nix/default.nix b/nix/default.nix index 1580f85..58512f1 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -95,6 +95,7 @@ socat pciutils dmidecode + openssl ]; nixpkgs.overlays = [ diff --git a/nix/flake/nixos.nix b/nix/flake/nixos.nix index a544fbd..9a963e4 100644 --- a/nix/flake/nixos.nix +++ b/nix/flake/nixos.nix @@ -1,4 +1,4 @@ -{ nixpkgs, nixpkgs-unstable, machines, physicalMachines, dns, microvm, disko, agenix, nixos-hardware, ... }: +{ nixpkgs, nixpkgs-unstable, machines, physicalMachines, dns, microvm, disko, agenix, nixos-hardware, kubenix, ... }: let mkNixosSystems = systemDef: builtins.mapAttrs @@ -11,7 +11,7 @@ in nixosConfigurations = mkNixosSystems (name: machine: { system = machine.arch; - specialArgs = { inherit nixpkgs-unstable machines machine dns microvm disko agenix nixos-hardware; }; + specialArgs = { inherit nixpkgs-unstable machines machine dns microvm disko agenix nixos-hardware kubenix; }; modules = [ ../. { networking.hostName = name; } diff --git a/nix/machines/jefke.nix b/nix/machines/jefke.nix index 44c7896..81ba3ed 100644 --- a/nix/machines/jefke.nix +++ b/nix/machines/jefke.nix @@ -15,6 +15,8 @@ hostCert = builtins.readFile ./certificates/jefke/host_ed25519.crt; userCert = builtins.readFile ./certificates/jefke/user_ed25519.crt; }; + + k3s.enable = true; }; }; } diff --git a/nix/modules/default.nix b/nix/modules/default.nix index c07bf26..4303f06 100644 --- a/nix/modules/default.nix +++ b/nix/modules/default.nix @@ -7,5 +7,6 @@ ./data-sharing.nix ./globals.nix ./monitoring + ./k3s ]; } diff --git a/nix/modules/k3s/bootstrap.nix b/nix/modules/k3s/bootstrap.nix new file mode 100644 index 0000000..20515d5 --- /dev/null +++ b/nix/modules/k3s/bootstrap.nix @@ -0,0 +1,16 @@ +{ kubenix, ... }: { + imports = [ kubenix.modules.k8s ]; + kubernetes.resources.clusterRoleBindings.pim-cluster-admin = { + roleRef = { + apiGroup = "rbac.authorization.k8s.io"; + kind = "ClusterRole"; + name = "cluster-admin"; + }; + subjects = [ + { + kind = "User"; + name = "pim"; + } + ]; + }; +} diff --git a/nix/modules/k3s/default.nix b/nix/modules/k3s/default.nix new file mode 100644 index 0000000..51f121d --- /dev/null +++ b/nix/modules/k3s/default.nix @@ -0,0 +1,36 @@ +{ pkgs, lib, config, kubenix, ... }: +let cfg = config.lab.k3s; +in { + options.lab.k3s.enable = lib.mkOption { + default = false; + type = lib.types.bool; + description = '' + Whether to start k3s with custom configuration. + ''; + }; + + config = lib.mkIf cfg.enable { + environment.systemPackages = with pkgs; [ k3s ]; + + networking = { + nftables.enable = lib.mkForce false; + firewall.enable = lib.mkForce false; + }; + + services.k3s = { + enable = true; + role = "server"; + extraFlags = "--tls-san ${config.networking.fqdn} --data-dir ${config.lab.storage.dataMountPoint}/k3s"; + }; + + system.activationScripts.k3s-bootstrap.text = + let + k3sBootstrapFile = (kubenix.evalModules.x86_64-linux { + module = import ./bootstrap.nix; + }).config.kubernetes.result; + in + '' + ln -sf ${k3sBootstrapFile} ${config.lab.storage.dataMountPoint}/k3s/server/manifests/k3s-bootstrap.json + ''; + }; +} diff --git a/nix/modules/monitoring/default.nix b/nix/modules/monitoring/default.nix index 85473c7..7fc634e 100644 --- a/nix/modules/monitoring/default.nix +++ b/nix/modules/monitoring/default.nix @@ -80,6 +80,15 @@ in }; }; + users = { + users.gatus = { + isSystemUser = true; + group = "gatus"; + }; + + groups.gatus = { }; + }; + system.activationScripts = lib.mkIf cfg.server.enable { gatus = '' mkdir -p /srv/gatus diff --git a/nix/physical.nix b/nix/physical.nix index e565dae..e131c79 100644 --- a/nix/physical.nix +++ b/nix/physical.nix @@ -1,4 +1,4 @@ -{ pkgs, nixpkgs-unstable, config, lib, microvm, disko, agenix, machine, machines, dns, nixos-hardware, ... }: { +{ pkgs, nixpkgs-unstable, config, lib, microvm, disko, agenix, machine, machines, dns, nixos-hardware, kubenix, ... }: { imports = [ microvm.nixosModules.host ] @@ -69,7 +69,7 @@ restartIfChanged = false; specialArgs = { - inherit agenix disko pkgs lib microvm dns nixpkgs-unstable; + inherit agenix disko pkgs lib microvm dns nixpkgs-unstable kubenix; machine = vm; hypervisorConfig = config; };