diff --git a/README.md b/README.md index 7ee81e5..b93db68 100644 --- a/README.md +++ b/README.md @@ -25,3 +25,43 @@ Additionally, it deploys an age identity, which is later used for decrypting sec ## Deployment Deployment can simply be done as follows: `deploy` + +## Creating an admin certificate for k3s + +Create the admin's private key: +``` +openssl genpkey -algorithm ed25519 -out -key.pem +``` + +Create a CSR for the admin: +``` +openssl req -new -key -key.pem -out .csr -subj "/CN=" +``` + +Create a Kubernetes CSR object on the cluster: +``` +k3s kubectl create -f - <-csr +spec: + request: $(cat .csr | base64 | tr -d '\n') + expirationSeconds: 307584000 # 10 years + signerName: kubernetes.io/kube-apiserver-client + usages: + - digital signature + - key encipherment + - client auth +EOF +``` + +Approve and sign the admin's CSR: +``` +k3s kubectl certificate approve -csr +``` + +Extract the resulting signed certificate from the CSR object: +``` +k3s kubectl get csr -csr -o jsonpath='{.status.certificate}' | base64 --decode > .crt +``` diff --git a/cluster/README.md b/cluster/README.md new file mode 100644 index 0000000..85cc5c8 --- /dev/null +++ b/cluster/README.md @@ -0,0 +1,3 @@ +``` +nix --extra-experimental-features nix-command --extra-experimental-features flakes run .#kubenix +``` diff --git a/cluster/flake.lock b/cluster/flake.lock new file mode 100644 index 0000000..7bc00f8 --- /dev/null +++ b/cluster/flake.lock @@ -0,0 +1,99 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "kubenix": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": "nixpkgs", + "systems": "systems", + "treefmt": "treefmt" + }, + "locked": { + "lastModified": 1700116223, + "narHash": "sha256-Pld/UXlBcIDnQMY0JkDzChJkbof/zEcRkaiXtzvArEE=", + "owner": "hall", + "repo": "kubenix", + "rev": "e4d036576436b9983216584a89388af3da995043", + "type": "github" + }, + "original": { + "owner": "hall", + "repo": "kubenix", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1686488075, + "narHash": "sha256-2otSBt2hbeD+5yY25NF3RhWx7l5SDt1aeU3cJ/9My4M=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9401a0c780b49faf6c28adf55764f230301d0dce", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "kubenix": "kubenix" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "id": "systems", + "type": "indirect" + } + }, + "treefmt": { + "inputs": { + "nixpkgs": [ + "kubenix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1688026376, + "narHash": "sha256-qJmkr9BWDpqblk4E9/rCsAEl39y2n4Ycw6KRopvpUcY=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "df3f32b0cc253dfc7009b7317e8f0e7ccd70b1cf", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/cluster/flake.nix b/cluster/flake.nix new file mode 100644 index 0000000..1b3f6c6 --- /dev/null +++ b/cluster/flake.nix @@ -0,0 +1,22 @@ +{ + inputs.kubenix.url = "github:hall/kubenix"; + outputs = { self, kubenix, ... }: + let + system = "x86_64-linux"; + in { + kubenix = kubenix.packages.${system}.default.override { + module = {kubenix, ...}: { + imports = [kubenix.modules.k8s]; + kubernetes = { + kubeconfig = "/etc/rancher/k3s/k3s.yaml"; + version = "1.24"; + }; + kubenix.project = "yeet"; + + kubernetes.resources.pods.web1.spec.containers.nginx.image = "nginx"; + kubernetes.resources.pods.web2.spec.containers.nginx.image = "nginx"; + }; + }; + }; +} + diff --git a/configuration.nix b/configuration.nix index 8e3c389..e4ddb31 100644 --- a/configuration.nix +++ b/configuration.nix @@ -112,6 +112,7 @@ ]; networking = { + domain = "hyp"; firewall.enable = false; useDHCP = false; @@ -169,10 +170,10 @@ }; }; - virtualisation.libvirtd.enable = true; - hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; age.identityPaths = [ "/root/age_ed25519" ]; + + virtualisation.libvirtd.enable = true; } diff --git a/flake.nix b/flake.nix index 58bf87f..4b99ab2 100644 --- a/flake.nix +++ b/flake.nix @@ -5,32 +5,41 @@ nixpkgs.url = "github:nixos/nixpkgs/nixos-23.05"; nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; deploy-rs.url = "github:serokell/deploy-rs"; + disko = { url = "github:nix-community/disko"; inputs.nixpkgs.follows = "nixpkgs"; }; + agenix = { url = "github:ryantm/agenix"; inputs.nixpkgs.follows = "nixpkgs"; }; }; - outputs = { self, nixpkgs, deploy-rs, disko, agenix, nixpkgs-unstable, ... }: + outputs = + { self, nixpkgs, deploy-rs, disko, agenix, nixpkgs-unstable, ... }: let system = "x86_64-linux"; pkgs = nixpkgs.legacyPackages.${system}; pkgs-unstable = nixpkgs-unstable.legacyPackages.${system}; machines = import ./machines; + # TODO: Maybe use mergeAttrLists mkNixosSystems = systemDef: - nixpkgs.lib.foldlAttrs (acc: name: machine: - acc // { - "${name}" = nixpkgs.lib.nixosSystem (systemDef machine); - }) { } machines; + nixpkgs.lib.foldlAttrs + (acc: name: machine: + acc // { + "${name}" = nixpkgs.lib.nixosSystem (systemDef machine); + }) + { } + machines; mkDeployNodes = nodeDef: nixpkgs.lib.foldlAttrs - (acc: name: machine: acc // { "${name}" = nodeDef machine; }) { } - machines; - in { + (acc: name: machine: acc // { "${name}" = nodeDef machine; }) + { } + machines; + in + { devShells.${system}.default = pkgs.mkShell { packages = [ pkgs.libsecret @@ -43,6 +52,7 @@ pkgs.postgresql_15 pkgs-unstable.opentofu pkgs.cdrtools + pkgs.kubectl ]; }; @@ -73,6 +83,7 @@ }; checks = builtins.mapAttrs - (system: deployLib: deployLib.deployChecks self.deploy) deploy-rs.lib; + (system: deployLib: deployLib.deployChecks self.deploy) + deploy-rs.lib; }; } diff --git a/machines/default.nix b/machines/default.nix index bf1e41f..0aa2aab 100644 --- a/machines/default.nix +++ b/machines/default.nix @@ -14,6 +14,8 @@ }; terraformDatabase.enable = true; + + k3s.enable = true; }; disko.devices = { diff --git a/modules/custom/data-disk.nix b/modules/custom/data-disk.nix index 4e2d485..99fa46d 100644 --- a/modules/custom/data-disk.nix +++ b/modules/custom/data-disk.nix @@ -3,17 +3,35 @@ let cfg = config.custom.dataDisk; in { options = { custom = { - dataDisk.enable = lib.mkOption { - default = false; - type = lib.types.bool; - description = '' - Whether to automatically mount /dev/sda1 on /mnt/data - ''; + dataDisk = { + enable = lib.mkOption { + default = false; + type = lib.types.bool; + description = '' + Whether to automatically mount a disk to be used as a data disk. + ''; + }; + + mountPoint = lib.mkOption { + default = "/mnt/data"; + type = lib.types.str; + description = '' + Mount point of the data disk (if enabled). + ''; + }; + + devicePath = lib.mkOption { + default = "/dev/sda1"; + type = lib.types.str; + description = '' + Path of the device to be used as a data disk. + ''; + }; }; }; }; config = lib.mkIf cfg.enable { - fileSystems."/mnt/data" = { device = "/dev/sda1"; }; + fileSystems.${cfg.mountPoint} = { device = cfg.devicePath; }; }; } diff --git a/modules/custom/default.nix b/modules/custom/default.nix index ceeaefa..6e7528d 100644 --- a/modules/custom/default.nix +++ b/modules/custom/default.nix @@ -1,3 +1,3 @@ { - imports = [ ./terraform-database.nix ./data-disk.nix ./ssh-certificates.nix ]; + imports = [ ./terraform-database.nix ./data-disk.nix ./ssh-certificates.nix ./k3s.nix ]; } diff --git a/modules/custom/k3s-bootstrap.yaml b/modules/custom/k3s-bootstrap.yaml new file mode 100644 index 0000000..efcc5e2 --- /dev/null +++ b/modules/custom/k3s-bootstrap.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pim-cluster-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: pim diff --git a/modules/custom/k3s.nix b/modules/custom/k3s.nix new file mode 100644 index 0000000..e37ff0a --- /dev/null +++ b/modules/custom/k3s.nix @@ -0,0 +1,31 @@ +{ pkgs, lib, config, ... }: +let cfg = config.custom.k3s; +in { + options = { + custom = { + k3s.enable = lib.mkOption { + default = false; + type = lib.types.bool; + description = '' + Whether to start k3s with custom configuration. + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + environment.systemPackages = [ pkgs.k3s ]; + services.k3s.enable = true; + services.k3s.role = "server"; + services.k3s.extraFlags = "--tls-san ${config.networking.fqdn} --data-dir ${config.custom.dataDisk.mountPoint}/k3s"; + + # TODO: use kubenix for this. + system.activationScripts.k3s-bootstrap.text = + let + k3sBootstrapFile = pkgs.writeText "k3s-bootstrap" (builtins.readFile ./k3s-bootstrap.yaml); + in + '' + ln -sf ${k3sBootstrapFile} ${config.custom.dataDisk.mountPoint}/k3s/server/manifests/k3s-bootstrap.yaml + ''; + }; +} diff --git a/modules/custom/terraform-database.nix b/modules/custom/terraform-database.nix index bdad8a7..01b7bb5 100644 --- a/modules/custom/terraform-database.nix +++ b/modules/custom/terraform-database.nix @@ -20,19 +20,21 @@ in { package = pkgs.postgresql_15; enableTCPIP = true; dataDir = lib.mkIf config.custom.dataDisk.enable - "/mnt/data/postgresql/${config.services.postgresql.package.psqlSchema}"; + "${config.custom.dataDisk.mountPoint}/postgresql/${config.services.postgresql.package.psqlSchema}"; authentication = '' hostssl terraformstates terraform all cert ''; - settings = let - serverCert = builtins.toFile "postgresql_server.crt" - (builtins.readFile ../../postgresql_server.crt); - in { - ssl = true; - ssl_cert_file = serverCert; - ssl_key_file = config.age.secrets."postgresql_server.key".path; - ssl_ca_file = serverCert; - }; + settings = + let + serverCert = builtins.toFile "postgresql_server.crt" + (builtins.readFile ../../postgresql_server.crt); + in + { + ssl = true; + ssl_cert_file = serverCert; + ssl_key_file = config.age.secrets."postgresql_server.key".path; + ssl_ca_file = serverCert; + }; ensureUsers = [{ name = "terraform"; ensurePermissions = { "DATABASE terraformstates" = "ALL PRIVILEGES"; }; diff --git a/nftables.conf b/nftables.conf index 46dd6cb..10d456f 100644 --- a/nftables.conf +++ b/nftables.conf @@ -15,6 +15,7 @@ table inet nixos-fw { chain input-allow { tcp dport 22 accept tcp dport 5432 accept comment "PostgreSQL server" + tcp dport 6443 accept comment "k3s" icmp type echo-request accept comment "allow ping" icmpv6 type != { nd-redirect, 139 } accept comment "Accept all ICMPv6 messages except redirects and node information queries (type 139). See RFC 4890, section 4.4." ip6 daddr fe80::/64 udp dport 546 accept comment "DHCPv6 client"