From 7a915f09287633be9eec042be8d15798c0d64964 Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Wed, 17 Jul 2024 16:25:41 +0200 Subject: [PATCH] refactor(kubenix): DRY deployment creation feat(kubenix): Create applysets for remaining resources --- flake-parts/kubenix.nix | 181 ++++++++++++++++-------- kubenix-modules/all.nix | 12 -- kubenix-modules/base.nix | 94 ------------ kubenix-modules/bootstrapDefault.nix | 164 +++++++++++++++++++++ kubenix-modules/bootstrapKubeSystem.nix | 36 +++++ kubenix-modules/cert-manager.nix | 15 -- kubenix-modules/longhorn.nix | 44 ------ kubenix-modules/metallb.nix | 7 - kubenix-modules/traefik.nix | 3 - kubenix-modules/volumes.nix | 69 --------- 10 files changed, 324 insertions(+), 301 deletions(-) delete mode 100644 kubenix-modules/all.nix delete mode 100644 kubenix-modules/base.nix create mode 100644 kubenix-modules/bootstrapDefault.nix create mode 100644 kubenix-modules/bootstrapKubeSystem.nix delete mode 100644 kubenix-modules/cert-manager.nix delete mode 100644 kubenix-modules/longhorn.nix delete mode 100644 kubenix-modules/metallb.nix delete mode 100644 kubenix-modules/volumes.nix diff --git a/flake-parts/kubenix.nix b/flake-parts/kubenix.nix index 2eefec0..320731b 100644 --- a/flake-parts/kubenix.nix +++ b/flake-parts/kubenix.nix @@ -1,17 +1,11 @@ { self, pkgs, machines, dns, myLib, flake-utils, kubenix, nixhelm, blog-pim, ... }: flake-utils.lib.eachDefaultSystem (system: let - mkKubenixPackage = module: kubenix.packages.${system}.default.override - { - specialArgs = { inherit myLib kubenix nixhelm system dns blog-pim machines; }; - module = { imports = [ module ]; }; - }; - deployScript = (pkgs.writeScriptBin "kubenix" (builtins.readFile ./kubenix-deploy.sh)).overrideAttrs (old: { buildCommand = "${old.buildCommand}\npatchShebangs $out"; }); - mkDeployScript = kubernetes: applyset: namespace: + mkDeployScript = kubernetes: applysetName: namespace: let kubeconfig = kubernetes.kubeconfig or ""; result = kubernetes.result or ""; @@ -28,17 +22,17 @@ --suffix PATH : "$out/bin" \ --run 'export KUBECONFIG=''${KUBECONFIG:-${toString kubeconfig}}' \ --set MANIFEST '${result}' \ - --set APPLYSET 'applyset-${applyset}' \ + --set APPLYSET 'applyset-${applysetName}' \ --set NAMESPACE '${namespace}' ''; }; in wrappedDeployScript; - mkDeployScriptAndManifest = module: applyset: namespace: + mkDeployScriptAndManifest = name: { module, namespace }: let kubernetes = (kubenix.evalModules.${system} { - specialArgs = { inherit namespace myLib blog-pim dns; }; + specialArgs = { inherit namespace myLib blog-pim dns nixhelm system machines; }; module = { kubenix, ... }: { @@ -51,7 +45,7 @@ ]; config = { - kubenix.project = applyset; + kubenix.project = name; kubernetes.namespace = namespace; }; }; @@ -59,53 +53,126 @@ in { manifest = kubernetes.result; - deploy = mkDeployScript kubernetes applyset namespace; + deploy = mkDeployScript kubernetes name namespace; }; + + deployers = { + bootstrap-default = { + module = "${self}/kubenix-modules/bootstrapDefault.nix"; + namespace = "default"; + }; + + bootstrap-kube-system = { + module = "${self}/kubenix-modules/bootstrapKubeSystem.nix"; + namespace = "kube-system"; + }; + + cyberchef = { + module = "${self}/kubenix-modules/cyberchef.nix"; + namespace = "static-websites"; + }; + + freshrss = { + module = "${self}/kubenix-modules/freshrss.nix"; + namespace = "freshrss"; + }; + + radicale = { + module = "${self}/kubenix-modules/radicale.nix"; + namespace = "radicale"; + }; + + kms = { + module = "${self}/kubenix-modules/kms.nix"; + namespace = "kms"; + }; + + atuin = { + module = "${self}/kubenix-modules/atuin.nix"; + namespace = "atuin"; + }; + + blog = { + module = "${self}/kubenix-modules/blog.nix"; + namespace = "static-websites"; + }; + + nextcloud = { + module = "${self}/kubenix-modules/nextcloud.nix"; + namespace = "nextcloud"; + }; + + hedgedoc = { + module = "${self}/kubenix-modules/hedgedoc.nix"; + namespace = "hedgedoc"; + }; + + kitchenowl = { + module = "${self}/kubenix-modules/kitchenowl.nix"; + namespace = "kitchenowl"; + }; + + forgejo = { + module = "${self}/kubenix-modules/forgejo"; + namespace = "forgejo"; + }; + + paperless = { + module = "${self}/kubenix-modules/paperless.nix"; + namespace = "paperless"; + }; + + syncthing = { + module = "${self}/kubenix-modules/syncthing.nix"; + namespace = "syncthing"; + }; + + pihole = { + module = "${self}/kubenix-modules/pihole.nix"; + namespace = "dns"; + }; + + immich = { + module = "${self}/kubenix-modules/immich.nix"; + namespace = "immich"; + }; + + attic = { + module = "${self}/kubenix-modules/attic.nix"; + namespace = "attic"; + }; + + inbucket = { + module = "${self}/kubenix-modules/inbucket.nix"; + namespace = "inbucket"; + }; + + dnsmasq = { + module = "${self}/kubenix-modules/dnsmasq.nix"; + namespace = "dns"; + }; + + bind9 = { + module = "${self}/kubenix-modules/bind9"; + namespace = "dns"; + }; + + media = { + module = "${self}/kubenix-modules/media.nix"; + namespace = "media"; + }; + + traefik = { + module = "${self}/kubenix-modules/traefik.nix"; + namespace = "kube-system"; + }; + + minecraft = { + module = "${self}/kubenix-modules/minecraft.nix"; + namespace = "minecraft"; + }; + }; in { - kubenix.all.deploy = mkKubenixPackage "${self}/kubenix-modules/all.nix"; - kubenix.bootstrap.deploy = mkKubenixPackage "${self}/kubenix-modules/base.nix"; - - kubenix.cyberchef = mkDeployScriptAndManifest - "${self}/kubenix-modules/cyberchef.nix" "cyberchef" "static-websites"; - kubenix.freshrss = mkDeployScriptAndManifest - "${self}/kubenix-modules/freshrss.nix" "freshrss" "freshrss"; - kubenix.radicale = mkDeployScriptAndManifest - "${self}/kubenix-modules/radicale.nix" "radicale" "radicale"; - kubenix.kms = mkDeployScriptAndManifest - "${self}/kubenix-modules/kms.nix" "kms" "kms"; - kubenix.atuin = mkDeployScriptAndManifest - "${self}/kubenix-modules/atuin.nix" "atuin" "atuin"; - kubenix.blog = mkDeployScriptAndManifest - "${self}/kubenix-modules/blog.nix" "blog" "static-websites"; - kubenix.nextcloud = mkDeployScriptAndManifest - "${self}/kubenix-modules/nextcloud.nix" "nextcloud" "nextcloud"; - kubenix.hedgedoc = mkDeployScriptAndManifest - "${self}/kubenix-modules/hedgedoc.nix" "hedgedoc" "hedgedoc"; - kubenix.kitchenowl = mkDeployScriptAndManifest - "${self}/kubenix-modules/kitchenowl.nix" "kitchenowl" "kitchenowl"; - kubenix.forgejo = mkDeployScriptAndManifest - "${self}/kubenix-modules/forgejo" "forgejo" "forgejo"; - kubenix.paperless = mkDeployScriptAndManifest - "${self}/kubenix-modules/paperless.nix" "paperless" "paperless"; - kubenix.syncthing = mkDeployScriptAndManifest - "${self}/kubenix-modules/syncthing.nix" "syncthing" "syncthing"; - kubenix.pihole = mkDeployScriptAndManifest - "${self}/kubenix-modules/pihole.nix" "pihole" "dns"; - kubenix.immich = mkDeployScriptAndManifest - "${self}/kubenix-modules/immich.nix" "immich" "immich"; - kubenix.attic = mkDeployScriptAndManifest - "${self}/kubenix-modules/attic.nix" "attic" "attic"; - kubenix.inbucket = mkDeployScriptAndManifest - "${self}/kubenix-modules/inbucket.nix" "inbucket" "inbucket"; - kubenix.dnsmasq = mkDeployScriptAndManifest - "${self}/kubenix-modules/dnsmasq.nix" "dnsmasq" "dns"; - kubenix.bind9 = mkDeployScriptAndManifest - "${self}/kubenix-modules/bind9" "bind9" "dns"; - kubenix.media = mkDeployScriptAndManifest - "${self}/kubenix-modules/media.nix" "media" "media"; - kubenix.traefik = mkDeployScriptAndManifest - "${self}/kubenix-modules/traefik.nix" "traefik" "kube-system"; - kubenix.minecraft = mkDeployScriptAndManifest - "${self}/kubenix-modules/minecraft.nix" "minecraft" "minecraft"; + kubenix = builtins.mapAttrs mkDeployScriptAndManifest deployers; }) diff --git a/kubenix-modules/all.nix b/kubenix-modules/all.nix deleted file mode 100644 index 055de68..0000000 --- a/kubenix-modules/all.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ - imports = [ - # ./argo.nix - ./base.nix - ./longhorn.nix - ./metallb.nix - ./cert-manager.nix - ./custom - ./volumes.nix - ./custom-types.nix - ]; -} diff --git a/kubenix-modules/base.nix b/kubenix-modules/base.nix deleted file mode 100644 index cb21e91..0000000 --- a/kubenix-modules/base.nix +++ /dev/null @@ -1,94 +0,0 @@ -# We deploy several resources that rely on "custom resource definitions". -# We must first import these resources definitions, before deploying resources that depend on them. -{ lib, kubenix, nixhelm, system, machines, ... }: { - imports = [ - kubenix.modules.k8s - kubenix.modules.helm - ]; - - config = { - kubenix.project = "home"; - - kubernetes = { - kubeconfig = "~/.kube/config"; - - # TODO: These were copied from https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.crds.yaml - # See https://cert-manager.io/docs/installation/helm/ - # Seems kubenix cannot import a list of resources, but only individual resources. - # Might be good to create a PR for this. - imports = [ - ./cert-manager-manifests/certificaterequest.yaml - ./cert-manager-manifests/certificate.yaml - ./cert-manager-manifests/challenge.yaml - ./cert-manager-manifests/clusterissuer.yaml - ./cert-manager-manifests/issuer.yaml - ./cert-manager-manifests/order.yaml - ]; - - helm.releases = { - metallb = { - chart = nixhelm.chartsDerivations.${system}.metallb.metallb; - includeCRDs = true; - }; - - cert-manager = { - chart = nixhelm.chartsDerivations.${system}.jetstack.cert-manager; - includeCRDs = false; - }; - - # argo-workflows = { - # chart = nixhelm.chartsDerivations.${system}.argoproj.argo-workflows; - # includeCRDs = true; - # }; - - longhorn = { - chart = nixhelm.chartsDerivations.${system}.longhorn.longhorn; - includeCRDs = true; - - values = { - persistence.defaultClassReplicaCount = 2; - - defaultSettings = { - defaultDataPath = "/mnt/longhorn"; - storageMinimalAvailablePercentage = 0; - allowRecurringJobWhileVolumeDetached = true; - backupTarget = "nfs://lewis.dmz:/mnt/longhorn/persistent/longhorn-backup"; - }; - }; - }; - }; - - resources = { - namespaces = { - static-websites = { }; - freshrss = { }; - radicale = { }; - kms = { }; - atuin = { }; - nextcloud = { }; - hedgedoc = { }; - kitchenowl = { }; - forgejo = { }; - paperless = { }; - syncthing = { }; - immich = { }; - attic = { }; - inbucket = { }; - dns = { }; - media = { }; - minecraft = { }; - }; - - nodes = - let - machinesWithKubernetesLabels = lib.filterAttrs (name: machine: machine.kubernetesNodeLabels != null) machines; - in - builtins.mapAttrs - (name: machine: { - metadata.labels = machine.kubernetesNodeLabels; - }) - machinesWithKubernetesLabels; - }; - }; - }; -} diff --git a/kubenix-modules/bootstrapDefault.nix b/kubenix-modules/bootstrapDefault.nix new file mode 100644 index 0000000..e581c96 --- /dev/null +++ b/kubenix-modules/bootstrapDefault.nix @@ -0,0 +1,164 @@ +{ lib, nixhelm, system, machines, ... }: { + kubernetes = { + + helm.releases = { + metallb = { + chart = nixhelm.chartsDerivations.${system}.metallb.metallb; + includeCRDs = true; + }; + + # argo-workflows = { + # chart = nixhelm.chartsDerivations.${system}.argoproj.argo-workflows; + # includeCRDs = true; + # }; + + longhorn = { + chart = nixhelm.chartsDerivations.${system}.longhorn.longhorn; + includeCRDs = true; + + values = { + persistence.defaultClassReplicaCount = 2; + + defaultSettings = { + defaultDataPath = "/mnt/longhorn"; + storageMinimalAvailablePercentage = 0; + allowRecurringJobWhileVolumeDetached = true; + backupTarget = "nfs://lewis.dmz:/mnt/longhorn/persistent/longhorn-backup"; + }; + }; + }; + + # argo-workflows = { + # chart = nixhelm.chartsDerivations.${system}.argoproj.argo-workflows; + # includeCRDs = true; + # }; + }; + + resources = { + namespaces = { + static-websites = { }; + freshrss = { }; + radicale = { }; + kms = { }; + atuin = { }; + nextcloud = { }; + hedgedoc = { }; + kitchenowl = { }; + forgejo = { }; + paperless = { }; + syncthing = { }; + immich = { }; + attic = { }; + inbucket = { }; + dns = { }; + media = { }; + minecraft = { }; + }; + + nodes = + let + machinesWithKubernetesLabels = lib.filterAttrs (name: machine: machine.kubernetesNodeLabels != null) machines; + in + builtins.mapAttrs + (name: machine: { + metadata.labels = machine.kubernetesNodeLabels; + }) + machinesWithKubernetesLabels; + + ingresses.longhorn = { + metadata.annotations = { + "cert-manager.io/cluster-issuer" = "letsencrypt"; + "traefik.ingress.kubernetes.io/router.entrypoints" = "localsecure"; + }; + + spec = { + ingressClassName = "traefik"; + + rules = [{ + host = "longhorn.kun.is"; + + http.paths = [{ + path = "/"; + pathType = "Prefix"; + + backend.service = { + name = "longhorn-frontend"; + port.number = 80; + }; + }]; + }]; + + tls = [{ + secretName = "longhorn-tls"; + hosts = [ "longhorn.kun.is" ]; + }]; + }; + }; + + recurringJobs.backup-nfs.spec = { + cron = "0 1 * * *"; # One o'clock at night + task = "backup"; + retain = 2; # We don't need many, as we also make Borg backups. + concurrency = 1; + }; + + ipAddressPools.main.spec.addresses = [ "192.168.30.128-192.168.30.200" "2a0d:6e00:1a77:30::2-2a0d:6e00:1a77:30:ffff:ffff:ffff:fffe" ]; + l2Advertisements.main.metadata = { }; + + persistentVolumes = { + music-syncthing.spec = { + capacity.storage = "1Gi"; + accessModes = [ "ReadWriteMany" ]; + + nfs = { + server = "lewis.dmz"; + path = "/mnt/longhorn/persistent/media/music"; + }; + }; + + media-media.spec = { + capacity.storage = "1Gi"; + accessModes = [ "ReadWriteMany" ]; + + nfs = { + server = "lewis.dmz"; + path = "/mnt/longhorn/persistent/media"; + }; + }; + }; + }; + }; + + lab = { + longhorn.persistentVolume = { + freshrss.storage = "1Gi"; + radicale.storage = "200Mi"; + atuin.storage = "300Mi"; + atuin-db.storage = "300Mi"; + nextcloud.storage = "50Gi"; + nextcloud-db.storage = "400Mi"; + hedgedoc-uploads.storage = "50Mi"; + hedgedoc-db.storage = "100Mi"; + kitchenowl.storage = "100Mi"; + forgejo.storage = "20Gi"; + paperless-data.storage = "10Gi"; + paperless-redisdata.storage = "20Mi"; + paperless-db.storage = "150Mi"; + syncthing.storage = "400Mi"; + pihole-data.storage = "750Mi"; + pihole-dnsmasq.storage = "16Mi"; + immich.storage = "50Gi"; + immich-db.storage = "5Gi"; + attic.storage = "15Gi"; + attic-db.storage = "150Mi"; + jellyfin.storage = "5Gi"; + transmission.storage = "25Mi"; + jellyseerr.storage = "75Mi"; + radarr.storage = "300Mi"; + prowlarr.storage = "150Mi"; + sonarr.storage = "150Mi"; + bazarr.storage = "25Mi"; + minecraft.storage = "1Gi"; + }; + }; +} diff --git a/kubenix-modules/bootstrapKubeSystem.nix b/kubenix-modules/bootstrapKubeSystem.nix new file mode 100644 index 0000000..43db164 --- /dev/null +++ b/kubenix-modules/bootstrapKubeSystem.nix @@ -0,0 +1,36 @@ +{ nixhelm, system, ... }: { + kubernetes = { + # TODO: These were copied from https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.crds.yaml + # See https://cert-manager.io/docs/installation/helm/ + # Seems kubenix cannot import a list of resources, only individual resources. + # Might be good to create a PR for this. + imports = [ + ./cert-manager-manifests/certificaterequest.yaml + ./cert-manager-manifests/certificate.yaml + ./cert-manager-manifests/challenge.yaml + ./cert-manager-manifests/clusterissuer.yaml + ./cert-manager-manifests/issuer.yaml + ./cert-manager-manifests/order.yaml + ]; + + helm.releases = { + cert-manager = { + chart = nixhelm.chartsDerivations.${system}.jetstack.cert-manager; + includeCRDs = false; + namespace = "kube-system"; + }; + }; + + resources.clusterIssuers.letsencrypt = { + spec.acme = { + server = "https://acme-v02.api.letsencrypt.org/directory"; + email = "pim@kunis.nl"; + privateKeySecretRef.name = "letsencrypt-private-key"; + solvers = [{ + selector = { }; + http01.ingress.class = "traefik"; + }]; + }; + }; + }; +} diff --git a/kubenix-modules/cert-manager.nix b/kubenix-modules/cert-manager.nix deleted file mode 100644 index ff3dbd1..0000000 --- a/kubenix-modules/cert-manager.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ - kubernetes.resources.clusterIssuers.letsencrypt = { - metadata.namespace = "kube-system"; - - spec.acme = { - server = "https://acme-v02.api.letsencrypt.org/directory"; - email = "pim@kunis.nl"; - privateKeySecretRef.name = "letsencrypt-private-key"; - solvers = [{ - selector = { }; - http01.ingress.class = "traefik"; - }]; - }; - }; -} diff --git a/kubenix-modules/longhorn.nix b/kubenix-modules/longhorn.nix deleted file mode 100644 index 48cb659..0000000 --- a/kubenix-modules/longhorn.nix +++ /dev/null @@ -1,44 +0,0 @@ -{ lib, nixhelm, system, ... }: { - config = { - kubernetes = { - resources = { - ingresses.longhorn = { - metadata.annotations = { - "cert-manager.io/cluster-issuer" = "letsencrypt"; - "traefik.ingress.kubernetes.io/router.entrypoints" = "localsecure"; - }; - - spec = { - ingressClassName = "traefik"; - - rules = [{ - host = "longhorn.kun.is"; - - http.paths = [{ - path = "/"; - pathType = "Prefix"; - - backend.service = { - name = "longhorn-frontend"; - port.number = 80; - }; - }]; - }]; - - tls = [{ - secretName = "longhorn-tls"; - hosts = [ "longhorn.kun.is" ]; - }]; - }; - }; - - recurringJobs.backup-nfs.spec = { - cron = "0 1 * * *"; # One o'clock at night - task = "backup"; - retain = 2; # We don't need many, as we also make Borg backups. - concurrency = 1; - }; - }; - }; - }; -} diff --git a/kubenix-modules/metallb.nix b/kubenix-modules/metallb.nix deleted file mode 100644 index 4adc926..0000000 --- a/kubenix-modules/metallb.nix +++ /dev/null @@ -1,7 +0,0 @@ -# TODO: These resources should probably exist within the kube-system namespace. -{ - kubernetes.resources = { - ipAddressPools.main.spec.addresses = [ "192.168.30.128-192.168.30.200" "2a0d:6e00:1a77:30::2-2a0d:6e00:1a77:30:ffff:ffff:ffff:fffe" ]; - l2Advertisements.main.metadata = { }; - }; -} diff --git a/kubenix-modules/traefik.nix b/kubenix-modules/traefik.nix index 293ac24..36f49ec 100644 --- a/kubenix-modules/traefik.nix +++ b/kubenix-modules/traefik.nix @@ -2,13 +2,10 @@ kubernetes.resources = { helmChartConfigs = { traefik = { - metadata.namespace = lib.mkForce "kube-system"; - # Override Traefik's service with a static load balancer IP. # Create endpoint for HTTPS on port 444. # Allow external name services for servers in LAN. spec.valuesContent = lib.generators.toYAML { } { - # service.annotations."metallb.universe.tf/loadBalancerIPs" = myLib.globals.traefikIPv4; providers.kubernetesIngress.allowExternalNameServices = true; service.loadBalancerIP = myLib.globals.traefikIPv4; diff --git a/kubenix-modules/volumes.nix b/kubenix-modules/volumes.nix deleted file mode 100644 index 8623b1a..0000000 --- a/kubenix-modules/volumes.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ - # kubernetes.resources.pods.testje.spec = { - # containers.testje = { - # image = "nginx"; - - # volumeMounts = [ - - # { name = "freshrss"; mountPath = "/freshrss"; } - # ]; - # }; - - # volumes.freshrss.persistentVolumeClaim.claimName = "freshrss"; - # }; - - kubernetes.resources.persistentVolumes = { - music-syncthing.spec = { - capacity.storage = "1Gi"; - accessModes = [ "ReadWriteMany" ]; - - nfs = { - server = "lewis.dmz"; - path = "/mnt/longhorn/persistent/media/music"; - }; - }; - - media-media.spec = { - capacity.storage = "1Gi"; - accessModes = [ "ReadWriteMany" ]; - - nfs = { - server = "lewis.dmz"; - path = "/mnt/longhorn/persistent/media"; - }; - }; - }; - - lab = { - longhorn.persistentVolume = { - freshrss.storage = "1Gi"; - radicale.storage = "200Mi"; - atuin.storage = "300Mi"; - atuin-db.storage = "300Mi"; - nextcloud.storage = "50Gi"; - nextcloud-db.storage = "400Mi"; - hedgedoc-uploads.storage = "50Mi"; - hedgedoc-db.storage = "100Mi"; - kitchenowl.storage = "100Mi"; - forgejo.storage = "20Gi"; - paperless-data.storage = "10Gi"; - paperless-redisdata.storage = "20Mi"; - paperless-db.storage = "150Mi"; - syncthing.storage = "400Mi"; - pihole-data.storage = "750Mi"; - pihole-dnsmasq.storage = "16Mi"; - immich.storage = "50Gi"; - immich-db.storage = "5Gi"; - attic.storage = "15Gi"; - attic-db.storage = "150Mi"; - jellyfin.storage = "5Gi"; - transmission.storage = "25Mi"; - jellyseerr.storage = "75Mi"; - radarr.storage = "300Mi"; - prowlarr.storage = "150Mi"; - sonarr.storage = "150Mi"; - bazarr.storage = "25Mi"; - minecraft.storage = "1Gi"; - }; - }; -}