From a21a09ad6bae705610421946857196b9a9c70d01 Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Sat, 18 May 2024 23:32:58 +0200 Subject: [PATCH] Install longhorn on k3s Introduce new storage standard with LVM --- kubenix-modules/all.nix | 3 +- kubenix-modules/atticd.nix | 1 + kubenix-modules/inbucket.nix | 13 ++++ kubenix-modules/longhorn.nix | 52 ++++++++++++++ machines/atlas.nix | 5 +- machines/jefke.nix | 3 +- machines/warwick.nix | 66 ++++++++--------- nixos-modules/k3s/default.nix | 52 +++++++++----- nixos-modules/storage.nix | 131 ++++++++++++++++++++++++++++++---- 9 files changed, 257 insertions(+), 69 deletions(-) create mode 100644 kubenix-modules/longhorn.nix diff --git a/kubenix-modules/all.nix b/kubenix-modules/all.nix index 1d02a5f..212e014 100644 --- a/kubenix-modules/all.nix +++ b/kubenix-modules/all.nix @@ -15,7 +15,7 @@ let ./media.nix ./bind9 ./dnsmasq.nix - ./minecraft.nix + # ./minecraft.nix ./blog.nix ./atticd.nix ./argo.nix @@ -25,6 +25,7 @@ in imports = [ ./base.nix ./custom-types.nix + ./longhorn.nix ./esrom.nix ./metallb.nix ./cert-manager.nix diff --git a/kubenix-modules/atticd.nix b/kubenix-modules/atticd.nix index 9cea734..6d0dada 100644 --- a/kubenix-modules/atticd.nix +++ b/kubenix-modules/atticd.nix @@ -93,6 +93,7 @@ lab = { ingresses.atticd = { host = "attic.kun.is"; + entrypoint = "localsecure"; service = { name = "atticd"; diff --git a/kubenix-modules/inbucket.nix b/kubenix-modules/inbucket.nix index 633c2ad..ff6b2ba 100644 --- a/kubenix-modules/inbucket.nix +++ b/kubenix-modules/inbucket.nix @@ -17,7 +17,14 @@ web.containerPort = 9000; smtp.containerPort = 2500; }; + + volumeMounts = [{ + name = "storage"; + mountPath = "/storage"; + }]; }; + + volumes.storage.persistentVolumeClaim.claimName = "inbucket"; }; }; }; @@ -44,6 +51,12 @@ }]; }; }; + + persistentVolumeClaims.inbucket.spec = { + accessModes = [ "ReadWriteOnce" ]; + storageClassName = "longhorn"; + resources.requests.storage = "30Mi"; + }; }; lab.ingresses.inbucket = { diff --git a/kubenix-modules/longhorn.nix b/kubenix-modules/longhorn.nix new file mode 100644 index 0000000..5f0f2bc --- /dev/null +++ b/kubenix-modules/longhorn.nix @@ -0,0 +1,52 @@ +{ nixhelm, system, ... }: { + config = { + kubernetes = { + helm.releases.longhorn = { + chart = nixhelm.chartsDerivations.${system}.longhorn.longhorn; + includeCRDs = true; + values = { + defaultSettings = { + defaultDataPath = "/mnt/longhorn"; + storageMinimalAvailablePercentage = 0; + }; + + persistence = { + defaultClassReplicaCount = 2; + }; + }; + }; + + resources = { + ingresses.longhorn = { + metadata.annotations = { + "cert-manager.io/cluster-issuer" = "letsencrypt"; + "traefik.ingress.kubernetes.io/router.entrypoints" = "localsecure"; + }; + + spec = { + ingressClassName = "traefik"; + + rules = [{ + host = "longhorn.kun.is"; + + http.paths = [{ + path = "/"; + pathType = "Prefix"; + + backend.service = { + name = "longhorn-frontend"; + port.number = 80; + }; + }]; + }]; + + tls = [{ + secretName = "longhorn-tls"; + hosts = [ "longhorn.kun.is" ]; + }]; + }; + }; + }; + }; + }; +} diff --git a/machines/atlas.nix b/machines/atlas.nix index 34f146e..cfed93b 100644 --- a/machines/atlas.nix +++ b/machines/atlas.nix @@ -4,13 +4,12 @@ nixosModule.lab = { storage = { - osDisk = "/dev/sda"; - dataPartition = "/dev/nvme0n1p1"; + kubernetesNode = true; }; k3s = { enable = true; - role = "agent"; + # role = "server"; serverAddr = "https://jefke.dmz:6443"; }; }; diff --git a/machines/jefke.nix b/machines/jefke.nix index d2df75c..d9e633d 100644 --- a/machines/jefke.nix +++ b/machines/jefke.nix @@ -4,8 +4,7 @@ nixosModule.lab = { storage = { - osDisk = "/dev/sda"; - dataPartition = "/dev/nvme0n1p1"; + kubernetesNode = true; }; k3s = { diff --git a/machines/warwick.nix b/machines/warwick.nix index 2cf5602..a53159c 100644 --- a/machines/warwick.nix +++ b/machines/warwick.nix @@ -11,46 +11,46 @@ config = builtins.readFile ./bird.conf; }; - systemd.network = { - netdevs = { - hamgre = { - netdevConfig = { - Name = "hamgre"; - Kind = "gre"; - MTUBytes = "1468"; - }; + #systemd.network = { + # netdevs = { + # hamgre = { + # netdevConfig = { + # Name = "hamgre"; + # Kind = "gre"; + # MTUBytes = "1468"; + # }; - tunnelConfig = { - Remote = "145.220.78.4"; - #Local = "192.145.57.90"; - }; - }; + # tunnelConfig = { + # Remote = "145.220.78.4"; + # #Local = "192.145.57.90"; + # }; + # }; - # hambr = { - # netdevConfig = { - # Name = "hambr"; - # Kind = "bridge"; - # }; - # }; - }; + # # hambr = { + # # netdevConfig = { + # # Name = "hambr"; + # # Kind = "bridge"; + # # }; + # # }; + # }; - networks = { - "30-main-nic".networkConfig.Tunnel = "hamgre"; + # networks = { + # "30-main-nic".networkConfig.Tunnel = "hamgre"; - "40-hamgre" = { - matchConfig.Name = "hamgre"; + # "40-hamgre" = { + # matchConfig.Name = "hamgre"; - networkConfig = { - Address = "44.137.61.34/30"; - }; - }; + # networkConfig = { + # Address = "44.137.61.34/30"; + # }; + # }; - # "40-hambr" = { - # matchConfig.Name = "hambr"; + # # "40-hambr" = { + # # matchConfig.Name = "hambr"; - # }; - }; - }; + # # }; + # }; + #}; }; }; } diff --git a/nixos-modules/k3s/default.nix b/nixos-modules/k3s/default.nix index b9e94fb..f268cf7 100644 --- a/nixos-modules/k3s/default.nix +++ b/nixos-modules/k3s/default.nix @@ -37,25 +37,45 @@ in { }; config = lib.mkIf cfg.enable { - environment.systemPackages = with pkgs; [ k3s ]; + environment.systemPackages = with pkgs; [ + k3s + openiscsi # Required for Longhorn + nfs-utils # Required for Longhorn + jq # Required for Longhorn + ]; networking = { nftables.enable = lib.mkForce false; firewall.enable = lib.mkForce false; }; - services.k3s = - let - serverFlags = "--tls-san ${config.networking.fqdn} --disable servicelb --cluster-cidr=10.42.0.0/16,2001:cafe:42::/56 --service-cidr=10.43.0.0/16,2001:cafe:43::/112"; - in - { + services = { + k3s = + let + serverFlags = "--tls-san ${config.networking.fqdn} --disable servicelb --cluster-cidr=10.42.0.0/16,2001:cafe:42::/56 --service-cidr=10.43.0.0/16,2001:cafe:43::/112"; + in + { + enable = true; + role = cfg.role; + tokenFile = config.age.secrets.k3s-server-token.path; + extraFlags = lib.mkIf (cfg.role == "server") serverFlags; + clusterInit = cfg.clusterInit; + serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr; + }; + + # Required for Longhorn + openiscsi = { enable = true; - role = cfg.role; - tokenFile = config.age.secrets.k3s-server-token.path; - extraFlags = lib.mkIf (cfg.role == "server") serverFlags; - clusterInit = cfg.clusterInit; - serverAddr = lib.mkIf (! (cfg.serverAddr == null)) cfg.serverAddr; + name = "iqn.2016-04.com.open-iscsi:${config.networking.fqdn}"; }; + }; + + # HACK: Symlink binaries to /usr/local/bin such that Longhorn can find them + # when they use nsenter. + # https://github.com/longhorn/longhorn/issues/2166#issuecomment-1740179416 + systemd.tmpfiles.rules = [ + "L+ /usr/local/bin - - - - /run/current-system/sw/bin/" + ]; system = lib.mkIf (cfg.role == "server") { activationScripts = { @@ -73,11 +93,11 @@ in { k3s-certs.text = '' mkdir -p /var/lib/rancher/k3s/server/tls/etcd - ln -sf ${./k3s-ca/server-ca.crt} /var/lib/rancher/k3s/server/tls/server-ca.crt - ln -sf ${./k3s-ca/client-ca.crt} /var/lib/rancher/k3s/server/tls/client-ca.crt - ln -sf ${./k3s-ca/request-header-ca.crt} /var/lib/rancher/k3s/server/tls/request-header-ca.crt - ln -sf ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt - ln -sf ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt + cp -f ${./k3s-ca/server-ca.crt} /var/lib/rancher/k3s/server/tls/server-ca.crt + cp -f ${./k3s-ca/client-ca.crt} /var/lib/rancher/k3s/server/tls/client-ca.crt + cp -f ${./k3s-ca/request-header-ca.crt} /var/lib/rancher/k3s/server/tls/request-header-ca.crt + cp -f ${./k3s-ca/etcd/peer-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt + cp -f ${./k3s-ca/etcd/server-ca.crt} /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt ''; }; }; diff --git a/nixos-modules/storage.nix b/nixos-modules/storage.nix index 0f12355..dca7e03 100644 --- a/nixos-modules/storage.nix +++ b/nixos-modules/storage.nix @@ -24,23 +24,126 @@ in { Mount point of the machine's data partition. ''; }; + + kubernetesNode = lib.mkOption { + default = false; + type = lib.types.bool; + description = '' + Whether to apply the Kubernetes disk setup. + ''; + }; }; config = { - fileSystems = lib.attrsets.mergeAttrsList [ - (lib.optionalAttrs (! machine.isRaspberryPi) { - "${cfg.dataMountPoint}".device = cfg.dataPartition; - }) - (lib.optionalAttrs machine.isRaspberryPi { - "/" = { - device = "/dev/disk/by-label/NIXOS_SD"; - fsType = "ext4"; - options = [ "noatime" ]; - }; - }) - ]; + fileSystems = { + "/" = lib.mkIf machine.isRaspberryPi { + device = "/dev/disk/by-label/NIXOS_SD"; + fsType = "ext4"; + options = [ "noatime" ]; + }; + }; - disko = lib.mkIf (! machine.isRaspberryPi) { + disko = lib.mkIf (! machine.isRaspberryPi) (if cfg.kubernetesNode then { + devices = { + disk = { + nvme = { + device = "/dev/nvme0n1"; + type = "disk"; + + content = { + type = "gpt"; + + partitions = { + boot = { + type = "EF00"; + size = "500M"; + + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + }; + }; + + pv_os = { + size = "79G"; + + content = { + type = "lvm_pv"; + vg = "vg_os"; + }; + }; + + pv_nvme_extra = { + size = "100%"; + + content = { + type = "lvm_pv"; + vg = "vg_data"; + }; + }; + }; + }; + }; + + sata = { + device = "/dev/sda"; + type = "disk"; + + content = { + type = "gpt"; + + partitions.pv_sata = { + size = "100%"; + + content = { + type = "lvm_pv"; + vg = "vg_data"; + }; + }; + }; + }; + }; + + lvm_vg = { + vg_os = { + type = "lvm_vg"; + + lvs = { + root = { + size = "75G"; + + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + mountOptions = [ "defaults" ]; + }; + }; + + swap = { + size = "100%FREE"; + content.type = "swap"; + }; + }; + }; + + vg_data = { + type = "lvm_vg"; + + lvs.longhorn = { + size = "100%FREE"; + + content = { + type = "filesystem"; + format = "xfs"; + mountpoint = "/mnt/longhorn"; + }; + }; + }; + }; + }; + } else { # TODO: Rename this to 'osDisk'. Unfortunately, we would need to run nixos-anywhere again then. devices.disk.vdb = { device = cfg.osDisk; @@ -75,6 +178,6 @@ in { }; }; }; - }; + }); }; }