cleanup more nix code

modules/backups.nix

@@ -0,0 +1,129 @@
+{ pkgs, lib, config, ... }:
+let
+  cfg = config.lab.backups;
+  beforeEverything = pkgs.writeShellScriptBin "beforeEverything" ''
+    if [ -d "${cfg.snapshotLocation}" ]; then
+      ${pkgs.btrfs-progs}/bin/btrfs subvolume delete ${cfg.snapshotLocation}
+    fi
+
+    ${pkgs.btrfs-progs}/bin/btrfs subvolume snapshot -r ${cfg.subvolumeLocation} ${cfg.snapshotLocation}
+  '';
+
+  borgmaticConfig = pkgs.writeTextFile {
+    name = "borgmatic-config";
+    text = ''
+      source_directories:
+      - ${cfg.snapshotLocation}
+      repositories:
+      - path: ${cfg.repoLocation}
+        label: nfs
+      - path: ssh://s6969ym3@s6969ym3.repo.borgbase.com/./repo
+        label: borgbase
+      exclude_patterns:
+        - ${cfg.snapshotLocation}/media
+      ssh_command: "${pkgs.openssh}/bin/ssh -i ${config.age.secrets."borgbase.pem".path} -o StrictHostKeychecking=no"
+      keep_daily: 7
+      keep_weekly: 4
+      keep_monthly: 6
+      encryption_passcommand: "${pkgs.coreutils}/bin/cat ''${BORG_PASSPHRASE_FILE}"
+      before_everything:
+      - ${beforeEverything}/bin/beforeEverything
+      postgresql_databases:
+      - name: nextcloud
+        hostname: lewis.dmz
+        username: nextcloud
+        password: ''${NEXTCLOUD_DATABASE_PASSWORD}
+        format: tar
+      - name: hedgedoc
+        hostname: lewis.dmz
+        username: hedgedoc
+        password: ''${HEDGEDOC_DATABASE_PASSWORD}
+        format: tar
+      - name: paperless
+        hostname: lewis.dmz
+        username: paperless
+        password: ''${PAPERLESS_DATABASE_PASSWORD}
+        format: tar
+    '';
+  };
+in
+{
+  options.lab.backups = {
+    enable = lib.mkOption {
+      default = false;
+      type = lib.types.bool;
+      description = ''
+        Whether to enable backups of persistent data on this machine.
+      '';
+    };
+
+    repoLocation = lib.mkOption {
+      default = "${config.lab.storage.dataMountPoint}/backups/nfs.borg";
+      type = lib.types.str;
+      description = ''
+        Location of the Borg repository to back up to.
+      '';
+    };
+
+    subvolumeLocation = lib.mkOption {
+      default = "${config.lab.storage.dataMountPoint}/nfs";
+      type = lib.types.str;
+      description = ''
+        Location of the btrfs subvolume holding the data.
+      '';
+    };
+
+    snapshotLocation = lib.mkOption {
+      default = "${config.lab.storage.dataMountPoint}/snapshot-nfs";
+      type = lib.types.str;
+      description = ''
+        Location to (temporary) create a snapshot of the subvolume.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = with pkgs; [ borgbackup postgresql ];
+    # Converted from:
+    # https://github.com/borgmatic-collective/borgmatic/tree/84823dfb912db650936e3492f6ead7e0e0d32a0f/sample/systemd
+    systemd.services.borgmatic = {
+      description = "borgmatic backup";
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      unitConfig.ConditionACPower = true;
+      preStart = "${pkgs.coreutils}/bin/sleep 10s";
+      path = with pkgs; [ postgresql ];
+
+      serviceConfig = {
+        Type = "oneshot";
+        Nice = 19;
+        CPUSchedulingPolicy = "batch";
+        IOSchedulingClass = "best-effort";
+        IOSchedulingPriority = 7;
+        IOWeight = 100;
+        Restart = "no";
+        LogRateLimitIntervalSec = 0;
+        EnvironmentFile = config.age.secrets."database_passwords.env".path;
+        Environment = "BORG_PASSPHRASE_FILE=${config.age.secrets."borg_passphrase".path}";
+      };
+
+      script = "${pkgs.systemd}/bin/systemd-inhibit --who=\"borgmatic\" --what=\"sleep:shutdown\" --why=\"Prevent interrupting scheduled backup\" ${pkgs.borgmatic}/bin/borgmatic --verbosity -2 --syslog-verbosity 1 -c ${borgmaticConfig}";
+    };
+
+    systemd.timers.borgmatic = {
+      description = "Run borgmatic backup";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "*-*-* 3:00:00";
+        Persistent = true;
+        RandomizedDelaySec = "3h";
+      };
+    };
+
+    age.secrets = {
+      "database_passwords.env".file = ../secrets/database_passwords.env.age;
+      "borg_passphrase".file = ../secrets/borg_passphrase.age;
+      "borgbase.pem".file = ../secrets/borgbase.pem.age;
+    };
+  };
+}

modules/data-sharing.nix

@@ -0,0 +1,97 @@
+{ pkgs, lib, config, ... }:
+let
+  cfg = config.lab.data-sharing;
+
+  nfsShares = [
+    "/nextcloud/data"
+    "/radicale"
+    "/freshrss/data"
+    "/freshrss/extensions"
+    "/pihole/data"
+    "/pihole/dnsmasq"
+    "/hedgedoc/uploads"
+    "/traefik/acme"
+    "/forgejo"
+    "/kitchenowl/data"
+    "/syncthing/config"
+    "/paperless-ngx/data"
+    "/paperless-ngx/redisdata"
+    "/media"
+    "/media/books"
+    "/media/movies"
+    "/media/music"
+    "/media/shows"
+    "/jellyfin/config"
+    "/transmission/config"
+    "/jellyseerr/config"
+    "/radarr/config"
+    "/prowlarr/config"
+    "/sonarr/config"
+    "/bazarr/config"
+  ];
+
+  nfsExports = lib.strings.concatLines (
+    builtins.map
+      (share:
+        "${cfg.nfsRoot}${share} 192.168.30.0/16(rw,sync,no_subtree_check,no_root_squash)"
+      )
+      nfsShares
+  );
+in
+{
+  options.lab.data-sharing = {
+    enable = lib.mkOption {
+      default = false;
+      type = lib.types.bool;
+      description = ''
+        Configure this server to serve our data using NFS and PostgreSQL.
+      '';
+    };
+
+    nfsRoot = lib.mkOption {
+      default = "/mnt/data/nfs";
+      type = lib.types.str;
+      description = ''
+        Root directory of NFS data.
+      '';
+    };
+
+    postgresDir = lib.mkOption {
+      default = "/mnt/data/postgresql/${config.services.postgresql.package.psqlSchema}";
+      type = lib.types.str;
+      description = ''
+        Postgresql data directory.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    networking.firewall.allowedTCPPorts = [
+      2049 # NFS
+      5432 # PostgeSQL
+      111 # NFS
+      20048 # NFS
+    ];
+
+    services = {
+      nfs.server = {
+        enable = true;
+        exports = nfsExports;
+      };
+
+      postgresql = {
+        enable = true;
+        package = pkgs.postgresql_15;
+        enableTCPIP = true;
+
+        dataDir = cfg.postgresDir;
+
+        authentication = ''
+          host	nextcloud	nextcloud	all	      md5
+          host	hedgedoc	hedgedoc	all	      md5
+          host	paperless	paperless	all	      md5
+        '';
+      };
+    };
+  };
+}

modules/default.nix

@@ -0,0 +1,11 @@
+{
+  imports = [
+    ./storage.nix
+    ./backups.nix
+    ./networking
+    ./data-sharing.nix
+    ./globals.nix
+    ./monitoring
+    ./k3s
+  ];
+}

modules/globals.nix

@@ -0,0 +1,89 @@
+{ lib, ... }: {
+  options.lab = {
+
+    networking = {
+      public = {
+        ipv4 = {
+          router = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              Public IPv4 address of the router.
+            '';
+          };
+        };
+
+        ipv6 = {
+          router = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              Publicly routable IPv6 address of the router.
+            '';
+          };
+        };
+      };
+
+      dmz = {
+        ipv4 = {
+          prefixLength = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              IPv4 prefix length of DMZ network.
+            '';
+          };
+
+          dockerSwarm = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              IPv4 address of the Docker Swarm in the DMZ.
+            '';
+          };
+
+
+          router = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              The router's IPv4 address on the DMZ network.
+            '';
+          };
+
+          services = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              The IPv4 address of the interface serving DHCP and DNS on the DMZ network.
+            '';
+          };
+        };
+
+        ipv6 = {
+          prefixLength = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              IPv6 prefix length of DMZ network.
+            '';
+          };
+
+          dockerSwarm = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              Globally routable IPv6 address of the Docker Swarm.
+            '';
+          };
+
+          router = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              The router's IPv6 address on the DMZ network.
+            '';
+          };
+
+          services = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              The IPv6 address of the interface serving DHCP and DNS on the DMZ network.
+            '';
+          };
+        };
+      };
+    };
+  };
+}

modules/k3s/bootstrap.nix

@@ -0,0 +1,20 @@
+{ kubenix, ... }: {
+  imports = [ kubenix.modules.k8s ];
+  kubernetes.resources.clusterRoleBindings.cluster-admins = {
+    roleRef = {
+      apiGroup = "rbac.authorization.k8s.io";
+      kind = "ClusterRole";
+      name = "cluster-admin";
+    };
+    subjects = [
+      {
+        kind = "User";
+        name = "pim";
+      }
+      {
+        kind = "User";
+        name = "niels";
+      }
+    ];
+  };
+}

modules/k3s/default.nix

@@ -0,0 +1,37 @@
+{ pkgs, lib, config, kubenix, ... }:
+let cfg = config.lab.k3s;
+in {
+  options.lab.k3s.enable = lib.mkOption {
+    default = false;
+    type = lib.types.bool;
+    description = ''
+      Whether to start k3s with custom configuration.
+    '';
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = with pkgs; [ k3s ];
+
+    networking = {
+      nftables.enable = lib.mkForce false;
+      firewall.enable = lib.mkForce false;
+    };
+
+    services.k3s = {
+      enable = true;
+      role = "server";
+      extraFlags = "--tls-san ${config.networking.fqdn} --disable servicelb";
+    };
+
+    system.activationScripts.k3s-bootstrap.text =
+      let
+        k3sBootstrapFile = (kubenix.evalModules.x86_64-linux {
+          module = import ./bootstrap.nix;
+        }).config.kubernetes.result;
+      in
+      ''
+        mkdir -p /var/lib/rancher/k3s/server/manifests
+        ln -sf ${k3sBootstrapFile} /var/lib/rancher/k3s/server/manifests/k3s-bootstrap.json
+      '';
+  };
+}

modules/monitoring/default.nix

@@ -0,0 +1,112 @@
+{ lib, pkgs, nixpkgs-unstable, config, machines, ... }:
+let
+  cfg = config.lab.monitoring;
+in
+{
+  imports = [
+    "${nixpkgs-unstable}/nixos/modules/services/monitoring/gatus.nix"
+    ./gatus-endpoints.nix
+  ];
+
+  options = {
+    lab.monitoring = {
+      enable = lib.mkOption {
+        default = true;
+        type = lib.types.bool;
+      };
+
+      server.enable = lib.mkOption {
+        default = false;
+        type = lib.types.bool;
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    networking.firewall.allowedTCPPorts = [ config.services.prometheus.exporters.node.port ]
+      ++ lib.lists.optionals cfg.server.enable [ 80 ];
+
+    services.prometheus = {
+      enable = cfg.server.enable;
+      webExternalUrl = "/prometheus";
+
+      exporters = {
+        node = {
+          enable = true;
+        };
+      };
+
+      scrapeConfigs = lib.mkIf cfg.server.enable (
+        lib.attrsets.mapAttrsToList
+          (name: machine: {
+            job_name = name;
+            static_configs = [{
+              targets = [ "${name}.dmz:${toString config.services.prometheus.exporters.node.port}" ];
+            }];
+          })
+          machines
+      );
+    };
+
+    services.gatus = lib.mkIf cfg.server.enable {
+      enable = true;
+      package = pkgs.unstable.gatus;
+
+      settings = {
+        storage = {
+          type = "sqlite";
+          path = "/srv/gatus/gatus.db";
+        };
+
+        alerting.email = {
+          from = "gatus@kun.is";
+          host = "mail.smtp2go.com";
+          port = 2525;
+          to = "pim@kunis.nl";
+          client.insecure = true;
+
+          default-alert = {
+            enabled = true;
+            failure-threshold = 2;
+            success-threshold = 1;
+            send-on-resolved = true;
+          };
+        };
+      };
+    };
+
+    users = {
+      users.gatus = {
+        isSystemUser = true;
+        group = "gatus";
+      };
+
+      groups.gatus = { };
+    };
+
+    system.activationScripts = lib.mkIf cfg.server.enable {
+      gatus = ''
+        mkdir -p /srv/gatus
+        chown gatus:gatus /srv/gatus
+      '';
+    };
+
+    services.nginx = lib.mkIf cfg.server.enable {
+      enable = true;
+
+      virtualHosts."${config.networking.fqdn}" = {
+        locations = {
+          "/" = {
+            proxyPass = "http://127.0.0.1:${toString config.services.gatus.settings.web.port}";
+            recommendedProxySettings = true;
+          };
+
+          "/prometheus/" = {
+            proxyPass = "http://127.0.0.1:${toString config.services.prometheus.port}";
+            recommendedProxySettings = true;
+          };
+        };
+      };
+    };
+  };
+}

modules/monitoring/gatus-endpoints.nix

@@ -0,0 +1,230 @@
+{ lib, config, machines, ... }:
+let
+  cfg = config.lab.monitoring;
+
+  status = code: "[STATUS] == ${toString code}";
+  bodyContains = text: "[BODY] == pat(*${text}*)";
+  maxResponseTime = ms: "[RESPONSE_TIME] < ${toString ms}";
+
+  machineEndpoints = lib.attrsets.mapAttrsToList
+    (name: machine: {
+      name = "Host ${name}";
+      url = "icmp://${name}.dmz";
+      conditions = [ "[RESPONSE_TIME] < 10" ];
+    })
+    machines;
+
+  otherEndpoints = [
+    {
+      name = "Forgejo";
+      url = "https://git.kun.is";
+      conditions = [
+        (status 200)
+        (bodyContains "Forgejo: Beyond coding. We forge.")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Nextcloud";
+      url = "https://cloud.kun.is/status.php";
+      conditions = [
+        (status 200)
+        "[BODY].installed == true"
+        "[BODY].maintenance == false"
+        "[BODY].needsDbUpgrade == false"
+        (maxResponseTime 2000)
+      ];
+    }
+    {
+      name = "Paperless-ngx";
+      url = "https://paperless.kun.is/accounts/login/";
+      conditions = [
+        (status 200)
+        (bodyContains "Please sign in.")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Radicale";
+      url = "https://dav.kun.is/.web/";
+      conditions = [
+        (status 200)
+        (bodyContains "Login")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "FreshRSS";
+      url = "https://rss.kun.is/i/";
+      conditions = [
+        (status 200)
+        (bodyContains "Login")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "KitchenOwl";
+      url = "https://boodschappen.kun.is/signin";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>KitchenOwl</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "HedgeDoc";
+      url = "https://md.kun.is/";
+      conditions = [
+        (status 200)
+        (bodyContains "The best platform to write and share markdown.")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Cyberchef";
+      url = "https://cyberchef.kun.is/";
+      conditions = [
+        (status 200)
+        (bodyContains "CyberChef - The Cyber Swiss Army Knife")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Pi-hole";
+      url = "https://pihole.kun.is:444/admin/login.php";
+      conditions = [
+        (status 200)
+        (bodyContains "Log in")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Inbucket";
+      url = "https://inbucket.kun.is:444/";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Inbucket</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "kms";
+      url = "tcp://kms.kun.is:1688";
+      conditions = [
+        "[CONNECTED] == true"
+      ];
+    }
+    {
+      name = "Bazarr";
+      url = "https://bazarr.kun.is:444/system/status";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Bazarr</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Sonarr";
+      url = "https://sonarr.kun.is:444/system/status";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Sonarr</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Radarr";
+      url = "https://radarr.kun.is:444/system/status";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Radarr</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Jellyfin";
+      url = "https://media.kun.is/web/index.html#!/login.html?";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Jellyfin</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Jellyseerr";
+      url = "https://jellyseerr.kun.is:444/login";
+      conditions = [
+        (status 200)
+        (bodyContains "Sign in to continue")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Prowlarr";
+      url = "https://prowlarr.kun.is:444/system/status";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Prowlarr</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Transmission";
+      url = "https://transmission.kun.is:444/transmission/web/";
+      conditions = [
+        (status 200)
+        (bodyContains "Transmission Web Interface")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Syncthing";
+      url = "https://sync.kun.is:444/";
+      conditions = [
+        (status 401)
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Traefik";
+      url = "https://traefik.kun.is:444/dashboard/#/";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Traefik</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "BIND";
+      url = "192.168.30.7";
+      dns = {
+        query-type = "SOA";
+        query-name = "kun.is";
+      };
+      conditions = [
+        "[DNS_RCODE] == NOERROR"
+      ];
+    }
+    {
+      name = "Pi-hole DNS";
+      url = "192.168.30.8";
+      dns = {
+        query-type = "SOA";
+        query-name = "kun.is";
+      };
+      conditions = [
+        "[DNS_RCODE] == NOERROR"
+      ];
+    }
+  ];
+in
+{
+  config = lib.mkIf cfg.server.enable {
+    services.gatus.settings.endpoints = map
+      (endpoint: endpoint // {
+        interval = "5m";
+        alerts = [{ type = "email"; }];
+      })
+      (machineEndpoints ++ otherEndpoints);
+  };
+}

modules/networking/default.nix

@@ -0,0 +1,38 @@
+{ lib, machine, ... }: {
+  config = {
+    networking = {
+      domain = "dmz";
+      nftables.enable = true;
+      useDHCP = false;
+
+      firewall = {
+        enable = true;
+      };
+    };
+
+    systemd.network = {
+      enable = true;
+
+      networks = lib.attrsets.mergeAttrsList [
+        (lib.optionalAttrs (! machine.isRaspberryPi) {
+          "30-main-nic" = {
+            matchConfig.Name = "en*";
+
+            networkConfig = {
+              DHCP = "yes";
+            };
+          };
+        })
+        (lib.optionalAttrs machine.isRaspberryPi {
+          "30-main-nic" = {
+            matchConfig.Name = "end*";
+            networkConfig = {
+              IPv6AcceptRA = true;
+              DHCP = "yes";
+            };
+          };
+        })
+      ];
+    };
+  };
+}

modules/storage.nix

@@ -0,0 +1,80 @@
+{ lib, config, machine, ... }:
+let cfg = config.lab.storage;
+in {
+  options.lab.storage = {
+    osDisk = lib.mkOption {
+      type = with lib.types; nullOr str;
+      description = ''
+        The disk to be used for the machine's operating system.
+      '';
+    };
+
+    dataPartition = lib.mkOption {
+      default = null;
+      type = lib.types.nullOr lib.types.str;
+      description = ''
+        Partition to be used for data storage on this machine.
+      '';
+    };
+
+    dataMountPoint = lib.mkOption {
+      default = "/mnt/data";
+      type = lib.types.str;
+      description = ''
+        Mount point of the machine's data partition.
+      '';
+    };
+  };
+
+  config = {
+    fileSystems = lib.attrsets.mergeAttrsList [
+      (lib.optionalAttrs (! machine.isRaspberryPi) {
+        "${cfg.dataMountPoint}".device = cfg.dataPartition;
+      })
+      (lib.optionalAttrs machine.isRaspberryPi {
+        "/" = {
+          device = "/dev/disk/by-label/NIXOS_SD";
+          fsType = "ext4";
+          options = [ "noatime" ];
+        };
+      })
+    ];
+
+    disko = lib.mkIf (! machine.isRaspberryPi) {
+      # TODO: Rename this to 'osDisk'. Unfortunately, we would need to run nixos-anywhere again then.
+      devices.disk.vdb = {
+        device = cfg.osDisk;
+        type = "disk";
+
+        content = {
+          type = "gpt";
+
+          partitions = {
+            swap.size = "100%";
+
+            ESP = {
+              type = "EF00";
+              size = "500M";
+
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+              };
+            };
+
+            root = {
+              end = "-4G";
+
+              content = {
+                type = "filesystem";
+                format = "btrfs";
+                mountpoint = "/";
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}