nixos-servers/modules/monitoring/default.nix

{
  lib,
  config,
  machines,
  ...
}: let
  cfg = config.lab.monitoring;
in {
  options = {
    lab.monitoring = {
      enable = lib.mkOption {
        default = true;
        type = lib.types.bool;
      };

      server.enable = lib.mkOption {
        default = false;
        type = lib.types.bool;
      };
    };
  };

  config = lib.mkIf cfg.enable {
    networking.firewall.allowedTCPPorts =
      [config.services.prometheus.exporters.node.port]
      ++ lib.lists.optionals cfg.server.enable [80];

    services.prometheus = {
      enable = cfg.server.enable;

      exporters = {
        node = {
          enable = true;
        };
      };

      scrapeConfigs = lib.mkIf cfg.server.enable (
        let
          generated =
            lib.attrsets.mapAttrsToList
            (name: machine: {
              job_name = name;
              static_configs = [
                {
                  targets = ["${name}.dmz:${toString config.services.prometheus.exporters.node.port}"];
                }
              ];
            })
            machines;

          pikvm = {
            job_name = "pikvm";
            metrics_path = "/api/export/prometheus/metrics";
            scheme = "https";
            tls_config.insecure_skip_verify = true;

            # We don't care about security here, it's behind a VPN.
            basic_auth = {
              username = "admin";
              password = "admin";
            };

            static_configs = [
              {
                targets = ["pikvm.dmz"];
              }
            ];
          };
        in
          generated ++ [pikvm]
      );
    };

    services.nginx = lib.mkIf cfg.server.enable {
      enable = true;

      virtualHosts."${config.networking.fqdn}" = {
        locations."/" = {
          proxyPass = "http://127.0.0.1:${toString config.services.prometheus.port}";
          recommendedProxySettings = true;
        };
      };
    };
  };
}