nixos-servers/nixos-modules/monitoring/default.nix

{ lib, config, machines, ... }:
let
  cfg = config.lab.monitoring;
in
{
  options = {
    lab.monitoring = {
      enable = lib.mkOption {
        default = true;
        type = lib.types.bool;
      };

      server.enable = lib.mkOption {
        default = false;
        type = lib.types.bool;
      };
    };
  };

  config = lib.mkIf cfg.enable {
    networking.firewall.allowedTCPPorts = [ config.services.prometheus.exporters.node.port ]
      ++ lib.lists.optionals cfg.server.enable [ 80 ];

    services.prometheus = {
      enable = cfg.server.enable;

      exporters = {
        node = {
          enable = true;
        };
      };

      scrapeConfigs = lib.mkIf cfg.server.enable (
        let
          generated = lib.attrsets.mapAttrsToList
            (name: machine: {
              job_name = name;
              static_configs = [{
                targets = [ "${name}.dmz:${toString config.services.prometheus.exporters.node.port}" ];
              }];
            })
            machines;

          pikvm = {
            job_name = "pikvm";
            metrics_path = "/api/export/prometheus/metrics";
            scheme = "https";
            tls_config.insecure_skip_verify = true;

            # We don't care about security here, it's behind a VPN.
            basic_auth = {
              username = "admin";
              password = "admin";
            };

            static_configs = [{
              targets = [ "pikvm.dmz" ];
            }];
          };
        in
        generated ++ [ pikvm ]
      );
    };

    services.nginx = lib.mkIf cfg.server.enable {
      enable = true;

      virtualHosts."${config.networking.fqdn}" = {
        locations."/" = {
          proxyPass = "http://127.0.0.1:${toString config.services.prometheus.port}";
          recommendedProxySettings = true;
        };
      };
    };
  };
}