nixos-servers/nix/modules/monitoring/default.nix

{ lib, pkgs, nixpkgs-unstable, config, machines, ... }:
let
  cfg = config.lab.monitoring;
in
{
  imports = [
    "${nixpkgs-unstable}/nixos/modules/services/monitoring/gatus.nix"
    ./gatus-endpoints.nix
  ];

  options = {
    lab.monitoring = {
      enable = lib.mkOption {
        default = true;
        type = lib.types.bool;
      };

      server.enable = lib.mkOption {
        default = false;
        type = lib.types.bool;
      };
    };
  };

  config = lib.mkIf cfg.enable {
    networking.firewall.allowedTCPPorts = [ config.services.prometheus.exporters.node.port ]
      ++ lib.lists.optionals cfg.server.enable [ config.services.prometheus.port ];

    services.prometheus = {
      enable = cfg.server.enable;

      exporters = {
        node = {
          enable = true;
        };
      };

      scrapeConfigs = lib.mkIf cfg.server.enable (
        lib.attrsets.mapAttrsToList
          (name: machine:
            let
              domain = if machine.isPhysical then "hyp" else "dmz";
            in
            {
              job_name = name;
              static_configs = [{
                targets = [ "${name}.${domain}:${toString config.services.prometheus.exporters.node.port}" ];
              }];
            })
          machines
      );
    };

    services.gatus = lib.mkIf cfg.server.enable {
      enable = true;
      package = pkgs.unstable.gatus;
      openFirewall = true;

      settings = {
        web.port = 4242;

        storage = {
          type = "sqlite";
          path = "/srv/gatus/gatus.db";
        };

        alerting.email = {
          from = "gatus@kun.is";
          host = "mail.smtp2go.com";
          port = 2525;
          to = "pim@kunis.nl";
          client.insecure = true;

          default-alert = {
            enabled = true;
            failure-threshold = 2;
            success-threshold = 1;
            send-on-resolved = true;
          };
        };
      };
    };

    system.activationScripts = lib.mkIf cfg.server.enable {
      gatus = ''
        mkdir -p /srv/gatus
        chown gatus:gatus /srv/gatus
      '';
    };
  };
}