monitoring websites with gatus

flake.lock

@@ -248,6 +248,22 @@
         "type": "github"
       }
     },
+    "nixpkgs-unstable": {
+      "locked": {
+        "lastModified": 1709499969,
+        "narHash": "sha256-PXi7pnvg+uYY5oUm8Vgw4pXZaGUGjGIOZ4gQ/yGVoPo=",
+        "owner": "pizzapim",
+        "repo": "nixpkgs",
+        "rev": "c74dae81760b4e48633133e689358ff4a120eabe",
+        "type": "github"
+      },
+      "original": {
+        "owner": "pizzapim",
+        "ref": "gatus",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "nixpkgs_2": {
       "locked": {
         "lastModified": 1707514827,
@@ -273,7 +289,8 @@
         "flake-utils": "flake-utils_2",
         "microvm": "microvm",
         "nixos-hardware": "nixos-hardware",
-        "nixpkgs": "nixpkgs_2"
+        "nixpkgs": "nixpkgs_2",
+        "nixpkgs-unstable": "nixpkgs-unstable"
       }
     },
     "spectrum": {

flake.nix

@@ -3,6 +3,7 @@
 
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-23.11";
+    nixpkgs-unstable.url = "github:pizzapim/nixpkgs/gatus";
     deploy-rs.url = "github:serokell/deploy-rs";
     nixos-hardware.url = "github:NixOS/nixos-hardware/master";
     flake-utils.url = "github:numtide/flake-utils";
@@ -29,7 +30,7 @@
   };
 
   outputs =
-    inputs@{ self, nixpkgs, deploy-rs, disko, agenix, dns, microvm, nixos-hardware, flake-utils, ... }:
+    inputs@{ self, nixpkgs, flake-utils, ... }:
     let
       hostSystem = "x86_64-linux";
       hostPkgs = import nixpkgs { system = hostSystem; };

nix/flake/checks.nix

@@ -1,11 +1,14 @@
 { self, hostPkgs, machines, flake-utils, deploy-rs, ... }: flake-utils.lib.eachDefaultSystem (system: {
   # Deploy-rs' flake checks seem broken for architectures different from the deployment machine.
   # We skip these here.
-  checks = deploy-rs.lib.${system}.deployChecks (self.deploy // {
+
-    nodes = (hostPkgs.lib.attrsets.filterAttrs
+  checks = deploy-rs.lib.${system}.deployChecks (
-      (name: node:
+    hostPkgs.lib.attrsets.updateManyAttrsByPath [{
+      path = [ "nodes" ];
+      update = hostPkgs.lib.attrsets.filterAttrs (name: node:
         machines.${name}.arch == system
-      )
+      );
-      self.deploy.nodes);
+    }]
-  });
+      self.deploy
+  );
 })

nix/flake/deploy.nix

@@ -10,12 +10,15 @@ in
     sshUser = "root";
     user = "root";
 
-    nodes = mkDeployNodes (name: machine: {
+    nodes = mkDeployNodes (name: machine:
-      hostname = self.nixosConfigurations.${name}.config.networking.fqdn;
+      let
+        nixosConfiguration = self.nixosConfigurations.${name};
+      in
+      {
+        hostname = nixosConfiguration.config.networking.fqdn;
         profiles.system = {
           remoteBuild = machine.arch != hostPkgs.stdenv.hostPlatform.system;
-        path = deploy-rs.lib.${machine.arch}.activate.nixos
+          path = deploy-rs.lib.${machine.arch}.activate.nixos nixosConfiguration;
-          self.nixosConfigurations.${name};
         };
       });
   };

nix/flake/nixos.nix

@@ -1,4 +1,4 @@
-{ nixpkgs, machines, physicalMachines, dns, microvm, disko, agenix, nixos-hardware, ... }:
+{ nixpkgs, nixpkgs-unstable, machines, physicalMachines, dns, microvm, disko, agenix, nixos-hardware, ... }:
 let
   mkNixosSystems = systemDef:
     builtins.mapAttrs
@@ -11,10 +11,19 @@ in
   nixosConfigurations = mkNixosSystems (name: machine: {
     system = machine.arch;
 
-    specialArgs = { inherit machines machine dns microvm disko agenix nixos-hardware; };
+    specialArgs = { inherit nixpkgs-unstable machines machine dns microvm disko agenix nixos-hardware; };
     modules = [
       ../.
       { networking.hostName = name; }
+      {
+        nixpkgs.overlays = [
+          (final: _prev: {
+            unstable = import nixpkgs-unstable {
+              system = machine.arch;
+            };
+          })
+        ];
+      }
     ];
   });
 }

nix/machines/warwick.nix

@@ -4,6 +4,8 @@
     arch = "aarch64-linux";
     isRaspberryPi = true;
 
-    nixosModule.lab.services.prometheus.server.enable = true;
+    nixosModule = {
+      lab.monitoring.server.enable = true;
+    };
   };
 }

nix/modules/default.nix

@@ -6,6 +6,6 @@
     ./networking
     ./data-sharing.nix
     ./globals.nix
-    ./prometheus.nix
+    ./monitoring
   ];
 }

nix/modules/monitoring/default.nix

@@ -1,10 +1,14 @@
-{ lib, config, machines, ... }:
+{ lib, pkgs, nixpkgs-unstable, config, machines, ... }:
 let
-  cfg = config.lab.services.prometheus;
+  cfg = config.lab.monitoring;
 in
 {
+  imports = [
+    "${nixpkgs-unstable}/nixos/modules/services/monitoring/gatus.nix"
+  ];
+
   options = {
-    lab.services.prometheus = {
+    lab.monitoring = {
       enable = lib.mkOption {
         default = true;
         type = lib.types.bool;
@@ -45,5 +49,16 @@ in
           machines
       );
     };
+
+    services.gatus = lib.mkIf cfg.server.enable {
+      enable = true;
+      package = pkgs.unstable.gatus;
+      openFirewall = true;
+
+      settings = {
+        web.port = 4242;
+        endpoints = import ./gatus-endpoints.nix;
+      };
+    };
   };
 }

nix/modules/monitoring/gatus-endpoints.nix

@@ -0,0 +1,187 @@
+let
+  status = code: "[STATUS] == ${toString code}";
+  bodyContains = text: "[BODY] == pat(*${text}*)";
+  maxResponseTime = ms: "[RESPONSE_TIME] < ${toString ms}";
+  endpoints = [
+    {
+      name = "Forgejo";
+      url = "https://git.kun.is";
+      conditions = [
+        (status 200)
+        (bodyContains "Forgejo: Beyond coding. We forge.")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Nextcloud";
+      url = "https://cloud.kun.is/status.php";
+      conditions = [
+        (status 200)
+        "[BODY].installed == true"
+        "[BODY].maintenance == false"
+        "[BODY].needsDbUpgrade == false"
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Paperless-ngx";
+      url = "https://paperless.kun.is/accounts/login/";
+      conditions = [
+        (status 200)
+        (bodyContains "Please sign in.")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Radicale";
+      url = "https://dav.kun.is/.web/";
+      conditions = [
+        (status 200)
+        (bodyContains "Login")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "FreshRSS";
+      url = "https://rss.kun.is/i/";
+      conditions = [
+        (status 200)
+        (bodyContains "Login")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "KitchenOwl";
+      url = "https://boodschappen.kun.is/signin";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>KitchenOwl</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "HedgeDoc";
+      url = "https://md.kun.is/";
+      conditions = [
+        (status 200)
+        (bodyContains "The best platform to write and share markdown.")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Cyberchef";
+      url = "https://cyberchef.kun.is/";
+      conditions = [
+        (status 200)
+        (bodyContains "CyberChef - The Cyber Swiss Army Knife")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Pi-hole";
+      url = "https://pihole.kun.is:444/admin/login.php";
+      conditions = [
+        (status 200)
+        (bodyContains "Log in")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Inbucket";
+      url = "https://inbucket.kun.is:444/";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Inbucket</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "kms";
+      url = "tcp://kms.kun.is:1688";
+      conditions = [
+        "[CONNECTED] == true"
+      ];
+    }
+    {
+      name = "Bazarr";
+      url = "https://bazarr.kun.is:444/system/status";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Bazarr</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Sonarr";
+      url = "https://sonarr.kun.is:444/system/status";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Sonarr</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Radarr";
+      url = "https://radarr.kun.is:444/system/status";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Radarr</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Jellyfin";
+      url = "https://media.kun.is/web/index.html#!/login.html?";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Jellyfin</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Jellyseerr";
+      url = "https://jellyseerr.kun.is:444/login";
+      conditions = [
+        (status 200)
+        (bodyContains "Sign in to continue")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Prowlarr";
+      url = "https://prowlarr.kun.is:444/system/status";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Prowlarr</title>")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Transmission";
+      url = "https://transmission.kun.is:444/transmission/web/";
+      conditions = [
+        (status 200)
+        (bodyContains "Transmission Web Interface")
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Syncthing";
+      url = "https://sync.kun.is:444/";
+      conditions = [
+        (status 401)
+        (maxResponseTime 750)
+      ];
+    }
+    {
+      name = "Traefik";
+      url = "https://traefik.kun.is:444/dashboard/#/";
+      conditions = [
+        (status 200)
+        (bodyContains "<title>Traefik</title>")
+        (maxResponseTime 750)
+      ];
+    }
+  ];
+in
+map (endpoint: endpoint // { interval = "5m"; }) endpoints

nix/physical.nix

@@ -1,4 +1,4 @@
-{ pkgs, config, lib, microvm, disko, agenix, machine, machines, dns, nixos-hardware, ... }: {
+{ pkgs, nixpkgs-unstable, config, lib, microvm, disko, agenix, machine, machines, dns, nixos-hardware, ... }: {
   imports = [
     microvm.nixosModules.host
   ]
@@ -69,7 +69,7 @@
             restartIfChanged = false;
 
             specialArgs = {
-              inherit agenix disko pkgs lib microvm dns;
+              inherit agenix disko pkgs lib microvm dns nixpkgs-unstable;
               machine = vm;
               hypervisorConfig = config;
             };