From 5e3245115f5acbb621a1cecd4394567e16ee2082 Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Sat, 7 Jun 2025 17:20:17 +0200 Subject: [PATCH] Enable Gatus service monitoring --- machines/warwick/configuration.nix | 271 ++++++++++++++++++++++++++++- nixos/server.nix | 7 +- secrets/servers.yaml | 13 +- 3 files changed, 281 insertions(+), 10 deletions(-) diff --git a/machines/warwick/configuration.nix b/machines/warwick/configuration.nix index 3e3052b..1371056 100644 --- a/machines/warwick/configuration.nix +++ b/machines/warwick/configuration.nix @@ -3,7 +3,9 @@ config, inputs, ... -}: { +}: let + gatusPort = 8080; +in { imports = [inputs.nixos-hardware.nixosModules.raspberry-pi-4]; config = { @@ -35,5 +37,272 @@ fsType = "ext4"; options = ["noatime"]; }; + + networking.firewall.allowedTCPPorts = [gatusPort]; + systemd.services.gatus.serviceConfig.EnvironmentFile = config.sops.secrets."gatus/env".path; + + services.gatus = { + enable = true; + + settings = { + alerting.email = { + from = "gatus@kun.is"; + host = "mail.smtp2go.com"; + port = 2525; + to = "pim@kunis.nl"; + client.insecure = true; + username = "$SMTP_USERNAME"; + password = "$SMTP_PASSWORD"; + + default-alert = { + enabled = true; + failure-threshold = 2; + success-threshold = 1; + send-on-resolved = true; + }; + }; + + web.port = gatusPort; + endpoints = let + status = code: "[STATUS] == ${toString code}"; + bodyContains = text: "[BODY] == pat(*${text}*)"; + maxResponseTime = ms: "[RESPONSE_TIME] < ${toString ms}"; + serviceEndpoints = [ + { + name = "Blog"; + url = "https://pim.kun.is"; + conditions = [ + (status 200) + (maxResponseTime 300) + ]; + } + { + name = "Cyberchef"; + url = "https://cyberchef.kun.is"; + conditions = [ + (status 200) + (maxResponseTime 300) + (bodyContains "CyberChef - The Cyber Swiss Army Knife") + ]; + } + { + name = "HedgeDoc"; + url = "https://md.kun.is/status"; + conditions = [ + (status 200) + (maxResponseTime 300) + "[BODY].notesCount > 0" + ]; + } + { + name = "Forgejo"; + url = "https://git.kun.is"; + conditions = [ + (status 200) + (maxResponseTime 300) + (bodyContains "Forgejo: Beyond coding. We forge.") + ]; + } + { + name = "Authentik"; + url = "https://authentik.kun.is/-/health/live/"; + conditions = [ + (status 200) + (maxResponseTime 300) + ]; + } + { + name = "Ntfy"; + url = "https://ntfy.kun.is"; + conditions = [ + (status 200) + (maxResponseTime 300) + ]; + } + { + name = "Jellyfin"; + url = "https://media.kun.is/health"; + conditions = [ + (status 200) + (maxResponseTime 300) + ]; + } + { + name = "Attic"; + url = "https://attic.kun.is"; + conditions = [ + (status 200) + (bodyContains "attic push") + (maxResponseTime 300) + ]; + } + { + name = "Esrom"; + url = "https://esrom.kun.is/seinlamp"; + conditions = [ + (status 200) + (bodyContains "Welcome to") + (maxResponseTime 300) + ]; + } + { + name = "Atuin"; + url = "https://atuin.kun.is"; + conditions = [ + (status 200) + (maxResponseTime 300) + "[BODY].total_history > 0" + ]; + } + { + name = "KitchenOwl"; + url = "https://boodschappen.kun.is"; + conditions = [ + (status 200) + (maxResponseTime 300) + (bodyContains "KitchenOwl") + ]; + } + { + name = "Inbucket"; + url = "https://inbucket.griffin-mermaid.ts.net/status"; + conditions = [ + (status 200) + (maxResponseTime 300) + ]; + } + { + name = "FreshRSS"; + url = "https://freshrss.griffin-mermaid.ts.net/i"; + conditions = [ + (status 401) + (maxResponseTime 300) + ]; + } + { + name = "Paperless-ngx"; + url = "https://paperless.griffin-mermaid.ts.net/accounts/login/"; + conditions = [ + (status 200) + (maxResponseTime 300) + (bodyContains "Please sign in.") + ]; + } + { + name = "Jellyseerr"; + url = "https://jellyseerr.griffin-mermaid.ts.net/login"; + conditions = [ + (status 200) + (maxResponseTime 300) + ]; + } + { + name = "Radarr"; + url = "https://radarr.griffin-mermaid.ts.net"; + conditions = [ + (status 200) + (maxResponseTime 300) + ]; + } + { + name = "Sonarr"; + url = "https://sonarr.griffin-mermaid.ts.net/login"; + conditions = [ + (status 200) + (maxResponseTime 300) + ]; + } + { + name = "Bazarr"; + url = "https://bazarr.griffin-mermaid.ts.net/system/status"; + conditions = [ + (status 200) + (maxResponseTime 300) + (bodyContains "Bazarr") + ]; + } + { + name = "Prowlarr"; + url = "https://prowlarr.griffin-mermaid.ts.net/login"; + conditions = [ + (status 200) + (maxResponseTime 300) + ]; + } + { + name = "Deluge"; + url = "https://deluge.griffin-mermaid.ts.net"; + conditions = [ + (status 200) + (maxResponseTime 300) + ]; + } + { + name = "SyncThing"; + url = "https://syncthing.griffin-mermaid.ts.net/"; + conditions = [ + (status 200) + (maxResponseTime 300) + ]; + } + { + name = "Radicale"; + url = "https://radicale.griffin-mermaid.ts.net/.web/"; + conditions = [ + (status 200) + (maxResponseTime 300) + (bodyContains "Sign in") + ]; + } + { + name = "Nextcloud"; + url = "https://nextcloud.griffin-mermaid.ts.net/status.php"; + conditions = [ + (status 200) + (maxResponseTime 300) + "[BODY].installed == true" + "[BODY].maintenance == false" + "[BODY].needsDbUpgrade == false" + ]; + } + { + name = "kms"; + url = "tcp://kms.kun.is:1688"; + conditions = [ + "[CONNECTED] == true" + ]; + } + { + name = "BIND"; + url = "192.168.30.134"; + dns = { + query-type = "SOA"; + query-name = "kun.is"; + }; + conditions = [ + "[DNS_RCODE] == NOERROR" + ]; + } + { + name = "Immich"; + url = "https://immich.griffin-mermaid.ts.net"; + conditions = [ + (status 200) + (maxResponseTime 300) + (bodyContains "To use Immich, you must enable JavaScript or use a JavaScript compatible browser.") + ]; + } + ]; + in + map + (endpoint: + endpoint + // { + interval = "5m"; + alerts = [{type = "email";}]; + }) + serviceEndpoints; + }; + }; }; } diff --git a/nixos/server.nix b/nixos/server.nix index 282edb6..54b5ee1 100644 --- a/nixos/server.nix +++ b/nixos/server.nix @@ -55,7 +55,11 @@ extraUpFlags = [ - "--accept-dns=false" + ( + if builtins.elem "kubernetes" config.deployment.tags + then "--accept-dns=false" + else "--accept-dns=true" + ) "--hostname=${config.networking.hostName}" ] ++ lib.lists.optional config.pim.tailscale.advertiseExitNode "--advertise-exit-node" @@ -67,6 +71,7 @@ "tailscale/authKey".sopsFile = "${self}/secrets/servers.yaml"; "borg/borgPassphrase".sopsFile = "${self}/secrets/servers.yaml"; "borg/borgbasePrivateKey".sopsFile = "${self}/secrets/servers.yaml"; + "gatus/env".sopsFile = "${self}/secrets/servers.yaml"; }; }; } diff --git a/secrets/servers.yaml b/secrets/servers.yaml index d947814..26c620b 100644 --- a/secrets/servers.yaml +++ b/secrets/servers.yaml @@ -3,11 +3,9 @@ tailscale: borg: borgPassphrase: ENC[AES256_GCM,data:UWA2sBLPi63MRVOPTYPWYLujF2M=,iv:FQq/IsZK7LWo30gZc7oT2E9feCLn7Oeg6wDGuezkhu8=,tag:fWYaZUwJrM8x6cemXzz6xg==,type:str] borgbasePrivateKey: ENC[AES256_GCM,data:O7eIY1yvbnBTS96tt5a8vcOEOzit4tEbIHmxnSbNsowC7YNk2g6MShQ6ll86GDiunLY33/Px0bqq9+4z/dk4N3FWQ1v5KQjr/gh+CS8VpIrv9zLv+Ru9UzeWQusbYxqnCu/IAQ1aB8UGV2LSCesQ0r/B6XEe51Phi65uWkbUYa/8voSiws3T3hnNrUDqiHdzfBgWZIQszz7zD92Q+aXu/kGNSxVKbXjWVfqBiyDEtuLEWoC1eENeEs41Ov5YT0Lm6+CUWadPqEwkDSvZWnBhoPwPLTZ/+ftZ/nizXUujsTdRwjcbOwJER+ObhgWDxbJE2WifxFOmHt3ggfSmAN842u5PjfT5gqpXMlTdCwYAYEJvDMqGsADe4p7+vPWJbetaahFrFGN8uBw7rs7W2wIiUKB5bAbAG0o6hdTpWfysuzMOFK/fROvCJsNhhKbbdiQbI0+SogtCkwv69+3uaRTFi33uqKCO6PQ6rMIahjo1lutm9iWq2nX4oI40W1KPC6EU/wF2,iv:rzkjjSnyrs58ZEO8XLsCSFsPHbtnL39SF6NJ6lUg3Ww=,tag:q0sunVc+9bLFoSdeykuT6g==,type:str] +gatus: + env: ENC[AES256_GCM,data:xe5yVFCGIpocPTGdTBw7dc9nZdwJ8697mDg/IX06qJo+a5H6u2DnXqAMZq1XbFs5C2gZR/0fyX8eysX3o6gbHCrIJ6e4vvPi6btKbkJ8xG8=,iv:yKlKJFbbuzqQLVweB/z2YNc8/bzk3Q33mApdV+iFYkc=,tag:kZg+t4RgZZXo2kPNgSEKuw==,type:str] sops: - kms: [] - gcp_kms: [] - azure_kv: [] - hc_vault: [] age: - recipient: age1th8rdw4fs3vmgy9gzc0k9xy88tddjj4vasepckfx9h4nlzsg3q3q4cjgwu enc: | @@ -72,8 +70,7 @@ sops: Y2cwK05uWXFhbndyRlhrSFNjYUlmZ1UKZ1vFRu1QhGGf7BIP8TxK2BIlMZlP3muA R3qLr1lEQmob4O0ilwn65nSCEd1/9W6dUWqeSlJ6CavjG59AvSHfIA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-05-25T14:53:30Z" - mac: ENC[AES256_GCM,data:jzjF+qjdptTI0Y1wNteZgYBGwF5dFWEBIFY3+k4Ty0YU/WB5AyUL6A8v0+PyoxoJK3pL+NAJEmLmAPFVh3+ExDlU9g3TAgpkOs7EsbJtWcjo8Ah08Hl8zoWqcMFcQhZ+aLnVKAE+tIBT4dWyV0AvOWmU8luvarsCp2tQ2OoBH20=,iv:PmbLg91onGz3kjxXMua/Thb904qDkWjHJcBY2dMAios=,tag:e0+fQqNysdiGvaodcimMVQ==,type:str] - pgp: [] + lastmodified: "2025-06-07T15:10:31Z" + mac: ENC[AES256_GCM,data:U39nM9BQp2UAJIcv8tx6X0HJoZrGyDUostNKkszEtWweKqCE5Cyc2iAel4WF005rHRwNCHIzLtj7cYCUihKJD7L7za90i+yh2sY5tr2lvgLYA5dSJJIQxtwEgxcV8rxdKl+Xz6cv5ZeYCiULYfnPx46M+lhlihCxHg1UBGhlc04=,iv:0wfV/4THddgX6c3AGrq0HYT9w8iQRBrDzyaOLAbaBGc=,tag:uHQTjO9IWOAwG1JQ5Sd0cw==,type:str] unencrypted_suffix: _unencrypted - version: 3.9.4 + version: 3.10.2