From 10bd58170bdc2403ced9f798bf62da2c1c8eabf6 Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Tue, 5 Mar 2024 20:56:00 +0100 Subject: [PATCH] monitoring websites with gatus --- flake.lock | 19 +- flake.nix | 3 +- nix/flake/checks.nix | 15 +- nix/flake/deploy.nix | 19 +- nix/flake/nixos.nix | 13 +- nix/machines/warwick.nix | 4 +- nix/modules/default.nix | 2 +- .../default.nix} | 21 +- nix/modules/monitoring/gatus-endpoints.nix | 187 ++++++++++++++++++ nix/physical.nix | 4 +- 10 files changed, 262 insertions(+), 25 deletions(-) rename nix/modules/{prometheus.nix => monitoring/default.nix} (70%) create mode 100644 nix/modules/monitoring/gatus-endpoints.nix diff --git a/flake.lock b/flake.lock index 1beb669..b6917b3 100644 --- a/flake.lock +++ b/flake.lock @@ -248,6 +248,22 @@ "type": "github" } }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1709499969, + "narHash": "sha256-PXi7pnvg+uYY5oUm8Vgw4pXZaGUGjGIOZ4gQ/yGVoPo=", + "owner": "pizzapim", + "repo": "nixpkgs", + "rev": "c74dae81760b4e48633133e689358ff4a120eabe", + "type": "github" + }, + "original": { + "owner": "pizzapim", + "ref": "gatus", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs_2": { "locked": { "lastModified": 1707514827, @@ -273,7 +289,8 @@ "flake-utils": "flake-utils_2", "microvm": "microvm", "nixos-hardware": "nixos-hardware", - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs_2", + "nixpkgs-unstable": "nixpkgs-unstable" } }, "spectrum": { diff --git a/flake.nix b/flake.nix index 742b33c..c20fa29 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,7 @@ inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixos-23.11"; + nixpkgs-unstable.url = "github:pizzapim/nixpkgs/gatus"; deploy-rs.url = "github:serokell/deploy-rs"; nixos-hardware.url = "github:NixOS/nixos-hardware/master"; flake-utils.url = "github:numtide/flake-utils"; @@ -29,7 +30,7 @@ }; outputs = - inputs@{ self, nixpkgs, deploy-rs, disko, agenix, dns, microvm, nixos-hardware, flake-utils, ... }: + inputs@{ self, nixpkgs, flake-utils, ... }: let hostSystem = "x86_64-linux"; hostPkgs = import nixpkgs { system = hostSystem; }; diff --git a/nix/flake/checks.nix b/nix/flake/checks.nix index c5d8b3a..3d1bd2a 100644 --- a/nix/flake/checks.nix +++ b/nix/flake/checks.nix @@ -1,11 +1,14 @@ { self, hostPkgs, machines, flake-utils, deploy-rs, ... }: flake-utils.lib.eachDefaultSystem (system: { # Deploy-rs' flake checks seem broken for architectures different from the deployment machine. # We skip these here. - checks = deploy-rs.lib.${system}.deployChecks (self.deploy // { - nodes = (hostPkgs.lib.attrsets.filterAttrs - (name: node: + + checks = deploy-rs.lib.${system}.deployChecks ( + hostPkgs.lib.attrsets.updateManyAttrsByPath [{ + path = [ "nodes" ]; + update = hostPkgs.lib.attrsets.filterAttrs (name: node: machines.${name}.arch == system - ) - self.deploy.nodes); - }); + ); + }] + self.deploy + ); }) diff --git a/nix/flake/deploy.nix b/nix/flake/deploy.nix index 39408b8..0977958 100644 --- a/nix/flake/deploy.nix +++ b/nix/flake/deploy.nix @@ -10,14 +10,17 @@ in sshUser = "root"; user = "root"; - nodes = mkDeployNodes (name: machine: { - hostname = self.nixosConfigurations.${name}.config.networking.fqdn; - profiles.system = { - remoteBuild = machine.arch != hostPkgs.stdenv.hostPlatform.system; - path = deploy-rs.lib.${machine.arch}.activate.nixos - self.nixosConfigurations.${name}; - }; - }); + nodes = mkDeployNodes (name: machine: + let + nixosConfiguration = self.nixosConfigurations.${name}; + in + { + hostname = nixosConfiguration.config.networking.fqdn; + profiles.system = { + remoteBuild = machine.arch != hostPkgs.stdenv.hostPlatform.system; + path = deploy-rs.lib.${machine.arch}.activate.nixos nixosConfiguration; + }; + }); }; } diff --git a/nix/flake/nixos.nix b/nix/flake/nixos.nix index 304d46e..a544fbd 100644 --- a/nix/flake/nixos.nix +++ b/nix/flake/nixos.nix @@ -1,4 +1,4 @@ -{ nixpkgs, machines, physicalMachines, dns, microvm, disko, agenix, nixos-hardware, ... }: +{ nixpkgs, nixpkgs-unstable, machines, physicalMachines, dns, microvm, disko, agenix, nixos-hardware, ... }: let mkNixosSystems = systemDef: builtins.mapAttrs @@ -11,10 +11,19 @@ in nixosConfigurations = mkNixosSystems (name: machine: { system = machine.arch; - specialArgs = { inherit machines machine dns microvm disko agenix nixos-hardware; }; + specialArgs = { inherit nixpkgs-unstable machines machine dns microvm disko agenix nixos-hardware; }; modules = [ ../. { networking.hostName = name; } + { + nixpkgs.overlays = [ + (final: _prev: { + unstable = import nixpkgs-unstable { + system = machine.arch; + }; + }) + ]; + } ]; }); } diff --git a/nix/machines/warwick.nix b/nix/machines/warwick.nix index e677e3e..174a001 100644 --- a/nix/machines/warwick.nix +++ b/nix/machines/warwick.nix @@ -4,6 +4,8 @@ arch = "aarch64-linux"; isRaspberryPi = true; - nixosModule.lab.services.prometheus.server.enable = true; + nixosModule = { + lab.monitoring.server.enable = true; + }; }; } diff --git a/nix/modules/default.nix b/nix/modules/default.nix index dbc70fa..c07bf26 100644 --- a/nix/modules/default.nix +++ b/nix/modules/default.nix @@ -6,6 +6,6 @@ ./networking ./data-sharing.nix ./globals.nix - ./prometheus.nix + ./monitoring ]; } diff --git a/nix/modules/prometheus.nix b/nix/modules/monitoring/default.nix similarity index 70% rename from nix/modules/prometheus.nix rename to nix/modules/monitoring/default.nix index 46bb5b3..49267ed 100644 --- a/nix/modules/prometheus.nix +++ b/nix/modules/monitoring/default.nix @@ -1,10 +1,14 @@ -{ lib, config, machines, ... }: +{ lib, pkgs, nixpkgs-unstable, config, machines, ... }: let - cfg = config.lab.services.prometheus; + cfg = config.lab.monitoring; in { + imports = [ + "${nixpkgs-unstable}/nixos/modules/services/monitoring/gatus.nix" + ]; + options = { - lab.services.prometheus = { + lab.monitoring = { enable = lib.mkOption { default = true; type = lib.types.bool; @@ -45,5 +49,16 @@ in machines ); }; + + services.gatus = lib.mkIf cfg.server.enable { + enable = true; + package = pkgs.unstable.gatus; + openFirewall = true; + + settings = { + web.port = 4242; + endpoints = import ./gatus-endpoints.nix; + }; + }; }; } diff --git a/nix/modules/monitoring/gatus-endpoints.nix b/nix/modules/monitoring/gatus-endpoints.nix new file mode 100644 index 0000000..3aae24d --- /dev/null +++ b/nix/modules/monitoring/gatus-endpoints.nix @@ -0,0 +1,187 @@ +let + status = code: "[STATUS] == ${toString code}"; + bodyContains = text: "[BODY] == pat(*${text}*)"; + maxResponseTime = ms: "[RESPONSE_TIME] < ${toString ms}"; + endpoints = [ + { + name = "Forgejo"; + url = "https://git.kun.is"; + conditions = [ + (status 200) + (bodyContains "Forgejo: Beyond coding. We forge.") + (maxResponseTime 750) + ]; + } + { + name = "Nextcloud"; + url = "https://cloud.kun.is/status.php"; + conditions = [ + (status 200) + "[BODY].installed == true" + "[BODY].maintenance == false" + "[BODY].needsDbUpgrade == false" + (maxResponseTime 750) + ]; + } + { + name = "Paperless-ngx"; + url = "https://paperless.kun.is/accounts/login/"; + conditions = [ + (status 200) + (bodyContains "Please sign in.") + (maxResponseTime 750) + ]; + } + { + name = "Radicale"; + url = "https://dav.kun.is/.web/"; + conditions = [ + (status 200) + (bodyContains "Login") + (maxResponseTime 750) + ]; + } + { + name = "FreshRSS"; + url = "https://rss.kun.is/i/"; + conditions = [ + (status 200) + (bodyContains "Login") + (maxResponseTime 750) + ]; + } + { + name = "KitchenOwl"; + url = "https://boodschappen.kun.is/signin"; + conditions = [ + (status 200) + (bodyContains "KitchenOwl") + (maxResponseTime 750) + ]; + } + { + name = "HedgeDoc"; + url = "https://md.kun.is/"; + conditions = [ + (status 200) + (bodyContains "The best platform to write and share markdown.") + (maxResponseTime 750) + ]; + } + { + name = "Cyberchef"; + url = "https://cyberchef.kun.is/"; + conditions = [ + (status 200) + (bodyContains "CyberChef - The Cyber Swiss Army Knife") + (maxResponseTime 750) + ]; + } + { + name = "Pi-hole"; + url = "https://pihole.kun.is:444/admin/login.php"; + conditions = [ + (status 200) + (bodyContains "Log in") + (maxResponseTime 750) + ]; + } + { + name = "Inbucket"; + url = "https://inbucket.kun.is:444/"; + conditions = [ + (status 200) + (bodyContains "Inbucket") + (maxResponseTime 750) + ]; + } + { + name = "kms"; + url = "tcp://kms.kun.is:1688"; + conditions = [ + "[CONNECTED] == true" + ]; + } + { + name = "Bazarr"; + url = "https://bazarr.kun.is:444/system/status"; + conditions = [ + (status 200) + (bodyContains "Bazarr") + (maxResponseTime 750) + ]; + } + { + name = "Sonarr"; + url = "https://sonarr.kun.is:444/system/status"; + conditions = [ + (status 200) + (bodyContains "Sonarr") + (maxResponseTime 750) + ]; + } + { + name = "Radarr"; + url = "https://radarr.kun.is:444/system/status"; + conditions = [ + (status 200) + (bodyContains "Radarr") + (maxResponseTime 750) + ]; + } + { + name = "Jellyfin"; + url = "https://media.kun.is/web/index.html#!/login.html?"; + conditions = [ + (status 200) + (bodyContains "Jellyfin") + (maxResponseTime 750) + ]; + } + { + name = "Jellyseerr"; + url = "https://jellyseerr.kun.is:444/login"; + conditions = [ + (status 200) + (bodyContains "Sign in to continue") + (maxResponseTime 750) + ]; + } + { + name = "Prowlarr"; + url = "https://prowlarr.kun.is:444/system/status"; + conditions = [ + (status 200) + (bodyContains "Prowlarr") + (maxResponseTime 750) + ]; + } + { + name = "Transmission"; + url = "https://transmission.kun.is:444/transmission/web/"; + conditions = [ + (status 200) + (bodyContains "Transmission Web Interface") + (maxResponseTime 750) + ]; + } + { + name = "Syncthing"; + url = "https://sync.kun.is:444/"; + conditions = [ + (status 401) + (maxResponseTime 750) + ]; + } + { + name = "Traefik"; + url = "https://traefik.kun.is:444/dashboard/#/"; + conditions = [ + (status 200) + (bodyContains "Traefik") + (maxResponseTime 750) + ]; + } + ]; +in +map (endpoint: endpoint // { interval = "5m"; }) endpoints diff --git a/nix/physical.nix b/nix/physical.nix index 281dc6f..e565dae 100644 --- a/nix/physical.nix +++ b/nix/physical.nix @@ -1,4 +1,4 @@ -{ pkgs, config, lib, microvm, disko, agenix, machine, machines, dns, nixos-hardware, ... }: { +{ pkgs, nixpkgs-unstable, config, lib, microvm, disko, agenix, machine, machines, dns, nixos-hardware, ... }: { imports = [ microvm.nixosModules.host ] @@ -69,7 +69,7 @@ restartIfChanged = false; specialArgs = { - inherit agenix disko pkgs lib microvm dns; + inherit agenix disko pkgs lib microvm dns nixpkgs-unstable; machine = vm; hypervisorConfig = config; };