add prometheus node exporters and agent

ansible/inventory/host_vars/lewis/apt.yml

@@ -4,3 +4,4 @@ apt_install_packages:
   - sudo
   - bridge-utils
   - borgmatic
+  - podman

ansible/prometheus.yml

@@ -1,6 +1,13 @@
 ---
-- name: Setup prometheus
+- name: Setup node exporters
   hosts: hypervisors
 
   roles:
     - role: geerlingguy.node_exporter
+
+- name: Setup Prometheus agent
+  hosts: lewis
+
+  roles:
+    - role: prometheus_agent
+      tags: agent

ansible/roles/prometheus_agent/container-prometheus.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Podman container-prometheus.service
+Documentation=man:podman-generate-systemd(1)
+Wants=network.target
+After=network-online.target
+
+[Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
+Restart=on-failure
+TimeoutStopSec=70
+ExecStartPre=/bin/rm -f %t/container-prometheus.pid %t/container-prometheus.ctr-id
+ExecStart=/usr/bin/podman run --network=host --conmon-pidfile %t/container-prometheus.pid --cidfile %t/container-prometheus.ctr-id --cgroups=no-conmon --replace --name prometheus -d -p 127.0.0.1:9100:9100 -p 9090:9090 -v /root/prometheus/data:/data/db:Z -v /root/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml prometheus --enable-feature=agent --config.file=/etc/prometheus/prometheus.yml --web.console.libraries=/usr/share/prometheus/console_libraries --web.console.templates=/usr/share/prometheus/consoles
+ExecStop=/usr/bin/podman stop --ignore --cidfile %t/container-prometheus.ctr-id -t 10
+ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/container-prometheus.ctr-id
+PIDFile=%t/container-prometheus.pid
+Type=forking
+
+[Install]
+WantedBy=multi-user.target default.target

ansible/roles/prometheus_agent/handlers/main.yml

@@ -0,0 +1,4 @@
+- name: systemd daemon reload
+  systemd:
+    name: container-prometheus.service
+    daemon_reload: true

ansible/roles/prometheus_agent/prometheus.yml

@@ -0,0 +1,21 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files: []
+
+scrape_configs:
+  - job_name: "lewis"
+    static_configs:
+      - targets: ["localhost:9100"]
+
+  - job_name: "atlas"
+    static_configs:
+      - targets: ["atlas.hyp:9100"]
+
+  - job_name: "jefke"
+    static_configs:
+      - targets: ["jefke.hyp:9100"]
+
+remote_write:
+  - url: https://metrics.kun.is:444/api/v1/write

ansible/roles/prometheus_agent/tasks/main.yml

@@ -0,0 +1,21 @@
+- name: Create prometheus directory
+  file:
+    path: /root/prometheus
+    state: directory
+
+- name: Copy prometheus config
+  copy:
+    src: "{{ role_path }}/prometheus.yml"
+    dest: /root/prometheus/prometheus.yml
+
+- name: Copy prometheus systemd unit
+  copy:
+    src: "{{ role_path }}/container-prometheus.service"
+    dest: /etc/systemd/system/container-prometheus.service
+  notify: systemd daemon reload
+
+- name: Enable prometheus systemd unit
+  systemd:
+    name: container-prometheus.service
+    enabled: true
+    state: started