From 0c7ee6b1bce1388e304ac851fac668b55cb93a4c Mon Sep 17 00:00:00 2001
From: Pim Kunis <pim@kunis.nl>
Date: Sun, 23 Jul 2023 16:58:00 +0200
Subject: [PATCH] add prometheus node exporters and agent

---
 ansible/inventory/host_vars/lewis/apt.yml     |  1 +
 ansible/prometheus.yml                        |  9 +++++++-
 .../container-prometheus.service              | 19 +++++++++++++++++
 .../roles/prometheus_agent/handlers/main.yml  |  4 ++++
 ansible/roles/prometheus_agent/prometheus.yml | 21 +++++++++++++++++++
 ansible/roles/prometheus_agent/tasks/main.yml | 21 +++++++++++++++++++
 6 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 ansible/roles/prometheus_agent/container-prometheus.service
 create mode 100644 ansible/roles/prometheus_agent/handlers/main.yml
 create mode 100644 ansible/roles/prometheus_agent/prometheus.yml
 create mode 100644 ansible/roles/prometheus_agent/tasks/main.yml

diff --git a/ansible/inventory/host_vars/lewis/apt.yml b/ansible/inventory/host_vars/lewis/apt.yml
index 84f31b0..3f8f79d 100644
--- a/ansible/inventory/host_vars/lewis/apt.yml
+++ b/ansible/inventory/host_vars/lewis/apt.yml
@@ -4,3 +4,4 @@ apt_install_packages:
   - sudo
   - bridge-utils
   - borgmatic
+  - podman
diff --git a/ansible/prometheus.yml b/ansible/prometheus.yml
index cd9f03e..84845bd 100644
--- a/ansible/prometheus.yml
+++ b/ansible/prometheus.yml
@@ -1,6 +1,13 @@
 ---
-- name: Setup prometheus
+- name: Setup node exporters
   hosts: hypervisors
 
   roles:
     - role: geerlingguy.node_exporter
+
+- name: Setup Prometheus agent
+  hosts: lewis
+
+  roles:
+    - role: prometheus_agent
+      tags: agent
diff --git a/ansible/roles/prometheus_agent/container-prometheus.service b/ansible/roles/prometheus_agent/container-prometheus.service
new file mode 100644
index 0000000..43468bc
--- /dev/null
+++ b/ansible/roles/prometheus_agent/container-prometheus.service
@@ -0,0 +1,19 @@
+[Unit]
+Description=Podman container-prometheus.service
+Documentation=man:podman-generate-systemd(1)
+Wants=network.target
+After=network-online.target
+
+[Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
+Restart=on-failure
+TimeoutStopSec=70
+ExecStartPre=/bin/rm -f %t/container-prometheus.pid %t/container-prometheus.ctr-id
+ExecStart=/usr/bin/podman run --network=host --conmon-pidfile %t/container-prometheus.pid --cidfile %t/container-prometheus.ctr-id --cgroups=no-conmon --replace --name prometheus -d -p 127.0.0.1:9100:9100 -p 9090:9090 -v /root/prometheus/data:/data/db:Z -v /root/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml prometheus --enable-feature=agent --config.file=/etc/prometheus/prometheus.yml --web.console.libraries=/usr/share/prometheus/console_libraries --web.console.templates=/usr/share/prometheus/consoles
+ExecStop=/usr/bin/podman stop --ignore --cidfile %t/container-prometheus.ctr-id -t 10
+ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/container-prometheus.ctr-id
+PIDFile=%t/container-prometheus.pid
+Type=forking
+
+[Install]
+WantedBy=multi-user.target default.target
diff --git a/ansible/roles/prometheus_agent/handlers/main.yml b/ansible/roles/prometheus_agent/handlers/main.yml
new file mode 100644
index 0000000..8e107b4
--- /dev/null
+++ b/ansible/roles/prometheus_agent/handlers/main.yml
@@ -0,0 +1,4 @@
+- name: systemd daemon reload
+  systemd:
+    name: container-prometheus.service
+    daemon_reload: true
diff --git a/ansible/roles/prometheus_agent/prometheus.yml b/ansible/roles/prometheus_agent/prometheus.yml
new file mode 100644
index 0000000..0891fdc
--- /dev/null
+++ b/ansible/roles/prometheus_agent/prometheus.yml
@@ -0,0 +1,21 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files: []
+
+scrape_configs:
+  - job_name: "lewis"
+    static_configs:
+      - targets: ["localhost:9100"]
+
+  - job_name: "atlas"
+    static_configs:
+      - targets: ["atlas.hyp:9100"]
+
+  - job_name: "jefke"
+    static_configs:
+      - targets: ["jefke.hyp:9100"]
+
+remote_write:
+  - url: https://metrics.kun.is:444/api/v1/write
diff --git a/ansible/roles/prometheus_agent/tasks/main.yml b/ansible/roles/prometheus_agent/tasks/main.yml
new file mode 100644
index 0000000..6e79294
--- /dev/null
+++ b/ansible/roles/prometheus_agent/tasks/main.yml
@@ -0,0 +1,21 @@
+- name: Create prometheus directory
+  file:
+    path: /root/prometheus
+    state: directory
+
+- name: Copy prometheus config
+  copy:
+    src: "{{ role_path }}/prometheus.yml"
+    dest: /root/prometheus/prometheus.yml
+
+- name: Copy prometheus systemd unit
+  copy:
+    src: "{{ role_path }}/container-prometheus.service"
+    dest: /etc/systemd/system/container-prometheus.service
+  notify: systemd daemon reload
+
+- name: Enable prometheus systemd unit
+  systemd:
+    name: container-prometheus.service
+    enabled: true
+    state: started