add monitoring stack

ansible/inventory/group_vars/all.yml

@@ -1,5 +1,6 @@
 data_directory_base: /mnt/data
 git_ssh_port: 56287
+elasticsearch_port: 14653
 concourse_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBSVLcr617iJt+hqLFSsOQy1JeueLIAj1eRfuI+KeZAu pim@x260"
 
 nfs_shares:
@@ -37,6 +38,12 @@ nfs_shares:
     path: /mnt/data/overleaf/mongodb
   - name: prometheus_data
     path: /mnt/data/prometheus/data
+  - name: elasticsearch_certs
+    path: /mnt/data/elasticsearch/certs
+  - name: elasticsearch_data
+    path: /mnt/data/elasticsearch/data
+  - name: grafana_data
+    path: /mnt/data/grafana/data
 
 database_passwords:
   nextcloud: !vault |

ansible/playbooks/setup.yml

@@ -53,3 +53,10 @@
       authorized_key:
         user: root
         key: "{{ concourse_public_key }}"
+
+- hosts: manager, workers
+  tasks:
+    - name: Increase vm.max_map_count
+      sysctl:
+        name: vm.max_map_count
+        value: 262144

ansible/playbooks/stacks.yml

@@ -20,3 +20,4 @@
     - {role: nextcloud, tags: nextcloud}
     - {role: syncthing, tags: syncthing}
     - {role: prometheus, tags: prometheus}
+    - {role: monitoring, tags: monitoring}

ansible/roles/monitoring/docker-stack.yml.j2

@@ -0,0 +1,98 @@
+# vi: ft=yaml
+version: "3.8"
+
+networks:
+  traefik:
+    external: true
+
+configs:
+  esdatasource:
+    external: true
+    name: "{{ esdatasource.config_name }}"
+
+volumes:
+  escerts:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.30.10,nolock,soft,rw"
+      device: ":/mnt/data/elasticsearch/certs"
+  esdata:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.30.10,nolock,soft,rw"
+      device: ":/mnt/data/elasticsearch/data"
+  grafanadata:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.30.10,nolock,soft,rw"
+      device: ":/mnt/data/grafana/data"
+
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.8.1
+    volumes:
+      - type: volume
+        source: escerts
+        target: /usr/share/elasticsearch/config/certs
+        volume:
+          nocopy: true
+      - type: volume
+        source: esdata
+        target: /usr/share/elasticsearch/data
+        volume:
+          nocopy: true
+    ports:
+      - {{ elasticsearch_port }}:9200
+    environment:
+      - node.name=es01
+      - cluster.name=shoarma
+      - cluster.initial_master_nodes=es01
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=false
+      - xpack.security.http.ssl.enabled=false
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.enabled=false
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=basic
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl http://localhost:9200 | grep -q 'You Know, for Search'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  grafana:
+    image: grafana/grafana-oss
+    networks:
+      - traefik
+    deploy:
+      labels:
+        - traefik.enable=true
+        - traefik.http.routers.grafana.entrypoints=localsecure
+        - traefik.http.routers.grafana.rule=Host(`grafana.kun.is`)
+        - traefik.http.routers.grafana.tls=true
+        - traefik.http.routers.grafana.tls.certresolver=letsencrypt
+        - traefik.http.routers.grafana.service=grafana
+        - traefik.http.services.grafana.loadbalancer.server.port=3000
+        - traefik.docker.network=traefik
+    volumes:
+      - type: volume
+        source: grafanadata
+        target: /var/lib/grafana
+        volume:
+          nocopy: true
+    configs:
+      - source: esdatasource
+        target: /etc/grafana/provisioning/datasources/elasticsearch.yaml

ansible/roles/monitoring/elasticsearch.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+datasources:
+  - name: Elasticsearch
+    type: elasticsearch
+    access: proxy
+    url: http://maestro.dmz:14653
+    jsonData:
+      # index: '[metrics-]YYYY.MM.DD'
+      interval: Daily
+      timeField: '@timestamp'
+

ansible/roles/monitoring/tasks/main.yml

@@ -0,0 +1,13 @@
+- name: Create elasticsearch data source config
+  docker_config:
+    name: esdatasource
+    data: "{{ lookup('template', '{{ role_path }}/elasticsearch.yml') }}"
+    use_ssh_client: true
+    rolling_versions: true
+  register: esdatasource
+
+- name: Deploy Docker stack
+  docker_stack:
+    name: monitoring
+    compose:
+      - "{{ lookup('template', '{{ role_path }}/docker-stack.yml.j2') | from_yaml }}"