From 3c4f505413c05dd1c25176a60ea2b2108893ed87 Mon Sep 17 00:00:00 2001 From: Pim Kunis Date: Tue, 1 Aug 2023 16:24:00 +0200 Subject: [PATCH] add monitoring stack --- ansible/inventory/group_vars/all.yml | 7 ++ ansible/playbooks/setup.yml | 7 ++ ansible/playbooks/stacks.yml | 1 + ansible/roles/monitoring/docker-stack.yml.j2 | 98 ++++++++++++++++++++ ansible/roles/monitoring/elasticsearch.yml | 12 +++ ansible/roles/monitoring/tasks/main.yml | 13 +++ 6 files changed, 138 insertions(+) create mode 100644 ansible/roles/monitoring/docker-stack.yml.j2 create mode 100644 ansible/roles/monitoring/elasticsearch.yml create mode 100644 ansible/roles/monitoring/tasks/main.yml diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index c5716bf..0734c92 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -1,5 +1,6 @@ data_directory_base: /mnt/data git_ssh_port: 56287 +elasticsearch_port: 14653 concourse_public_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBSVLcr617iJt+hqLFSsOQy1JeueLIAj1eRfuI+KeZAu pim@x260" nfs_shares: @@ -37,6 +38,12 @@ nfs_shares: path: /mnt/data/overleaf/mongodb - name: prometheus_data path: /mnt/data/prometheus/data + - name: elasticsearch_certs + path: /mnt/data/elasticsearch/certs + - name: elasticsearch_data + path: /mnt/data/elasticsearch/data + - name: grafana_data + path: /mnt/data/grafana/data database_passwords: nextcloud: !vault | diff --git a/ansible/playbooks/setup.yml b/ansible/playbooks/setup.yml index a731fd3..7b06092 100644 --- a/ansible/playbooks/setup.yml +++ b/ansible/playbooks/setup.yml @@ -53,3 +53,10 @@ authorized_key: user: root key: "{{ concourse_public_key }}" + +- hosts: manager, workers + tasks: + - name: Increase vm.max_map_count + sysctl: + name: vm.max_map_count + value: 262144 diff --git a/ansible/playbooks/stacks.yml b/ansible/playbooks/stacks.yml index a46c7d8..5453fd0 100644 --- a/ansible/playbooks/stacks.yml +++ b/ansible/playbooks/stacks.yml @@ -20,3 +20,4 @@ - {role: nextcloud, tags: nextcloud} - {role: syncthing, tags: syncthing} - {role: prometheus, tags: prometheus} + - {role: monitoring, tags: monitoring} diff --git a/ansible/roles/monitoring/docker-stack.yml.j2 b/ansible/roles/monitoring/docker-stack.yml.j2 new file mode 100644 index 0000000..bf8f9db --- /dev/null +++ b/ansible/roles/monitoring/docker-stack.yml.j2 @@ -0,0 +1,98 @@ +# vi: ft=yaml +version: "3.8" + +networks: + traefik: + external: true + +configs: + esdatasource: + external: true + name: "{{ esdatasource.config_name }}" + +volumes: + escerts: + driver_opts: + type: "nfs" + o: "addr=192.168.30.10,nolock,soft,rw" + device: ":/mnt/data/elasticsearch/certs" + esdata: + driver_opts: + type: "nfs" + o: "addr=192.168.30.10,nolock,soft,rw" + device: ":/mnt/data/elasticsearch/data" + grafanadata: + driver_opts: + type: "nfs" + o: "addr=192.168.30.10,nolock,soft,rw" + device: ":/mnt/data/grafana/data" + +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.8.1 + volumes: + - type: volume + source: escerts + target: /usr/share/elasticsearch/config/certs + volume: + nocopy: true + - type: volume + source: esdata + target: /usr/share/elasticsearch/data + volume: + nocopy: true + ports: + - {{ elasticsearch_port }}:9200 + environment: + - node.name=es01 + - cluster.name=shoarma + - cluster.initial_master_nodes=es01 + - bootstrap.memory_lock=true + - xpack.security.enabled=false + - xpack.security.http.ssl.enabled=false + - xpack.security.http.ssl.key=certs/es01/es01.key + - xpack.security.http.ssl.certificate=certs/es01/es01.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.enabled=false + - xpack.security.transport.ssl.key=certs/es01/es01.key + - xpack.security.transport.ssl.certificate=certs/es01/es01.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=basic + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl http://localhost:9200 | grep -q 'You Know, for Search'", + ] + interval: 10s + timeout: 10s + retries: 120 + + grafana: + image: grafana/grafana-oss + networks: + - traefik + deploy: + labels: + - traefik.enable=true + - traefik.http.routers.grafana.entrypoints=localsecure + - traefik.http.routers.grafana.rule=Host(`grafana.kun.is`) + - traefik.http.routers.grafana.tls=true + - traefik.http.routers.grafana.tls.certresolver=letsencrypt + - traefik.http.routers.grafana.service=grafana + - traefik.http.services.grafana.loadbalancer.server.port=3000 + - traefik.docker.network=traefik + volumes: + - type: volume + source: grafanadata + target: /var/lib/grafana + volume: + nocopy: true + configs: + - source: esdatasource + target: /etc/grafana/provisioning/datasources/elasticsearch.yaml diff --git a/ansible/roles/monitoring/elasticsearch.yml b/ansible/roles/monitoring/elasticsearch.yml new file mode 100644 index 0000000..a898f3e --- /dev/null +++ b/ansible/roles/monitoring/elasticsearch.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +datasources: + - name: Elasticsearch + type: elasticsearch + access: proxy + url: http://maestro.dmz:14653 + jsonData: + # index: '[metrics-]YYYY.MM.DD' + interval: Daily + timeField: '@timestamp' + diff --git a/ansible/roles/monitoring/tasks/main.yml b/ansible/roles/monitoring/tasks/main.yml new file mode 100644 index 0000000..c5958c3 --- /dev/null +++ b/ansible/roles/monitoring/tasks/main.yml @@ -0,0 +1,13 @@ +- name: Create elasticsearch data source config + docker_config: + name: esdatasource + data: "{{ lookup('template', '{{ role_path }}/elasticsearch.yml') }}" + use_ssh_client: true + rolling_versions: true + register: esdatasource + +- name: Deploy Docker stack + docker_stack: + name: monitoring + compose: + - "{{ lookup('template', '{{ role_path }}/docker-stack.yml.j2') | from_yaml }}"