From db38d9c6bb27104db3a943422ec3f25eeb4a0830 Mon Sep 17 00:00:00 2001
From: Pim Kunis <pim@kunis.nl>
Date: Tue, 22 Aug 2023 22:53:40 +0200
Subject: [PATCH] collect docker logs using fluent

---
 ansible/playbooks/setup.yml                  | 12 ++++++++++-
 ansible/roles/monitoring/docker-stack.yml.j2 |  2 +-
 ansible/roles/monitoring/fluent.conf.j2      |  2 +-
 terraform/elasticsearch/main.tf              | 22 ++++++++++++++++++++
 4 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/ansible/playbooks/setup.yml b/ansible/playbooks/setup.yml
index 7b06092..f6130d3 100644
--- a/ansible/playbooks/setup.yml
+++ b/ansible/playbooks/setup.yml
@@ -17,7 +17,17 @@
 
   roles:
     - setup_apt
-    - docker
+
+  post_tasks:
+    - name: Install Docker
+      include_role:
+        name: docker
+      vars:
+        docker_daemon_config:
+          log-driver: fluentd
+          log-opts:
+            fluentd-address: "localhost:22222"
+            tag: "docker.{{ '{{' }}.Name{{ '}}' }}"
 
 - name: Setup Docker Swarm manager
   hosts: manager
diff --git a/ansible/roles/monitoring/docker-stack.yml.j2 b/ansible/roles/monitoring/docker-stack.yml.j2
index 04730f8..9a61c12 100644
--- a/ansible/roles/monitoring/docker-stack.yml.j2
+++ b/ansible/roles/monitoring/docker-stack.yml.j2
@@ -122,7 +122,7 @@ services:
       - BAUTH_PASS=test
 
   fluentd:
-    image: git.kun.is/pim/fluentd:1.0.1
+    image: git.kun.is/pim/fluentd:1.0.2
     depends_on:
       - elasticsearch
     ports:
diff --git a/ansible/roles/monitoring/fluent.conf.j2 b/ansible/roles/monitoring/fluent.conf.j2
index 61b52b3..63d2f5a 100644
--- a/ansible/roles/monitoring/fluent.conf.j2
+++ b/ansible/roles/monitoring/fluent.conf.j2
@@ -6,7 +6,7 @@
   port {{ fluent_forward_port }}
 </source>
 
-<match *>
+<match **>
   @type elasticsearch
   host maestro.dmz
   port {{ elasticsearch_port }}
diff --git a/terraform/elasticsearch/main.tf b/terraform/elasticsearch/main.tf
index 818dba0..b6d0e4f 100644
--- a/terraform/elasticsearch/main.tf
+++ b/terraform/elasticsearch/main.tf
@@ -38,3 +38,25 @@ resource "elasticstack_elasticsearch_index_template" "metrics_template" {
     })
   }
 }
+
+resource "elasticstack_elasticsearch_index_lifecycle" "logs_ilm" {
+  name = "logs_ilm"
+
+  delete {
+    min_age = "2d"
+    delete {}
+  }
+}
+
+resource "elasticstack_elasticsearch_index_template" "logs_template" {
+  name = "logs_template"
+
+  priority = 42
+  index_patterns = ["fluentd.docker.**"]
+
+  template {
+    settings = jsonencode({
+      "index.lifecycle.name" = elasticstack_elasticsearch_index_lifecycle.logs_ilm.name
+    })
+  }
+}