create ES index every day

ansible/roles/monitoring/elasticsearch.yml.j2

@@ -7,7 +7,7 @@ datasources:
     access: proxy
     url: http://maestro.dmz:{{ elasticsearch_port }}
     jsonData:
-      index: 'fluentd.cpu'
+      index: 'fluentd.cpu.*'
       timeField: '@timestamp'
 
   - name: memory
@@ -15,5 +15,5 @@ datasources:
     access: proxy
     url: http://maestro.dmz:{{ elasticsearch_port }}
     jsonData:
-      index: 'fluentd.memory'
+      index: 'fluentd.memory.*'
       timeField: '@timestamp'

ansible/roles/monitoring/fluent.conf.j2

@@ -11,7 +11,12 @@
   host maestro.dmz
   port {{ elasticsearch_port }}
   include_timestamp true
-  index_name fluentd.${tag}
+  index_name fluentd.${tag}.%Y%m%d
+  <buffer tag, time>
+    timekey 1d
+    flush_mode interval
+    flush_interval 1s
+  </buffer>
 </match>
 
 <system>

terraform/elasticsearch/main.tf

@@ -0,0 +1,40 @@
+terraform {
+  backend "pg" {
+    schema_name = "shoarma-elasticsearch"
+  }
+
+  required_providers {
+    elasticstack = {
+      source = "elastic/elasticstack"
+      version = "0.6.2"
+    }
+  }
+}
+
+provider "elasticstack" {
+  elasticsearch {
+    endpoints = ["http://maestro.dmz:14653"]
+  }
+}
+
+resource "elasticstack_elasticsearch_index_lifecycle" "metrics_ilm" {
+  name = "metrics_ilm"
+
+  delete {
+    min_age = "7d"
+    delete {}
+  }
+}
+
+resource "elasticstack_elasticsearch_index_template" "metrics_template" {
+  name = "metrics_template"
+
+  priority = 42
+  index_patterns = ["fluentd.cpu*", "fluentd.memory*"]
+
+  template {
+    settings = jsonencode({
+      "index.lifecycle.name" = elasticstack_elasticsearch_index_lifecycle.metrics_ilm.name
+    })
+  }
+}