From b53f52efbbcbe9655f5113d8e8f5249739719b99 Mon Sep 17 00:00:00 2001
From: kev <vimagick@gmail.com>
Date: Tue, 5 Nov 2019 02:09:26 +0800
Subject: [PATCH] update elk

---
 elk/README.md                                 | 12 +++++++
 elk/data/logstash.yml                         |  7 ----
 elk/data/pipelines.yml                        |  6 ++++
 .../main}/logstash.conf                       |  5 ++-
 elk/docker-compose.yml                        | 34 ++++++++++---------
 5 files changed, 40 insertions(+), 24 deletions(-)
 delete mode 100644 elk/data/logstash.yml
 create mode 100644 elk/data/pipelines.yml
 rename elk/data/{pipeline => pipelines/main}/logstash.conf (66%)

diff --git a/elk/README.md b/elk/README.md
index 06dbb49..3ce22de 100644
--- a/elk/README.md
+++ b/elk/README.md
@@ -20,6 +20,18 @@ $ chown -R 1000:1000 data
 $ docker-compose up -d
 ```
 
+## Docker Config
+
+- https://github.com/elastic/dockerfiles
+- https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html
+- https://www.elastic.co/guide/en/logstash/current/docker-config.html
+- https://www.elastic.co/guide/en/kibana/current/docker.html
+
+## Logstash Config
+
+- https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html
+- https://www.elastic.co/guide/en/logstash/current/event-dependent-configuration.html
+
 ## Delete indices older than 7 days
 
 File: delete-indices.yml
diff --git a/elk/data/logstash.yml b/elk/data/logstash.yml
deleted file mode 100644
index e9c1daa..0000000
--- a/elk/data/logstash.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-http.host: "0.0.0.0"
-path.config: /usr/share/logstash/pipeline
-#xpack.monitoring.enabled: true
-#xpack.monitoring.elasticsearch.hosts:
-#- http://elasticsearch:9200
-#xpack.monitoring.elasticsearch.username: logstash_system
-#xpack.monitoring.elasticsearch.password: changeme
diff --git a/elk/data/pipelines.yml b/elk/data/pipelines.yml
new file mode 100644
index 0000000..dc26b1a
--- /dev/null
+++ b/elk/data/pipelines.yml
@@ -0,0 +1,6 @@
+#
+# https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html
+#
+
+- pipeline.id: main
+  path.config: "/usr/share/logstash/pipelines/main"
diff --git a/elk/data/pipeline/logstash.conf b/elk/data/pipelines/main/logstash.conf
similarity index 66%
rename from elk/data/pipeline/logstash.conf
rename to elk/data/pipelines/main/logstash.conf
index dc8fd83..d221018 100644
--- a/elk/data/pipeline/logstash.conf
+++ b/elk/data/pipelines/main/logstash.conf
@@ -9,9 +9,12 @@ input {
 
 output {
   stdout {
-    codec => rubydebug
+    codec => rubydebug {
+      metadata => true
+    }
   }
   elasticsearch {
     hosts => [ "elasticsearch:9200" ]
+    ilm_enabled => false
   }
 }
diff --git a/elk/docker-compose.yml b/elk/docker-compose.yml
index f51f17c..a63365e 100644
--- a/elk/docker-compose.yml
+++ b/elk/docker-compose.yml
@@ -1,9 +1,9 @@
-version: '3.5'
+version: '3.7'
 
 services:
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.3.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.4.2
     ports:
       - "9200:9200"
     volumes:
@@ -13,7 +13,7 @@ services:
       - cluster.name=docker-cluster
       - cluster.initial_master_nodes=es1
       - bootstrap.memory_lock=true
-      - "ES_JAVA_OPTS=-Xms16g -Xmx16g"
+      - ES_JAVA_OPTS=-Xms8g -Xmx8g
     ulimits:
       memlock:
         soft: -1
@@ -21,38 +21,40 @@ services:
     restart: unless-stopped
 
   logstash:
-    image: docker.elastic.co/logstash/logstash:7.3.0
+    image: docker.elastic.co/logstash/logstash:7.4.2
     ports:
       - "5044:5044"
       - "9600:9600"
       - "12201:12201/udp"
-    depends_on:
-      - elasticsearch
     volumes:
-      - ./data/logstash.yml:/usr/share/logstash/config/logstash.yml
-      - ./data/pipeline:/usr/share/logstash/pipeline
+      - ./data/pipelines.yml:/usr/share/logstash/config/pipelines.yml
+      - ./data/pipelines:/usr/share/logstash/pipelines
     environment:
       - node.name=ls1
-      - "LS_JAVA_OPTS=-Xms4g -Xmx4g"
+      - xpack.monitoring.enabled=true
+      - xpack.monitoring.elasticsearch_hosts=[http://elasticsearch:9200]
+      - LS_JAVA_OPTS=-Xms4g -Xmx4g
+    depends_on:
+      - elasticsearch
     restart: unless-stopped
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:7.3.0
+    image: docker.elastic.co/kibana/kibana:7.4.2
     ports:
       - "5601:5601"
+    environment:
+      - SERVER_NAME=kibana
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
     depends_on:
       - elasticsearch
-    environment:
-      SERVER_NAME: kibana
-      ELASTICSEARCH_URL: http://elasticsearch:9200
     restart: unless-stopped
 
   apm-server:
-    image: docker.elastic.co/apm/apm-server:7.3.0
+    image: docker.elastic.co/apm/apm-server:7.4.2
     ports:
       - "8200:8200"
-    depends_on:
-      - elasticsearch
     volumes:
       - ./data/apm-server.yml:/usr/share/apm-server/apm-server.yml
+    depends_on:
+      - elasticsearch
     restart: unless-stopped