From 3bd949af5def0d90b8dd7879b66f0b02b5f23ce6 Mon Sep 17 00:00:00 2001
From: kev <vimagick@gmail.com>
Date: Tue, 26 Dec 2017 21:22:11 +0800
Subject: [PATCH] add logstash

---
 README.md                            |  1 +
 logstash/README.md                   | 81 ++++++++++++++++++++++++++++
 logstash/data/logstash.yml           |  6 +++
 logstash/data/pipeline/logstash.conf | 21 ++++++++
 logstash/docker-compose.yml          | 11 ++++
 5 files changed, 120 insertions(+)
 create mode 100644 logstash/README.md
 create mode 100644 logstash/data/logstash.yml
 create mode 100644 logstash/data/pipeline/logstash.conf
 create mode 100644 logstash/docker-compose.yml

diff --git a/README.md b/README.md
index 4f3d89b..9200331 100644
--- a/README.md
+++ b/README.md
@@ -75,6 +75,7 @@ A collection of delicious docker recipes.
 - [x] h2o
 - [x] httpbin :+1:
 - [x] influxdb
+- [x] logstash
 - [x] luigi
 - [x] mariadb
 - [x] mariadb-arm
diff --git a/logstash/README.md b/logstash/README.md
new file mode 100644
index 0000000..9f2a363
--- /dev/null
+++ b/logstash/README.md
@@ -0,0 +1,81 @@
+logstash
+========
+
+## How It Works
+
+![](https://www.elastic.co/assets/blt203883a0718cdc5a/filebeat-diagram.png)
+
+```
+log files ---> filebeat agent --+
+...                             |
+log files ---> filebeat agent --+> logstash container ---> aliyun™ log service
+...                             |
+log files ---> filebeat agent --+
+```
+
+## docker-compose.yml
+
+```yaml
+logstash:
+  image: docker.elastic.co/logstash/logstash:6.1.1
+  ports:
+    - "5044:5044"
+    - "9600:9600"
+  volumes:
+    - ./data/logstash.yml:/usr/share/logstash/config/logstash.yml
+    - ./data/pipeline:/usr/share/logstash/pipeline
+  environment:
+    LS_JAVA_OPTS: "-Xms1g -Xmx1g"
+  restart: always
+```
+
+## Up and Running
+
+```bash
+$ docker-compose up -d
+$ docker-compose exec logstash bash
+>>> logstash-plugin install logstash-output-logservice
+Validating logstash-output-logservice
+Installing logstash-output-logservice
+Installation successful
+>>> logstash-plugin list
+logstash-output-logservice
+>>> exit
+$ vim data/pipeline/logstash.conf
+$ docker-compose restart
+$ docker-compose logs -f
+$ curl http://localhost:9600
+{
+  "host": "easypi",
+  "version": "6.1.1",
+  "http_address": "0.0.0.0:9600",
+  "id": "c7c4f9d7-5621-4375-bfc9-96abb0f1b4c3",
+  "name": "6848fe4c533f",
+  "build_date": "2017-12-17T21:51:17+00:00",
+  "build_sha": "d46ca0de31662d29b8c5c94d4162e4c760d3f8fb",
+  "build_snapshot": false
+}
+```
+
+## Setup Filebeat
+
+```bash
+# https://www.elastic.co/downloads/beats/filebeat
+$ wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.1.1-amd64.deb
+$ dpkg -i filebeat-6.1.1-amd64.deb
+$ vim /etc/filebeat/filebeat.yml
+#output.elasticsearch:
+  # Array of hosts to connect to.
+  #hosts: ["localhost:9200"]
+output.logstash:
+  # The Logstash hosts
+  hosts: ["1.2.3.4:5044"]
+$ systemctl start filebeat
+$ systemctl enable filebeat
+```
+
+## References
+
+- https://www.elastic.co/guide/en/logstash/current/getting-started-with-logstash.html
+- https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-getting-started.html
+- https://www.alibabacloud.com/help/zh/doc-detail/28984.htm
diff --git a/logstash/data/logstash.yml b/logstash/data/logstash.yml
new file mode 100644
index 0000000..26372e3
--- /dev/null
+++ b/logstash/data/logstash.yml
@@ -0,0 +1,6 @@
+http.host: "0.0.0.0"
+path.config: /usr/share/logstash/pipeline
+xpack.monitoring.enabled: false
+#xpack.monitoring.elasticsearch.url: http://elasticsearch:9200
+#xpack.monitoring.elasticsearch.username: logstash_system
+#xpack.monitoring.elasticsearch.password: changeme
diff --git a/logstash/data/pipeline/logstash.conf b/logstash/data/pipeline/logstash.conf
new file mode 100644
index 0000000..d5a020d
--- /dev/null
+++ b/logstash/data/pipeline/logstash.conf
@@ -0,0 +1,21 @@
+input {
+  beats {
+    port => 5044
+  }
+}
+
+output {
+  stdout {
+    codec => rubydebug
+  }
+# logservice {
+#   endpoint => "cn-shanghai.log.aliyuncs.com"
+#   project => "logging"
+#   logstore => "logstore"
+#   source => "default"
+#   topic => "default"
+#   access_key_id => "XXXXXXXXXXXXXXXX"
+#   access_key_secret => "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+#   max_send_retry => 3
+# }
+}
diff --git a/logstash/docker-compose.yml b/logstash/docker-compose.yml
new file mode 100644
index 0000000..5feca43
--- /dev/null
+++ b/logstash/docker-compose.yml
@@ -0,0 +1,11 @@
+logstash:
+  image: docker.elastic.co/logstash/logstash:6.1.1
+  ports:
+    - "5044:5044"
+    - "9600:9600"
+  volumes:
+    - ./data/logstash.yml:/usr/share/logstash/config/logstash.yml
+    - ./data/pipeline:/usr/share/logstash/pipeline
+  environment:
+    LS_JAVA_OPTS: "-Xms1g -Xmx1g"
+  restart: always