update elk

elk/conf/01-lumberjack-input.conf

@@ -1,6 +0,0 @@
-input {
-  lumberjack {
-    port => 5000
-    type => "logs"
-  }
-}

elk/conf/02-beats-input.conf

@@ -1,6 +0,0 @@
-input {
-  beats {
-    port => 5044
-    ssl => false
-  }
-}

elk/conf/10-syslog.conf

@@ -1,13 +0,0 @@
-filter {
-  if [type] == "syslog" {
-    grok {
-      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
-      add_field => [ "received_at", "%{@timestamp}" ]
-      add_field => [ "received_from", "%{host}" ]
-    }
-    syslog_pri { }
-    date {
-      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
-    }
-  }
-}

elk/conf/11-nginx.conf

@@ -1,7 +0,0 @@
-filter {
-  if [type] == "nginx-access" {
-    grok {
-      match => { "message" => "%{NGINXACCESS}" }
-    }
-  }
-}

elk/conf/30-output.conf

@@ -1,9 +0,0 @@
-output {
-  elasticsearch {
-    hosts => ["elasticsearch"]
-    sniffing => true
-    manage_template => false
-    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
-    document_type => "%{[@metadata][type]}"
-  }
-}

elk/data/logstash.yml

@@ -0,0 +1,6 @@
+http.host: "0.0.0.0"
+path.config: /usr/share/logstash/pipeline
+#xpack.monitoring.enabled: false
+xpack.monitoring.elasticsearch.url: http://elasticsearch:9200
+#xpack.monitoring.elasticsearch.username: logstash_system
+#xpack.monitoring.elasticsearch.password: changeme

elk/data/pipeline/logstash.conf

@@ -0,0 +1,14 @@
+input {
+  beats {
+    port => 5044
+  }
+}
+
+output {
+  stdout {
+    codec => rubydebug
+  }
+  elasticsearch {
+    hosts => [ "elasticsearch:9200" ]
+  }
+}

elk/docker-compose.yml

@@ -1,33 +1,42 @@
-version: '2'
+elasticsearch:
-services:
+  image: docker.elastic.co/elasticsearch/elasticsearch:6.1.1
-  elasticsearch:
+  ports:
-    image: elasticsearch:5.0
+    - "9200:9200"
-    ports:
+  volumes:
-      - '9200:9200'
+    - ./data:/usr/share/elasticsearch/data
-      - '9300:9300'
+  environment:
-    volumes:
+    - node_name=elastic-search
-      - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+    - cluster.name=docker-cluster
-      - ./data:/usr/share/elasticsearch/data
+    - bootstrap.memory_lock=true
-    mem_limit: 4g
+    - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
-    ulimits:
+  ulimits:
-      memlock:
+    memlock:
-        soft: -1
+      soft: -1
-        hard: -1
+      hard: -1
-      nofile:
+  restart: unless-stopped
-        soft: 65536
+
-        hard: 65536
+logstash:
-    cap_add:
+  image: docker.elastic.co/logstash/logstash:6.1.1
-      - IPC_LOCK
+  ports:
-  logstash:
+    - "5044:5044"
-    image: logstash:5.0
+    - "9600:9600"
-    ports:
+  links:
-      - '5000:5000'
+    - elasticsearch
-      - '5044:5044'
+  volumes:
-    volumes:
+    - ./data/logstash.yml:/usr/share/logstash/config/logstash.yml
-      - ./conf:/etc/logstash/conf.d
+    - ./data/pipeline:/usr/share/logstash/pipeline
-  kibana:
+    - /usr/share/logstash/vendor/bundle
-    image: kibana:5.0
+  environment:
-    ports:
+    LS_JAVA_OPTS: "-Xms1g -Xmx1g"
-      - '5601:5601'
+  restart: unless-stopped
-    environment:
+
-      - ELASTICSEARCH_URL=http://elasticsearch:9200
+kibana:
+  image: docker.elastic.co/kibana/kibana:6.1.1
+  ports:
+    - "5601:5601"
+  links:
+    - elasticsearch
+  environment:
+    SERVER_NAME: kibana
+    ELASTICSEARCH_URL: http://elasticsearch:9200
+  restart: unless-stopped

elk/elasticsearch.yml

@@ -1,2 +0,0 @@
-network.host: 0.0.0.0
-bootstrap.mlockall: true