From 83d151a51ce761034d2997f398ce11a47063d991 Mon Sep 17 00:00:00 2001 From: kev Date: Sat, 6 Jan 2018 21:38:57 +0800 Subject: [PATCH] update elk --- elk/conf/01-lumberjack-input.conf | 6 --- elk/conf/02-beats-input.conf | 6 --- elk/conf/10-syslog.conf | 13 ------ elk/conf/11-nginx.conf | 7 --- elk/conf/30-output.conf | 9 ---- elk/data/logstash.yml | 6 +++ elk/data/pipeline/logstash.conf | 14 ++++++ elk/docker-compose.yml | 75 +++++++++++++++++-------------- elk/elasticsearch.yml | 2 - 9 files changed, 62 insertions(+), 76 deletions(-) delete mode 100644 elk/conf/01-lumberjack-input.conf delete mode 100644 elk/conf/02-beats-input.conf delete mode 100644 elk/conf/10-syslog.conf delete mode 100644 elk/conf/11-nginx.conf delete mode 100644 elk/conf/30-output.conf create mode 100644 elk/data/logstash.yml create mode 100644 elk/data/pipeline/logstash.conf delete mode 100644 elk/elasticsearch.yml diff --git a/elk/conf/01-lumberjack-input.conf b/elk/conf/01-lumberjack-input.conf deleted file mode 100644 index e4da550..0000000 --- a/elk/conf/01-lumberjack-input.conf +++ /dev/null @@ -1,6 +0,0 @@ -input { - lumberjack { - port => 5000 - type => "logs" - } -} diff --git a/elk/conf/02-beats-input.conf b/elk/conf/02-beats-input.conf deleted file mode 100644 index a00d3f5..0000000 --- a/elk/conf/02-beats-input.conf +++ /dev/null @@ -1,6 +0,0 @@ -input { - beats { - port => 5044 - ssl => false - } -} diff --git a/elk/conf/10-syslog.conf b/elk/conf/10-syslog.conf deleted file mode 100644 index acce463..0000000 --- a/elk/conf/10-syslog.conf +++ /dev/null @@ -1,13 +0,0 @@ -filter { - if [type] == "syslog" { - grok { - match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } - add_field => [ "received_at", "%{@timestamp}" ] - add_field => [ "received_from", "%{host}" ] - } - syslog_pri { } - date { - match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] - } - } -} diff --git a/elk/conf/11-nginx.conf b/elk/conf/11-nginx.conf deleted file mode 100644 index d4a45db..0000000 --- a/elk/conf/11-nginx.conf +++ /dev/null @@ -1,7 +0,0 @@ -filter { - if [type] == "nginx-access" { - grok { - match => { "message" => "%{NGINXACCESS}" } - } - } -} diff --git a/elk/conf/30-output.conf b/elk/conf/30-output.conf deleted file mode 100644 index 07c9faf..0000000 --- a/elk/conf/30-output.conf +++ /dev/null @@ -1,9 +0,0 @@ -output { - elasticsearch { - hosts => ["elasticsearch"] - sniffing => true - manage_template => false - index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" - document_type => "%{[@metadata][type]}" - } -} diff --git a/elk/data/logstash.yml b/elk/data/logstash.yml new file mode 100644 index 0000000..9bc7d05 --- /dev/null +++ b/elk/data/logstash.yml @@ -0,0 +1,6 @@ +http.host: "0.0.0.0" +path.config: /usr/share/logstash/pipeline +#xpack.monitoring.enabled: false +xpack.monitoring.elasticsearch.url: http://elasticsearch:9200 +#xpack.monitoring.elasticsearch.username: logstash_system +#xpack.monitoring.elasticsearch.password: changeme diff --git a/elk/data/pipeline/logstash.conf b/elk/data/pipeline/logstash.conf new file mode 100644 index 0000000..e5e15c4 --- /dev/null +++ b/elk/data/pipeline/logstash.conf @@ -0,0 +1,14 @@ +input { + beats { + port => 5044 + } +} + +output { + stdout { + codec => rubydebug + } + elasticsearch { + hosts => [ "elasticsearch:9200" ] + } +} diff --git a/elk/docker-compose.yml b/elk/docker-compose.yml index 56eaae5..2ce4fee 100644 --- a/elk/docker-compose.yml +++ b/elk/docker-compose.yml @@ -1,33 +1,42 @@ -version: '2' -services: - elasticsearch: - image: elasticsearch:5.0 - ports: - - '9200:9200' - - '9300:9300' - volumes: - - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml - - ./data:/usr/share/elasticsearch/data - mem_limit: 4g - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - cap_add: - - IPC_LOCK - logstash: - image: logstash:5.0 - ports: - - '5000:5000' - - '5044:5044' - volumes: - - ./conf:/etc/logstash/conf.d - kibana: - image: kibana:5.0 - ports: - - '5601:5601' - environment: - - ELASTICSEARCH_URL=http://elasticsearch:9200 +elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:6.1.1 + ports: + - "9200:9200" + volumes: + - ./data:/usr/share/elasticsearch/data + environment: + - node_name=elastic-search + - cluster.name=docker-cluster + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + ulimits: + memlock: + soft: -1 + hard: -1 + restart: unless-stopped + +logstash: + image: docker.elastic.co/logstash/logstash:6.1.1 + ports: + - "5044:5044" + - "9600:9600" + links: + - elasticsearch + volumes: + - ./data/logstash.yml:/usr/share/logstash/config/logstash.yml + - ./data/pipeline:/usr/share/logstash/pipeline + - /usr/share/logstash/vendor/bundle + environment: + LS_JAVA_OPTS: "-Xms1g -Xmx1g" + restart: unless-stopped + +kibana: + image: docker.elastic.co/kibana/kibana:6.1.1 + ports: + - "5601:5601" + links: + - elasticsearch + environment: + SERVER_NAME: kibana + ELASTICSEARCH_URL: http://elasticsearch:9200 + restart: unless-stopped diff --git a/elk/elasticsearch.yml b/elk/elasticsearch.yml deleted file mode 100644 index 4e561a2..0000000 --- a/elk/elasticsearch.yml +++ /dev/null @@ -1,2 +0,0 @@ -network.host: 0.0.0.0 -bootstrap.mlockall: true