ELK (draft)

# apt-get install default-jre
# update-alternatives --config java
There is only one alternative in link group java (providing /usr/bin/java): /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
Nothing to configure.
# cat /etc/environment  | grep JAVA
JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java"
# source /etc/environment
# echo $JAVA_HOME

wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
sudo apt-get install apt-transport-https
echo "deb https://artifacts.elastic.co/packages/5.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-5.x.list

root@ubuntu-xenial:/etc/elasticsearch# ls /etc/elasticsearch/elasticsearch.yml
/etc/elasticsearch/elasticsearch.yml
root@ubuntu-xenial:/etc/elasticsearch# diff elasticsearch.yml.orig elasticsearch.yml
55a56
> network.host: 0.0.0.0

# Cluster Health
$ curl -XGET 'localhost:9200/_cat/health?v&pretty'
# 노드 상태 확인하기
$ curl -XGET "http://localhost:9200/_cat/nodes?v&h=id,name,ip,port,v,m&s=name"

root@ubuntu-xenial:/etc/logstash# cat /etc/environment | grep JAVACMD
JAVACMD=`which java`

/usr/share/logstash/bin/logstash -e 'input { stdin { } } output { stdout {} }'

root@ubuntu-xenial:/etc/kibana# ls /etc/kibana/kibana.yml
/etc/kibana/kibana.yml
root@ubuntu-xenial:/etc/kibana# diff kibana.yml.orig kibana.yml
7a8
> server.host: "0.0.0.0"

curl -XGET 'localhost:9200/_cat/indices?v&pretty'

sudo apt-get update && sudo apt-get install filebeat
sudo systemctl enable filebeat

$ cat /etc/filebeat/filebeat.yml  | egrep -v '(^\s)*#|^$'
filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/*.log
output.elasticsearch:
  hosts: ["localhost:9200"]

# cat filebeat.yml | egrep -v '(^\s)*#|^$'
filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/*.log
output.logstash:
  hosts: ["localhost:5044"]

# cat /etc/logstash/conf.d/logstash.conf
input {
  beats {
    port => 5044
  }
}

# The filter part of this file is commented out to indicate that it is
# optional.
# filter {
#
# }

output {
  elasticsearch {
    hosts => "localhost:9200"
    manage_template => false
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
    document_type => "%{[@metadata][type]}"
  }
}
# /usr/share/logstash/bin/logstash-plugin update logstash-input-beats 

# systemctl stop filebeat
# systemctl stop logstash

# cat /etc/filebeat/filebeat.test.yml
filebeat.prospectors:
- input_type: log
  paths:
    - /var/log/nginx/access.log
  fields:
    log_type: nginx
    logsource: taejoon_desktop
- input_type: log
  paths:
    - /var/log/*.log
  fields:
    log_type: syslog

output.logstash:
  hosts: ["localhost:5044"]
# filebeat.sh -e -c filebeat.test.yml -d "publish"

# cat /etc/logstash/grok/nginx
NGINXACCESS %{IPORHOST:remote_addr} - %{USERNAME:remote_user} \[%{HTTPDATE:time_local}\] \"%{DATA:request}\" %{INT:status} %{NUMBER:bytes_sent} \"%{DATA:http_referer}\" \"%{DATA:http_user_agent}\"

# cat /etc/logstash/conf.d/test.conf
input {
  beats {
    port => 5044
  }
}

filter {
   if [fields][log_type] == "nginx" {
    grok {
#        match => { "message" => "%{IPORHOST:remote_addr} - %{USERNAME:remote_user} \[%{HTTPDATE:time_local}\] \"%{DATA:request}\" %{INT:status} %{NUMBER:bytes_sent} \"%{DATA:http_referer}\" \"%{DATA:http_user_agent}\""}
        match => { "message" => "%{NGINXACCESS}"}
        patterns_dir => ["/etc/logstash/grok"]
    }
   }
}

output {
   if [fields][log_type] == "nginx" {
        stdout { codec => rubydebug }
        elasticsearch {
            hosts => [ "localhost:9200" ]
            index    => "nginx-%{+YYYY.MM.dd}"
        }
    }

   if [fields][log_type] == "syslog" {
        stdout { codec => rubydebug }
        elasticsearch {
            hosts => [ "localhost:9200" ]
            index    => "syslog-%{+YYYY.MM.dd}"
        }
    }
}

# /usr/share/logstash/bin/logstash -f test.conf --config.test_and_exit
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path //usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
Configuration OK

# /usr/share/logstash/bin/logstash -f test.conf --config.reload.automatic