从0搭建elk日志分析平台

字数统计: 3k字   |   阅读时长≈ 16分

为什么需要搭建日志分析平台

1、由于Linux服务器并不是谁都可以上的，并且查看日志需要tail -f xxx.log非常麻烦，有时候还需要人家进行复现
2、保证其他开发人员能快速分析排错
3、保证服务器只有部分人可以进入(防小人)

准备工作,logback 配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
<?xml version="1.0" encoding="UTF-8" ?>
<configuration>
    <springProperty scope="context" name="logDir" source="app.log.dir"/>
    <springProperty scope="context" name="logMaxHistory" source="app.log.maxHistory"/>
    <springProperty scope="context" name="logLevel" source="app.log.level"/>

    <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
    <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
    <conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />

    <conversionRule conversionWord="ex" converterClass="com.globalmodule.logback.StackTraceConverter" />
    <conversionRule conversionWord="msgext" converterClass="com.globalmodule.logback.MsgConverter" />

    <appender name="INFO_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">

        <file>${logDir}/info.log</file>
        <encoder>
            <pattern>{ "date":"%date", "level":"%level", "logger":"%logger{10}", "file_line":"%file:%line", "msg":%msgext, "exception":%ex }%n%n</pattern>
            <charset>UTF-8</charset>
        </encoder>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${logDir}/history/info-%d{yyyy-MM-dd}.txt</fileNamePattern>
            <maxHistory>${logMaxHistory}</maxHistory>
        </rollingPolicy>

        <filter class="com.globalmodule.logback.LogbackLevelFilter">
            <level>DEBUG</level>
            <level>INFO</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
    </appender>

    <appender name="SMS_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">

        <file>${logDir}/sms.log</file>
        <encoder>
            <pattern>{ "date":"%date", "level":"%level", "logger":"%logger{10}", "file_line":"%file:%line", "msg":%msgext, "exception":%ex }%n%n</pattern>
            <charset>UTF-8</charset>
        </encoder>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${logDir}/history/sms-%d{yyyy-MM-dd}.txt</fileNamePattern>
            <maxHistory>${logMaxHistory}</maxHistory>
        </rollingPolicy>

        <filter class="com.globalmodule.logback.LogbackLevelFilter">
            <level>DEBUG</level>
            <level>INFO</level>
            <level>ERROR</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
    </appender>

    <appender name="ERROR_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">

        <file>${logDir}/error.log</file>
        <encoder>
            <pattern>{ "date":"%date", "level":"%level", "logger":"%logger{10}", "file_line":"%file:%line", "msg":%msgext, "exception":%ex }%n%n</pattern>
            <charset>UTF-8</charset>
        </encoder>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${logDir}/history/error-%d{yyyy-MM-dd}.txt</fileNamePattern>
            <maxHistory>${logMaxHistory}</maxHistory>
        </rollingPolicy>
        <filter class="com.globalmodule.logback.LogbackLevelFilter">
            <level>ERROR</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
    </appender>

    <appender name="ACCESS_LOG" class="ch.qos.logback.core.rolling.RollingFileAppender">

        <file>${logDir}/access.log</file>
        <encoder>
            <pattern>{ "date":"%date", "level":"%level", "logger":"%logger{10}", "file_line":"%file:%line", "msg":%msgext, "exception":%ex }%n%n</pattern>
            <charset>UTF-8</charset>
        </encoder>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${logDir}/history/access-%d{yyyy-MM-dd}.txt</fileNamePattern>
            <maxHistory>${logMaxHistory}</maxHistory>
        </rollingPolicy>

        <filter class="com.globalmodule.logback.LogbackLevelFilter">
            <level>DEBUG</level>
            <level>INFO</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
    </appender>

    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>${CONSOLE_LOG_PATTERN:-%clr([%d{yyyy-MM-dd HH:mm:ss.SSS}]){faint} %clr(%-5p) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%20.20t]){faint} %clr(%-40.40logger{40}){cyan} : %m%n}</pattern>
            <charset>UTF-8</charset>
        </encoder>
    </appender>


    <logger name="com.hyfj.soft.springbootdemo.commons.SmsUtils">
        <appender-ref ref="SMS_FILE"/>
    </logger>

    <logger name="com.globalmodule.logback.log.AccessLog">
        <appender-ref ref="ACCESS_LOG"/>
    </logger>

    <logger name="com.leyongleshi.">
        <appender-ref ref="INFO_FILE"/>
    </logger>

    <root level="${logLevel}">
        <appender-ref ref="ERROR_FILE"/>

        <!--虚拟机参数设置-Dlogback.env=local 运行就会在本地控制台打印日志-->
        <!--<if condition='property("logback.env").contains("local")'>-->
            <!--<then>-->
                <appender-ref ref="STDOUT"/>
            <!--</then>-->
        <!--</if>-->
    </root>

</configuration>

2、核心配置参数:

1
2
3
4
5
6
7
8
<encoder>
           <pattern>{ "date":"%date", "level":"%level", "logger":"%logger{10}", "file_line":"%file:%line", "msg":%msgext, "exception":%ex }%n%n</pattern>
           <charset>UTF-8</charset>
       </encoder>
       <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
           <fileNamePattern>${logDir}/history/access-%d{yyyy-MM-dd}.txt</fileNamePattern>
           <maxHistory>${logMaxHistory}</maxHistory>
       </rollingPolicy>

这里是把日志转json,日志:

1
{ "date":"2020-05-07 12:03:02,784", "level":"INFO", "logger":"c.g.l.l.AccessLog", "file_line":"BaseAuthAndLoginAop.java:71", "msg":{"appName":"hyfj","request":{"headers":{"Origin":"http://localhost:8083","Accept":"*/*","Connection":"keep-alive","User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36","Referer":"http://localhost:8083/swagger-ui.html","Host":"localhost:8083","Accept-Encoding":"gzip, deflate, br","Accept-Language":"zh-CN,zh;q=0.9,en;q=0.8","Content-Length":"0","Content-Type":"application/json"},"params":{"currentPage":["1"],"phone":["15196705987"],"password":["123"]},"queryString":"phone=15196705987&password=123&currentPage=1","clientIp":"192.168.2.60","uri":"http://localhost:8083/user/page","method":"POST","time":"2020-05-07 12:03:02","host":"localhost:8083"},"response":"{\"list\":[],\"pagination\":{\"current\":1,\"pageSize\":20,\"total\":0}}","requestCurl":"","checkTokenTime":-1,"controllerTime":-1}, "exception":"" }

转json需要取继承两个类，并且去重写父类的方法

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
public class StackTraceConverter extends ThrowableProxyConverter {
    public StackTraceConverter() {
    }

    @Override
    public String convert(ILoggingEvent event) {
		//拿到异常信息
        IThrowableProxy tp = event.getThrowableProxy();
		
		//如果没有异信息
        if (tp == null) {
			//返回字符串  ： "\"\""
            return JsonUtils.serialize("");
        }
		//如果有异常信息
        String ex = super.convert(event);
		
		//json化
        return JsonUtils.serialize(ex);
    }
}

public class MsgConverter extends ClassicConverter {
    public MsgConverter() {
    }

    @Override
    public String convert(ILoggingEvent event) {
        String msg = event.getFormattedMessage();

        try {
            JsonUtils.deserialize(msg, Map.class);
            return msg;
        } catch (Exception e) {
            Map<String,String> map = new HashMap<>(1);
            map.put("value",msg);
            return JsonUtils.serialize(map);
        }

    }
}

开始搭建 docker

这里我使用的是centos7搭建的docker

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
移除旧版本：
sudo yum remove docker \
                  docker-client \
                  docker-client-latest \
                  docker-common \
                  docker-latest \
                  docker-latest-logrotate \
                  docker-logrotate \
                  docker-selinux \
                  docker-engine-selinux \
                  docker-engine
				 
安装必要的工具：
sudo yum install -y yum-utils device-mapper-persistent-data lvm2

添加软件源：
sudo yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

更新 yum 缓存：
sudo yum makecache fast

安装 Docker-ce：
sudo yum -y install docker-ce

国内配置镜像：
vim /etc/docker/daemon.json （这个文件也许不存在，新建一个即可）
把下面的镜像复制到daemon.json文件中
{
  "registry-mirrors": ["http://hub-mirror.c.163.com"]
}

启动 Docker 后台服务：
sudo systemctl start docker

测试运行 hello-world：
docker run hello-world

安装elk

拉取镜像
docker pull sebp/elk
创建es数据存放目录：
mkdir -p /var/data/elk
以sebp/elk镜像启动容器，并命名为：myelk
docker run -d -p 5044:5044 -p 5601:5601 -p 9200:9200 -p 9300:9300 -v /var/data/elk:/var/lib/elasticsearch --name=myelk sebp/elk
端口占用:
logstash，占用5044端口
kibana，占用5601端口
elasticsearch，占用9200和9300端口

docker基础语法:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
查看容器的标准输出：
docker attach --sig-proxy=false myelk

停止容器：
docker stop myelk

启动容器：
docker start myelk

查看运行中的容器：
docker ps

进入容器命令行：
docker exec -it myelk /bin/bash

进入容器后:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
cd /etc/logstash/conf.d/

vim 02-beats-input.conf    # 注释掉ssl ，这个根据情况选择，我这里选择不启用ssl

vim myfilter.conf 
输入如下内容并保持

filter {
  json {
    source => "message"
    #target => "doc"
	
	# message字段保留的就是整个json数据，如果把下面的语句生效，则流入es的数据中，没有message这个字段。具体效果可以自行做实验
    #remove_field => ["message"]
  }

  # 这里是在处理日期时间，因为日志本身有时间，然后logstash会自行生成一个时间字段：@timestamp ，显然应该以日志里的时间为准。所以有如下配置：替换@timestamp字段的值为日志本身的时间。
  date {
    match => ["date","yyyy-MM-dd HH:mm:ss,SSS"]
    target => "@timestamp"
    locale => "cn"
    timezone => "Asia/Chongqing"
  }        
}

重启logstash：
service logstash restart
如果提示stop fail，则再试一次。


单独测试logstash 

1
2
启动：service logstash restart 
关闭：service logstash stop 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
进入容器，先执行：
service logstash stop 

保证logstash已经停止

cd /opt/logstash

vim test-logstash.conf
输入以下内容：

input {
  stdin {}
}

filter {
  json {
    source => "message"
    #target => "doc"
    #remove_field => ["message"]
  }

  date {
    match => ["date","yyyy-MM-dd HH:mm:ss,SSS"]
    target => "@timestamp"
    locale => "cn"
    timezone => "Asia/Chongqing"
  }        

}

output {
  stdout {}
}

保存，然后执行：

bin/logstash -f test-logstash.conf

等待一会，直到看到如下字样：

The stdin plugin is now waiting for input:
[2018-12-29T07:57:25,159][INFO ][logstash.pipeline        ] Pipeline started successfully {:pipeline_id=>"main", :thread=>"#<Thread:0x2d752f07 run>"}
[2018-12-29T07:57:25,210][INFO ][logstash.agent           ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[2018-12-29T07:57:25,458][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}

然后直接在命令行输入一个json字符串，比如这里输入：

{ "date":"2018-12-28 18:54:04,052", "level":"INFO", "logger":"c.l.s.l.c.u.GeTuiPushUtils", "file_line":"GeTuiPushUtils.java:54", "msg":{"value":"初始化个推服务成功"}, "exception":"" }

然后会输出一个字符串，大致如下：

{
    "msg" => { "value" => "初始化个推服务成功" },
    "host" => "d6e142cd797d",
    "level" => "INFO",
    "exception" => "",
    "file_line" => "GeTuiPushUtils.java:54",
    "logger" => "c.l.s.l.c.u.GeTuiPushUtils",
    "message" => "{ \"date\":\"2018-12-28 18:54:04,052\", \"level\":\"INFO\", \"logger\":\"c.l.s.l.c.u.GeTuiPushUtils\", \"file_line\":\"GeTuiPushUtils.java:54\", \"msg\":{\"value\":\"初始化个推服务成功\"}, \"exception\":\"\" }",
    "@version" => "1",
    "@timestamp" => 2018-12-28T10:54:04.052Z,
    "date" => "2018-12-28 18:54:04,052"
}

如果是这样的，说明配置成功。然后按住  ctrl + c 停止测试。

安装filebeat

centos 7 安装filebeat

1
2
3
curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.4.3-x86_64.rpm

sudo rpm -vi filebeat-5.4.3-x86_64.rpm

如果不能安装，可自行查看官方文档进行安装:
https://www.elastic.co/guide/en/beats/filebeat/5.4/filebeat-installation.html

安装好后，剩下的就行进行配置，filebeat的配置文件在：/etc/filebeat
配置文件为filebeat.yml 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
filebeat:
  prospectors:
    -
      paths:
        - /var/log/laijiandu-test/*.log
      tags: ["test-log"]
      input_type: log
      document_type: test-log
      scan_frequency: 2s
      harvester_buffer_size: 16384
      max_bytes: 10485760
  registry_file: /var/log/registry
output:
  logstash:
    hosts: ["logstashserverip:5044"]
    worker: 1
    bulk_max_size: 2048
    compression_level: 3
    index: filebeat
shipper:
logging:
  files:
    rotateeverybytes: 10485760

注意：上面配置中的tags需要和logstash中的对应起来。

filebeat的启动与停止

1
2
3
4
5
启动：
sudo /etc/init.d/filebeat start 

停止：
sudo /etc/init.d/filebeat stop

kibana

一切配置完成后,只需要输入ip:5601 即可进入kibana配置中心，这个时候进行基础配置就可以了

扫一扫，分享到微信

tag:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia-plus根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true

很忏愧,只是一条咸鱼