Fork me on GitHub

springBoot引入elk实现日志系统

字数统计: 2.6k字   |   阅读时长≈ 14分

一丶安装

1、安装包下载：
elastic产品下载地址：https://www.elastic.co/cn/downloads/

elasticsearch：https://www.elastic.co/cn/downloads/elasticsearch

kibana：https://www.elastic.co/cn/downloads/kibana

logstash：https://artifacts.elastic.co/cn/downloads/logstash

二丶部署

1. 部署elasticsearch

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
# 把下载好的包放到/usr/local/下,并跳转
cd /usr/local

# 解压文件
tar -zxvf elasticsearch-7.6.2-linux-x86_64.tar.gz

# 创建新的账号(出于安全考虑es默认是不允许使用root账号的)
useradd es

# 设置密码
passwd es

# 创建data文件和logs文件夹
mkdir -p /home/es/elasticsearch/data
mkdir -p /home/es/elasticsearch/logs

# 给文件授权
chmod -R 777 /home/es/elasticsearch
chmod -R 777 /usr/local/elasticsearch-7.6.2

# 进入elasticsearch-7.6.2/config文件夹
cd /usr/local/elasticsearch-7.6.2/config

# 设置elasticsearch.yml
vim elasticsearch.yml

>>
http.cors.enabled: true
http.cors.allow-origin: "*"
network.host: 0.0.0.0
cluster.name: laokou-elasticsearch #可自定义
node.name: node-elasticsearch #可自定义
http.port: 9200
cluster.initial_master_nodes: ["node-elasticsearch"] #这里就是node.name
path.data: /home/es/elasticsearch/data # 数据目录位置
path.logs: /home/es/elasticsearch/logs # 日志目录位置
>>

# 设置jvm.option(默认是1g)
vim jvm.options

>>
-Xms512m
-Xmx512m
>>

#运行报错
报错信息：max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]

vim /etc/sysctl.conf
>>
vm.max_map_count = 655360
>>

# 设置好后刷新
sysctl -p

# 设置limlts.conf

vim /etc/security/limits.conf

>>
es soft nofile 65535
es hard nofile 65537
>>

# 用新账号启动
su es

# 运行es
cd /usr/local/elasticsearch-7.6.2/bin
./elasticsearch

# 启动http://localhost:9200,如果有数据返回则成功
设置启动脚本
>>
nohup ./bin/elasticsearch > nohup.out 2>&1 &
>>


# 报错问题及解决方案
1. 启动jdk版本警告(es最低jdk1.8)
>>
future versions of Elasticsearch will require Java 11; your Java version from [/data/Java/jdk1.8.0_291/jre] does not meet this requirement
>>

解决方案：
修改elasticsearch安装目录/bin/elasticsearch-env 文件，注释掉以下代码部分即可解决！

安装ik分词器

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
>su root

>mkdir -p /usr/local/elasticsearch-7.6.2/plugins/analysis-synonym

>mkdir -p /usr/local/elasticsearch-7.6.2/plugins/analysis-ik

>mkdir -p /usr/local/elasticsearch-7.6.2/plugins/analysis-pinyin

>yum install -y unzip zip

>unzip -d /usr/local/elasticsearch-7.6.2/plugins/analysis-ik /opt/elasticsearch-analysis-ik-7.6.2.zip

>unzip -d /usr/local/elasticsearch-7.6.2/plugins/analysis-pinyin /opt/elasticsearch-analysis-pinyin-7.6.2.zip

>unzip -d /usr/local/elasticsearch-7.6.2/plugins/analysis-synonym /opt/elasticsearch-analysis-dynamic-synonym-7.6.2.zip

2. 部署kibana

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 把安装包放到/usr/local下，并且解压
cd /usr/local/
tar -zxvf kibana-7.6.2-linux-x86_64.tar.gz

# 进入kibana目录下面
cd kibana-7.6.2-linux-x86_64/

# 编辑配置文件
vim config/kibana.yml

# 添加一下内容,也可以自己找找放开注释
service.port: 5601
service.host: "0.0.0.0"
elasticsearch.hosts: ["http://localhost:9200"]

# 启动kibana
./bin/kibana

# 进入页面localhost:5601

#设置启动脚本
>>
nohup ./bin/kibana > nohup.out 2>&1 &
>>

3.部署logstash

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
# 下载包到/usr/local/
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.6.2.tar.gz

# 解压
tar -zxvf logstash-7.6.2.tar.gz

# 创建logstash.conf文件到config下
touch config/logstash.conf
vi config/logstash.conf

# 根据具体业务来编写脚本
>>
input {
  tcp {
    mode => "server"
    host => "0.0.0.0"
    port => 4560
    codec => json_lines
    type => "debug"
  }
  tcp {
    mode => "server"
    host => "0.0.0.0"
    port => 4561
    codec => json_lines
    type => "error"
  }
  tcp {
    mode => "server"
    host => "0.0.0.0"
    port => 4562
    codec => json_lines
    type => "business"
  }
  tcp {
    mode => "server"
    host => "0.0.0.0"
    port => 4563
    codec => json_lines
    type => "record"
  }
}
filter{
  if [type] == "record" {
    mutate {
      remove_field => "port"
      remove_field => "host"
      remove_field => "@version"
    }
    json {
      source => "message"
      remove_field => ["message"]
    }
  }
}
output {
  elasticsearch {
    hosts => "localhost:9200"
    index => "sjzdry-%{type}-%{+YYYY.MM.dd}"
  }
}
>>

# 启动程序
cd ../
./bin/logstash -f ./config/logstash.conf

三丶springboot集成elk

1.配置springboot脚本

  1. 在项目的资源文件夹下添加logback-spring.xml脚本
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE configuration>
<configuration>
    <!--引用默认日志配置-->
    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
    <!--使用默认的控制台日志输出实现-->
    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
    <!--应用名称-->
    <springProperty scope="context" name="APP_NAME" source="spring.application.name" defaultValue="springBoot"/>
    <!--日志文件保存路径-->
    <property name="LOG_FILE_PATH" value="${LOG_FILE:-${LOG_PATH:-${LOG_TEMP:-${java.io.tmpdir:-/tmp}}}/logs}"/>
    <!--LogStash访问host-->
    <springProperty name="LOG_STASH_HOST" scope="context" source="logstash.host" defaultValue="localhost"/>

    <!--DEBUG日志输出到文件-->
    <appender name="FILE_DEBUG"
              class="ch.qos.logback.core.rolling.RollingFileAppender">
        <!--输出DEBUG以上级别日志-->
        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
            <level>DEBUG</level>
        </filter>
        <encoder>
            <!--设置为默认的文件日志格式-->
            <pattern>${FILE_LOG_PATTERN}</pattern>
            <charset>UTF-8</charset>
        </encoder>
        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
            <!--设置文件命名格式-->
            <fileNamePattern>${LOG_FILE_PATH}/debug/${APP_NAME}-%d{yyyy-MM-dd}-%i.log</fileNamePattern>
            <!--设置日志文件大小，超过就重新生成文件，默认10M-->
            <maxFileSize>${LOG_FILE_MAX_SIZE:-10MB}</maxFileSize>
            <!--日志文件保留天数，默认30天-->
            <maxHistory>${LOG_FILE_MAX_HISTORY:-30}</maxHistory>
        </rollingPolicy>
    </appender>

    <!--ERROR日志输出到文件-->
    <appender name="FILE_ERROR"
              class="ch.qos.logback.core.rolling.RollingFileAppender">
        <!--只输出ERROR级别的日志-->
        <filter class="ch.qos.logback.classic.filter.LevelFilter">
            <level>ERROR</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
        <encoder>
            <!--设置为默认的文件日志格式-->
            <pattern>${FILE_LOG_PATTERN}</pattern>
            <charset>UTF-8</charset>
        </encoder>
        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
            <!--设置文件命名格式-->
            <fileNamePattern>${LOG_FILE_PATH}/error/${APP_NAME}-%d{yyyy-MM-dd}-%i.log</fileNamePattern>
            <!--设置日志文件大小，超过就重新生成文件，默认10M-->
            <maxFileSize>${LOG_FILE_MAX_SIZE:-10MB}</maxFileSize>
            <!--日志文件保留天数，默认30天-->
            <maxHistory>${LOG_FILE_MAX_HISTORY:-30}</maxHistory>
        </rollingPolicy>
    </appender>

    <!--DEBUG日志输出到LogStash-->
    <appender name="LOG_STASH_DEBUG" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
            <level>DEBUG</level>
        </filter>
        <destination>${LOG_STASH_HOST}:4560</destination>
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
            <providers>
                <timestamp>
                    <timeZone>Asia/Shanghai</timeZone>
                </timestamp>
                <!--自定义日志输出格式-->
                <pattern>
                    <pattern>
                        {
                        "project": "mall-swarm",
                        "level": "%level",
                        "service": "${APP_NAME:-}",
                        "pid": "${PID:-}",
                        "thread": "%thread",
                        "class": "%logger",
                        "message": "%message",
                        "stack_trace": "%exception{20}"
                        }
                    </pattern>
                </pattern>
            </providers>
        </encoder>
        <!--当有多个LogStash服务时，设置访问策略为轮询-->
        <connectionStrategy>
            <roundRobin>
                <connectionTTL>5 minutes</connectionTTL>
            </roundRobin>
        </connectionStrategy>
    </appender>

    <!--ERROR日志输出到LogStash-->
    <appender name="LOG_STASH_ERROR" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <filter class="ch.qos.logback.classic.filter.LevelFilter">
            <level>ERROR</level>
            <onMatch>ACCEPT</onMatch>
            <onMismatch>DENY</onMismatch>
        </filter>
        <destination>${LOG_STASH_HOST}:4561</destination>
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
            <providers>
                <timestamp>
                    <timeZone>Asia/Shanghai</timeZone>
                </timestamp>
                <!--自定义日志输出格式-->
                <pattern>
                    <pattern>
                        {
                        "project": "mall-swarm",
                        "level": "%level",
                        "service": "${APP_NAME:-}",
                        "pid": "${PID:-}",
                        "thread": "%thread",
                        "class": "%logger",
                        "message": "%message",
                        "stack_trace": "%exception{20}"
                        }
                    </pattern>
                </pattern>
            </providers>
        </encoder>
        <!--当有多个LogStash服务时，设置访问策略为轮询-->
        <connectionStrategy>
            <roundRobin>
                <connectionTTL>5 minutes</connectionTTL>
            </roundRobin>
        </connectionStrategy>
    </appender>

    <!--业务日志输出到LogStash-->
    <appender name="LOG_STASH_BUSINESS" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <destination>${LOG_STASH_HOST}:4562</destination>
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
            <providers>
                <timestamp>
                    <timeZone>Asia/Shanghai</timeZone>
                </timestamp>
                <!--自定义日志输出格式-->
                <pattern>
                    <pattern>
                        {
                        "project": "mall-swarm",
                        "level": "%level",
                        "service": "${APP_NAME:-}",
                        "pid": "${PID:-}",
                        "thread": "%thread",
                        "class": "%logger",
                        "message": "%message",
                        "stack_trace": "%exception{20}"
                        }
                    </pattern>
                </pattern>
            </providers>
        </encoder>
        <!--当有多个LogStash服务时，设置访问策略为轮询-->
        <connectionStrategy>
            <roundRobin>
                <connectionTTL>5 minutes</connectionTTL>
            </roundRobin>
        </connectionStrategy>
    </appender>

    <!--接口访问记录日志输出到LogStash-->
    <appender name="LOG_STASH_RECORD" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <destination>${LOG_STASH_HOST}:4563</destination>
        <encoder charset="UTF-8" class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
            <providers>
                <timestamp>
                    <timeZone>Asia/Shanghai</timeZone>
                </timestamp>
                <!--自定义日志输出格式-->
                <pattern>
                    <pattern>
                        {
                        "project": "mall-swarm",
                        "level": "%level",
                        "service": "${APP_NAME:-}",
                        "class": "%logger",
                        "message": "%message"
                        }
                    </pattern>
                </pattern>
            </providers>
        </encoder>
        <!--当有多个LogStash服务时，设置访问策略为轮询-->
        <connectionStrategy>
            <roundRobin>
                <connectionTTL>5 minutes</connectionTTL>
            </roundRobin>
        </connectionStrategy>
    </appender>

    <!--控制框架输出日志-->
    <logger name="org.slf4j" level="INFO"/>
    <logger name="springfox" level="INFO"/>
    <logger name="io.swagger" level="INFO"/>
    <logger name="org.springframework" level="INFO"/>
    <logger name="org.hibernate.validator" level="INFO"/>
    <logger name="com.alibaba.nacos.client.naming" level="INFO"/>

    <root level="DEBUG">
        <appender-ref ref="CONSOLE"/>
        <appender-ref ref="FILE_DEBUG"/>
        <appender-ref ref="FILE_ERROR"/>
        <appender-ref ref="LOG_STASH_DEBUG"/>
        <appender-ref ref="LOG_STASH_ERROR"/>
    </root>

    <logger name="com.macro.mall.common.log.WebLogAspect" level="DEBUG">
        <appender-ref ref="LOG_STASH_RECORD"/>
    </logger>

    <logger name="com.macro.mall" level="DEBUG">
        <appender-ref ref="LOG_STASH_BUSINESS"/>
    </logger>
</configuration>

运行就行！

具体方式可以查看mall源码

2.配置kibana查询日志

  1. 第一步进入kibana,模糊搜索一下你的日志索引

  1. 选择日期格式，点击创建刷新一下就行

  1. 完成

赏

谢谢你请我吃糖果

支付宝
微信

扫一扫，分享到微信

× 文章目录
  1. 1. 一丶安装
  2. 2. 二丶部署
    1. 2.1. 1. 部署elasticsearch
    2. 2.2. 2. 部署kibana
    3. 2.3. 3.部署logstash
  3. 3. 三丶springboot集成elk
    1. 3.1. 1.配置springboot脚本
    2. 3.2. 2.配置kibana查询日志

tag:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia-plus根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true

主要涉及生活和技术：
Java后端开发，热爱生活

联系QQ:2627233351

只做了一点微小的工作
谢谢大家