前言

这篇主要是记录自己整合的过程，其中有参考其他博文，侵删。

创建rabbit-log.conf

进入docker

[root@localhost /]# docker exec -it elk /bin/bash

搭建elk链接: Docker部署ELK

创建rabbit-log.conf

[root@localhost /]# vim /opt/logstash/config/rabbit-log.conf

编写内容

 input {
   rabbitmq{
        host=>"127.0.0.1"                # 这里填写Rabbitmq的地址，确保可以ping通
        port=> 15672                            # 这里填写Rabbitmq的端口
        user=>"guest"                          # 这里填写Rabbitmq的用户名
        password=>"guest"                      # 这里填写Rabbitmq的密码
        exchange => "LOG-EXCHANGE"      # 这里填写Rabbitmq的交换机的名称
        key => "LOG"                   #路由键
        queue=>"LOG-QUEUE"                     # 这里填写Rabbitmq的队列的名称
        durable=> true                         # 这里填写Rabbitmq的队列的durable属性
        codec=>json                            # 这里填写Rabbitmq的队列的内容是什么格式
   }
}
filter {

}
output {
   elasticsearch {
        hosts => ["localhost:9200"]          # ElasticSearch的地址加端口
        index => "%{client}-%{+YYYYMMdd}"    # ElasticSearch的保存文档的index名称，client是各个模块的应用名，用于区别索引
   }
}

重启logstash

[root@localhost /]# service logstash restart

创建logback.xml

引入jar包

        <!--用于logback的if标签-->
        <dependency>
            <groupId>org.codehaus.janino</groupId>
            <artifactId>janino</artifactId>
            <version>3.1.2</version>
        </dependency>
        <!--mq-->
        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-starter-bus-amqp</artifactId>
        </dependency>

编写logback.xml

        <?xml version="1.0" encoding="UTF-8"?>
<configuration scan="true" scanPeriod="10 seconds">

    <!-- 读取application.yml中的属性 -->
    <springProperty scope="context" name="springAppName" source="spring.application.name"/>
    <springProperty scope="context" name="active" source="spring.profiles.active"/>

    <!--&lt;!&ndash; 因为rabbitmq的配置没有第一时间读取到，所以不走配置文件 &ndash;&gt;
    <springProperty scope="context" name="rabbitmqHost" source="spring.rabbitmq.host"/>
    <springProperty scope="context" name="port" source="spring.rabbitmq.port"/>
    <springProperty scope="context" name="username" source="spring.rabbitmq.username"/>
    <springProperty scope="context" name="password" source="spring.rabbitmq.password"/>-->

    <!-- 定义日志文件 输入位置 -->
    <property name="log_dir" value="/home/log" />
    <!-- 日志最大的历史 30天 -->
    <property name="maxHistory" value="30"/>

    <!-- 输出到控制台 -->
    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符 -->
            <pattern>
                %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{50}) - %highlight(%msg) %n
            </pattern>
        </encoder>
    </appender>
    <!-- ALL级别日志 appender -->
    <appender name="ALL" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <!-- 按天回滚 daily -->
            <fileNamePattern>
                ${log_dir}/${springAppName}/${springAppName}.%d{yyyy-MM-dd}.log
            </fileNamePattern>
            <!-- 日志最大的历史 30天 -->
            <maxHistory>${maxHistory}</maxHistory>
            <totalSizeCap>1GB</totalSizeCap>
            <cleanHistoryOnStart>true</cleanHistoryOnStart>
        </rollingPolicy>
        <encoder>
            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n</pattern>
        </encoder>
    </appender>
    <!-- 输出到mq -->
    <appender name="AMQP" class="org.springframework.amqp.rabbit.logback.AmqpAppender">
        <encoder class="com.xxx.common.core.utils.log.EsEncoder" >
            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n</pattern>
        </encoder>
        
        <!-- 能读到配置文件的属性就直接用这个 -->
        <!--<host>${rabbitmqHost}</host>
        <port>${port}</port>
        <username>${username}</username>
        <password>${password}</password>-->
       
        <host>127.0.0.1</host>
        <port>15672</port>
        <username>guest</username>
        <password>guest</password>
        
        <applicationId>${springAppName}</applicationId>
        <routingKeyPattern>LOG</routingKeyPattern>
        <declareExchange>true</declareExchange>
        <exchangeType>direct</exchangeType>
        <exchangeName>LOG-EXCHANGE</exchangeName>
        <generateId>true</generateId>
        <charset>UTF-8</charset>
        <durable>true</durable>
        <deliveryMode>PERSISTENT</deliveryMode>
    </appender>
    <!-- 异步输入到logstash -->
    <appender name="ASYNC" class="ch.qos.logback.classic.AsyncAppender">
        <appender-ref ref="AMQP" />
        <includeCallerData>true</includeCallerData>
    </appender>

    <!-- 日志输出级别 这样设置不打印日志 -->
    <root level="INFO">
        <appender-ref ref="STDOUT"/>
        <appender-ref ref="ALL" />
        <if condition='"${active}".contains("test")'>
            <then>
                <appender-ref ref="ASYNC"/>
            </then>
        </if>
    </root>
</configuration>

编写EsEncoder.java

package com.xxx.common.core.utils.log;

import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.classic.spi.IThrowableProxy;
import ch.qos.logback.classic.spi.StackTraceElementProxy;
import cn.hutool.core.util.StrUtil;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;

import java.time.LocalDateTime;
import java.util.LinkedHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @Description 发送至es的日志格式化
 * @Author Top
 * @CreateDate 2020/12/13 20:37
 * @Version 1.0
 */
public class EsEncoder extends PatternLayoutEncoder {


    @Override
    public byte[] encode(ILoggingEvent event) {
        return convertEventWithError(event).getBytes();
    }

    private final static Pattern pattern = Pattern.compile("\\{\\}");

    private String convertEventWithError(ILoggingEvent event) {
        try {
            // 检查-------------------------------------------------------
            if (event == null) {
                return StrUtil.EMPTY;
            }
            if (StrUtil.isBlank(event.getMessage())) {
                return StrUtil.EMPTY;
            }
            // 获取源Msg替换结果-------------------------------------------
            String msg = printReplace(event.getMessage(), event.getArgumentArray());
            // 尝试获取错误日志--------------------------------------------
            IThrowableProxy err = event.getThrowableProxy();
            StringBuilder errStr = null;
            if (err != null) {
                errStr = new StringBuilder();
                errStr.append(err.getClassName());
                errStr.append(" : ");
                errStr.append(err.getMessage());
                errStr.append(" \n ");
                StackTraceElementProxy[] errTrack = err.getStackTraceElementProxyArray();
                for (StackTraceElementProxy stackTraceElementProxy : errTrack) {
                    errStr.append(stackTraceElementProxy.getStackTraceElement());
                    errStr.append(" \n ");
                }
                errStr.delete(errStr.length() - 3, errStr.length());
            }
            // 构建Json--------------------------------------------------
            LinkedHashMap<String, String> result = new LinkedHashMap();
            //client用于区别es的索引，springAppName属性取得logback.xml哪里的 logback.xml的springAppName是取得配置文件的
            result.put("client", event.getLoggerContextVO().getPropertyMap().get("springAppName"));
            result.put("logTime", LocalDateTime.now().toString() + "Z");
            result.put("level", event.getLevel().levelStr);
            result.put("thread", event.getThreadName());
            result.put("logger", event.getLoggerName());
            result.put("msg", msg);
            if (errStr != null && errStr.length() != 0) {
                result.put("err", errStr.toString());
            }
            // 返回--------------------------------------------------
            return (new ObjectMapper()).writeValueAsString(result) + "\n";
        } catch (Throwable e) {
            try {
                return (new ObjectMapper()).writeValueAsString(
                        ImmutableMap.of("err", "EsEncoder日志工具错误:" + e.toString())) + "\n";
            } catch (Throwable e2) {
                return "{\"err\":\"EsEncoder日志工具错误\"}\n";
            }
        }
    }

    public static String printReplace(String first, Object... replaces) {
        try {
            int replaceLen;
            if (StrUtil.isBlank(first)) {
                return first;
            }
            if (replaces == null || (replaceLen = replaces.length) == 0) {
                return first;
            }

            StringBuilder result = new StringBuilder();
            int replaceIdx = 0;
            int curCpIdx = 0;

            Matcher m = pattern.matcher(first);
            boolean tailed = false;
            while (m.find()) {
                if (replaceIdx < replaceLen) {
                    result.append(first, curCpIdx, m.start());
                    result.append(replaces[replaceIdx] == null ? StrUtil.NULL : replaces[replaceIdx].toString());
                    curCpIdx = m.end();
                } else {
                    result.append(first.substring(curCpIdx));
                    tailed = true;
                    break;
                }
                replaceIdx++;
            }
            if (!tailed) {
                result.append(first.substring(curCpIdx));
            }
            return result.toString();
        } catch (Throwable e) {
            return first;
        }
    }
}

application.yml加上rabbitmq的属性，不加也能用，但是rabbitmq的健康检查会一直报连接不上的信息

spring:
  rabbitmq:
    host: 127.0.0.1
    port: 15672
    username: guest
    password: guest

Kibana登录认证设置

安装Apache密码生产工具

[root@localhost /]# yum install httpd-tools

生成密码文件

[root@localhost /]# htpasswd -c -b /etc/nginx/passwd/kibana.passwd user password

配置Nginx

[root@localhost /]# vim /usr/local/nginx/conf/nginx.conf

配置内容

server {
listen 192.168.75.150:5601;
 
auth_basic "Kibana Auth";
auth_basic_user_file /etc/nginx/passwd/kibana.passwd;
 
location / {
proxy_pass http://127.0.0.1:5601;
proxy_redirect off;
}
}

重启nginx

[root@localhost /]# cd /usr/local/nginx/sbin/
[root@localhost sbin]# ./nginx -s reload

引用链接

链接: logback.xml if 条件判断
链接: Java-Logback-使用Logback输出日志为JSON
链接: Kibana登录认证设置