FluentdBit 采集K8s中服务日志

以Rainbond插件形式获取各个服务日志

1.构建插件

1.创建插件信息

image-20230623105138195

2.添加配置文件

  1. fluent-bit

    挂载路径：/fluent-bit/etc/fluent-bit.conf

    配置文件内容：

    1
    2
    3
    4
    5
    6
    7
    8
    9
    [SERVICE]
        flush         1
        log_level     info
        Daemon        off
        parsers_file  parsers.conf

    @INCLUDE input-kubernetes.conf
    @INCLUDE filter-kubernetes.conf
    @INCLUDE output-kafka.conf

  2. parsers

    挂载路径：/fluent-bit/etc/parsers.conf

    配置文件内容：

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    [PARSER]
        Name   njc_service_log
        Format regex
        Regex  ^(?<time>[^ ]*) [^ ]* (?<level>[^ ]*) [^ ]* (?<class>[^ ]*): (?<message>[^ \]].*)
        Time_Key time
        Time_Format %d/%b/%Y:%H:%M:%S %z
        
    [PARSER]
        Name multiline_pattern
        Format regex
        Regex ^(?<log>\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}\.\d{3}.*)
        Time_Key  time
        Time_Format %b %d %H:%M:%S

  3. input-kubernetes

    挂载路径：/fluent-bit/etc/input-kubernetes.conf

    配置文件内容：

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    [INPUT]
        Name              tail
        Tag               kube.njc.dev.permission
        Path              /app/logs/loky-im-system/info.log,/app/logs/loky-im-system/error.log
        DB                /var/log/flb.db
        Mem_Buf_Limit     5MB
        Rotate_Wait       60
        Skip_Long_Lines   On
        Refresh_Interval  10
        multiline.parser  java

  4. filter-kubernetes

    挂载路径：/fluent-bit/etc/filter-kubernetes.conf

    配置文件内容：

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    [FILTER]
        Name                kubernetes
        Match               kube.njc.*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Kube_Tag_Prefix     kube.var.log.containers.
        Merge_Log           On
        Merge_Log_Key       log_processed
        K8S-Logging.Parser  On
        K8S-Logging.Exclude Off

    [FILTER]
        Name         parser
        Match        kube.njc.dev.permission
        Key_Name     log
        Parser       njc_service_log
        Reserve_Data  On
        Preserve_Key  On

  5. output-kafka

    挂载路径：/fluent-bit/etc/output-kafka.conf

    配置文件内容：

    1
    2
    3
    4
    5
    [OUTPUT]
        Name           kafka
        Match          kube.njc.dev.permission
        Brokers        192.168.5.31:9092,192.168.5.32:9092,192.168.5.33:9092
        Topics         service.log.dev.permission