如何使用 docker-compose 配置 fluent-bit、Fluentd、Loki 和 Grafana？

Question

我正在尝试在 docker 中运行 Fluent-bit 并使用 Loki 在 Grafana 中查看日志，但我在 Grafana 中看不到任何标签。 Loki 数据源报告它可以工作并找到标签。

我需要弄清楚如何从 fluent-bit -> loki -> grafana 获取 docker 日志。任何日志。

这是我的docker-compose.yaml

version: "3.3"

networks:
  loki:
    external: true

services:
  fluent-bit:
    image: grafana/fluent-bit-plugin-loki:latest
    container_name: fluent-bit
    environment:
      LOKI_URL: http://loki:3100/loki/api/v1/push
    networks:
      - loki
    volumes:
      - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
    logging:
      options:
         tag: infra.monitoring

这是我的配置文件。

[INPUT]
    Name        forward
    Listen      0.0.0.0
    Port        24224
[Output]
    Name loki
    Match *
    Url ${LOKI_URL}
    RemoveKeys source
    Labels {job="fluent-bit"}
    LabelKeys container_name
    BatchWait 1
    BatchSize 1001024
    LineFormat json
    LogLevel info

这是我的 Grafana 和 Loki 设置

grafana:
  image: grafana/grafana
  depends_on:
    - prometheus
  container_name: grafana
  volumes:
    - grafana_data:/var/lib/grafana:rw
    - ./grafana/provisioning:/etc/grafana/provisioning
  environment:
    - GF_SECURITY_ADMIN_USER=admin
    - GF_SECURITY_ADMIN_PASSWORD=admin
    - GF_USERS_ALLOW_SIGN_UP=false
    - GF_INSTALL_PLUGINS=grafana-piechart-panel
    - GF_RENDERING_SERVER_URL=http://renderer:8081/render
    - GF_RENDERING_CALLBACK_URL=http://grafana:3000/
    - GF_LOG_FILTERS=rendering:debug
  restart: unless-stopped
  networks:
    - traefik
    - loki
  labels:
    - "traefik.enable=true"
    - "traefik.http.routers.grafana.rule=Host(`grafana-int.mydomain.com`)"
    - "traefik.http.services.grafana.loadbalancer.server.port=3000"
    - "traefik.docker.network=traefik"


loki:
  image: grafana/loki:latest
  container_name: loki
  expose:
    - "3100"
  networks:
    - loki

renderer:
  image: grafana/grafana-image-renderer:2.0.0
  container_name: grafana-image-renderer
  expose:
    - "8081"
  environment:
    ENABLE_METRICS: "true"
  networks:
    - loki

我已尝试使用下面评论中链接的文档中描述的以下配置，但仍然没有标签。

[SERVICE]
    Flush        1
    Log_Level    info
    Parsers_File parsers.conf
[INPUT]
    Name                syslog
    Path                /tmp/in_syslog
    Buffer_Chunk_Size   32000
    Buffer_Max_Size     64000
[OUTPUT]
    Name loki
    Match *
    Url ${LOKI_URL}
    RemoveKeys source
    Labels {job="fluent-bit"}
    LabelKeys container_name
    BatchWait 1
    BatchSize 1001024
    LineFormat json
    LogLevel info

我尝试了这个配置，但仍然没有标签。

[INPUT]
   @type tail
   format json
   read_from_head true
   path /var/log/syslog
   pos_file /tmp/container-logs.pos
[OUTPUT]
   Name loki
   Match *
   Url ${LOKI_URL}
   RemoveKeys source
   LabelKeys container_name
   BatchWait 1
   BatchSize 1001024
   LineFormat json
   LogLevel info  

Answer 1

玩了一段时间后，我认为最好的方法是收集 fluent-bit 中的日志并将它们转发到 Fluentd，然后输出到 Loki 并在 Grafana 中读取这些文件。

这是一个可以在本地工作的配置。

用于 Fluentd 和 Loki 的 docker-compose.yaml。

version: "3.8"

networks:
  appnet:
    external: true

volumes:
  host_logs:
    
services:
  fluentd:
    image: grafana/fluent-plugin-loki:master
    command:
      - "fluentd"
      - "-v"
      - "-p"
      - "/fluentd/plugins"
    environment:
      LOKI_URL: http://loki:3100
      LOKI_USERNAME:
      LOKI_PASSWORD:
    container_name: "fluentd"
    restart: always
    ports:
      - '24224:24224'
    networks:
      - appnet
    volumes:
      - host_logs:/var/log
      # Needed for journald log ingestion:
      - /etc/machine-id:/etc/machine-id
      - /dev/log:/dev/log
      - /var/run/systemd/journal/:/var/run/systemd/journal/      
      - type: bind
        source: ./config/fluent.conf
        target: /fluentd/etc/fluent.conf
      - type: bind
        source: /var/lib/docker/containers
        target: /fluentd/log/containers
    logging:
      options:
        tag: docker.monitoring
  
  loki:
    image: grafana/loki:master
    container_name: "loki"
    restart: always
    networks:
      - appnet
    ports:
      - 3100
    volumes:
      - type: bind
        source: ./config/loki.conf
        target: /loki/etc/loki.conf
    depends_on:
      - fluentd

fluent.conf

<source>
  @type forward
  bind 0.0.0.0
  port 24224
</source>

<match **>
  @type loki
  url "http://loki:3100"
  flush_interval 1s
  flush_at_shutdown true
  buffer_chunk_limit 1m
  extra_labels {"job":"localhost_logs", "host":"localhost", "agent":"fluentd"}
  <label>
      fluentd_worker
  </label>  
</match>

loki.conf

auth_enabled: false

server:
  http_listen_port: 3100

ingester:
  lifecycler:
    address: 127.0.0.1
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 5m
  chunk_retain_period: 30s

schema_config:
  configs:
  - from: 2020-10-16
    store: boltdb
    object_store: filesystem
    schema: v11
    index:
      prefix: index_
      period: 168h

storage_config:
  boltdb:
    directory: /tmp/loki/index

  filesystem:
    directory: /tmp/loki/chunks

limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h

docker-compose.yaml for fluent-bit

version: "3.8"

networks:
    appnet:
      external: true
  
services:

  fluent-bit:
    image: fluent/fluent-bit:latest
    container_name: "fluent-bit"
    restart: always
    ports:
      - '2020:2020'
    networks:
      - appnet
    volumes:
      - type: bind
        source: ./config/fluent-bit.conf
        target: /fluent-bit/etc/fluent-bit.conf
        read_only: true
      - type: bind
        source: ./config/parsers.conf
        target: /fluent-bit/etc/parsers.conf
        read_only: true
      - type: bind
        source: /var/log/
        target: /var/log/
      - type: bind
        source: /var/lib/docker/containers
        target: /fluent-bit/log/containers

fluent-bit.conf

[SERVICE]
    Flush   2
    Log_Level   info
    Parsers_File parsers.conf

[INPUT]
    Name    tail
    Path    /fluent-bit/log/containers/*/*-json.log
    Tag     docker.logs
    Parser  docker

[OUTPUT]
    Name    forward
    Match   *
    Host    fluentd

parsers.conf

[PARSER]
    Name        docker
    Format      json
    Time_Key    time
    Time_Format    %Y-%m-%dT%H:%M:%S.%L
    Time_Keep    On