微服务从代码到k8s部署应有尽有系列（十一、日志收集）

#消息队列
kafka:
  image: wurstmeister/kafka
  container_name: kafka
  ports:
    - 9092:9092
  environment:
    KAFKA_ADVERTISED_HOST_NAME: kafka
    KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    TZ: Asia/Shanghai
  restart: always
  volumes:
    - /var/run/docker.sock:/var/run/docker.sock
  networks:
    - looklook_net
  depends_on:
    - zookeeperCopy

$  docker exec -it kafka /bin/shCopy

$ vi /opt/kafka/config/server.properties
listeners=PLAINTEXT://kafka:9092 # 原文件中，要加kafka listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://kafka:9092 #源文件中，要加kafka advertised.listeners=PLAINTEXT://:9092Copy

$  cd /opt/kafka/bin
$ ./kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 1 -partitions 1 --topic looklook-logCopy

filebeat.inputs:
  - type: log
    enabled: true
    paths:
      - /var/lib/docker/containers/*/*-json.log

filebeat.config:
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false

processors:
  - add_cloud_metadata: ~
  - add_docker_metadata: ~

output.kafka:
  enabled: true
  hosts: ["kafka:9092"]
  #要提前创建topic
  topic: "looklook-log"
  partition.hash:
    reachable_only: true
  compression: gzip
  max_message_bytes: 1000000
  required_acks: 1
Copy

Clusters:
  - Input:
      Kafka:
        Name: gostash
        Brokers:
          - "kafka:9092"
        Topics:
          - looklook-log
        Group: pro
        Consumers: 16
    Filters:
      - Action: drop
        Conditions:
          - Key: k8s_container_name
            Value: "-rpc"
            Type: contains
          - Key: level
            Value: info
            Type: match
            Op: and
      - Action: remove_field
        Fields:
          # - message
          - _source
          - _type
          - _score
          - _id
          - "@version"
          - topic
          - index
          - beat
          - docker_container
          - offset
          - prospector
          - source
          - stream
          - "@metadata"
      - Action: transfer
        Field: message
        Target: data
    Output:
      ElasticSearch:
        Hosts:
          - "http://elasticsearch:9200"
        Index: "looklook-{{yyyy-MM-dd}}"Copy

func (l *BusinessListLogic) BusinessList(req types.BusinessListReq) (*types.BusinessListResp, error) {

    logx.Error("测试的日志")

    ...
}Copy