Filebeat: Receiving Docker logs in Kafka

1. Install Filebeat
2. echo "deb https://artifacts.elastic.co/packages/8.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-8.x.list
3. sudo apt-get update && sudo apt-get install filebeat

filebeat.config:
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false

processors:
  - add_cloud_metadata: ~
  - add_docker_metadata: ~

filebeat.inputs:
- type: container
  paths:
    - '/var/lib/docker/containers/*/*.log'

# ============================= Kafka Output =============================

output.kafka:
  hosts: ["kafka-server:9093"]
  topic: "docker-logs"
  ssl.certificate_authorities: "/etc/filebeat/certs/caroot.pem"
  ssl.certificate: "/etc/filebeat/certs/cert.pem"
  ssl.key: "/etc/filebeat/certs/key.pem"
  max_message_bytes: 2000000

 3. Enable Filebeat service sudo systemctl enable filebeat
 4. Restart Filebeat.