From ce41f608884b2f82cc700cfa14e4f8d1065ff153 Mon Sep 17 00:00:00 2001 From: Germain Masse Date: Thu, 19 Mar 2020 16:04:21 +0100 Subject: [PATCH] Move filebeat to its own container --- Dockerfile | 11 --------- README.md | 2 +- config/filebeat.docker.yml | 16 ++++++++++++ docker-compose.elk.yml.dist | 13 ++++++++++ docker-compose.filebeat.yml.dist | 42 ++++++++++++++++++++++++++++++++ target/start-mailserver.sh | 25 ------------------- 6 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 config/filebeat.docker.yml create mode 100644 docker-compose.filebeat.yml.dist diff --git a/Dockerfile b/Dockerfile index 1d4f0a88..f6d0ba6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -119,17 +119,6 @@ RUN \ rm -f /etc/postsrsd.secret && \ rm -f /etc/cron.daily/00logwatch -# install filebeat for logging -# SKIP and run in an external container instead -#RUN curl https://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add - && \ -# echo "deb http://packages.elastic.co/beats/apt stable main" | tee -a /etc/apt/sources.list.d/beats.list && \ -# apt-get update -q --fix-missing && \ -# apt-get -y install --no-install-recommends \ -# filebeat \ -# && apt-get clean \ -# && rm -rf /var/lib/apt/lists/* -#COPY target/filebeat.yml.tmpl /etc/filebeat/filebeat.yml.tmpl - RUN echo "0 */6 * * * clamav /usr/bin/freshclam --quiet" > /etc/cron.d/clamav-freshclam && \ chmod 644 /etc/clamav/freshclam.conf && \ freshclam && \ diff --git a/README.md b/README.md index 04bf351f..5d304172 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ If you got any problems with SPF and/or forwarding mails, give [SRS](https://git Your config folder will be mounted in `/tmp/docker-mailserver/`. To understand how things work on boot, please have a look at [start-mailserver.sh](https://github.com/tomav/docker-mailserver/blob/master/target/start-mailserver.sh) -`restart: always` ensures that the mail server container (and ELK container when using the mail server together with ELK stack) is automatically restarted by Docker in cases like a Docker service or host restart or container exit. +`restart: always` ensures that the mail server container (and Filebeat/ELK containers when using the mail server together with ELK stack) is automatically restarted by Docker in cases like a Docker service or host restart or container exit. #### Exposed ports * 25 receiving email from other mailservers diff --git a/config/filebeat.docker.yml b/config/filebeat.docker.yml new file mode 100644 index 00000000..cfa132ee --- /dev/null +++ b/config/filebeat.docker.yml @@ -0,0 +1,16 @@ +filebeat.config: + modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + +filebeat.autodiscover: + providers: + - type: docker + hints.enabled: true + hints.default_config.enabled: false + +processors: +- add_cloud_metadata: ~ + +output.logstash: + hosts: ["127.0.0.1:5044"] diff --git a/docker-compose.elk.yml.dist b/docker-compose.elk.yml.dist index 0c308a3c..95f3aa2f 100644 --- a/docker-compose.elk.yml.dist +++ b/docker-compose.elk.yml.dist @@ -8,6 +8,10 @@ services: container_name: ${CONTAINER_NAME} links: - elk + labels: + - "co.elastic.logs/enabled=true" + - "co.elastic.logs/module=system" + - "co.elastic.logs/fileset.stdout=syslog" ports: - "25:25" - "143:143" @@ -24,6 +28,15 @@ services: - NET_ADMIN - SYS_PTRACE restart: always + filebeat: + image: docker.elastic.co/beats/filebeat:7.6.1 + user: root + volumes: + - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers/:/var/lib/docker/containers/:ro + command: ["filebeat", "-e", "--strict.perms=false"] + restart: always elk: build: elk ports: diff --git a/docker-compose.filebeat.yml.dist b/docker-compose.filebeat.yml.dist new file mode 100644 index 00000000..5dc483c1 --- /dev/null +++ b/docker-compose.filebeat.yml.dist @@ -0,0 +1,42 @@ +version: '2' +services: + mail: + image: tvial/docker-mailserver:latest + hostname: ${HOSTNAME} + domainname: ${DOMAINNAME} + container_name: ${CONTAINER_NAME} + ports: + - "25:25" + - "143:143" + - "587:587" + - "993:993" + labels: + - "co.elastic.logs/enabled=true" + - "co.elastic.logs/module=system" + - "co.elastic.logs/fileset.stdout=syslog" + volumes: + - maildata:/var/mail + - mailstate:/var/mail-state + - maillogs:/var/log/mail + - ./config/:/tmp/docker-mailserver/ + env_file: + - .env + - env-mailserver + cap_add: + - NET_ADMIN + - SYS_PTRACE + restart: always + filebeat: + image: docker.elastic.co/beats/filebeat:7.6.1 + user: root + volumes: + - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/lib/docker/containers/:/var/lib/docker/containers/:ro + command: ["filebeat", "-e", "--strict.perms=false"] + restart: always +volumes: + maildata: + driver: local + maillogs: + driver: local diff --git a/target/start-mailserver.sh b/target/start-mailserver.sh index 5b3c04b7..dc31e3cc 100644 --- a/target/start-mailserver.sh +++ b/target/start-mailserver.sh @@ -95,10 +95,6 @@ function register_functions() { _register_setup_function "_setup_default_vars" _register_setup_function "_setup_file_permissions" - if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then - _register_setup_function "_setup_elk_forwarder" - fi - if [ "$SMTP_ONLY" != 1 ]; then _register_setup_function "_setup_dovecot" _register_setup_function "_setup_dovecot_dhparam" @@ -208,10 +204,6 @@ function register_functions() { _register_start_daemon "_start_daemons_cron" _register_start_daemon "_start_daemons_rsyslog" - if [ "$ENABLE_ELK_FORWARDER" = 1 ]; then - _register_start_daemon "_start_daemons_filebeat" - fi - if [ "$SMTP_ONLY" != 1 ]; then _register_start_daemon "_start_daemons_dovecot" fi @@ -1452,18 +1444,6 @@ function _setup_security_stack() { fi } -function _setup_elk_forwarder() { - notify 'task' 'Setting up Elk forwarder' - - ELK_PORT=${ELK_PORT:="5044"} - ELK_HOST=${ELK_HOST:="elk"} - notify 'inf' "Enabling log forwarding to ELK ($ELK_HOST:$ELK_PORT)" - cat /etc/filebeat/filebeat.yml.tmpl \ - | sed "s@\$ELK_HOST@$ELK_HOST@g" \ - | sed "s@\$ELK_PORT@$ELK_PORT@g" \ - > /etc/filebeat/filebeat.yml -} - function _setup_logrotate() { notify 'inf' "Setting up logrotate" @@ -1750,11 +1730,6 @@ function _start_daemons_dovecot() { #fi } -function _start_daemons_filebeat() { - notify 'task' 'Starting filebeat' 'n' - supervisorctl start filebeat -} - function _start_daemons_fetchmail() { notify 'task' 'Starting fetchmail' 'n' /usr/local/bin/setup-fetchmail