From 9b5d4d307c46119942b5ebb035f9e9e87ba403b1 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+aendeavor@users.noreply.github.com> Date: Mon, 21 Sep 2020 13:54:31 +0200 Subject: [PATCH] getting rid of ELK & updating Compose (#1614) * removed elk diretory * removed filebeat / elk compose.yml.dist; adjusted other compose files * final README update for env variables * updated compose files --- README.md | 26 +++++++------- config/filebeat.docker.yml | 16 --------- docker-compose.elk.yml.dist | 58 -------------------------------- docker-compose.filebeat.yml.dist | 42 ----------------------- docker-compose.yml.dist | 31 +++++++++-------- elk/.env.dist | 1 - elk/02-beats-input.conf | 6 ---- elk/10-syslog.conf | 15 --------- elk/16-amavis.conf | 23 ------------- elk/Dockerfile | 30 ----------------- elk/amavis.grok | 1 - elk/docker-compose.yml.dist | 16 --------- setup.sh | 9 ++--- 13 files changed, 33 insertions(+), 241 deletions(-) delete mode 100644 config/filebeat.docker.yml delete mode 100644 docker-compose.elk.yml.dist delete mode 100644 docker-compose.filebeat.yml.dist delete mode 100644 elk/.env.dist delete mode 100644 elk/02-beats-input.conf delete mode 100644 elk/10-syslog.conf delete mode 100644 elk/16-amavis.conf delete mode 100644 elk/Dockerfile delete mode 100644 elk/amavis.grok delete mode 100644 elk/docker-compose.yml.dist diff --git a/README.md b/README.md index f7f8ad6e..8fe3b127 100644 --- a/README.md +++ b/README.md @@ -179,9 +179,9 @@ version: '3.8' services: mail: image: tvial/docker-mailserver:latest - hostname: mail - domainname: domain.com - container_name: mail + hostname: mail # ${HOSTNAME} + domainname: domain.com # ${DOMAINNAME} + container_name: mail # ${CONTAINER_NAME} ports: - "25:25" - "143:143" @@ -203,14 +203,12 @@ services: cap_add: - NET_ADMIN - SYS_PTRACE + restart: always volumes: maildata: - driver: local mailstate: - driver: local maillogs: - driver: local ``` #### LDAP setup @@ -221,9 +219,9 @@ version: '3.8' services: mail: image: tvial/docker-mailserver:latest - hostname: mail - domainname: domain.com - container_name: mail + hostname: mail # ${HOSTNAME} + domainname: domain.com # ${DOMAINNAME} + container_name: mail # ${CONTAINER_NAME} ports: - "25:25" - "143:143" @@ -265,19 +263,21 @@ services: cap_add: - NET_ADMIN - SYS_PTRACE + restart: always volumes: maildata: - driver: local mailstate: - driver: local maillogs: - driver: local ``` ## Environment variables -If an option doesn't work as documented here, check if you are running the latest image! Value in **bold** is the default value. +If an option doesn't work as documented here, check if you are running the latest image! Values in **bold** are the default values. + +### Reminder + +Please note: Variables in `.env` are expanded in the `docker-compose.yml` file **only** and **not** in the container. The file `env-mailserver` serves this case where environment variables are used in the container. ### Assignments diff --git a/config/filebeat.docker.yml b/config/filebeat.docker.yml deleted file mode 100644 index cfa132ee..00000000 --- a/config/filebeat.docker.yml +++ /dev/null @@ -1,16 +0,0 @@ -filebeat.config: - modules: - path: ${path.config}/modules.d/*.yml - reload.enabled: false - -filebeat.autodiscover: - providers: - - type: docker - hints.enabled: true - hints.default_config.enabled: false - -processors: -- add_cloud_metadata: ~ - -output.logstash: - hosts: ["127.0.0.1:5044"] diff --git a/docker-compose.elk.yml.dist b/docker-compose.elk.yml.dist deleted file mode 100644 index 2621b577..00000000 --- a/docker-compose.elk.yml.dist +++ /dev/null @@ -1,58 +0,0 @@ -version: '2' - -services: - mail: - image: tvial/docker-mailserver:latest - hostname: ${HOSTNAME} - domainname: ${DOMAINNAME} - container_name: ${CONTAINER_NAME} - links: - - elk - labels: - - "co.elastic.logs/enabled=true" - - "co.elastic.logs/module=system" - - "co.elastic.logs/fileset.stdout=syslog" - ports: - - "25:25" - - "143:143" - - "587:587" - - "993:993" - volumes: - - maildata:/var/mail - - maillogs:/var/log/mail - - ./config/:/tmp/docker-mailserver/ - env_file: - - .env - - env-mailserver - cap_add: - - NET_ADMIN - - SYS_PTRACE - restart: always - filebeat: - image: docker.elastic.co/beats/filebeat:7.6.1 - user: root - volumes: - - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - - /var/lib/docker/containers/:/var/lib/docker/containers/:ro - command: ["filebeat", "-e", "--strict.perms=false"] - restart: always - elk: - build: - context: elk - args: - - MAXMIND_LICENSE - ports: - - "5601:5601" - - "9200:9200" - - "5044:5044" - - "5000:5000" - env_file: - - elk/.env - restart: always - -volumes: - maildata: - driver: local - maillogs: - driver: local diff --git a/docker-compose.filebeat.yml.dist b/docker-compose.filebeat.yml.dist deleted file mode 100644 index 5dc483c1..00000000 --- a/docker-compose.filebeat.yml.dist +++ /dev/null @@ -1,42 +0,0 @@ -version: '2' -services: - mail: - image: tvial/docker-mailserver:latest - hostname: ${HOSTNAME} - domainname: ${DOMAINNAME} - container_name: ${CONTAINER_NAME} - ports: - - "25:25" - - "143:143" - - "587:587" - - "993:993" - labels: - - "co.elastic.logs/enabled=true" - - "co.elastic.logs/module=system" - - "co.elastic.logs/fileset.stdout=syslog" - volumes: - - maildata:/var/mail - - mailstate:/var/mail-state - - maillogs:/var/log/mail - - ./config/:/tmp/docker-mailserver/ - env_file: - - .env - - env-mailserver - cap_add: - - NET_ADMIN - - SYS_PTRACE - restart: always - filebeat: - image: docker.elastic.co/beats/filebeat:7.6.1 - user: root - volumes: - - ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - - /var/lib/docker/containers/:/var/lib/docker/containers/:ro - command: ["filebeat", "-e", "--strict.perms=false"] - restart: always -volumes: - maildata: - driver: local - maillogs: - driver: local diff --git a/docker-compose.yml.dist b/docker-compose.yml.dist index 58cb4f6f..0dbaec1e 100644 --- a/docker-compose.yml.dist +++ b/docker-compose.yml.dist @@ -1,4 +1,5 @@ -version: '2' +version: '3.7' + services: mail: image: tvial/docker-mailserver:latest @@ -6,26 +7,24 @@ services: domainname: ${DOMAINNAME} container_name: ${CONTAINER_NAME} ports: - - "25:25" - - "143:143" - - "587:587" - - "993:993" + - "25:25" + - "143:143" + - "587:587" + - "993:993" volumes: - - maildata:/var/mail - - mailstate:/var/mail-state - - maillogs:/var/log/mail - - ./config/:/tmp/docker-mailserver/ + - maildata:/var/mail + - mailstate:/var/mail-state + - maillogs:/var/log/mail + - ./config/:/tmp/docker-mailserver/ env_file: - - .env - - env-mailserver + - .env + - env-mailserver cap_add: - - NET_ADMIN - - SYS_PTRACE + - NET_ADMIN + - SYS_PTRACE restart: always + volumes: maildata: - driver: local mailstate: - driver: local maillogs: - driver: local diff --git a/elk/.env.dist b/elk/.env.dist deleted file mode 100644 index 43a62e16..00000000 --- a/elk/.env.dist +++ /dev/null @@ -1 +0,0 @@ -MAXMIND_LICENSE= diff --git a/elk/02-beats-input.conf b/elk/02-beats-input.conf deleted file mode 100644 index a00d3f5b..00000000 --- a/elk/02-beats-input.conf +++ /dev/null @@ -1,6 +0,0 @@ -input { - beats { - port => 5044 - ssl => false - } -} diff --git a/elk/10-syslog.conf b/elk/10-syslog.conf deleted file mode 100644 index 17ff59e6..00000000 --- a/elk/10-syslog.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - grok { - overwrite => [ "message" ] - match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:message}" } - add_field => [ "received_at", "%{@timestamp}" ] - add_field => [ "received_from", "%{host}" ] - add_field => [ "program", "%{syslog_program}" ] - } - syslog_pri { } - date { - match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] - } -} - - diff --git a/elk/16-amavis.conf b/elk/16-amavis.conf deleted file mode 100644 index ad60eff6..00000000 --- a/elk/16-amavis.conf +++ /dev/null @@ -1,23 +0,0 @@ -filter { - # grok log lines by program name - if [program] == 'amavis' { - grok { - patterns_dir => "/etc/logstash/patterns.d" - match => [ "message", "%{AMAVIS}" ] - tag_on_failure => [ "_grok_amavis_nomatch" ] - add_tag => [ "_grok_amavis_success" ] - } - } - - # Do some data type conversions - mutate { - convert => [ - # list of integer fields - "amavis_size", "integer", - "amavis_duration", "integer", - - # list of float fields - "amavis_hits", "float" - ] - } -} diff --git a/elk/Dockerfile b/elk/Dockerfile deleted file mode 100644 index 98a35316..00000000 --- a/elk/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM sebp/elk:761 - -RUN mkdir /etc/logstash/patterns.d -#postfix grok and filter -RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/postfix.grok > /etc/logstash/patterns.d/postfix.grok -RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/50-filter-postfix.conf > /etc/logstash/conf.d/15-filter-postfix.conf -# custom amavis grok and filter -COPY amavis.grok /etc/logstash/patterns.d -COPY 16-amavis.conf /etc/logstash/conf.d -# dovecot grok and filter -RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok -RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf -# FIXME: may be a cron job? -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -ARG MAXMIND_LICENSE -RUN mkdir -p /usr/share/GeoIP && \ -curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_LICENSE}&suffix=tar.gz" \ -| tar zx --to-stdout --wildcards --no-anchored '*.mmdb' > /usr/share/GeoIP/GeoLiteCity.dat - -WORKDIR ${LOGSTASH_HOME} -RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip - -# override beats input -COPY 02-beats-input.conf /etc/logstash/conf.d/ -# override syslog -COPY 10-syslog.conf /etc/logstash/conf.d/ - -# avoid Bootstrap Checks failure on production -RUN /bin/grep -q -F 'transport.host' /etc/elasticsearch/elasticsearch.yml || echo "transport.host: 127.0.0.1" >> /etc/elasticsearch/elasticsearch.yml diff --git a/elk/amavis.grok b/elk/amavis.grok deleted file mode 100644 index 36713188..00000000 --- a/elk/amavis.grok +++ /dev/null @@ -1 +0,0 @@ -AMAVIS \(%{DATA:amavis_id}\) %{DATA:amavis_action} %{DATA:amavis_status} {%{DATA:amavis_relaytype}},( %{GREEDYDATA:amavis_policybank})? \[%{IP:remote_ip}\]:%{POSINT:remote_port} \[%{IP:amavis_ip}\] <%{DATA:from}> -> <%{DATA:to}>(, quarantine: %{DATA:quarantine_id})?, Queue-ID: %{DATA:queue_id}(, Message-ID: <%{DATA:message_id}>)?(, mail_id: %{DATA:mail_id})?, Hits: %{NUMBER:amavis_hits}, size: %{POSINT:amavis_size}(, queued_as: %{DATA:amavis_queue_id})?(, dkim_sd=%{DATA:amavis_dkim})?, %{NUMBER:amavis_duration} ms diff --git a/elk/docker-compose.yml.dist b/elk/docker-compose.yml.dist deleted file mode 100644 index 1bdfbb25..00000000 --- a/elk/docker-compose.yml.dist +++ /dev/null @@ -1,16 +0,0 @@ -version: '2' - -services: - elk: - build: - context: . - args: - - MAXMIND_LICENSE - ports: - - "5601:5601" - - "9200:9200" - - "5044:5044" - - "5000:5000" - env_file: - - .env - restart: always diff --git a/setup.sh b/setup.sh index dbc15f5f..c0c3455c 100755 --- a/setup.sh +++ b/setup.sh @@ -4,7 +4,7 @@ # included in the docker-mailserver set -euEo pipefail -trap '_report_err ${_:-"SOURCE UNKNOWN"} ${LINENO} ${?}' ERR +trap '_report_err ${_} ${LINENO} ${?}' ERR function _report_err() { @@ -321,13 +321,14 @@ function _main() _docker_container /bin/bash -c "${@}" fi ;; - * ) _usage ; exit 1 ;; + * ) _usage ; _unset_vars ; exit 1 ;; esac ;; - * ) _usage ; exit 1 ;; + * ) _usage ; _unset_vars ; exit 1 ;; esac + + _unset_vars } _main "${@}" -_unset_vars