getting rid of ELK & updating Compose (#1614)

* removed elk diretory
* removed filebeat / elk compose.yml.dist; adjusted other compose files
* final README update for env variables
* updated compose files
This commit is contained in:
Georg Lauterbach 2020-09-21 13:54:31 +02:00 committed by GitHub
parent fa84895ab2
commit 9b5d4d307c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 33 additions and 241 deletions

View File

@ -179,9 +179,9 @@ version: '3.8'
services:
mail:
image: tvial/docker-mailserver:latest
hostname: mail
domainname: domain.com
container_name: mail
hostname: mail # ${HOSTNAME}
domainname: domain.com # ${DOMAINNAME}
container_name: mail # ${CONTAINER_NAME}
ports:
- "25:25"
- "143:143"
@ -203,14 +203,12 @@ services:
cap_add:
- NET_ADMIN
- SYS_PTRACE
restart: always
volumes:
maildata:
driver: local
mailstate:
driver: local
maillogs:
driver: local
```
#### LDAP setup
@ -221,9 +219,9 @@ version: '3.8'
services:
mail:
image: tvial/docker-mailserver:latest
hostname: mail
domainname: domain.com
container_name: mail
hostname: mail # ${HOSTNAME}
domainname: domain.com # ${DOMAINNAME}
container_name: mail # ${CONTAINER_NAME}
ports:
- "25:25"
- "143:143"
@ -265,19 +263,21 @@ services:
cap_add:
- NET_ADMIN
- SYS_PTRACE
restart: always
volumes:
maildata:
driver: local
mailstate:
driver: local
maillogs:
driver: local
```
## Environment variables
If an option doesn't work as documented here, check if you are running the latest image! Value in **bold** is the default value.
If an option doesn't work as documented here, check if you are running the latest image! Values in **bold** are the default values.
### Reminder
Please note: Variables in `.env` are expanded in the `docker-compose.yml` file **only** and **not** in the container. The file `env-mailserver` serves this case where environment variables are used in the container.
### Assignments

View File

@ -1,16 +0,0 @@
filebeat.config:
modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
filebeat.autodiscover:
providers:
- type: docker
hints.enabled: true
hints.default_config.enabled: false
processors:
- add_cloud_metadata: ~
output.logstash:
hosts: ["127.0.0.1:5044"]

View File

@ -1,58 +0,0 @@
version: '2'
services:
mail:
image: tvial/docker-mailserver:latest
hostname: ${HOSTNAME}
domainname: ${DOMAINNAME}
container_name: ${CONTAINER_NAME}
links:
- elk
labels:
- "co.elastic.logs/enabled=true"
- "co.elastic.logs/module=system"
- "co.elastic.logs/fileset.stdout=syslog"
ports:
- "25:25"
- "143:143"
- "587:587"
- "993:993"
volumes:
- maildata:/var/mail
- maillogs:/var/log/mail
- ./config/:/tmp/docker-mailserver/
env_file:
- .env
- env-mailserver
cap_add:
- NET_ADMIN
- SYS_PTRACE
restart: always
filebeat:
image: docker.elastic.co/beats/filebeat:7.6.1
user: root
volumes:
- ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
- /var/lib/docker/containers/:/var/lib/docker/containers/:ro
command: ["filebeat", "-e", "--strict.perms=false"]
restart: always
elk:
build:
context: elk
args:
- MAXMIND_LICENSE
ports:
- "5601:5601"
- "9200:9200"
- "5044:5044"
- "5000:5000"
env_file:
- elk/.env
restart: always
volumes:
maildata:
driver: local
maillogs:
driver: local

View File

@ -1,42 +0,0 @@
version: '2'
services:
mail:
image: tvial/docker-mailserver:latest
hostname: ${HOSTNAME}
domainname: ${DOMAINNAME}
container_name: ${CONTAINER_NAME}
ports:
- "25:25"
- "143:143"
- "587:587"
- "993:993"
labels:
- "co.elastic.logs/enabled=true"
- "co.elastic.logs/module=system"
- "co.elastic.logs/fileset.stdout=syslog"
volumes:
- maildata:/var/mail
- mailstate:/var/mail-state
- maillogs:/var/log/mail
- ./config/:/tmp/docker-mailserver/
env_file:
- .env
- env-mailserver
cap_add:
- NET_ADMIN
- SYS_PTRACE
restart: always
filebeat:
image: docker.elastic.co/beats/filebeat:7.6.1
user: root
volumes:
- ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
- /var/run/docker.sock:/var/run/docker.sock:ro
- /var/lib/docker/containers/:/var/lib/docker/containers/:ro
command: ["filebeat", "-e", "--strict.perms=false"]
restart: always
volumes:
maildata:
driver: local
maillogs:
driver: local

View File

@ -1,4 +1,5 @@
version: '2'
version: '3.7'
services:
mail:
image: tvial/docker-mailserver:latest
@ -6,26 +7,24 @@ services:
domainname: ${DOMAINNAME}
container_name: ${CONTAINER_NAME}
ports:
- "25:25"
- "143:143"
- "587:587"
- "993:993"
- "25:25"
- "143:143"
- "587:587"
- "993:993"
volumes:
- maildata:/var/mail
- mailstate:/var/mail-state
- maillogs:/var/log/mail
- ./config/:/tmp/docker-mailserver/
- maildata:/var/mail
- mailstate:/var/mail-state
- maillogs:/var/log/mail
- ./config/:/tmp/docker-mailserver/
env_file:
- .env
- env-mailserver
- .env
- env-mailserver
cap_add:
- NET_ADMIN
- SYS_PTRACE
- NET_ADMIN
- SYS_PTRACE
restart: always
volumes:
maildata:
driver: local
mailstate:
driver: local
maillogs:
driver: local

View File

@ -1 +0,0 @@
MAXMIND_LICENSE=

View File

@ -1,6 +0,0 @@
input {
beats {
port => 5044
ssl => false
}
}

View File

@ -1,15 +0,0 @@
filter {
grok {
overwrite => [ "message" ]
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:message}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
add_field => [ "program", "%{syslog_program}" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}

View File

@ -1,23 +0,0 @@
filter {
# grok log lines by program name
if [program] == 'amavis' {
grok {
patterns_dir => "/etc/logstash/patterns.d"
match => [ "message", "%{AMAVIS}" ]
tag_on_failure => [ "_grok_amavis_nomatch" ]
add_tag => [ "_grok_amavis_success" ]
}
}
# Do some data type conversions
mutate {
convert => [
# list of integer fields
"amavis_size", "integer",
"amavis_duration", "integer",
# list of float fields
"amavis_hits", "float"
]
}
}

View File

@ -1,30 +0,0 @@
FROM sebp/elk:761
RUN mkdir /etc/logstash/patterns.d
#postfix grok and filter
RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/postfix.grok > /etc/logstash/patterns.d/postfix.grok
RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/50-filter-postfix.conf > /etc/logstash/conf.d/15-filter-postfix.conf
# custom amavis grok and filter
COPY amavis.grok /etc/logstash/patterns.d
COPY 16-amavis.conf /etc/logstash/conf.d
# dovecot grok and filter
RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok
RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf
# FIXME: may be a cron job?
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
ARG MAXMIND_LICENSE
RUN mkdir -p /usr/share/GeoIP && \
curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_LICENSE}&suffix=tar.gz" \
| tar zx --to-stdout --wildcards --no-anchored '*.mmdb' > /usr/share/GeoIP/GeoLiteCity.dat
WORKDIR ${LOGSTASH_HOME}
RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip
# override beats input
COPY 02-beats-input.conf /etc/logstash/conf.d/
# override syslog
COPY 10-syslog.conf /etc/logstash/conf.d/
# avoid Bootstrap Checks failure on production
RUN /bin/grep -q -F 'transport.host' /etc/elasticsearch/elasticsearch.yml || echo "transport.host: 127.0.0.1" >> /etc/elasticsearch/elasticsearch.yml

View File

@ -1 +0,0 @@
AMAVIS \(%{DATA:amavis_id}\) %{DATA:amavis_action} %{DATA:amavis_status} {%{DATA:amavis_relaytype}},( %{GREEDYDATA:amavis_policybank})? \[%{IP:remote_ip}\]:%{POSINT:remote_port} \[%{IP:amavis_ip}\] <%{DATA:from}> -> <%{DATA:to}>(, quarantine: %{DATA:quarantine_id})?, Queue-ID: %{DATA:queue_id}(, Message-ID: <%{DATA:message_id}>)?(, mail_id: %{DATA:mail_id})?, Hits: %{NUMBER:amavis_hits}, size: %{POSINT:amavis_size}(, queued_as: %{DATA:amavis_queue_id})?(, dkim_sd=%{DATA:amavis_dkim})?, %{NUMBER:amavis_duration} ms

View File

@ -1,16 +0,0 @@
version: '2'
services:
elk:
build:
context: .
args:
- MAXMIND_LICENSE
ports:
- "5601:5601"
- "9200:9200"
- "5044:5044"
- "5000:5000"
env_file:
- .env
restart: always

View File

@ -4,7 +4,7 @@
# included in the docker-mailserver
set -euEo pipefail
trap '_report_err ${_:-"SOURCE UNKNOWN"} ${LINENO} ${?}' ERR
trap '_report_err ${_} ${LINENO} ${?}' ERR
function _report_err()
{
@ -321,13 +321,14 @@ function _main()
_docker_container /bin/bash -c "${@}"
fi
;;
* ) _usage ; exit 1 ;;
* ) _usage ; _unset_vars ; exit 1 ;;
esac
;;
* ) _usage ; exit 1 ;;
* ) _usage ; _unset_vars ; exit 1 ;;
esac
_unset_vars
}
_main "${@}"
_unset_vars