getting rid of ELK & updating Compose (#1614)
* removed elk diretory * removed filebeat / elk compose.yml.dist; adjusted other compose files * final README update for env variables * updated compose files
This commit is contained in:
parent
fa84895ab2
commit
9b5d4d307c
26
README.md
26
README.md
|
@ -179,9 +179,9 @@ version: '3.8'
|
||||||
services:
|
services:
|
||||||
mail:
|
mail:
|
||||||
image: tvial/docker-mailserver:latest
|
image: tvial/docker-mailserver:latest
|
||||||
hostname: mail
|
hostname: mail # ${HOSTNAME}
|
||||||
domainname: domain.com
|
domainname: domain.com # ${DOMAINNAME}
|
||||||
container_name: mail
|
container_name: mail # ${CONTAINER_NAME}
|
||||||
ports:
|
ports:
|
||||||
- "25:25"
|
- "25:25"
|
||||||
- "143:143"
|
- "143:143"
|
||||||
|
@ -203,14 +203,12 @@ services:
|
||||||
cap_add:
|
cap_add:
|
||||||
- NET_ADMIN
|
- NET_ADMIN
|
||||||
- SYS_PTRACE
|
- SYS_PTRACE
|
||||||
|
restart: always
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
maildata:
|
maildata:
|
||||||
driver: local
|
|
||||||
mailstate:
|
mailstate:
|
||||||
driver: local
|
|
||||||
maillogs:
|
maillogs:
|
||||||
driver: local
|
|
||||||
```
|
```
|
||||||
|
|
||||||
#### LDAP setup
|
#### LDAP setup
|
||||||
|
@ -221,9 +219,9 @@ version: '3.8'
|
||||||
services:
|
services:
|
||||||
mail:
|
mail:
|
||||||
image: tvial/docker-mailserver:latest
|
image: tvial/docker-mailserver:latest
|
||||||
hostname: mail
|
hostname: mail # ${HOSTNAME}
|
||||||
domainname: domain.com
|
domainname: domain.com # ${DOMAINNAME}
|
||||||
container_name: mail
|
container_name: mail # ${CONTAINER_NAME}
|
||||||
ports:
|
ports:
|
||||||
- "25:25"
|
- "25:25"
|
||||||
- "143:143"
|
- "143:143"
|
||||||
|
@ -265,19 +263,21 @@ services:
|
||||||
cap_add:
|
cap_add:
|
||||||
- NET_ADMIN
|
- NET_ADMIN
|
||||||
- SYS_PTRACE
|
- SYS_PTRACE
|
||||||
|
restart: always
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
maildata:
|
maildata:
|
||||||
driver: local
|
|
||||||
mailstate:
|
mailstate:
|
||||||
driver: local
|
|
||||||
maillogs:
|
maillogs:
|
||||||
driver: local
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Environment variables
|
## Environment variables
|
||||||
|
|
||||||
If an option doesn't work as documented here, check if you are running the latest image! Value in **bold** is the default value.
|
If an option doesn't work as documented here, check if you are running the latest image! Values in **bold** are the default values.
|
||||||
|
|
||||||
|
### Reminder
|
||||||
|
|
||||||
|
Please note: Variables in `.env` are expanded in the `docker-compose.yml` file **only** and **not** in the container. The file `env-mailserver` serves this case where environment variables are used in the container.
|
||||||
|
|
||||||
### Assignments
|
### Assignments
|
||||||
|
|
||||||
|
|
|
@ -1,16 +0,0 @@
|
||||||
filebeat.config:
|
|
||||||
modules:
|
|
||||||
path: ${path.config}/modules.d/*.yml
|
|
||||||
reload.enabled: false
|
|
||||||
|
|
||||||
filebeat.autodiscover:
|
|
||||||
providers:
|
|
||||||
- type: docker
|
|
||||||
hints.enabled: true
|
|
||||||
hints.default_config.enabled: false
|
|
||||||
|
|
||||||
processors:
|
|
||||||
- add_cloud_metadata: ~
|
|
||||||
|
|
||||||
output.logstash:
|
|
||||||
hosts: ["127.0.0.1:5044"]
|
|
|
@ -1,58 +0,0 @@
|
||||||
version: '2'
|
|
||||||
|
|
||||||
services:
|
|
||||||
mail:
|
|
||||||
image: tvial/docker-mailserver:latest
|
|
||||||
hostname: ${HOSTNAME}
|
|
||||||
domainname: ${DOMAINNAME}
|
|
||||||
container_name: ${CONTAINER_NAME}
|
|
||||||
links:
|
|
||||||
- elk
|
|
||||||
labels:
|
|
||||||
- "co.elastic.logs/enabled=true"
|
|
||||||
- "co.elastic.logs/module=system"
|
|
||||||
- "co.elastic.logs/fileset.stdout=syslog"
|
|
||||||
ports:
|
|
||||||
- "25:25"
|
|
||||||
- "143:143"
|
|
||||||
- "587:587"
|
|
||||||
- "993:993"
|
|
||||||
volumes:
|
|
||||||
- maildata:/var/mail
|
|
||||||
- maillogs:/var/log/mail
|
|
||||||
- ./config/:/tmp/docker-mailserver/
|
|
||||||
env_file:
|
|
||||||
- .env
|
|
||||||
- env-mailserver
|
|
||||||
cap_add:
|
|
||||||
- NET_ADMIN
|
|
||||||
- SYS_PTRACE
|
|
||||||
restart: always
|
|
||||||
filebeat:
|
|
||||||
image: docker.elastic.co/beats/filebeat:7.6.1
|
|
||||||
user: root
|
|
||||||
volumes:
|
|
||||||
- ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
|
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
|
||||||
- /var/lib/docker/containers/:/var/lib/docker/containers/:ro
|
|
||||||
command: ["filebeat", "-e", "--strict.perms=false"]
|
|
||||||
restart: always
|
|
||||||
elk:
|
|
||||||
build:
|
|
||||||
context: elk
|
|
||||||
args:
|
|
||||||
- MAXMIND_LICENSE
|
|
||||||
ports:
|
|
||||||
- "5601:5601"
|
|
||||||
- "9200:9200"
|
|
||||||
- "5044:5044"
|
|
||||||
- "5000:5000"
|
|
||||||
env_file:
|
|
||||||
- elk/.env
|
|
||||||
restart: always
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
maildata:
|
|
||||||
driver: local
|
|
||||||
maillogs:
|
|
||||||
driver: local
|
|
|
@ -1,42 +0,0 @@
|
||||||
version: '2'
|
|
||||||
services:
|
|
||||||
mail:
|
|
||||||
image: tvial/docker-mailserver:latest
|
|
||||||
hostname: ${HOSTNAME}
|
|
||||||
domainname: ${DOMAINNAME}
|
|
||||||
container_name: ${CONTAINER_NAME}
|
|
||||||
ports:
|
|
||||||
- "25:25"
|
|
||||||
- "143:143"
|
|
||||||
- "587:587"
|
|
||||||
- "993:993"
|
|
||||||
labels:
|
|
||||||
- "co.elastic.logs/enabled=true"
|
|
||||||
- "co.elastic.logs/module=system"
|
|
||||||
- "co.elastic.logs/fileset.stdout=syslog"
|
|
||||||
volumes:
|
|
||||||
- maildata:/var/mail
|
|
||||||
- mailstate:/var/mail-state
|
|
||||||
- maillogs:/var/log/mail
|
|
||||||
- ./config/:/tmp/docker-mailserver/
|
|
||||||
env_file:
|
|
||||||
- .env
|
|
||||||
- env-mailserver
|
|
||||||
cap_add:
|
|
||||||
- NET_ADMIN
|
|
||||||
- SYS_PTRACE
|
|
||||||
restart: always
|
|
||||||
filebeat:
|
|
||||||
image: docker.elastic.co/beats/filebeat:7.6.1
|
|
||||||
user: root
|
|
||||||
volumes:
|
|
||||||
- ./config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro
|
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
|
||||||
- /var/lib/docker/containers/:/var/lib/docker/containers/:ro
|
|
||||||
command: ["filebeat", "-e", "--strict.perms=false"]
|
|
||||||
restart: always
|
|
||||||
volumes:
|
|
||||||
maildata:
|
|
||||||
driver: local
|
|
||||||
maillogs:
|
|
||||||
driver: local
|
|
|
@ -1,4 +1,5 @@
|
||||||
version: '2'
|
version: '3.7'
|
||||||
|
|
||||||
services:
|
services:
|
||||||
mail:
|
mail:
|
||||||
image: tvial/docker-mailserver:latest
|
image: tvial/docker-mailserver:latest
|
||||||
|
@ -22,10 +23,8 @@ services:
|
||||||
- NET_ADMIN
|
- NET_ADMIN
|
||||||
- SYS_PTRACE
|
- SYS_PTRACE
|
||||||
restart: always
|
restart: always
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
maildata:
|
maildata:
|
||||||
driver: local
|
|
||||||
mailstate:
|
mailstate:
|
||||||
driver: local
|
|
||||||
maillogs:
|
maillogs:
|
||||||
driver: local
|
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
MAXMIND_LICENSE=
|
|
|
@ -1,6 +0,0 @@
|
||||||
input {
|
|
||||||
beats {
|
|
||||||
port => 5044
|
|
||||||
ssl => false
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,15 +0,0 @@
|
||||||
filter {
|
|
||||||
grok {
|
|
||||||
overwrite => [ "message" ]
|
|
||||||
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:message}" }
|
|
||||||
add_field => [ "received_at", "%{@timestamp}" ]
|
|
||||||
add_field => [ "received_from", "%{host}" ]
|
|
||||||
add_field => [ "program", "%{syslog_program}" ]
|
|
||||||
}
|
|
||||||
syslog_pri { }
|
|
||||||
date {
|
|
||||||
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
|
@ -1,23 +0,0 @@
|
||||||
filter {
|
|
||||||
# grok log lines by program name
|
|
||||||
if [program] == 'amavis' {
|
|
||||||
grok {
|
|
||||||
patterns_dir => "/etc/logstash/patterns.d"
|
|
||||||
match => [ "message", "%{AMAVIS}" ]
|
|
||||||
tag_on_failure => [ "_grok_amavis_nomatch" ]
|
|
||||||
add_tag => [ "_grok_amavis_success" ]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Do some data type conversions
|
|
||||||
mutate {
|
|
||||||
convert => [
|
|
||||||
# list of integer fields
|
|
||||||
"amavis_size", "integer",
|
|
||||||
"amavis_duration", "integer",
|
|
||||||
|
|
||||||
# list of float fields
|
|
||||||
"amavis_hits", "float"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,30 +0,0 @@
|
||||||
FROM sebp/elk:761
|
|
||||||
|
|
||||||
RUN mkdir /etc/logstash/patterns.d
|
|
||||||
#postfix grok and filter
|
|
||||||
RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/postfix.grok > /etc/logstash/patterns.d/postfix.grok
|
|
||||||
RUN curl -L https://raw.githubusercontent.com/whyscream/postfix-grok-patterns/master/50-filter-postfix.conf > /etc/logstash/conf.d/15-filter-postfix.conf
|
|
||||||
# custom amavis grok and filter
|
|
||||||
COPY amavis.grok /etc/logstash/patterns.d
|
|
||||||
COPY 16-amavis.conf /etc/logstash/conf.d
|
|
||||||
# dovecot grok and filter
|
|
||||||
RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/patterns.d/dovecot.grok > /etc/logstash/patterns.d/dovecot.grok
|
|
||||||
RUN curl -L https://raw.githubusercontent.com/ninech/logstash-patterns/master/exmples/50-filter-dovecot.conf > /etc/logstash/conf.d/17-filter-dovecot.conf
|
|
||||||
# FIXME: may be a cron job?
|
|
||||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
|
||||||
|
|
||||||
ARG MAXMIND_LICENSE
|
|
||||||
RUN mkdir -p /usr/share/GeoIP && \
|
|
||||||
curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_LICENSE}&suffix=tar.gz" \
|
|
||||||
| tar zx --to-stdout --wildcards --no-anchored '*.mmdb' > /usr/share/GeoIP/GeoLiteCity.dat
|
|
||||||
|
|
||||||
WORKDIR ${LOGSTASH_HOME}
|
|
||||||
RUN gosu logstash bin/logstash-plugin install --local --no-verify logstash-filter-geoip
|
|
||||||
|
|
||||||
# override beats input
|
|
||||||
COPY 02-beats-input.conf /etc/logstash/conf.d/
|
|
||||||
# override syslog
|
|
||||||
COPY 10-syslog.conf /etc/logstash/conf.d/
|
|
||||||
|
|
||||||
# avoid Bootstrap Checks failure on production
|
|
||||||
RUN /bin/grep -q -F 'transport.host' /etc/elasticsearch/elasticsearch.yml || echo "transport.host: 127.0.0.1" >> /etc/elasticsearch/elasticsearch.yml
|
|
|
@ -1 +0,0 @@
|
||||||
AMAVIS \(%{DATA:amavis_id}\) %{DATA:amavis_action} %{DATA:amavis_status} {%{DATA:amavis_relaytype}},( %{GREEDYDATA:amavis_policybank})? \[%{IP:remote_ip}\]:%{POSINT:remote_port} \[%{IP:amavis_ip}\] <%{DATA:from}> -> <%{DATA:to}>(, quarantine: %{DATA:quarantine_id})?, Queue-ID: %{DATA:queue_id}(, Message-ID: <%{DATA:message_id}>)?(, mail_id: %{DATA:mail_id})?, Hits: %{NUMBER:amavis_hits}, size: %{POSINT:amavis_size}(, queued_as: %{DATA:amavis_queue_id})?(, dkim_sd=%{DATA:amavis_dkim})?, %{NUMBER:amavis_duration} ms
|
|
|
@ -1,16 +0,0 @@
|
||||||
version: '2'
|
|
||||||
|
|
||||||
services:
|
|
||||||
elk:
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
args:
|
|
||||||
- MAXMIND_LICENSE
|
|
||||||
ports:
|
|
||||||
- "5601:5601"
|
|
||||||
- "9200:9200"
|
|
||||||
- "5044:5044"
|
|
||||||
- "5000:5000"
|
|
||||||
env_file:
|
|
||||||
- .env
|
|
||||||
restart: always
|
|
9
setup.sh
9
setup.sh
|
@ -4,7 +4,7 @@
|
||||||
# included in the docker-mailserver
|
# included in the docker-mailserver
|
||||||
|
|
||||||
set -euEo pipefail
|
set -euEo pipefail
|
||||||
trap '_report_err ${_:-"SOURCE UNKNOWN"} ${LINENO} ${?}' ERR
|
trap '_report_err ${_} ${LINENO} ${?}' ERR
|
||||||
|
|
||||||
function _report_err()
|
function _report_err()
|
||||||
{
|
{
|
||||||
|
@ -321,13 +321,14 @@ function _main()
|
||||||
_docker_container /bin/bash -c "${@}"
|
_docker_container /bin/bash -c "${@}"
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
* ) _usage ; exit 1 ;;
|
* ) _usage ; _unset_vars ; exit 1 ;;
|
||||||
esac
|
esac
|
||||||
;;
|
;;
|
||||||
|
|
||||||
* ) _usage ; exit 1 ;;
|
* ) _usage ; _unset_vars ; exit 1 ;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
_unset_vars
|
||||||
}
|
}
|
||||||
|
|
||||||
_main "${@}"
|
_main "${@}"
|
||||||
_unset_vars
|
|
||||||
|
|
Loading…
Reference in New Issue