From 7af7546d88375a67c38d8d78eafe87a00ec9563e Mon Sep 17 00:00:00 2001 From: LucidityCrash <49124523+LucidityCrash@users.noreply.github.com> Date: Tue, 23 May 2023 16:25:08 +0100 Subject: [PATCH] feature: adding `getmail` as an alternative to `fetchmail` (#2803) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Co-authored-by: Casper --- Dockerfile | 7 +- README.md | 1 + docs/content/config/advanced/mail-getmail.md | 101 +++++++++++++++++++ docs/content/config/environment.md | 12 +++ docs/mkdocs.yml | 3 + mailserver.env | 9 ++ target/bin/debug-getmail | 21 ++++ target/bin/getmail-cron | 9 ++ target/getmail/getmailrc | 7 ++ target/scripts/build/packages.sh | 16 +++ target/scripts/start-mailserver.sh | 2 + target/scripts/startup/setup.d/getmail.sh | 42 ++++++++ target/scripts/startup/setup.d/mail_state.sh | 1 + target/scripts/startup/variables-stack.sh | 2 + test/config/getmail/getmail-user3.cf | 11 ++ test/tests/parallel/set1/getmail.bats | 79 +++++++++++++++ 16 files changed, 320 insertions(+), 3 deletions(-) create mode 100644 docs/content/config/advanced/mail-getmail.md create mode 100644 target/bin/debug-getmail create mode 100644 target/bin/getmail-cron create mode 100644 target/getmail/getmailrc create mode 100644 target/scripts/startup/setup.d/getmail.sh create mode 100644 test/config/getmail/getmail-user3.cf create mode 100644 test/tests/parallel/set1/getmail.bats diff --git a/Dockerfile b/Dockerfile index 1af200de..018c39c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -158,15 +158,16 @@ COPY target/opendmarc/opendmarc.conf /etc/opendmarc.conf COPY target/opendmarc/default-opendmarc /etc/default/opendmarc COPY target/opendmarc/ignore.hosts /etc/opendmarc/ignore.hosts -# ----------------------------------------------- -# --- Fetchmail, Postfix & Let'sEncrypt --------- -# ----------------------------------------------- +# -------------------------------------------------- +# --- Fetchmail, Getmail, Postfix & Let'sEncrypt --- +# -------------------------------------------------- # Remove invalid URL from SPF message # https://bugs.launchpad.net/spf-engine/+bug/1896912 RUN echo 'Reason_Message = Message {rejectdefer} due to: {spf}.' >>/etc/postfix-policyd-spf-python/policyd-spf.conf COPY target/fetchmail/fetchmailrc /etc/fetchmailrc_general +COPY target/getmail/getmailrc /etc/getmailrc_general COPY target/postfix/main.cf target/postfix/master.cf /etc/postfix/ # DH parameters for DHE cipher suites, ffdhe4096 is the official standard 4096-bit DH params now part of TLS 1.3 diff --git a/README.md b/README.md index 1115249f..b3072cd9 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ If you have issues, please search through [the documentation][documentation::web - [OpenDKIM](http://www.opendkim.org) & [OpenDMARC](https://github.com/trusteddomainproject/OpenDMARC) - [Fail2ban](https://www.fail2ban.org/wiki/index.php/Main_Page) - [Fetchmail](http://www.fetchmail.info/fetchmail-man.html) +- [Getmail6](https://getmail6.org/documentation.html) - [Postscreen](http://www.postfix.org/POSTSCREEN_README.html) - [Postgrey](https://postgrey.schweikert.ch/) - Support for [LetsEncrypt](https://letsencrypt.org/), manual and self-signed certificates diff --git a/docs/content/config/advanced/mail-getmail.md b/docs/content/config/advanced/mail-getmail.md new file mode 100644 index 00000000..d08f5d07 --- /dev/null +++ b/docs/content/config/advanced/mail-getmail.md @@ -0,0 +1,101 @@ +--- +title: 'Advanced | Email Gathering with Getmail' +--- + +To enable the [getmail][getmail-website] service to retrieve e-mails set the environment variable `ENABLE_GETMAIL` to `1`. Your `compose.yaml` file should include the following: + +```yaml +environment: + - ENABLE_GETMAIL=1 + - GETMAIL_POLL=5 +``` + +In your DMS config volume (eg: `docker-data/dms/config/`), create a `getmail-.cf` file for each remote account that you want to retrieve mail and store into a local DMS account. `` should be replaced by you, and is just the rest of the filename (eg: `getmail-example.cf`). The contents of each file should be configuration like documented below. + +The directory structure should similar to this: + +```txt +├── docker-data/dms/config +│   ├── dovecot.cf +│   ├── getmail-example.cf +│   ├── postfix-accounts.cf +│   └── postfix-virtual.cf +├── docker-compose.yml +└── README.md +``` + +## Configuration + +A detailed description of the configuration options can be found in the [online version of the manual page][getmail-docs]. + +### Common Options + +The default options added to each `getmail` config are: + +```getmailrc +[options] +verbose = 0 +read_all = false +delete = false +max_messages_per_session = 500 +received = false +delivered_to = false +``` + +If you want to use a different base config, mount a file to `/etc/getmailrc_general`. This file will replace the default "Common Options" base config above, that all `getmail-.cf` files will extend with their configs when used. + +??? example "IMAP Configuration" + + This example will: + + 1. Connect to the remote IMAP server from Gmail. + 2. Retrieve mail from the gmail account `alice` with password `notsecure`. + 3. Store any mail retrieved from the remote mail-server into DMS for the `user1@example.com` account that DMS manages. + + ```getmailrc + [retriever] + type = SimpleIMAPRetriever + server = imap.gmail.com + username = alice + password = notsecure + [destination] + type = MDA_external + path = /usr/lib/dovecot/deliver + allow_root_commands = true + arguments =("-d","user1@example.com") + ``` + +??? example "POP3 Configuration" + + Just like the IMAP example above, but instead via POP3 protocol if you prefer that over IMAP. + + ```getmailrc + [retriever] + type = SimplePOP3Retriever + server = pop3.gmail.com + username = alice + password = notsecure + [destination] + type = MDA_external + path = /usr/lib/dovecot/deliver + allow_root_commands = true + arguments =("-d","user1@example.com") + ``` + +### Polling Interval + +By default the `getmail` service checks external mail accounts for new mail every 5 minutes. That polling interval is configurable via the `GETMAIL_POLL` ENV variable, with a value in minutes (_default: 5, min: 1, max: 30_): + +```yaml +environment: + - GETMAIL_POLL=1 +``` + +### XOAUTH2 Authentication + +It is possible to utilize the `getmail-gmail-xoauth-tokens` helper to provide authentication using `xoauth2` for [gmail (example 12)][getmail-docs-xoauth-12] or [Microsoft Office 365 (example 13)][getmail-docs-xoauth-13] + +[getmail-website]: https://www.getmail.org +[getmail-docs]: https://getmail6.org/configuration.html +[getmail-docs-xoauth-12]: https://github.com/getmail6/getmail6/blob/1f95606156231f1e074ba62a9baa64f892a92ef8/docs/getmailrc-examples#L286 +[getmail-docs-xoauth-13]: https://github.com/getmail6/getmail6/blob/1f95606156231f1e074ba62a9baa64f892a92ef8/docs/getmailrc-examples#L351 diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 3d204cce..64c0b0d0 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -548,6 +548,18 @@ Note: activate this only if you are confident in your bayes database for identif **1** => `/etc/fetchmailrc` is split per poll entry. For every poll entry a separate fetchmail instance is started to allow having multiple imap idle configurations defined. Note: The defaults of your fetchmailrc file need to be at the top of the file. Otherwise it won't be added correctly to all separate `fetchmail` instances. +#### Getmail + +##### ENABLE_GETMAIL + +Enable or disable `getmail`. + +- **0** => Disabled +- 1 => Enabled + +##### GETMAIL_POLL + +- **5** => `getmail` The number of minutes for the interval. Min: 1; Max: 30; Default: 5. #### LDAP diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index b4bef9bf..cef4a082 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -104,6 +104,8 @@ markdown_extensions: lang: shell-session - name: fetchmailrc lang: txt + - name: getmailrc + lang: txt - name: caddyfile lang: txt @@ -142,6 +144,7 @@ nav: - 'LDAP Authentication': config/advanced/auth-ldap.md - 'Email Filtering with Sieve': config/advanced/mail-sieve.md - 'Email Gathering with Fetchmail': config/advanced/mail-fetchmail.md + - 'Email Gathering with Getmail': config/advanced/mail-getmail.md - 'Email Forwarding': - 'Relay Hosts': config/advanced/mail-forwarding/relay-hosts.md - 'AWS SES': config/advanced/mail-forwarding/aws-ses.md diff --git a/mailserver.env b/mailserver.env index 7c1af20f..c857aff9 100644 --- a/mailserver.env +++ b/mailserver.env @@ -382,6 +382,15 @@ ENABLE_FETCHMAIL=0 # The interval to fetch mail in seconds FETCHMAIL_POLL=300 +# Enable or disable `getmail`. +# +# - **0** => Disabled +# - 1 => Enabled +ENABLE_GETMAIL=0 + +# The number of minutes for the interval. Min: 1; Max: 30. +GETMAIL_POLL=5 + # ----------------------------------------------- # --- LDAP Section ------------------------------ # ----------------------------------------------- diff --git a/target/bin/debug-getmail b/target/bin/debug-getmail new file mode 100644 index 00000000..404855b1 --- /dev/null +++ b/target/bin/debug-getmail @@ -0,0 +1,21 @@ +#! /bin/bash + +# shellcheck source=../scripts/helpers/log.sh +source /usr/local/bin/helpers/log.sh +# shellcheck source=../scripts/startup/setup-stack.sh +source /usr/local/bin/setup.d/getmail.sh + +_setup_getmail + +if [[ -d /var/lib/getmail ]] +then + GETMAILDIR=/var/lib/getmail +else + mkdir -p /tmp/docker-mailserver/getmail + GETMAILDIR=/tmp/docker-mailserver/getmail +fi + +for FILE in /etc/getmailrc.d/getmailrc* +do + /usr/local/bin/getmail --getmaildir "${GETMAILDIR}" --rcfile "${FILE}" --dump | tail -n +7 +done diff --git a/target/bin/getmail-cron b/target/bin/getmail-cron new file mode 100644 index 00000000..04710d1b --- /dev/null +++ b/target/bin/getmail-cron @@ -0,0 +1,9 @@ +#! /bin/bash + +for FILE in /etc/getmailrc.d/getmailrc* +do + if ! pgrep -f "${FILE}$" &>/dev/null + then + /usr/local/bin/getmail --getmaildir /var/lib/getmail --rcfile "${FILE}" + fi +done diff --git a/target/getmail/getmailrc b/target/getmail/getmailrc new file mode 100644 index 00000000..57c2b4a9 --- /dev/null +++ b/target/getmail/getmailrc @@ -0,0 +1,7 @@ +[options] +verbose = 0 +read_all = false +delete = false +max_messages_per_session = 500 +received = false +delivered_to = false diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 74dc5ed2..5a5dc47c 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -201,6 +201,21 @@ function _install_fail2ban sedfile -i -r 's/^_nft_add_set = .+/_nft_add_set = add set \\{ type \\; flags interval\\; \\}/' /etc/fail2ban/action.d/nftables.conf } +# Presently the getmail6 package is v6.14, which is too old. +# v6.18 contains fixes for Google and Microsoft OAuth support. +# using pip to install getmail. +# TODO This can be removed when the base image is updated to Debian 12 (Bookworm) +function _install_getmail +{ + _log 'debug' 'Installing getmail6' + apt-get "${QUIET}" --no-install-recommends install python3-pip + pip3 install --no-cache-dir 'getmail6~=6.18.12' + ln -s /usr/local/bin/getmail /usr/bin/getmail + ln -s /usr/local/bin/getmail-gmail-xoauth-tokens /usr/bin/getmail-gmail-xoauth-tokens + apt-get "${QUIET}" purge python3-pip + apt-get "${QUIET}" autoremove +} + function _remove_data_after_package_installations { _log 'debug' 'Deleting sensitive files (secrets)' @@ -225,5 +240,6 @@ _install_packages _install_dovecot _install_rspamd _install_fail2ban +_install_getmail _remove_data_after_package_installations _post_installation_steps diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 3532311f..39f0db2b 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -97,6 +97,8 @@ function _register_functions # needs to come after _setup_postfix_early _register_setup_function '_setup_spoof_protection' +_register_setup_function '_setup_getmail' + if [[ ${ENABLE_SRS} -eq 1 ]] then _register_setup_function '_setup_SRS' diff --git a/target/scripts/startup/setup.d/getmail.sh b/target/scripts/startup/setup.d/getmail.sh new file mode 100644 index 00000000..c0cd4df8 --- /dev/null +++ b/target/scripts/startup/setup.d/getmail.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +function _setup_getmail +{ + if [[ ${ENABLE_GETMAIL} -eq 1 ]] + then + _log 'trace' 'Preparing Getmail configuration' + + local GETMAILRC ID CONFIGS + + GETMAILRC='/etc/getmailrc.d' + CONFIGS=0 + + mkdir -p "${GETMAILRC}" + + # Generate getmailrc configs, starting with the `/etc/getmailrc_general` base config, + # Add a unique `message_log` config, then append users own config to the end. + for FILE in /tmp/docker-mailserver/getmail-*.cf + do + if [[ -f ${FILE} ]] + then + CONFIGS=1 + ID=$(cut -d '-' -f 3 <<< "${FILE}" | cut -d '.' -f 1) + local GETMAIL_CONFIG="${GETMAILRC}/getmailrc-${ID}" + cat /etc/getmailrc_general >"${GETMAIL_CONFIG}.tmp" + echo -e "message_log = /var/log/mail/getmail-${ID}.log\n" >>"${GETMAIL_CONFIG}.tmp" + cat "${GETMAIL_CONFIG}.tmp" "${FILE}" >"${GETMAIL_CONFIG}" + rm "${GETMAIL_CONFIG}.tmp" + fi + done + + if [[ ${CONFIGS} -eq 1 ]] + then + cat >/etc/cron.d/getmail << EOF +*/${GETMAIL_POLL} * * * * root /usr/local/bin/getmail-cron +EOF + chmod -R 600 "${GETMAILRC}" + fi + else + _log 'debug' 'Getmail is disabled' + fi +} diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index b96a457c..161d06e6 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -25,6 +25,7 @@ function _setup_save_states [[ ${ENABLE_CLAMAV} -eq 1 ]] && SERVICEDIRS+=('lib/clamav') [[ ${ENABLE_FAIL2BAN} -eq 1 ]] && SERVICEDIRS+=('lib/fail2ban') [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/fetchmail') + [[ ${ENABLE_GETMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/getmail') [[ ${ENABLE_POSTGREY} -eq 1 ]] && SERVICEDIRS+=('lib/postgrey') [[ ${ENABLE_RSPAMD} -eq 1 ]] && SERVICEDIRS+=('lib/rspamd') [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && SERVICEDIRS+=('lib/redis') diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 0746c55c..f5e66a99 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -74,6 +74,7 @@ function __environment_variables_general_setup VARS[ENABLE_DNSBL]="${ENABLE_DNSBL:=0}" VARS[ENABLE_FAIL2BAN]="${ENABLE_FAIL2BAN:=0}" VARS[ENABLE_FETCHMAIL]="${ENABLE_FETCHMAIL:=0}" + VARS[ENABLE_GETMAIL]="${ENABLE_GETMAIL:=0}" VARS[ENABLE_MANAGESIEVE]="${ENABLE_MANAGESIEVE:=0}" VARS[ENABLE_OPENDKIM]="${ENABLE_OPENDKIM:=1}" VARS[ENABLE_OPENDMARC]="${ENABLE_OPENDMARC:=1}" @@ -125,6 +126,7 @@ function __environment_variables_general_setup VARS[ACCOUNT_PROVISIONER]="${ACCOUNT_PROVISIONER:=FILE}" VARS[FETCHMAIL_PARALLEL]="${FETCHMAIL_PARALLEL:=0}" VARS[FETCHMAIL_POLL]="${FETCHMAIL_POLL:=300}" + VARS[GETMAIL_POLL]="${GETMAIL_POLL:=5}" VARS[LOG_LEVEL]="${LOG_LEVEL:=info}" VARS[LOGROTATE_INTERVAL]="${LOGROTATE_INTERVAL:=weekly}" VARS[LOGWATCH_INTERVAL]="${LOGWATCH_INTERVAL:=none}" diff --git a/test/config/getmail/getmail-user3.cf b/test/config/getmail/getmail-user3.cf new file mode 100644 index 00000000..cad73c98 --- /dev/null +++ b/test/config/getmail/getmail-user3.cf @@ -0,0 +1,11 @@ +[retriever] +type = SimpleIMAPSSLRetriever +server = imap.remote-service.test +username = user3 +password=secret + +[destination] +type = MDA_external +path = /usr/lib/dovecot/deliver +allow_root_commands = true +arguments =("-d","user3@example.test") diff --git a/test/tests/parallel/set1/getmail.bats b/test/tests/parallel/set1/getmail.bats new file mode 100644 index 00000000..4b5c528a --- /dev/null +++ b/test/tests/parallel/set1/getmail.bats @@ -0,0 +1,79 @@ +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +BATS_TEST_NAME_PREFIX='[Getmail] ' +CONTAINER_NAME='dms-test_getmail' + +function setup_file() { + + local CUSTOM_SETUP_ARGUMENTS=(--env 'ENABLE_GETMAIL=1') + _init_with_defaults + mv "${TEST_TMP_CONFIG}/getmail/getmail-user3.cf" "${TEST_TMP_CONFIG}/getmail-user3.cf" + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' +} + +function teardown_file() { _default_teardown ; } + +@test 'default configuration exists and is correct' { + _run_in_container cat /etc/getmailrc_general + assert_success + assert_line '[options]' + assert_line 'verbose = 0' + assert_line 'read_all = false' + assert_line 'delete = false' + assert_line 'max_messages_per_session = 500' + assert_line 'received = false' + assert_line 'delivered_to = false' + + _run_in_container stat /usr/local/bin/debug-getmail + assert_success + _run_in_container stat /usr/local/bin/getmail-cron + assert_success +} + +@test 'debug-getmail works as expected' { + _run_in_container cat /etc/getmailrc.d/getmailrc-user3 + assert_success + assert_line '[options]' + assert_line 'verbose = 0' + assert_line 'read_all = false' + assert_line 'delete = false' + assert_line 'max_messages_per_session = 500' + assert_line 'received = false' + assert_line 'delivered_to = false' + assert_line 'message_log = /var/log/mail/getmail-user3.log' + assert_line '[retriever]' + assert_line 'type = SimpleIMAPSSLRetriever' + assert_line 'server = imap.remote-service.test' + assert_line 'username = user3' + assert_line 'password=secret' + assert_line '[destination]' + assert_line 'type = MDA_external' + assert_line 'path = /usr/lib/dovecot/deliver' + assert_line 'allow_root_commands = true' + assert_line 'arguments =("-d","user3@example.test")' + + _run_in_container /usr/local/bin/debug-getmail + assert_success + assert_line --regexp 'retriever:.*SimpleIMAPSSLRetriever\(ca_certs="None", certfile="None", getmaildir="\/tmp\/docker-mailserver\/getmail", imap_on_delete="None", imap_search="None", keyfile="None", mailboxes="\(.*INBOX.*\)", move_on_delete="None", password="\*", password_command="\(\)", port="993", record_mailbox="True", server="imap.remote-service.test", ssl_cert_hostname="None", ssl_ciphers="None", ssl_fingerprints="\(\)", ssl_version="None", timeout="180", use_cram_md5="False", use_kerberos="False", use_peek="True", use_xoauth2="False", username="user3"\)' + assert_line --regexp 'destination:.*MDA_external\(allow_root_commands="True", arguments="\(.*-d.*user3@example.test.*\)", command="deliver", group="None", ignore_stderr="False", path="\/usr\/lib\/dovecot\/deliver", pipe_stdout="True", unixfrom="False", user="None"\)' + assert_line ' delete : False' + assert_line ' delete_after : 0' + assert_line ' delete_bigger_than : 0' + assert_line ' delivered_to : False' + assert_line ' fingerprint : False' + assert_line ' logfile : logfile(filename="/var/log/mail/getmail-user3.log")' + assert_line ' max_bytes_per_session : 0' + assert_line ' max_message_size : 0' + assert_line ' max_messages_per_session : 500' + assert_line ' message_log : /var/log/mail/getmail-user3.log' + assert_line ' message_log_syslog : False' + assert_line ' message_log_verbose : False' + assert_line ' netrc_file : None' + assert_line ' read_all : False' + assert_line ' received : False' + assert_line ' skip_imap_fetch_size : False' + assert_line ' to_oldmail_on_each_mail : False' + assert_line ' use_netrc : False' + assert_line ' verbose : 0' +}