From 061fe12aa7bab6ed4d1e69932b9884e3153da9fd Mon Sep 17 00:00:00 2001 From: brainkiller Date: Sun, 17 Jan 2021 10:39:09 +0100 Subject: [PATCH] Solve Fetchmail imap idle issue (#10) * Migrate PR#1730 from tomav/docker-mailserver repo to new docker-mailserver/docker-mailserver repo * Resolved review comments * Moved counter increment to have consistency between fetchmail process and fetchmail config files * Added tests for new fetchmail option Co-authored-by: Georg Lauterbach <44545919+aendeavor@users.noreply.github.com> --- ENVIRONMENT.md | 9 ++- target/bin/fetchmailrc_split | 54 +++++++++++++++++ target/scripts/start-mailserver.sh | 43 ++++++++++++- test/config/fetchmail.cf | 6 ++ test/mail_fetchmail_parallel.bats | 97 ++++++++++++++++++++++++++++++ 5 files changed, 206 insertions(+), 3 deletions(-) create mode 100755 target/bin/fetchmailrc_split create mode 100644 test/mail_fetchmail_parallel.bats diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md index f64ef8b2..72852a1d 100644 --- a/ENVIRONMENT.md +++ b/ENVIRONMENT.md @@ -131,7 +131,7 @@ Set the mailbox size limit for all users. If set to zero, the size will be unlim - **1** => Dovecot quota is enabled - 0 => Dovecot quota is disabled - + See [mailbox quota](https://github.com/tomav/docker-mailserver/wiki/Configure-Accounts#mailbox-quota). ##### POSTFIX\_MESSAGE\_SIZE\_LIMIT @@ -335,6 +335,13 @@ Note: activate this only if you are confident in your bayes database for identif - **300** => `fetchmail` The number of seconds for the interval +##### FETCHMAIL_PARALLEL + + **0** => `fetchmail` runs with a single config file `/etc/fetchmailrc` + **1** => `/etc/fetchmailrc` is split per poll entry. For every poll entry a seperate fetchmail instance is started to allow having multiple imap idle configurations defined. + +Note: The defaults of your fetchmailrc file need to be at the top of the file. Otherwise it won't be added correctly to all separate `fetchmail` instances. + #### LDAP ##### ENABLE_LDAP diff --git a/target/bin/fetchmailrc_split b/target/bin/fetchmailrc_split new file mode 100755 index 00000000..9f9d1647 --- /dev/null +++ b/target/bin/fetchmailrc_split @@ -0,0 +1,54 @@ +#! /bin/bash + +# Description: This script will split the content of /etc/fetchmailrc into +# smaller fetchmailrc files per server [poll] entries. Each +# separate fetchmailrc file is stored in /etc/fetchmailrc.d +# +# The mail purpose for this is to work around what is known +# as the Fetchmail IMAP idle issue. +# + +FETCHMAILRC="/etc/fetchmailrc" +FETCHMAILRCD="/etc/fetchmailrc.d" +DEFAULT_FILE="${FETCHMAILRCD}/defaults" + +if [[ ! -r "${FETCHMAILRC}" ]] +then + echo "Error: File ${FETCHMAILRC} not found" + exit 1 +fi + +if [[ ! -d ${FETCHMAILRCD} ]] +then + if ! mkdir "${FETCHMAILRCD}" + then + echo "Error: Unable to create folder ${FETCHMAILRCD}" + exit 1 + fi +fi + +COUNTER=0 +SERVER=0 +while read -r LINE +do + if [[ ${LINE} =~ poll ]] + then + # If we read "poll" then we reached a new server definition + # We need to create a new file with fetchmail defaults from + # /etc/fetcmailrc + COUNTER=$((COUNTER+1)) + SERVER=1 + cat "${DEFAULT_FILE}" > "${FETCHMAILRCD}/fetchmail-${COUNTER}.rc" + echo "${LINE}" >> "${FETCHMAILRCD}/fetchmail-${COUNTER}.rc" + elif [[ ${SERVER} -eq 0 ]] + then + # We have not yet found "poll". Let's assume we are still reading + # the default settings from /etc/fetchmailrc file + echo "${LINE}" >> "${DEFAULT_FILE}" + else + # Just the server settings that need to be added to the specific rc.d file + echo "${LINE}" >> "${FETCHMAILRCD}/fetchmail-${COUNTER}.rc" + fi +done < "${FETCHMAILRC}" + +rm "${DEFAULT_FILE}" diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 4435e588..8b005f84 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -18,6 +18,7 @@ ENABLE_SASLAUTHD="${ENABLE_SASLAUTHD:=0}" ENABLE_SPAMASSASSIN="${ENABLE_SPAMASSASSIN:=0}" ENABLE_SRS="${ENABLE_SRS:=0}" FETCHMAIL_POLL="${FETCHMAIL_POLL:=300}" +FETCHMAIL_PARALLEL="${FETCHMAIL_PARALLEL:=0}" LDAP_START_TLS="${LDAP_START_TLS:=no}" LOGROTATE_INTERVAL="${LOGROTATE_INTERVAL:=${REPORT_INTERVAL:-daily}}" LOGWATCH_INTERVAL="${LOGWATCH_INTERVAL:=none}" @@ -396,6 +397,7 @@ function _setup_default_vars echo "ENABLE_SPAMASSASSIN=${ENABLE_SPAMASSASSIN}" echo "ENABLE_SRS=${ENABLE_SRS}" echo "FETCHMAIL_POLL=${FETCHMAIL_POLL}" + echo "FETCHMAIL_PARALLEL=${FETCHMAIL_PARALLEL}" echo "LDAP_START_TLS=${LDAP_START_TLS}" echo "LOGROTATE_INTERVAL=${LOGROTATE_INTERVAL}" echo "LOGWATCH_INTERVAL=${LOGWATCH_INTERVAL}" @@ -2068,9 +2070,46 @@ function _start_daemons_dovecot function _start_daemons_fetchmail { - _notify 'task' 'Starting fetchmail' 'n' + _notify 'task' 'Preparing fetchmail config' /usr/local/bin/setup-fetchmail - supervisorctl start fetchmail + if [[ ${FETCHMAIL_PARALLEL} -eq 1 ]] + then + mkdir /etc/fetchmailrc.d/ + /usr/local/bin/fetchmailrc_split + + COUNTER=0 + for RC in /etc/fetchmailrc.d/fetchmail-*.rc + do + COUNTER=$((COUNTER+1)) + cat < "/etc/supervisor/conf.d/fetchmail-${COUNTER}.conf" +[program:fetchmail-${COUNTER}] +startsecs=0 +autostart=false +autorestart=true +stdout_logfile=/var/log/supervisor/%(program_name)s.log +stderr_logfile=/var/log/supervisor/%(program_name)s.log +user=fetchmail +command=/usr/bin/fetchmail -f ${RC} -v --nodetach --daemon %(ENV_FETCHMAIL_POLL)s -i /var/lib/fetchmail/.fetchmail-UIDL-cache --pidfile /var/run/fetchmail/%(program_name)s.pid +EOF + chmod 700 "${RC}" + chown fetchmail:root "${RC}" + done + + supervisorctl reread + supervisorctl update + + COUNTER=0 + for _ in /etc/fetchmailrc.d/fetchmail-*.rc + do + COUNTER=$((COUNTER+1)) + _notify 'task' "Starting fetchmail instance ${COUNTER}" 'n' + supervisorctl start "fetchmail-${COUNTER}" + done + + else + _notify 'task' 'Starting fetchmail' 'n' + supervisorctl start fetchmail + fi } function _start_daemons_clamav diff --git a/test/config/fetchmail.cf b/test/config/fetchmail.cf index e4a3c414..0e6bfc40 100644 --- a/test/config/fetchmail.cf +++ b/test/config/fetchmail.cf @@ -3,3 +3,9 @@ poll pop3.example.com. with proto POP3 password 'secret' is 'user2@domain.tld' here options keep ssl + +poll pop3-2.example.com. with proto POP3 + user 'username' there with + password 'secret' + is 'user3@domain.tld' + here options keep ssl diff --git a/test/mail_fetchmail_parallel.bats b/test/mail_fetchmail_parallel.bats new file mode 100644 index 00000000..5c641de0 --- /dev/null +++ b/test/mail_fetchmail_parallel.bats @@ -0,0 +1,97 @@ +load 'test_helper/common' + +function setup() { + run_setup_file_if_necessary +} + +function teardown() { + run_teardown_file_if_necessary +} + +function setup_file() { + local PRIVATE_CONFIG + PRIVATE_CONFIG="$(duplicate_config_for_container .)" + docker run -d --name mail_fetchmail_parallel \ + -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ + -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ + -e ENABLE_FETCHMAIL=1 \ + -e FETCHMAIL_PARALLEL=1 \ + --cap-add=NET_ADMIN \ + -e DMS_DEBUG=0 \ + -h mail.my-domain.com -t "${NAME}" + wait_for_finished_setup_in_container mail_fetchmail_parallel +} + +function teardown_file() { + docker rm -f mail_fetchmail_parallel +} + +@test "first" { + skip 'this test must come first to reliably identify when to run setup_file' +} + +# +# processes +# + +@test "checking process: fetchmail 1 (fetchmail server enabled)" { + run docker exec mail_fetchmail_parallel /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/bin/fetchmail -f /etc/fetchmailrc.d/fetchmail-1.rc'" + assert_success +} + +@test "checking process: fetchmail 2 (fetchmail server enabled)" { + run docker exec mail_fetchmail_parallel /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/bin/fetchmail -f /etc/fetchmailrc.d/fetchmail-2.rc'" + assert_success +} + +# +# fetchmail +# + +@test "checking fetchmail: gerneral options in fetchmail-1.rc are loaded" { + run docker exec mail_fetchmail_parallel grep 'set syslog' /etc/fetchmailrc.d/fetchmail-1.rc + assert_success +} + +@test "checking fetchmail: gerneral options in fetchmail-2.rc are loaded" { + run docker exec mail_fetchmail_parallel grep 'set syslog' /etc/fetchmailrc.d/fetchmail-2.rc + assert_success +} + +@test "checking fetchmail: fetchmail-1.rc is loaded with pop3.example.com" { + run docker exec mail_fetchmail_parallel grep 'pop3.example.com' /etc/fetchmailrc.d/fetchmail-1.rc + assert_success +} + +@test "checking fetchmail: fetchmail-1.rc is loaded without pop3-2.example.com" { + run docker exec mail_fetchmail_parallel grep 'pop3-2.example.com' /etc/fetchmailrc.d/fetchmail-1.rc + assert_failure +} + +@test "checking fetchmail: fetchmail-2.rc is loaded without pop3.example.com" { + run docker exec mail_fetchmail_parallel grep 'pop3.example.com' /etc/fetchmailrc.d/fetchmail-2.rc + assert_failure +} + +@test "checking fetchmail: fetchmail-2.rc is loaded with pop3-2.example.com" { + run docker exec mail_fetchmail_parallel grep 'pop3-2.example.com' /etc/fetchmailrc.d/fetchmail-2.rc + assert_success +} + +# +# supervisor +# + +@test "checking restart of process: fetchmail-1" { + run docker exec mail_fetchmail_parallel /bin/bash -c "pkill fetchmail && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/bin/fetchmail -f /etc/fetchmailrc.d/fetchmail-1.rc'" + assert_success +} + +@test "checking restart of process: fetchmail-2" { + run docker exec mail_fetchmail_parallel /bin/bash -c "pkill fetchmail && sleep 10 && ps aux --forest | grep -v grep | grep '/usr/bin/fetchmail -f /etc/fetchmailrc.d/fetchmail-2.rc'" + assert_success +} + +@test "last" { + skip 'this test is only there to reliably mark the end for the teardown_file' +}