From 5908d9f060b1c74d7f168a578d11869ff15f7ad4 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:33:38 +1300 Subject: [PATCH] tests(refactor): Dovecot quotas (#3068) * chore: Extract out Dovecot Quota test cases into new test file Test cases are just cut + paste, no logic changed there yet. * chore: Rename test case descriptions * chore: Use `setup ...` methods instead of direct calls * chore: Adjust `_run_in_container_bash` to `_run_in_container` Plus some additional bug fixes in the disabled test case * tests(refactor): Revise ENV test cases for max mailbox and message sizes * tests(refactor): Revise ENV test cases for mailbox and message limits v2 Removes the extra variables and filtering in favour of explicit values instead of matching for comparison. - Easier at a glance to know what is actually expected. - Additionally reworks the quota limit checks in other test cases. Using a different formatter for `doveadm` is easier to match the desired value (`Limit`). * chore: Sync improvement from `tests.bats` master --- NOTE: This PR has been merged to avoid additional maintenance burden without losing the improvements. It was not considered complete, but remaining tasks were not documented in the PR. --- docs/content/config/environment.md | 8 +- mailserver.env | 4 +- target/bin/setquota | 6 +- .../parallel/set1/dovecot/dovecot_quotas.bats | 246 ++++++++++++++++++ test/tests/serial/tests.bats | 193 -------------- 5 files changed, 257 insertions(+), 200 deletions(-) create mode 100644 test/tests/parallel/set1/dovecot/dovecot_quotas.bats diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 5ef3b968..71807c87 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -228,9 +228,9 @@ Provide any valid URI. Examples: - `lmtps:inet::` (secure lmtp with starttls) - `lmtp::2003` (use kopano as mailstore) -##### POSTFIX\_MAILBOX\_SIZE\_LIMIT +##### POSTFIX_MAILBOX_SIZE_LIMIT -Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). +Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). Size is in bytes. - **empty** => 0 (no limit) @@ -241,9 +241,9 @@ Set the mailbox size limit for all users. If set to zero, the size will be unlim See [mailbox quota][docs-accounts-quota]. -##### POSTFIX\_MESSAGE\_SIZE\_LIMIT +##### POSTFIX_MESSAGE_SIZE_LIMIT -Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!) +Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!). Size is in bytes. - **empty** => 10240000 (~10 MB) diff --git a/mailserver.env b/mailserver.env index 48b537bc..e84157a2 100644 --- a/mailserver.env +++ b/mailserver.env @@ -254,7 +254,7 @@ VIRUSMAILS_DELETE_DELAY= # `lmtp::2003` (use kopano as mailstore) POSTFIX_DAGENT= -# Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). +# Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). Size is in bytes. # # empty => 0 POSTFIX_MAILBOX_SIZE_LIMIT= @@ -264,7 +264,7 @@ POSTFIX_MAILBOX_SIZE_LIMIT= # 1 => Dovecot quota is enabled ENABLE_QUOTAS=1 -# Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!) +# Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!). Size is in bytes. # # empty => 10240000 (~10 MB) POSTFIX_MESSAGE_SIZE_LIMIT= diff --git a/target/bin/setquota b/target/bin/setquota index 5b2bba41..039421b2 100755 --- a/target/bin/setquota +++ b/target/bin/setquota @@ -59,10 +59,14 @@ function _quota_request_if_missing() { fi } + +# Dovecot docs incorrectly refer to these units with names for SI types (base 10), +# But then mentions they're actually treated as IEC type (base 2): +# https://doc.dovecot.org/settings/types/#size function _quota_unit_is_valid() { if ! grep -qE "^([0-9]+(B|k|M|G|T)|0)\$" <<< "${QUOTA}"; then __usage - _exit_with_error 'Invalid quota format. e.g. 302M (B (byte), k (kilobyte), M (megabyte), G (gigabyte) or T (terabyte))' + _exit_with_error 'Invalid quota format. e.g. 302M (B (byte), k (kibibyte), M (mebibyte), G (gibibyte) or T (tebibyte))' fi } diff --git a/test/tests/parallel/set1/dovecot/dovecot_quotas.bats b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats new file mode 100644 index 00000000..2c176235 --- /dev/null +++ b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats @@ -0,0 +1,246 @@ +load "${REPOSITORY_ROOT}/test/helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" + +# upstream default: 10 240 000 +# https://www.postfix.org/postconf.5.html#message_size_limit +# > The maximal size in bytes of a message, including envelope information. +# > The value cannot exceed LONG_MAX (typically, a 32-bit or 64-bit signed integer). +# > Note: Be careful when making changes. Excessively small values will result in the loss of non-delivery notifications, when a bounce message size exceeds the local or remote MTA's message size limit. + +# upstream default: 51 200 000 +# https://www.postfix.org/postconf.5.html#mailbox_size_limit +# > The maximal size of any local(8) individual mailbox or maildir file, or zero (no limit). +# > In fact, this limits the size of any file that is written to upon local delivery, including files written by external commands that are executed by the local(8) delivery agent. +# > The value cannot exceed LONG_MAX (typically, a 32-bit or 64-bit signed integer). +# > This limit must not be smaller than the message size limit. + +# upstream default: 51 200 000 +# https://www.postfix.org/postconf.5.html#virtual_mailbox_limit +# > The maximal size in bytes of an individual virtual(8) mailbox or maildir file, or zero (no limit). +# > This parameter is specific to the virtual(8) delivery agent. +# > It does not apply when mail is delivered with a different mail delivery program. + +BATS_TEST_NAME_PREFIX='[Dovecot Quotas] ' +CONTAINER_NAME='dms-test_dovecot-quotas' + +function setup_file() { + _init_with_defaults + + local CONTAINER_ARGS_ENV_CUSTOM=( + --env ENABLE_QUOTAS=1 + --env POSTFIX_MAILBOX_SIZE_LIMIT=4096000 + --env POSTFIX_MESSAGE_SIZE_LIMIT=2048000 + --env PERMIT_DOCKER=container + ) + _common_container_setup 'CONTAINER_ARGS_ENV_CUSTOM' +} + +function teardown_file() { _default_teardown ; } + +@test 'should only support setting quota for a valid account' { + # Prepare + _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' + + # Actual tests + _run_in_container setup quota set quota_user 50M + assert_failure + + _run_in_container setup quota set username@fulldomain 50M + assert_failure + + _run_in_container setup quota set quota_user@domain.tld 50M + assert_success + + # Cleanup + _run_in_container setup email del -y quota_user@domain.tld + assert_success +} + +@test 'should only allow valid units as quota size' { + # Prepare + _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' + + # Actual tests + _run_in_container setup quota set quota_user@domain.tld 26GIGOTS + assert_failure + _run_in_container setup quota set quota_user@domain.tld 123 + assert_failure + _run_in_container setup quota set quota_user@domain.tld M + assert_failure + _run_in_container setup quota set quota_user@domain.tld -60M + assert_failure + + + _run_in_container setup quota set quota_user@domain.tld 10B + assert_success + _run_in_container setup quota set quota_user@domain.tld 10k + assert_success + _run_in_container setup quota set quota_user@domain.tld 10M + assert_success + _run_in_container setup quota set quota_user@domain.tld 10G + assert_success + _run_in_container setup quota set quota_user@domain.tld 10T + assert_success + + # Cleanup + _run_in_container setup email del -y quota_user@domain.tld + assert_success +} + +@test 'should only support removing quota from a valid account' { + # Prepare + _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' + + # Actual tests + _run_in_container setup quota del uota_user@domain.tld + assert_failure + _run_in_container setup quota del quota_user + assert_failure + _run_in_container setup quota del dontknowyou@domain.tld + assert_failure + + _run_in_container setup quota set quota_user@domain.tld 10T + assert_success + _run_in_container setup quota del quota_user@domain.tld + assert_success + _run_in_container grep -i 'quota_user@domain.tld' /tmp/docker-mailserver/dovecot-quotas.cf + assert_failure + + # Cleanup + _run_in_container setup email del -y quota_user@domain.tld + assert_success +} + +@test 'should not error when there is no quota to remove for an account' { + # Prepare + _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' + + # Actual tests + _run_in_container grep -i 'quota_user@domain.tld' /tmp/docker-mailserver/dovecot-quotas.cf + assert_failure + + _run_in_container setup quota del quota_user@domain.tld + assert_success + _run_in_container setup quota del quota_user@domain.tld + assert_success + + # Cleanup + _run_in_container setup email del -y quota_user@domain.tld + assert_success +} + +@test 'should have configured Postfix to use the Dovecot quota-status service' { + _run_in_container postconf + assert_success + assert_output --partial 'check_policy_service inet:localhost:65265' +} + +@test '(ENV POSTFIX_MAILBOX_SIZE_LIMIT) should be configured for both Postfix and Dovecot' { + _run_in_container postconf -h mailbox_size_limit + assert_output 4096000 + + # Dovecot mailbox is sized by `virtual_mailbox_size` from Postfix: + _run_in_container postconf -h virtual_mailbox_limit + assert_output 4096000 + + # Quota support: + _run_in_container doveconf -h plugin/quota_rule + # Global default storage limit quota for each mailbox 4 MiB: + assert_output '*:storage=4M' + + # Sizes are equivalent - Bytes to MiB (rounded): + run numfmt --to=iec --format '%.0f' 4096000 + assert_output '4M' +} + +@test '(ENV POSTFIX_MESSAGE_SIZE_LIMIT) should be configured for both Postfix and Dovecot' { + _run_in_container postconf -h message_size_limit + assert_output 2048000 + + _run_in_container doveconf -h plugin/quota_max_mail_size + assert_output '2M' + + # Sizes are equivalent - Bytes to MiB (rounded): + run numfmt --to=iec --format '%.0f' 2048000 + assert_output '2M' +} + +@test 'Deleting an mailbox account should also remove that account from dovecot-quotas.cf' { + _add_mail_account_then_wait_until_ready 'quserremoved@domain.tld' + + _run_in_container setup quota set quserremoved@domain.tld 12M + assert_success + + _run_in_container cat '/tmp/docker-mailserver/dovecot-quotas.cf' + assert_success + assert_output 'quserremoved@domain.tld:12M' + + _run_in_container setup email del -y quserremoved@domain.tld + assert_success + + _run_in_container cat /tmp/docker-mailserver/dovecot-quotas.cf + assert_success + refute_output --partial 'quserremoved@domain.tld:12M' +} + +@test 'Dovecot should acknowledge quota configured for accounts' { + # sed -nE 's/.*STORAGE.*Limit=([0-9]+).*/\1/p' | numfmt --from-unit=1024 --to=iec --format '%.0f' + local CMD_GET_QUOTA="doveadm -f flow quota get -u 'user1@localhost.localdomain'" + + # 4M == 4096 kiB (numfmt --to-unit=1024 --from=iec 4M) + _run_in_container_bash "${CMD_GET_QUOTA}" + assert_line --partial 'Type=STORAGE Value=0 Limit=4096' + + # Setting a new limit for the user: + _run_in_container setup quota set 'user1@localhost.localdomain' 50M + assert_success + # 50M (50 * 1024^2) == 51200 kiB (numfmt --to-unit=1024 --from=iec 52428800) + run _repeat_until_success_or_timeout 20 _exec_in_container_bash "${CMD_GET_QUOTA} | grep -o 'Type=STORAGE Value=0 Limit=51200'" + assert_success + + # Deleting quota resets it to default global quota limit (`plugin/quota_rule`): + _run_in_container setup quota del 'user1@localhost.localdomain' + assert_success + run _repeat_until_success_or_timeout 20 _exec_in_container_bash "${CMD_GET_QUOTA} | grep -o 'Type=STORAGE Value=0 Limit=4096'" + assert_success +} + +@test 'should receive a warning mail from Dovecot when quota is exceeded' { + # skip 'disabled as it fails randomly: https://github.com/docker-mailserver/docker-mailserver/pull/2511' + + # Prepare + _add_mail_account_then_wait_until_ready 'quotauser@otherdomain.tld' + + # Actual tests + _run_in_container setup quota set quotauser@otherdomain.tld 10k + assert_success + + # wait until quota has been updated + run _repeat_until_success_or_timeout 20 _exec_in_container_bash "doveadm -f flow quota get -u 'quotauser@otherdomain.tld' | grep -o 'Type=STORAGE Value=0 Limit=10'" + assert_success + + # dovecot and postfix has been restarted + _wait_for_service postfix + _wait_for_service dovecot + sleep 10 + + # send some big emails + _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' + _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' + _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' + # check for quota warn message existence + run _repeat_until_success_or_timeout 20 _exec_in_container grep -R 'Subject: quota warning' /var/mail/otherdomain.tld/quotauser/new/ + assert_success + + run _repeat_until_success_or_timeout 20 sh -c "docker logs ${CONTAINER_NAME} | grep 'Quota exceeded (mailbox for user is full)'" + assert_success + + # ensure only the first big message and the warn message are present (other messages are rejected: mailbox is full) + _run_in_container sh -c 'ls /var/mail/otherdomain.tld/quotauser/new/ | wc -l' + assert_success + assert_output "2" + + # Cleanup + _run_in_container setup email del -y quotauser@otherdomain.tld + assert_success +} diff --git a/test/tests/serial/tests.bats b/test/tests/serial/tests.bats index 20ee0dd1..094454f0 100644 --- a/test/tests/serial/tests.bats +++ b/test/tests/serial/tests.bats @@ -17,7 +17,6 @@ function setup_file() { local CONTAINER_ARGS_ENV_CUSTOM=( --env ENABLE_AMAVIS=1 --env AMAVIS_LOGLEVEL=2 - --env ENABLE_QUOTAS=1 --env ENABLE_SRS=1 --env PERMIT_DOCKER=host --env PFLOGSUMM_TRIGGER=logrotate @@ -244,198 +243,6 @@ zip EOF } -@test "quota: setquota user must be existing" { - _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' - - _run_in_container_bash "setquota quota_user 50M" - assert_failure - _run_in_container_bash "setquota quota_user@domain.tld 50M" - assert_success - - _run_in_container_bash "setquota username@fulldomain 50M" - assert_failure - - _run_in_container_bash "delmailuser -y quota_user@domain.tld" - assert_success -} - -@test "quota: setquota must be well formatted" { - _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' - - _run_in_container_bash "setquota quota_user@domain.tld 26GIGOTS" - assert_failure - _run_in_container_bash "setquota quota_user@domain.tld 123" - assert_failure - _run_in_container_bash "setquota quota_user@domain.tld M" - assert_failure - _run_in_container_bash "setquota quota_user@domain.tld -60M" - assert_failure - - - _run_in_container_bash "setquota quota_user@domain.tld 10B" - assert_success - _run_in_container_bash "setquota quota_user@domain.tld 10k" - assert_success - _run_in_container_bash "setquota quota_user@domain.tld 10M" - assert_success - _run_in_container_bash "setquota quota_user@domain.tld 10G" - assert_success - _run_in_container_bash "setquota quota_user@domain.tld 10T" - assert_success - - - _run_in_container_bash "delmailuser -y quota_user@domain.tld" - assert_success -} - -@test "quota: delquota user must be existing" { - _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' - - _run_in_container_bash "delquota uota_user@domain.tld" - assert_failure - _run_in_container_bash "delquota quota_user" - assert_failure - _run_in_container_bash "delquota dontknowyou@domain.tld" - assert_failure - - _run_in_container_bash "setquota quota_user@domain.tld 10T" - assert_success - _run_in_container_bash "delquota quota_user@domain.tld" - assert_success - _run_in_container_bash "grep -i 'quota_user@domain.tld' /tmp/docker-mailserver/dovecot-quotas.cf" - assert_failure - - _run_in_container_bash "delmailuser -y quota_user@domain.tld" - assert_success -} - -@test "quota: delquota allow when no quota for existing user" { - _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' - - _run_in_container_bash "grep -i 'quota_user@domain.tld' /tmp/docker-mailserver/dovecot-quotas.cf" - assert_failure - - _run_in_container_bash "delquota quota_user@domain.tld" - assert_success - _run_in_container_bash "delquota quota_user@domain.tld" - assert_success - - _run_in_container_bash "delmailuser -y quota_user@domain.tld" - assert_success -} - -@test "quota: dovecot quota present in postconf" { - _run_in_container_bash "postconf | grep 'check_policy_service inet:localhost:65265'" - assert_success -} - - -@test "quota: dovecot mailbox max size must be equal to postfix mailbox max size" { - postfix_mailbox_size=$(_exec_in_container_bash "postconf | grep -Po '(?<=mailbox_size_limit = )[0-9]+'") - run echo "${postfix_mailbox_size}" - refute_output "" - - # dovecot relies on virtual_mailbox_size by default - postfix_virtual_mailbox_size=$(_exec_in_container_bash "postconf | grep -Po '(?<=virtual_mailbox_limit = )[0-9]+'") - assert_equal "${postfix_virtual_mailbox_size}" "${postfix_mailbox_size}" - - postfix_mailbox_size_mb=$(( postfix_mailbox_size / 1000000)) - - dovecot_mailbox_size_mb=$(_exec_in_container_bash "doveconf | grep -oP '(?<=quota_rule \= \*\:storage=)[0-9]+'") - run echo "${dovecot_mailbox_size_mb}" - refute_output "" - - assert_equal "${postfix_mailbox_size_mb}" "${dovecot_mailbox_size_mb}" -} - - -@test "quota: dovecot message max size must be equal to postfix messsage max size" { - postfix_message_size=$(_exec_in_container_bash "postconf | grep -Po '(?<=message_size_limit = )[0-9]+'") - run echo "${postfix_message_size}" - refute_output "" - - postfix_message_size_mb=$(( postfix_message_size / 1000000)) - - dovecot_message_size_mb=$(_exec_in_container_bash "doveconf | grep -oP '(?<=quota_max_mail_size = )[0-9]+'") - run echo "${dovecot_message_size_mb}" - refute_output "" - - assert_equal "${postfix_message_size_mb}" "${dovecot_message_size_mb}" -} - -@test "quota: quota directive is removed when mailbox is removed" { - _add_mail_account_then_wait_until_ready 'quserremoved@domain.tld' - - _run_in_container_bash "setquota quserremoved@domain.tld 12M" - assert_success - - _run_in_container_bash 'cat /tmp/docker-mailserver/dovecot-quotas.cf | grep -E "^quserremoved@domain.tld\:12M\$" | wc -l | grep 1' - assert_success - - _run_in_container_bash "delmailuser -y quserremoved@domain.tld" - assert_success - - _run_in_container_bash 'cat /tmp/docker-mailserver/dovecot-quotas.cf | grep -E "^quserremoved@domain.tld\:12M\$"' - assert_failure -} - -@test "quota: dovecot applies user quota" { - _run_in_container_bash "doveadm quota get -u 'user1@localhost.localdomain' | grep 'User quota STORAGE'" - assert_output --partial "- 0" - - _run_in_container_bash "setquota user1@localhost.localdomain 50M" - assert_success - - # wait until quota has been updated - run _repeat_until_success_or_timeout 20 _exec_in_container_bash 'doveadm quota get -u user1@localhost.localdomain | grep -oP "(User quota STORAGE\s+[0-9]+\s+)51200(.*)"' - assert_success - - _run_in_container_bash "delquota user1@localhost.localdomain" - assert_success - - # wait until quota has been updated - run _repeat_until_success_or_timeout 20 _exec_in_container_bash 'doveadm quota get -u user1@localhost.localdomain | grep -oP "(User quota STORAGE\s+[0-9]+\s+)-(.*)"' - assert_success -} - -@test "quota: warn message received when quota exceeded" { - skip 'disabled as it fails randomly: https://github.com/docker-mailserver/docker-mailserver/pull/2511' - - # create user - _add_mail_account_then_wait_until_ready 'quotauser@otherdomain.tld' - _run_in_container_bash 'setquota quotauser@otherdomain.tld 10k' - assert_success - - # wait until quota has been updated - run _repeat_until_success_or_timeout 20 _exec_in_container_bash 'doveadm quota get -u quotauser@otherdomain.tld | grep -oP \"(User quota STORAGE\s+[0-9]+\s+)10(.*)\"' - assert_success - - # dovecot and postfix has been restarted - _wait_for_service postfix - _wait_for_service dovecot - sleep 10 - - # send some big emails - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - - # check for quota warn message existence - run _repeat_until_success_or_timeout 20 _exec_in_container_bash 'grep \"Subject: quota warning\" /var/mail/otherdomain.tld/quotauser/new/ -R' - assert_success - - run _repeat_until_success_or_timeout 20 sh -c "docker logs mail | grep 'Quota exceeded (mailbox for user is full)'" - assert_success - - # ensure only the first big message and the warn message are present (other messages are rejected: mailbox is full) - _run_in_container sh -c 'ls /var/mail/otherdomain.tld/quotauser/new/ | wc -l' - assert_success - assert_output "2" - - _run_in_container_bash "delmailuser -y quotauser@otherdomain.tld" - assert_success -} - # # PERMIT_DOCKER mynetworks #