From e58dd1b95b3552ea850dad6b2ef0b87f8a9bae78 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 18 Mar 2023 23:32:48 +0800 Subject: [PATCH] Rspamd: more features (#3159) --- Dockerfile | 3 - docs/content/config/environment.md | 102 +++++++++++------- docs/content/config/security/rspamd.md | 7 ++ target/dovecot/90-sieve.conf | 2 +- target/dovecot/sieve/before/60-spam.sieve | 4 - target/scripts/start-mailserver.sh | 2 + target/scripts/startup/setup.d/dovecot.sh | 89 +++++++-------- .../scripts/startup/setup.d/security/misc.sh | 26 +++++ .../startup/setup.d/security/rspamd.sh | 55 +++++++++- target/scripts/startup/variables-stack.sh | 1 + test/config/rspamd/postfix-accounts.cf | 2 + test/config/rspamd/user-patches.sh | 13 +++ .../email-templates/rspamd-pass.txt | 12 +++ .../email-templates/rspamd-spam-header.txt | 12 +++ .../email-templates/rspamd-spam.txt | 1 - .../email-templates/rspamd-virus.txt | 2 +- .../rspamd_imap_move_to_inbox.txt | 4 + .../nc_templates/rspamd_imap_move_to_junk.txt | 4 + .../parallel/set1/dovecot/dovecot_sieve.bats | 11 +- .../parallel/set1/spam_virus/rspamd.bats | 96 ++++++++++++++++- 20 files changed, 339 insertions(+), 109 deletions(-) delete mode 100644 target/dovecot/sieve/before/60-spam.sieve create mode 100644 test/config/rspamd/postfix-accounts.cf create mode 100644 test/config/rspamd/user-patches.sh create mode 100644 test/test-files/email-templates/rspamd-pass.txt create mode 100644 test/test-files/email-templates/rspamd-spam-header.txt create mode 100644 test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt create mode 100644 test/test-files/nc_templates/rspamd_imap_move_to_junk.txt diff --git a/Dockerfile b/Dockerfile index acadbe54..b66070de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,7 +54,6 @@ EOF # ----------------------------------------------- COPY target/dovecot/*.inc target/dovecot/*.conf /etc/dovecot/conf.d/ -COPY target/dovecot/sieve/ /etc/dovecot/sieve/ COPY target/dovecot/dovecot-purge.cron /etc/cron.d/dovecot-purge.disabled RUN chmod 0 /etc/cron.d/dovecot-purge.disabled WORKDIR /usr/share/dovecot @@ -66,8 +65,6 @@ RUN < disabled -- 1 => enabled - -##### ENABLE_RSPAMD_REDIS - -Explicit control over running a Redis instance within the container. By default, this value will match what is set for [`ENABLE_RSPAMD`](#enable_rspamd). - -The purpose of this setting is to opt-out of starting an internal Redis instance when enabling Rspamd, replacing it with your own external instance. - -??? note "Configuring rspamd for an external Redis instance" - - You will need to [provide configuration][config-rspamd-redis] at `/etc/rspamd/local.d/redis.conf` similar to: - - ``` - servers = "redis.example.test:6379"; - expand_keys = true; - ``` - -[config-rspamd-redis]: https://rspamd.com/doc/configuration/redis.html - -- 0 => Disabled -- 1 => Enabled - ##### ENABLE_AMAVIS Amavis content filter (used for ClamAV & SpamAssassin) @@ -316,6 +285,69 @@ Note: More details at Note: More information at +##### MOVE_SPAM_TO_JUNK + +When enabled, e-mails marked with the + +1. `X-Spam: Yes` header added by Rspamd +2. `X-Spam-Flag: YES` header added by SpamAssassin (requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)) + +will be automatically moved to the Junk folder (with the help of a Sieve script). + +- 0 => Spam messages will be delivered in the mailbox. +- **1** => Spam messages will be delivered in the `Junk` folder. + +#### Rspamd + +##### ENABLE_RSPAMD + +Enable or disable Rspamd. + +!!! warning "Current State" + + Rspamd-support is under active development. Be aware that breaking changes can happen at any time. To get more information, see [the detailed documentation page for Rspamd][docs-rspamd]. + +- **0** => disabled +- 1 => enabled + +##### ENABLE_RSPAMD_REDIS + +Explicit control over running a Redis instance within the container. By default, this value will match what is set for [`ENABLE_RSPAMD`](#enable_rspamd). + +The purpose of this setting is to opt-out of starting an internal Redis instance when enabling Rspamd, replacing it with your own external instance. + +??? note "Configuring Rspamd for an external Redis instance" + + You will need to [provide configuration][rspamd-redis-config] at `/etc/rspamd/local.d/redis.conf` similar to: + + ``` + servers = "redis.example.test:6379"; + expand_keys = true; + ``` + +[rspamd-redis-config]: https://rspamd.com/doc/configuration/redis.html + +- 0 => Disabled +- 1 => Enabled + +##### RSPAMD_LEARN + +When enabled, + +1. the "[autolearning][rspamd-autolearn]" feature is turned on; +2. the Bayes classifier will be trained when moving mails from or to the Junk folder (with the help of Sieve scripts). + +!!! attention + + As of now, the spam learning database is global (i.e. available to all users). If one user deliberately trains it with malicious data, then it will ruin your detection rate. + + This feature is suitably only for users who can tell ham from spam and users that can be trusted. + +[rspamd-autolearn]: https://rspamd.com/doc/configuration/statistic.html#autolearning + +- **0** => Disabled +- 1 => Enabled + #### Reports ##### PFLOGSUMM_TRIGGER @@ -418,14 +450,6 @@ Changes the interval in which log files are rotated. - **0** => KAM disabled - 1 => KAM enabled -##### MOVE_SPAM_TO_JUNK - -Spam messages can be moved in the Junk folder. -Note: this setting needs `SPAMASSASSIN_SPAM_TO_INBOX=1` - -- 0 => Spam messages will be delivered in the mailbox. -- **1** => Spam messages will be delivered in the `Junk` folder. - ##### SA_TAG - **2.0** => add spam info headers if at, or above that level diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index e9a95700..dc81b1fc 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -20,6 +20,13 @@ If you want to have a look at the default configuration files for Rspamd that DM Maintainers noticed only few differences, some of them with a big impact though. For those running Rspamd on ARM64, we recommend [disabling](#with-the-help-of-a-custom-file) the [DKIM signing module][dkim-signing-module] if you don't use it. +The following environment variables are related to Rspamd: + +1. [`ENABLE_RSPAMD`](../environment.md#enable_rspamd) +2. [`ENABLE_RSPAMD_REDIS`](../environment.md#enable_rspamd_redis) +3. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) +4. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) + ## The Default Configuration ### Mode of Operation diff --git a/target/dovecot/90-sieve.conf b/target/dovecot/90-sieve.conf index 78fbe47b..8b559be2 100644 --- a/target/dovecot/90-sieve.conf +++ b/target/dovecot/90-sieve.conf @@ -71,7 +71,7 @@ plugin { # (http://pigeonhole.dovecot.org) for available plugins. # The sieve_extprograms plugin is included in this release. #sieve_plugins = - sieve_plugins = sieve_extprograms + sieve_plugins = sieve_imapsieve sieve_extprograms # The separator that is expected between the :user and :detail # address parts introduced by the subaddress extension. This may diff --git a/target/dovecot/sieve/before/60-spam.sieve b/target/dovecot/sieve/before/60-spam.sieve deleted file mode 100644 index 4fa29fab..00000000 --- a/target/dovecot/sieve/before/60-spam.sieve +++ /dev/null @@ -1,4 +0,0 @@ -require "fileinto"; -if header :contains "X-Spam-Flag" "YES" { - fileinto "Junk"; -} diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index c98639b8..83209267 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -45,6 +45,7 @@ function _register_functions if [[ ${SMTP_ONLY} -ne 1 ]] then _register_setup_function '_setup_dovecot' + _register_setup_function '_setup_dovecot_sieve' _register_setup_function '_setup_dovecot_dhparam' _register_setup_function '_setup_dovecot_quota' fi @@ -81,6 +82,7 @@ function _register_functions _register_setup_function '_setup_opendmarc' # must come after `_setup_opendkim` _register_setup_function '_setup_security_stack' + _register_setup_function '_setup_spam_to_junk' _register_setup_function '_setup_rspamd' _register_setup_function '_setup_ssl' diff --git a/target/scripts/startup/setup.d/dovecot.sh b/target/scripts/startup/setup.d/dovecot.sh index cf77c979..fbb01871 100644 --- a/target/scripts/startup/setup.d/dovecot.sh +++ b/target/scripts/startup/setup.d/dovecot.sh @@ -40,54 +40,6 @@ function _setup_dovecot esac - # enable Managesieve service by setting the symlink - # to the configuration file Dovecot will actually find - if [[ ${ENABLE_MANAGESIEVE} -eq 1 ]] - then - _log 'trace' 'Sieve management enabled' - mv /etc/dovecot/protocols.d/managesieved.protocol.disab /etc/dovecot/protocols.d/managesieved.protocol - fi - - # copy pipe and filter programs, if any - rm -f /usr/lib/dovecot/sieve-filter/* - rm -f /usr/lib/dovecot/sieve-pipe/* - [[ -d /tmp/docker-mailserver/sieve-filter ]] && cp /tmp/docker-mailserver/sieve-filter/* /usr/lib/dovecot/sieve-filter/ - [[ -d /tmp/docker-mailserver/sieve-pipe ]] && cp /tmp/docker-mailserver/sieve-pipe/* /usr/lib/dovecot/sieve-pipe/ - - # create global sieve directories - mkdir -p /usr/lib/dovecot/sieve-global/before - mkdir -p /usr/lib/dovecot/sieve-global/after - - if [[ -f /tmp/docker-mailserver/before.dovecot.sieve ]] - then - cp /tmp/docker-mailserver/before.dovecot.sieve /usr/lib/dovecot/sieve-global/before/50-before.dovecot.sieve - sievec /usr/lib/dovecot/sieve-global/before/50-before.dovecot.sieve - else - rm -f /usr/lib/dovecot/sieve-global/before/50-before.dovecot.sieve /usr/lib/dovecot/sieve-global/before/50-before.dovecot.svbin - fi - - if [[ -f /tmp/docker-mailserver/after.dovecot.sieve ]] - then - cp /tmp/docker-mailserver/after.dovecot.sieve /usr/lib/dovecot/sieve-global/after/50-after.dovecot.sieve - sievec /usr/lib/dovecot/sieve-global/after/50-after.dovecot.sieve - else - rm -f /usr/lib/dovecot/sieve-global/after/50-after.dovecot.sieve /usr/lib/dovecot/sieve-global/after/50-after.dovecot.svbin - fi - - # sieve will move spams to .Junk folder when SPAMASSASSIN_SPAM_TO_INBOX=1 and MOVE_SPAM_TO_JUNK=1 - if [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 1 ]] && [[ ${MOVE_SPAM_TO_JUNK} -eq 1 ]] - then - _log 'debug' 'Spam messages will be moved to the Junk folder' - cp /etc/dovecot/sieve/before/60-spam.sieve /usr/lib/dovecot/sieve-global/before/ - sievec /usr/lib/dovecot/sieve-global/before/60-spam.sieve - else - rm -f /usr/lib/dovecot/sieve-global/before/60-spam.sieve /usr/lib/dovecot/sieve-global/before/60-spam.svbin - fi - - chown docker:docker -R /usr/lib/dovecot/sieve* - chmod 550 -R /usr/lib/dovecot/sieve* - chmod -f +x /usr/lib/dovecot/sieve-pipe/* - if [[ ${ENABLE_POP3} -eq 1 ]] then _log 'debug' 'Enabling POP3 services' @@ -97,6 +49,47 @@ function _setup_dovecot [[ -f /tmp/docker-mailserver/dovecot.cf ]] && cp /tmp/docker-mailserver/dovecot.cf /etc/dovecot/local.conf } +function _setup_dovecot_sieve +{ + mkdir -p /usr/lib/dovecot/sieve-{filter,global,pipe} + mkdir -p /usr/lib/dovecot/sieve-global/{before,after} + + # enable Managesieve service by setting the symlink + # to the configuration file Dovecot will actually find + if [[ ${ENABLE_MANAGESIEVE} -eq 1 ]] + then + _log 'trace' 'Sieve management enabled' + mv /etc/dovecot/protocols.d/managesieved.protocol.disab /etc/dovecot/protocols.d/managesieved.protocol + fi + + if [[ -d /tmp/docker-mailserver/sieve-filter ]] + then + cp /tmp/docker-mailserver/sieve-filter/* /usr/lib/dovecot/sieve-filter/ + fi + if [[ -d /tmp/docker-mailserver/sieve-pipe ]] + then + cp /tmp/docker-mailserver/sieve-pipe/* /usr/lib/dovecot/sieve-pipe/ + fi + + if [[ -f /tmp/docker-mailserver/before.dovecot.sieve ]] + then + cp \ + /tmp/docker-mailserver/before.dovecot.sieve \ + /usr/lib/dovecot/sieve-global/before/50-before.dovecot.sieve + sievec /usr/lib/dovecot/sieve-global/before/50-before.dovecot.sieve + fi + if [[ -f /tmp/docker-mailserver/after.dovecot.sieve ]] + then + cp \ + /tmp/docker-mailserver/after.dovecot.sieve \ + /usr/lib/dovecot/sieve-global/after/50-after.dovecot.sieve + sievec /usr/lib/dovecot/sieve-global/after/50-after.dovecot.sieve + fi + + chown dovecot:root -R /usr/lib/dovecot/sieve-* + find /usr/lib/dovecot/sieve-* -type d -exec chmod 755 {} \; + chmod +x /usr/lib/dovecot/sieve-{filter,pipe}/* +} function _setup_dovecot_quota { diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index d441f7d1..085b632b 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -263,3 +263,29 @@ function __setup__security__amavis fi fi } + +# We can use Sieve to move spam emails to the "Junk" folder. +function _setup_spam_to_junk +{ + if [[ ${MOVE_SPAM_TO_JUNK} -eq 1 ]] + then + _log 'debug' 'Spam emails will be moved to the Junk folder' + cat >/usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve << EOF +require ["fileinto","mailbox"]; + +if anyof (header :contains "X-Spam-Flag" "YES", + header :contains "X-Spam" "Yes") { + fileinto "Junk"; +} +EOF + sievec /usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve + chown dovecot:root /usr/lib/dovecot/sieve-global/after/spam_to_junk.{sieve,svbin} + + if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 0 ]] + then + _log 'warning' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MOVE_SPAM_TO_JUNK=1' to work" + fi + else + _log 'debug' 'Spam emails will not be moved to the Junk folder' + fi +} diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index a476bd46..8b478904 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -7,10 +7,18 @@ function _setup_rspamd _log 'warn' 'Rspamd integration is work in progress - expect (breaking) changes at any time' _log 'debug' 'Enabling and configuring Rspamd' - __rspamd__preflight_checks + __rspamd__preflight_checks_and_setup __rspamd__adjust_postfix_configuration __rspamd__disable_default_modules __rspamd__handle_modules_configuration + + if [[ ${RSPAMD_LEARN} -eq 1 ]] + then + __rspamd__log 'debug' 'Enabling and configuring learning' + __rspamd__setup_learning + else + __rspamd__log 'debug' 'Learning is disabled' + fi else _log 'debug' 'Rspamd is disabled' fi @@ -28,7 +36,7 @@ function __rspamd__log { _log "${1:-}" "(Rspamd setup) ${2:-}" ; } # # This will also check whether Amavis is enabled and emit a warning as # we discourage users from running Amavis & Rspamd at the same time. -function __rspamd__preflight_checks +function __rspamd__preflight_checks_and_setup { touch /var/lib/rspamd/stats.ucl @@ -225,3 +233,46 @@ function __rspamd__handle_modules_configuration done < <(_get_valid_lines_from_file "${RSPAMD_CUSTOM_COMMANDS_FILE}") fi } + +# This function sets up intelligent learning of Junk, by +# +# 1. enabling auto-learn for the classifier-bayes module +# 2. setting up sieve scripts that detect when a user is moving e-mail +# from or to the "Junk" folder, and learning them as ham or spam. +function __rspamd__setup_learning +{ + __rspamd__log 'debug' 'Setting up intelligent learning of spam and ham' + + local SIEVE_PIPE_BIN_DIR='/usr/lib/dovecot/sieve-pipe' + ln -s "$(type -f -P rspamc)" "${SIEVE_PIPE_BIN_DIR}/rspamc" + + sedfile -i -E 's|(mail_plugins =.*)|\1 imap_sieve|' /etc/dovecot/conf.d/20-imap.conf + sedfile -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf + cat >>/etc/dovecot/conf.d/90-sieve.conf << EOF + + # From elsewhere to Junk folder + imapsieve_mailbox1_name = Junk + imapsieve_mailbox1_causes = COPY + imapsieve_mailbox1_before = file:${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve + + # From Junk folder to elsewhere + imapsieve_mailbox2_name = * + imapsieve_mailbox2_from = Junk + imapsieve_mailbox2_causes = COPY + imapsieve_mailbox2_before = file:${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve +} +EOF + + cat >"${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve" << EOF +require ["vnd.dovecot.pipe", "copy", "imapsieve"]; +pipe :copy "rspamc" ["-h", "127.0.0.1:11334", "learn_spam"]; +EOF + + cat >"${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve" << EOF +require ["vnd.dovecot.pipe", "copy", "imapsieve"]; +pipe :copy "rspamc" ["-h", "127.0.0.1:11334", "learn_ham"]; +EOF + + sievec "${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve" + sievec "${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve" +} diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 20a94847..ffdb4528 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -55,6 +55,7 @@ function __environment_variables_general_setup VARS[POSTGREY_MAX_AGE]="${POSTGREY_MAX_AGE:=35}" VARS[POSTGREY_TEXT]="${POSTGREY_TEXT:=Delayed by Postgrey}" VARS[POSTSCREEN_ACTION]="${POSTSCREEN_ACTION:=enforce}" + VARS[RSPAMD_LEARN]="${RSPAMD_LEARN:=0}" VARS[SA_KILL]=${SA_KILL:="6.31"} VARS[SA_SPAM_SUBJECT]=${SA_SPAM_SUBJECT:="***SPAM*** "} VARS[SA_TAG]=${SA_TAG:="2.0"} diff --git a/test/config/rspamd/postfix-accounts.cf b/test/config/rspamd/postfix-accounts.cf new file mode 100644 index 00000000..c1b89d3c --- /dev/null +++ b/test/config/rspamd/postfix-accounts.cf @@ -0,0 +1,2 @@ +# password is 123 +user1@localhost.localdomain|{SHA512-CRYPT}$6$ARrjj74S83.7c53r$AZXMChav0r03LPEGAJJBmfUt59139rZ1zRIxrPhICSh.Y70Zjq6gClnF/cHDUG95dMoFt4Bkj6N4hvFSZ7L301 diff --git a/test/config/rspamd/user-patches.sh b/test/config/rspamd/user-patches.sh new file mode 100644 index 00000000..00e6ca97 --- /dev/null +++ b/test/config/rspamd/user-patches.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +cat >/etc/rspamd/override.d/testmodule_complicated.conf << EOF +complicated { + anOption = someValue; +} +EOF + +echo "enable_test_patterns = true;" >>/etc/rspamd/local.d/options.inc + +echo 'mail_debug = yes' >>/etc/dovecot/dovecot.conf +sed -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf +echo -e 'sieve_trace_debug = yes\n}' >>/etc/dovecot/conf.d/90-sieve.conf diff --git a/test/test-files/email-templates/rspamd-pass.txt b/test/test-files/email-templates/rspamd-pass.txt new file mode 100644 index 00000000..0f244740 --- /dev/null +++ b/test/test-files/email-templates/rspamd-pass.txt @@ -0,0 +1,12 @@ +HELO mail.external.tld +MAIL FROM: pass@example.test +RCPT TO: user1@localhost.localdomain +DATA +From: Docker Mail Server +To: Existing Local User +Date: Sat, 22 May 2010 07:43:25 -0400 +Subject: Test Message rspamd-pass.txt +This mail should pass and Rspamd should not mark it. + +. +QUIT diff --git a/test/test-files/email-templates/rspamd-spam-header.txt b/test/test-files/email-templates/rspamd-spam-header.txt new file mode 100644 index 00000000..7be1a56d --- /dev/null +++ b/test/test-files/email-templates/rspamd-spam-header.txt @@ -0,0 +1,12 @@ +HELO mail.example.test +MAIL FROM: spam-header@example.test +RCPT TO: user1@localhost.localdomain +DATA +From: Docker Mail Server +To: Existing Local User +Date: Sat, 21 Jan 2023 11:11:11 +0000 +Subject: Test Message rspamd-spam-header.txt +YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X + +. +QUIT diff --git a/test/test-files/email-templates/rspamd-spam.txt b/test/test-files/email-templates/rspamd-spam.txt index 2e6d566a..88bd719c 100644 --- a/test/test-files/email-templates/rspamd-spam.txt +++ b/test/test-files/email-templates/rspamd-spam.txt @@ -6,7 +6,6 @@ From: Docker Mail Server To: Existing Local User Date: Sat, 21 Jan 2023 11:11:11 +0000 Subject: Test Message rspamd-spam.txt -This is a test mail. XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X . diff --git a/test/test-files/email-templates/rspamd-virus.txt b/test/test-files/email-templates/rspamd-virus.txt index cdacf9af..c745f261 100644 --- a/test/test-files/email-templates/rspamd-virus.txt +++ b/test/test-files/email-templates/rspamd-virus.txt @@ -2,7 +2,7 @@ HELO mail.example.test MAIL FROM: virus@example.test RCPT TO: user1@localhost.localdomain DATA -From: Docker Mail Server +From: Docker Mail Server To: Existing Local User Date: Sat, 21 Jan 2023 11:11:11 +0000 Subject: Test Message rspamd-virus.txt diff --git a/test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt b/test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt new file mode 100644 index 00000000..a3f6dc13 --- /dev/null +++ b/test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt @@ -0,0 +1,4 @@ +A LOGIN user1@localhost.localdomain 123 +B SELECT Junk +A UID MOVE 1:1 INBOX +A4 LOGOUT diff --git a/test/test-files/nc_templates/rspamd_imap_move_to_junk.txt b/test/test-files/nc_templates/rspamd_imap_move_to_junk.txt new file mode 100644 index 00000000..0039d106 --- /dev/null +++ b/test/test-files/nc_templates/rspamd_imap_move_to_junk.txt @@ -0,0 +1,4 @@ +A LOGIN user1@localhost.localdomain 123 +B SELECT INBOX +A UID MOVE 1:1 Junk +A4 LOGOUT diff --git a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats index 889890d2..c4b0112f 100644 --- a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats +++ b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats @@ -37,27 +37,24 @@ function teardown_file() { _default_teardown ; } # dovecot-sieve/dovecot.sieve @test "User Sieve - should store mail from 'spam@spam.com' into recipient (user1) mailbox 'INBOX.spam'" { - _run_in_container_bash 'ls -A /var/mail/localhost.localdomain/user1/.INBOX.spam/new' - assert_success - _should_output_number_of_lines 1 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/.INBOX.spam/new 1 } # dovecot-sieve/before.dovecot.sieve @test "Global Sieve - should have copied mail from 'spam@spam.com' to recipient (user1) inbox" { - _run_in_container grep 'Spambot ' -R /var/mail/localhost.localdomain/user1/new/ + _run_in_container grep -R 'Spambot ' /var/mail/localhost.localdomain/user1/new/ assert_success } # dovecot-sieve/sieve-pipe + dovecot-sieve/user2@otherdomain.tld.dovecot.sieve @test "Sieve Pipe - should pipe mail received for user2 into '/tmp/pipe-test.out'" { - _run_in_container_bash 'ls -A /tmp/pipe-test.out' + _run_in_container_bash '[[ -f /tmp/pipe-test.out ]]' assert_success - _should_output_number_of_lines 1 } # Only test coverage for feature is to check that the service is listening on the expected port: # https://doc.dovecot.org/admin_manual/pigeonhole_managesieve_server/ @test "ENV 'ENABLE_MANAGESIEVE' - should have enabled service on port 4190" { - _run_in_container_bash 'nc -z 0.0.0.0 4190' + _run_in_container nc -z 0.0.0.0 4190 assert_success } diff --git a/test/tests/parallel/set1/spam_virus/rspamd.bats b/test/tests/parallel/set1/spam_virus/rspamd.bats index 6c385553..658a0e68 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd.bats @@ -16,8 +16,11 @@ function setup_file() { --env ENABLE_OPENDMARC=0 --env PERMIT_DOCKER=host --env LOG_LEVEL=trace + --env MOVE_SPAM_TO_JUNK=1 + --env RSPAMD_LEARN=1 ) + mv "${TEST_TMP_CONFIG}"/rspamd/* "${TEST_TMP_CONFIG}/" _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' # wait for ClamAV to be fully setup or we will get errors on the log @@ -31,12 +34,15 @@ function setup_file() { # We will send 3 emails: the first one should pass just fine; the second one should # be rejected due to spam; the third one should be rejected due to a virus. - export MAIL_ID1=$(_send_email_and_get_id 'email-templates/existing-user1') + export MAIL_ID1=$(_send_email_and_get_id 'email-templates/rspamd-pass') export MAIL_ID2=$(_send_email_and_get_id 'email-templates/rspamd-spam') export MAIL_ID3=$(_send_email_and_get_id 'email-templates/rspamd-virus') + export MAIL_ID4=$(_send_email_and_get_id 'email-templates/rspamd-spam-header') - # add a nested option to a module - _exec_in_container_bash "echo -e 'complicated {\n anOption = someValue;\n}' >/etc/rspamd/override.d/testmodule_complicated.conf" + for ID in MAIL_ID{1,2,3,4} + do + [[ -n ${!ID} ]] || { echo "${ID} is empty - aborting!" ; return 1 ; } + done } function teardown_file() { _default_teardown ; } @@ -44,6 +50,9 @@ function teardown_file() { _default_teardown ; } @test "Postfix's main.cf was adjusted" { _run_in_container grep -F 'smtpd_milters = $rspamd_milter' /etc/postfix/main.cf assert_success + _run_in_container postconf rspamd_milter + assert_success + assert_output 'rspamd_milter = inet:localhost:11332' } @test 'logs exist and contains proper content' { @@ -62,6 +71,8 @@ function teardown_file() { _default_teardown ; } _print_mail_log_for_id "${MAIL_ID1}" assert_output --partial "stored mail into mailbox 'INBOX'" + + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 } @test 'detects and rejects spam' { @@ -71,6 +82,8 @@ function teardown_file() { _default_teardown ; } _print_mail_log_for_id "${MAIL_ID2}" assert_output --partial 'milter-reject' assert_output --partial '5.7.1 Gtube pattern' + + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 } @test 'detects and rejects virus' { @@ -81,6 +94,8 @@ function teardown_file() { _default_teardown ; } assert_output --partial 'milter-reject' assert_output --partial '5.7.1 ClamAV FOUND VIRUS "Eicar-Signature"' refute_output --partial "stored mail into mailbox 'INBOX'" + + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 } @test 'custom commands work correctly' { @@ -153,3 +168,78 @@ function teardown_file() { _default_teardown ; } _run_in_container grep -F 'OhMy = "PraiseBeLinters !";' "${MODULE_PATH}" assert_success } + +@test 'Check MOVE_SPAM_TO_JUNK works for Rspamd' { + _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve ]]' + assert_success + _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.svbin ]]' + assert_success + + _service_log_should_contain_string 'rspamd' 'S \(add header\)' + _service_log_should_contain_string 'rspamd' 'add header "Gtube pattern"' + + _print_mail_log_for_id "${MAIL_ID4}" + assert_output --partial "fileinto action: stored mail into mailbox 'Junk'" + + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/.Junk/new/ 1 +} + +@test 'Check RSPAMD_LEARN works' { + for FILE in learn-{ham,spam}.{sieve,svbin} + do + _run_in_container_bash "[[ -f /usr/lib/dovecot/sieve-pipe/${FILE} ]]" + assert_success + done + + _run_in_container grep 'mail_plugins.*imap_sieve' /etc/dovecot/conf.d/20-imap.conf + local SIEVE_CONFIG_FILE='/etc/dovecot/conf.d/90-sieve.conf' + _run_in_container grep 'sieve_plugins.*sieve_imapsieve' "${SIEVE_CONFIG_FILE}" + _run_in_container grep 'sieve_global_extensions.*\+vnd\.dovecot\.pipe' "${SIEVE_CONFIG_FILE}" + _run_in_container grep -F 'sieve_pipe_bin_dir = /usr/lib/dovecot/sieve-pipe' "${SIEVE_CONFIG_FILE}" + + # Move an email to the "Junk" folder from "INBOX"; the first email we + # sent should pass fine, hence we can now move it + _send_email 'nc_templates/rspamd_imap_move_to_junk' '0.0.0.0 143' + sleep 1 # wait for the transaction to finish + + local MOVE_TO_JUNK_LINES=( + 'imapsieve: mailbox Junk: MOVE event' + 'imapsieve: Matched static mailbox rule [1]' + "sieve: file storage: script: Opened script \`learn-spam'" + 'sieve: file storage: Using Sieve script path: /usr/lib/dovecot/sieve-pipe/learn-spam.sieve' + "sieve: Executing script from \`/usr/lib/dovecot/sieve-pipe/learn-spam.svbin'" + "Finished running script \`/usr/lib/dovecot/sieve-pipe/learn-spam.svbin'" + 'sieve: action pipe: running program: rspamc' + "pipe action: piped message to program \`rspamc'" + "left message in mailbox 'Junk'" + ) + + _run_in_container cat /var/log/mail/mail.log + assert_success + for LINE in "${MOVE_TO_JUNK_LINES[@]}" + do + assert_output --partial "${LINE}" + done + + # Move an email to the "INBOX" folder from "Junk"; there should be two mails + # in the "Junk" folder + _send_email 'nc_templates/rspamd_imap_move_to_inbox' '0.0.0.0 143' + sleep 1 # wait for the transaction to finish + + local MOVE_TO_JUNK_LINES=( + 'imapsieve: Matched static mailbox rule [2]' + "sieve: file storage: script: Opened script \`learn-ham'" + 'sieve: file storage: Using Sieve script path: /usr/lib/dovecot/sieve-pipe/learn-ham.sieve' + "sieve: Executing script from \`/usr/lib/dovecot/sieve-pipe/learn-ham.svbin'" + "Finished running script \`/usr/lib/dovecot/sieve-pipe/learn-ham.svbin'" + "left message in mailbox 'INBOX'" + ) + + _run_in_container cat /var/log/mail/mail.log + assert_success + for LINE in "${MOVE_TO_JUNK_LINES[@]}" + do + assert_output --partial "${LINE}" + done +}