fix: `postfix-main.cf` may depend upon `postfix-master.cf` (#3880)

Custom parameters must be referenced to be retained when `postconf -n` is run. If those parameters are referenced by `postfix-master.cf` this needs to update `master.cf` before updating `main.cf`.

CHANGELOG.md

@@ -96,7 +96,9 @@ The most noteworthy change of this release is the update of the container's base
 
 ### Fixes
 
-- DMS config files that are parsed line by line are now more robust to parse by detecting and fixing line-endings ([#3819](https://github.com/docker-mailserver/docker-mailserver/pull/3819))
+- DMS config:
+  - Files that are parsed line by line are now more robust to parse by detecting and fixing line-endings ([#3819](https://github.com/docker-mailserver/docker-mailserver/pull/3819))
+  - The override config `postfix-main.cf` now retains custom parameters intended for use with `postfix-master.cf` ([#3880](https://github.com/docker-mailserver/docker-mailserver/pull/3880))
 - Variables related to Rspamd are declared as `readonly`, which would cause warnings in the log when being re-declared; we now guard against this issue ([#3837](https://github.com/docker-mailserver/docker-mailserver/pull/3837))
 - Relay host feature refactored ([#3845](https://github.com/docker-mailserver/docker-mailserver/pull/3845))
   - `DEFAULT_RELAY_HOST` ENV can now also use the `RELAY_USER` + `RELAY_PASSWORD` ENV for supplying credentials.

target/scripts/startup/setup.d/postfix.sh

@@ -109,21 +109,6 @@ function _setup_postfix_late() {
 function __postfix__setup_override_configuration() {
   __postfix__log 'debug' 'Overriding / adjusting configuration with user-supplied values'
 
-  local OVERRIDE_CONFIG_POSTFIX_MAIN='/tmp/docker-mailserver/postfix-main.cf'
-  if [[ -f ${OVERRIDE_CONFIG_POSTFIX_MAIN} ]]; then
-    cat "${OVERRIDE_CONFIG_POSTFIX_MAIN}" >>/etc/postfix/main.cf
-    _adjust_mtime_for_postfix_maincf
-
-    # do not directly output to 'main.cf' as this causes a read-write-conflict
-    postconf -n >/tmp/postfix-main-new.cf 2>/dev/null
-
-    mv /tmp/postfix-main-new.cf /etc/postfix/main.cf
-    _adjust_mtime_for_postfix_maincf
-    __postfix__log 'trace' "Adjusted '/etc/postfix/main.cf' according to '${OVERRIDE_CONFIG_POSTFIX_MAIN}'"
-  else
-    __postfix__log 'trace' "No extra Postfix settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MAIN}' was not provided"
-  fi
-
   local OVERRIDE_CONFIG_POSTFIX_MASTER='/tmp/docker-mailserver/postfix-master.cf'
   if [[ -f ${OVERRIDE_CONFIG_POSTFIX_MASTER} ]]; then
     while read -r LINE; do
@@ -131,7 +116,26 @@ function __postfix__setup_override_configuration() {
     done < <(_get_valid_lines_from_file "${OVERRIDE_CONFIG_POSTFIX_MASTER}")
     __postfix__log 'trace' "Adjusted '/etc/postfix/master.cf' according to '${OVERRIDE_CONFIG_POSTFIX_MASTER}'"
   else
-    __postfix__log 'trace' "No extra Postfix settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MASTER}' was not provided"
+    __postfix__log 'trace' "No extra Postfix master settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MASTER}' was not provided"
+  fi
+
+  # NOTE: `postfix-main.cf` should be handled after `postfix-master.cf` as custom parameters require an existing reference
+  # in either `main.cf` or `master.cf` prior to `postconf` reading `main.cf`, otherwise it is discarded from output.
+  local OVERRIDE_CONFIG_POSTFIX_MAIN='/tmp/docker-mailserver/postfix-main.cf'
+  if [[ -f ${OVERRIDE_CONFIG_POSTFIX_MAIN} ]]; then
+    cat "${OVERRIDE_CONFIG_POSTFIX_MAIN}" >>/etc/postfix/main.cf
+    _adjust_mtime_for_postfix_maincf
+
+    # Do not directly output to 'main.cf' as this causes a read-write-conflict.
+    # `postconf` output is filtered to skip expected warnings regarding overrides:
+    # https://github.com/docker-mailserver/docker-mailserver/pull/3880#discussion_r1510414576
+    postconf -n >/tmp/postfix-main-new.cf 2> >(grep -v 'overriding earlier entry')
+
+    mv /tmp/postfix-main-new.cf /etc/postfix/main.cf
+    _adjust_mtime_for_postfix_maincf
+    __postfix__log 'trace' "Adjusted '/etc/postfix/main.cf' according to '${OVERRIDE_CONFIG_POSTFIX_MAIN}'"
+  else
+    __postfix__log 'trace' "No extra Postfix main settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MAIN}' was not provided"
   fi
 }
 

test/config/override-configs/postfix-main.cf

@@ -2,3 +2,6 @@ max_idle = 600s
 # this is a comment
     # this is also a comment
 readme_directory = /tmp
+
+# This parameter is referenced by an override in `postfix-master.cf`:
+custom_parameter = cidr:{{!172.16.0.42 REJECT}}, permit_sasl_authenticated, reject

test/config/override-configs/postfix-master.cf

@@ -1,3 +1,7 @@
 submission/inet/smtpd_sasl_security_options=noanonymous
 # this is a test comment, please don't delete me :'(
     # this is also a test comment, :O
+
+# DMS must first process `postfix-master.cf` to `master.cf` when any custom parameters are from `postfix-main.cf`
+# before an updated `main.cf` is read via `postconf`, otherwise without an existing reference the parameter is excluded.
+submission/inet/smtpd_client_restrictions=$custom_parameter

test/tests/parallel/set1/config_overrides.bats

@@ -15,6 +15,9 @@ function setup_file() {
 
 function teardown_file() { _default_teardown ; }
 
+# The `postconf` command can query both `main.cf` and `master.cf` at `/etc/postfix/`.
+# Reference: http://www.postfix.org/postconf.1.html
+
 @test "Postfix - 'postfix-main.cf' overrides applied to '/etc/postfix/main.cf'" {
   _run_in_container grep -q 'max_idle = 600s' /tmp/docker-mailserver/postfix-main.cf
   assert_success
@@ -37,6 +40,24 @@ function teardown_file() { _default_teardown ; }
   assert_output --partial '-o smtpd_sasl_security_options=noanonymous'
 }
 
+# Custom parameter support works correctly:
+# NOTE: This would only fail on a fresh container state, any restart would pass successfully:
+# https://github.com/docker-mailserver/docker-mailserver/pull/3880
+@test "Postfix - 'postfix-master.cf' should apply before 'postfix-main.cf'" {
+  # Retrieve the value for this setting, `postfix-master.cf` should have the override set:
+  _run_in_container postconf -Ph 'submission/inet/smtpd_client_restrictions'
+  assert_success
+  refute_output --partial 'postconf: warning: /etc/postfix/master.cf: undefined parameter: custom_parameter'
+  #shellcheck disable=SC2016
+  assert_output '$custom_parameter'
+
+  # As it's a custom parameter (`$` prefix), ensure the parameter value expands correctly:
+  _run_in_container postconf -Phx 'submission/inet/smtpd_client_restrictions'
+  assert_success
+  refute_output --partial 'postconf: warning: /etc/postfix/master.cf: undefined parameter: custom_parameter'
+  assert_output 'cidr:{{!172.16.0.42 REJECT}}, permit_sasl_authenticated, reject'
+}
+
 @test "Dovecot - 'dovecot.cf' overrides applied to '/etc/dovecot/local.conf'" {
   _run_in_container grep -q 'mail_max_userip_connections = 69' /tmp/docker-mailserver/dovecot.cf
   assert_success