email: Fix broken SpamAssassin integration and improve filtering fram…

…ework.

mod_spamassassin: Fix bug in header prepend logic that caused
  SpamAssassin headers to be ignored.
mod_spamassassin: Also process headers previously prepended
  by previously executed filters.
utils.c: Fix bbs_splice always returning 0.
net_smtp: Add better logging for incoming SMTP connections.
net_smtp: Allow global MailScript rules to be executed
  for all incoming messages. This allows, for example, all
  definite spam messages to be dropped immediately by system policy.
smtp_msg_process: Allow mproc->mbox to be NULL, which allows us to
  filter mail not associated with a local mailbox, e.g. using global
  MailScript rules. As part of this, message processor callbacks
  are now invoked from within mod_smtp_delivery_external and
  mod_smtp_mailing_lists.
smtp_msg_process: Allow global Sieve and MailScript to be run both
  before and after a mailbox's rules. The global .rules file has thus
  been split into before.rules and after.rules (not backwards compatible).
  3 passes of message processors are now done, and message processors
  now execute the appropriate callback based on the iteration.
mod_mailscript: Add more tailored CONTAINS condition for substring match.
mod_mailscript: Add floating point comparison operator conditions for
  header values.
mod_mailscript: Fix documentation to clarify that a leading delimiter
  is not needed for the MOVETO action.
mod_mailscript: Add REJECT action that is similar to Sieve "reject" action.
  The BOUNCE action continues to return a failure code without dropping the
  message.
mod_mailscript: Rename DROP action to DISCARD for consistency with Sieve
  nomenclature.
mod_mailscript: Rename FORWARD action to REDIRECT for consistency with Sieve
  nomenclature.
mod_mailscript: The EXIT action may only be used to abort all rules
  processing in global rules, not mailbox rules. This is because
  if allowed in mailbox rules, that would allow a user to skip the
  "after" global rules from running, which may enforce certain behavior.
  In practice, this does not change behavior, since prior to this commit,
  this would only result in rules being skipped if run from the single
  global .rules, which is now equivalent to before.rules.
mod_sieve: The "reject" action now implicitly drops messages, since this
  is how the Sieve action works. Only in MailScript is it possible to
  trigger a rejection message but accept the message (using the BOUNCE action).
mod_smtp_filter: Don't add empty "Authentication-Results" headers.
README: Add additional SpamAssassin tips.

configs/.rules

@@ -5,8 +5,9 @@
 # By default, rules apply to ALL messages, and you need to use MATCH directives to explicitly
 # filter to the rules you want.
 # Rules are evaluated in a single ordered pass from top to bottom.
-# The global rules file (e.g. /home/bbs/maildir/.rules) is executed first, followed by
-# any .rules file in each user's individual maildir (e.g. /home/bbs/maildir/1/.rules)
+# The global before rules file (e.g. /home/bbs/maildir/before.rules ) is executed first, followed by
+# any .rules file in each user's individual maildir (e.g. /home/bbs/maildir/1/.rules),
+# followed by the global after rules file (e.g. /home/bbs/maildir/after.rules).
 # Rule processing occurs once the entire message has been received and before delivery is attempted.
 
 # WARNING: DO NOT ALLOW USERS to directly create or modify these rules.
@@ -40,15 +41,23 @@
 # DIRECTION <IN|OUT> - direction of message. IN=incoming message for a local recipient, OUT=message sent by a local user for delivery (either locally or externally)
 # MAILFROM <EQUALS|LIKE> <arg> - envelope MAIL FROM (envelope sender)
 # RECIPIENT <EQUALS|LIKE> <arg> - envelope RCPT TO (envelope recipient)
-# HEADER <header name> <EQUALS|LIKE|EXISTS> <arg> - header in the message. Since certain headers may be duplicated (To:, Cc:, etc.), this is a match on ANY of these headers.
+# HEADER <header name> <EXISTS|EQUALS|CONTAINS|LIKE|>=|>|<|<=|==> <arg> - header in the message. Since certain headers may be duplicated (To:, Cc:, etc.), this is a match on ANY of these headers.
 # FILE <file> <EXISTS> - whether a named file exists in the user's maildir. May be used for both incoming and outgoing messages.
 # RETVAL (>,>=,==,<,<=) <value> - check return value of last command
 # SIZE (>,>=,==,<,<=) <size> - size of message, in bytes
 
 # Condition Keywords:
+# EXISTS - specified value exists
 # EQUALS - exact string match
+# CONTAINS - contains substring
 # LIKE - regular expression match
-# EXISTS - specified value exists
+# >= - greater than or equal to
+# > - greater than
+# <= - less than or equal to
+# < less than
+# == equal to
+
+# Note that floating point comparisons are supported for HEADER conditions, but not for RETVAL or SIZE.
 
 # Other Keywords:
 # RETVAL - return code of previous TEST or ACTION statement, useful for conditional execution of certain MATCH or ACTION statements.
@@ -57,15 +66,18 @@
 # List of actions
 # BREAK - Stop executing the current rule
 # RETURN - Stop executing all rules in the current rules file
-# EXIT - Stop executing all rules in all rules files
-# BOUNCE - Reject SMTP acceptance of the message and return an SMTP error code to the sending server/client. Optionally, a custom bounce message may be specified as an argument.
-# DROP - Drop the message (prevent delivery). Typically used in conjunction with BOUNCE. If you wish to delete the message but retain it temporarily in the Trash folder, you should use MOVETO .Trash instead.
-# FORWARD - Forward the message to another address. Can be used to implement conditional or unconditional forwarding.
+# EXIT - Stop executing all rules in all rules files. May not be used in mailbox rules, can only be used in the global rules. Use with caution, you probably should use BREAK or RETURN instead!
+# BOUNCE - Return an SMTP error code to the sending server/client. Optionally, a custom bounce message may be specified as an argument. This rule does not reject acceptance on its own. Use with caution, you probably want to use REJECT instead unless you are trying to be clever.
+# REJECT - Same as Sieve 'reject' action. Reject SMTP acceptance of the message and return an SMTP error code to the sending server/client. Optionally, a custom bounce message may be specified as an argument.
+# DISCARD - Same as Sieve 'discard' action. Drop and discard the message (prevent delivery). This rule does not trigger a reject message on its own. If you wish to delete the message but retain it temporarily in the Trash folder, you should use MOVETO Trash instead. This rule is typically not implicit, only the REJECT action will automatically discard a message as well.
+# REDIRECT - Similar to the Sieve 'redirect' action. Forward the message to another address. Can be used to implement conditional or unconditional forwarding.
+#            Unlike the Sieve 'redirect' action, "keep copy" is implicit. Additionally, it can be used multiple times to forward copies to multiple destinations.
+#            To drop the message after forwarding it, also use the DISCARD action afterwards.
 # RELAY - Outgoing messages only. Relay the message via another SMTP server. Useful for SMTP proxying. Format is smtp:// or smtps://user:password@host:port
 #                       Note that STARTTLS is always attempted for smtp://, while smtps:// will force Implicit TLS to be used, and smtp:// will use Explicit TLS if possible.
-#                       Currently, RELAY implicitly results in a DROP, normal message processing will not continue afterwards.
+#                       Currently, RELAY implicitly results in a DISCARD, normal message processing will not continue afterwards and no copy of the message is retained.
 # REPLY - Reply to the sent message, to the original sender
-# MOVETO - Move the message to a specified folder in the maildir, e.g. to .Junk, .Trash, etc. Can be used to implement filtering. If this action is executed multiple times, the last one wins. For outgoing messages, an IMAP URI may also be used, e.g. to APPEND the sent message to a remote IMAP mailbox.
+# MOVETO - Move the message to a specified folder in the maildir, e.g. to Junk, Trash, Folder.subfolder, etc. Can be used to implement filtering. If this action is executed multiple times, the last one wins. For outgoing messages, an IMAP URI may also be used, e.g. to APPEND the sent message to a remote IMAP mailbox.
 # EXEC - Execute a system program.
 # NOOP - Does nothing and always returns 0. Possibly useful for debugging rule execution with debug enabled.
 
@@ -77,11 +89,11 @@ COMMENT
 
 # Reject all messages, incoming and outgoing
 RULE
-# You can use BOUNCE or DROP in isolation but typically these are used together
-# BOUNCE used by itself would send a bounce but actually deliver the message
-# DROP used by itself would drop the message but not send a bounce
-ACTION BOUNCE This message is not allowed # Send a custom bounce message
-ACTION DROP # Drop the message
+# You can use REJECT or DISCARD in isolation but typically these are used together
+# REJECT used by itself would send a bounce and not deliver the message
+# DISCARD used by itself would silently drop the message and not send a bounce
+ACTION REJECT This message is not allowed # Send a custom bounce message
+ACTION DISCARD # Drop and discard the message
 ENDRULE
 
 RULE
@@ -90,8 +102,8 @@ MATCH SIZE >= 700 # greater than 700 bytes
 MATCH NOT HEADER Precedence EQUALS Bulk # if Precedence header is not bulk
 MATCH HEADER Precedence EXISTS # ... and it's present (not missing)
 # if all these rules match:
-ACTION FORWARD <[email protected]> # forward the message to [email protected]
-ACTION MOVETO .Trash # delete the message after forwarding it
+ACTION REDIRECT <[email protected]> # forward the message to [email protected]
+ACTION MOVETO Trash # delete the message after forwarding it
 ACTION BREAK # stop processing the CURRENT rules. In this example, this isn't meaningful, but if there were further actions it would be.
 ENDRULE
 
@@ -108,19 +120,38 @@ ENDIF
 IF RETVAL == 0
 	ACTION BREAK
 ENDIF
-# This rule will bounce and drop the message if it came from <1|2|3>@example.com
-ACTION BOUNCE
-ACTION DROP
+# This rule will reject the message if it came from <1|2|3>@example.com
+ACTION REJECT
+ENDRULE
+
+# Forward any messages to [email protected] smaller than 1 KB to [email protected]
+RULE
+MATCH RECIPIENT EQUALS [email protected]
+IF SIZE > 1024
+	ACTION REJECT "Message is too large"
+	ACTION BREAK # This is necessary! If the explicit BREAK is missing, we will end up forwarding a copy nonetheless! REJECT is implicit drop, it does NOT implicitly stop processing rules!
+ENDIF
+ACTION REDIRECT <[email protected]>
 ENDRULE
 
 # A simple spam filter
 RULE
 MATCH DIRECTION IN # all incoming email (you don't want to spam filter the mail you send, right?)
+MATCH HEADER X-Spam-Level CONTAINS *****
+IF NOT RETVAL 0
+	ACTION MOVETO Junk
+	ACTION RETURN # stop processing ALL rules immediately (why bother if it's spam?)
+ENDIF
+ENDRULE
+
+# Alternate simple spam filter
+RULE
+MATCH DIRECTION IN # all incoming email (you don't want to spam filter the mail you send, right?)
 MATCH NOT FILE .nospamfilter MISSING # if .nospamfilter file is present in a user's root maildir folder, don't do any spam filtering
 MATCH NOT HEADER From LIKE ^[A-Za-z0-9._%+-]+@safe\.example\.com$ # safe/allowed sender
 ACTION EXEC spamassasin -e ${MAILFILE} # run spamassasin on this file, which will return nonzero if it's spam
 IF NOT RETVAL 0
-	ACTION MOVETO .Junk
+	ACTION MOVETO Junk
 	ACTION RETURN # stop processing ALL rules immediately (why bother if it's spam?)
 ENDIF
 ENDRULE
@@ -150,12 +181,12 @@ ENDRULE
 
 # Prevent accidentally sending emails to the wrong people. e.g. the same message should never contain [email protected] and [email protected].
 # You can see how you could use this to prevent accidentally replying all to email lists you never intend to post to, etc. which could be quite handy!
+# mod_smtp_recipient_monitor does more in-depth sender/recipient message analysis
 RULE
 MATCH DIRECTION OUT
 MATCH HEADER To EQUALS [email protected]
 MATCH HEADER To EQUALS [email protected]
-ACTION BOUNCE # send bounce message
-ACTION DROP # drop the message, prevent it from being sent.
+ACTION REJECT # reject message, prevent it from being sent
 ENDRULE
 
 ENDCOMMENT

include/net_smtp.h

@@ -230,6 +230,12 @@ int smtp_message_quarantinable(struct smtp_session *smtp);
 #define SMTP_MSG_DIRECTION_IN 0
 #define SMTP_MSG_DIRECTION_OUT 1
 
+enum msg_process_iteration {
+	FILTER_BEFORE_MAILBOX = 0,	/*!< Execute before the mailbox filters */
+	FILTER_MAILBOX,				/*!< Mailbox filter execution */
+	FILTER_AFTER_MAILBOX,		/*!< Execute after the mailbox filters */
+};
+
 struct smtp_msg_process {
 	/* Inputs */
 	struct smtp_session *smtp;	/*!< SMTP session. Not originally included, so try to avoid using this! */
@@ -246,8 +252,9 @@ struct smtp_msg_process {
 	unsigned int userid;		/*!< User ID (outgoing only) */
 	enum smtp_direction dir;	/*!< Full direction (IN, OUT, or SUBMIT) */
 	unsigned int direction:1;	/*!< 0 = incoming, 1 = outgoing */
+	enum msg_process_iteration iteration; /*!< Which processing pass this is */
 	/* Outputs */
-	unsigned int bounce:1;		/*!< Whether to send a bounce */
+	unsigned int bounce:1;		/*!< Whether to send a bounce. This on its own does not also implicitly drop the message, that bit must be explicitly set. */
 	unsigned int drop:1;		/*!< Whether message should be dropped */
 	int res;					/*!< General return code */
 	char *newdir;				/*!< New message location (incoming only) */
@@ -260,7 +267,7 @@ struct smtp_msg_process {
 void smtp_mproc_init(struct smtp_session *smtp, struct smtp_msg_process *mproc);
 
 /*!
- * \brief Register an SMTP processor callback to run on each message received or sent
+ * \brief Register an SMTP processor callback to run on each message received or sent. Callback will be called 3x, once for each msg_process_order.
  * \param cb Callback that should return nonzero to stop processing further callbacks
  */
 #define smtp_register_processor(cb) __smtp_register_processor(cb, BBS_MODULE_SELF)
@@ -272,6 +279,8 @@ int smtp_unregister_processor(int (*cb)(struct smtp_msg_process *mproc));
 
 /*!
  * \brief Run SMTP callbacks for a message (only called by net_smtp)
+ * \param mproc
+ * \retval 0 to continue, -1 to abort transaction immediately
  */
 int smtp_run_callbacks(struct smtp_msg_process *mproc);
 
@@ -282,6 +291,20 @@ struct smtp_response {
 	const char *reply;
 };
 
+/*!
+ * \brief Wrapper around smtp_run_callbacks, for use by delivery handlers
+ * \param smtp
+ * \param mproc Does not need to be (and should not be) initialized
+ * \param mbox Mailbox or NULL
+ * \param resp
+ * \param dir
+ * \param recipient Recipient, with <>
+ * \param datalen Message size
+ * \param freedata
+ * \retval 0 to continue, nonzero to return the return value
+ */
+int smtp_run_delivery_callbacks(struct smtp_session *smtp, struct smtp_msg_process *mproc, struct mailbox *mbox, struct smtp_response **restrict resp, enum smtp_direction dir, const char *recipient, size_t datalen, void **freedata);
+
 #define smtp_abort(r, c, sub, msg) \
 	r->code = c; \
 	r->subcode = #sub; \