//
// This is the new, exciting, clever version that makes libcurl do all the work :)
//
-// Copyright (c) 1997-2023 by the citadel.org team
+// Copyright (c) 1997-2024 by the citadel.org team
//
// This program is open source software. Use, duplication, or disclosure
// is subject to the terms of the GNU General Public License, version 3.
#include <libcitadel.h>
#include <curl/curl.h>
#include "../../sysconfig.h"
-#include "../../citadel.h"
+#include "../../citadel_defs.h"
#include "../../server.h"
#include "../../citserver.h"
#include "../../support.h"
#include "../../citadel_dirs.h"
#include "../smtp/smtp_util.h"
+long last_queue_job_submitted = 0;
+long last_queue_job_processed = 0;
+
struct smtpmsgsrc { // Data passed in and out of libcurl for message upload
StrBuf *TheMessage;
int bytes_total;
int bytes_sent;
};
-static int doing_smtpclient = 0;
-
// Initialize the SMTP outbound queue
void smtp_init_spoolout(void) {
struct ctdlroom qrbuf;
}
-// For internet mail, generate delivery instructions.
+// For internet mail, generate a delivery job.
// Yes, this is recursive. Deal with it. Infinite recursion does
-// not happen because the delivery instructions message does not
-// contain a recipient.
+// not happen because the message containing the delivery job does not
+// have a recipient.
int smtp_aftersave(struct CtdlMessage *msg, struct recptypes *recps) {
if ((recps != NULL) && (recps->num_internet > 0)) {
struct CtdlMessage *imsg = NULL;
long nTokens;
int i;
- syslog(LOG_DEBUG, "smtpclient: generating delivery instructions");
+ syslog(LOG_DEBUG, "smtpclient: generating delivery job");
StrBufPrintf(SpoolMsg,
"Content-type: " SPOOLMIME "\n"
imsg->cm_magic = CTDLMESSAGE_MAGIC;
imsg->cm_anon_type = MES_NORMAL;
imsg->cm_format_type = FMT_RFC822;
- CM_SetField(imsg, eMsgSubject, HKEY("QMSG"));
- CM_SetField(imsg, eAuthor, HKEY("Citadel"));
- CM_SetField(imsg, eJournal, HKEY("do not journal"));
- CM_SetAsFieldSB(imsg, eMesageText, &SpoolMsg);
- CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM);
+ CM_SetField(imsg, eMsgSubject, "QMSG");
+ CM_SetField(imsg, eAuthor, "Citadel");
+ CM_SetField(imsg, eJournal, "do not journal");
+ CM_SetAsFieldSB(imsg, eMessageText, &SpoolMsg);
+ last_queue_job_submitted = CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM);
CM_Free(imsg);
}
return 0;
return;
}
- char *t = malloc(strlen(response));
- if (!t) {
- return;
- }
- t[0] = 0;
-
char *p;
for (p = response; *p != 0; ++p) {
if ( (*p != '\n') && (!isprint(*p)) ) { // expunge any nonprintables except for newlines
process_rfc822_addr(recp, user, node, name); // split recipient address into username, hostname, displayname
num_mx = getmx(mxes, node);
if (num_mx < 1) {
- return (421);
+ return(421);
}
CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
+
+ // If we have a source room, it's probably a mailing list message; generate an unsubscribe header
if (!IsEmptyStr(source_room)) {
- // If we have a source room, it's probably a mailing list message; generate an unsubscribe header
- char esc_room[ROOMNAMELEN*2];
- char esc_email[1024];
- urlesc(esc_room, sizeof esc_room, source_room);
- urlesc(esc_email, sizeof esc_email, recp);
- cprintf("List-Unsubscribe: <http://%s/listsub?cmd=unsubscribe&room=%s&email=%s>\r\n",
- CtdlGetConfigStr("c_fqdn"),
- esc_room,
- esc_email
- );
+ char base_url[SIZ];
+ char unsubscribe_url[SIZ];
+ snprintf(base_url, sizeof base_url, "https://%s/listsub", CtdlGetConfigStr("c_fqdn"));
+ generate_one_click_url(unsubscribe_url, base_url, "unsubscribe", source_room, recp);
+ cprintf("List-Unsubscribe: %s\r\n", unsubscribe_url);
+ cprintf("List-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n"); // RFC 8058
+
}
+
CtdlOutputMsg(msgid, MT_RFC822, HEADERS_ALL, 0, 1, NULL, 0, NULL, &fromaddr, NULL);
s.TheMessage = CC->redirect_buffer;
- s.bytes_total = StrLength(CC->redirect_buffer);
- s.bytes_sent = 0;
CC->redirect_buffer = NULL;
+ s.bytes_total = StrLength(s.TheMessage);
+ s.bytes_sent = 0;
response_code = 421;
- // keep trying MXes until one works or we run out
+
+ // Future enhancement: if we implement DKIM signing, this is where it must happen.
+
+ // Keep trying MXes until one works or we run out.
for (i = 0; ((i < num_mx) && ((response_code / 100) == 4)); ++i) {
response_code = 421; // default 421 makes non-protocol errors transient
s.bytes_sent = 0; // rewind our buffer in case we try multiple MXes
// Construct an SMTP URL in the form of:
// smtp[s]://target_host/source_host
// This looks weird but libcurl uses that last part to set our name for EHLO or HELO.
- // We check for "smtp://" and "smtps://" because the admin may have put those prefixes in a smart-host entry.
+ // We check for "smtp://" and "smtps://" because an admin may have put those prefixes in a smart-host entry
// If there is no prefix we add "smtp://"
extract_token(try_this_mx, mxes, i, '|', (sizeof try_this_mx - 7));
snprintf(smtp_url, sizeof smtp_url,
msg = CtdlFetchMessage(qmsgnum, 1);
if (msg == NULL) {
- syslog(LOG_WARNING, "smtpclient: %ld does not exist", qmsgnum);
+ syslog(LOG_WARNING, "smtpclient: msg#%ld does not exist", qmsgnum);
return;
}
- instr = msg->cm_fields[eMesageText];
- msg->cm_fields[eMesageText] = NULL;
+ instr = msg->cm_fields[eMessageText];
+ msg->cm_fields[eMessageText] = NULL;
CM_Free(msg);
- // if the queue message has any CRLF's convert them to LF's
+ // if the queue job message has any CRLF's convert them to LF's
char *crlf = NULL;
while (crlf = strstr(instr, "\r\n"), crlf != NULL) {
strcpy(crlf, crlf + 1);
char cfgline[SIZ];
for (i = 0; i < num_tokens(instr, '\n'); ++i) {
extract_token(cfgline, instr, i, '\n', sizeof cfgline);
- if (!strncasecmp(cfgline, HKEY("msgid|")))
- msgid = atol(&cfgline[6]);
- if (!strncasecmp(cfgline, HKEY("submitted|")))
- submitted = atol(&cfgline[10]);
- if (!strncasecmp(cfgline, HKEY("attempted|")))
- attempted = atol(&cfgline[10]);
- if (!strncasecmp(cfgline, HKEY("bounceto|")))
- bounceto = strdup(&cfgline[9]);
- if (!strncasecmp(cfgline, HKEY("envelope_from|")))
- envelope_from = strdup(&cfgline[14]);
- if (!strncasecmp(cfgline, HKEY("source_room|")))
- source_room = strdup(&cfgline[12]);
+ if (!strncasecmp(cfgline, HKEY("msgid|"))) msgid = atol(&cfgline[6]);
+ if (!strncasecmp(cfgline, HKEY("submitted|"))) submitted = atol(&cfgline[10]);
+ if (!strncasecmp(cfgline, HKEY("attempted|"))) attempted = atol(&cfgline[10]);
+ if (!strncasecmp(cfgline, HKEY("bounceto|"))) bounceto = strdup(&cfgline[9]);
+ if (!strncasecmp(cfgline, HKEY("envelope_from|"))) envelope_from = strdup(&cfgline[14]);
+ if (!strncasecmp(cfgline, HKEY("source_room|"))) source_room = strdup(&cfgline[12]);
}
int should_try_now = 0;
}
}
else {
- syslog(LOG_DEBUG, "smtpclient: %ld retry time not reached", qmsgnum);
+ syslog(LOG_DEBUG, "smtpclient: msg#%ld retry time not reached", qmsgnum);
}
if (bounceto != NULL) {
}
+enum {
+ FULL_QUEUE_RUN, // try to process the entire queue, including messages that have already been attempted
+ QUICK_QUEUE_RUN // only process jobs in the queue that have not been tried yet
+};
+
+
// Run through the queue sending out messages.
-void smtp_do_queue(void) {
+void smtp_do_queue(int type_of_queue_run) {
+ static int doing_smtpclient = 0;
int i = 0;
- // This is a simple concurrency check to make sure only one smtpclient
- // run is done at a time. We could do this with a mutex, but since we
- // don't really require extremely fine granularity here, we'll do it
- // with a static variable instead.
+ // This is a concurrency check to make sure only one smtpclient run is done at a time.
+ begin_critical_section(S_SMTPQUEUE);
if (doing_smtpclient) {
+ end_critical_section(S_SMTPQUEUE);
return;
}
doing_smtpclient = 1;
+ end_critical_section(S_SMTPQUEUE);
- syslog(LOG_DEBUG, "smtpclient: start queue run");
+ syslog(LOG_DEBUG, "smtpclient: start %s queue run , last_queue_job_processed=%ld , last_queue_job_submitted=%ld",
+ (type_of_queue_run == QUICK_QUEUE_RUN ? "quick" : "full"),
+ last_queue_job_processed, last_queue_job_submitted
+ );
if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
syslog(LOG_WARNING, "smtpclient: cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
return;
}
- // This array will hold the list of queue instruction messages
+ // This array will hold the list of queue job messages
Array *smtp_queue = array_new(sizeof(long));
if (smtp_queue == NULL) {
syslog(LOG_WARNING, "smtpclient: cannot allocate queue array");
}
// Put the queue in memory so we can close the db cursor
- CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_add_msg, (void *)smtp_queue);
+ CtdlForEachMessage(
+ (type_of_queue_run == QUICK_QUEUE_RUN ? MSGS_GT : MSGS_ALL), // quick = new jobs; full = all jobs
+ (type_of_queue_run == QUICK_QUEUE_RUN ? last_queue_job_processed : 0), // quick = new jobs; full = all jobs
+ NULL,
+ SPOOLMIME, // Searching for Content-type of SPOOLIME will give us only queue instruction messages
+ NULL,
+ smtp_add_msg, // That's our callback function to add a job to the queue
+ (void *)smtp_queue
+ );
// We are ready to run through the queue now.
+ syslog(LOG_DEBUG, "smtpclient: %d messages to be processed", array_len(smtp_queue));
for (i = 0; i < array_len(smtp_queue); ++i) {
long m;
memcpy(&m, array_get_element_at(smtp_queue, i), sizeof(long));
}
array_free(smtp_queue);
+ last_queue_job_processed = last_queue_job_submitted;
doing_smtpclient = 0;
- syslog(LOG_DEBUG, "smtpclient: end queue run");
+ syslog(LOG_DEBUG, "smtpclient: end %s queue run , last_queue_job_processed=%ld , last_queue_job_submitted=%ld",
+ (type_of_queue_run == QUICK_QUEUE_RUN ? "quick" : "full"),
+ last_queue_job_processed, last_queue_job_submitted
+ );
+}
+
+
+// The "full" queue run goes through the entire queue, attempting delivery for newly submitted messages,
+// retrying failed deliveries periodically, and handling undeliverable messages.
+void smtp_do_queue_full(void) {
+ smtp_do_queue(FULL_QUEUE_RUN);
+}
+
+
+// The "quick" queue run only handles newly submitted messages, allowing them to be delivered immediately
+// instead of waiting for the next "full" queue run.
+void smtp_do_queue_quick(void) {
+ if (last_queue_job_submitted > last_queue_job_processed) {
+ smtp_do_queue(QUICK_QUEUE_RUN);
+ }
}
char *ctdl_module_init_smtpclient(void) {
if (!threading) {
CtdlRegisterMessageHook(smtp_aftersave, EVT_AFTERSAVE);
- CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER, PRIO_AGGR + 51);
+ CtdlRegisterSessionHook(smtp_do_queue_quick, EVT_HOUSE, PRIO_AGGR + 51);
+ CtdlRegisterSessionHook(smtp_do_queue_full, EVT_TIMER, PRIO_AGGR + 51);
smtp_init_spoolout();
}