//
// This is the new, exciting, clever version that makes libcurl do all the work :)
//
-// Copyright (c) 1997-2022 by the citadel.org team
+// Copyright (c) 1997-2023 by the citadel.org team
//
-// This program is open source software; you can redistribute it and/or
-// modify it under the terms of the GNU General Public License as published
-// by the Free Software Foundation; either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// This program is open source software. Use, duplication, or disclosure
+// is subject to the terms of the GNU General Public License, version 3.
#include <stdlib.h>
#include <unistd.h>
#include <libcitadel.h>
#include <curl/curl.h>
#include "../../sysconfig.h"
-#include "../../citadel.h"
+#include "../../citadel_defs.h"
#include "../../server.h"
#include "../../citserver.h"
#include "../../support.h"
#include "../../citadel_dirs.h"
#include "../smtp/smtp_util.h"
+long last_queue_job_submitted = 0;
+long last_queue_job_processed = 0;
+
struct smtpmsgsrc { // Data passed in and out of libcurl for message upload
StrBuf *TheMessage;
int bytes_total;
int bytes_sent;
};
-static int doing_smtpclient = 0;
-long *smtpq = NULL; // array of msgnums containing queue instructions
-int smtpq_count = 0; // number of queue messages in smtpq
-int smtpq_alloc = 0; // current allocation size for smtpq
-
-
// Initialize the SMTP outbound queue
void smtp_init_spoolout(void) {
struct ctdlroom qrbuf;
}
-// For internet mail, generate delivery instructions.
+// For internet mail, generate a delivery job.
// Yes, this is recursive. Deal with it. Infinite recursion does
-// not happen because the delivery instructions message does not
-// contain a recipient.
+// not happen because the message containing the delivery job does not
+// have a recipient.
int smtp_aftersave(struct CtdlMessage *msg, struct recptypes *recps) {
if ((recps != NULL) && (recps->num_internet > 0)) {
struct CtdlMessage *imsg = NULL;
long nTokens;
int i;
- syslog(LOG_DEBUG, "smtpclient: generating delivery instructions");
+ syslog(LOG_DEBUG, "smtpclient: generating delivery job");
StrBufPrintf(SpoolMsg,
"Content-type: " SPOOLMIME "\n"
imsg->cm_magic = CTDLMESSAGE_MAGIC;
imsg->cm_anon_type = MES_NORMAL;
imsg->cm_format_type = FMT_RFC822;
- CM_SetField(imsg, eMsgSubject, HKEY("QMSG"));
- CM_SetField(imsg, eAuthor, HKEY("Citadel"));
- CM_SetField(imsg, eJournal, HKEY("do not journal"));
- CM_SetAsFieldSB(imsg, eMesageText, &SpoolMsg);
- CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM);
+ CM_SetField(imsg, eMsgSubject, "QMSG");
+ CM_SetField(imsg, eAuthor, "Citadel");
+ CM_SetField(imsg, eJournal, "do not journal");
+ CM_SetAsFieldSB(imsg, eMessageText, &SpoolMsg);
+ last_queue_job_submitted = CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM);
CM_Free(imsg);
}
return 0;
return;
}
- char *t = malloc(strlen(response));
- if (!t) {
- return;
- }
- t[0] = 0;
-
char *p;
for (p = response; *p != 0; ++p) {
if ( (*p != '\n') && (!isprint(*p)) ) { // expunge any nonprintables except for newlines
// Construct an SMTP URL in the form of:
// smtp[s]://target_host/source_host
// This looks weird but libcurl uses that last part to set our name for EHLO or HELO.
- // We check for "smtp://" and "smtps://" because the admin may have put those prefixes in a smart-host entry.
+ // We check for "smtp://" and "smtps://" because an admin may have put those prefixes in a smart-host entry
// If there is no prefix we add "smtp://"
extract_token(try_this_mx, mxes, i, '|', (sizeof try_this_mx - 7));
snprintf(smtp_url, sizeof smtp_url,
return;
}
- instr = msg->cm_fields[eMesageText];
- msg->cm_fields[eMesageText] = NULL;
+ instr = msg->cm_fields[eMessageText];
+ msg->cm_fields[eMessageText] = NULL;
CM_Free(msg);
- // if the queue message has any CRLF's convert them to LF's
+ // if the queue job message has any CRLF's convert them to LF's
char *crlf = NULL;
while (crlf = strstr(instr, "\r\n"), crlf != NULL) {
strcpy(crlf, crlf + 1);
char cfgline[SIZ];
for (i = 0; i < num_tokens(instr, '\n'); ++i) {
extract_token(cfgline, instr, i, '\n', sizeof cfgline);
- if (!strncasecmp(cfgline, HKEY("msgid|")))
- msgid = atol(&cfgline[6]);
- if (!strncasecmp(cfgline, HKEY("submitted|")))
- submitted = atol(&cfgline[10]);
- if (!strncasecmp(cfgline, HKEY("attempted|")))
- attempted = atol(&cfgline[10]);
- if (!strncasecmp(cfgline, HKEY("bounceto|")))
- bounceto = strdup(&cfgline[9]);
- if (!strncasecmp(cfgline, HKEY("envelope_from|")))
- envelope_from = strdup(&cfgline[14]);
- if (!strncasecmp(cfgline, HKEY("source_room|")))
- source_room = strdup(&cfgline[12]);
+ if (!strncasecmp(cfgline, HKEY("msgid|"))) msgid = atol(&cfgline[6]);
+ if (!strncasecmp(cfgline, HKEY("submitted|"))) submitted = atol(&cfgline[10]);
+ if (!strncasecmp(cfgline, HKEY("attempted|"))) attempted = atol(&cfgline[10]);
+ if (!strncasecmp(cfgline, HKEY("bounceto|"))) bounceto = strdup(&cfgline[9]);
+ if (!strncasecmp(cfgline, HKEY("envelope_from|"))) envelope_from = strdup(&cfgline[14]);
+ if (!strncasecmp(cfgline, HKEY("source_room|"))) source_room = strdup(&cfgline[12]);
}
int should_try_now = 0;
if (!strncasecmp(cfgline, HKEY("remote|"))) {
char recp[SIZ];
int previous_result = extract_int(cfgline, 2);
- if ((previous_result == 0)
- || (previous_result == 4)) {
+ if ((previous_result == 0) || (previous_result == 4)) {
int new_result = 421;
extract_token(recp, cfgline, 1, '|', sizeof recp);
new_result = smtp_attempt_delivery(msgid, recp, envelope_from, source_room, server_response);
else {
++num_delayed;
}
- StrBufAppendPrintf
- (NewInstr,
- "remote|%s|%ld|%ld (%s)\n",
- recp, (new_result / 100), new_result, server_response);
+ StrBufAppendPrintf(NewInstr, "remote|%s|%ld|%ld (%s)\n", recp, (new_result / 100), new_result, server_response);
}
}
}
delete_this_queue = 1;
}
// If it's been more than four hours but less than five days, warn the sender that delivery is delayed
- else if (((attempted - submitted) < SMTP_DELIVER_WARN)
- && ((time(NULL) - submitted) >= SMTP_DELIVER_WARN)) {
+ else if (((attempted - submitted) < SMTP_DELIVER_WARN) && ((time(NULL) - submitted) >= SMTP_DELIVER_WARN)) {
smtp_do_bounce(ChrPtr(NewInstr), SDB_WARN);
}
// Callback for smtp_do_queue()
void smtp_add_msg(long msgnum, void *userdata) {
+ Array *smtp_queue = (Array *) userdata;
+ array_append(smtp_queue, &msgnum);
+}
- if (smtpq == NULL) {
- smtpq_count = 0;
- smtpq_alloc = 100;
- smtpq = malloc(smtpq_alloc * sizeof(long));
- }
-
- if (smtpq_alloc >= smtpq_count) {
- smtpq_alloc += 100;
- smtpq = realloc(smtpq, (smtpq_alloc * sizeof(long)));
- }
- smtpq[smtpq_count++] = msgnum;
-}
+enum {
+ FULL_QUEUE_RUN, // try to process the entire queue, including messages that have already been attempted
+ QUICK_QUEUE_RUN // only process jobs in the queue that have not been tried yet
+};
// Run through the queue sending out messages.
-void smtp_do_queue(void) {
+void smtp_do_queue(int type_of_queue_run) {
+ static int doing_smtpclient = 0;
int i = 0;
- // This is a simple concurrency check to make sure only one smtpclient
- // run is done at a time. We could do this with a mutex, but since we
- // don't really require extremely fine granularity here, we'll do it
- // with a static variable instead.
+ // This is a concurrency check to make sure only one smtpclient run is done at a time.
+ begin_critical_section(S_SMTPQUEUE);
if (doing_smtpclient) {
+ end_critical_section(S_SMTPQUEUE);
return;
}
doing_smtpclient = 1;
+ end_critical_section(S_SMTPQUEUE);
- syslog(LOG_DEBUG, "smtpclient: start queue run");
+ syslog(LOG_DEBUG, "smtpclient: start %s queue run , last_queue_job_processed=%ld , last_queue_job_submitted=%ld",
+ (type_of_queue_run == QUICK_QUEUE_RUN ? "quick" : "full"),
+ last_queue_job_processed, last_queue_job_submitted
+ );
if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
- syslog(LOG_WARNING, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
+ syslog(LOG_WARNING, "smtpclient: cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
doing_smtpclient = 0;
return;
}
+
+ // This array will hold the list of queue job messages
+ Array *smtp_queue = array_new(sizeof(long));
+ if (smtp_queue == NULL) {
+ syslog(LOG_WARNING, "smtpclient: cannot allocate queue array");
+ doing_smtpclient = 0;
+ return;
+ }
+
// Put the queue in memory so we can close the db cursor
- CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_add_msg, NULL);
+ CtdlForEachMessage(
+ (type_of_queue_run == QUICK_QUEUE_RUN ? MSGS_GT : MSGS_ALL), // quick = new jobs; full = all jobs
+ (type_of_queue_run == QUICK_QUEUE_RUN ? last_queue_job_processed : 0), // quick = new jobs; full = all jobs
+ NULL,
+ SPOOLMIME, // Searching for Content-type of SPOOLIME will give us only queue instruction messages
+ NULL,
+ smtp_add_msg, // That's our callback function to add a job to the queue
+ (void *)smtp_queue
+ );
// We are ready to run through the queue now.
- for (i = 0; i < smtpq_count; ++i) {
- smtp_process_one_msg(smtpq[i]);
+ syslog(LOG_DEBUG, "smtpclient: %d messages to be processed", array_len(smtp_queue));
+ for (i = 0; i < array_len(smtp_queue); ++i) {
+ long m;
+ memcpy(&m, array_get_element_at(smtp_queue, i), sizeof(long));
+ smtp_process_one_msg(m);
}
- smtpq_count = 0; // don't free it, we will use this memory on the next run
+ array_free(smtp_queue);
+ last_queue_job_processed = last_queue_job_submitted;
doing_smtpclient = 0;
- syslog(LOG_DEBUG, "smtpclient: end queue run");
+ syslog(LOG_DEBUG, "smtpclient: end %s queue run , last_queue_job_processed=%ld , last_queue_job_submitted=%ld",
+ (type_of_queue_run == QUICK_QUEUE_RUN ? "quick" : "full"),
+ last_queue_job_processed, last_queue_job_submitted
+ );
+}
+
+
+// The "full" queue run goes through the entire queue, attempting delivery for newly submitted messages,
+// retrying failed deliveries periodically, and handling undeliverable messages.
+void smtp_do_queue_full(void) {
+ smtp_do_queue(FULL_QUEUE_RUN);
+}
+
+
+// The "quick" queue run only handles newly submitted messages, allowing them to be delivered immediately
+// instead of waiting for the next "full" queue run.
+void smtp_do_queue_quick(void) {
+ if (last_queue_job_submitted > last_queue_job_processed) {
+ smtp_do_queue(QUICK_QUEUE_RUN);
+ }
}
char *ctdl_module_init_smtpclient(void) {
if (!threading) {
CtdlRegisterMessageHook(smtp_aftersave, EVT_AFTERSAVE);
- CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER, PRIO_AGGR + 51);
+ CtdlRegisterSessionHook(smtp_do_queue_quick, EVT_HOUSE, PRIO_AGGR + 51);
+ CtdlRegisterSessionHook(smtp_do_queue_full, EVT_TIMER, PRIO_AGGR + 51);
smtp_init_spoolout();
}