1 // Transmit outbound SMTP mail to the big wide world of the Internet
3 // This is the new, exciting, clever version that makes libcurl do all the work :)
5 // Copyright (c) 1997-2024 by the citadel.org team
7 // This program is open source software. Use, duplication, or disclosure
8 // is subject to the terms of the GNU General Public License, version 3.
17 #include <sys/types.h>
19 #include <libcitadel.h>
20 #include <curl/curl.h>
21 #include "../../sysconfig.h"
22 #include "../../citadel_defs.h"
23 #include "../../server.h"
24 #include "../../citserver.h"
25 #include "../../support.h"
26 #include "../../config.h"
27 #include "../../ctdl_module.h"
28 #include "../../clientsocket.h"
29 #include "../../msgbase.h"
30 #include "../../domain.h"
31 #include "../../internet_addressing.h"
32 #include "../../citadel_dirs.h"
33 #include "../smtp/smtp_util.h"
35 long last_queue_job_submitted = 0;
36 long last_queue_job_processed = 0;
38 struct smtpmsgsrc { // Data passed in and out of libcurl for message upload
44 // Initialize the SMTP outbound queue
45 void smtp_init_spoolout(void) {
46 struct ctdlroom qrbuf;
48 // Create the room. This will silently fail if the room already
49 // exists, and that's perfectly ok, because we want it to exist.
50 CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
52 // Make sure it's set to be a "system room" so it doesn't show up
53 // in the <K>nown rooms list for administrators.
54 if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
55 qrbuf.QRflags2 |= QR2_SYSTEM;
56 CtdlPutRoomLock(&qrbuf);
61 // For internet mail, generate a delivery job.
62 // Yes, this is recursive. Deal with it. Infinite recursion does
63 // not happen because the message containing the delivery job does not
65 int smtp_aftersave(struct CtdlMessage *msg, struct recptypes *recps) {
66 if ((recps != NULL) && (recps->num_internet > 0)) {
67 struct CtdlMessage *imsg = NULL;
69 StrBuf *SpoolMsg = NewStrBuf();
73 syslog(LOG_DEBUG, "smtpclient: generating delivery job");
75 StrBufPrintf(SpoolMsg,
76 "Content-type: " SPOOLMIME "\n"
79 "submitted|%ld\n" "bounceto|%s\n", msg->cm_fields[eVltMsgNum], (long) time(NULL), recps->bounce_to);
81 if (recps->envelope_from != NULL) {
82 StrBufAppendBufPlain(SpoolMsg, HKEY("envelope_from|"), 0);
83 StrBufAppendBufPlain(SpoolMsg, recps->envelope_from, -1, 0);
84 StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
86 if (recps->sending_room != NULL) {
87 StrBufAppendBufPlain(SpoolMsg, HKEY("source_room|"), 0);
88 StrBufAppendBufPlain(SpoolMsg, recps->sending_room, -1, 0);
89 StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
92 nTokens = num_tokens(recps->recp_internet, '|');
93 for (i = 0; i < nTokens; i++) {
95 len = extract_token(recipient, recps->recp_internet, i, '|', sizeof recipient);
97 StrBufAppendBufPlain(SpoolMsg, HKEY("remote|"), 0);
98 StrBufAppendBufPlain(SpoolMsg, recipient, len, 0);
99 StrBufAppendBufPlain(SpoolMsg, HKEY("|0||\n"), 0);
103 imsg = malloc(sizeof(struct CtdlMessage));
104 memset(imsg, 0, sizeof(struct CtdlMessage));
105 imsg->cm_magic = CTDLMESSAGE_MAGIC;
106 imsg->cm_anon_type = MES_NORMAL;
107 imsg->cm_format_type = FMT_RFC822;
108 CM_SetField(imsg, eMsgSubject, "QMSG");
109 CM_SetField(imsg, eAuthor, "Citadel");
110 CM_SetField(imsg, eJournal, "do not journal");
111 CM_SetAsFieldSB(imsg, eMessageText, &SpoolMsg);
112 last_queue_job_submitted = CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM);
119 // Callback for smtp_attempt_delivery() to supply libcurl with upload data.
120 static size_t upload_source(void *ptr, size_t size, size_t nmemb, void *userp) {
121 struct smtpmsgsrc *s = (struct smtpmsgsrc *) userp;
123 const char *send_this = NULL;
125 sendbytes = (size * nmemb);
127 if (s->bytes_sent >= s->bytes_total) {
128 return (0); // no data remaining; we are done
131 if (sendbytes > (s->bytes_total - s->bytes_sent)) {
132 sendbytes = s->bytes_total - s->bytes_sent; // can't send more than we have
135 send_this = ChrPtr(s->TheMessage);
136 send_this += s->bytes_sent; // start where we last left off
138 memcpy(ptr, send_this, sendbytes);
139 s->bytes_sent += sendbytes;
140 return(sendbytes); // return the number of bytes _actually_ copied
144 // The libcurl API doesn't provide a way to capture the actual SMTP result message returned
145 // by the remote server. This is an ugly way to extract it, by capturing debug data from
146 // the library and filtering on the lines we want.
147 int ctdl_libcurl_smtp_debug_callback(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr) {
148 if (type != CURLINFO_HEADER_IN)
152 char *debugbuf = (char *) userptr;
154 int len = strlen(debugbuf);
155 if (len + size > SIZ)
158 memcpy(&debugbuf[len], data, size);
159 debugbuf[len + size] = 0;
164 // Go through the debug output of an SMTP transaction, and boil it down to just the final success or error response message.
165 void trim_response(long response_code, char *response) {
166 if ((response_code < 100) || (response_code > 999) || (IsEmptyStr(response))) {
171 for (p = response; *p != 0; ++p) {
172 if ( (*p != '\n') && (!isprint(*p)) ) { // expunge any nonprintables except for newlines
177 char response_code_str[4];
178 snprintf(response_code_str, sizeof response_code_str, "%ld", response_code);
179 char *respstart = strstr(response, response_code_str);
180 if (respstart == NULL) { // If we have a response code but no response text,
181 strcpy(response, smtpstatus(response_code)); // use one of our canned messages.
184 strcpy(response, respstart);
186 p = strstr(response, "\n");
193 // Attempt a delivery to one recipient.
194 // Returns a three-digit SMTP status code.
195 int smtp_attempt_delivery(long msgid, char *recp, char *envelope_from, char *source_room, char *response) {
197 char *fromaddr = NULL;
199 CURLcode res = CURLE_OK;
200 struct curl_slist *recipients = NULL;
201 long response_code = 421;
207 char try_this_mx[256];
211 syslog(LOG_DEBUG, "smtpclient: smtp_attempt_delivery(%ld, %s)", msgid, recp);
213 process_rfc822_addr(recp, user, node, name); // split recipient address into username, hostname, displayname
214 num_mx = getmx(mxes, node);
219 CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
221 // If we have a source room, it's probably a mailing list message; generate an unsubscribe header
222 if (!IsEmptyStr(source_room)) {
224 char unsubscribe_url[SIZ];
225 snprintf(base_url, sizeof base_url, "https://%s/listsub", CtdlGetConfigStr("c_fqdn"));
226 generate_one_click_url(unsubscribe_url, base_url, "unsubscribe", source_room, recp);
227 cprintf("List-Unsubscribe: %s\r\n", unsubscribe_url);
228 cprintf("List-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n"); // RFC 8058
232 CtdlOutputMsg(msgid, MT_RFC822, HEADERS_ALL, 0, 1, NULL, 0, NULL, &fromaddr, NULL);
233 s.TheMessage = CC->redirect_buffer;
234 CC->redirect_buffer = NULL;
235 syslog(LOG_DEBUG, "fromaddr=<%s>",fromaddr);
237 // If we have a DKIM key, try to sign the message.
238 char *dkim_private_key = CtdlGetConfigStr("dkim_private_key");
239 char *dkim_selector = CtdlGetConfigStr("dkim_selector");
240 char *dkim_from_domain = (strchr(fromaddr, '@') ? strchr(fromaddr, '@')+1 : NULL);
242 !IsEmptyStr(dkim_from_domain) // Is the sending domain non-empty?
243 && IsDirectory(fromaddr, 0) // and is it one of "our" domains?
244 && !IsEmptyStr(dkim_private_key) // Do we have a private signing key?
245 && !IsEmptyStr(dkim_selector) // and a selector to go with it?
247 char *pkey = strdup(dkim_private_key); // If you answered "yes" to all of the above questions,
248 if (pkey) { // congratulations! We get to DKIM-sign the message!
250 while (sp = strchr(pkey, '_')) { // The dkim_private_key record contains our RSA private key,
251 *sp = '\n'; // but we have to convert all the newlines back to underscores.
253 syslog(LOG_DEBUG, "smtpclient: dkim-signing with private key for selector <%s> domain <%s>",
254 dkim_selector, dkim_from_domain);
255 dkim_sign(s.TheMessage, pkey, dkim_from_domain, dkim_selector);
260 // Prepare the buffer for transmittal
261 s.bytes_total = StrLength(s.TheMessage);
266 // Keep trying MXes until one works or we run out.
267 for (i = 0; ((i < num_mx) && ((response_code / 100) == 4)); ++i) {
268 response_code = 421; // default 421 makes non-protocol errors transient
269 s.bytes_sent = 0; // rewind our buffer in case we try multiple MXes
271 curl = curl_easy_init();
275 if (!IsEmptyStr(envelope_from)) {
276 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, envelope_from);
279 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, fromaddr);
282 recipients = curl_slist_append(recipients, recp);
283 curl_easy_setopt(curl, CURLOPT_MAIL_RCPT, recipients);
284 curl_easy_setopt(curl, CURLOPT_READFUNCTION, upload_source);
285 curl_easy_setopt(curl, CURLOPT_READDATA, &s);
286 curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); // tell libcurl we are uploading
287 curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L); // Time out after 20 seconds
288 if (CtdlGetConfigInt("c_smtpclient_disable_starttls") == 0) {
289 curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY); // Attempt STARTTLS if offered
291 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
292 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
293 curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, ctdl_libcurl_smtp_debug_callback);
294 curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void *) response);
295 curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
297 // Construct an SMTP URL in the form of:
298 // smtp[s]://target_host/source_host
299 // This looks weird but libcurl uses that last part to set our name for EHLO or HELO.
300 // We check for "smtp://" and "smtps://" because an admin may have put those prefixes in a smart-host entry
301 // If there is no prefix we add "smtp://"
302 extract_token(try_this_mx, mxes, i, '|', (sizeof try_this_mx - 7));
303 snprintf(smtp_url, sizeof smtp_url,
305 (((!strncasecmp(try_this_mx, HKEY("smtp://")))
306 || (!strncasecmp(try_this_mx, HKEY("smtps://")))) ? "" : "smtp://"),
307 try_this_mx, CtdlGetConfigStr("c_fqdn")
309 curl_easy_setopt(curl, CURLOPT_URL, smtp_url);
310 syslog(LOG_DEBUG, "smtpclient: trying MX %d of %d <%s>", i+1, num_mx, smtp_url); // send the message
311 res = curl_easy_perform(curl);
312 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
314 "smtpclient: libcurl returned %d (%s) , SMTP response %ld",
315 res, curl_easy_strerror(res), response_code
318 if ((res != CURLE_OK) && (response_code == 0)) { // check for errors
322 curl_slist_free_all(recipients);
323 recipients = NULL; // this gets reused; avoid double-free
324 curl_easy_cleanup(curl);
325 curl = NULL; // this gets reused; avoid double-free
327 // Trim the error message buffer down to just the actual message
328 trim_response(response_code, response);
332 FreeStrBuf(&s.TheMessage);
336 return ((int) response_code);
340 // Process one outbound message.
341 void smtp_process_one_msg(long qmsgnum) {
342 struct CtdlMessage *msg = NULL;
349 int delete_this_queue = 0;
350 char server_response[SIZ];
352 msg = CtdlFetchMessage(qmsgnum, 1);
354 syslog(LOG_WARNING, "smtpclient: msg#%ld does not exist", qmsgnum);
358 instr = msg->cm_fields[eMessageText];
359 msg->cm_fields[eMessageText] = NULL;
362 // if the queue job message has any CRLF's convert them to LF's
364 while (crlf = strstr(instr, "\r\n"), crlf != NULL) {
365 strcpy(crlf, crlf + 1);
368 // Strip out the headers and we are now left with just the instructions.
369 char *soi = strstr(instr, "\n\n");
371 strcpy(instr, soi + 2);
375 time_t submitted = time(NULL);
376 time_t attempted = 0;
377 char *bounceto = NULL;
378 char *envelope_from = NULL;
379 char *source_room = NULL;
382 for (i = 0; i < num_tokens(instr, '\n'); ++i) {
383 extract_token(cfgline, instr, i, '\n', sizeof cfgline);
384 if (!strncasecmp(cfgline, HKEY("msgid|"))) msgid = atol(&cfgline[6]);
385 if (!strncasecmp(cfgline, HKEY("submitted|"))) submitted = atol(&cfgline[10]);
386 if (!strncasecmp(cfgline, HKEY("attempted|"))) attempted = atol(&cfgline[10]);
387 if (!strncasecmp(cfgline, HKEY("bounceto|"))) bounceto = strdup(&cfgline[9]);
388 if (!strncasecmp(cfgline, HKEY("envelope_from|"))) envelope_from = strdup(&cfgline[14]);
389 if (!strncasecmp(cfgline, HKEY("source_room|"))) source_room = strdup(&cfgline[12]);
392 int should_try_now = 0;
393 if (attempted < submitted) { // If no attempts have been made yet, try now
396 else if ((attempted - submitted) <= 14400) {
397 if ((time(NULL) - attempted) > 1800) { // First four hours, retry every 30 minutes
402 if ((time(NULL) - attempted) > 14400) { // After that, retry once every 4 hours
407 if (should_try_now) {
408 syslog(LOG_DEBUG, "smtpclient: attempting delivery of message <%ld> now", qmsgnum);
410 syslog(LOG_DEBUG, "smtpclient: this message originated in <%s>", source_room);
412 StrBuf *NewInstr = NewStrBuf();
413 StrBufAppendPrintf(NewInstr, "Content-type: " SPOOLMIME "\n\n");
414 StrBufAppendPrintf(NewInstr, "msgid|%ld\n", msgid);
415 StrBufAppendPrintf(NewInstr, "submitted|%ld\n", submitted);
417 StrBufAppendPrintf(NewInstr, "bounceto|%s\n", bounceto);
420 StrBufAppendPrintf(NewInstr, "envelope_from|%s\n", envelope_from);
422 for (i = 0; i < num_tokens(instr, '\n'); ++i) {
423 extract_token(cfgline, instr, i, '\n', sizeof cfgline);
424 if (!strncasecmp(cfgline, HKEY("remote|"))) {
426 int previous_result = extract_int(cfgline, 2);
427 if ((previous_result == 0) || (previous_result == 4)) {
428 int new_result = 421;
429 extract_token(recp, cfgline, 1, '|', sizeof recp);
430 new_result = smtp_attempt_delivery(msgid, recp, envelope_from, source_room, server_response);
431 syslog(LOG_DEBUG, "smtpclient: recp: <%s> , result: %d (%s)", recp, new_result, server_response);
432 if ((new_result / 100) == 2) {
436 if ((new_result / 100) == 5) {
442 StrBufAppendPrintf(NewInstr, "remote|%s|%ld|%ld (%s)\n", recp, (new_result / 100), new_result, server_response);
448 StrBufAppendPrintf(NewInstr, "attempted|%ld\n", time(NULL));
450 // All deliveries have now been attempted. Now determine the disposition of this queue entry.
452 time_t age = time(NULL) - submitted;
454 "smtpclient: submission age: %ldd%ldh%ldm%lds",
455 (age / 86400), ((age % 86400) / 3600), ((age % 3600) / 60), (age % 60));
456 syslog(LOG_DEBUG, "smtpclient: num_success=%d , num_fail=%d , num_delayed=%d", num_success, num_fail, num_delayed);
458 // If there are permanent fails on this attempt, deliver a bounce to the user.
459 // The 5XX fails will be recorded in the rewritten queue, but they will be removed before the next attempt.
461 smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_FATALS);
463 // If all deliveries have either succeeded or failed, we are finished with this queue entry.
464 if (num_delayed == 0) {
465 delete_this_queue = 1;
467 // If it's been more than five days, give up and tell the sender that delivery failed
468 else if ((time(NULL) - submitted) > SMTP_DELIVER_FAIL) {
469 smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_ALL);
470 delete_this_queue = 1;
472 // If it's been more than four hours but less than five days, warn the sender that delivery is delayed
473 else if (((attempted - submitted) < SMTP_DELIVER_WARN) && ((time(NULL) - submitted) >= SMTP_DELIVER_WARN)) {
474 smtp_do_bounce(ChrPtr(NewInstr), SDB_WARN);
477 if (delete_this_queue) {
478 syslog(LOG_DEBUG, "smtpclient: %ld deleting", qmsgnum);
479 deletes[0] = qmsgnum;
481 CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, deletes, 2, "");
482 FreeStrBuf(&NewInstr); // We have to free NewInstr here, no longer needed
485 // replace the old queue entry with the new one
486 syslog(LOG_DEBUG, "smtpclient: %ld rewriting", qmsgnum);
487 msg = convert_internet_message_buf(&NewInstr); // This function will free NewInstr for us
488 CtdlSubmitMsg(msg, NULL, SMTP_SPOOLOUT_ROOM);
490 CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, &qmsgnum, 1, "");
494 syslog(LOG_DEBUG, "smtpclient: msg#%ld retry time not reached", qmsgnum);
497 if (bounceto != NULL) {
500 if (envelope_from != NULL) {
503 if (source_room != NULL) {
510 // Callback for smtp_do_queue()
511 void smtp_add_msg(long msgnum, void *userdata) {
512 Array *smtp_queue = (Array *) userdata;
513 array_append(smtp_queue, &msgnum);
518 FULL_QUEUE_RUN, // try to process the entire queue, including messages that have already been attempted
519 QUICK_QUEUE_RUN // only process jobs in the queue that have not been tried yet
523 // Run through the queue sending out messages.
524 void smtp_do_queue(int type_of_queue_run) {
525 static int doing_smtpclient = 0;
528 // This is a concurrency check to make sure only one smtpclient run is done at a time.
529 begin_critical_section(S_SMTPQUEUE);
530 if (doing_smtpclient) {
531 end_critical_section(S_SMTPQUEUE);
534 doing_smtpclient = 1;
535 end_critical_section(S_SMTPQUEUE);
537 syslog(LOG_DEBUG, "smtpclient: start %s queue run , last_queue_job_processed=%ld , last_queue_job_submitted=%ld",
538 (type_of_queue_run == QUICK_QUEUE_RUN ? "quick" : "full"),
539 last_queue_job_processed, last_queue_job_submitted
542 if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
543 syslog(LOG_WARNING, "smtpclient: cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
544 doing_smtpclient = 0;
548 // This array will hold the list of queue job messages
549 Array *smtp_queue = array_new(sizeof(long));
550 if (smtp_queue == NULL) {
551 syslog(LOG_WARNING, "smtpclient: cannot allocate queue array");
552 doing_smtpclient = 0;
556 // Put the queue in memory so we can close the db cursor
558 (type_of_queue_run == QUICK_QUEUE_RUN ? MSGS_GT : MSGS_ALL), // quick = new jobs; full = all jobs
559 (type_of_queue_run == QUICK_QUEUE_RUN ? last_queue_job_processed : 0), // quick = new jobs; full = all jobs
561 SPOOLMIME, // Searching for Content-type of SPOOLIME will give us only queue instruction messages
563 smtp_add_msg, // That's our callback function to add a job to the queue
567 // We are ready to run through the queue now.
568 syslog(LOG_DEBUG, "smtpclient: %d messages to be processed", array_len(smtp_queue));
569 for (i = 0; i < array_len(smtp_queue); ++i) {
571 memcpy(&m, array_get_element_at(smtp_queue, i), sizeof(long));
572 smtp_process_one_msg(m);
575 array_free(smtp_queue);
576 last_queue_job_processed = last_queue_job_submitted;
577 doing_smtpclient = 0;
578 syslog(LOG_DEBUG, "smtpclient: end %s queue run , last_queue_job_processed=%ld , last_queue_job_submitted=%ld",
579 (type_of_queue_run == QUICK_QUEUE_RUN ? "quick" : "full"),
580 last_queue_job_processed, last_queue_job_submitted
585 // The "full" queue run goes through the entire queue, attempting delivery for newly submitted messages,
586 // retrying failed deliveries periodically, and handling undeliverable messages.
587 void smtp_do_queue_full(void) {
588 smtp_do_queue(FULL_QUEUE_RUN);
592 // The "quick" queue run only handles newly submitted messages, allowing them to be delivered immediately
593 // instead of waiting for the next "full" queue run.
594 void smtp_do_queue_quick(void) {
595 if (last_queue_job_submitted > last_queue_job_processed) {
596 smtp_do_queue(QUICK_QUEUE_RUN);
601 // Initialization function, called from modules_init.c
602 char *ctdl_module_init_smtpclient(void) {
604 CtdlRegisterMessageHook(smtp_aftersave, EVT_AFTERSAVE);
605 CtdlRegisterSessionHook(smtp_do_queue_quick, EVT_HOUSE, PRIO_AGGR + 51);
606 CtdlRegisterSessionHook(smtp_do_queue_full, EVT_TIMER, PRIO_AGGR + 51);
607 smtp_init_spoolout();
610 // return our module id for the log