2 * Transmit outbound SMTP mail to the big wide world of the Internet
4 * This is the new, exciting, clever version that makes libcurl do all the work :)
6 * Copyright (c) 1997-2020 by the citadel.org team
8 * This program is open source software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License as published
10 * by the Free Software Foundation; either version 3 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
22 #include <sysconfig.h>
27 #include <sys/types.h>
29 #include <libcitadel.h>
30 #include <curl/curl.h>
33 #include "citserver.h"
36 #include "ctdl_module.h"
37 #include "clientsocket.h"
40 #include "internet_addressing.h"
41 #include "citadel_dirs.h"
42 #include "modules/smtp/smtp_util.h"
44 struct smtpmsgsrc { // Data passed in and out of libcurl for message upload
50 static int doing_smtpclient = 0;
51 long *smtpq = NULL; // array of msgnums containing queue instructions
52 int smtpq_count = 0; // number of queue messages in smtpq
53 int smtpq_alloc = 0; // current allocation size for smtpq
57 * Initialize the SMTP outbound queue
59 void smtp_init_spoolout(void)
61 struct ctdlroom qrbuf;
64 * Create the room. This will silently fail if the room already
65 * exists, and that's perfectly ok, because we want it to exist.
67 CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
70 * Make sure it's set to be a "system room" so it doesn't show up
71 * in the <K>nown rooms list for administrators.
73 if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
74 qrbuf.QRflags2 |= QR2_SYSTEM;
75 CtdlPutRoomLock(&qrbuf);
80 /* For internet mail, generate delivery instructions.
81 * Yes, this is recursive. Deal with it. Infinite recursion does
82 * not happen because the delivery instructions message does not
83 * contain a recipient.
85 int smtp_aftersave(struct CtdlMessage *msg, recptypes * recps)
87 if ((recps != NULL) && (recps->num_internet > 0)) {
88 struct CtdlMessage *imsg = NULL;
90 StrBuf *SpoolMsg = NewStrBuf();
94 syslog(LOG_DEBUG, "smtpclient: generating delivery instructions");
96 StrBufPrintf(SpoolMsg,
97 "Content-type: " SPOOLMIME "\n"
100 "submitted|%ld\n" "bounceto|%s\n", msg->cm_fields[eVltMsgNum], (long) time(NULL), recps->bounce_to);
102 if (recps->envelope_from != NULL) {
103 StrBufAppendBufPlain(SpoolMsg, HKEY("envelope_from|"), 0);
104 StrBufAppendBufPlain(SpoolMsg, recps->envelope_from, -1, 0);
105 StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
107 if (recps->sending_room != NULL) {
108 StrBufAppendBufPlain(SpoolMsg, HKEY("source_room|"), 0);
109 StrBufAppendBufPlain(SpoolMsg, recps->sending_room, -1, 0);
110 StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
113 nTokens = num_tokens(recps->recp_internet, '|');
114 for (i = 0; i < nTokens; i++) {
116 len = extract_token(recipient, recps->recp_internet, i, '|', sizeof recipient);
118 StrBufAppendBufPlain(SpoolMsg, HKEY("remote|"), 0);
119 StrBufAppendBufPlain(SpoolMsg, recipient, len, 0);
120 StrBufAppendBufPlain(SpoolMsg, HKEY("|0||\n"), 0);
124 imsg = malloc(sizeof(struct CtdlMessage));
125 memset(imsg, 0, sizeof(struct CtdlMessage));
126 imsg->cm_magic = CTDLMESSAGE_MAGIC;
127 imsg->cm_anon_type = MES_NORMAL;
128 imsg->cm_format_type = FMT_RFC822;
129 CM_SetField(imsg, eMsgSubject, HKEY("QMSG"));
130 CM_SetField(imsg, eAuthor, HKEY("Citadel"));
131 CM_SetField(imsg, eJournal, HKEY("do not journal"));
132 CM_SetAsFieldSB(imsg, eMesageText, &SpoolMsg);
133 CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM);
141 * Callback for smtp_attempt_delivery() to supply libcurl with upload data.
143 static size_t upload_source(void *ptr, size_t size, size_t nmemb, void *userp)
145 struct smtpmsgsrc *s = (struct smtpmsgsrc *) userp;
147 const char *send_this = NULL;
149 sendbytes = (size * nmemb);
151 if (s->bytes_sent >= s->bytes_total) {
152 return (0); // no data remaining; we are done
155 if (sendbytes > (s->bytes_total - s->bytes_sent)) {
156 sendbytes = s->bytes_total - s->bytes_sent; // can't send more than we have
159 send_this = ChrPtr(s->TheMessage);
160 send_this += s->bytes_sent; // start where we last left off
162 memcpy(ptr, send_this, sendbytes);
163 s->bytes_sent += sendbytes;
164 return (sendbytes); // return the number of bytes _actually_ copied
169 * The libcurl API doesn't provide a way to capture the actual SMTP result message returned
170 * by the remote server. This is an ugly way to extract it, by capturing debug data from
171 * the library and filtering on the lines we want.
173 int ctdl_libcurl_smtp_debug_callback(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr)
175 if (type != CURLINFO_HEADER_IN)
179 char *debugbuf = (char *) userptr;
181 int len = strlen(debugbuf);
182 if (len + size > SIZ)
185 memcpy(&debugbuf[len], data, size);
186 debugbuf[len + size] = 0;
192 * Go through the debug output of an SMTP transaction, and boil it down to just the final success or error response message.
194 void trim_response(long response_code, char *response)
196 if ((response_code < 100) || (response_code > 999) || (IsEmptyStr(response))) {
200 char *t = malloc(strlen(response));
207 for (p = response; *p != 0; ++p) {
208 if ( (*p != '\n') && (!isprint(*p)) ) { // expunge any nonprintables except for newlines
213 char response_code_str[4];
214 snprintf(response_code_str, sizeof response_code_str, "%ld", response_code);
215 char *respstart = strstr(response, response_code_str);
216 if (respstart == NULL) {
217 strcpy(response, smtpstatus(response_code));
220 strcpy(response, respstart);
222 p = strstr(response, "\n");
230 * Attempt a delivery to one recipient.
231 * Returns a three-digit SMTP status code.
233 int smtp_attempt_delivery(long msgid, char *recp, char *envelope_from, char *response)
236 char *fromaddr = NULL;
238 CURLcode res = CURLE_OK;
239 struct curl_slist *recipients = NULL;
240 long response_code = 421;
246 char try_this_mx[256];
250 syslog(LOG_DEBUG, "smtpclient: smtp_attempt_delivery(%ld, %s)", msgid, recp);
252 process_rfc822_addr(recp, user, node, name); // split recipient address into username, hostname, displayname
253 num_mx = getmx(mxes, node);
258 CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
259 CtdlOutputMsg(msgid, MT_RFC822, HEADERS_ALL, 0, 1, NULL, 0, NULL, &fromaddr, NULL);
260 s.TheMessage = CC->redirect_buffer;
261 s.bytes_total = StrLength(CC->redirect_buffer);
263 CC->redirect_buffer = NULL;
266 for (i = 0; ((i < num_mx) && ((response_code / 100) == 4)); ++i) { // keep trying MXes until one works or we run out
267 response_code = 421; // default 421 makes non-protocol errors transient
268 s.bytes_sent = 0; // rewind our buffer in case we try multiple MXes
270 curl = curl_easy_init();
274 if (!IsEmptyStr(envelope_from)) {
275 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, envelope_from);
277 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, fromaddr);
280 recipients = curl_slist_append(recipients, recp);
281 curl_easy_setopt(curl, CURLOPT_MAIL_RCPT, recipients);
282 curl_easy_setopt(curl, CURLOPT_READFUNCTION, upload_source);
283 curl_easy_setopt(curl, CURLOPT_READDATA, &s);
284 curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); // tell libcurl we are uploading
285 curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L); // Time out after 20 seconds
286 if (CtdlGetConfigInt("c_smtpclient_disable_starttls") == 0) {
287 curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY); // Attempt STARTTLS if offered
289 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
290 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
291 curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, ctdl_libcurl_smtp_debug_callback);
292 curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void *) response);
293 curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
295 // Construct an SMTP URL in the form of:
296 // smtp[s]://target_host/source_host
297 // This looks weird but libcurl uses that last part to set our name for EHLO or HELO.
298 // We check for "smtp://" and "smtps://" because the admin may have put those prefixes in a smart-host entry.
299 // If there is no prefix we add "smtp://"
300 extract_token(try_this_mx, mxes, i, '|', (sizeof try_this_mx - 7));
301 snprintf(smtp_url, sizeof smtp_url, "%s%s/%s", (((!strncasecmp(try_this_mx, HKEY("smtp://")))
305 HKEY("smtps://")))) ? "" : "smtp://"),
306 try_this_mx, CtdlGetConfigStr("c_fqdn")
308 curl_easy_setopt(curl, CURLOPT_URL, smtp_url);
309 syslog(LOG_DEBUG, "smtpclient: trying %s", smtp_url); // send the message
310 res = curl_easy_perform(curl);
311 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
313 "smtpclient: libcurl returned %d (%s) , SMTP response %ld",
314 res, curl_easy_strerror(res), response_code);
316 if ((res != CURLE_OK) && (response_code == 0)) { // check for errors
320 curl_slist_free_all(recipients);
321 recipients = NULL; // this gets reused; avoid double-free
322 curl_easy_cleanup(curl);
323 curl = NULL; // this gets reused; avoid double-free
325 /* Trim the error message buffer down to just the actual message */
326 trim_response(response_code, response);
330 FreeStrBuf(&s.TheMessage);
333 return ((int) response_code);
338 * Process one outbound message.
340 void smtp_process_one_msg(long qmsgnum)
342 struct CtdlMessage *msg = NULL;
349 int delete_this_queue = 0;
350 char server_response[SIZ];
352 msg = CtdlFetchMessage(qmsgnum, 1);
354 syslog(LOG_WARNING, "smtpclient: %ld does not exist", qmsgnum);
358 instr = msg->cm_fields[eMesageText];
359 msg->cm_fields[eMesageText] = NULL;
362 // if the queue message has any CRLF's convert them to LF's
364 while (crlf = strstr(instr, "\r\n"), crlf != NULL) {
365 strcpy(crlf, crlf + 1);
368 // Strip out the headers and we are now left with just the instructions.
369 char *soi = strstr(instr, "\n\n");
371 strcpy(instr, soi + 2);
375 time_t submitted = time(NULL);
376 time_t attempted = 0;
377 char *bounceto = NULL;
378 char *envelope_from = NULL;
381 for (i = 0; i < num_tokens(instr, '\n'); ++i) {
382 extract_token(cfgline, instr, i, '\n', sizeof cfgline);
383 if (!strncasecmp(cfgline, HKEY("msgid|")))
384 msgid = atol(&cfgline[6]);
385 if (!strncasecmp(cfgline, HKEY("submitted|")))
386 submitted = atol(&cfgline[10]);
387 if (!strncasecmp(cfgline, HKEY("attempted|")))
388 attempted = atol(&cfgline[10]);
389 if (!strncasecmp(cfgline, HKEY("bounceto|")))
390 bounceto = strdup(&cfgline[9]);
391 if (!strncasecmp(cfgline, HKEY("envelope_from|")))
392 envelope_from = strdup(&cfgline[14]);
395 int should_try_now = 0;
396 if (attempted < submitted) { // If no attempts have been made yet, try now
398 } else if ((attempted - submitted) <= 14400) {
399 if ((time(NULL) - attempted) > 1800) { // First four hours, retry every 30 minutes
403 if ((time(NULL) - attempted) > 14400) { // After that, retry once every 4 hours
408 if (should_try_now) {
409 syslog(LOG_DEBUG, "smtpclient: %ld attempting delivery now", qmsgnum);
410 StrBuf *NewInstr = NewStrBuf();
411 StrBufAppendPrintf(NewInstr, "Content-type: " SPOOLMIME "\n\n");
412 StrBufAppendPrintf(NewInstr, "msgid|%ld\n", msgid);
413 StrBufAppendPrintf(NewInstr, "submitted|%ld\n", submitted);
415 StrBufAppendPrintf(NewInstr, "bounceto|%s\n", bounceto);
417 StrBufAppendPrintf(NewInstr, "envelope_from|%s\n", envelope_from);
419 for (i = 0; i < num_tokens(instr, '\n'); ++i) {
420 extract_token(cfgline, instr, i, '\n', sizeof cfgline);
421 if (!strncasecmp(cfgline, HKEY("remote|"))) {
423 int previous_result = extract_int(cfgline, 2);
424 if ((previous_result == 0)
425 || (previous_result == 4)) {
426 int new_result = 421;
427 extract_token(recp, cfgline, 1, '|', sizeof recp);
428 new_result = smtp_attempt_delivery(msgid, recp, envelope_from, server_response);
430 "smtpclient: recp: <%s> , result: %d (%s)", recp, new_result, server_response);
431 if ((new_result / 100) == 2) {
434 if ((new_result / 100) == 5) {
441 "remote|%s|%ld|%ld (%s)\n",
442 recp, (new_result / 100), new_result, server_response);
448 StrBufAppendPrintf(NewInstr, "attempted|%ld\n", time(NULL));
450 // All deliveries have now been attempted. Now determine the disposition of this queue entry.
452 time_t age = time(NULL) - submitted;
454 "smtpclient: submission age: %ldd%ldh%ldm%lds",
455 (age / 86400), ((age % 86400) / 3600), ((age % 3600) / 60), (age % 60));
456 syslog(LOG_DEBUG, "smtpclient: num_success=%d , num_fail=%d , num_delayed=%d", num_success, num_fail, num_delayed);
458 // If there are permanent fails on this attempt, deliver a bounce to the user.
459 // The 5XX fails will be recorded in the rewritten queue, but they will be removed before the next attempt.
461 smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_FATALS);
463 // If all deliveries have either succeeded or failed, we are finished with this queue entry.
465 if (num_delayed == 0) {
466 delete_this_queue = 1;
468 // If it's been more than five days, give up and tell the sender that delivery failed
470 else if ((time(NULL) - submitted) > SMTP_DELIVER_FAIL) {
471 smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_ALL);
472 delete_this_queue = 1;
474 // If it's been more than four hours but less than five days, warn the sender that delivery is delayed
476 else if (((attempted - submitted) < SMTP_DELIVER_WARN)
477 && ((time(NULL) - submitted) >= SMTP_DELIVER_WARN)) {
478 smtp_do_bounce(ChrPtr(NewInstr), SDB_WARN);
481 if (delete_this_queue) {
482 syslog(LOG_DEBUG, "smtpclient: %ld deleting", qmsgnum);
483 deletes[0] = qmsgnum;
485 CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, deletes, 2, "");
486 FreeStrBuf(&NewInstr); // We have to free NewInstr here, no longer needed
488 // replace the old queue entry with the new one
489 syslog(LOG_DEBUG, "smtpclient: %ld rewriting", qmsgnum);
490 msg = convert_internet_message_buf(&NewInstr); // This function will free NewInstr for us
491 CtdlSubmitMsg(msg, NULL, SMTP_SPOOLOUT_ROOM);
493 CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, &qmsgnum, 1, "");
496 syslog(LOG_DEBUG, "smtpclient: %ld retry time not reached", qmsgnum);
499 if (bounceto != NULL)
501 if (envelope_from != NULL)
508 * Callback for smtp_do_queue()
510 void smtp_add_msg(long msgnum, void *userdata)
516 smtpq = malloc(smtpq_alloc * sizeof(long));
519 if (smtpq_alloc >= smtpq_count) {
521 smtpq = realloc(smtpq, (smtpq_alloc * sizeof(long)));
524 smtpq[smtpq_count++] = msgnum;
529 * Run through the queue sending out messages.
531 void smtp_do_queue(void)
536 * This is a simple concurrency check to make sure only one smtpclient
537 * run is done at a time. We could do this with a mutex, but since we
538 * don't really require extremely fine granularity here, we'll do it
539 * with a static variable instead.
541 if (doing_smtpclient)
543 doing_smtpclient = 1;
545 syslog(LOG_DEBUG, "smtpclient: start queue run");
547 if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
548 syslog(LOG_WARNING, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
549 doing_smtpclient = 0;
552 // Put the queue in memory so we can close the db cursor
553 CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_add_msg, NULL);
555 // We are ready to run through the queue now.
556 for (i = 0; i < smtpq_count; ++i) {
557 smtp_process_one_msg(smtpq[i]);
560 smtpq_count = 0; // don't free it, we will use this memory on the next run
561 doing_smtpclient = 0;
562 syslog(LOG_DEBUG, "smtpclient: end queue run");
569 CTDL_MODULE_INIT(smtpclient)
572 CtdlRegisterMessageHook(smtp_aftersave, EVT_AFTERSAVE);
573 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER, PRIO_AGGR + 50);
574 smtp_init_spoolout();
577 /* return our module id for the log */