2 * Transmit outbound SMTP mail to the big wide world of the Internet
4 * This is the new, exciting, clever version that makes libcurl do all the work :)
6 * Copyright (c) 1997-2017 by the citadel.org team
8 * This program is open source software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License as published
10 * by the Free Software Foundation; either version 3 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
22 #include <sysconfig.h>
24 #if TIME_WITH_SYS_TIME
25 # include <sys/time.h>
29 # include <sys/time.h>
38 #include <sys/types.h>
40 #include <libcitadel.h>
41 #include <curl/curl.h>
44 #include "citserver.h"
47 #include "ctdl_module.h"
48 #include "clientsocket.h"
51 #include "internet_addressing.h"
52 #include "citadel_dirs.h"
53 #include "modules/smtp/smtp_util.h"
55 struct smtpmsgsrc { // Data passed in and out of libcurl for message upload
61 struct CitContext smtp_client_CC;
62 static int doing_smtpclient = 0;
63 long *smtpq = NULL; // array of msgnums containing queue instructions
64 int smtpq_count = 0; // number of queue messages in smtpq
65 int smtpq_alloc = 0; // current allocation size for smtpq
69 * Initialize the SMTP outbound queue
71 void smtp_init_spoolout(void) {
72 struct ctdlroom qrbuf;
75 * Create the room. This will silently fail if the room already
76 * exists, and that's perfectly ok, because we want it to exist.
78 CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
81 * Make sure it's set to be a "system room" so it doesn't show up
82 * in the <K>nown rooms list for Aides.
84 if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
85 qrbuf.QRflags2 |= QR2_SYSTEM;
86 CtdlPutRoomLock(&qrbuf);
91 /* For internet mail, generate delivery instructions.
92 * Yes, this is recursive. Deal with it. Infinite recursion does
93 * not happen because the delivery instructions message does not
94 * contain a recipient.
96 int smtp_aftersave(struct CtdlMessage *msg, recptypes *recps)
98 if ((recps != NULL) && (recps->num_internet > 0)) {
99 struct CtdlMessage *imsg = NULL;
101 StrBuf *SpoolMsg = NewStrBuf();
105 syslog(LOG_DEBUG, "smtpclient: generating delivery instructions");
107 StrBufPrintf(SpoolMsg,
108 "Content-type: "SPOOLMIME"\n"
113 msg->cm_fields[eVltMsgNum],
117 if (recps->envelope_from != NULL) {
118 StrBufAppendBufPlain(SpoolMsg, HKEY("envelope_from|"), 0);
119 StrBufAppendBufPlain(SpoolMsg, recps->envelope_from, -1, 0);
120 StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
122 if (recps->sending_room != NULL) {
123 StrBufAppendBufPlain(SpoolMsg, HKEY("source_room|"), 0);
124 StrBufAppendBufPlain(SpoolMsg, recps->sending_room, -1, 0);
125 StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
128 nTokens = num_tokens(recps->recp_internet, '|');
129 for (i = 0; i < nTokens; i++) {
131 len = extract_token(recipient, recps->recp_internet, i, '|', sizeof recipient);
133 StrBufAppendBufPlain(SpoolMsg, HKEY("remote|"), 0);
134 StrBufAppendBufPlain(SpoolMsg, recipient, len, 0);
135 StrBufAppendBufPlain(SpoolMsg, HKEY("|0||\n"), 0);
139 imsg = malloc(sizeof(struct CtdlMessage));
140 memset(imsg, 0, sizeof(struct CtdlMessage));
141 imsg->cm_magic = CTDLMESSAGE_MAGIC;
142 imsg->cm_anon_type = MES_NORMAL;
143 imsg->cm_format_type = FMT_RFC822;
144 CM_SetField(imsg, eMsgSubject, HKEY("QMSG"));
145 CM_SetField(imsg, eAuthor, HKEY("Citadel"));
146 CM_SetField(imsg, eJournal, HKEY("do not journal"));
147 CM_SetAsFieldSB(imsg, eMesageText, &SpoolMsg);
148 CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM, QP_EADDR);
156 * Callback for smtp_attempt_delivery() to supply libcurl with upload data.
158 static size_t upload_source(void *ptr, size_t size, size_t nmemb, void *userp)
160 struct smtpmsgsrc *s = (struct smtpmsgsrc *) userp;
162 const char *send_this = NULL;
164 sendbytes = (size * nmemb);
166 if (s->bytes_sent >= s->bytes_total) {
167 return(0); // we are donez0r
170 if (sendbytes > (s->bytes_total - s->bytes_sent)) {
171 sendbytes = s->bytes_total - s->bytes_sent; // can't send more than we have
174 send_this = ChrPtr(s->TheMessage);
175 send_this += s->bytes_sent; // start where we last left off
177 memcpy(ptr, send_this, sendbytes);
178 s->bytes_sent += sendbytes;
179 return(sendbytes); // return the number of bytes _actually_ copied
184 * Attempt a delivery to one recipient.
185 * Returns a three-digit SMTP status code.
187 int smtp_attempt_delivery(long msgid, char *recp, char *envelope_from)
190 char *fromaddr = NULL;
192 CURLcode res = CURLE_OK;
193 struct curl_slist *recipients = NULL;
194 long response_code = 421;
200 char try_this_mx[256];
203 syslog(LOG_DEBUG, "smtpclient: smtp_attempt_delivery(%ld, %s)", msgid, recp);
205 process_rfc822_addr(recp, user, node, name); // split recipient address into username, hostname, displayname
206 num_mx = getmx(mxes, node);
211 CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
212 CtdlOutputMsg(msgid, MT_RFC822, HEADERS_ALL, 0, 1, NULL, 0, NULL, &fromaddr, NULL);
213 s.TheMessage = CC->redirect_buffer;
214 s.bytes_total = StrLength(CC->redirect_buffer);
216 CC->redirect_buffer = NULL;
219 for (i=0; ((i<num_mx)&&((response_code/100)==4)); ++i) { // keep trying MXes until one works or we run out
220 response_code = 421; // default 421 makes non-protocol errors transient
221 s.bytes_sent = 0; // rewind our buffer in case we try multiple MXes
223 curl = curl_easy_init();
226 if (!IsEmptyStr(envelope_from)) {
227 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, envelope_from);
230 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, fromaddr);
233 recipients = curl_slist_append(recipients, recp);
234 curl_easy_setopt(curl, CURLOPT_MAIL_RCPT, recipients);
235 curl_easy_setopt(curl, CURLOPT_READFUNCTION, upload_source);
236 curl_easy_setopt(curl, CURLOPT_READDATA, &s);
237 curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); // tell libcurl we are uploading
238 curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L); // Time out after 20 seconds
239 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
240 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
242 strcpy(try_this_mx, "smtp://");
243 extract_token(&try_this_mx[7], mxes, i, '|', (sizeof try_this_mx - 7));
245 (!strncasecmp(try_this_mx, HKEY("smtp://smtp://"))) // This can happen if the administrator
246 || (!strncasecmp(try_this_mx, HKEY("smtp://smtps://"))) // puts a full smtp[s] URI as the smart-host
248 strcpy(try_this_mx, &try_this_mx[7]);
252 curl_easy_setopt(curl, CURLOPT_URL, try_this_mx);
254 syslog(LOG_DEBUG, "smtpclient: trying %s", try_this_mx); // send the message
255 res = curl_easy_perform(curl);
256 curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
257 syslog(LOG_DEBUG, "smtpclient: libcurl returned %d (%s) , SMTP response %ld",
258 res, curl_easy_strerror(res), response_code
261 if ((res != CURLE_OK) && (response_code == 0)) { // check for errors
265 curl_slist_free_all(recipients);
266 curl_easy_cleanup(curl);
270 FreeStrBuf(&s.TheMessage);
271 if (fromaddr) free(fromaddr);
272 return((int)response_code);
277 * Process one outbound message.
279 void smtp_process_one_msg(long qmsgnum)
281 struct CtdlMessage *msg = NULL;
288 int delete_this_queue = 0;
290 syslog(LOG_DEBUG, "smtpclient: smtp_process_one_msg(%ld)", qmsgnum);
292 msg = CtdlFetchMessage(qmsgnum, 1, 1);
294 syslog(LOG_WARNING, "smtpclient: queue message %ld does not exist", qmsgnum);
298 instr = msg->cm_fields[eMesageText];
299 msg->cm_fields[eMesageText] = NULL;
302 // if the queue message has any CRLF's convert them to LF's
304 while (crlf = strstr(instr, "\r\n"), crlf != NULL) {
305 strcpy(crlf, crlf+1);
308 // Strip out the headers and we are now left with just the instructions.
309 char *soi = strstr(instr, "\n\n");
311 strcpy(instr, soi+2);
315 time_t submitted = time(NULL);
316 time_t attempted = 0;
317 char *bounceto = NULL;
318 char *envelope_from = NULL;
320 // Example queue instructions
323 // submitted|1489343934
324 // bounceto|IGnatius T Foobar@dev
325 // attempted|1489344257
326 // remote|unreachable@example.com|4|Timeout while connecting example.com [93.184.216.34]:25
327 // remote|unreachable@example.org|4|Timeout while connecting example.org [93.184.216.34]:25
328 // remote|unreachable@example.gov|0|1) A-lookup example.gov - Domain name not found; 2) AAAA-lookup example.gov - Domain name not found;
331 for (i=0; i<num_tokens(instr, '\n'); ++i) {
332 extract_token(cfgline, instr, i, '\n', sizeof cfgline);
333 if (!strncasecmp(cfgline, HKEY("msgid|"))) msgid = atol(&cfgline[6]);
334 if (!strncasecmp(cfgline, HKEY("submitted|"))) submitted = atol(&cfgline[10]);
335 if (!strncasecmp(cfgline, HKEY("attempted|"))) attempted = atol(&cfgline[10]);
336 if (!strncasecmp(cfgline, HKEY("bounceto|"))) bounceto = strdup(&cfgline[9]);
337 if (!strncasecmp(cfgline, HKEY("envelope_from|"))) envelope_from = strdup(&cfgline[14]);
340 int should_try_now = 1;
341 if (attempted < submitted) { // If no attempts have been made yet, try now
344 else if ((attempted - submitted) <= 14400) {
345 if ((time(NULL) - attempted) > 1800) { // First four hours, retry every 30 minutes
350 if ((time(NULL) - attempted) > 14400) { // After that, retry once every 4 hours
355 if (should_try_now) {
356 syslog(LOG_DEBUG, "smtpclient: attempting delivery");
358 StrBuf *NewInstr = NewStrBuf();
359 StrBufAppendPrintf(NewInstr, "Content-type: "SPOOLMIME"\n\n");
360 StrBufAppendPrintf(NewInstr, "msgid|%ld\n", msgid);
361 StrBufAppendPrintf(NewInstr, "submitted|%ld\n", submitted);
362 if (bounceto) StrBufAppendPrintf(NewInstr, "bounceto|%s\n", bounceto);
363 if (envelope_from) StrBufAppendPrintf(NewInstr, "envelope_from|%s\n", envelope_from);
365 for (i=0; i<num_tokens(instr, '\n'); ++i) {
366 extract_token(cfgline, instr, i, '\n', sizeof cfgline);
367 if (!strncasecmp(cfgline, HKEY("remote|"))) {
369 int previous_result = extract_int(cfgline, 2);
370 if ((previous_result == 0) || (previous_result == 4)) {
371 int new_result = 421;
372 extract_token(recp, cfgline, 1, '|', sizeof recp);
373 new_result = smtp_attempt_delivery(msgid, recp, envelope_from);
374 syslog(LOG_DEBUG, "smtpclient: recp: <%s> , result: %d (%s)", recp, new_result, smtpstatus(new_result));
375 if ((new_result / 100) == 2) {
379 if ((new_result / 100) == 5) {
385 StrBufAppendPrintf(NewInstr, "remote|%s|%ld|%ld (%s)\n",
386 recp, (new_result / 100) , new_result, smtpstatus(new_result)
393 StrBufAppendPrintf(NewInstr, "attempted|%ld\n", time(NULL));
395 // All deliveries have now been attempted. Now determine the disposition of this queue entry.
397 time_t age = time(NULL) - submitted;
398 syslog(LOG_DEBUG, "smtpclient: submission age: %ldd%ldh%ldm%lds", (age/86400) , ((age%86400)/3600) , ((age%3600)/60) , (age%60));
399 syslog(LOG_DEBUG, "smtpclient: num_success=%d , num_fail=%d , num_delayed=%d", num_success, num_fail, num_delayed);
401 // If there are permanent fails on this attempt, deliver a bounce to the user.
402 // The 5XX fails will be recorded in the rewritten queue, but they will be removed before the next attempt.
404 smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_FATALS);
407 // If all deliveries have either succeeded or failed, we are finished with this queue entry.
409 if (num_delayed == 0) {
410 delete_this_queue = 1;
413 // If it's been more than five days, give up and tell the sender we #failed
415 else if ((time(NULL) - submitted) > SMTP_DELIVER_FAIL) {
416 smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_ALL);
417 delete_this_queue = 1;
420 // If it's been more than four hours but less than five days, warn the sender that I've Been Delayed
422 else if ( ((attempted - submitted) < SMTP_DELIVER_WARN) && ((time(NULL) - submitted) >= SMTP_DELIVER_WARN) ) {
423 smtp_do_bounce(ChrPtr(NewInstr), SDB_WARN);
426 if (delete_this_queue) {
427 syslog(LOG_DEBUG, "smtpclient: deleting this queue entry");
428 deletes[0] = qmsgnum;
430 CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, deletes, 2, "");
431 FreeStrBuf(&NewInstr); // We have to free NewInstr here, no longer needed
434 // replace the old qmsg with the new one
435 syslog(LOG_DEBUG, "smtpclient: rewriting this queue entry");
436 msg = convert_internet_message_buf(&NewInstr); // This function will free NewInstr for us
437 CtdlSubmitMsg(msg, NULL, SMTP_SPOOLOUT_ROOM, 0);
439 CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, &qmsgnum, 1, "");
443 syslog(LOG_DEBUG, "smtpclient: retry time not reached");
446 if (bounceto != NULL) free(bounceto);
447 if (envelope_from != NULL) free(envelope_from);
454 * Callback for smtp_do_queue()
456 void smtp_add_msg(long msgnum, void *userdata) {
461 smtpq = malloc(smtpq_alloc * sizeof(long));
464 if (smtpq_alloc >= smtpq_count) {
466 smtpq = realloc(smtpq, (smtpq_alloc * sizeof(long)));
469 smtpq[smtpq_count++] = msgnum;
474 * Run through the queue sending out messages.
476 void smtp_do_queue(void) {
480 * This is a simple concurrency check to make sure only one pop3client
481 * run is done at a time. We could do this with a mutex, but since we
482 * don't really require extremely fine granularity here, we'll do it
483 * with a static variable instead.
485 if (doing_smtpclient) return;
486 doing_smtpclient = 1;
488 syslog(LOG_DEBUG, "smtpclient: start queue run");
489 pthread_setspecific(MyConKey, (void *)&smtp_client_CC);
491 if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
492 syslog(LOG_WARNING, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
493 doing_smtpclient = 0;
497 // Put the queue in memory so we can close the db cursor
498 CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_add_msg, NULL);
500 // We are ready to run through the queue now.
501 for (i=0; i<smtpq_count; ++i) {
502 smtp_process_one_msg(smtpq[i]);
505 smtpq_count = 0; // don't free it, we will use this memory on the next run
506 doing_smtpclient = 0;
507 syslog(LOG_DEBUG, "smtpclient: end queue run");
514 CTDL_MODULE_INIT(smtpclient)
518 CtdlFillSystemContext(&smtp_client_CC, "SMTP_Send");
519 CtdlRegisterMessageHook(smtp_aftersave, EVT_AFTERSAVE);
520 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER, PRIO_AGGR + 50);
521 smtp_init_spoolout();
524 /* return our module id for the log */