]> code.citadel.org Git - citadel.git/blob - citadel/server/modules/smtp/serv_smtpclient.c
comment change
[citadel.git] / citadel / server / modules / smtp / serv_smtpclient.c
1 // Transmit outbound SMTP mail to the big wide world of the Internet
2 //
3 // This is the new, exciting, clever version that makes libcurl do all the work  :)
4 //
5 // Copyright (c) 1997-2023 by the citadel.org team
6 //
7 // This program is open source software.  Use, duplication, or disclosure
8 // is subject to the terms of the GNU General Public License, version 3.
9
10 #include <stdlib.h>
11 #include <unistd.h>
12 #include <stdio.h>
13 #include <time.h>
14 #include <ctype.h>
15 #include <string.h>
16 #include <errno.h>
17 #include <sys/types.h>
18 #include <sys/stat.h>
19 #include <libcitadel.h>
20 #include <curl/curl.h>
21 #include "../../sysconfig.h"
22 #include "../../citadel.h"
23 #include "../../server.h"
24 #include "../../citserver.h"
25 #include "../../support.h"
26 #include "../../config.h"
27 #include "../../ctdl_module.h"
28 #include "../../clientsocket.h"
29 #include "../../msgbase.h"
30 #include "../../domain.h"
31 #include "../../internet_addressing.h"
32 #include "../../citadel_dirs.h"
33 #include "../smtp/smtp_util.h"
34
35 struct smtpmsgsrc {             // Data passed in and out of libcurl for message upload
36         StrBuf *TheMessage;
37         int bytes_total;
38         int bytes_sent;
39 };
40
41 // Initialize the SMTP outbound queue
42 void smtp_init_spoolout(void) {
43         struct ctdlroom qrbuf;
44
45         // Create the room.  This will silently fail if the room already
46         // exists, and that's perfectly ok, because we want it to exist.
47         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
48
49         // Make sure it's set to be a "system room" so it doesn't show up
50         // in the <K>nown rooms list for administrators.
51         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
52                 qrbuf.QRflags2 |= QR2_SYSTEM;
53                 CtdlPutRoomLock(&qrbuf);
54         }
55 }
56
57
58 // For internet mail, generate delivery instructions.
59 // Yes, this is recursive.  Deal with it.  Infinite recursion does
60 // not happen because the delivery instructions message does not
61 // contain a recipient.
62 int smtp_aftersave(struct CtdlMessage *msg, struct recptypes *recps) {
63         if ((recps != NULL) && (recps->num_internet > 0)) {
64                 struct CtdlMessage *imsg = NULL;
65                 char recipient[SIZ];
66                 StrBuf *SpoolMsg = NewStrBuf();
67                 long nTokens;
68                 int i;
69
70                 syslog(LOG_DEBUG, "smtpclient: generating delivery instructions");
71
72                 StrBufPrintf(SpoolMsg,
73                              "Content-type: " SPOOLMIME "\n"
74                              "\n"
75                              "msgid|%s\n"
76                              "submitted|%ld\n" "bounceto|%s\n", msg->cm_fields[eVltMsgNum], (long) time(NULL), recps->bounce_to);
77
78                 if (recps->envelope_from != NULL) {
79                         StrBufAppendBufPlain(SpoolMsg, HKEY("envelope_from|"), 0);
80                         StrBufAppendBufPlain(SpoolMsg, recps->envelope_from, -1, 0);
81                         StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
82                 }
83                 if (recps->sending_room != NULL) {
84                         StrBufAppendBufPlain(SpoolMsg, HKEY("source_room|"), 0);
85                         StrBufAppendBufPlain(SpoolMsg, recps->sending_room, -1, 0);
86                         StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
87                 }
88
89                 nTokens = num_tokens(recps->recp_internet, '|');
90                 for (i = 0; i < nTokens; i++) {
91                         long len;
92                         len = extract_token(recipient, recps->recp_internet, i, '|', sizeof recipient);
93                         if (len > 0) {
94                                 StrBufAppendBufPlain(SpoolMsg, HKEY("remote|"), 0);
95                                 StrBufAppendBufPlain(SpoolMsg, recipient, len, 0);
96                                 StrBufAppendBufPlain(SpoolMsg, HKEY("|0||\n"), 0);
97                         }
98                 }
99
100                 imsg = malloc(sizeof(struct CtdlMessage));
101                 memset(imsg, 0, sizeof(struct CtdlMessage));
102                 imsg->cm_magic = CTDLMESSAGE_MAGIC;
103                 imsg->cm_anon_type = MES_NORMAL;
104                 imsg->cm_format_type = FMT_RFC822;
105                 CM_SetField(imsg, eMsgSubject, HKEY("QMSG"));
106                 CM_SetField(imsg, eAuthor, HKEY("Citadel"));
107                 CM_SetField(imsg, eJournal, HKEY("do not journal"));
108                 CM_SetAsFieldSB(imsg, eMesageText, &SpoolMsg);
109                 CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM);
110                 CM_Free(imsg);
111         }
112         return 0;
113 }
114
115
116 // Callback for smtp_attempt_delivery() to supply libcurl with upload data.
117 static size_t upload_source(void *ptr, size_t size, size_t nmemb, void *userp) {
118         struct smtpmsgsrc *s = (struct smtpmsgsrc *) userp;
119         int sendbytes = 0;
120         const char *send_this = NULL;
121
122         sendbytes = (size * nmemb);
123
124         if (s->bytes_sent >= s->bytes_total) {
125                 return (0);                                     // no data remaining; we are done
126         }
127
128         if (sendbytes > (s->bytes_total - s->bytes_sent)) {
129                 sendbytes = s->bytes_total - s->bytes_sent;     // can't send more than we have
130         }
131
132         send_this = ChrPtr(s->TheMessage);
133         send_this += s->bytes_sent;                             // start where we last left off
134
135         memcpy(ptr, send_this, sendbytes);
136         s->bytes_sent += sendbytes;
137         return(sendbytes);                                      // return the number of bytes _actually_ copied
138 }
139
140
141 // The libcurl API doesn't provide a way to capture the actual SMTP result message returned
142 // by the remote server.  This is an ugly way to extract it, by capturing debug data from
143 // the library and filtering on the lines we want.
144 int ctdl_libcurl_smtp_debug_callback(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr) {
145         if (type != CURLINFO_HEADER_IN)
146                 return 0;
147         if (!userptr)
148                 return 0;
149         char *debugbuf = (char *) userptr;
150
151         int len = strlen(debugbuf);
152         if (len + size > SIZ)
153                 return 0;
154
155         memcpy(&debugbuf[len], data, size);
156         debugbuf[len + size] = 0;
157         return 0;
158 }
159
160
161 // Go through the debug output of an SMTP transaction, and boil it down to just the final success or error response message.
162 void trim_response(long response_code, char *response) {
163         if ((response_code < 100) || (response_code > 999) || (IsEmptyStr(response))) {
164                 return;
165         }
166
167         char *t = malloc(strlen(response));
168         if (!t) {
169                 return;
170         }
171         t[0] = 0;
172
173         char *p;
174         for (p = response; *p != 0; ++p) {
175                 if ( (*p != '\n') && (!isprint(*p)) ) {         // expunge any nonprintables except for newlines
176                         *p = ' ';
177                 }
178         }
179
180         char response_code_str[4];
181         snprintf(response_code_str, sizeof response_code_str, "%ld", response_code);
182         char *respstart = strstr(response, response_code_str);
183         if (respstart == NULL) {                                // If we have a response code but no response text,
184                 strcpy(response, smtpstatus(response_code));    // use one of our canned messages.
185                 return;
186         }
187         strcpy(response, respstart);
188
189         p = strstr(response, "\n");
190         if (p != NULL) {
191                 *p = 0;
192         }
193 }
194
195
196 // Attempt a delivery to one recipient.
197 // Returns a three-digit SMTP status code.
198 int smtp_attempt_delivery(long msgid, char *recp, char *envelope_from, char *source_room, char *response) {
199         struct smtpmsgsrc s;
200         char *fromaddr = NULL;
201         CURL *curl;
202         CURLcode res = CURLE_OK;
203         struct curl_slist *recipients = NULL;
204         long response_code = 421;
205         int num_mx = 0;
206         char mxes[SIZ];
207         char user[1024];
208         char node[1024];
209         char name[1024];
210         char try_this_mx[256];
211         char smtp_url[512];
212         int i;
213
214         syslog(LOG_DEBUG, "smtpclient: smtp_attempt_delivery(%ld, %s)", msgid, recp);
215
216         process_rfc822_addr(recp, user, node, name);    // split recipient address into username, hostname, displayname
217         num_mx = getmx(mxes, node);
218         if (num_mx < 1) {
219                 return (421);
220         }
221
222         CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
223         if (!IsEmptyStr(source_room)) {
224                 // If we have a source room, it's probably a mailing list message; generate an unsubscribe header
225                 char esc_room[ROOMNAMELEN*2];
226                 char esc_email[1024];
227                 urlesc(esc_room, sizeof esc_room, source_room);
228                 urlesc(esc_email, sizeof esc_email, recp);
229                 cprintf("List-Unsubscribe: <http://%s/listsub?cmd=unsubscribe&room=%s&email=%s>\r\n",
230                         CtdlGetConfigStr("c_fqdn"),
231                         esc_room,
232                         esc_email
233                 );
234         }
235         CtdlOutputMsg(msgid, MT_RFC822, HEADERS_ALL, 0, 1, NULL, 0, NULL, &fromaddr, NULL);
236         s.TheMessage = CC->redirect_buffer;
237         s.bytes_total = StrLength(CC->redirect_buffer);
238         s.bytes_sent = 0;
239         CC->redirect_buffer = NULL;
240         response_code = 421;
241                                         // keep trying MXes until one works or we run out
242         for (i = 0; ((i < num_mx) && ((response_code / 100) == 4)); ++i) {
243                 response_code = 421;    // default 421 makes non-protocol errors transient
244                 s.bytes_sent = 0;       // rewind our buffer in case we try multiple MXes
245
246                 curl = curl_easy_init();
247                 if (curl) {
248                         response[0] = 0;
249
250                         if (!IsEmptyStr(envelope_from)) {
251                                 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, envelope_from);
252                         }
253                         else {
254                                 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, fromaddr);
255                         }
256
257                         recipients = curl_slist_append(recipients, recp);
258                         curl_easy_setopt(curl, CURLOPT_MAIL_RCPT, recipients);
259                         curl_easy_setopt(curl, CURLOPT_READFUNCTION, upload_source);
260                         curl_easy_setopt(curl, CURLOPT_READDATA, &s);
261                         curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);                              // tell libcurl we are uploading
262                         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L);                           // Time out after 20 seconds
263                         if (CtdlGetConfigInt("c_smtpclient_disable_starttls") == 0) {
264                                 curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY);        // Attempt STARTTLS if offered
265                         }
266                         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
267                         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
268                         curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, ctdl_libcurl_smtp_debug_callback);
269                         curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void *) response);
270                         curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
271
272                         // Construct an SMTP URL in the form of:
273                         //      smtp[s]://target_host/source_host
274                         // This looks weird but libcurl uses that last part to set our name for EHLO or HELO.
275                         // We check for "smtp://" and "smtps://" because the admin may have put those prefixes in a smart-host entry.
276                         // If there is no prefix we add "smtp://"
277                         extract_token(try_this_mx, mxes, i, '|', (sizeof try_this_mx - 7));
278                         snprintf(smtp_url, sizeof smtp_url,
279                                 "%s%s/%s",
280                                 (((!strncasecmp(try_this_mx, HKEY("smtp://")))
281                                 || (!strncasecmp(try_this_mx, HKEY("smtps://")))) ? "" : "smtp://"),
282                                 try_this_mx, CtdlGetConfigStr("c_fqdn")
283                         );
284                         curl_easy_setopt(curl, CURLOPT_URL, smtp_url);
285                         syslog(LOG_DEBUG, "smtpclient: trying MX %d of %d <%s>", i+1, num_mx, smtp_url);        // send the message
286                         res = curl_easy_perform(curl);
287                         curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
288                         syslog(LOG_DEBUG,
289                                "smtpclient: libcurl returned %d (%s) , SMTP response %ld",
290                                res, curl_easy_strerror(res), response_code
291                         );
292
293                         if ((res != CURLE_OK) && (response_code == 0)) {        // check for errors
294                                 response_code = 421;
295                         }
296
297                         curl_slist_free_all(recipients);
298                         recipients = NULL;                                      // this gets reused; avoid double-free
299                         curl_easy_cleanup(curl);
300                         curl = NULL;                                            // this gets reused; avoid double-free
301
302                         // Trim the error message buffer down to just the actual message
303                         trim_response(response_code, response);
304                 }
305         }
306
307         FreeStrBuf(&s.TheMessage);
308         if (fromaddr) {
309                 free(fromaddr);
310         }
311         return ((int) response_code);
312 }
313
314
315 // Process one outbound message.
316 void smtp_process_one_msg(long qmsgnum) {
317         struct CtdlMessage *msg = NULL;
318         char *instr = NULL;
319         int i;
320         int num_success = 0;
321         int num_fail = 0;
322         int num_delayed = 0;
323         long deletes[2];
324         int delete_this_queue = 0;
325         char server_response[SIZ];
326
327         msg = CtdlFetchMessage(qmsgnum, 1);
328         if (msg == NULL) {
329                 syslog(LOG_WARNING, "smtpclient: %ld does not exist", qmsgnum);
330                 return;
331         }
332
333         instr = msg->cm_fields[eMesageText];
334         msg->cm_fields[eMesageText] = NULL;
335         CM_Free(msg);
336
337         // if the queue message has any CRLF's convert them to LF's
338         char *crlf = NULL;
339         while (crlf = strstr(instr, "\r\n"), crlf != NULL) {
340                 strcpy(crlf, crlf + 1);
341         }
342
343         // Strip out the headers and we are now left with just the instructions.
344         char *soi = strstr(instr, "\n\n");
345         if (soi) {
346                 strcpy(instr, soi + 2);
347         }
348
349         long msgid = 0;
350         time_t submitted = time(NULL);
351         time_t attempted = 0;
352         char *bounceto = NULL;
353         char *envelope_from = NULL;
354         char *source_room = NULL;
355
356         char cfgline[SIZ];
357         for (i = 0; i < num_tokens(instr, '\n'); ++i) {
358                 extract_token(cfgline, instr, i, '\n', sizeof cfgline);
359                 if (!strncasecmp(cfgline, HKEY("msgid|")))
360                         msgid = atol(&cfgline[6]);
361                 if (!strncasecmp(cfgline, HKEY("submitted|")))
362                         submitted = atol(&cfgline[10]);
363                 if (!strncasecmp(cfgline, HKEY("attempted|")))
364                         attempted = atol(&cfgline[10]);
365                 if (!strncasecmp(cfgline, HKEY("bounceto|")))
366                         bounceto = strdup(&cfgline[9]);
367                 if (!strncasecmp(cfgline, HKEY("envelope_from|")))
368                         envelope_from = strdup(&cfgline[14]);
369                 if (!strncasecmp(cfgline, HKEY("source_room|")))
370                         source_room = strdup(&cfgline[12]);
371         }
372
373         int should_try_now = 0;
374         if (attempted < submitted) {                    // If no attempts have been made yet, try now
375                 should_try_now = 1;
376         }
377         else if ((attempted - submitted) <= 14400) {
378                 if ((time(NULL) - attempted) > 1800) {  // First four hours, retry every 30 minutes
379                         should_try_now = 1;
380                 }
381         }
382         else {
383                 if ((time(NULL) - attempted) > 14400) { // After that, retry once every 4 hours
384                         should_try_now = 1;
385                 }
386         }
387
388         if (should_try_now) {
389                 syslog(LOG_DEBUG, "smtpclient: attempting delivery of message <%ld> now", qmsgnum);
390                 if (source_room) {
391                         syslog(LOG_DEBUG, "smtpclient: this message originated in <%s>", source_room);
392                 }
393                 StrBuf *NewInstr = NewStrBuf();
394                 StrBufAppendPrintf(NewInstr, "Content-type: " SPOOLMIME "\n\n");
395                 StrBufAppendPrintf(NewInstr, "msgid|%ld\n", msgid);
396                 StrBufAppendPrintf(NewInstr, "submitted|%ld\n", submitted);
397                 if (bounceto) {
398                         StrBufAppendPrintf(NewInstr, "bounceto|%s\n", bounceto);
399                 }
400                 if (envelope_from) {
401                         StrBufAppendPrintf(NewInstr, "envelope_from|%s\n", envelope_from);
402                 }
403                 for (i = 0; i < num_tokens(instr, '\n'); ++i) {
404                         extract_token(cfgline, instr, i, '\n', sizeof cfgline);
405                         if (!strncasecmp(cfgline, HKEY("remote|"))) {
406                                 char recp[SIZ];
407                                 int previous_result = extract_int(cfgline, 2);
408                                 if ((previous_result == 0) || (previous_result == 4)) {
409                                         int new_result = 421;
410                                         extract_token(recp, cfgline, 1, '|', sizeof recp);
411                                         new_result = smtp_attempt_delivery(msgid, recp, envelope_from, source_room, server_response);
412                                         syslog(LOG_DEBUG, "smtpclient: recp: <%s> , result: %d (%s)", recp, new_result, server_response);
413                                         if ((new_result / 100) == 2) {
414                                                 ++num_success;
415                                         }
416                                         else {
417                                                 if ((new_result / 100) == 5) {
418                                                         ++num_fail;
419                                                 }
420                                                 else {
421                                                         ++num_delayed;
422                                                 }
423                                                 StrBufAppendPrintf(NewInstr, "remote|%s|%ld|%ld (%s)\n", recp, (new_result / 100), new_result, server_response);
424                                         }
425                                 }
426                         }
427                 }
428
429                 StrBufAppendPrintf(NewInstr, "attempted|%ld\n", time(NULL));
430
431                 // All deliveries have now been attempted.  Now determine the disposition of this queue entry.
432
433                 time_t age = time(NULL) - submitted;
434                 syslog(LOG_DEBUG,
435                        "smtpclient: submission age: %ldd%ldh%ldm%lds",
436                        (age / 86400), ((age % 86400) / 3600), ((age % 3600) / 60), (age % 60));
437                 syslog(LOG_DEBUG, "smtpclient: num_success=%d , num_fail=%d , num_delayed=%d", num_success, num_fail, num_delayed);
438
439                 // If there are permanent fails on this attempt, deliver a bounce to the user.
440                 // The 5XX fails will be recorded in the rewritten queue, but they will be removed before the next attempt.
441                 if (num_fail > 0) {
442                         smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_FATALS);
443                 }
444                 // If all deliveries have either succeeded or failed, we are finished with this queue entry.
445                 if (num_delayed == 0) {
446                         delete_this_queue = 1;
447                 }
448                 // If it's been more than five days, give up and tell the sender that delivery failed
449                 else if ((time(NULL) - submitted) > SMTP_DELIVER_FAIL) {
450                         smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_ALL);
451                         delete_this_queue = 1;
452                 }
453                 // If it's been more than four hours but less than five days, warn the sender that delivery is delayed
454                 else if (((attempted - submitted) < SMTP_DELIVER_WARN) && ((time(NULL) - submitted) >= SMTP_DELIVER_WARN)) {
455                         smtp_do_bounce(ChrPtr(NewInstr), SDB_WARN);
456                 }
457
458                 if (delete_this_queue) {
459                         syslog(LOG_DEBUG, "smtpclient: %ld deleting", qmsgnum);
460                         deletes[0] = qmsgnum;
461                         deletes[1] = msgid;
462                         CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, deletes, 2, "");
463                         FreeStrBuf(&NewInstr);  // We have to free NewInstr here, no longer needed
464                 }
465                 else {
466                         // replace the old queue entry with the new one
467                         syslog(LOG_DEBUG, "smtpclient: %ld rewriting", qmsgnum);
468                         msg = convert_internet_message_buf(&NewInstr);  // This function will free NewInstr for us
469                         CtdlSubmitMsg(msg, NULL, SMTP_SPOOLOUT_ROOM);
470                         CM_Free(msg);
471                         CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, &qmsgnum, 1, "");
472                 }
473         }
474         else {
475                 syslog(LOG_DEBUG, "smtpclient: %ld retry time not reached", qmsgnum);
476         }
477
478         if (bounceto != NULL) {
479                 free(bounceto);
480         }
481         if (envelope_from != NULL) {
482                 free(envelope_from);
483         }
484         if (source_room != NULL) {
485                 free(source_room);
486         }
487         free(instr);
488 }
489
490
491 // Callback for smtp_do_queue()
492 void smtp_add_msg(long msgnum, void *userdata) {
493         Array *smtp_queue = (Array *) userdata;
494         array_append(smtp_queue, &msgnum);
495 }
496
497
498 // Run through the queue sending out messages.
499 void smtp_do_queue(void) {
500         static int doing_smtpclient = 0;
501         int i = 0;
502
503         // This is a concurrency check to make sure only one smtpclient run is done at a time.
504         begin_critical_section(S_SMTPQUEUE);
505         if (doing_smtpclient) {
506                 end_critical_section(S_SMTPQUEUE);
507                 return;
508         }
509         doing_smtpclient = 1;
510         end_critical_section(S_SMTPQUEUE);
511
512         syslog(LOG_DEBUG, "smtpclient: start queue run");
513
514         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
515                 syslog(LOG_WARNING, "smtpclient: cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
516                 doing_smtpclient = 0;
517                 return;
518         }
519
520         // This array will hold the list of queue instruction messages
521         Array *smtp_queue = array_new(sizeof(long));
522         if (smtp_queue == NULL) {
523                 syslog(LOG_WARNING, "smtpclient: cannot allocate queue array");
524                 doing_smtpclient = 0;
525                 return;
526         }
527
528         // Put the queue in memory so we can close the db cursor
529         // Searching for messages with a top level Content-type of SPOOLIME will give us only queue instruction messages.
530         CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_add_msg, (void *)smtp_queue);
531
532         // We are ready to run through the queue now.
533         for (i = 0; i < array_len(smtp_queue); ++i) {
534                 long m;
535                 memcpy(&m, array_get_element_at(smtp_queue, i), sizeof(long));
536                 smtp_process_one_msg(m);
537         }
538
539         array_free(smtp_queue);
540         doing_smtpclient = 0;
541         syslog(LOG_DEBUG, "smtpclient: end queue run");
542 }
543
544
545 // Initialization function, called from modules_init.c
546 char *ctdl_module_init_smtpclient(void) {
547         if (!threading) {
548                 CtdlRegisterMessageHook(smtp_aftersave, EVT_AFTERSAVE);
549                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER, PRIO_AGGR + 51);
550                 smtp_init_spoolout();
551         }
552
553         // return our module id for the log
554         return "smtpclient";
555 }