]> code.citadel.org Git - citadel.git/blob - citadel/modules/smtp/serv_smtpclient.c
473b310e519ceabb1d9aa6e7ab3d5fd4cbe08a7e
[citadel.git] / citadel / modules / smtp / serv_smtpclient.c
1 // Transmit outbound SMTP mail to the big wide world of the Internet
2 //
3 // This is the new, exciting, clever version that makes libcurl do all the work  :)
4 //
5 // Copyright (c) 1997-2022 by the citadel.org team
6 //
7 // This program is open source software; you can redistribute it and/or
8 // modify it under the terms of the GNU General Public License as published
9 // by the Free Software Foundation; either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // This program is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 // GNU General Public License for more details.
16
17 #include <stdlib.h>
18 #include <unistd.h>
19 #include <stdio.h>
20 #include <sysconfig.h>
21 #include <time.h>
22 #include <ctype.h>
23 #include <string.h>
24 #include <errno.h>
25 #include <sys/types.h>
26 #include <sys/stat.h>
27 #include <libcitadel.h>
28 #include <curl/curl.h>
29 #include "citadel.h"
30 #include "server.h"
31 #include "citserver.h"
32 #include "support.h"
33 #include "config.h"
34 #include "ctdl_module.h"
35 #include "clientsocket.h"
36 #include "msgbase.h"
37 #include "domain.h"
38 #include "internet_addressing.h"
39 #include "citadel_dirs.h"
40 #include "modules/smtp/smtp_util.h"
41
42 struct smtpmsgsrc {             // Data passed in and out of libcurl for message upload
43         StrBuf *TheMessage;
44         int bytes_total;
45         int bytes_sent;
46 };
47
48 static int doing_smtpclient = 0;
49 long *smtpq = NULL;             // array of msgnums containing queue instructions
50 int smtpq_count = 0;            // number of queue messages in smtpq
51 int smtpq_alloc = 0;            // current allocation size for smtpq
52
53
54 // Initialize the SMTP outbound queue
55 void smtp_init_spoolout(void) {
56         struct ctdlroom qrbuf;
57
58         // Create the room.  This will silently fail if the room already
59         // exists, and that's perfectly ok, because we want it to exist.
60         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
61
62         // Make sure it's set to be a "system room" so it doesn't show up
63         // in the <K>nown rooms list for administrators.
64         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
65                 qrbuf.QRflags2 |= QR2_SYSTEM;
66                 CtdlPutRoomLock(&qrbuf);
67         }
68 }
69
70
71 // For internet mail, generate delivery instructions.
72 // Yes, this is recursive.  Deal with it.  Infinite recursion does
73 // not happen because the delivery instructions message does not
74 // contain a recipient.
75 int smtp_aftersave(struct CtdlMessage *msg, struct recptypes *recps) {
76         if ((recps != NULL) && (recps->num_internet > 0)) {
77                 struct CtdlMessage *imsg = NULL;
78                 char recipient[SIZ];
79                 StrBuf *SpoolMsg = NewStrBuf();
80                 long nTokens;
81                 int i;
82
83                 syslog(LOG_DEBUG, "smtpclient: generating delivery instructions");
84
85                 StrBufPrintf(SpoolMsg,
86                              "Content-type: " SPOOLMIME "\n"
87                              "\n"
88                              "msgid|%s\n"
89                              "submitted|%ld\n" "bounceto|%s\n", msg->cm_fields[eVltMsgNum], (long) time(NULL), recps->bounce_to);
90
91                 if (recps->envelope_from != NULL) {
92                         StrBufAppendBufPlain(SpoolMsg, HKEY("envelope_from|"), 0);
93                         StrBufAppendBufPlain(SpoolMsg, recps->envelope_from, -1, 0);
94                         StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
95                 }
96                 if (recps->sending_room != NULL) {
97                         StrBufAppendBufPlain(SpoolMsg, HKEY("source_room|"), 0);
98                         StrBufAppendBufPlain(SpoolMsg, recps->sending_room, -1, 0);
99                         StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
100                 }
101
102                 nTokens = num_tokens(recps->recp_internet, '|');
103                 for (i = 0; i < nTokens; i++) {
104                         long len;
105                         len = extract_token(recipient, recps->recp_internet, i, '|', sizeof recipient);
106                         if (len > 0) {
107                                 StrBufAppendBufPlain(SpoolMsg, HKEY("remote|"), 0);
108                                 StrBufAppendBufPlain(SpoolMsg, recipient, len, 0);
109                                 StrBufAppendBufPlain(SpoolMsg, HKEY("|0||\n"), 0);
110                         }
111                 }
112
113                 imsg = malloc(sizeof(struct CtdlMessage));
114                 memset(imsg, 0, sizeof(struct CtdlMessage));
115                 imsg->cm_magic = CTDLMESSAGE_MAGIC;
116                 imsg->cm_anon_type = MES_NORMAL;
117                 imsg->cm_format_type = FMT_RFC822;
118                 CM_SetField(imsg, eMsgSubject, HKEY("QMSG"));
119                 CM_SetField(imsg, eAuthor, HKEY("Citadel"));
120                 CM_SetField(imsg, eJournal, HKEY("do not journal"));
121                 CM_SetAsFieldSB(imsg, eMesageText, &SpoolMsg);
122                 CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM);
123                 CM_Free(imsg);
124         }
125         return 0;
126 }
127
128
129 // Callback for smtp_attempt_delivery() to supply libcurl with upload data.
130 static size_t upload_source(void *ptr, size_t size, size_t nmemb, void *userp) {
131         struct smtpmsgsrc *s = (struct smtpmsgsrc *) userp;
132         int sendbytes = 0;
133         const char *send_this = NULL;
134
135         sendbytes = (size * nmemb);
136
137         if (s->bytes_sent >= s->bytes_total) {
138                 return (0);                                     // no data remaining; we are done
139         }
140
141         if (sendbytes > (s->bytes_total - s->bytes_sent)) {
142                 sendbytes = s->bytes_total - s->bytes_sent;     // can't send more than we have
143         }
144
145         send_this = ChrPtr(s->TheMessage);
146         send_this += s->bytes_sent;                             // start where we last left off
147
148         memcpy(ptr, send_this, sendbytes);
149         s->bytes_sent += sendbytes;
150         return(sendbytes);                                      // return the number of bytes _actually_ copied
151 }
152
153
154 // The libcurl API doesn't provide a way to capture the actual SMTP result message returned
155 // by the remote server.  This is an ugly way to extract it, by capturing debug data from
156 // the library and filtering on the lines we want.
157 int ctdl_libcurl_smtp_debug_callback(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr) {
158         if (type != CURLINFO_HEADER_IN)
159                 return 0;
160         if (!userptr)
161                 return 0;
162         char *debugbuf = (char *) userptr;
163
164         int len = strlen(debugbuf);
165         if (len + size > SIZ)
166                 return 0;
167
168         memcpy(&debugbuf[len], data, size);
169         debugbuf[len + size] = 0;
170         return 0;
171 }
172
173
174 // Go through the debug output of an SMTP transaction, and boil it down to just the final success or error response message.
175 void trim_response(long response_code, char *response) {
176         if ((response_code < 100) || (response_code > 999) || (IsEmptyStr(response))) {
177                 return;
178         }
179
180         char *t = malloc(strlen(response));
181         if (!t) {
182                 return;
183         }
184         t[0] = 0;
185
186         char *p;
187         for (p = response; *p != 0; ++p) {
188                 if ( (*p != '\n') && (!isprint(*p)) ) {         // expunge any nonprintables except for newlines
189                         *p = ' ';
190                 }
191         }
192
193         char response_code_str[4];
194         snprintf(response_code_str, sizeof response_code_str, "%ld", response_code);
195         char *respstart = strstr(response, response_code_str);
196         if (respstart == NULL) {                                // If we have a response code but no response text,
197                 strcpy(response, smtpstatus(response_code));    // use one of our canned messages.
198                 return;
199         }
200         strcpy(response, respstart);
201
202         p = strstr(response, "\n");
203         if (p != NULL) {
204                 *p = 0;
205         }
206 }
207
208
209 // Attempt a delivery to one recipient.
210 // Returns a three-digit SMTP status code.
211 int smtp_attempt_delivery(long msgid, char *recp, char *envelope_from, char *source_room, char *response) {
212         struct smtpmsgsrc s;
213         char *fromaddr = NULL;
214         CURL *curl;
215         CURLcode res = CURLE_OK;
216         struct curl_slist *recipients = NULL;
217         long response_code = 421;
218         int num_mx = 0;
219         char mxes[SIZ];
220         char user[1024];
221         char node[1024];
222         char name[1024];
223         char try_this_mx[256];
224         char smtp_url[512];
225         int i;
226
227         syslog(LOG_DEBUG, "smtpclient: smtp_attempt_delivery(%ld, %s)", msgid, recp);
228
229         process_rfc822_addr(recp, user, node, name);    // split recipient address into username, hostname, displayname
230         num_mx = getmx(mxes, node);
231         if (num_mx < 1) {
232                 return (421);
233         }
234
235         CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
236         if (!IsEmptyStr(source_room)) {
237                 // FIXME this is a useless header; we really want to generate a List-Unsubscribe header here.
238                 cprintf("X-Citadel-Source-Room: %s\r\n", source_room);
239         }
240         CtdlOutputMsg(msgid, MT_RFC822, HEADERS_ALL, 0, 1, NULL, 0, NULL, &fromaddr, NULL);
241         s.TheMessage = CC->redirect_buffer;
242         s.bytes_total = StrLength(CC->redirect_buffer);
243         s.bytes_sent = 0;
244         CC->redirect_buffer = NULL;
245         response_code = 421;
246                                         // keep trying MXes until one works or we run out
247         for (i = 0; ((i < num_mx) && ((response_code / 100) == 4)); ++i) {
248                 response_code = 421;    // default 421 makes non-protocol errors transient
249                 s.bytes_sent = 0;       // rewind our buffer in case we try multiple MXes
250
251                 curl = curl_easy_init();
252                 if (curl) {
253                         response[0] = 0;
254
255                         if (!IsEmptyStr(envelope_from)) {
256                                 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, envelope_from);
257                         }
258                         else {
259                                 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, fromaddr);
260                         }
261
262                         recipients = curl_slist_append(recipients, recp);
263                         curl_easy_setopt(curl, CURLOPT_MAIL_RCPT, recipients);
264                         curl_easy_setopt(curl, CURLOPT_READFUNCTION, upload_source);
265                         curl_easy_setopt(curl, CURLOPT_READDATA, &s);
266                         curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);                              // tell libcurl we are uploading
267                         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L);                           // Time out after 20 seconds
268                         if (CtdlGetConfigInt("c_smtpclient_disable_starttls") == 0) {
269                                 curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY);        // Attempt STARTTLS if offered
270                         }
271                         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
272                         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
273                         curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, ctdl_libcurl_smtp_debug_callback);
274                         curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void *) response);
275                         curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
276
277                         // Construct an SMTP URL in the form of:
278                         //      smtp[s]://target_host/source_host
279                         // This looks weird but libcurl uses that last part to set our name for EHLO or HELO.
280                         // We check for "smtp://" and "smtps://" because the admin may have put those prefixes in a smart-host entry.
281                         // If there is no prefix we add "smtp://"
282                         extract_token(try_this_mx, mxes, i, '|', (sizeof try_this_mx - 7));
283                         snprintf(smtp_url, sizeof smtp_url,
284                                 "%s%s/%s",
285                                 (((!strncasecmp(try_this_mx, HKEY("smtp://")))
286                                 || (!strncasecmp(try_this_mx, HKEY("smtps://")))) ? "" : "smtp://"),
287                                 try_this_mx, CtdlGetConfigStr("c_fqdn")
288                         );
289                         curl_easy_setopt(curl, CURLOPT_URL, smtp_url);
290                         syslog(LOG_DEBUG, "smtpclient: trying MX %d of %d <%s>", i+1, num_mx, smtp_url);        // send the message
291                         res = curl_easy_perform(curl);
292                         curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
293                         syslog(LOG_DEBUG,
294                                "smtpclient: libcurl returned %d (%s) , SMTP response %ld",
295                                res, curl_easy_strerror(res), response_code
296                         );
297
298                         if ((res != CURLE_OK) && (response_code == 0)) {        // check for errors
299                                 response_code = 421;
300                         }
301
302                         curl_slist_free_all(recipients);
303                         recipients = NULL;                                      // this gets reused; avoid double-free
304                         curl_easy_cleanup(curl);
305                         curl = NULL;                                            // this gets reused; avoid double-free
306
307                         // Trim the error message buffer down to just the actual message
308                         trim_response(response_code, response);
309                 }
310         }
311
312         FreeStrBuf(&s.TheMessage);
313         if (fromaddr) {
314                 free(fromaddr);
315         }
316         return ((int) response_code);
317 }
318
319
320 // Process one outbound message.
321 void smtp_process_one_msg(long qmsgnum) {
322         struct CtdlMessage *msg = NULL;
323         char *instr = NULL;
324         int i;
325         int num_success = 0;
326         int num_fail = 0;
327         int num_delayed = 0;
328         long deletes[2];
329         int delete_this_queue = 0;
330         char server_response[SIZ];
331
332         msg = CtdlFetchMessage(qmsgnum, 1);
333         if (msg == NULL) {
334                 syslog(LOG_WARNING, "smtpclient: %ld does not exist", qmsgnum);
335                 return;
336         }
337
338         instr = msg->cm_fields[eMesageText];
339         msg->cm_fields[eMesageText] = NULL;
340         CM_Free(msg);
341
342         // if the queue message has any CRLF's convert them to LF's
343         char *crlf = NULL;
344         while (crlf = strstr(instr, "\r\n"), crlf != NULL) {
345                 strcpy(crlf, crlf + 1);
346         }
347
348         // Strip out the headers and we are now left with just the instructions.
349         char *soi = strstr(instr, "\n\n");
350         if (soi) {
351                 strcpy(instr, soi + 2);
352         }
353
354         long msgid = 0;
355         time_t submitted = time(NULL);
356         time_t attempted = 0;
357         char *bounceto = NULL;
358         char *envelope_from = NULL;
359         char *source_room = NULL;
360
361         char cfgline[SIZ];
362         for (i = 0; i < num_tokens(instr, '\n'); ++i) {
363                 extract_token(cfgline, instr, i, '\n', sizeof cfgline);
364                 if (!strncasecmp(cfgline, HKEY("msgid|")))
365                         msgid = atol(&cfgline[6]);
366                 if (!strncasecmp(cfgline, HKEY("submitted|")))
367                         submitted = atol(&cfgline[10]);
368                 if (!strncasecmp(cfgline, HKEY("attempted|")))
369                         attempted = atol(&cfgline[10]);
370                 if (!strncasecmp(cfgline, HKEY("bounceto|")))
371                         bounceto = strdup(&cfgline[9]);
372                 if (!strncasecmp(cfgline, HKEY("envelope_from|")))
373                         envelope_from = strdup(&cfgline[14]);
374                 if (!strncasecmp(cfgline, HKEY("source_room|")))
375                         source_room = strdup(&cfgline[12]);
376         }
377
378         int should_try_now = 0;
379         if (attempted < submitted) {                    // If no attempts have been made yet, try now
380                 should_try_now = 1;
381         }
382         else if ((attempted - submitted) <= 14400) {
383                 if ((time(NULL) - attempted) > 1800) {  // First four hours, retry every 30 minutes
384                         should_try_now = 1;
385                 }
386         }
387         else {
388                 if ((time(NULL) - attempted) > 14400) { // After that, retry once every 4 hours
389                         should_try_now = 1;
390                 }
391         }
392
393         if (should_try_now) {
394                 syslog(LOG_DEBUG, "smtpclient: attempting delivery of message <%ld> now", qmsgnum);
395                 if (source_room) {
396                         syslog(LOG_DEBUG, "smtpclient: this message originated in <%s>", source_room);
397                 }
398                 StrBuf *NewInstr = NewStrBuf();
399                 StrBufAppendPrintf(NewInstr, "Content-type: " SPOOLMIME "\n\n");
400                 StrBufAppendPrintf(NewInstr, "msgid|%ld\n", msgid);
401                 StrBufAppendPrintf(NewInstr, "submitted|%ld\n", submitted);
402                 if (bounceto) {
403                         StrBufAppendPrintf(NewInstr, "bounceto|%s\n", bounceto);
404                 }
405                 if (envelope_from) {
406                         StrBufAppendPrintf(NewInstr, "envelope_from|%s\n", envelope_from);
407                 }
408                 for (i = 0; i < num_tokens(instr, '\n'); ++i) {
409                         extract_token(cfgline, instr, i, '\n', sizeof cfgline);
410                         if (!strncasecmp(cfgline, HKEY("remote|"))) {
411                                 char recp[SIZ];
412                                 int previous_result = extract_int(cfgline, 2);
413                                 if ((previous_result == 0)
414                                     || (previous_result == 4)) {
415                                         int new_result = 421;
416                                         extract_token(recp, cfgline, 1, '|', sizeof recp);
417                                         new_result = smtp_attempt_delivery(msgid, recp, envelope_from, source_room, server_response);
418                                         syslog(LOG_DEBUG, "smtpclient: recp: <%s> , result: %d (%s)", recp, new_result, server_response);
419                                         if ((new_result / 100) == 2) {
420                                                 ++num_success;
421                                         }
422                                         else {
423                                                 if ((new_result / 100) == 5) {
424                                                         ++num_fail;
425                                                 }
426                                                 else {
427                                                         ++num_delayed;
428                                                 }
429                                                 StrBufAppendPrintf
430                                                     (NewInstr,
431                                                      "remote|%s|%ld|%ld (%s)\n",
432                                                      recp, (new_result / 100), new_result, server_response);
433                                         }
434                                 }
435                         }
436                 }
437
438                 StrBufAppendPrintf(NewInstr, "attempted|%ld\n", time(NULL));
439
440                 // All deliveries have now been attempted.  Now determine the disposition of this queue entry.
441
442                 time_t age = time(NULL) - submitted;
443                 syslog(LOG_DEBUG,
444                        "smtpclient: submission age: %ldd%ldh%ldm%lds",
445                        (age / 86400), ((age % 86400) / 3600), ((age % 3600) / 60), (age % 60));
446                 syslog(LOG_DEBUG, "smtpclient: num_success=%d , num_fail=%d , num_delayed=%d", num_success, num_fail, num_delayed);
447
448                 // If there are permanent fails on this attempt, deliver a bounce to the user.
449                 // The 5XX fails will be recorded in the rewritten queue, but they will be removed before the next attempt.
450                 if (num_fail > 0) {
451                         smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_FATALS);
452                 }
453                 // If all deliveries have either succeeded or failed, we are finished with this queue entry.
454                 if (num_delayed == 0) {
455                         delete_this_queue = 1;
456                 }
457                 // If it's been more than five days, give up and tell the sender that delivery failed
458                 else if ((time(NULL) - submitted) > SMTP_DELIVER_FAIL) {
459                         smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_ALL);
460                         delete_this_queue = 1;
461                 }
462                 // If it's been more than four hours but less than five days, warn the sender that delivery is delayed
463                 else if (((attempted - submitted) < SMTP_DELIVER_WARN)
464                          && ((time(NULL) - submitted) >= SMTP_DELIVER_WARN)) {
465                         smtp_do_bounce(ChrPtr(NewInstr), SDB_WARN);
466                 }
467
468                 if (delete_this_queue) {
469                         syslog(LOG_DEBUG, "smtpclient: %ld deleting", qmsgnum);
470                         deletes[0] = qmsgnum;
471                         deletes[1] = msgid;
472                         CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, deletes, 2, "");
473                         FreeStrBuf(&NewInstr);  // We have to free NewInstr here, no longer needed
474                 }
475                 else {
476                         // replace the old queue entry with the new one
477                         syslog(LOG_DEBUG, "smtpclient: %ld rewriting", qmsgnum);
478                         msg = convert_internet_message_buf(&NewInstr);  // This function will free NewInstr for us
479                         CtdlSubmitMsg(msg, NULL, SMTP_SPOOLOUT_ROOM);
480                         CM_Free(msg);
481                         CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, &qmsgnum, 1, "");
482                 }
483         }
484         else {
485                 syslog(LOG_DEBUG, "smtpclient: %ld retry time not reached", qmsgnum);
486         }
487
488         if (bounceto != NULL) {
489                 free(bounceto);
490         }
491         if (envelope_from != NULL) {
492                 free(envelope_from);
493         }
494         if (source_room != NULL) {
495                 free(source_room);
496         }
497         free(instr);
498 }
499
500
501 // Callback for smtp_do_queue()
502 void smtp_add_msg(long msgnum, void *userdata) {
503
504         if (smtpq == NULL) {
505                 smtpq_count = 0;
506                 smtpq_alloc = 100;
507                 smtpq = malloc(smtpq_alloc * sizeof(long));
508         }
509
510         if (smtpq_alloc >= smtpq_count) {
511                 smtpq_alloc += 100;
512                 smtpq = realloc(smtpq, (smtpq_alloc * sizeof(long)));
513         }
514
515         smtpq[smtpq_count++] = msgnum;
516 }
517
518
519 // Run through the queue sending out messages.
520 void smtp_do_queue(void) {
521         int i = 0;
522
523         // This is a simple concurrency check to make sure only one smtpclient
524         // run is done at a time.  We could do this with a mutex, but since we
525         // don't really require extremely fine granularity here, we'll do it
526         // with a static variable instead.
527         if (doing_smtpclient) {
528                 return;
529         }
530         doing_smtpclient = 1;
531
532         syslog(LOG_DEBUG, "smtpclient: start queue run");
533
534         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
535                 syslog(LOG_WARNING, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
536                 doing_smtpclient = 0;
537                 return;
538         }
539         // Put the queue in memory so we can close the db cursor
540         CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_add_msg, NULL);
541
542         // We are ready to run through the queue now.
543         for (i = 0; i < smtpq_count; ++i) {
544                 smtp_process_one_msg(smtpq[i]);
545         }
546
547         smtpq_count = 0;        // don't free it, we will use this memory on the next run
548         doing_smtpclient = 0;
549         syslog(LOG_DEBUG, "smtpclient: end queue run");
550 }
551
552
553 // Module entry point
554 CTDL_MODULE_INIT(smtpclient)
555 {
556         if (!threading) {
557                 CtdlRegisterMessageHook(smtp_aftersave, EVT_AFTERSAVE);
558                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER, PRIO_AGGR + 51);
559                 smtp_init_spoolout();
560         }
561
562         // return our module id for the log
563         return "smtpclient";
564 }