Trim SMTP responses down to one line when putting them into bounce records. Also...
[citadel.git] / citadel / modules / smtp / serv_smtpclient.c
1 /*
2  * Transmit outbound SMTP mail to the big wide world of the Internet
3  *
4  * This is the new, exciting, clever version that makes libcurl do all the work  :)
5  *
6  * Copyright (c) 1997-2018 by the citadel.org team
7  *
8  * This program is open source software; you can redistribute it and/or
9  * modify it under the terms of the GNU General Public License as published
10  * by the Free Software Foundation; either version 3 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16  * GNU General Public License for more details.
17  */
18
19 #include <stdlib.h>
20 #include <unistd.h>
21 #include <stdio.h>
22 #include <sysconfig.h>
23
24 #if TIME_WITH_SYS_TIME
25 #include <sys/time.h>
26 #include <time.h>
27 #else
28 #if HAVE_SYS_TIME_H
29 #include <sys/time.h>
30 #else
31 #include <time.h>
32 #endif
33 #endif
34
35 #include <ctype.h>
36 #include <string.h>
37 #include <errno.h>
38 #include <sys/types.h>
39 #include <sys/stat.h>
40 #include <libcitadel.h>
41 #include <curl/curl.h>
42 #include "citadel.h"
43 #include "server.h"
44 #include "citserver.h"
45 #include "support.h"
46 #include "config.h"
47 #include "ctdl_module.h"
48 #include "clientsocket.h"
49 #include "msgbase.h"
50 #include "domain.h"
51 #include "internet_addressing.h"
52 #include "citadel_dirs.h"
53 #include "modules/smtp/smtp_util.h"
54
55 struct smtpmsgsrc {             // Data passed in and out of libcurl for message upload
56         StrBuf *TheMessage;
57         int bytes_total;
58         int bytes_sent;
59 };
60
61 struct CitContext smtp_client_CC;
62 static int doing_smtpclient = 0;
63 long *smtpq = NULL;             // array of msgnums containing queue instructions
64 int smtpq_count = 0;            // number of queue messages in smtpq
65 int smtpq_alloc = 0;            // current allocation size for smtpq
66
67
68 /*
69  * Initialize the SMTP outbound queue
70  */
71 void smtp_init_spoolout(void)
72 {
73         struct ctdlroom qrbuf;
74
75         /*
76          * Create the room.  This will silently fail if the room already
77          * exists, and that's perfectly ok, because we want it to exist.
78          */
79         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
80
81         /*
82          * Make sure it's set to be a "system room" so it doesn't show up
83          * in the <K>nown rooms list for administrators.
84          */
85         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
86                 qrbuf.QRflags2 |= QR2_SYSTEM;
87                 CtdlPutRoomLock(&qrbuf);
88         }
89 }
90
91
92 /* For internet mail, generate delivery instructions.
93  * Yes, this is recursive.  Deal with it.  Infinite recursion does
94  * not happen because the delivery instructions message does not
95  * contain a recipient.
96  */
97 int smtp_aftersave(struct CtdlMessage *msg, recptypes * recps)
98 {
99         if ((recps != NULL) && (recps->num_internet > 0)) {
100                 struct CtdlMessage *imsg = NULL;
101                 char recipient[SIZ];
102                 StrBuf *SpoolMsg = NewStrBuf();
103                 long nTokens;
104                 int i;
105
106                 syslog(LOG_DEBUG, "smtpclient: generating delivery instructions");
107
108                 StrBufPrintf(SpoolMsg,
109                              "Content-type: " SPOOLMIME "\n"
110                              "\n"
111                              "msgid|%s\n"
112                              "submitted|%ld\n" "bounceto|%s\n", msg->cm_fields[eVltMsgNum], (long) time(NULL), recps->bounce_to);
113
114                 if (recps->envelope_from != NULL) {
115                         StrBufAppendBufPlain(SpoolMsg, HKEY("envelope_from|"), 0);
116                         StrBufAppendBufPlain(SpoolMsg, recps->envelope_from, -1, 0);
117                         StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
118                 }
119                 if (recps->sending_room != NULL) {
120                         StrBufAppendBufPlain(SpoolMsg, HKEY("source_room|"), 0);
121                         StrBufAppendBufPlain(SpoolMsg, recps->sending_room, -1, 0);
122                         StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
123                 }
124
125                 nTokens = num_tokens(recps->recp_internet, '|');
126                 for (i = 0; i < nTokens; i++) {
127                         long len;
128                         len = extract_token(recipient, recps->recp_internet, i, '|', sizeof recipient);
129                         if (len > 0) {
130                                 StrBufAppendBufPlain(SpoolMsg, HKEY("remote|"), 0);
131                                 StrBufAppendBufPlain(SpoolMsg, recipient, len, 0);
132                                 StrBufAppendBufPlain(SpoolMsg, HKEY("|0||\n"), 0);
133                         }
134                 }
135
136                 imsg = malloc(sizeof(struct CtdlMessage));
137                 memset(imsg, 0, sizeof(struct CtdlMessage));
138                 imsg->cm_magic = CTDLMESSAGE_MAGIC;
139                 imsg->cm_anon_type = MES_NORMAL;
140                 imsg->cm_format_type = FMT_RFC822;
141                 CM_SetField(imsg, eMsgSubject, HKEY("QMSG"));
142                 CM_SetField(imsg, eAuthor, HKEY("Citadel"));
143                 CM_SetField(imsg, eJournal, HKEY("do not journal"));
144                 CM_SetAsFieldSB(imsg, eMesageText, &SpoolMsg);
145                 CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM, QP_EADDR);
146                 CM_Free(imsg);
147         }
148         return 0;
149 }
150
151
152 /*
153  * Callback for smtp_attempt_delivery() to supply libcurl with upload data.
154  */
155 static size_t upload_source(void *ptr, size_t size, size_t nmemb, void *userp)
156 {
157         struct smtpmsgsrc *s = (struct smtpmsgsrc *) userp;
158         int sendbytes = 0;
159         const char *send_this = NULL;
160
161         sendbytes = (size * nmemb);
162
163         if (s->bytes_sent >= s->bytes_total) {
164                 return (0);     // no data remaining; we are done
165         }
166
167         if (sendbytes > (s->bytes_total - s->bytes_sent)) {
168                 sendbytes = s->bytes_total - s->bytes_sent;     // can't send more than we have
169         }
170
171         send_this = ChrPtr(s->TheMessage);
172         send_this += s->bytes_sent;     // start where we last left off
173
174         memcpy(ptr, send_this, sendbytes);
175         s->bytes_sent += sendbytes;
176         return (sendbytes);     // return the number of bytes _actually_ copied
177 }
178
179
180 /*
181  * The libcurl API doesn't provide a way to capture the actual SMTP result message returned
182  * by the remote server.  This is an ugly way to extract it, by capturing debug data from
183  * the library and filtering on the lines we want.
184  */
185 int ctdl_libcurl_smtp_debug_callback(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr)
186 {
187         if (type != CURLINFO_HEADER_IN)
188                 return 0;
189         if (!userptr)
190                 return 0;
191         char *debugbuf = (char *) userptr;
192
193         int len = strlen(debugbuf);
194         if (len + size > SIZ)
195                 return 0;
196
197         memcpy(&debugbuf[len], data, size);
198         debugbuf[len + size] = 0;
199         return 0;
200 }
201
202
203 /*
204  * Go through the debug output of an SMTP transaction, and boil it down to just the final success or error response message.
205  */
206 void trim_response(long response_code, char *response)
207 {
208         if ((response_code < 100) || (response_code > 999) || (IsEmptyStr(response))) {
209                 return;
210         }
211
212         char *t = malloc(strlen(response));
213         if (!t) {
214                 return;
215         }
216         t[0] = 0;
217
218         char *p;
219         for (p = response; *p != 0; ++p) {
220                 if ( (*p != '\n') && (!isprint(*p)) ) {         // expunge any nonprintables except for newlines
221                         *p = ' ';
222                 }
223         }
224
225         char response_code_str[4];
226         snprintf(response_code_str, sizeof response_code_str, "%ld", response_code);
227         char *respstart = strstr(response, response_code_str);
228         if (respstart == NULL) {
229                 strcpy(response, smtpstatus(response_code));
230                 return;
231         }
232         strcpy(response, respstart);
233
234         p = strstr(response, "\n");
235         if (p != NULL) {
236                 *p = 0;
237         }
238 }
239
240
241 /*
242  * Attempt a delivery to one recipient.
243  * Returns a three-digit SMTP status code.
244  */
245 int smtp_attempt_delivery(long msgid, char *recp, char *envelope_from, char *response)
246 {
247         struct smtpmsgsrc s;
248         char *fromaddr = NULL;
249         CURL *curl;
250         CURLcode res = CURLE_OK;
251         struct curl_slist *recipients = NULL;
252         long response_code = 421;
253         int num_mx = 0;
254         char mxes[SIZ];
255         char user[1024];
256         char node[1024];
257         char name[1024];
258         char try_this_mx[256];
259         char smtp_url[512];
260         int i;
261
262         syslog(LOG_DEBUG, "smtpclient: smtp_attempt_delivery(%ld, %s)", msgid, recp);
263
264         process_rfc822_addr(recp, user, node, name);    // split recipient address into username, hostname, displayname
265         num_mx = getmx(mxes, node);
266         if (num_mx < 1) {
267                 return (421);
268         }
269
270         CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
271         CtdlOutputMsg(msgid, MT_RFC822, HEADERS_ALL, 0, 1, NULL, 0, NULL, &fromaddr, NULL);
272         s.TheMessage = CC->redirect_buffer;
273         s.bytes_total = StrLength(CC->redirect_buffer);
274         s.bytes_sent = 0;
275         CC->redirect_buffer = NULL;
276         response_code = 421;
277
278         for (i = 0; ((i < num_mx) && ((response_code / 100) == 4)); ++i) {      // keep trying MXes until one works or we run out
279                 response_code = 421;    // default 421 makes non-protocol errors transient
280                 s.bytes_sent = 0;       // rewind our buffer in case we try multiple MXes
281
282                 curl = curl_easy_init();
283                 if (curl) {
284                         response[0] = 0;
285
286                         if (!IsEmptyStr(envelope_from)) {
287                                 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, envelope_from);
288                         } else {
289                                 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, fromaddr);
290                         }
291
292                         recipients = curl_slist_append(recipients, recp);
293                         curl_easy_setopt(curl, CURLOPT_MAIL_RCPT, recipients);
294                         curl_easy_setopt(curl, CURLOPT_READFUNCTION, upload_source);
295                         curl_easy_setopt(curl, CURLOPT_READDATA, &s);
296                         curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);      // tell libcurl we are uploading
297                         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L);   // Time out after 20 seconds
298                         if (CtdlGetConfigInt("c_smtpclient_disable_starttls") == 0) {
299                                 curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY);        // Attempt STARTTLS if offered
300                         }
301                         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
302                         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
303                         curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, ctdl_libcurl_smtp_debug_callback);
304                         curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void *) response);
305                         curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
306
307                         // Construct an SMTP URL in the form of:
308                         //      smtp[s]://target_host/source_host
309                         // This looks weird but libcurl uses that last part to set our name for EHLO or HELO.
310                         // We check for "smtp://" and "smtps://" because the admin may have put those prefixes in a smart-host entry.
311                         // If there is no prefix we add "smtp://"
312                         extract_token(try_this_mx, mxes, i, '|', (sizeof try_this_mx - 7));
313                         snprintf(smtp_url, sizeof smtp_url, "%s%s/%s", (((!strncasecmp(try_this_mx, HKEY("smtp://")))
314                                                                          ||
315                                                                          (!strncasecmp
316                                                                           (try_this_mx,
317                                                                            HKEY("smtps://")))) ? "" : "smtp://"),
318                                  try_this_mx, CtdlGetConfigStr("c_fqdn")
319                             );
320                         curl_easy_setopt(curl, CURLOPT_URL, smtp_url);
321                         syslog(LOG_DEBUG, "smtpclient: trying %s", smtp_url);   // send the message
322                         res = curl_easy_perform(curl);
323                         curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
324                         syslog(LOG_DEBUG,
325                                "smtpclient: libcurl returned %d (%s) , SMTP response %ld",
326                                res, curl_easy_strerror(res), response_code);
327
328                         if ((res != CURLE_OK) && (response_code == 0)) {        // check for errors
329                                 response_code = 421;
330                         }
331
332                         curl_slist_free_all(recipients);
333                         recipients = NULL;      // this gets reused; avoid double-free
334                         curl_easy_cleanup(curl);
335                         curl = NULL;    // this gets reused; avoid double-free
336
337                         /* Trim the error message buffer down to just the actual message */
338                         trim_response(response_code, response);
339                 }
340         }
341
342         FreeStrBuf(&s.TheMessage);
343         if (fromaddr)
344                 free(fromaddr);
345         return ((int) response_code);
346 }
347
348
349 /*
350  * Process one outbound message.
351  */
352 void smtp_process_one_msg(long qmsgnum)
353 {
354         struct CtdlMessage *msg = NULL;
355         char *instr = NULL;
356         int i;
357         int num_success = 0;
358         int num_fail = 0;
359         int num_delayed = 0;
360         long deletes[2];
361         int delete_this_queue = 0;
362         char server_response[SIZ];
363
364         msg = CtdlFetchMessage(qmsgnum, 1, 1);
365         if (msg == NULL) {
366                 syslog(LOG_WARNING, "smtpclient: %ld does not exist", qmsgnum);
367                 return;
368         }
369
370         instr = msg->cm_fields[eMesageText];
371         msg->cm_fields[eMesageText] = NULL;
372         CM_Free(msg);
373
374         // if the queue message has any CRLF's convert them to LF's
375         char *crlf = NULL;
376         while (crlf = strstr(instr, "\r\n"), crlf != NULL) {
377                 strcpy(crlf, crlf + 1);
378         }
379
380         // Strip out the headers and we are now left with just the instructions.
381         char *soi = strstr(instr, "\n\n");
382         if (soi) {
383                 strcpy(instr, soi + 2);
384         }
385
386         long msgid = 0;
387         time_t submitted = time(NULL);
388         time_t attempted = 0;
389         char *bounceto = NULL;
390         char *envelope_from = NULL;
391
392         char cfgline[SIZ];
393         for (i = 0; i < num_tokens(instr, '\n'); ++i) {
394                 extract_token(cfgline, instr, i, '\n', sizeof cfgline);
395                 if (!strncasecmp(cfgline, HKEY("msgid|")))
396                         msgid = atol(&cfgline[6]);
397                 if (!strncasecmp(cfgline, HKEY("submitted|")))
398                         submitted = atol(&cfgline[10]);
399                 if (!strncasecmp(cfgline, HKEY("attempted|")))
400                         attempted = atol(&cfgline[10]);
401                 if (!strncasecmp(cfgline, HKEY("bounceto|")))
402                         bounceto = strdup(&cfgline[9]);
403                 if (!strncasecmp(cfgline, HKEY("envelope_from|")))
404                         envelope_from = strdup(&cfgline[14]);
405         }
406
407         int should_try_now = 0;
408         if (attempted < submitted) {    // If no attempts have been made yet, try now
409                 should_try_now = 1;
410         } else if ((attempted - submitted) <= 14400) {
411                 if ((time(NULL) - attempted) > 1800) {  // First four hours, retry every 30 minutes
412                         should_try_now = 1;
413                 }
414         } else {
415                 if ((time(NULL) - attempted) > 14400) { // After that, retry once every 4 hours
416                         should_try_now = 1;
417                 }
418         }
419
420         if (should_try_now) {
421                 syslog(LOG_DEBUG, "smtpclient: %ld attempting delivery now", qmsgnum);
422                 StrBuf *NewInstr = NewStrBuf();
423                 StrBufAppendPrintf(NewInstr, "Content-type: " SPOOLMIME "\n\n");
424                 StrBufAppendPrintf(NewInstr, "msgid|%ld\n", msgid);
425                 StrBufAppendPrintf(NewInstr, "submitted|%ld\n", submitted);
426                 if (bounceto)
427                         StrBufAppendPrintf(NewInstr, "bounceto|%s\n", bounceto);
428                 if (envelope_from)
429                         StrBufAppendPrintf(NewInstr, "envelope_from|%s\n", envelope_from);
430
431                 for (i = 0; i < num_tokens(instr, '\n'); ++i) {
432                         extract_token(cfgline, instr, i, '\n', sizeof cfgline);
433                         if (!strncasecmp(cfgline, HKEY("remote|"))) {
434                                 char recp[SIZ];
435                                 int previous_result = extract_int(cfgline, 2);
436                                 if ((previous_result == 0)
437                                     || (previous_result == 4)) {
438                                         int new_result = 421;
439                                         extract_token(recp, cfgline, 1, '|', sizeof recp);
440                                         new_result = smtp_attempt_delivery(msgid, recp, envelope_from, server_response);
441                                         syslog(LOG_DEBUG,
442                                                "smtpclient: recp: <%s> , result: %d (%s)", recp, new_result, server_response);
443                                         if ((new_result / 100) == 2) {
444                                                 ++num_success;
445                                         } else {
446                                                 if ((new_result / 100) == 5) {
447                                                         ++num_fail;
448                                                 } else {
449                                                         ++num_delayed;
450                                                 }
451                                                 StrBufAppendPrintf
452                                                     (NewInstr,
453                                                      "remote|%s|%ld|%ld (%s)\n",
454                                                      recp, (new_result / 100), new_result, server_response);
455                                         }
456                                 }
457                         }
458                 }
459
460                 StrBufAppendPrintf(NewInstr, "attempted|%ld\n", time(NULL));
461
462                 // All deliveries have now been attempted.  Now determine the disposition of this queue entry.
463
464                 time_t age = time(NULL) - submitted;
465                 syslog(LOG_DEBUG,
466                        "smtpclient: submission age: %ldd%ldh%ldm%lds",
467                        (age / 86400), ((age % 86400) / 3600), ((age % 3600) / 60), (age % 60));
468                 syslog(LOG_DEBUG, "smtpclient: num_success=%d , num_fail=%d , num_delayed=%d", num_success, num_fail, num_delayed);
469
470                 // If there are permanent fails on this attempt, deliver a bounce to the user.
471                 // The 5XX fails will be recorded in the rewritten queue, but they will be removed before the next attempt.
472                 if (num_fail > 0) {
473                         smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_FATALS);
474                 }
475                 // If all deliveries have either succeeded or failed, we are finished with this queue entry.
476                 //
477                 if (num_delayed == 0) {
478                         delete_this_queue = 1;
479                 }
480                 // If it's been more than five days, give up and tell the sender that delivery failed
481                 //
482                 else if ((time(NULL) - submitted) > SMTP_DELIVER_FAIL) {
483                         smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_ALL);
484                         delete_this_queue = 1;
485                 }
486                 // If it's been more than four hours but less than five days, warn the sender that delivery is delayed
487                 //
488                 else if (((attempted - submitted) < SMTP_DELIVER_WARN)
489                          && ((time(NULL) - submitted) >= SMTP_DELIVER_WARN)) {
490                         smtp_do_bounce(ChrPtr(NewInstr), SDB_WARN);
491                 }
492
493                 if (delete_this_queue) {
494                         syslog(LOG_DEBUG, "smtpclient: %ld deleting", qmsgnum);
495                         deletes[0] = qmsgnum;
496                         deletes[1] = msgid;
497                         CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, deletes, 2, "");
498                         FreeStrBuf(&NewInstr);  // We have to free NewInstr here, no longer needed
499                 } else {
500                         // replace the old queue entry with the new one
501                         syslog(LOG_DEBUG, "smtpclient: %ld rewriting", qmsgnum);
502                         msg = convert_internet_message_buf(&NewInstr);  // This function will free NewInstr for us
503                         CtdlSubmitMsg(msg, NULL, SMTP_SPOOLOUT_ROOM, 0);
504                         CM_Free(msg);
505                         CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, &qmsgnum, 1, "");
506                 }
507         } else {
508                 syslog(LOG_DEBUG, "smtpclient: %ld retry time not reached", qmsgnum);
509         }
510
511         if (bounceto != NULL)
512                 free(bounceto);
513         if (envelope_from != NULL)
514                 free(envelope_from);
515         free(instr);
516 }
517
518
519 /*
520  * Callback for smtp_do_queue()
521  */
522 void smtp_add_msg(long msgnum, void *userdata)
523 {
524
525         if (smtpq == NULL) {
526                 smtpq_count = 0;
527                 smtpq_alloc = 100;
528                 smtpq = malloc(smtpq_alloc * sizeof(long));
529         }
530
531         if (smtpq_alloc >= smtpq_count) {
532                 smtpq_alloc += 100;
533                 smtpq = realloc(smtpq, (smtpq_alloc * sizeof(long)));
534         }
535
536         smtpq[smtpq_count++] = msgnum;
537 }
538
539
540 /*
541  * Run through the queue sending out messages.
542  */
543 void smtp_do_queue(void)
544 {
545         int i = 0;
546
547         /*
548          * This is a simple concurrency check to make sure only one smtpclient
549          * run is done at a time.  We could do this with a mutex, but since we
550          * don't really require extremely fine granularity here, we'll do it
551          * with a static variable instead.
552          */
553         if (doing_smtpclient)
554                 return;
555         doing_smtpclient = 1;
556
557         syslog(LOG_DEBUG, "smtpclient: start queue run");
558         pthread_setspecific(MyConKey, (void *) &smtp_client_CC);
559
560         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
561                 syslog(LOG_WARNING, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
562                 doing_smtpclient = 0;
563                 return;
564         }
565         // Put the queue in memory so we can close the db cursor
566         CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_add_msg, NULL);
567
568         // We are ready to run through the queue now.
569         for (i = 0; i < smtpq_count; ++i) {
570                 smtp_process_one_msg(smtpq[i]);
571         }
572
573         smtpq_count = 0;        // don't free it, we will use this memory on the next run
574         doing_smtpclient = 0;
575         syslog(LOG_DEBUG, "smtpclient: end queue run");
576 }
577
578
579 /*
580  * Module entry point
581  */
582 CTDL_MODULE_INIT(smtpclient)
583 {
584         if (!threading) {
585                 CtdlFillSystemContext(&smtp_client_CC, "SMTP_Send");
586                 CtdlRegisterMessageHook(smtp_aftersave, EVT_AFTERSAVE);
587                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER, PRIO_AGGR + 50);
588                 smtp_init_spoolout();
589         }
590
591         /* return our module id for the log */
592         return "smtpclient";
593 }