citadel/modules/smtp/serv_smtpclient.c

   1 /*
   2  * Transmit outbound SMTP mail to the big wide world of the Internet
   3  *
   4  * This is the new, exciting, clever version that makes libcurl do all the work  :)
   5  *
   6  * Copyright (c) 1997-2017 by the citadel.org team
   7  *
   8  * This program is open source software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as published
  10  * by the Free Software Foundation; either version 3 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  */
  18
  19 #include <stdlib.h>
  20 #include <unistd.h>
  21 #include <stdio.h>
  22 #include <sysconfig.h>
  23
  24 #if TIME_WITH_SYS_TIME
  25 # include <sys/time.h>
  26 # include <time.h>
  27 #else
  28 # if HAVE_SYS_TIME_H
  29 #  include <sys/time.h>
  30 # else
  31 #  include <time.h>
  32 # endif
  33 #endif
  34
  35 #include <ctype.h>
  36 #include <string.h>
  37 #include <errno.h>
  38 #include <sys/types.h>
  39 #include <sys/stat.h>
  40 #include <libcitadel.h>
  41 #include <curl/curl.h>
  42 #include "citadel.h"
  43 #include "server.h"
  44 #include "citserver.h"
  45 #include "support.h"
  46 #include "config.h"
  47 #include "ctdl_module.h"
  48 #include "clientsocket.h"
  49 #include "msgbase.h"
  50 #include "domain.h"
  51 #include "internet_addressing.h"
  52 #include "citadel_dirs.h"
  53 #include "modules/smtp/smtp_util.h"
  54
  55 struct smtpmsgsrc {             // Data passed in and out of libcurl for message upload
  56         StrBuf *TheMessage;
  57         int bytes_total;
  58         int bytes_sent;
  59 };
  60
  61 struct CitContext smtp_client_CC;
  62 static int doing_smtpclient = 0;
  63 long *smtpq = NULL;             // array of msgnums containing queue instructions
  64 int smtpq_count = 0;            // number of queue messages in smtpq
  65 int smtpq_alloc = 0;            // current allocation size for smtpq
  66
  67
  68 /*
  69  * Initialize the SMTP outbound queue
  70  */
  71 void smtp_init_spoolout(void) {
  72         struct ctdlroom qrbuf;
  73
  74         /*
  75          * Create the room.  This will silently fail if the room already
  76          * exists, and that's perfectly ok, because we want it to exist.
  77          */
  78         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
  79
  80         /*
  81          * Make sure it's set to be a "system room" so it doesn't show up
  82          * in the <K>nown rooms list for administrators.
  83          */
  84         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
  85                 qrbuf.QRflags2 |= QR2_SYSTEM;
  86                 CtdlPutRoomLock(&qrbuf);
  87         }
  88 }
  89
  90
  91 /* For internet mail, generate delivery instructions.
  92  * Yes, this is recursive.  Deal with it.  Infinite recursion does
  93  * not happen because the delivery instructions message does not
  94  * contain a recipient.
  95  */
  96 int smtp_aftersave(struct CtdlMessage *msg, recptypes *recps)
  97 {
  98         if ((recps != NULL) && (recps->num_internet > 0)) {
  99                 struct CtdlMessage *imsg = NULL;
 100                 char recipient[SIZ];
 101                 StrBuf *SpoolMsg = NewStrBuf();
 102                 long nTokens;
 103                 int i;
 104
 105                 syslog(LOG_DEBUG, "smtpclient: generating delivery instructions");
 106
 107                 StrBufPrintf(SpoolMsg,
 108                              "Content-type: "SPOOLMIME"\n"
 109                              "\n"
 110                              "msgid|%s\n"
 111                              "submitted|%ld\n"
 112                              "bounceto|%s\n",
 113                              msg->cm_fields[eVltMsgNum],
 114                              (long)time(NULL),
 115                              recps->bounce_to
 116                 );
 117
 118                 if (recps->envelope_from != NULL) {
 119                         StrBufAppendBufPlain(SpoolMsg, HKEY("envelope_from|"), 0);
 120                         StrBufAppendBufPlain(SpoolMsg, recps->envelope_from, -1, 0);
 121                         StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
 122                 }
 123                 if (recps->sending_room != NULL) {
 124                         StrBufAppendBufPlain(SpoolMsg, HKEY("source_room|"), 0);
 125                         StrBufAppendBufPlain(SpoolMsg, recps->sending_room, -1, 0);
 126                         StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
 127                 }
 128
 129                 nTokens = num_tokens(recps->recp_internet, '|');
 130                 for (i = 0; i < nTokens; i++) {
 131                         long len;
 132                         len = extract_token(recipient, recps->recp_internet, i, '|', sizeof recipient);
 133                         if (len > 0) {
 134                                 StrBufAppendBufPlain(SpoolMsg, HKEY("remote|"), 0);
 135                                 StrBufAppendBufPlain(SpoolMsg, recipient, len, 0);
 136                                 StrBufAppendBufPlain(SpoolMsg, HKEY("|0||\n"), 0);
 137                         }
 138                 }
 139
 140                 imsg = malloc(sizeof(struct CtdlMessage));
 141                 memset(imsg, 0, sizeof(struct CtdlMessage));
 142                 imsg->cm_magic = CTDLMESSAGE_MAGIC;
 143                 imsg->cm_anon_type = MES_NORMAL;
 144                 imsg->cm_format_type = FMT_RFC822;
 145                 CM_SetField(imsg, eMsgSubject, HKEY("QMSG"));
 146                 CM_SetField(imsg, eAuthor, HKEY("Citadel"));
 147                 CM_SetField(imsg, eJournal, HKEY("do not journal"));
 148                 CM_SetAsFieldSB(imsg, eMesageText, &SpoolMsg);
 149                 CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM, QP_EADDR);
 150                 CM_Free(imsg);
 151         }
 152         return 0;
 153 }
 154
 155
 156 /*
 157  * Callback for smtp_attempt_delivery() to supply libcurl with upload data.
 158  */
 159 static size_t upload_source(void *ptr, size_t size, size_t nmemb, void *userp)
 160 {
 161         struct smtpmsgsrc *s = (struct smtpmsgsrc *) userp;
 162         int sendbytes = 0;
 163         const char *send_this = NULL;
 164
 165         sendbytes = (size * nmemb);
 166
 167         if (s->bytes_sent >= s->bytes_total) {
 168                 return(0);                                      // no data remaining; we are done
 169         }
 170
 171         if (sendbytes > (s->bytes_total - s->bytes_sent)) {
 172                 sendbytes = s->bytes_total - s->bytes_sent;     // can't send more than we have
 173         }
 174
 175         send_this = ChrPtr(s->TheMessage);
 176         send_this += s->bytes_sent;                             // start where we last left off
 177
 178         memcpy(ptr, send_this, sendbytes);
 179         s->bytes_sent += sendbytes;
 180         return(sendbytes);                                      // return the number of bytes _actually_ copied
 181 }
 182
 183
 184 /*
 185  * Attempt a delivery to one recipient.
 186  * Returns a three-digit SMTP status code.
 187  */
 188 int smtp_attempt_delivery(long msgid, char *recp, char *envelope_from)
 189 {
 190         struct smtpmsgsrc s;
 191         char *fromaddr = NULL;
 192         CURL *curl;
 193         CURLcode res = CURLE_OK;
 194         struct curl_slist *recipients = NULL;
 195         long response_code = 421;
 196         int num_mx = 0;
 197         char mxes[SIZ];
 198         char user[1024];
 199         char node[1024];
 200         char name[1024];
 201         char try_this_mx[256];
 202         char smtp_url[512];
 203         int i;
 204
 205         syslog(LOG_DEBUG, "smtpclient: smtp_attempt_delivery(%ld, %s)", msgid, recp);
 206
 207         process_rfc822_addr(recp, user, node, name);            // split recipient address into username, hostname, displayname
 208         num_mx = getmx(mxes, node);
 209         if (num_mx < 1) {
 210                 return(421);
 211         }
 212
 213         CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
 214         CtdlOutputMsg(msgid, MT_RFC822, HEADERS_ALL, 0, 1, NULL, 0, NULL, &fromaddr, NULL);
 215         s.TheMessage = CC->redirect_buffer;
 216         s.bytes_total =  StrLength(CC->redirect_buffer);
 217         s.bytes_sent = 0;
 218         CC->redirect_buffer = NULL;
 219         response_code = 421;
 220
 221         for (i=0; ((i<num_mx)&&((response_code/100)==4)); ++i) {                                // keep trying MXes until one works or we run out
 222                 response_code = 421;                                                            // default 421 makes non-protocol errors transient
 223                 s.bytes_sent = 0;                                                               // rewind our buffer in case we try multiple MXes
 224
 225                 curl = curl_easy_init();
 226                 if (curl) {
 227
 228                         if (!IsEmptyStr(envelope_from)) {
 229                                 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, envelope_from);
 230                         }
 231                         else {
 232                                 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, fromaddr);
 233                         }
 234
 235                         recipients = curl_slist_append(recipients, recp);
 236                         curl_easy_setopt(curl, CURLOPT_MAIL_RCPT, recipients);
 237                         curl_easy_setopt(curl, CURLOPT_READFUNCTION, upload_source);
 238                         curl_easy_setopt(curl, CURLOPT_READDATA, &s);
 239                         curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);                                              // tell libcurl we are uploading
 240                         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L);                                           // Time out after 20 seconds
 241                         if (CtdlGetConfigInt("c_smtpclient_try_starttls") != 0) {
 242                                 curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY);                        // Attempt STARTTLS if offered
 243                         }
 244                         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
 245                         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
 246                         // curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_error_buffer);
 247
 248                         // Construct an SMTP URL in the form of:
 249                         //      smtp[s]://target_host/source_host
 250                         // This looks weird but libcurl uses that last part to set our name for EHLO or HELO.
 251                         // We check for "smtp://" and "smtps://" because the admin may have put those prefixes in a smart-host entry.
 252                         // If there is no prefix we add "smtp://"
 253                         extract_token(try_this_mx, mxes, i, '|', (sizeof try_this_mx - 7));
 254                         snprintf(smtp_url, sizeof smtp_url,
 255                                 "%s%s/%s",
 256                                 (((!strncasecmp(try_this_mx, HKEY("smtp://"))) || (!strncasecmp(try_this_mx, HKEY("smtps://")))) ? "" : "smtp://"),
 257                                 try_this_mx,
 258                                 CtdlGetConfigStr("c_fqdn")
 259                         );
 260                         curl_easy_setopt(curl, CURLOPT_URL, smtp_url);
 261                         syslog(LOG_DEBUG, "smtpclient: trying %s", smtp_url);                           // send the message
 262                         res = curl_easy_perform(curl);
 263                         curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
 264                         syslog(LOG_DEBUG, "smtpclient: libcurl returned %d (%s) , SMTP response %ld",
 265                                 res, curl_easy_strerror(res), response_code
 266                         );
 267
 268                         if ((res != CURLE_OK) && (response_code == 0)) {                                // check for errors
 269                                 response_code = 421;
 270                         }
 271
 272                         curl_slist_free_all(recipients);
 273                         recipients = NULL;                                                              // this gets reused; avoid double-free
 274                         curl_easy_cleanup(curl);
 275                         curl = NULL;                                                                    // this gets reused; avoid double-free
 276                 }
 277         }
 278
 279         FreeStrBuf(&s.TheMessage);
 280         if (fromaddr) free(fromaddr);
 281         return((int)response_code);
 282 }
 283
 284
 285 /*
 286  * Process one outbound message.
 287  */
 288 void smtp_process_one_msg(long qmsgnum)
 289 {
 290         struct CtdlMessage *msg = NULL;
 291         char *instr = NULL;
 292         int i;
 293         int num_success = 0;
 294         int num_fail = 0;
 295         int num_delayed = 0;
 296         long deletes[2];
 297         int delete_this_queue = 0;
 298
 299         msg = CtdlFetchMessage(qmsgnum, 1, 1);
 300         if (msg == NULL) {
 301                 syslog(LOG_WARNING, "smtpclient: %ld does not exist", qmsgnum);
 302                 return;
 303         }
 304
 305         instr = msg->cm_fields[eMesageText];
 306         msg->cm_fields[eMesageText] = NULL;
 307         CM_Free(msg);
 308
 309         // if the queue message has any CRLF's convert them to LF's
 310         char *crlf = NULL;
 311         while (crlf = strstr(instr, "\r\n"), crlf != NULL) {
 312                 strcpy(crlf, crlf+1);
 313         }
 314
 315         // Strip out the headers and we are now left with just the instructions.
 316         char *soi = strstr(instr, "\n\n");
 317         if (soi) {
 318                 strcpy(instr, soi+2);
 319         }
 320
 321         long msgid = 0;
 322         time_t submitted = time(NULL);
 323         time_t attempted = 0;
 324         char *bounceto = NULL;
 325         char *envelope_from = NULL;
 326
 327         char cfgline[SIZ];
 328         for (i=0; i<num_tokens(instr, '\n'); ++i) {
 329                 extract_token(cfgline, instr, i, '\n', sizeof cfgline);
 330                 if (!strncasecmp(cfgline, HKEY("msgid|")))              msgid = atol(&cfgline[6]);
 331                 if (!strncasecmp(cfgline, HKEY("submitted|")))          submitted = atol(&cfgline[10]);
 332                 if (!strncasecmp(cfgline, HKEY("attempted|")))          attempted = atol(&cfgline[10]);
 333                 if (!strncasecmp(cfgline, HKEY("bounceto|")))           bounceto = strdup(&cfgline[9]);
 334                 if (!strncasecmp(cfgline, HKEY("envelope_from|")))      envelope_from = strdup(&cfgline[14]);
 335         }
 336
 337         int should_try_now = 0;
 338         if (attempted < submitted) {                            // If no attempts have been made yet, try now
 339                 should_try_now = 1;
 340         }
 341         else if ((attempted - submitted) <= 14400) {
 342                 if ((time(NULL) - attempted) > 1800) {          // First four hours, retry every 30 minutes
 343                         should_try_now = 1;
 344                 }
 345         }
 346         else {
 347                 if ((time(NULL) - attempted) > 14400) {         // After that, retry once every 4 hours
 348                         should_try_now = 1;
 349                 }
 350         }
 351
 352         if (should_try_now) {
 353                 syslog(LOG_DEBUG, "smtpclient: %ld attempting delivery now", qmsgnum);
 354                 StrBuf *NewInstr = NewStrBuf();
 355                 StrBufAppendPrintf(NewInstr, "Content-type: "SPOOLMIME"\n\n");
 356                 StrBufAppendPrintf(NewInstr, "msgid|%ld\n", msgid);
 357                 StrBufAppendPrintf(NewInstr, "submitted|%ld\n", submitted);
 358                 if (bounceto) StrBufAppendPrintf(NewInstr, "bounceto|%s\n", bounceto);
 359                 if (envelope_from) StrBufAppendPrintf(NewInstr, "envelope_from|%s\n", envelope_from);
 360
 361                 for (i=0; i<num_tokens(instr, '\n'); ++i) {
 362                         extract_token(cfgline, instr, i, '\n', sizeof cfgline);
 363                         if (!strncasecmp(cfgline, HKEY("remote|"))) {
 364                                 char recp[SIZ];
 365                                 int previous_result = extract_int(cfgline, 2);
 366                                 if ((previous_result == 0) || (previous_result == 4)) {
 367                                         int new_result = 421;
 368                                         extract_token(recp, cfgline, 1, '|', sizeof recp);
 369                                         new_result = smtp_attempt_delivery(msgid, recp, envelope_from);
 370                                         syslog(LOG_DEBUG, "smtpclient: recp: <%s> , result: %d (%s)", recp, new_result, smtpstatus(new_result));
 371                                         if ((new_result / 100) == 2) {
 372                                                 ++num_success;
 373                                         }
 374                                         else {
 375                                                 if ((new_result / 100) == 5) {
 376                                                         ++num_fail;
 377                                                 }
 378                                                 else {
 379                                                         ++num_delayed;
 380                                                 }
 381                                                 StrBufAppendPrintf(NewInstr, "remote|%s|%ld|%ld (%s)\n",
 382                                                         recp, (new_result / 100) , new_result, smtpstatus(new_result)
 383                                                 );
 384                                         }
 385                                 }
 386                         }
 387                 }
 388
 389                 StrBufAppendPrintf(NewInstr, "attempted|%ld\n", time(NULL));
 390
 391                 // All deliveries have now been attempted.  Now determine the disposition of this queue entry.
 392
 393                 time_t age = time(NULL) - submitted;
 394                 syslog(LOG_DEBUG, "smtpclient: submission age: %ldd%ldh%ldm%lds", (age/86400) , ((age%86400)/3600) , ((age%3600)/60) , (age%60));
 395                 syslog(LOG_DEBUG, "smtpclient: num_success=%d , num_fail=%d , num_delayed=%d", num_success, num_fail, num_delayed);
 396
 397                 // If there are permanent fails on this attempt, deliver a bounce to the user.
 398                 // The 5XX fails will be recorded in the rewritten queue, but they will be removed before the next attempt.
 399                 if (num_fail > 0) {
 400                         smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_FATALS);
 401                 }
 402
 403                 // If all deliveries have either succeeded or failed, we are finished with this queue entry.
 404                 //
 405                 if (num_delayed == 0) {
 406                         delete_this_queue = 1;
 407                 }
 408
 409                 // If it's been more than five days, give up and tell the sender that delivery failed
 410                 //
 411                 else if ((time(NULL) - submitted) > SMTP_DELIVER_FAIL) {
 412                         smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_ALL);
 413                         delete_this_queue = 1;
 414                 }
 415
 416                 // If it's been more than four hours but less than five days, warn the sender that delivery is delayed
 417                 //
 418                 else if ( ((attempted - submitted) < SMTP_DELIVER_WARN) && ((time(NULL) - submitted) >= SMTP_DELIVER_WARN) ) {
 419                         smtp_do_bounce(ChrPtr(NewInstr), SDB_WARN);
 420                 }
 421
 422                 if (delete_this_queue) {
 423                         syslog(LOG_DEBUG, "smtpclient: %ld deleting", qmsgnum);
 424                         deletes[0] = qmsgnum;
 425                         deletes[1] = msgid;
 426                         CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, deletes, 2, "");
 427                         FreeStrBuf(&NewInstr);                                          // We have to free NewInstr here, no longer needed
 428                 }
 429                 else {
 430                         // replace the old queue entry with the new one
 431                         syslog(LOG_DEBUG, "smtpclient: %ld rewriting", qmsgnum);
 432                         msg = convert_internet_message_buf(&NewInstr);                  // This function will free NewInstr for us
 433                         CtdlSubmitMsg(msg, NULL, SMTP_SPOOLOUT_ROOM, 0);
 434                         CM_Free(msg);
 435                         CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, &qmsgnum, 1, "");
 436                 }
 437         }
 438         else {
 439                 syslog(LOG_DEBUG, "smtpclient: %ld retry time not reached", qmsgnum);
 440         }
 441
 442         if (bounceto != NULL)           free(bounceto);
 443         if (envelope_from != NULL)      free(envelope_from);
 444         free(instr);
 445 }
 446
 447
 448 /*
 449  * Callback for smtp_do_queue()
 450  */
 451 void smtp_add_msg(long msgnum, void *userdata) {
 452
 453         if (smtpq == NULL) {
 454                 smtpq_count = 0;
 455                 smtpq_alloc = 100;
 456                 smtpq = malloc(smtpq_alloc * sizeof(long));
 457         }
 458
 459         if (smtpq_alloc >= smtpq_count) {
 460                 smtpq_alloc += 100;
 461                 smtpq = realloc(smtpq, (smtpq_alloc * sizeof(long)));
 462         }
 463
 464         smtpq[smtpq_count++] = msgnum;
 465 }
 466
 467
 468 /*
 469  * Run through the queue sending out messages.
 470  */
 471 void smtp_do_queue(void) {
 472         int i = 0;
 473
 474         /*
 475          * This is a simple concurrency check to make sure only one smtpclient
 476          * run is done at a time.  We could do this with a mutex, but since we
 477          * don't really require extremely fine granularity here, we'll do it
 478          * with a static variable instead.
 479          */
 480         if (doing_smtpclient) return;
 481         doing_smtpclient = 1;
 482
 483         syslog(LOG_DEBUG, "smtpclient: start queue run");
 484         pthread_setspecific(MyConKey, (void *)&smtp_client_CC);
 485
 486         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
 487                 syslog(LOG_WARNING, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
 488                 doing_smtpclient = 0;
 489                 return;
 490         }
 491
 492         // Put the queue in memory so we can close the db cursor
 493         CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_add_msg, NULL);
 494
 495         // We are ready to run through the queue now.
 496         for (i=0; i<smtpq_count; ++i) {
 497                 smtp_process_one_msg(smtpq[i]);
 498         }
 499
 500         smtpq_count = 0;                // don't free it, we will use this memory on the next run
 501         doing_smtpclient = 0;
 502         syslog(LOG_DEBUG, "smtpclient: end queue run");
 503 }
 504
 505
 506 /*
 507  * Module entry point
 508  */
 509 CTDL_MODULE_INIT(smtpclient)
 510 {
 511         if (!threading)
 512         {
 513                 CtdlFillSystemContext(&smtp_client_CC, "SMTP_Send");
 514                 CtdlRegisterMessageHook(smtp_aftersave, EVT_AFTERSAVE);
 515                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER, PRIO_AGGR + 50);
 516                 smtp_init_spoolout();
 517         }
 518
 519         /* return our module id for the log */
 520         return "smtpclient";
 521 }