citadel/server/modules/smtp/serv_smtpclient.c

   1 // Transmit outbound SMTP mail to the big wide world of the Internet
   2 //
   3 // This is the new, exciting, clever version that makes libcurl do all the work  :)
   4 //
   5 // Copyright (c) 1997-2024 by the citadel.org team
   6 //
   7 // This program is open source software.  Use, duplication, or disclosure
   8 // is subject to the terms of the GNU General Public License, version 3.
   9
  10 #include <stdlib.h>
  11 #include <unistd.h>
  12 #include <stdio.h>
  13 #include <time.h>
  14 #include <ctype.h>
  15 #include <string.h>
  16 #include <errno.h>
  17 #include <sys/types.h>
  18 #include <sys/stat.h>
  19 #include <libcitadel.h>
  20 #include <curl/curl.h>
  21 #include "../../sysconfig.h"
  22 #include "../../citadel_defs.h"
  23 #include "../../server.h"
  24 #include "../../citserver.h"
  25 #include "../../support.h"
  26 #include "../../config.h"
  27 #include "../../ctdl_module.h"
  28 #include "../../clientsocket.h"
  29 #include "../../msgbase.h"
  30 #include "../../domain.h"
  31 #include "../../internet_addressing.h"
  32 #include "../../citadel_dirs.h"
  33 #include "../smtp/smtp_util.h"
  34
  35 long last_queue_job_submitted = 0;
  36 long last_queue_job_processed = 0;
  37
  38 struct smtpmsgsrc {             // Data passed in and out of libcurl for message upload
  39         StrBuf *TheMessage;
  40         int bytes_total;
  41         int bytes_sent;
  42 };
  43
  44 // Initialize the SMTP outbound queue
  45 void smtp_init_spoolout(void) {
  46         struct ctdlroom qrbuf;
  47
  48         // Create the room.  This will silently fail if the room already
  49         // exists, and that's perfectly ok, because we want it to exist.
  50         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
  51
  52         // Make sure it's set to be a "system room" so it doesn't show up
  53         // in the <K>nown rooms list for administrators.
  54         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
  55                 qrbuf.QRflags2 |= QR2_SYSTEM;
  56                 CtdlPutRoomLock(&qrbuf);
  57         }
  58 }
  59
  60
  61 // For internet mail, generate a delivery job.
  62 // Yes, this is recursive.  Deal with it.  Infinite recursion does
  63 // not happen because the message containing the delivery job does not
  64 // have a recipient.
  65 int smtp_aftersave(struct CtdlMessage *msg, struct recptypes *recps) {
  66         if ((recps != NULL) && (recps->num_internet > 0)) {
  67                 struct CtdlMessage *imsg = NULL;
  68                 char recipient[SIZ];
  69                 StrBuf *SpoolMsg = NewStrBuf();
  70                 long nTokens;
  71                 int i;
  72
  73                 syslog(LOG_DEBUG, "smtpclient: generating delivery job");
  74
  75                 StrBufPrintf(SpoolMsg,
  76                              "Content-type: " SPOOLMIME "\n"
  77                              "\n"
  78                              "msgid|%s\n"
  79                              "submitted|%ld\n" "bounceto|%s\n", msg->cm_fields[eVltMsgNum], (long) time(NULL), recps->bounce_to);
  80
  81                 if (recps->envelope_from != NULL) {
  82                         StrBufAppendBufPlain(SpoolMsg, HKEY("envelope_from|"), 0);
  83                         StrBufAppendBufPlain(SpoolMsg, recps->envelope_from, -1, 0);
  84                         StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
  85                 }
  86                 if (recps->sending_room != NULL) {
  87                         StrBufAppendBufPlain(SpoolMsg, HKEY("source_room|"), 0);
  88                         StrBufAppendBufPlain(SpoolMsg, recps->sending_room, -1, 0);
  89                         StrBufAppendBufPlain(SpoolMsg, HKEY("\n"), 0);
  90                 }
  91
  92                 nTokens = num_tokens(recps->recp_internet, '|');
  93                 for (i = 0; i < nTokens; i++) {
  94                         long len;
  95                         len = extract_token(recipient, recps->recp_internet, i, '|', sizeof recipient);
  96                         if (len > 0) {
  97                                 StrBufAppendBufPlain(SpoolMsg, HKEY("remote|"), 0);
  98                                 StrBufAppendBufPlain(SpoolMsg, recipient, len, 0);
  99                                 StrBufAppendBufPlain(SpoolMsg, HKEY("|0||\n"), 0);
 100                         }
 101                 }
 102
 103                 imsg = malloc(sizeof(struct CtdlMessage));
 104                 memset(imsg, 0, sizeof(struct CtdlMessage));
 105                 imsg->cm_magic = CTDLMESSAGE_MAGIC;
 106                 imsg->cm_anon_type = MES_NORMAL;
 107                 imsg->cm_format_type = FMT_RFC822;
 108                 CM_SetField(imsg, eMsgSubject, "QMSG");
 109                 CM_SetField(imsg, eAuthor, "Citadel");
 110                 CM_SetField(imsg, eJournal, "do not journal");
 111                 CM_SetAsFieldSB(imsg, eMessageText, &SpoolMsg);
 112                 last_queue_job_submitted = CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM);
 113                 CM_Free(imsg);
 114         }
 115         return 0;
 116 }
 117
 118
 119 // Callback for smtp_attempt_delivery() to supply libcurl with upload data.
 120 static size_t upload_source(void *ptr, size_t size, size_t nmemb, void *userp) {
 121         struct smtpmsgsrc *s = (struct smtpmsgsrc *) userp;
 122         int sendbytes = 0;
 123         const char *send_this = NULL;
 124
 125         sendbytes = (size * nmemb);
 126
 127         if (s->bytes_sent >= s->bytes_total) {
 128                 return (0);                                     // no data remaining; we are done
 129         }
 130
 131         if (sendbytes > (s->bytes_total - s->bytes_sent)) {
 132                 sendbytes = s->bytes_total - s->bytes_sent;     // can't send more than we have
 133         }
 134
 135         send_this = ChrPtr(s->TheMessage);
 136         send_this += s->bytes_sent;                             // start where we last left off
 137
 138         memcpy(ptr, send_this, sendbytes);
 139         s->bytes_sent += sendbytes;
 140         return(sendbytes);                                      // return the number of bytes _actually_ copied
 141 }
 142
 143
 144 // The libcurl API doesn't provide a way to capture the actual SMTP result message returned
 145 // by the remote server.  This is an ugly way to extract it, by capturing debug data from
 146 // the library and filtering on the lines we want.
 147 int ctdl_libcurl_smtp_debug_callback(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr) {
 148         if (type != CURLINFO_HEADER_IN)
 149                 return 0;
 150         if (!userptr)
 151                 return 0;
 152         char *debugbuf = (char *) userptr;
 153
 154         int len = strlen(debugbuf);
 155         if (len + size > SIZ)
 156                 return 0;
 157
 158         memcpy(&debugbuf[len], data, size);
 159         debugbuf[len + size] = 0;
 160         return 0;
 161 }
 162
 163
 164 // Go through the debug output of an SMTP transaction, and boil it down to just the final success or error response message.
 165 void trim_response(long response_code, char *response) {
 166         if ((response_code < 100) || (response_code > 999) || (IsEmptyStr(response))) {
 167                 return;
 168         }
 169
 170         char *p;
 171         for (p = response; *p != 0; ++p) {
 172                 if ( (*p != '\n') && (!isprint(*p)) ) {         // expunge any nonprintables except for newlines
 173                         *p = ' ';
 174                 }
 175         }
 176
 177         char response_code_str[4];
 178         snprintf(response_code_str, sizeof response_code_str, "%ld", response_code);
 179         char *respstart = strstr(response, response_code_str);
 180         if (respstart == NULL) {                                // If we have a response code but no response text,
 181                 strcpy(response, smtpstatus(response_code));    // use one of our canned messages.
 182                 return;
 183         }
 184         strcpy(response, respstart);
 185
 186         p = strstr(response, "\n");
 187         if (p != NULL) {
 188                 *p = 0;
 189         }
 190 }
 191
 192
 193 // Attempt a delivery to one recipient.
 194 // Returns a three-digit SMTP status code.
 195 int smtp_attempt_delivery(long msgid, char *recp, char *envelope_from, char *source_room, char *response) {
 196         struct smtpmsgsrc s;
 197         char *fromaddr = NULL;
 198         CURL *curl;
 199         CURLcode res = CURLE_OK;
 200         struct curl_slist *recipients = NULL;
 201         long response_code = 421;
 202         int num_mx = 0;
 203         char mxes[SIZ];
 204         char user[1024];
 205         char node[1024];
 206         char name[1024];
 207         char try_this_mx[256];
 208         char smtp_url[512];
 209         int i;
 210
 211         syslog(LOG_DEBUG, "smtpclient: smtp_attempt_delivery(%ld, %s)", msgid, recp);
 212
 213         process_rfc822_addr(recp, user, node, name);    // split recipient address into username, hostname, displayname
 214         num_mx = getmx(mxes, node);
 215         if (num_mx < 1) {
 216                 return(421);
 217         }
 218
 219         CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
 220
 221         // If we have a source room, it's probably a mailing list message; generate an unsubscribe header
 222         if (!IsEmptyStr(source_room)) {
 223                 char base_url[SIZ];
 224                 char unsubscribe_url[SIZ];
 225                 snprintf(base_url, sizeof base_url, "https://%s/listsub", CtdlGetConfigStr("c_fqdn"));
 226                 generate_one_click_url(unsubscribe_url, base_url, "unsubscribe", source_room, recp);
 227                 cprintf("List-Unsubscribe: %s\r\n", unsubscribe_url);
 228                 cprintf("List-Unsubscribe-Post: List-Unsubscribe=One-Click\r\n");       // RFC 8058
 229
 230         }
 231
 232         CtdlOutputMsg(msgid, MT_RFC822, HEADERS_ALL, 0, 1, NULL, 0, NULL, &fromaddr, NULL);
 233         s.TheMessage = CC->redirect_buffer;
 234         CC->redirect_buffer = NULL;
 235
 236 #if 0
 237         // FIXME genericize this
 238         char *pkey_in =
 239                 "-----BEGIN PRIVATE KEY-----\n"
 240                 "MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDfuefcepokRrnp\n"
 241                 "SSDsxu+QDqeD8GL9QnZz/N6IxTdBv6Wc10ExBe2IjS5dKI7AvhSSEK0zGE8Hkpmw\n"
 242                 "eccbiepQqeueteWzAMZ1uT43bD3k7eye7vWobiOP9QtoYGR6sG25h2W5Tbc91W4f\n"
 243                 "dvYnxYVJjx8wIVF0f3o25v+rQueoo0HlvGyA9/xi9GAaJL05OmK1xnMJgSvW/Q8Q\n"
 244                 "zq7apf1D6XPXHuhv5tevElkZ5jlvM2w0cTVyAzMrUh6Rkcn9xM4/NPWYghBc3jO4\n"
 245                 "TrPnSrobQGrX0fcizE/FN6I0in0Ke8Z+gMM8NeFcsjvLZe9MpY9i0pw/ygLIh5t3\n"
 246                 "O4qpwC1JAgMBAAECggEAIwiTCMEAGzciDKhhagJ66BWLYMtHTP5X2zDZThSH4xlW\n"
 247                 "HznL4RfbCtuEy5y6we7h/L90x8ACPB7WRz7CkYrmsMvy9A7q0b2I1k10MyyVgqBJ\n"
 248                 "QdgMitv4YKYQK7+QbG/tNrS/lqVXUOz3iiDQSgkRpqOtUBWfkj0WD7vbhF99NDhV\n"
 249                 "dxaehFkKv3yNy0bXJlHJBJ6KtOUnDwub8TExh8dyj3kB8Qzj4I98shaXPNUSSaOw\n"
 250                 "zG6QG72yrxlMs495jkIPbF2JDidmLrX+oVISwKyaBWx+BkFV/KFAEKgaB5/nCw7+\n"
 251                 "qq/jxsmXim3HuQ3MIAjq1yw9aGRH1HMi8Gn7tYlNGwKBgQDy6EEKpuEiW9wwlI2+\n"
 252                 "GVuSkhSTTX1h6qK/ay8Jtyb8yJM/BxogAQlfjdgFixiZHy5MaomTbfeT2GDji553\n"
 253                 "+RsnZ60+g7FI9nHwabSxtuCQ+vjbFqCsdMPAiSeG0bEzo0zf5TjASdUtuZL0vXjl\n"
 254                 "yMZWDEuESoVNlYlvCOVkw2nvIwKBgQDryPuSq6PNVHRWsKRRs5ju4wKs/1ucBOg5\n"
 255                 "gCcN8lE03mFCWAlZhypE4/fAhTQ/a5KQoAzc0QZcXRueDyNsnc+QWw3/QWf8/fkV\n"
 256                 "HPfTWS3Dcuj+4RnWUucaZ/mKFlTC3+eNSlpyaPIMlCjXGsJ9GlPrsaAi9KPbD2v/\n"
 257                 "XcMq/PMOowKBgHVf7S3sfZVQthFzdxqIvksQ84hKRW/vJT1B2bTkH56+fQhTsjgM\n"
 258                 "yC64J85l7DjxbDnYsSngVWXHhOnvKV/nq0tbOcefcydCjsQREBNfvxvPajjTskgj\n"
 259                 "FAQRQlxPL0U4f4khBk9EXhJ+PZithaHjZpNl1YfTSp62x3Yz4kTSeHnpAoGAGn5m\n"
 260                 "5kArE7NdrzACBrwrfww7DL1Uyd8zSOLBgKutvEcQnqfNxSWO9la3TAarrESmH2Ic\n"
 261                 "j+Nc15wOsl/5FwdUf1/73qa2zJKtHlY28qSeo8uRqrIYeSCvnyP3wjBoLc2C8zlb\n"
 262                 "mGd6azdqr2DuYahHrcAzwjnC/6Zn+DXM7FOn7AkCgYBp1xxY88cCoF24yffkD3MC\n"
 263                 "ACUury4qRSDTGx6/qCCkIyWxg1vuiDrlPWhSwQznxHvovcfpdjdbWcFY87IK6mpG\n"
 264                 "aJHwMJ7Kw+baoxGPZWHwdg6BgvUCihe3xlcaq6rOBoLviD6FOzbogg++Tvi0LemG\n"
 265                 "y/wEs/mZkaRzW4n41ir0Xw==\n"
 266                 "-----END PRIVATE KEY-----\n"
 267         ;
 268         dkim_sign(s.TheMessage, pkey_in, "dev.citadel.org", "foo");
 269 #endif
 270
 271         // Prepare the buffer for transmittal
 272         s.bytes_total = StrLength(s.TheMessage);
 273         s.bytes_sent = 0;
 274         response_code = 421;
 275
 276
 277         // Keep trying MXes until one works or we run out.
 278         for (i = 0; ((i < num_mx) && ((response_code / 100) == 4)); ++i) {
 279                 response_code = 421;    // default 421 makes non-protocol errors transient
 280                 s.bytes_sent = 0;       // rewind our buffer in case we try multiple MXes
 281
 282                 curl = curl_easy_init();
 283                 if (curl) {
 284                         response[0] = 0;
 285
 286                         if (!IsEmptyStr(envelope_from)) {
 287                                 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, envelope_from);
 288                         }
 289                         else {
 290                                 curl_easy_setopt(curl, CURLOPT_MAIL_FROM, fromaddr);
 291                         }
 292
 293                         recipients = curl_slist_append(recipients, recp);
 294                         curl_easy_setopt(curl, CURLOPT_MAIL_RCPT, recipients);
 295                         curl_easy_setopt(curl, CURLOPT_READFUNCTION, upload_source);
 296                         curl_easy_setopt(curl, CURLOPT_READDATA, &s);
 297                         curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);                              // tell libcurl we are uploading
 298                         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L);                           // Time out after 20 seconds
 299                         if (CtdlGetConfigInt("c_smtpclient_disable_starttls") == 0) {
 300                                 curl_easy_setopt(curl, CURLOPT_USE_SSL, CURLUSESSL_TRY);        // Attempt STARTTLS if offered
 301                         }
 302                         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
 303                         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
 304                         curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, ctdl_libcurl_smtp_debug_callback);
 305                         curl_easy_setopt(curl, CURLOPT_DEBUGDATA, (void *) response);
 306                         curl_easy_setopt(curl, CURLOPT_VERBOSE, 1L);
 307
 308                         // Construct an SMTP URL in the form of:
 309                         //      smtp[s]://target_host/source_host
 310                         // This looks weird but libcurl uses that last part to set our name for EHLO or HELO.
 311                         // We check for "smtp://" and "smtps://" because an admin may have put those prefixes in a smart-host entry
 312                         // If there is no prefix we add "smtp://"
 313                         extract_token(try_this_mx, mxes, i, '|', (sizeof try_this_mx - 7));
 314                         snprintf(smtp_url, sizeof smtp_url,
 315                                 "%s%s/%s",
 316                                 (((!strncasecmp(try_this_mx, HKEY("smtp://")))
 317                                 || (!strncasecmp(try_this_mx, HKEY("smtps://")))) ? "" : "smtp://"),
 318                                 try_this_mx, CtdlGetConfigStr("c_fqdn")
 319                         );
 320                         curl_easy_setopt(curl, CURLOPT_URL, smtp_url);
 321                         syslog(LOG_DEBUG, "smtpclient: trying MX %d of %d <%s>", i+1, num_mx, smtp_url);        // send the message
 322                         res = curl_easy_perform(curl);
 323                         curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code);
 324                         syslog(LOG_DEBUG,
 325                                "smtpclient: libcurl returned %d (%s) , SMTP response %ld",
 326                                res, curl_easy_strerror(res), response_code
 327                         );
 328
 329                         if ((res != CURLE_OK) && (response_code == 0)) {        // check for errors
 330                                 response_code = 421;
 331                         }
 332
 333                         curl_slist_free_all(recipients);
 334                         recipients = NULL;                                      // this gets reused; avoid double-free
 335                         curl_easy_cleanup(curl);
 336                         curl = NULL;                                            // this gets reused; avoid double-free
 337
 338                         // Trim the error message buffer down to just the actual message
 339                         trim_response(response_code, response);
 340                 }
 341         }
 342
 343         FreeStrBuf(&s.TheMessage);
 344         if (fromaddr) {
 345                 free(fromaddr);
 346         }
 347         return ((int) response_code);
 348 }
 349
 350
 351 // Process one outbound message.
 352 void smtp_process_one_msg(long qmsgnum) {
 353         struct CtdlMessage *msg = NULL;
 354         char *instr = NULL;
 355         int i;
 356         int num_success = 0;
 357         int num_fail = 0;
 358         int num_delayed = 0;
 359         long deletes[2];
 360         int delete_this_queue = 0;
 361         char server_response[SIZ];
 362
 363         msg = CtdlFetchMessage(qmsgnum, 1);
 364         if (msg == NULL) {
 365                 syslog(LOG_WARNING, "smtpclient: msg#%ld does not exist", qmsgnum);
 366                 return;
 367         }
 368
 369         instr = msg->cm_fields[eMessageText];
 370         msg->cm_fields[eMessageText] = NULL;
 371         CM_Free(msg);
 372
 373         // if the queue job message has any CRLF's convert them to LF's
 374         char *crlf = NULL;
 375         while (crlf = strstr(instr, "\r\n"), crlf != NULL) {
 376                 strcpy(crlf, crlf + 1);
 377         }
 378
 379         // Strip out the headers and we are now left with just the instructions.
 380         char *soi = strstr(instr, "\n\n");
 381         if (soi) {
 382                 strcpy(instr, soi + 2);
 383         }
 384
 385         long msgid = 0;
 386         time_t submitted = time(NULL);
 387         time_t attempted = 0;
 388         char *bounceto = NULL;
 389         char *envelope_from = NULL;
 390         char *source_room = NULL;
 391
 392         char cfgline[SIZ];
 393         for (i = 0; i < num_tokens(instr, '\n'); ++i) {
 394                 extract_token(cfgline, instr, i, '\n', sizeof cfgline);
 395                 if (!strncasecmp(cfgline, HKEY("msgid|")))              msgid = atol(&cfgline[6]);
 396                 if (!strncasecmp(cfgline, HKEY("submitted|")))          submitted = atol(&cfgline[10]);
 397                 if (!strncasecmp(cfgline, HKEY("attempted|")))          attempted = atol(&cfgline[10]);
 398                 if (!strncasecmp(cfgline, HKEY("bounceto|")))           bounceto = strdup(&cfgline[9]);
 399                 if (!strncasecmp(cfgline, HKEY("envelope_from|")))      envelope_from = strdup(&cfgline[14]);
 400                 if (!strncasecmp(cfgline, HKEY("source_room|")))        source_room = strdup(&cfgline[12]);
 401         }
 402
 403         int should_try_now = 0;
 404         if (attempted < submitted) {                    // If no attempts have been made yet, try now
 405                 should_try_now = 1;
 406         }
 407         else if ((attempted - submitted) <= 14400) {
 408                 if ((time(NULL) - attempted) > 1800) {  // First four hours, retry every 30 minutes
 409                         should_try_now = 1;
 410                 }
 411         }
 412         else {
 413                 if ((time(NULL) - attempted) > 14400) { // After that, retry once every 4 hours
 414                         should_try_now = 1;
 415                 }
 416         }
 417
 418         if (should_try_now) {
 419                 syslog(LOG_DEBUG, "smtpclient: attempting delivery of message <%ld> now", qmsgnum);
 420                 if (source_room) {
 421                         syslog(LOG_DEBUG, "smtpclient: this message originated in <%s>", source_room);
 422                 }
 423                 StrBuf *NewInstr = NewStrBuf();
 424                 StrBufAppendPrintf(NewInstr, "Content-type: " SPOOLMIME "\n\n");
 425                 StrBufAppendPrintf(NewInstr, "msgid|%ld\n", msgid);
 426                 StrBufAppendPrintf(NewInstr, "submitted|%ld\n", submitted);
 427                 if (bounceto) {
 428                         StrBufAppendPrintf(NewInstr, "bounceto|%s\n", bounceto);
 429                 }
 430                 if (envelope_from) {
 431                         StrBufAppendPrintf(NewInstr, "envelope_from|%s\n", envelope_from);
 432                 }
 433                 for (i = 0; i < num_tokens(instr, '\n'); ++i) {
 434                         extract_token(cfgline, instr, i, '\n', sizeof cfgline);
 435                         if (!strncasecmp(cfgline, HKEY("remote|"))) {
 436                                 char recp[SIZ];
 437                                 int previous_result = extract_int(cfgline, 2);
 438                                 if ((previous_result == 0) || (previous_result == 4)) {
 439                                         int new_result = 421;
 440                                         extract_token(recp, cfgline, 1, '|', sizeof recp);
 441                                         new_result = smtp_attempt_delivery(msgid, recp, envelope_from, source_room, server_response);
 442                                         syslog(LOG_DEBUG, "smtpclient: recp: <%s> , result: %d (%s)", recp, new_result, server_response);
 443                                         if ((new_result / 100) == 2) {
 444                                                 ++num_success;
 445                                         }
 446                                         else {
 447                                                 if ((new_result / 100) == 5) {
 448                                                         ++num_fail;
 449                                                 }
 450                                                 else {
 451                                                         ++num_delayed;
 452                                                 }
 453                                                 StrBufAppendPrintf(NewInstr, "remote|%s|%ld|%ld (%s)\n", recp, (new_result / 100), new_result, server_response);
 454                                         }
 455                                 }
 456                         }
 457                 }
 458
 459                 StrBufAppendPrintf(NewInstr, "attempted|%ld\n", time(NULL));
 460
 461                 // All deliveries have now been attempted.  Now determine the disposition of this queue entry.
 462
 463                 time_t age = time(NULL) - submitted;
 464                 syslog(LOG_DEBUG,
 465                        "smtpclient: submission age: %ldd%ldh%ldm%lds",
 466                        (age / 86400), ((age % 86400) / 3600), ((age % 3600) / 60), (age % 60));
 467                 syslog(LOG_DEBUG, "smtpclient: num_success=%d , num_fail=%d , num_delayed=%d", num_success, num_fail, num_delayed);
 468
 469                 // If there are permanent fails on this attempt, deliver a bounce to the user.
 470                 // The 5XX fails will be recorded in the rewritten queue, but they will be removed before the next attempt.
 471                 if (num_fail > 0) {
 472                         smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_FATALS);
 473                 }
 474                 // If all deliveries have either succeeded or failed, we are finished with this queue entry.
 475                 if (num_delayed == 0) {
 476                         delete_this_queue = 1;
 477                 }
 478                 // If it's been more than five days, give up and tell the sender that delivery failed
 479                 else if ((time(NULL) - submitted) > SMTP_DELIVER_FAIL) {
 480                         smtp_do_bounce(ChrPtr(NewInstr), SDB_BOUNCE_ALL);
 481                         delete_this_queue = 1;
 482                 }
 483                 // If it's been more than four hours but less than five days, warn the sender that delivery is delayed
 484                 else if (((attempted - submitted) < SMTP_DELIVER_WARN) && ((time(NULL) - submitted) >= SMTP_DELIVER_WARN)) {
 485                         smtp_do_bounce(ChrPtr(NewInstr), SDB_WARN);
 486                 }
 487
 488                 if (delete_this_queue) {
 489                         syslog(LOG_DEBUG, "smtpclient: %ld deleting", qmsgnum);
 490                         deletes[0] = qmsgnum;
 491                         deletes[1] = msgid;
 492                         CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, deletes, 2, "");
 493                         FreeStrBuf(&NewInstr);  // We have to free NewInstr here, no longer needed
 494                 }
 495                 else {
 496                         // replace the old queue entry with the new one
 497                         syslog(LOG_DEBUG, "smtpclient: %ld rewriting", qmsgnum);
 498                         msg = convert_internet_message_buf(&NewInstr);  // This function will free NewInstr for us
 499                         CtdlSubmitMsg(msg, NULL, SMTP_SPOOLOUT_ROOM);
 500                         CM_Free(msg);
 501                         CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, &qmsgnum, 1, "");
 502                 }
 503         }
 504         else {
 505                 syslog(LOG_DEBUG, "smtpclient: msg#%ld retry time not reached", qmsgnum);
 506         }
 507
 508         if (bounceto != NULL) {
 509                 free(bounceto);
 510         }
 511         if (envelope_from != NULL) {
 512                 free(envelope_from);
 513         }
 514         if (source_room != NULL) {
 515                 free(source_room);
 516         }
 517         free(instr);
 518 }
 519
 520
 521 // Callback for smtp_do_queue()
 522 void smtp_add_msg(long msgnum, void *userdata) {
 523         Array *smtp_queue = (Array *) userdata;
 524         array_append(smtp_queue, &msgnum);
 525 }
 526
 527
 528 enum {
 529         FULL_QUEUE_RUN,         // try to process the entire queue, including messages that have already been attempted
 530         QUICK_QUEUE_RUN         // only process jobs in the queue that have not been tried yet
 531 };
 532
 533
 534 // Run through the queue sending out messages.
 535 void smtp_do_queue(int type_of_queue_run) {
 536         static int doing_smtpclient = 0;
 537         int i = 0;
 538
 539         // This is a concurrency check to make sure only one smtpclient run is done at a time.
 540         begin_critical_section(S_SMTPQUEUE);
 541         if (doing_smtpclient) {
 542                 end_critical_section(S_SMTPQUEUE);
 543                 return;
 544         }
 545         doing_smtpclient = 1;
 546         end_critical_section(S_SMTPQUEUE);
 547
 548         syslog(LOG_DEBUG, "smtpclient: start %s queue run , last_queue_job_processed=%ld , last_queue_job_submitted=%ld",
 549                 (type_of_queue_run == QUICK_QUEUE_RUN ? "quick" : "full"),
 550                 last_queue_job_processed, last_queue_job_submitted
 551         );
 552
 553         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
 554                 syslog(LOG_WARNING, "smtpclient: cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
 555                 doing_smtpclient = 0;
 556                 return;
 557         }
 558
 559         // This array will hold the list of queue job messages
 560         Array *smtp_queue = array_new(sizeof(long));
 561         if (smtp_queue == NULL) {
 562                 syslog(LOG_WARNING, "smtpclient: cannot allocate queue array");
 563                 doing_smtpclient = 0;
 564                 return;
 565         }
 566
 567         // Put the queue in memory so we can close the db cursor
 568         CtdlForEachMessage(
 569                 (type_of_queue_run == QUICK_QUEUE_RUN ? MSGS_GT : MSGS_ALL),            // quick = new jobs; full = all jobs
 570                 (type_of_queue_run == QUICK_QUEUE_RUN ? last_queue_job_processed : 0),  // quick = new jobs; full = all jobs
 571                 NULL,
 572                 SPOOLMIME,              // Searching for Content-type of SPOOLIME will give us only queue instruction messages
 573                 NULL,
 574                 smtp_add_msg,           // That's our callback function to add a job to the queue
 575                 (void *)smtp_queue
 576         );
 577
 578         // We are ready to run through the queue now.
 579         syslog(LOG_DEBUG, "smtpclient: %d messages to be processed", array_len(smtp_queue));
 580         for (i = 0; i < array_len(smtp_queue); ++i) {
 581                 long m;
 582                 memcpy(&m, array_get_element_at(smtp_queue, i), sizeof(long));
 583                 smtp_process_one_msg(m);
 584         }
 585
 586         array_free(smtp_queue);
 587         last_queue_job_processed = last_queue_job_submitted;
 588         doing_smtpclient = 0;
 589         syslog(LOG_DEBUG, "smtpclient: end %s queue run , last_queue_job_processed=%ld , last_queue_job_submitted=%ld",
 590                 (type_of_queue_run == QUICK_QUEUE_RUN ? "quick" : "full"),
 591                 last_queue_job_processed, last_queue_job_submitted
 592         );
 593 }
 594
 595
 596 // The "full" queue run goes through the entire queue, attempting delivery for newly submitted messages,
 597 // retrying failed deliveries periodically, and handling undeliverable messages.
 598 void smtp_do_queue_full(void) {
 599         smtp_do_queue(FULL_QUEUE_RUN);
 600 }
 601
 602
 603 // The "quick" queue run only handles newly submitted messages, allowing them to be delivered immediately
 604 // instead of waiting for the next "full" queue run.
 605 void smtp_do_queue_quick(void) {
 606         if (last_queue_job_submitted > last_queue_job_processed) {
 607                 smtp_do_queue(QUICK_QUEUE_RUN);
 608         }
 609 }
 610
 611
 612 // Initialization function, called from modules_init.c
 613 char *ctdl_module_init_smtpclient(void) {
 614         if (!threading) {
 615                 CtdlRegisterMessageHook(smtp_aftersave, EVT_AFTERSAVE);
 616                 CtdlRegisterSessionHook(smtp_do_queue_quick, EVT_HOUSE, PRIO_AGGR + 51);
 617                 CtdlRegisterSessionHook(smtp_do_queue_full, EVT_TIMER, PRIO_AGGR + 51);
 618                 smtp_init_spoolout();
 619         }
 620
 621         // return our module id for the log
 622         return "smtpclient";
 623 }