citadel/modules/smtp/serv_smtpqueue.c

   1 /*
   2  * This module is an SMTP and ESMTP implementation for the Citadel system.
   3  * It is compliant with all of the following:
   4  *
   5  * RFC  821 - Simple Mail Transfer Protocol
   6  * RFC  876 - Survey of SMTP Implementations
   7  * RFC 1047 - Duplicate messages and SMTP
   8  * RFC 1652 - 8 bit MIME
   9  * RFC 1869 - Extended Simple Mail Transfer Protocol
  10  * RFC 1870 - SMTP Service Extension for Message Size Declaration
  11  * RFC 2033 - Local Mail Transfer Protocol
  12  * RFC 2197 - SMTP Service Extension for Command Pipelining
  13  * RFC 2476 - Message Submission
  14  * RFC 2487 - SMTP Service Extension for Secure SMTP over TLS
  15  * RFC 2554 - SMTP Service Extension for Authentication
  16  * RFC 2821 - Simple Mail Transfer Protocol
  17  * RFC 2822 - Internet Message Format
  18  * RFC 2920 - SMTP Service Extension for Command Pipelining
  19  *
  20  * The VRFY and EXPN commands have been removed from this implementation
  21  * because nobody uses these commands anymore, except for spammers.
  22  *
  23  * Copyright (c) 1998-2012 by the citadel.org team
  24  *
  25  *  This program is open source software; you can redistribute it and/or modify
  26  *  it under the terms of the GNU General Public License version 3.
  27  *
  28  *
  29  *
  30  *  This program is distributed in the hope that it will be useful,
  31  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  32  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  33  *  GNU General Public License for more details.
  34  *
  35  *
  36  *
  37  *
  38  */
  39
  40 #include "sysdep.h"
  41 #include <stdlib.h>
  42 #include <unistd.h>
  43 #include <stdio.h>
  44 #include <termios.h>
  45 #include <fcntl.h>
  46 #include <signal.h>
  47 #include <pwd.h>
  48 #include <errno.h>
  49 #include <sys/types.h>
  50 #include <syslog.h>
  51
  52 #if TIME_WITH_SYS_TIME
  53 # include <sys/time.h>
  54 # include <time.h>
  55 #else
  56 # if HAVE_SYS_TIME_H
  57 #  include <sys/time.h>
  58 # else
  59 #  include <time.h>
  60 # endif
  61 #endif
  62 #include <sys/wait.h>
  63 #include <ctype.h>
  64 #include <string.h>
  65 #include <limits.h>
  66 #include <sys/socket.h>
  67 #include <netinet/in.h>
  68 #include <arpa/inet.h>
  69 #include <libcitadel.h>
  70 #include "citadel.h"
  71 #include "server.h"
  72 #include "citserver.h"
  73 #include "support.h"
  74 #include "config.h"
  75 #include "control.h"
  76 #include "user_ops.h"
  77 #include "database.h"
  78 #include "msgbase.h"
  79 #include "internet_addressing.h"
  80 #include "genstamp.h"
  81 #include "domain.h"
  82 #include "clientsocket.h"
  83 #include "locate_host.h"
  84 #include "citadel_dirs.h"
  85
  86 #include "ctdl_module.h"
  87
  88 #include "smtpqueue.h"
  89 #include "event_client.h"
  90
  91
  92 struct CitContext smtp_queue_CC;
  93 pthread_mutex_t ActiveQItemsLock;
  94 HashList *ActiveQItems  = NULL;
  95 HashList *QItemHandlers = NULL;
  96 int max_sessions_for_outbound_smtp = 500; /* how many sessions might be active till we stop adding more smtp jobs */
  97 int ndelay_count = 50; /* every n queued messages we will sleep... */
  98 int delay_msec = 5000; /* this many seconds. */
  99
 100 static const long MaxRetry = SMTP_RETRY_INTERVAL * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2;
 101 int MsgCount            = 0;
 102 int run_queue_now       = 0;    /* Set to 1 to ignore SMTP send retry times */
 103
 104 void smtp_try_one_queue_entry(OneQueItem *MyQItem,
 105                               MailQEntry *MyQEntry,
 106                               StrBuf *MsgText,
 107 /* KeepMsgText allows us to use MsgText as ours. */
 108                               int KeepMsgText,
 109                               int MsgCount,
 110                               ParsedURL *RelayUrls);
 111
 112
 113 void smtp_evq_cleanup(void)
 114 {
 115
 116         pthread_mutex_lock(&ActiveQItemsLock);
 117         DeleteHash(&QItemHandlers);
 118         DeleteHash(&ActiveQItems);
 119         pthread_mutex_unlock(&ActiveQItemsLock);
 120         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 121 /*      citthread_mutex_destroy(&ActiveQItemsLock); TODO */
 122 }
 123
 124 int DecreaseQReference(OneQueItem *MyQItem)
 125 {
 126         int IDestructQueItem;
 127
 128         pthread_mutex_lock(&ActiveQItemsLock);
 129         MyQItem->ActiveDeliveries--;
 130         IDestructQueItem = MyQItem->ActiveDeliveries == 0;
 131         pthread_mutex_unlock(&ActiveQItemsLock);
 132         return IDestructQueItem;
 133 }
 134
 135 void RemoveQItem(OneQueItem *MyQItem)
 136 {
 137         long len;
 138         const char* Key;
 139         void *VData;
 140         HashPos  *It;
 141
 142         pthread_mutex_lock(&ActiveQItemsLock);
 143         It = GetNewHashPos(ActiveQItems, 0);
 144         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 145                 DeleteEntryFromHash(ActiveQItems, It);
 146         else
 147         {
 148                 syslog(LOG_WARNING,
 149                        "SMTP cleanup: unable to find QItem with ID[%ld]",
 150                        MyQItem->MessageID);
 151                 while (GetNextHashPos(ActiveQItems, It, &len, &Key, &VData))
 152                         syslog(LOG_WARNING,
 153                                "SMTP cleanup: have_: ID[%ld]",
 154                                ((OneQueItem *)VData)->MessageID);
 155         }
 156         pthread_mutex_unlock(&ActiveQItemsLock);
 157         DeleteHashPos(&It);
 158 }
 159
 160
 161 void FreeMailQEntry(void *qv)
 162 {
 163         MailQEntry *Q = qv;
 164 /*
 165         syslog(LOG_DEBUG, "---------------%s--------------", __FUNCTION__);
 166         cit_backtrace();
 167 */
 168         FreeStrBuf(&Q->Recipient);
 169         FreeStrBuf(&Q->StatusMessage);
 170
 171         memset(Q, 0, sizeof(MailQEntry));
 172         free(Q);
 173 }
 174 void FreeQueItem(OneQueItem **Item)
 175 {
 176 /*
 177         syslog(LOG_DEBUG, "---------------%s--------------", __FUNCTION__);
 178         cit_backtrace();
 179 */
 180         DeleteHash(&(*Item)->MailQEntries);
 181         FreeStrBuf(&(*Item)->EnvelopeFrom);
 182         FreeStrBuf(&(*Item)->BounceTo);
 183         FreeStrBuf(&(*Item)->SenderRoom);
 184         FreeURL(&(*Item)->URL);
 185         memset(*Item, 0, sizeof(OneQueItem));
 186         free(*Item);
 187         Item = NULL;
 188 }
 189 void HFreeQueItem(void *Item)
 190 {
 191         FreeQueItem((OneQueItem**)&Item);
 192 }
 193
 194 /* inspect recipients with a status of:
 195  * - 0 (no delivery yet attempted)
 196  * - 3/4 (transient errors
 197  *        were experienced and it's time to try again)
 198  */
 199 int CheckQEntryActive(MailQEntry *ThisItem)
 200 {
 201         if ((ThisItem->Status == 0) ||
 202             (ThisItem->Status == 3) ||
 203             (ThisItem->Status == 4))
 204         {
 205                 return 1;
 206         }
 207         else
 208                 return 0;
 209 }
 210 int CheckQEntryIsBounce(MailQEntry *ThisItem)
 211 {
 212         if ((ThisItem->Status == 3) ||
 213             (ThisItem->Status == 4) ||
 214             (ThisItem->Status == 5))
 215         {
 216                 return 1;
 217         }
 218         else
 219                 return 0;
 220 }
 221
 222 int CountActiveQueueEntries(OneQueItem *MyQItem)
 223 {
 224         HashPos  *It;
 225         long len;
 226         long ActiveDeliveries;
 227         const char *Key;
 228         void *vQE;
 229
 230         ActiveDeliveries = 0;
 231         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 232         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 233         {
 234                 MailQEntry *ThisItem = vQE;
 235
 236                 if (CheckQEntryActive(ThisItem))
 237                 {
 238                         ActiveDeliveries++;
 239                         ThisItem->Active = 1;
 240                 }
 241                 else
 242                         ThisItem->Active = 0;
 243         }
 244         DeleteHashPos(&It);
 245         return ActiveDeliveries;
 246 }
 247
 248 OneQueItem *DeserializeQueueItem(StrBuf *RawQItem, long QueMsgID)
 249 {
 250         OneQueItem *Item;
 251         const char *pLine = NULL;
 252         StrBuf *Line;
 253         StrBuf *Token;
 254         void *v;
 255
 256         Item = (OneQueItem*)malloc(sizeof(OneQueItem));
 257         memset(Item, 0, sizeof(OneQueItem));
 258         Item->Retry = SMTP_RETRY_INTERVAL;
 259         Item->MessageID = -1;
 260         Item->QueMsgID = QueMsgID;
 261
 262         Token = NewStrBuf();
 263         Line = NewStrBufPlain(NULL, 128);
 264         while (pLine != StrBufNOTNULL) {
 265                 const char *pItemPart = NULL;
 266                 void *vHandler;
 267
 268                 StrBufExtract_NextToken(Line, RawQItem, &pLine, '\n');
 269                 if (StrLength(Line) == 0) continue;
 270                 StrBufExtract_NextToken(Token, Line, &pItemPart, '|');
 271                 if (GetHash(QItemHandlers, SKEY(Token), &vHandler))
 272                 {
 273                         QItemHandler H;
 274                         H = (QItemHandler) vHandler;
 275                         H(Item, Line, &pItemPart);
 276                 }
 277         }
 278         FreeStrBuf(&Line);
 279         FreeStrBuf(&Token);
 280
 281         if (Item->Retry >= MaxRetry)
 282                 Item->FailNow = 1;
 283
 284         pthread_mutex_lock(&ActiveQItemsLock);
 285         if (GetHash(ActiveQItems,
 286                     LKEY(Item->MessageID),
 287                     &v))
 288         {
 289                 /* WHOOPS. somebody else is already working on this. */
 290                 pthread_mutex_unlock(&ActiveQItemsLock);
 291                 FreeQueItem(&Item);
 292                 return NULL;
 293         }
 294         else {
 295                 /* mark our claim on this. */
 296                 Put(ActiveQItems,
 297                     LKEY(Item->MessageID),
 298                     Item,
 299                     HFreeQueItem);
 300                 pthread_mutex_unlock(&ActiveQItemsLock);
 301         }
 302
 303         return Item;
 304 }
 305
 306 StrBuf *SerializeQueueItem(OneQueItem *MyQItem)
 307 {
 308         StrBuf *QMessage;
 309         HashPos  *It;
 310         const char *Key;
 311         long len;
 312         void *vQE;
 313
 314         QMessage = NewStrBufPlain(NULL, SIZ);
 315         StrBufPrintf(QMessage, "Content-type: %s\n", SPOOLMIME);
 316 //      "attempted|%ld\n"  "retry|%ld\n",, (long)time(NULL), (long)retry );
 317         StrBufAppendBufPlain(QMessage, HKEY("\nmsgid|"), 0);
 318         StrBufAppendPrintf(QMessage, "%ld", MyQItem->MessageID);
 319
 320         StrBufAppendBufPlain(QMessage, HKEY("\nsubmitted|"), 0);
 321         StrBufAppendPrintf(QMessage, "%ld", MyQItem->Submitted);
 322
 323         if (StrLength(MyQItem->BounceTo) > 0) {
 324                 StrBufAppendBufPlain(QMessage, HKEY("\nbounceto|"), 0);
 325                 StrBufAppendBuf(QMessage, MyQItem->BounceTo, 0);
 326         }
 327
 328         if (StrLength(MyQItem->EnvelopeFrom) > 0) {
 329                 StrBufAppendBufPlain(QMessage, HKEY("\nenvelope_from|"), 0);
 330                 StrBufAppendBuf(QMessage, MyQItem->EnvelopeFrom, 0);
 331         }
 332
 333         if (StrLength(MyQItem->SenderRoom) > 0) {
 334                 StrBufAppendBufPlain(QMessage, HKEY("\nsource_room|"), 0);
 335                 StrBufAppendBuf(QMessage, MyQItem->SenderRoom, 0);
 336         }
 337
 338         StrBufAppendBufPlain(QMessage, HKEY("\nretry|"), 0);
 339         StrBufAppendPrintf(QMessage, "%ld",
 340                            MyQItem->Retry);
 341
 342         StrBufAppendBufPlain(QMessage, HKEY("\nattempted|"), 0);
 343         StrBufAppendPrintf(QMessage, "%ld",
 344                            MyQItem->ReattemptWhen);
 345
 346         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 347         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 348         {
 349                 MailQEntry *ThisItem = vQE;
 350
 351                 StrBufAppendBufPlain(QMessage, HKEY("\nremote|"), 0);
 352                 StrBufAppendBuf(QMessage, ThisItem->Recipient, 0);
 353                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 354                 StrBufAppendPrintf(QMessage, "%d", ThisItem->Status);
 355                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 356                 StrBufAppendBuf(QMessage, ThisItem->StatusMessage, 0);
 357         }
 358         DeleteHashPos(&It);
 359         StrBufAppendBufPlain(QMessage, HKEY("\n"), 0);
 360         return QMessage;
 361 }
 362
 363
 364
 365
 366
 367 void NewMailQEntry(OneQueItem *Item)
 368 {
 369         Item->Current = (MailQEntry*) malloc(sizeof(MailQEntry));
 370         memset(Item->Current, 0, sizeof(MailQEntry));
 371
 372         if (Item->MailQEntries == NULL)
 373                 Item->MailQEntries = NewHash(1, Flathash);
 374         Item->Current->StatusMessage = NewStrBuf();
 375         Item->Current->n = GetCount(Item->MailQEntries);
 376         Put(Item->MailQEntries,
 377             IKEY(Item->Current->n),
 378             Item->Current,
 379             FreeMailQEntry);
 380 }
 381
 382 void QItem_Handle_MsgID(OneQueItem *Item, StrBuf *Line, const char **Pos)
 383 {
 384         Item->MessageID = StrBufExtractNext_long(Line, Pos, '|');
 385 }
 386
 387 void QItem_Handle_EnvelopeFrom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 388 {
 389         if (Item->EnvelopeFrom == NULL)
 390                 Item->EnvelopeFrom = NewStrBufPlain(NULL, StrLength(Line));
 391         StrBufExtract_NextToken(Item->EnvelopeFrom, Line, Pos, '|');
 392 }
 393
 394 void QItem_Handle_BounceTo(OneQueItem *Item, StrBuf *Line, const char **Pos)
 395 {
 396         if (Item->BounceTo == NULL)
 397                 Item->BounceTo = NewStrBufPlain(NULL, StrLength(Line));
 398         StrBufExtract_NextToken(Item->BounceTo, Line, Pos, '|');
 399 }
 400
 401 void QItem_Handle_SenderRoom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 402 {
 403         if (Item->SenderRoom == NULL)
 404                 Item->SenderRoom = NewStrBufPlain(NULL, StrLength(Line));
 405         StrBufExtract_NextToken(Item->SenderRoom, Line, Pos, '|');
 406 }
 407
 408 void QItem_Handle_Recipient(OneQueItem *Item, StrBuf *Line, const char **Pos)
 409 {
 410         if (Item->Current == NULL)
 411                 NewMailQEntry(Item);
 412         if (Item->Current->Recipient == NULL)
 413                 Item->Current->Recipient=NewStrBufPlain(NULL, StrLength(Line));
 414         StrBufExtract_NextToken(Item->Current->Recipient, Line, Pos, '|');
 415         Item->Current->Status = StrBufExtractNext_int(Line, Pos, '|');
 416         StrBufExtract_NextToken(Item->Current->StatusMessage, Line, Pos, '|');
 417         Item->Current = NULL; // TODO: is this always right?
 418 }
 419
 420
 421 void QItem_Handle_retry(OneQueItem *Item, StrBuf *Line, const char **Pos)
 422 {
 423         Item->Retry =
 424                 StrBufExtractNext_int(Line, Pos, '|');
 425         Item->Retry *= 2;
 426 }
 427
 428
 429 void QItem_Handle_Submitted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 430 {
 431         Item->Submitted = atol(*Pos);
 432
 433 }
 434
 435 void QItem_Handle_Attempted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 436 {
 437         Item->ReattemptWhen = StrBufExtractNext_int(Line, Pos, '|');
 438 }
 439
 440
 441
 442 /**
 443  * this one has to have the context for loading the message via the redirect buffer...
 444  */
 445 StrBuf *smtp_load_msg(OneQueItem *MyQItem, int n)
 446 {
 447         CitContext *CCC=CC;
 448         StrBuf *SendMsg;
 449
 450         CCC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
 451         CtdlOutputMsg(MyQItem->MessageID,
 452                       MT_RFC822, HEADERS_ALL,
 453                       0, 1, NULL,
 454                       (ESC_DOT|SUPPRESS_ENV_TO) );
 455
 456         SendMsg = CCC->redirect_buffer;
 457         CCC->redirect_buffer = NULL;
 458         if ((StrLength(SendMsg) > 0) &&
 459             ChrPtr(SendMsg)[StrLength(SendMsg) - 1] != '\n') {
 460                 syslog(LOG_WARNING,
 461                        "SMTP client[%d]: Possible problem: message did not "
 462                        "correctly terminate. (expecting 0x10, got 0x%02x)\n",
 463                        MsgCount, //yes uncool, but best choice here...
 464                        ChrPtr(SendMsg)[StrLength(SendMsg) - 1] );
 465                 StrBufAppendBufPlain(SendMsg, HKEY("\r\n"), 0);
 466         }
 467         return SendMsg;
 468 }
 469
 470
 471
 472 /*
 473  * smtp_do_bounce() is caled by smtp_do_procmsg() to scan a set of delivery
 474  * instructions for "5" codes (permanent fatal errors) and produce/deliver
 475  * a "bounce" message (delivery status notification).
 476  */
 477 void smtpq_do_bounce(OneQueItem *MyQItem, StrBuf *OMsgTxt)
 478 {
 479         static int seq = 0;
 480
 481         struct CtdlMessage *bmsg = NULL;
 482         StrBuf *boundary;
 483         StrBuf *Msg = NULL;
 484         StrBuf *BounceMB;
 485         struct recptypes *valid;
 486         time_t now;
 487
 488         HashPos *It;
 489         void *vQE;
 490         long len;
 491         const char *Key;
 492
 493         int first_attempt = 0;
 494         int successful_bounce = 0;
 495         int num_bounces = 0;
 496         int give_up = 0;
 497
 498         syslog(LOG_DEBUG, "smtp_do_bounce() called\n");
 499
 500         if (MyQItem->SendBounceMail == 0)
 501                 return;
 502
 503         now = ev_time();
 504
 505         if ( (now - MyQItem->Submitted) > SMTP_GIVE_UP ) {
 506                 give_up = 1;
 507         }
 508
 509         if (MyQItem->Retry == SMTP_RETRY_INTERVAL) {
 510                 first_attempt = 1;
 511         }
 512
 513         /*
 514          * Now go through the instructions checking for stuff.
 515          */
 516         Msg = NewStrBufPlain(NULL, 1024);
 517         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 518         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 519         {
 520                 MailQEntry *ThisItem = vQE;
 521                 if ((ThisItem->Active && (ThisItem->Status == 5)) || /* failed now? */
 522                     ((give_up == 1) && (ThisItem->Status != 2)) ||
 523                     ((first_attempt == 1) && (ThisItem->Status != 2)))
 524                         /* giving up after failed attempts... */
 525                 {
 526                         ++num_bounces;
 527
 528                         StrBufAppendBuf(Msg, ThisItem->Recipient, 0);
 529                         StrBufAppendBufPlain(Msg, HKEY(": "), 0);
 530                         StrBufAppendBuf(Msg, ThisItem->StatusMessage, 0);
 531                         StrBufAppendBufPlain(Msg, HKEY("\r\n"), 0);
 532                 }
 533         }
 534         DeleteHashPos(&It);
 535
 536         /* Deliver the bounce if there's anything worth mentioning */
 537         syslog(LOG_DEBUG, "num_bounces = %d\n", num_bounces);
 538
 539         if (num_bounces == 0) {
 540                 FreeStrBuf(&Msg);
 541                 return;
 542         }
 543
 544         boundary = NewStrBufPlain(HKEY("=_Citadel_Multipart_"));
 545         StrBufAppendPrintf(boundary,
 546                            "%s_%04x%04x",
 547                            config.c_fqdn,
 548                            getpid(),
 549                            ++seq);
 550
 551         /* Start building our bounce message; go shopping for memory first. */
 552         BounceMB = NewStrBufPlain(
 553                 NULL,
 554                 1024 + /* mime stuff.... */
 555                 StrLength(Msg) +  /* the bounce information... */
 556                 StrLength(OMsgTxt)); /* the original message */
 557         if (BounceMB == NULL) {
 558                 FreeStrBuf(&boundary);
 559                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 560
 561                 return;
 562         }
 563
 564         bmsg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
 565         if (bmsg == NULL) {
 566                 FreeStrBuf(&boundary);
 567                 FreeStrBuf(&BounceMB);
 568                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 569
 570                 return;
 571         }
 572         memset(bmsg, 0, sizeof(struct CtdlMessage));
 573
 574
 575         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: multipart/mixed; boundary=\""), 0);
 576         StrBufAppendBuf(BounceMB, boundary, 0);
 577         StrBufAppendBufPlain(BounceMB, HKEY("\"\r\n"), 0);
 578         StrBufAppendBufPlain(BounceMB, HKEY("MIME-Version: 1.0\r\n"), 0);
 579         StrBufAppendBufPlain(BounceMB, HKEY("X-Mailer: " CITADEL "\r\n"), 0);
 580         StrBufAppendBufPlain(BounceMB, HKEY("\r\nThis is a multipart message in MIME format.\r\n\r\n"), 0);
 581         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 582         StrBufAppendBuf(BounceMB, boundary, 0);
 583         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 584         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: text/plain\r\n\r\n"), 0);
 585
 586         if (give_up)
 587                 StrBufAppendBufPlain(
 588                         BounceMB,
 589                         HKEY(
 590                                 "A message you sent could not be delivered "
 591                                 "to some or all of its recipients\n"
 592                                 "due to prolonged unavailability "
 593                                 "of its destination(s).\n"
 594                                 "Giving up on the following addresses:\n\n"
 595                                 ), 0);
 596         else
 597                 StrBufAppendBufPlain(
 598                         BounceMB,
 599                         HKEY(
 600                                 "A message you sent could not be delivered "
 601                                 "to some or all of its recipients.\n"
 602                                 "The following addresses "
 603                                 "were undeliverable:\n\n"
 604                                 ), 0);
 605
 606         StrBufAppendBuf(BounceMB, Msg, 0);
 607         FreeStrBuf(&Msg);
 608
 609         if (StrLength(MyQItem->SenderRoom) > 0)
 610         {
 611                 StrBufAppendBufPlain(
 612                         BounceMB,
 613                         HKEY("The message was originaly posted in: "), 0);
 614                 StrBufAppendBuf(BounceMB, MyQItem->SenderRoom, 0);
 615                 StrBufAppendBufPlain(
 616                         BounceMB,
 617                         HKEY("\n"), 0);
 618         }
 619
 620         /* Attach the original message */
 621         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 622         StrBufAppendBuf(BounceMB, boundary, 0);
 623         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 624         StrBufAppendBufPlain(BounceMB,
 625                              HKEY("Content-type: message/rfc822\r\n"), 0);
 626         StrBufAppendBufPlain(BounceMB,
 627                              HKEY("Content-Transfer-Encoding: 7bit\r\n"), 0);
 628         StrBufAppendBufPlain(BounceMB,
 629                              HKEY("Content-Disposition: inline\r\n"), 0);
 630         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 631         StrBufAppendBuf(BounceMB, OMsgTxt, 0);
 632
 633         /* Close the multipart MIME scope */
 634         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 635         StrBufAppendBuf(BounceMB, boundary, 0);
 636         StrBufAppendBufPlain(BounceMB, HKEY("--\r\n"), 0);
 637
 638         bmsg->cm_magic = CTDLMESSAGE_MAGIC;
 639         bmsg->cm_anon_type = MES_NORMAL;
 640         bmsg->cm_format_type = FMT_RFC822;
 641
 642         bmsg->cm_fields['O'] = strdup(MAILROOM);
 643         bmsg->cm_fields['A'] = strdup("Citadel");
 644         bmsg->cm_fields['N'] = strdup(config.c_nodename);
 645         bmsg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
 646         bmsg->cm_fields['M'] = SmashStrBuf(&BounceMB);
 647
 648         /* First try the user who sent the message */
 649         if (StrLength(MyQItem->BounceTo) == 0)
 650                 syslog(LOG_ERR, "No bounce address specified\n");
 651         else
 652                 syslog(LOG_DEBUG, "bounce to user? <%s>\n",
 653                        ChrPtr(MyQItem->BounceTo));
 654
 655         /* Can we deliver the bounce to the original sender? */
 656         valid = validate_recipients(ChrPtr(MyQItem->BounceTo), NULL, 0);
 657         if ((valid != NULL) && (valid->num_error == 0)) {
 658                 CtdlSubmitMsg(bmsg, valid, "", QP_EADDR);
 659                 successful_bounce = 1;
 660         }
 661
 662         /* If not, post it in the Aide> room */
 663         if (successful_bounce == 0) {
 664                 CtdlSubmitMsg(bmsg, NULL, config.c_aideroom, QP_EADDR);
 665         }
 666
 667         /* Free up the memory we used */
 668         free_recipients(valid);
 669         FreeStrBuf(&boundary);
 670         CtdlFreeMessage(bmsg);
 671         syslog(LOG_DEBUG, "Done processing bounces\n");
 672 }
 673
 674 /*
 675  * smtp_do_procmsg()
 676  *
 677  * Called by smtp_do_queue() to handle an individual message.
 678  */
 679 void smtp_do_procmsg(long msgnum, void *userdata) {
 680         int mynumsessions = num_sessions;
 681         struct CtdlMessage *msg = NULL;
 682         char *instr = NULL;
 683         StrBuf *PlainQItem;
 684         OneQueItem *MyQItem;
 685         char *pch;
 686         HashPos  *It;
 687         void *vQE;
 688         long len;
 689         const char *Key;
 690         int nRelays = 0;
 691         ParsedURL *RelayUrls = NULL;
 692         int HaveBuffers = 0;
 693         StrBuf *Msg =NULL;
 694
 695         if (mynumsessions > max_sessions_for_outbound_smtp) {
 696                 syslog(LOG_DEBUG,
 697                        "SMTP Queue: skipping because of num jobs %d > %d max_sessions_for_outbound_smtp",
 698                        mynumsessions,
 699                        max_sessions_for_outbound_smtp);
 700         }
 701
 702         syslog(LOG_DEBUG, "SMTP Queue: smtp_do_procmsg(%ld)\n", msgnum);
 703         ///strcpy(envelope_from, "");
 704
 705         msg = CtdlFetchMessage(msgnum, 1);
 706         if (msg == NULL) {
 707                 syslog(LOG_ERR, "SMTP Queue: tried %ld but no such message!\n",
 708                        msgnum);
 709                 return;
 710         }
 711
 712         pch = instr = msg->cm_fields['M'];
 713
 714         /* Strip out the headers (no not amd any other non-instruction) line */
 715         while (pch != NULL) {
 716                 pch = strchr(pch, '\n');
 717                 if ((pch != NULL) && (*(pch + 1) == '\n')) {
 718                         instr = pch + 2;
 719                         pch = NULL;
 720                 }
 721         }
 722         PlainQItem = NewStrBufPlain(instr, -1);
 723         CtdlFreeMessage(msg);
 724         MyQItem = DeserializeQueueItem(PlainQItem, msgnum);
 725         FreeStrBuf(&PlainQItem);
 726
 727         if (MyQItem == NULL) {
 728                 syslog(LOG_ERR,
 729                        "SMTP Queue: Msg No %ld: already in progress!\n",
 730                        msgnum);
 731                 return; /* s.b. else is already processing... */
 732         }
 733
 734         /*
 735          * Postpone delivery if we've already tried recently.
 736          */
 737         if ((MyQItem->ReattemptWhen != 0) &&
 738             (time(NULL) < MyQItem->ReattemptWhen) &&
 739             (run_queue_now == 0))
 740         {
 741                 syslog(LOG_DEBUG, "SMTP client: Retry time not yet reached.\n");
 742
 743                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 744                 pthread_mutex_lock(&ActiveQItemsLock);
 745                 {
 746                         if (GetHashPosFromKey(ActiveQItems,
 747                                               LKEY(MyQItem->MessageID),
 748                                               It))
 749                         {
 750                                 DeleteEntryFromHash(ActiveQItems, It);
 751                         }
 752                 }
 753                 pthread_mutex_unlock(&ActiveQItemsLock);
 754                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 755                 DeleteHashPos(&It);
 756                 return;
 757         }
 758
 759         /*
 760          * Bail out if there's no actual message associated with this
 761          */
 762         if (MyQItem->MessageID < 0L) {
 763                 syslog(LOG_ERR, "SMTP Queue: no 'msgid' directive found!\n");
 764                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 765                 pthread_mutex_lock(&ActiveQItemsLock);
 766                 {
 767                         if (GetHashPosFromKey(ActiveQItems,
 768                                               LKEY(MyQItem->MessageID),
 769                                               It))
 770                         {
 771                                 DeleteEntryFromHash(ActiveQItems, It);
 772                         }
 773                 }
 774                 pthread_mutex_unlock(&ActiveQItemsLock);
 775                 DeleteHashPos(&It);
 776                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 777                 return;
 778         }
 779
 780         {
 781                 char mxbuf[SIZ];
 782                 ParsedURL **Url = &MyQItem->URL;
 783                 nRelays = get_hosts(mxbuf, "smarthost");
 784                 if (nRelays > 0) {
 785                         StrBuf *All;
 786                         StrBuf *One;
 787                         const char *Pos = NULL;
 788                         All = NewStrBufPlain(mxbuf, -1);
 789                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 790
 791                         while ((Pos != StrBufNOTNULL) &&
 792                                ((Pos == NULL) ||
 793                                 !IsEmptyStr(Pos)))
 794                         {
 795                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 796                                 if (!ParseURL(Url, One, 25))
 797                                         syslog(LOG_DEBUG,
 798                                                "Failed to parse: %s\n",
 799                                                ChrPtr(One));
 800                                 else {
 801                                         ///if (!Url->IsIP)) // todo dupe me fork ipv6
 802                                         Url = &(*Url)->Next;
 803                                 }
 804                         }
 805                         FreeStrBuf(&All);
 806                         FreeStrBuf(&One);
 807                 }
 808
 809                 Url = &MyQItem->FallBackHost;
 810                 nRelays = get_hosts(mxbuf, "fallbackhost");
 811                 if (nRelays > 0) {
 812                         StrBuf *All;
 813                         StrBuf *One;
 814                         const char *Pos = NULL;
 815                         All = NewStrBufPlain(mxbuf, -1);
 816                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 817
 818                         while ((Pos != StrBufNOTNULL) &&
 819                                ((Pos == NULL) ||
 820                                 !IsEmptyStr(Pos)))
 821                         {
 822                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 823                                 if (!ParseURL(Url, One, 25))
 824                                         syslog(LOG_DEBUG,
 825                                                "Failed to parse: %s\n",
 826                                                ChrPtr(One));
 827                                 else
 828                                         Url = &(*Url)->Next;
 829                         }
 830                         FreeStrBuf(&All);
 831                         FreeStrBuf(&One);
 832                 }
 833         }
 834
 835         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 836         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 837         {
 838                 MailQEntry *ThisItem = vQE;
 839                 syslog(LOG_DEBUG, "SMTP Queue: Task: <%s> %d\n",
 840                        ChrPtr(ThisItem->Recipient),
 841                        ThisItem->Active);
 842         }
 843         DeleteHashPos(&It);
 844
 845         MyQItem->ActiveDeliveries = CountActiveQueueEntries(MyQItem);
 846
 847         /* failsafe against overload:
 848          * will we exceed the limit set?
 849          */
 850         if ((MyQItem->ActiveDeliveries + mynumsessions > max_sessions_for_outbound_smtp) &&
 851             /* if yes, did we reach more than half of the quota? */
 852             ((mynumsessions * 2) > max_sessions_for_outbound_smtp) &&
 853             /* if... would we ever fit into half of the quota?? */
 854             (((MyQItem->ActiveDeliveries * 2)  < max_sessions_for_outbound_smtp)))
 855         {
 856                 /* abort delivery for another time. */
 857                 syslog(LOG_DEBUG,
 858                        "SMTP Queue: skipping because of num jobs %d + %ld > %d max_sessions_for_outbound_smtp",
 859                        mynumsessions,
 860                        MyQItem->ActiveDeliveries,
 861                        max_sessions_for_outbound_smtp);
 862
 863                 FreeQueItem(&MyQItem);
 864
 865                 return;
 866         }
 867
 868
 869         if (MyQItem->ActiveDeliveries > 0)
 870         {
 871                 int nActivated = 0;
 872                 int n = MsgCount++;
 873                 int m = MyQItem->ActiveDeliveries;
 874                 int i = 1;
 875                 Msg = smtp_load_msg(MyQItem, n);
 876                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 877                 while ((i <= m) &&
 878                        (GetNextHashPos(MyQItem->MailQEntries,
 879                                        It, &len, &Key, &vQE)))
 880                 {
 881                         MailQEntry *ThisItem = vQE;
 882
 883                         if (ThisItem->Active == 1)
 884                         {
 885                                 int KeepBuffers = (i == m);
 886
 887                                 nActivated++;
 888                                 if (nActivated % ndelay_count == 0)
 889                                         usleep(delay_msec);
 890
 891                                 if (i > 1) n = MsgCount++;
 892                                 syslog(LOG_DEBUG,
 893                                        "SMTPQ: Trying <%ld> <%s> %d / %d \n",
 894                                        MyQItem->MessageID,
 895                                        ChrPtr(ThisItem->Recipient),
 896                                        i,
 897                                        m);
 898                                 smtp_try_one_queue_entry(MyQItem,
 899                                                          ThisItem,
 900                                                          Msg,
 901                                                          KeepBuffers,
 902                                                          n,
 903                                                          RelayUrls);
 904
 905                                 if (KeepBuffers) HaveBuffers = 1;
 906
 907                                 i++;
 908                         }
 909                 }
 910                 DeleteHashPos(&It);
 911         }
 912         else
 913         {
 914                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 915                 pthread_mutex_lock(&ActiveQItemsLock);
 916                 {
 917                         if (GetHashPosFromKey(ActiveQItems,
 918                                               LKEY(MyQItem->MessageID),
 919                                               It))
 920                         {
 921                                 DeleteEntryFromHash(ActiveQItems, It);
 922                         }
 923                         else
 924                         {
 925                                 long len;
 926                                 const char* Key;
 927                                 void *VData;
 928
 929                                 syslog(LOG_WARNING,
 930                                        "SMTP cleanup: unable to find "
 931                                        "QItem with ID[%ld]",
 932                                        MyQItem->MessageID);
 933                                 while (GetNextHashPos(ActiveQItems,
 934                                                       It,
 935                                                       &len,
 936                                                       &Key,
 937                                                       &VData))
 938                                 {
 939                                         syslog(LOG_WARNING,
 940                                                "SMTP cleanup: have: ID[%ld]",
 941                                               ((OneQueItem *)VData)->MessageID);
 942                                 }
 943                         }
 944
 945                 }
 946                 pthread_mutex_unlock(&ActiveQItemsLock);
 947                 DeleteHashPos(&It);
 948                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 949
 950 // TODO: bounce & delete?
 951
 952         }
 953         if (!HaveBuffers) {
 954                 FreeStrBuf (&Msg);
 955 // TODO : free RelayUrls
 956         }
 957 }
 958
 959
 960
 961 /*
 962  * smtp_queue_thread()
 963  *
 964  * Run through the queue sending out messages.
 965  */
 966 void smtp_do_queue(void) {
 967         static int is_running = 0;
 968         int num_processed = 0;
 969
 970         if (is_running)
 971                 return; /* Concurrency check - only one can run */
 972         is_running = 1;
 973
 974         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 975         syslog(LOG_INFO, "SMTP client: processing outbound queue");
 976
 977         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
 978                 syslog(LOG_ERR, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
 979         }
 980         else {
 981                 num_processed = CtdlForEachMessage(MSGS_ALL,
 982                                                    0L,
 983                                                    NULL,
 984                                                    SPOOLMIME,
 985                                                    NULL,
 986                                                    smtp_do_procmsg,
 987                                                    NULL);
 988         }
 989         syslog(LOG_INFO,
 990                "SMTP client: queue run completed; %d messages processed",
 991                num_processed);
 992
 993         run_queue_now = 0;
 994         is_running = 0;
 995 }
 996
 997
 998
 999 /*
1000  * Initialize the SMTP outbound queue
1001  */
1002 void smtp_init_spoolout(void) {
1003         struct ctdlroom qrbuf;
1004
1005         /*
1006          * Create the room.  This will silently fail if the room already
1007          * exists, and that's perfectly ok, because we want it to exist.
1008          */
1009         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
1010
1011         /*
1012          * Make sure it's set to be a "system room" so it doesn't show up
1013          * in the <K>nown rooms list for Aides.
1014          */
1015         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
1016                 qrbuf.QRflags2 |= QR2_SYSTEM;
1017                 CtdlPutRoomLock(&qrbuf);
1018         }
1019 }
1020
1021
1022
1023
1024 /*****************************************************************************/
1025 /*                          SMTP UTILITY COMMANDS                            */
1026 /*****************************************************************************/
1027
1028 void cmd_smtp(char *argbuf) {
1029         char cmd[64];
1030         char node[256];
1031         char buf[1024];
1032         int i;
1033         int num_mxhosts;
1034
1035         if (CtdlAccessCheck(ac_aide)) return;
1036
1037         extract_token(cmd, argbuf, 0, '|', sizeof cmd);
1038
1039         if (!strcasecmp(cmd, "mx")) {
1040                 extract_token(node, argbuf, 1, '|', sizeof node);
1041                 num_mxhosts = getmx(buf, node);
1042                 cprintf("%d %d MX hosts listed for %s\n",
1043                         LISTING_FOLLOWS, num_mxhosts, node);
1044                 for (i=0; i<num_mxhosts; ++i) {
1045                         extract_token(node, buf, i, '|', sizeof node);
1046                         cprintf("%s\n", node);
1047                 }
1048                 cprintf("000\n");
1049                 return;
1050         }
1051
1052         else if (!strcasecmp(cmd, "runqueue")) {
1053                 run_queue_now = 1;
1054                 cprintf("%d All outbound SMTP will be retried now.\n", CIT_OK);
1055                 return;
1056         }
1057
1058         else {
1059                 cprintf("%d Invalid command.\n", ERROR + ILLEGAL_VALUE);
1060         }
1061
1062 }
1063
1064
1065 CTDL_MODULE_INIT(smtp_queu)
1066 {
1067         char *pstr;
1068
1069         if (!threading)
1070         {
1071                 pstr = getenv("CITSERVER_n_session_max");
1072                 if ((pstr != NULL) && (*pstr != '\0'))
1073                         max_sessions_for_outbound_smtp = atol(pstr); /* how many sessions might be active till we stop adding more smtp jobs */
1074
1075                 pstr = getenv("CITSERVER_smtp_n_delay_count");
1076                 if ((pstr != NULL) && (*pstr != '\0'))
1077                         ndelay_count = atol(pstr); /* every n queued messages we will sleep... */
1078
1079                 pstr = getenv("CITSERVER_smtp_delay");
1080                 if ((pstr != NULL) && (*pstr != '\0'))
1081                         delay_msec = atol(pstr) * 1000; /* this many seconds. */
1082
1083
1084
1085
1086                 CtdlFillSystemContext(&smtp_queue_CC, "SMTP_Send");
1087                 ActiveQItems = NewHash(1, lFlathash);
1088                 pthread_mutex_init(&ActiveQItemsLock, NULL);
1089
1090                 QItemHandlers = NewHash(0, NULL);
1091
1092                 Put(QItemHandlers, HKEY("msgid"), QItem_Handle_MsgID, reference_free_handler);
1093                 Put(QItemHandlers, HKEY("envelope_from"), QItem_Handle_EnvelopeFrom, reference_free_handler);
1094                 Put(QItemHandlers, HKEY("retry"), QItem_Handle_retry, reference_free_handler);
1095                 Put(QItemHandlers, HKEY("attempted"), QItem_Handle_Attempted, reference_free_handler);
1096                 Put(QItemHandlers, HKEY("remote"), QItem_Handle_Recipient, reference_free_handler);
1097                 Put(QItemHandlers, HKEY("bounceto"), QItem_Handle_BounceTo, reference_free_handler);
1098                 Put(QItemHandlers, HKEY("source_room"), QItem_Handle_SenderRoom, reference_free_handler);
1099                 Put(QItemHandlers, HKEY("submitted"), QItem_Handle_Submitted, reference_free_handler);
1100                 smtp_init_spoolout();
1101
1102                 CtdlRegisterEVCleanupHook(smtp_evq_cleanup);
1103
1104                 CtdlRegisterProtoHook(cmd_smtp, "SMTP", "SMTP utility commands");
1105                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER);
1106         }
1107
1108         /* return our Subversion id for the Log */
1109         return "smtpeventclient";
1110 }