citadel/modules/smtp/serv_smtpqueue.c

   1 /*
   2  * This module is an SMTP and ESMTP implementation for the Citadel system.
   3  * It is compliant with all of the following:
   4  *
   5  * RFC  821 - Simple Mail Transfer Protocol
   6  * RFC  876 - Survey of SMTP Implementations
   7  * RFC 1047 - Duplicate messages and SMTP
   8  * RFC 1652 - 8 bit MIME
   9  * RFC 1869 - Extended Simple Mail Transfer Protocol
  10  * RFC 1870 - SMTP Service Extension for Message Size Declaration
  11  * RFC 2033 - Local Mail Transfer Protocol
  12  * RFC 2197 - SMTP Service Extension for Command Pipelining
  13  * RFC 2476 - Message Submission
  14  * RFC 2487 - SMTP Service Extension for Secure SMTP over TLS
  15  * RFC 2554 - SMTP Service Extension for Authentication
  16  * RFC 2821 - Simple Mail Transfer Protocol
  17  * RFC 2822 - Internet Message Format
  18  * RFC 2920 - SMTP Service Extension for Command Pipelining
  19  *
  20  * The VRFY and EXPN commands have been removed from this implementation
  21  * because nobody uses these commands anymore, except for spammers.
  22  *
  23  * Copyright (c) 1998-2012 by the citadel.org team
  24  *
  25  *  This program is open source software; you can redistribute it and/or modify
  26  *  it under the terms of the GNU General Public License version 3.
  27  *
  28  *
  29  *
  30  *  This program is distributed in the hope that it will be useful,
  31  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  32  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  33  *  GNU General Public License for more details.
  34  *
  35  *
  36  *
  37  *
  38  */
  39
  40 #include "sysdep.h"
  41 #include <stdlib.h>
  42 #include <unistd.h>
  43 #include <stdio.h>
  44 #include <termios.h>
  45 #include <fcntl.h>
  46 #include <signal.h>
  47 #include <pwd.h>
  48 #include <errno.h>
  49 #include <sys/types.h>
  50 #include <syslog.h>
  51
  52 #if TIME_WITH_SYS_TIME
  53 # include <sys/time.h>
  54 # include <time.h>
  55 #else
  56 # if HAVE_SYS_TIME_H
  57 #  include <sys/time.h>
  58 # else
  59 #  include <time.h>
  60 # endif
  61 #endif
  62 #include <sys/wait.h>
  63 #include <ctype.h>
  64 #include <string.h>
  65 #include <limits.h>
  66 #include <sys/socket.h>
  67 #include <netinet/in.h>
  68 #include <arpa/inet.h>
  69 #include <libcitadel.h>
  70 #include "citadel.h"
  71 #include "server.h"
  72 #include "citserver.h"
  73 #include "support.h"
  74 #include "config.h"
  75 #include "control.h"
  76 #include "user_ops.h"
  77 #include "database.h"
  78 #include "msgbase.h"
  79 #include "internet_addressing.h"
  80 #include "genstamp.h"
  81 #include "domain.h"
  82 #include "clientsocket.h"
  83 #include "locate_host.h"
  84 #include "citadel_dirs.h"
  85
  86 #include "ctdl_module.h"
  87
  88 #include "smtpqueue.h"
  89 #include "smtp_clienthandlers.h"
  90 #include "event_client.h"
  91
  92
  93 struct CitContext smtp_queue_CC;
  94 pthread_mutex_t ActiveQItemsLock;
  95 HashList *ActiveQItems  = NULL;
  96 HashList *QItemHandlers = NULL;
  97 int max_sessions_for_outbound_smtp = 500; /* how many sessions might be active till we stop adding more smtp jobs */
  98 int ndelay_count = 50; /* every n queued messages we will sleep... */
  99 int delay_msec = 5000; /* this many seconds. */
 100
 101 static const long MaxRetry = SMTP_RETRY_INTERVAL * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2;
 102 int MsgCount            = 0;
 103 int run_queue_now       = 0;    /* Set to 1 to ignore SMTP send retry times */
 104
 105 void RegisterQItemHandler(const char *Key, long Len, QItemHandler H)
 106 {
 107         QItemHandlerStruct *HS = (QItemHandlerStruct*)malloc(sizeof(QItemHandlerStruct));
 108         HS->H = H;
 109         Put(QItemHandlers, Key, Len, HS, NULL);
 110 }
 111
 112
 113
 114 void smtp_try_one_queue_entry(OneQueItem *MyQItem,
 115                               MailQEntry *MyQEntry,
 116                               StrBuf *MsgText,
 117 /* KeepMsgText allows us to use MsgText as ours. */
 118                               int KeepMsgText,
 119                               int MsgCount,
 120                               ParsedURL *RelayUrls);
 121
 122
 123 void smtp_evq_cleanup(void)
 124 {
 125
 126         pthread_mutex_lock(&ActiveQItemsLock);
 127         DeleteHash(&QItemHandlers);
 128         DeleteHash(&ActiveQItems);
 129         pthread_mutex_unlock(&ActiveQItemsLock);
 130         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 131 /*      citthread_mutex_destroy(&ActiveQItemsLock); TODO */
 132 }
 133
 134 int DecreaseQReference(OneQueItem *MyQItem)
 135 {
 136         int IDestructQueItem;
 137
 138         pthread_mutex_lock(&ActiveQItemsLock);
 139         MyQItem->ActiveDeliveries--;
 140         IDestructQueItem = MyQItem->ActiveDeliveries == 0;
 141         pthread_mutex_unlock(&ActiveQItemsLock);
 142         return IDestructQueItem;
 143 }
 144
 145 void DecreaseShutdownDeliveries(OneQueItem *MyQItem)
 146 {
 147         pthread_mutex_lock(&ActiveQItemsLock);
 148         MyQItem->NotYetShutdownDeliveries--;
 149         pthread_mutex_unlock(&ActiveQItemsLock);
 150 }
 151
 152 int GetShutdownDeliveries(OneQueItem *MyQItem)
 153 {
 154         int DestructNow;
 155
 156         pthread_mutex_lock(&ActiveQItemsLock);
 157         DestructNow = MyQItem->ActiveDeliveries == 0;
 158         pthread_mutex_unlock(&ActiveQItemsLock);
 159         return DestructNow;
 160 }
 161 void RemoveQItem(OneQueItem *MyQItem)
 162 {
 163         long len;
 164         const char* Key;
 165         void *VData;
 166         HashPos  *It;
 167
 168         pthread_mutex_lock(&ActiveQItemsLock);
 169         It = GetNewHashPos(ActiveQItems, 0);
 170         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 171                 DeleteEntryFromHash(ActiveQItems, It);
 172         else
 173         {
 174                 SMTPC_syslog(LOG_WARNING,
 175                              "unable to find QItem with ID[%ld]",
 176                              MyQItem->MessageID);
 177                 while (GetNextHashPos(ActiveQItems, It, &len, &Key, &VData))
 178                         SMTPC_syslog(LOG_WARNING,
 179                                      "have_: ID[%ld]",
 180                                      ((OneQueItem *)VData)->MessageID);
 181         }
 182         pthread_mutex_unlock(&ActiveQItemsLock);
 183         DeleteHashPos(&It);
 184 }
 185
 186
 187 void FreeMailQEntry(void *qv)
 188 {
 189         MailQEntry *Q = qv;
 190 /*
 191         SMTPC_syslog(LOG_DEBUG, "---------------%s--------------", __FUNCTION__);
 192         cit_backtrace();
 193 */
 194         FreeStrBuf(&Q->Recipient);
 195         FreeStrBuf(&Q->StatusMessage);
 196
 197         memset(Q, 0, sizeof(MailQEntry));
 198         free(Q);
 199 }
 200 void FreeQueItem(OneQueItem **Item)
 201 {
 202 /*
 203         SMTPC_syslog(LOG_DEBUG, "---------------%s--------------", __FUNCTION__);
 204         cit_backtrace();
 205 */
 206         DeleteHash(&(*Item)->MailQEntries);
 207         FreeStrBuf(&(*Item)->EnvelopeFrom);
 208         FreeStrBuf(&(*Item)->BounceTo);
 209         FreeStrBuf(&(*Item)->SenderRoom);
 210         FreeURL(&(*Item)->URL);
 211         memset(*Item, 0, sizeof(OneQueItem));
 212         free(*Item);
 213         Item = NULL;
 214 }
 215 void HFreeQueItem(void *Item)
 216 {
 217         FreeQueItem((OneQueItem**)&Item);
 218 }
 219
 220 /* inspect recipients with a status of:
 221  * - 0 (no delivery yet attempted)
 222  * - 3/4 (transient errors
 223  *        were experienced and it's time to try again)
 224  */
 225 int CheckQEntryActive(MailQEntry *ThisItem)
 226 {
 227         if ((ThisItem->Status == 0) ||
 228             (ThisItem->Status == 3) ||
 229             (ThisItem->Status == 4))
 230         {
 231                 return 1;
 232         }
 233         else
 234                 return 0;
 235 }
 236 int CheckQEntryIsBounce(MailQEntry *ThisItem)
 237 {
 238         if ((ThisItem->Status == 3) ||
 239             (ThisItem->Status == 4) ||
 240             (ThisItem->Status == 5))
 241         {
 242                 return 1;
 243         }
 244         else
 245                 return 0;
 246 }
 247
 248 int CountActiveQueueEntries(OneQueItem *MyQItem)
 249 {
 250         HashPos  *It;
 251         long len;
 252         long ActiveDeliveries;
 253         const char *Key;
 254         void *vQE;
 255
 256         ActiveDeliveries = 0;
 257         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 258         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 259         {
 260                 MailQEntry *ThisItem = vQE;
 261
 262                 if (CheckQEntryActive(ThisItem))
 263                 {
 264                         ActiveDeliveries++;
 265                         ThisItem->Active = 1;
 266                 }
 267                 else
 268                         ThisItem->Active = 0;
 269         }
 270         DeleteHashPos(&It);
 271         return ActiveDeliveries;
 272 }
 273
 274 OneQueItem *DeserializeQueueItem(StrBuf *RawQItem, long QueMsgID)
 275 {
 276         OneQueItem *Item;
 277         const char *pLine = NULL;
 278         StrBuf *Line;
 279         StrBuf *Token;
 280         void *v;
 281
 282         Item = (OneQueItem*)malloc(sizeof(OneQueItem));
 283         memset(Item, 0, sizeof(OneQueItem));
 284         Item->Retry = SMTP_RETRY_INTERVAL;
 285         Item->MessageID = -1;
 286         Item->QueMsgID = QueMsgID;
 287
 288         Token = NewStrBuf();
 289         Line = NewStrBufPlain(NULL, 128);
 290         while (pLine != StrBufNOTNULL) {
 291                 const char *pItemPart = NULL;
 292                 void *vHandler;
 293
 294                 StrBufExtract_NextToken(Line, RawQItem, &pLine, '\n');
 295                 if (StrLength(Line) == 0) continue;
 296                 StrBufExtract_NextToken(Token, Line, &pItemPart, '|');
 297                 if (GetHash(QItemHandlers, SKEY(Token), &vHandler))
 298                 {
 299                         QItemHandlerStruct *HS;
 300                         HS = (QItemHandlerStruct*) vHandler;
 301                         HS->H(Item, Line, &pItemPart);
 302                 }
 303         }
 304         FreeStrBuf(&Line);
 305         FreeStrBuf(&Token);
 306
 307         if (Item->Retry >= MaxRetry)
 308                 Item->FailNow = 1;
 309
 310         pthread_mutex_lock(&ActiveQItemsLock);
 311         if (GetHash(ActiveQItems,
 312                     LKEY(Item->MessageID),
 313                     &v))
 314         {
 315                 /* WHOOPS. somebody else is already working on this. */
 316                 pthread_mutex_unlock(&ActiveQItemsLock);
 317                 FreeQueItem(&Item);
 318                 return NULL;
 319         }
 320         else {
 321                 /* mark our claim on this. */
 322                 Put(ActiveQItems,
 323                     LKEY(Item->MessageID),
 324                     Item,
 325                     HFreeQueItem);
 326                 pthread_mutex_unlock(&ActiveQItemsLock);
 327         }
 328
 329         return Item;
 330 }
 331
 332 StrBuf *SerializeQueueItem(OneQueItem *MyQItem)
 333 {
 334         StrBuf *QMessage;
 335         HashPos  *It;
 336         const char *Key;
 337         long len;
 338         void *vQE;
 339
 340         QMessage = NewStrBufPlain(NULL, SIZ);
 341         StrBufPrintf(QMessage, "Content-type: %s\n", SPOOLMIME);
 342 //      "attempted|%ld\n"  "retry|%ld\n",, (long)time(NULL), (long)retry );
 343         StrBufAppendBufPlain(QMessage, HKEY("\nmsgid|"), 0);
 344         StrBufAppendPrintf(QMessage, "%ld", MyQItem->MessageID);
 345
 346         StrBufAppendBufPlain(QMessage, HKEY("\nsubmitted|"), 0);
 347         StrBufAppendPrintf(QMessage, "%ld", MyQItem->Submitted);
 348
 349         if (StrLength(MyQItem->BounceTo) > 0) {
 350                 StrBufAppendBufPlain(QMessage, HKEY("\nbounceto|"), 0);
 351                 StrBufAppendBuf(QMessage, MyQItem->BounceTo, 0);
 352         }
 353
 354         if (StrLength(MyQItem->EnvelopeFrom) > 0) {
 355                 StrBufAppendBufPlain(QMessage, HKEY("\nenvelope_from|"), 0);
 356                 StrBufAppendBuf(QMessage, MyQItem->EnvelopeFrom, 0);
 357         }
 358
 359         if (StrLength(MyQItem->SenderRoom) > 0) {
 360                 StrBufAppendBufPlain(QMessage, HKEY("\nsource_room|"), 0);
 361                 StrBufAppendBuf(QMessage, MyQItem->SenderRoom, 0);
 362         }
 363
 364         StrBufAppendBufPlain(QMessage, HKEY("\nretry|"), 0);
 365         StrBufAppendPrintf(QMessage, "%ld",
 366                            MyQItem->Retry);
 367
 368         StrBufAppendBufPlain(QMessage, HKEY("\nattempted|"), 0);
 369         StrBufAppendPrintf(QMessage, "%ld",
 370                            time(NULL) /*ctdl_ev_now()*/ + MyQItem->Retry);
 371
 372         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 373         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 374         {
 375                 MailQEntry *ThisItem = vQE;
 376
 377                 StrBufAppendBufPlain(QMessage, HKEY("\nremote|"), 0);
 378                 StrBufAppendBuf(QMessage, ThisItem->Recipient, 0);
 379                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 380                 StrBufAppendPrintf(QMessage, "%d", ThisItem->Status);
 381                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 382                 StrBufAppendBuf(QMessage, ThisItem->StatusMessage, 0);
 383         }
 384         DeleteHashPos(&It);
 385         StrBufAppendBufPlain(QMessage, HKEY("\n"), 0);
 386         return QMessage;
 387 }
 388
 389
 390
 391
 392
 393 void NewMailQEntry(OneQueItem *Item)
 394 {
 395         Item->Current = (MailQEntry*) malloc(sizeof(MailQEntry));
 396         memset(Item->Current, 0, sizeof(MailQEntry));
 397
 398         if (Item->MailQEntries == NULL)
 399                 Item->MailQEntries = NewHash(1, Flathash);
 400         /* alocate big buffer so we won't get problems reallocating later. */
 401         Item->Current->StatusMessage = NewStrBufPlain(NULL, SIZ);
 402         Item->Current->n = GetCount(Item->MailQEntries);
 403         Put(Item->MailQEntries,
 404             IKEY(Item->Current->n),
 405             Item->Current,
 406             FreeMailQEntry);
 407 }
 408
 409 void QItem_Handle_MsgID(OneQueItem *Item, StrBuf *Line, const char **Pos)
 410 {
 411         Item->MessageID = StrBufExtractNext_long(Line, Pos, '|');
 412 }
 413
 414 void QItem_Handle_EnvelopeFrom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 415 {
 416         if (Item->EnvelopeFrom == NULL)
 417                 Item->EnvelopeFrom = NewStrBufPlain(NULL, StrLength(Line));
 418         StrBufExtract_NextToken(Item->EnvelopeFrom, Line, Pos, '|');
 419 }
 420
 421 void QItem_Handle_BounceTo(OneQueItem *Item, StrBuf *Line, const char **Pos)
 422 {
 423         if (Item->BounceTo == NULL)
 424                 Item->BounceTo = NewStrBufPlain(NULL, StrLength(Line));
 425         StrBufExtract_NextToken(Item->BounceTo, Line, Pos, '|');
 426 }
 427
 428 void QItem_Handle_SenderRoom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 429 {
 430         if (Item->SenderRoom == NULL)
 431                 Item->SenderRoom = NewStrBufPlain(NULL, StrLength(Line));
 432         StrBufExtract_NextToken(Item->SenderRoom, Line, Pos, '|');
 433 }
 434
 435 void QItem_Handle_Recipient(OneQueItem *Item, StrBuf *Line, const char **Pos)
 436 {
 437         if (Item->Current == NULL)
 438                 NewMailQEntry(Item);
 439         if (Item->Current->Recipient == NULL)
 440                 Item->Current->Recipient=NewStrBufPlain(NULL, StrLength(Line));
 441         StrBufExtract_NextToken(Item->Current->Recipient, Line, Pos, '|');
 442         Item->Current->Status = StrBufExtractNext_int(Line, Pos, '|');
 443         StrBufExtract_NextToken(Item->Current->StatusMessage, Line, Pos, '|');
 444         Item->Current = NULL; // TODO: is this always right?
 445 }
 446
 447
 448 void QItem_Handle_retry(OneQueItem *Item, StrBuf *Line, const char **Pos)
 449 {
 450         Item->Retry =
 451                 StrBufExtractNext_int(Line, Pos, '|');
 452         if (Item->Retry == 0)
 453                 Item->Retry = SMTP_RETRY_INTERVAL;
 454         else
 455                 Item->Retry *= 2;
 456 }
 457
 458
 459 void QItem_Handle_Submitted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 460 {
 461         Item->Submitted = atol(*Pos);
 462
 463 }
 464
 465 void QItem_Handle_Attempted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 466 {
 467         Item->ReattemptWhen = StrBufExtractNext_int(Line, Pos, '|');
 468 }
 469
 470
 471
 472 /**
 473  * this one has to have the context for loading the message via the redirect buffer...
 474  */
 475 StrBuf *smtp_load_msg(OneQueItem *MyQItem, int n)
 476 {
 477         CitContext *CCC=CC;
 478         StrBuf *SendMsg;
 479
 480         CCC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
 481         CtdlOutputMsg(MyQItem->MessageID,
 482                       MT_RFC822, HEADERS_ALL,
 483                       0, 1, NULL,
 484                       (ESC_DOT|SUPPRESS_ENV_TO) );
 485
 486         SendMsg = CCC->redirect_buffer;
 487         CCC->redirect_buffer = NULL;
 488         if ((StrLength(SendMsg) > 0) &&
 489             ChrPtr(SendMsg)[StrLength(SendMsg) - 1] != '\n') {
 490                 SMTPC_syslog(LOG_WARNING,
 491                              "[%d] Possible problem: message did not "
 492                              "correctly terminate. (expecting 0x10, got 0x%02x)\n",
 493                              MsgCount, //yes uncool, but best choice here...
 494                              ChrPtr(SendMsg)[StrLength(SendMsg) - 1] );
 495                 StrBufAppendBufPlain(SendMsg, HKEY("\r\n"), 0);
 496         }
 497         return SendMsg;
 498 }
 499
 500
 501
 502 /*
 503  * smtp_do_bounce() is caled by smtp_do_procmsg() to scan a set of delivery
 504  * instructions for "5" codes (permanent fatal errors) and produce/deliver
 505  * a "bounce" message (delivery status notification).
 506  */
 507 void smtpq_do_bounce(OneQueItem *MyQItem, StrBuf *OMsgTxt)
 508 {
 509         static int seq = 0;
 510
 511         struct CtdlMessage *bmsg = NULL;
 512         StrBuf *boundary;
 513         StrBuf *Msg = NULL;
 514         StrBuf *BounceMB;
 515         struct recptypes *valid;
 516         time_t now;
 517
 518         HashPos *It;
 519         void *vQE;
 520         long len;
 521         const char *Key;
 522
 523         int first_attempt = 0;
 524         int successful_bounce = 0;
 525         int num_bounces = 0;
 526         int give_up = 0;
 527
 528         SMTPCM_syslog(LOG_DEBUG, "smtp_do_bounce() called\n");
 529
 530         if (MyQItem->SendBounceMail == 0)
 531                 return;
 532
 533         now = time (NULL); //ev_time();
 534
 535         if ( (now - MyQItem->Submitted) > SMTP_GIVE_UP ) {
 536                 give_up = 1;
 537         }
 538
 539         if (MyQItem->Retry == SMTP_RETRY_INTERVAL) {
 540                 first_attempt = 1;
 541         }
 542
 543         /*
 544          * Now go through the instructions checking for stuff.
 545          */
 546         Msg = NewStrBufPlain(NULL, 1024);
 547         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 548         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 549         {
 550                 MailQEntry *ThisItem = vQE;
 551                 if ((ThisItem->Active && (ThisItem->Status == 5)) || /* failed now? */
 552                     ((give_up == 1) && (ThisItem->Status != 2)) ||
 553                     ((first_attempt == 1) && (ThisItem->Status != 2)))
 554                         /* giving up after failed attempts... */
 555                 {
 556                         ++num_bounces;
 557
 558                         StrBufAppendBuf(Msg, ThisItem->Recipient, 0);
 559                         StrBufAppendBufPlain(Msg, HKEY(": "), 0);
 560                         StrBufAppendBuf(Msg, ThisItem->StatusMessage, 0);
 561                         StrBufAppendBufPlain(Msg, HKEY("\r\n"), 0);
 562                 }
 563         }
 564         DeleteHashPos(&It);
 565
 566         /* Deliver the bounce if there's anything worth mentioning */
 567         SMTPC_syslog(LOG_DEBUG, "num_bounces = %d\n", num_bounces);
 568
 569         if (num_bounces == 0) {
 570                 FreeStrBuf(&Msg);
 571                 return;
 572         }
 573
 574         boundary = NewStrBufPlain(HKEY("=_Citadel_Multipart_"));
 575         StrBufAppendPrintf(boundary,
 576                            "%s_%04x%04x",
 577                            config.c_fqdn,
 578                            getpid(),
 579                            ++seq);
 580
 581         /* Start building our bounce message; go shopping for memory first. */
 582         BounceMB = NewStrBufPlain(
 583                 NULL,
 584                 1024 + /* mime stuff.... */
 585                 StrLength(Msg) +  /* the bounce information... */
 586                 StrLength(OMsgTxt)); /* the original message */
 587         if (BounceMB == NULL) {
 588                 FreeStrBuf(&boundary);
 589                 SMTPCM_syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 590
 591                 return;
 592         }
 593
 594         bmsg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
 595         if (bmsg == NULL) {
 596                 FreeStrBuf(&boundary);
 597                 FreeStrBuf(&BounceMB);
 598                 SMTPCM_syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 599
 600                 return;
 601         }
 602         memset(bmsg, 0, sizeof(struct CtdlMessage));
 603
 604
 605         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: multipart/mixed; boundary=\""), 0);
 606         StrBufAppendBuf(BounceMB, boundary, 0);
 607         StrBufAppendBufPlain(BounceMB, HKEY("\"\r\n"), 0);
 608         StrBufAppendBufPlain(BounceMB, HKEY("MIME-Version: 1.0\r\n"), 0);
 609         StrBufAppendBufPlain(BounceMB, HKEY("X-Mailer: " CITADEL "\r\n"), 0);
 610         StrBufAppendBufPlain(BounceMB, HKEY("\r\nThis is a multipart message in MIME format.\r\n\r\n"), 0);
 611         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 612         StrBufAppendBuf(BounceMB, boundary, 0);
 613         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 614         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: text/plain\r\n\r\n"), 0);
 615
 616         if (give_up)
 617                 StrBufAppendBufPlain(
 618                         BounceMB,
 619                         HKEY(
 620                                 "A message you sent could not be delivered "
 621                                 "to some or all of its recipients\n"
 622                                 "due to prolonged unavailability "
 623                                 "of its destination(s).\n"
 624                                 "Giving up on the following addresses:\n\n"
 625                                 ), 0);
 626         else
 627                 StrBufAppendBufPlain(
 628                         BounceMB,
 629                         HKEY(
 630                                 "A message you sent could not be delivered "
 631                                 "to some or all of its recipients.\n"
 632                                 "The following addresses "
 633                                 "were undeliverable:\n\n"
 634                                 ), 0);
 635
 636         StrBufAppendBuf(BounceMB, Msg, 0);
 637         FreeStrBuf(&Msg);
 638
 639         if (StrLength(MyQItem->SenderRoom) > 0)
 640         {
 641                 StrBufAppendBufPlain(
 642                         BounceMB,
 643                         HKEY("The message was originaly posted in: "), 0);
 644                 StrBufAppendBuf(BounceMB, MyQItem->SenderRoom, 0);
 645                 StrBufAppendBufPlain(
 646                         BounceMB,
 647                         HKEY("\n"), 0);
 648         }
 649
 650         /* Attach the original message */
 651         StrBufAppendBufPlain(BounceMB, HKEY("\r\n--"), 0);
 652         StrBufAppendBuf(BounceMB, boundary, 0);
 653         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 654         StrBufAppendBufPlain(BounceMB,
 655                              HKEY("Content-type: message/rfc822\r\n"), 0);
 656         StrBufAppendBufPlain(BounceMB,
 657                              HKEY("Content-Transfer-Encoding: 7bit\r\n"), 0);
 658         StrBufAppendBufPlain(BounceMB,
 659                              HKEY("Content-Disposition: inline\r\n"), 0);
 660         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 661         StrBufAppendBuf(BounceMB, OMsgTxt, 0);
 662
 663         /* Close the multipart MIME scope */
 664         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 665         StrBufAppendBuf(BounceMB, boundary, 0);
 666         StrBufAppendBufPlain(BounceMB, HKEY("--\r\n"), 0);
 667
 668         bmsg->cm_magic = CTDLMESSAGE_MAGIC;
 669         bmsg->cm_anon_type = MES_NORMAL;
 670         bmsg->cm_format_type = FMT_RFC822;
 671
 672         bmsg->cm_fields['O'] = strdup(MAILROOM);
 673         bmsg->cm_fields['A'] = strdup("Citadel");
 674         bmsg->cm_fields['N'] = strdup(config.c_nodename);
 675         bmsg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
 676         bmsg->cm_fields['M'] = SmashStrBuf(&BounceMB);
 677
 678         /* First try the user who sent the message */
 679         if (StrLength(MyQItem->BounceTo) == 0) {
 680                 SMTPCM_syslog(LOG_ERR, "No bounce address specified\n");
 681         }
 682         else {
 683                 SMTPC_syslog(LOG_DEBUG, "bounce to user? <%s>\n",
 684                        ChrPtr(MyQItem->BounceTo));
 685         }
 686
 687         /* Can we deliver the bounce to the original sender? */
 688         valid = validate_recipients(ChrPtr(MyQItem->BounceTo), NULL, 0);
 689         if ((valid != NULL) && (valid->num_error == 0)) {
 690                 CtdlSubmitMsg(bmsg, valid, "", QP_EADDR);
 691                 successful_bounce = 1;
 692         }
 693
 694         /* If not, post it in the Aide> room */
 695         if (successful_bounce == 0) {
 696                 CtdlSubmitMsg(bmsg, NULL, config.c_aideroom, QP_EADDR);
 697         }
 698
 699         /* Free up the memory we used */
 700         free_recipients(valid);
 701         FreeStrBuf(&boundary);
 702         CtdlFreeMessage(bmsg);
 703         SMTPCM_syslog(LOG_DEBUG, "Done processing bounces\n");
 704 }
 705
 706 /*
 707  * smtp_do_procmsg()
 708  *
 709  * Called by smtp_do_queue() to handle an individual message.
 710  */
 711 void smtp_do_procmsg(long msgnum, void *userdata) {
 712         time_t now;
 713         int mynumsessions = num_sessions;
 714         struct CtdlMessage *msg = NULL;
 715         char *instr = NULL;
 716         StrBuf *PlainQItem;
 717         OneQueItem *MyQItem;
 718         char *pch;
 719         HashPos  *It;
 720         void *vQE;
 721         long len;
 722         const char *Key;
 723         int nRelays = 0;
 724         ParsedURL *RelayUrls = NULL;
 725         int HaveBuffers = 0;
 726         StrBuf *Msg =NULL;
 727
 728         if (mynumsessions > max_sessions_for_outbound_smtp) {
 729                 SMTPC_syslog(LOG_DEBUG,
 730                              "skipping because of num jobs %d > %d max_sessions_for_outbound_smtp",
 731                              mynumsessions,
 732                              max_sessions_for_outbound_smtp);
 733         }
 734
 735         SMTPC_syslog(LOG_DEBUG, "smtp_do_procmsg(%ld)\n", msgnum);
 736         ///strcpy(envelope_from, "");
 737
 738         msg = CtdlFetchMessage(msgnum, 1);
 739         if (msg == NULL) {
 740                 SMTPC_syslog(LOG_ERR, "tried %ld but no such message!\n",
 741                        msgnum);
 742                 return;
 743         }
 744
 745         pch = instr = msg->cm_fields['M'];
 746
 747         /* Strip out the headers (no not amd any other non-instruction) line */
 748         while (pch != NULL) {
 749                 pch = strchr(pch, '\n');
 750                 if ((pch != NULL) && (*(pch + 1) == '\n')) {
 751                         instr = pch + 2;
 752                         pch = NULL;
 753                 }
 754         }
 755         PlainQItem = NewStrBufPlain(instr, -1);
 756         CtdlFreeMessage(msg);
 757         MyQItem = DeserializeQueueItem(PlainQItem, msgnum);
 758         FreeStrBuf(&PlainQItem);
 759
 760         if (MyQItem == NULL) {
 761                 SMTPC_syslog(LOG_ERR,
 762                              "Msg No %ld: already in progress!\n",
 763                              msgnum);
 764                 return; /* s.b. else is already processing... */
 765         }
 766
 767         /*
 768          * Postpone delivery if we've already tried recently.
 769          */
 770         now = time(NULL);
 771         if ((MyQItem->ReattemptWhen != 0) &&
 772             (now < MyQItem->ReattemptWhen) &&
 773             (run_queue_now == 0))
 774         {
 775                 SMTPC_syslog(LOG_DEBUG,
 776                              "Retry time not yet reached. %ld seconds left.",
 777                              MyQItem->ReattemptWhen - now);
 778
 779                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 780                 pthread_mutex_lock(&ActiveQItemsLock);
 781                 {
 782                         if (GetHashPosFromKey(ActiveQItems,
 783                                               LKEY(MyQItem->MessageID),
 784                                               It))
 785                         {
 786                                 DeleteEntryFromHash(ActiveQItems, It);
 787                         }
 788                 }
 789                 pthread_mutex_unlock(&ActiveQItemsLock);
 790                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 791                 DeleteHashPos(&It);
 792                 return;
 793         }
 794
 795         /*
 796          * Bail out if there's no actual message associated with this
 797          */
 798         if (MyQItem->MessageID < 0L) {
 799                 SMTPCM_syslog(LOG_ERR, "no 'msgid' directive found!\n");
 800                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 801                 pthread_mutex_lock(&ActiveQItemsLock);
 802                 {
 803                         if (GetHashPosFromKey(ActiveQItems,
 804                                               LKEY(MyQItem->MessageID),
 805                                               It))
 806                         {
 807                                 DeleteEntryFromHash(ActiveQItems, It);
 808                         }
 809                 }
 810                 pthread_mutex_unlock(&ActiveQItemsLock);
 811                 DeleteHashPos(&It);
 812                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 813                 return;
 814         }
 815
 816         {
 817                 char mxbuf[SIZ];
 818                 ParsedURL **Url = &MyQItem->URL;
 819                 nRelays = get_hosts(mxbuf, "smarthost");
 820                 if (nRelays > 0) {
 821                         StrBuf *All;
 822                         StrBuf *One;
 823                         const char *Pos = NULL;
 824                         All = NewStrBufPlain(mxbuf, -1);
 825                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 826
 827                         while ((Pos != StrBufNOTNULL) &&
 828                                ((Pos == NULL) ||
 829                                 !IsEmptyStr(Pos)))
 830                         {
 831                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 832                                 if (!ParseURL(Url, One, 25)) {
 833                                         SMTPC_syslog(LOG_DEBUG,
 834                                                      "Failed to parse: %s\n",
 835                                                      ChrPtr(One));
 836                                 }
 837                                 else {
 838                                         ///if (!Url->IsIP)) // todo dupe me fork ipv6
 839                                         Url = &(*Url)->Next;
 840                                 }
 841                         }
 842                         FreeStrBuf(&All);
 843                         FreeStrBuf(&One);
 844                 }
 845
 846                 Url = &MyQItem->FallBackHost;
 847                 nRelays = get_hosts(mxbuf, "fallbackhost");
 848                 if (nRelays > 0) {
 849                         StrBuf *All;
 850                         StrBuf *One;
 851                         const char *Pos = NULL;
 852                         All = NewStrBufPlain(mxbuf, -1);
 853                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 854
 855                         while ((Pos != StrBufNOTNULL) &&
 856                                ((Pos == NULL) ||
 857                                 !IsEmptyStr(Pos)))
 858                         {
 859                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 860                                 if (!ParseURL(Url, One, 25)) {
 861                                         SMTPC_syslog(LOG_DEBUG,
 862                                                      "Failed to parse: %s\n",
 863                                                      ChrPtr(One));
 864                                 }
 865                                 else
 866                                         Url = &(*Url)->Next;
 867                         }
 868                         FreeStrBuf(&All);
 869                         FreeStrBuf(&One);
 870                 }
 871         }
 872
 873         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 874         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 875         {
 876                 MailQEntry *ThisItem = vQE;
 877                 SMTPC_syslog(LOG_DEBUG, "SMTP Queue: Task: <%s> %d\n",
 878                              ChrPtr(ThisItem->Recipient),
 879                              ThisItem->Active);
 880         }
 881         DeleteHashPos(&It);
 882
 883         MyQItem->NotYetShutdownDeliveries =
 884         MyQItem->ActiveDeliveries = CountActiveQueueEntries(MyQItem);
 885
 886         /* failsafe against overload:
 887          * will we exceed the limit set?
 888          */
 889         if ((MyQItem->ActiveDeliveries + mynumsessions > max_sessions_for_outbound_smtp) &&
 890             /* if yes, did we reach more than half of the quota? */
 891             ((mynumsessions * 2) > max_sessions_for_outbound_smtp) &&
 892             /* if... would we ever fit into half of the quota?? */
 893             (((MyQItem->ActiveDeliveries * 2)  < max_sessions_for_outbound_smtp)))
 894         {
 895                 /* abort delivery for another time. */
 896                 SMTPC_syslog(LOG_DEBUG,
 897                              "SMTP Queue: skipping because of num jobs %d + %ld > %d max_sessions_for_outbound_smtp",
 898                              mynumsessions,
 899                              MyQItem->ActiveDeliveries,
 900                              max_sessions_for_outbound_smtp);
 901
 902                 FreeQueItem(&MyQItem);
 903
 904                 return;
 905         }
 906
 907
 908         if (MyQItem->ActiveDeliveries > 0)
 909         {
 910                 int nActivated = 0;
 911                 int n = MsgCount++;
 912                 int m = MyQItem->ActiveDeliveries;
 913                 int i = 1;
 914                 Msg = smtp_load_msg(MyQItem, n);
 915                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 916                 while ((i <= m) &&
 917                        (GetNextHashPos(MyQItem->MailQEntries,
 918                                        It, &len, &Key, &vQE)))
 919                 {
 920                         MailQEntry *ThisItem = vQE;
 921
 922                         if (ThisItem->Active == 1)
 923                         {
 924                                 int KeepBuffers = (i == m);
 925
 926                                 nActivated++;
 927                                 if (nActivated % ndelay_count == 0)
 928                                         usleep(delay_msec);
 929
 930                                 if (i > 1) n = MsgCount++;
 931                                 syslog(LOG_DEBUG,
 932                                        "SMTPC: Trying <%ld> <%s> %d / %d \n",
 933                                        MyQItem->MessageID,
 934                                        ChrPtr(ThisItem->Recipient),
 935                                        i,
 936                                        m);
 937                                 (*((int*) userdata)) ++;
 938                                 smtp_try_one_queue_entry(MyQItem,
 939                                                          ThisItem,
 940                                                          Msg,
 941                                                          KeepBuffers,
 942                                                          n,
 943                                                          RelayUrls);
 944
 945                                 if (KeepBuffers) HaveBuffers = 1;
 946
 947                                 i++;
 948                         }
 949                 }
 950                 DeleteHashPos(&It);
 951         }
 952         else
 953         {
 954                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 955                 pthread_mutex_lock(&ActiveQItemsLock);
 956                 {
 957                         if (GetHashPosFromKey(ActiveQItems,
 958                                               LKEY(MyQItem->MessageID),
 959                                               It))
 960                         {
 961                                 DeleteEntryFromHash(ActiveQItems, It);
 962                         }
 963                         else
 964                         {
 965                                 long len;
 966                                 const char* Key;
 967                                 void *VData;
 968
 969                                 SMTPC_syslog(LOG_WARNING,
 970                                              "unable to find QItem with ID[%ld]",
 971                                              MyQItem->MessageID);
 972                                 while (GetNextHashPos(ActiveQItems,
 973                                                       It,
 974                                                       &len,
 975                                                       &Key,
 976                                                       &VData))
 977                                 {
 978                                         SMTPC_syslog(LOG_WARNING,
 979                                                      "have: ID[%ld]",
 980                                                      ((OneQueItem *)VData)->MessageID);
 981                                 }
 982                         }
 983
 984                 }
 985                 pthread_mutex_unlock(&ActiveQItemsLock);
 986                 DeleteHashPos(&It);
 987                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 988
 989 // TODO: bounce & delete?
 990
 991         }
 992         if (!HaveBuffers) {
 993                 FreeStrBuf (&Msg);
 994 // TODO : free RelayUrls
 995         }
 996 }
 997
 998
 999
1000 /*
1001  * smtp_queue_thread()
1002  *
1003  * Run through the queue sending out messages.
1004  */
1005 void smtp_do_queue(void) {
1006         static int is_running = 0;
1007         int num_processed = 0;
1008         int num_activated = 0;
1009
1010         if (is_running)
1011                 return; /* Concurrency check - only one can run */
1012         is_running = 1;
1013
1014         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
1015         SMTPCM_syslog(LOG_INFO, "processing outbound queue");
1016
1017         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
1018                 SMTPC_syslog(LOG_ERR, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
1019         }
1020         else {
1021                 num_processed = CtdlForEachMessage(MSGS_ALL,
1022                                                    0L,
1023                                                    NULL,
1024                                                    SPOOLMIME,
1025                                                    NULL,
1026                                                    smtp_do_procmsg,
1027                                                    &num_activated);
1028         }
1029         SMTPC_syslog(LOG_INFO,
1030                      "queue run completed; %d messages processed %d activated",
1031                      num_processed, num_activated);
1032
1033         run_queue_now = 0;
1034         is_running = 0;
1035 }
1036
1037
1038
1039 /*
1040  * Initialize the SMTP outbound queue
1041  */
1042 void smtp_init_spoolout(void) {
1043         struct ctdlroom qrbuf;
1044
1045         /*
1046          * Create the room.  This will silently fail if the room already
1047          * exists, and that's perfectly ok, because we want it to exist.
1048          */
1049         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
1050
1051         /*
1052          * Make sure it's set to be a "system room" so it doesn't show up
1053          * in the <K>nown rooms list for Aides.
1054          */
1055         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
1056                 qrbuf.QRflags2 |= QR2_SYSTEM;
1057                 CtdlPutRoomLock(&qrbuf);
1058         }
1059 }
1060
1061
1062
1063
1064 /*****************************************************************************/
1065 /*                          SMTP UTILITY COMMANDS                            */
1066 /*****************************************************************************/
1067
1068 void cmd_smtp(char *argbuf) {
1069         char cmd[64];
1070         char node[256];
1071         char buf[1024];
1072         int i;
1073         int num_mxhosts;
1074
1075         if (CtdlAccessCheck(ac_aide)) return;
1076
1077         extract_token(cmd, argbuf, 0, '|', sizeof cmd);
1078
1079         if (!strcasecmp(cmd, "mx")) {
1080                 extract_token(node, argbuf, 1, '|', sizeof node);
1081                 num_mxhosts = getmx(buf, node);
1082                 cprintf("%d %d MX hosts listed for %s\n",
1083                         LISTING_FOLLOWS, num_mxhosts, node);
1084                 for (i=0; i<num_mxhosts; ++i) {
1085                         extract_token(node, buf, i, '|', sizeof node);
1086                         cprintf("%s\n", node);
1087                 }
1088                 cprintf("000\n");
1089                 return;
1090         }
1091
1092         else if (!strcasecmp(cmd, "runqueue")) {
1093                 run_queue_now = 1;
1094                 cprintf("%d All outbound SMTP will be retried now.\n", CIT_OK);
1095                 return;
1096         }
1097
1098         else {
1099                 cprintf("%d Invalid command.\n", ERROR + ILLEGAL_VALUE);
1100         }
1101
1102 }
1103
1104
1105 CTDL_MODULE_INIT(smtp_queu)
1106 {
1107         char *pstr;
1108
1109         if (!threading)
1110         {
1111                 pstr = getenv("CITSERVER_n_session_max");
1112                 if ((pstr != NULL) && (*pstr != '\0'))
1113                         max_sessions_for_outbound_smtp = atol(pstr); /* how many sessions might be active till we stop adding more smtp jobs */
1114
1115                 pstr = getenv("CITSERVER_smtp_n_delay_count");
1116                 if ((pstr != NULL) && (*pstr != '\0'))
1117                         ndelay_count = atol(pstr); /* every n queued messages we will sleep... */
1118
1119                 pstr = getenv("CITSERVER_smtp_delay");
1120                 if ((pstr != NULL) && (*pstr != '\0'))
1121                         delay_msec = atol(pstr) * 1000; /* this many seconds. */
1122
1123
1124
1125
1126                 CtdlFillSystemContext(&smtp_queue_CC, "SMTP_Send");
1127                 ActiveQItems = NewHash(1, lFlathash);
1128                 pthread_mutex_init(&ActiveQItemsLock, NULL);
1129
1130                 QItemHandlers = NewHash(0, NULL);
1131
1132                 RegisterQItemHandler(HKEY("msgid"),             QItem_Handle_MsgID);
1133                 RegisterQItemHandler(HKEY("envelope_from"),     QItem_Handle_EnvelopeFrom);
1134                 RegisterQItemHandler(HKEY("retry"),             QItem_Handle_retry);
1135                 RegisterQItemHandler(HKEY("attempted"),         QItem_Handle_Attempted);
1136                 RegisterQItemHandler(HKEY("remote"),            QItem_Handle_Recipient);
1137                 RegisterQItemHandler(HKEY("bounceto"),          QItem_Handle_BounceTo);
1138                 RegisterQItemHandler(HKEY("source_room"),       QItem_Handle_SenderRoom);
1139                 RegisterQItemHandler(HKEY("submitted"),         QItem_Handle_Submitted);
1140
1141                 smtp_init_spoolout();
1142
1143                 CtdlRegisterEVCleanupHook(smtp_evq_cleanup);
1144
1145                 CtdlRegisterProtoHook(cmd_smtp, "SMTP", "SMTP utility commands");
1146                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER);
1147         }
1148
1149         /* return our Subversion id for the Log */
1150         return "smtpeventclient";
1151 }