citadel/modules/smtp/serv_smtpqueue.c

   1 /*
   2  * This module is an SMTP and ESMTP implementation for the Citadel system.
   3  * It is compliant with all of the following:
   4  *
   5  * RFC  821 - Simple Mail Transfer Protocol
   6  * RFC  876 - Survey of SMTP Implementations
   7  * RFC 1047 - Duplicate messages and SMTP
   8  * RFC 1652 - 8 bit MIME
   9  * RFC 1869 - Extended Simple Mail Transfer Protocol
  10  * RFC 1870 - SMTP Service Extension for Message Size Declaration
  11  * RFC 2033 - Local Mail Transfer Protocol
  12  * RFC 2197 - SMTP Service Extension for Command Pipelining
  13  * RFC 2476 - Message Submission
  14  * RFC 2487 - SMTP Service Extension for Secure SMTP over TLS
  15  * RFC 2554 - SMTP Service Extension for Authentication
  16  * RFC 2821 - Simple Mail Transfer Protocol
  17  * RFC 2822 - Internet Message Format
  18  * RFC 2920 - SMTP Service Extension for Command Pipelining
  19  *
  20  * The VRFY and EXPN commands have been removed from this implementation
  21  * because nobody uses these commands anymore, except for spammers.
  22  *
  23  * Copyright (c) 1998-2012 by the citadel.org team
  24  *
  25  *  This program is open source software; you can redistribute it and/or modify
  26  *  it under the terms of the GNU General Public License version 3.
  27  *
  28  *
  29  *
  30  *  This program is distributed in the hope that it will be useful,
  31  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  32  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  33  *  GNU General Public License for more details.
  34  *
  35  *
  36  *
  37  *
  38  */
  39
  40 #include "sysdep.h"
  41 #include <stdlib.h>
  42 #include <unistd.h>
  43 #include <stdio.h>
  44 #include <termios.h>
  45 #include <fcntl.h>
  46 #include <signal.h>
  47 #include <pwd.h>
  48 #include <errno.h>
  49 #include <sys/types.h>
  50 #include <syslog.h>
  51
  52 #if TIME_WITH_SYS_TIME
  53 # include <sys/time.h>
  54 # include <time.h>
  55 #else
  56 # if HAVE_SYS_TIME_H
  57 #  include <sys/time.h>
  58 # else
  59 #  include <time.h>
  60 # endif
  61 #endif
  62 #include <sys/wait.h>
  63 #include <ctype.h>
  64 #include <string.h>
  65 #include <limits.h>
  66 #include <sys/socket.h>
  67 #include <netinet/in.h>
  68 #include <arpa/inet.h>
  69 #include <libcitadel.h>
  70 #include "citadel.h"
  71 #include "server.h"
  72 #include "citserver.h"
  73 #include "support.h"
  74 #include "config.h"
  75 #include "control.h"
  76 #include "user_ops.h"
  77 #include "database.h"
  78 #include "msgbase.h"
  79 #include "internet_addressing.h"
  80 #include "genstamp.h"
  81 #include "domain.h"
  82 #include "clientsocket.h"
  83 #include "locate_host.h"
  84 #include "citadel_dirs.h"
  85
  86 #include "ctdl_module.h"
  87
  88 #include "smtpqueue.h"
  89 #include "event_client.h"
  90
  91
  92 struct CitContext smtp_queue_CC;
  93 pthread_mutex_t ActiveQItemsLock;
  94 HashList *ActiveQItems  = NULL;
  95 HashList *QItemHandlers = NULL;
  96 int max_sessions_for_outbound_smtp = 500; /* how many sessions might be active till we stop adding more smtp jobs */
  97 int ndelay_count = 50; /* every n queued messages we will sleep... */
  98 int delay_msec = 5000; /* this many seconds. */
  99
 100 static const long MaxRetry = SMTP_RETRY_INTERVAL * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2;
 101 int MsgCount            = 0;
 102 int run_queue_now       = 0;    /* Set to 1 to ignore SMTP send retry times */
 103
 104 void RegisterQItemHandler(const char *Key, long Len, QItemHandler H)
 105 {
 106         QItemHandlerStruct *HS = (QItemHandlerStruct*)malloc(sizeof(QItemHandlerStruct));
 107         HS->H = H;
 108         Put(QItemHandlers, Key, Len, HS, NULL);
 109 }
 110
 111
 112
 113 void smtp_try_one_queue_entry(OneQueItem *MyQItem,
 114                               MailQEntry *MyQEntry,
 115                               StrBuf *MsgText,
 116 /* KeepMsgText allows us to use MsgText as ours. */
 117                               int KeepMsgText,
 118                               int MsgCount,
 119                               ParsedURL *RelayUrls);
 120
 121
 122 void smtp_evq_cleanup(void)
 123 {
 124
 125         pthread_mutex_lock(&ActiveQItemsLock);
 126         DeleteHash(&QItemHandlers);
 127         DeleteHash(&ActiveQItems);
 128         pthread_mutex_unlock(&ActiveQItemsLock);
 129         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 130 /*      citthread_mutex_destroy(&ActiveQItemsLock); TODO */
 131 }
 132
 133 int DecreaseQReference(OneQueItem *MyQItem)
 134 {
 135         int IDestructQueItem;
 136
 137         pthread_mutex_lock(&ActiveQItemsLock);
 138         MyQItem->ActiveDeliveries--;
 139         IDestructQueItem = MyQItem->ActiveDeliveries == 0;
 140         pthread_mutex_unlock(&ActiveQItemsLock);
 141         return IDestructQueItem;
 142 }
 143
 144 void DecreaseShutdownDeliveries(OneQueItem *MyQItem)
 145 {
 146         pthread_mutex_lock(&ActiveQItemsLock);
 147         MyQItem->NotYetShutdownDeliveries--;
 148         pthread_mutex_unlock(&ActiveQItemsLock);
 149 }
 150
 151 int GetShutdownDeliveries(OneQueItem *MyQItem)
 152 {
 153         int DestructNow;
 154
 155         pthread_mutex_lock(&ActiveQItemsLock);
 156         DestructNow = MyQItem->ActiveDeliveries == 0;
 157         pthread_mutex_unlock(&ActiveQItemsLock);
 158         return DestructNow;
 159 }
 160 void RemoveQItem(OneQueItem *MyQItem)
 161 {
 162         long len;
 163         const char* Key;
 164         void *VData;
 165         HashPos  *It;
 166
 167         pthread_mutex_lock(&ActiveQItemsLock);
 168         It = GetNewHashPos(ActiveQItems, 0);
 169         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 170                 DeleteEntryFromHash(ActiveQItems, It);
 171         else
 172         {
 173                 syslog(LOG_WARNING,
 174                        "SMTP cleanup: unable to find QItem with ID[%ld]",
 175                        MyQItem->MessageID);
 176                 while (GetNextHashPos(ActiveQItems, It, &len, &Key, &VData))
 177                         syslog(LOG_WARNING,
 178                                "SMTP cleanup: have_: ID[%ld]",
 179                                ((OneQueItem *)VData)->MessageID);
 180         }
 181         pthread_mutex_unlock(&ActiveQItemsLock);
 182         DeleteHashPos(&It);
 183 }
 184
 185
 186 void FreeMailQEntry(void *qv)
 187 {
 188         MailQEntry *Q = qv;
 189 /*
 190         syslog(LOG_DEBUG, "---------------%s--------------", __FUNCTION__);
 191         cit_backtrace();
 192 */
 193         FreeStrBuf(&Q->Recipient);
 194         FreeStrBuf(&Q->StatusMessage);
 195
 196         memset(Q, 0, sizeof(MailQEntry));
 197         free(Q);
 198 }
 199 void FreeQueItem(OneQueItem **Item)
 200 {
 201 /*
 202         syslog(LOG_DEBUG, "---------------%s--------------", __FUNCTION__);
 203         cit_backtrace();
 204 */
 205         DeleteHash(&(*Item)->MailQEntries);
 206         FreeStrBuf(&(*Item)->EnvelopeFrom);
 207         FreeStrBuf(&(*Item)->BounceTo);
 208         FreeStrBuf(&(*Item)->SenderRoom);
 209         FreeURL(&(*Item)->URL);
 210         memset(*Item, 0, sizeof(OneQueItem));
 211         free(*Item);
 212         Item = NULL;
 213 }
 214 void HFreeQueItem(void *Item)
 215 {
 216         FreeQueItem((OneQueItem**)&Item);
 217 }
 218
 219 /* inspect recipients with a status of:
 220  * - 0 (no delivery yet attempted)
 221  * - 3/4 (transient errors
 222  *        were experienced and it's time to try again)
 223  */
 224 int CheckQEntryActive(MailQEntry *ThisItem)
 225 {
 226         if ((ThisItem->Status == 0) ||
 227             (ThisItem->Status == 3) ||
 228             (ThisItem->Status == 4))
 229         {
 230                 return 1;
 231         }
 232         else
 233                 return 0;
 234 }
 235 int CheckQEntryIsBounce(MailQEntry *ThisItem)
 236 {
 237         if ((ThisItem->Status == 3) ||
 238             (ThisItem->Status == 4) ||
 239             (ThisItem->Status == 5))
 240         {
 241                 return 1;
 242         }
 243         else
 244                 return 0;
 245 }
 246
 247 int CountActiveQueueEntries(OneQueItem *MyQItem)
 248 {
 249         HashPos  *It;
 250         long len;
 251         long ActiveDeliveries;
 252         const char *Key;
 253         void *vQE;
 254
 255         ActiveDeliveries = 0;
 256         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 257         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 258         {
 259                 MailQEntry *ThisItem = vQE;
 260
 261                 if (CheckQEntryActive(ThisItem))
 262                 {
 263                         ActiveDeliveries++;
 264                         ThisItem->Active = 1;
 265                 }
 266                 else
 267                         ThisItem->Active = 0;
 268         }
 269         DeleteHashPos(&It);
 270         return ActiveDeliveries;
 271 }
 272
 273 OneQueItem *DeserializeQueueItem(StrBuf *RawQItem, long QueMsgID)
 274 {
 275         OneQueItem *Item;
 276         const char *pLine = NULL;
 277         StrBuf *Line;
 278         StrBuf *Token;
 279         void *v;
 280
 281         Item = (OneQueItem*)malloc(sizeof(OneQueItem));
 282         memset(Item, 0, sizeof(OneQueItem));
 283         Item->Retry = SMTP_RETRY_INTERVAL;
 284         Item->MessageID = -1;
 285         Item->QueMsgID = QueMsgID;
 286
 287         Token = NewStrBuf();
 288         Line = NewStrBufPlain(NULL, 128);
 289         while (pLine != StrBufNOTNULL) {
 290                 const char *pItemPart = NULL;
 291                 void *vHandler;
 292
 293                 StrBufExtract_NextToken(Line, RawQItem, &pLine, '\n');
 294                 if (StrLength(Line) == 0) continue;
 295                 StrBufExtract_NextToken(Token, Line, &pItemPart, '|');
 296                 if (GetHash(QItemHandlers, SKEY(Token), &vHandler))
 297                 {
 298                         QItemHandlerStruct *HS;
 299                         HS = (QItemHandlerStruct*) vHandler;
 300                         HS->H(Item, Line, &pItemPart);
 301                 }
 302         }
 303         FreeStrBuf(&Line);
 304         FreeStrBuf(&Token);
 305
 306         if (Item->Retry >= MaxRetry)
 307                 Item->FailNow = 1;
 308
 309         pthread_mutex_lock(&ActiveQItemsLock);
 310         if (GetHash(ActiveQItems,
 311                     LKEY(Item->MessageID),
 312                     &v))
 313         {
 314                 /* WHOOPS. somebody else is already working on this. */
 315                 pthread_mutex_unlock(&ActiveQItemsLock);
 316                 FreeQueItem(&Item);
 317                 return NULL;
 318         }
 319         else {
 320                 /* mark our claim on this. */
 321                 Put(ActiveQItems,
 322                     LKEY(Item->MessageID),
 323                     Item,
 324                     HFreeQueItem);
 325                 pthread_mutex_unlock(&ActiveQItemsLock);
 326         }
 327
 328         return Item;
 329 }
 330
 331 StrBuf *SerializeQueueItem(OneQueItem *MyQItem)
 332 {
 333         StrBuf *QMessage;
 334         HashPos  *It;
 335         const char *Key;
 336         long len;
 337         void *vQE;
 338
 339         QMessage = NewStrBufPlain(NULL, SIZ);
 340         StrBufPrintf(QMessage, "Content-type: %s\n", SPOOLMIME);
 341 //      "attempted|%ld\n"  "retry|%ld\n",, (long)time(NULL), (long)retry );
 342         StrBufAppendBufPlain(QMessage, HKEY("\nmsgid|"), 0);
 343         StrBufAppendPrintf(QMessage, "%ld", MyQItem->MessageID);
 344
 345         StrBufAppendBufPlain(QMessage, HKEY("\nsubmitted|"), 0);
 346         StrBufAppendPrintf(QMessage, "%ld", MyQItem->Submitted);
 347
 348         if (StrLength(MyQItem->BounceTo) > 0) {
 349                 StrBufAppendBufPlain(QMessage, HKEY("\nbounceto|"), 0);
 350                 StrBufAppendBuf(QMessage, MyQItem->BounceTo, 0);
 351         }
 352
 353         if (StrLength(MyQItem->EnvelopeFrom) > 0) {
 354                 StrBufAppendBufPlain(QMessage, HKEY("\nenvelope_from|"), 0);
 355                 StrBufAppendBuf(QMessage, MyQItem->EnvelopeFrom, 0);
 356         }
 357
 358         if (StrLength(MyQItem->SenderRoom) > 0) {
 359                 StrBufAppendBufPlain(QMessage, HKEY("\nsource_room|"), 0);
 360                 StrBufAppendBuf(QMessage, MyQItem->SenderRoom, 0);
 361         }
 362
 363         StrBufAppendBufPlain(QMessage, HKEY("\nretry|"), 0);
 364         StrBufAppendPrintf(QMessage, "%ld",
 365                            MyQItem->Retry);
 366
 367         StrBufAppendBufPlain(QMessage, HKEY("\nattempted|"), 0);
 368         StrBufAppendPrintf(QMessage, "%ld",
 369                            time(NULL) /*ctdl_ev_now()*/ + MyQItem->Retry);
 370
 371         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 372         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 373         {
 374                 MailQEntry *ThisItem = vQE;
 375
 376                 StrBufAppendBufPlain(QMessage, HKEY("\nremote|"), 0);
 377                 StrBufAppendBuf(QMessage, ThisItem->Recipient, 0);
 378                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 379                 StrBufAppendPrintf(QMessage, "%d", ThisItem->Status);
 380                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 381                 StrBufAppendBuf(QMessage, ThisItem->StatusMessage, 0);
 382         }
 383         DeleteHashPos(&It);
 384         StrBufAppendBufPlain(QMessage, HKEY("\n"), 0);
 385         return QMessage;
 386 }
 387
 388
 389
 390
 391
 392 void NewMailQEntry(OneQueItem *Item)
 393 {
 394         Item->Current = (MailQEntry*) malloc(sizeof(MailQEntry));
 395         memset(Item->Current, 0, sizeof(MailQEntry));
 396
 397         if (Item->MailQEntries == NULL)
 398                 Item->MailQEntries = NewHash(1, Flathash);
 399         Item->Current->StatusMessage = NewStrBuf();
 400         Item->Current->n = GetCount(Item->MailQEntries);
 401         Put(Item->MailQEntries,
 402             IKEY(Item->Current->n),
 403             Item->Current,
 404             FreeMailQEntry);
 405 }
 406
 407 void QItem_Handle_MsgID(OneQueItem *Item, StrBuf *Line, const char **Pos)
 408 {
 409         Item->MessageID = StrBufExtractNext_long(Line, Pos, '|');
 410 }
 411
 412 void QItem_Handle_EnvelopeFrom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 413 {
 414         if (Item->EnvelopeFrom == NULL)
 415                 Item->EnvelopeFrom = NewStrBufPlain(NULL, StrLength(Line));
 416         StrBufExtract_NextToken(Item->EnvelopeFrom, Line, Pos, '|');
 417 }
 418
 419 void QItem_Handle_BounceTo(OneQueItem *Item, StrBuf *Line, const char **Pos)
 420 {
 421         if (Item->BounceTo == NULL)
 422                 Item->BounceTo = NewStrBufPlain(NULL, StrLength(Line));
 423         StrBufExtract_NextToken(Item->BounceTo, Line, Pos, '|');
 424 }
 425
 426 void QItem_Handle_SenderRoom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 427 {
 428         if (Item->SenderRoom == NULL)
 429                 Item->SenderRoom = NewStrBufPlain(NULL, StrLength(Line));
 430         StrBufExtract_NextToken(Item->SenderRoom, Line, Pos, '|');
 431 }
 432
 433 void QItem_Handle_Recipient(OneQueItem *Item, StrBuf *Line, const char **Pos)
 434 {
 435         if (Item->Current == NULL)
 436                 NewMailQEntry(Item);
 437         if (Item->Current->Recipient == NULL)
 438                 Item->Current->Recipient=NewStrBufPlain(NULL, StrLength(Line));
 439         StrBufExtract_NextToken(Item->Current->Recipient, Line, Pos, '|');
 440         Item->Current->Status = StrBufExtractNext_int(Line, Pos, '|');
 441         StrBufExtract_NextToken(Item->Current->StatusMessage, Line, Pos, '|');
 442         Item->Current = NULL; // TODO: is this always right?
 443 }
 444
 445
 446 void QItem_Handle_retry(OneQueItem *Item, StrBuf *Line, const char **Pos)
 447 {
 448         Item->Retry =
 449                 StrBufExtractNext_int(Line, Pos, '|');
 450         if (Item->Retry == 0)
 451                 Item->Retry = SMTP_RETRY_INTERVAL;
 452         else
 453                 Item->Retry *= 2;
 454 }
 455
 456
 457 void QItem_Handle_Submitted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 458 {
 459         Item->Submitted = atol(*Pos);
 460
 461 }
 462
 463 void QItem_Handle_Attempted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 464 {
 465         Item->ReattemptWhen = StrBufExtractNext_int(Line, Pos, '|');
 466 }
 467
 468
 469
 470 /**
 471  * this one has to have the context for loading the message via the redirect buffer...
 472  */
 473 StrBuf *smtp_load_msg(OneQueItem *MyQItem, int n)
 474 {
 475         CitContext *CCC=CC;
 476         StrBuf *SendMsg;
 477
 478         CCC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
 479         CtdlOutputMsg(MyQItem->MessageID,
 480                       MT_RFC822, HEADERS_ALL,
 481                       0, 1, NULL,
 482                       (ESC_DOT|SUPPRESS_ENV_TO) );
 483
 484         SendMsg = CCC->redirect_buffer;
 485         CCC->redirect_buffer = NULL;
 486         if ((StrLength(SendMsg) > 0) &&
 487             ChrPtr(SendMsg)[StrLength(SendMsg) - 1] != '\n') {
 488                 syslog(LOG_WARNING,
 489                        "SMTP client[%d]: Possible problem: message did not "
 490                        "correctly terminate. (expecting 0x10, got 0x%02x)\n",
 491                        MsgCount, //yes uncool, but best choice here...
 492                        ChrPtr(SendMsg)[StrLength(SendMsg) - 1] );
 493                 StrBufAppendBufPlain(SendMsg, HKEY("\r\n"), 0);
 494         }
 495         return SendMsg;
 496 }
 497
 498
 499
 500 /*
 501  * smtp_do_bounce() is caled by smtp_do_procmsg() to scan a set of delivery
 502  * instructions for "5" codes (permanent fatal errors) and produce/deliver
 503  * a "bounce" message (delivery status notification).
 504  */
 505 void smtpq_do_bounce(OneQueItem *MyQItem, StrBuf *OMsgTxt)
 506 {
 507         static int seq = 0;
 508
 509         struct CtdlMessage *bmsg = NULL;
 510         StrBuf *boundary;
 511         StrBuf *Msg = NULL;
 512         StrBuf *BounceMB;
 513         struct recptypes *valid;
 514         time_t now;
 515
 516         HashPos *It;
 517         void *vQE;
 518         long len;
 519         const char *Key;
 520
 521         int first_attempt = 0;
 522         int successful_bounce = 0;
 523         int num_bounces = 0;
 524         int give_up = 0;
 525
 526         syslog(LOG_DEBUG, "smtp_do_bounce() called\n");
 527
 528         if (MyQItem->SendBounceMail == 0)
 529                 return;
 530
 531         now = time (NULL); //ev_time();
 532
 533         if ( (now - MyQItem->Submitted) > SMTP_GIVE_UP ) {
 534                 give_up = 1;
 535         }
 536
 537         if (MyQItem->Retry == SMTP_RETRY_INTERVAL) {
 538                 first_attempt = 1;
 539         }
 540
 541         /*
 542          * Now go through the instructions checking for stuff.
 543          */
 544         Msg = NewStrBufPlain(NULL, 1024);
 545         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 546         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 547         {
 548                 MailQEntry *ThisItem = vQE;
 549                 if ((ThisItem->Active && (ThisItem->Status == 5)) || /* failed now? */
 550                     ((give_up == 1) && (ThisItem->Status != 2)) ||
 551                     ((first_attempt == 1) && (ThisItem->Status != 2)))
 552                         /* giving up after failed attempts... */
 553                 {
 554                         ++num_bounces;
 555
 556                         StrBufAppendBuf(Msg, ThisItem->Recipient, 0);
 557                         StrBufAppendBufPlain(Msg, HKEY(": "), 0);
 558                         StrBufAppendBuf(Msg, ThisItem->StatusMessage, 0);
 559                         StrBufAppendBufPlain(Msg, HKEY("\r\n"), 0);
 560                 }
 561         }
 562         DeleteHashPos(&It);
 563
 564         /* Deliver the bounce if there's anything worth mentioning */
 565         syslog(LOG_DEBUG, "num_bounces = %d\n", num_bounces);
 566
 567         if (num_bounces == 0) {
 568                 FreeStrBuf(&Msg);
 569                 return;
 570         }
 571
 572         boundary = NewStrBufPlain(HKEY("=_Citadel_Multipart_"));
 573         StrBufAppendPrintf(boundary,
 574                            "%s_%04x%04x",
 575                            config.c_fqdn,
 576                            getpid(),
 577                            ++seq);
 578
 579         /* Start building our bounce message; go shopping for memory first. */
 580         BounceMB = NewStrBufPlain(
 581                 NULL,
 582                 1024 + /* mime stuff.... */
 583                 StrLength(Msg) +  /* the bounce information... */
 584                 StrLength(OMsgTxt)); /* the original message */
 585         if (BounceMB == NULL) {
 586                 FreeStrBuf(&boundary);
 587                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 588
 589                 return;
 590         }
 591
 592         bmsg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
 593         if (bmsg == NULL) {
 594                 FreeStrBuf(&boundary);
 595                 FreeStrBuf(&BounceMB);
 596                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 597
 598                 return;
 599         }
 600         memset(bmsg, 0, sizeof(struct CtdlMessage));
 601
 602
 603         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: multipart/mixed; boundary=\""), 0);
 604         StrBufAppendBuf(BounceMB, boundary, 0);
 605         StrBufAppendBufPlain(BounceMB, HKEY("\"\r\n"), 0);
 606         StrBufAppendBufPlain(BounceMB, HKEY("MIME-Version: 1.0\r\n"), 0);
 607         StrBufAppendBufPlain(BounceMB, HKEY("X-Mailer: " CITADEL "\r\n"), 0);
 608         StrBufAppendBufPlain(BounceMB, HKEY("\r\nThis is a multipart message in MIME format.\r\n\r\n"), 0);
 609         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 610         StrBufAppendBuf(BounceMB, boundary, 0);
 611         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 612         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: text/plain\r\n\r\n"), 0);
 613
 614         if (give_up)
 615                 StrBufAppendBufPlain(
 616                         BounceMB,
 617                         HKEY(
 618                                 "A message you sent could not be delivered "
 619                                 "to some or all of its recipients\n"
 620                                 "due to prolonged unavailability "
 621                                 "of its destination(s).\n"
 622                                 "Giving up on the following addresses:\n\n"
 623                                 ), 0);
 624         else
 625                 StrBufAppendBufPlain(
 626                         BounceMB,
 627                         HKEY(
 628                                 "A message you sent could not be delivered "
 629                                 "to some or all of its recipients.\n"
 630                                 "The following addresses "
 631                                 "were undeliverable:\n\n"
 632                                 ), 0);
 633
 634         StrBufAppendBuf(BounceMB, Msg, 0);
 635         FreeStrBuf(&Msg);
 636
 637         if (StrLength(MyQItem->SenderRoom) > 0)
 638         {
 639                 StrBufAppendBufPlain(
 640                         BounceMB,
 641                         HKEY("The message was originaly posted in: "), 0);
 642                 StrBufAppendBuf(BounceMB, MyQItem->SenderRoom, 0);
 643                 StrBufAppendBufPlain(
 644                         BounceMB,
 645                         HKEY("\n"), 0);
 646         }
 647
 648         /* Attach the original message */
 649         StrBufAppendBufPlain(BounceMB, HKEY("\r\n--"), 0);
 650         StrBufAppendBuf(BounceMB, boundary, 0);
 651         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 652         StrBufAppendBufPlain(BounceMB,
 653                              HKEY("Content-type: message/rfc822\r\n"), 0);
 654         StrBufAppendBufPlain(BounceMB,
 655                              HKEY("Content-Transfer-Encoding: 7bit\r\n"), 0);
 656         StrBufAppendBufPlain(BounceMB,
 657                              HKEY("Content-Disposition: inline\r\n"), 0);
 658         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 659         StrBufAppendBuf(BounceMB, OMsgTxt, 0);
 660
 661         /* Close the multipart MIME scope */
 662         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 663         StrBufAppendBuf(BounceMB, boundary, 0);
 664         StrBufAppendBufPlain(BounceMB, HKEY("--\r\n"), 0);
 665
 666         bmsg->cm_magic = CTDLMESSAGE_MAGIC;
 667         bmsg->cm_anon_type = MES_NORMAL;
 668         bmsg->cm_format_type = FMT_RFC822;
 669
 670         bmsg->cm_fields['O'] = strdup(MAILROOM);
 671         bmsg->cm_fields['A'] = strdup("Citadel");
 672         bmsg->cm_fields['N'] = strdup(config.c_nodename);
 673         bmsg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
 674         bmsg->cm_fields['M'] = SmashStrBuf(&BounceMB);
 675
 676         /* First try the user who sent the message */
 677         if (StrLength(MyQItem->BounceTo) == 0)
 678                 syslog(LOG_ERR, "No bounce address specified\n");
 679         else
 680                 syslog(LOG_DEBUG, "bounce to user? <%s>\n",
 681                        ChrPtr(MyQItem->BounceTo));
 682
 683         /* Can we deliver the bounce to the original sender? */
 684         valid = validate_recipients(ChrPtr(MyQItem->BounceTo), NULL, 0);
 685         if ((valid != NULL) && (valid->num_error == 0)) {
 686                 CtdlSubmitMsg(bmsg, valid, "", QP_EADDR);
 687                 successful_bounce = 1;
 688         }
 689
 690         /* If not, post it in the Aide> room */
 691         if (successful_bounce == 0) {
 692                 CtdlSubmitMsg(bmsg, NULL, config.c_aideroom, QP_EADDR);
 693         }
 694
 695         /* Free up the memory we used */
 696         free_recipients(valid);
 697         FreeStrBuf(&boundary);
 698         CtdlFreeMessage(bmsg);
 699         syslog(LOG_DEBUG, "Done processing bounces\n");
 700 }
 701
 702 /*
 703  * smtp_do_procmsg()
 704  *
 705  * Called by smtp_do_queue() to handle an individual message.
 706  */
 707 void smtp_do_procmsg(long msgnum, void *userdata) {
 708         time_t now;
 709         int mynumsessions = num_sessions;
 710         struct CtdlMessage *msg = NULL;
 711         char *instr = NULL;
 712         StrBuf *PlainQItem;
 713         OneQueItem *MyQItem;
 714         char *pch;
 715         HashPos  *It;
 716         void *vQE;
 717         long len;
 718         const char *Key;
 719         int nRelays = 0;
 720         ParsedURL *RelayUrls = NULL;
 721         int HaveBuffers = 0;
 722         StrBuf *Msg =NULL;
 723
 724         if (mynumsessions > max_sessions_for_outbound_smtp) {
 725                 syslog(LOG_DEBUG,
 726                        "SMTP Queue: skipping because of num jobs %d > %d max_sessions_for_outbound_smtp",
 727                        mynumsessions,
 728                        max_sessions_for_outbound_smtp);
 729         }
 730
 731         syslog(LOG_DEBUG, "SMTP Queue: smtp_do_procmsg(%ld)\n", msgnum);
 732         ///strcpy(envelope_from, "");
 733
 734         msg = CtdlFetchMessage(msgnum, 1);
 735         if (msg == NULL) {
 736                 syslog(LOG_ERR, "SMTP Queue: tried %ld but no such message!\n",
 737                        msgnum);
 738                 return;
 739         }
 740
 741         pch = instr = msg->cm_fields['M'];
 742
 743         /* Strip out the headers (no not amd any other non-instruction) line */
 744         while (pch != NULL) {
 745                 pch = strchr(pch, '\n');
 746                 if ((pch != NULL) && (*(pch + 1) == '\n')) {
 747                         instr = pch + 2;
 748                         pch = NULL;
 749                 }
 750         }
 751         PlainQItem = NewStrBufPlain(instr, -1);
 752         CtdlFreeMessage(msg);
 753         MyQItem = DeserializeQueueItem(PlainQItem, msgnum);
 754         FreeStrBuf(&PlainQItem);
 755
 756         if (MyQItem == NULL) {
 757                 syslog(LOG_ERR,
 758                        "SMTP Queue: Msg No %ld: already in progress!\n",
 759                        msgnum);
 760                 return; /* s.b. else is already processing... */
 761         }
 762
 763         /*
 764          * Postpone delivery if we've already tried recently.
 765          */
 766         now = time(NULL);
 767         if ((MyQItem->ReattemptWhen != 0) &&
 768             (now < MyQItem->ReattemptWhen) &&
 769             (run_queue_now == 0))
 770         {
 771                 syslog(LOG_DEBUG, "SMTP client: Retry time not yet reached. %ld seconds left.",  MyQItem->ReattemptWhen - now);
 772
 773                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 774                 pthread_mutex_lock(&ActiveQItemsLock);
 775                 {
 776                         if (GetHashPosFromKey(ActiveQItems,
 777                                               LKEY(MyQItem->MessageID),
 778                                               It))
 779                         {
 780                                 DeleteEntryFromHash(ActiveQItems, It);
 781                         }
 782                 }
 783                 pthread_mutex_unlock(&ActiveQItemsLock);
 784                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 785                 DeleteHashPos(&It);
 786                 return;
 787         }
 788
 789         /*
 790          * Bail out if there's no actual message associated with this
 791          */
 792         if (MyQItem->MessageID < 0L) {
 793                 syslog(LOG_ERR, "SMTP Queue: no 'msgid' directive found!\n");
 794                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 795                 pthread_mutex_lock(&ActiveQItemsLock);
 796                 {
 797                         if (GetHashPosFromKey(ActiveQItems,
 798                                               LKEY(MyQItem->MessageID),
 799                                               It))
 800                         {
 801                                 DeleteEntryFromHash(ActiveQItems, It);
 802                         }
 803                 }
 804                 pthread_mutex_unlock(&ActiveQItemsLock);
 805                 DeleteHashPos(&It);
 806                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 807                 return;
 808         }
 809
 810         {
 811                 char mxbuf[SIZ];
 812                 ParsedURL **Url = &MyQItem->URL;
 813                 nRelays = get_hosts(mxbuf, "smarthost");
 814                 if (nRelays > 0) {
 815                         StrBuf *All;
 816                         StrBuf *One;
 817                         const char *Pos = NULL;
 818                         All = NewStrBufPlain(mxbuf, -1);
 819                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 820
 821                         while ((Pos != StrBufNOTNULL) &&
 822                                ((Pos == NULL) ||
 823                                 !IsEmptyStr(Pos)))
 824                         {
 825                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 826                                 if (!ParseURL(Url, One, 25))
 827                                         syslog(LOG_DEBUG,
 828                                                "Failed to parse: %s\n",
 829                                                ChrPtr(One));
 830                                 else {
 831                                         ///if (!Url->IsIP)) // todo dupe me fork ipv6
 832                                         Url = &(*Url)->Next;
 833                                 }
 834                         }
 835                         FreeStrBuf(&All);
 836                         FreeStrBuf(&One);
 837                 }
 838
 839                 Url = &MyQItem->FallBackHost;
 840                 nRelays = get_hosts(mxbuf, "fallbackhost");
 841                 if (nRelays > 0) {
 842                         StrBuf *All;
 843                         StrBuf *One;
 844                         const char *Pos = NULL;
 845                         All = NewStrBufPlain(mxbuf, -1);
 846                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 847
 848                         while ((Pos != StrBufNOTNULL) &&
 849                                ((Pos == NULL) ||
 850                                 !IsEmptyStr(Pos)))
 851                         {
 852                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 853                                 if (!ParseURL(Url, One, 25))
 854                                         syslog(LOG_DEBUG,
 855                                                "Failed to parse: %s\n",
 856                                                ChrPtr(One));
 857                                 else
 858                                         Url = &(*Url)->Next;
 859                         }
 860                         FreeStrBuf(&All);
 861                         FreeStrBuf(&One);
 862                 }
 863         }
 864
 865         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 866         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 867         {
 868                 MailQEntry *ThisItem = vQE;
 869                 syslog(LOG_DEBUG, "SMTP Queue: Task: <%s> %d\n",
 870                        ChrPtr(ThisItem->Recipient),
 871                        ThisItem->Active);
 872         }
 873         DeleteHashPos(&It);
 874
 875         MyQItem->NotYetShutdownDeliveries =
 876         MyQItem->ActiveDeliveries = CountActiveQueueEntries(MyQItem);
 877
 878         /* failsafe against overload:
 879          * will we exceed the limit set?
 880          */
 881         if ((MyQItem->ActiveDeliveries + mynumsessions > max_sessions_for_outbound_smtp) &&
 882             /* if yes, did we reach more than half of the quota? */
 883             ((mynumsessions * 2) > max_sessions_for_outbound_smtp) &&
 884             /* if... would we ever fit into half of the quota?? */
 885             (((MyQItem->ActiveDeliveries * 2)  < max_sessions_for_outbound_smtp)))
 886         {
 887                 /* abort delivery for another time. */
 888                 syslog(LOG_DEBUG,
 889                        "SMTP Queue: skipping because of num jobs %d + %ld > %d max_sessions_for_outbound_smtp",
 890                        mynumsessions,
 891                        MyQItem->ActiveDeliveries,
 892                        max_sessions_for_outbound_smtp);
 893
 894                 FreeQueItem(&MyQItem);
 895
 896                 return;
 897         }
 898
 899
 900         if (MyQItem->ActiveDeliveries > 0)
 901         {
 902                 int nActivated = 0;
 903                 int n = MsgCount++;
 904                 int m = MyQItem->ActiveDeliveries;
 905                 int i = 1;
 906                 Msg = smtp_load_msg(MyQItem, n);
 907                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 908                 while ((i <= m) &&
 909                        (GetNextHashPos(MyQItem->MailQEntries,
 910                                        It, &len, &Key, &vQE)))
 911                 {
 912                         MailQEntry *ThisItem = vQE;
 913
 914                         if (ThisItem->Active == 1)
 915                         {
 916                                 int KeepBuffers = (i == m);
 917
 918                                 nActivated++;
 919                                 if (nActivated % ndelay_count == 0)
 920                                         usleep(delay_msec);
 921
 922                                 if (i > 1) n = MsgCount++;
 923                                 syslog(LOG_DEBUG,
 924                                        "SMTPQ: Trying <%ld> <%s> %d / %d \n",
 925                                        MyQItem->MessageID,
 926                                        ChrPtr(ThisItem->Recipient),
 927                                        i,
 928                                        m);
 929                                 smtp_try_one_queue_entry(MyQItem,
 930                                                          ThisItem,
 931                                                          Msg,
 932                                                          KeepBuffers,
 933                                                          n,
 934                                                          RelayUrls);
 935
 936                                 if (KeepBuffers) HaveBuffers = 1;
 937
 938                                 i++;
 939                         }
 940                 }
 941                 DeleteHashPos(&It);
 942         }
 943         else
 944         {
 945                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 946                 pthread_mutex_lock(&ActiveQItemsLock);
 947                 {
 948                         if (GetHashPosFromKey(ActiveQItems,
 949                                               LKEY(MyQItem->MessageID),
 950                                               It))
 951                         {
 952                                 DeleteEntryFromHash(ActiveQItems, It);
 953                         }
 954                         else
 955                         {
 956                                 long len;
 957                                 const char* Key;
 958                                 void *VData;
 959
 960                                 syslog(LOG_WARNING,
 961                                        "SMTP cleanup: unable to find "
 962                                        "QItem with ID[%ld]",
 963                                        MyQItem->MessageID);
 964                                 while (GetNextHashPos(ActiveQItems,
 965                                                       It,
 966                                                       &len,
 967                                                       &Key,
 968                                                       &VData))
 969                                 {
 970                                         syslog(LOG_WARNING,
 971                                                "SMTP cleanup: have: ID[%ld]",
 972                                               ((OneQueItem *)VData)->MessageID);
 973                                 }
 974                         }
 975
 976                 }
 977                 pthread_mutex_unlock(&ActiveQItemsLock);
 978                 DeleteHashPos(&It);
 979                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 980
 981 // TODO: bounce & delete?
 982
 983         }
 984         if (!HaveBuffers) {
 985                 FreeStrBuf (&Msg);
 986 // TODO : free RelayUrls
 987         }
 988 }
 989
 990
 991
 992 /*
 993  * smtp_queue_thread()
 994  *
 995  * Run through the queue sending out messages.
 996  */
 997 void smtp_do_queue(void) {
 998         static int is_running = 0;
 999         int num_processed = 0;
1000
1001         if (is_running)
1002                 return; /* Concurrency check - only one can run */
1003         is_running = 1;
1004
1005         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
1006         syslog(LOG_INFO, "SMTP client: processing outbound queue");
1007
1008         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
1009                 syslog(LOG_ERR, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
1010         }
1011         else {
1012                 num_processed = CtdlForEachMessage(MSGS_ALL,
1013                                                    0L,
1014                                                    NULL,
1015                                                    SPOOLMIME,
1016                                                    NULL,
1017                                                    smtp_do_procmsg,
1018                                                    NULL);
1019         }
1020         syslog(LOG_INFO,
1021                "SMTP client: queue run completed; %d messages processed",
1022                num_processed);
1023
1024         run_queue_now = 0;
1025         is_running = 0;
1026 }
1027
1028
1029
1030 /*
1031  * Initialize the SMTP outbound queue
1032  */
1033 void smtp_init_spoolout(void) {
1034         struct ctdlroom qrbuf;
1035
1036         /*
1037          * Create the room.  This will silently fail if the room already
1038          * exists, and that's perfectly ok, because we want it to exist.
1039          */
1040         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
1041
1042         /*
1043          * Make sure it's set to be a "system room" so it doesn't show up
1044          * in the <K>nown rooms list for Aides.
1045          */
1046         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
1047                 qrbuf.QRflags2 |= QR2_SYSTEM;
1048                 CtdlPutRoomLock(&qrbuf);
1049         }
1050 }
1051
1052
1053
1054
1055 /*****************************************************************************/
1056 /*                          SMTP UTILITY COMMANDS                            */
1057 /*****************************************************************************/
1058
1059 void cmd_smtp(char *argbuf) {
1060         char cmd[64];
1061         char node[256];
1062         char buf[1024];
1063         int i;
1064         int num_mxhosts;
1065
1066         if (CtdlAccessCheck(ac_aide)) return;
1067
1068         extract_token(cmd, argbuf, 0, '|', sizeof cmd);
1069
1070         if (!strcasecmp(cmd, "mx")) {
1071                 extract_token(node, argbuf, 1, '|', sizeof node);
1072                 num_mxhosts = getmx(buf, node);
1073                 cprintf("%d %d MX hosts listed for %s\n",
1074                         LISTING_FOLLOWS, num_mxhosts, node);
1075                 for (i=0; i<num_mxhosts; ++i) {
1076                         extract_token(node, buf, i, '|', sizeof node);
1077                         cprintf("%s\n", node);
1078                 }
1079                 cprintf("000\n");
1080                 return;
1081         }
1082
1083         else if (!strcasecmp(cmd, "runqueue")) {
1084                 run_queue_now = 1;
1085                 cprintf("%d All outbound SMTP will be retried now.\n", CIT_OK);
1086                 return;
1087         }
1088
1089         else {
1090                 cprintf("%d Invalid command.\n", ERROR + ILLEGAL_VALUE);
1091         }
1092
1093 }
1094
1095
1096 CTDL_MODULE_INIT(smtp_queu)
1097 {
1098         char *pstr;
1099
1100         if (!threading)
1101         {
1102                 pstr = getenv("CITSERVER_n_session_max");
1103                 if ((pstr != NULL) && (*pstr != '\0'))
1104                         max_sessions_for_outbound_smtp = atol(pstr); /* how many sessions might be active till we stop adding more smtp jobs */
1105
1106                 pstr = getenv("CITSERVER_smtp_n_delay_count");
1107                 if ((pstr != NULL) && (*pstr != '\0'))
1108                         ndelay_count = atol(pstr); /* every n queued messages we will sleep... */
1109
1110                 pstr = getenv("CITSERVER_smtp_delay");
1111                 if ((pstr != NULL) && (*pstr != '\0'))
1112                         delay_msec = atol(pstr) * 1000; /* this many seconds. */
1113
1114
1115
1116
1117                 CtdlFillSystemContext(&smtp_queue_CC, "SMTP_Send");
1118                 ActiveQItems = NewHash(1, lFlathash);
1119                 pthread_mutex_init(&ActiveQItemsLock, NULL);
1120
1121                 QItemHandlers = NewHash(0, NULL);
1122
1123                 RegisterQItemHandler(HKEY("msgid"),             QItem_Handle_MsgID);
1124                 RegisterQItemHandler(HKEY("envelope_from"),     QItem_Handle_EnvelopeFrom);
1125                 RegisterQItemHandler(HKEY("retry"),             QItem_Handle_retry);
1126                 RegisterQItemHandler(HKEY("attempted"),         QItem_Handle_Attempted);
1127                 RegisterQItemHandler(HKEY("remote"),            QItem_Handle_Recipient);
1128                 RegisterQItemHandler(HKEY("bounceto"),          QItem_Handle_BounceTo);
1129                 RegisterQItemHandler(HKEY("source_room"),       QItem_Handle_SenderRoom);
1130                 RegisterQItemHandler(HKEY("submitted"),         QItem_Handle_Submitted);
1131
1132                 smtp_init_spoolout();
1133
1134                 CtdlRegisterEVCleanupHook(smtp_evq_cleanup);
1135
1136                 CtdlRegisterProtoHook(cmd_smtp, "SMTP", "SMTP utility commands");
1137                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER);
1138         }
1139
1140         /* return our Subversion id for the Log */
1141         return "smtpeventclient";
1142 }