citadel/modules/smtp/serv_smtpqueue.c

   1 /*
   2  * This module is an SMTP and ESMTP implementation for the Citadel system.
   3  * It is compliant with all of the following:
   4  *
   5  * RFC  821 - Simple Mail Transfer Protocol
   6  * RFC  876 - Survey of SMTP Implementations
   7  * RFC 1047 - Duplicate messages and SMTP
   8  * RFC 1652 - 8 bit MIME
   9  * RFC 1869 - Extended Simple Mail Transfer Protocol
  10  * RFC 1870 - SMTP Service Extension for Message Size Declaration
  11  * RFC 2033 - Local Mail Transfer Protocol
  12  * RFC 2197 - SMTP Service Extension for Command Pipelining
  13  * RFC 2476 - Message Submission
  14  * RFC 2487 - SMTP Service Extension for Secure SMTP over TLS
  15  * RFC 2554 - SMTP Service Extension for Authentication
  16  * RFC 2821 - Simple Mail Transfer Protocol
  17  * RFC 2822 - Internet Message Format
  18  * RFC 2920 - SMTP Service Extension for Command Pipelining
  19  *
  20  * The VRFY and EXPN commands have been removed from this implementation
  21  * because nobody uses these commands anymore, except for spammers.
  22  *
  23  * Copyright (c) 1998-2012 by the citadel.org team
  24  *
  25  *  This program is open source software; you can redistribute it and/or modify
  26  *  it under the terms of the GNU General Public License version 3.
  27  *
  28  *
  29  *
  30  *  This program is distributed in the hope that it will be useful,
  31  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  32  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  33  *  GNU General Public License for more details.
  34  *
  35  *
  36  *
  37  *
  38  */
  39
  40 #include "sysdep.h"
  41 #include <stdlib.h>
  42 #include <unistd.h>
  43 #include <stdio.h>
  44 #include <termios.h>
  45 #include <fcntl.h>
  46 #include <signal.h>
  47 #include <pwd.h>
  48 #include <errno.h>
  49 #include <sys/types.h>
  50 #include <syslog.h>
  51
  52 #if TIME_WITH_SYS_TIME
  53 # include <sys/time.h>
  54 # include <time.h>
  55 #else
  56 # if HAVE_SYS_TIME_H
  57 #  include <sys/time.h>
  58 # else
  59 #  include <time.h>
  60 # endif
  61 #endif
  62 #include <sys/wait.h>
  63 #include <ctype.h>
  64 #include <string.h>
  65 #include <limits.h>
  66 #include <sys/socket.h>
  67 #include <netinet/in.h>
  68 #include <arpa/inet.h>
  69 #include <libcitadel.h>
  70 #include "citadel.h"
  71 #include "server.h"
  72 #include "citserver.h"
  73 #include "support.h"
  74 #include "config.h"
  75 #include "control.h"
  76 #include "user_ops.h"
  77 #include "database.h"
  78 #include "msgbase.h"
  79 #include "internet_addressing.h"
  80 #include "genstamp.h"
  81 #include "domain.h"
  82 #include "clientsocket.h"
  83 #include "locate_host.h"
  84 #include "citadel_dirs.h"
  85
  86 #include "ctdl_module.h"
  87
  88 #include "smtpqueue.h"
  89 #include "smtp_clienthandlers.h"
  90 #include "event_client.h"
  91
  92
  93 struct CitContext smtp_queue_CC;
  94 pthread_mutex_t ActiveQItemsLock;
  95 HashList *ActiveQItems  = NULL;
  96 HashList *QItemHandlers = NULL;
  97 const unsigned short DefaultMXPort = 25;
  98 int max_sessions_for_outbound_smtp = 500; /* how many sessions might be active till we stop adding more smtp jobs */
  99 int ndelay_count = 50; /* every n queued messages we will sleep... */
 100 int delay_msec = 5000; /* this many seconds. */
 101
 102 static const long MaxRetry = SMTP_RETRY_INTERVAL * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2;
 103 int MsgCount            = 0;
 104 int run_queue_now       = 0;    /* Set to 1 to ignore SMTP send retry times */
 105
 106 void RegisterQItemHandler(const char *Key, long Len, QItemHandler H)
 107 {
 108         QItemHandlerStruct *HS = (QItemHandlerStruct*)malloc(sizeof(QItemHandlerStruct));
 109         HS->H = H;
 110         Put(QItemHandlers, Key, Len, HS, NULL);
 111 }
 112
 113
 114
 115 void smtp_try_one_queue_entry(OneQueItem *MyQItem,
 116                               MailQEntry *MyQEntry,
 117                               StrBuf *MsgText,
 118 /* KeepMsgText allows us to use MsgText as ours. */
 119                               int KeepMsgText,
 120                               int MsgCount,
 121                               ParsedURL *RelayUrls);
 122
 123
 124 void smtp_evq_cleanup(void)
 125 {
 126
 127         pthread_mutex_lock(&ActiveQItemsLock);
 128         DeleteHash(&QItemHandlers);
 129         DeleteHash(&ActiveQItems);
 130         pthread_mutex_unlock(&ActiveQItemsLock);
 131         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 132 /*      citthread_mutex_destroy(&ActiveQItemsLock); TODO */
 133 }
 134
 135 int DecreaseQReference(OneQueItem *MyQItem)
 136 {
 137         int IDestructQueItem;
 138
 139         pthread_mutex_lock(&ActiveQItemsLock);
 140         MyQItem->ActiveDeliveries--;
 141         IDestructQueItem = MyQItem->ActiveDeliveries == 0;
 142         pthread_mutex_unlock(&ActiveQItemsLock);
 143         return IDestructQueItem;
 144 }
 145
 146 void DecreaseShutdownDeliveries(OneQueItem *MyQItem)
 147 {
 148         pthread_mutex_lock(&ActiveQItemsLock);
 149         MyQItem->NotYetShutdownDeliveries--;
 150         pthread_mutex_unlock(&ActiveQItemsLock);
 151 }
 152
 153 int GetShutdownDeliveries(OneQueItem *MyQItem)
 154 {
 155         int DestructNow;
 156
 157         pthread_mutex_lock(&ActiveQItemsLock);
 158         DestructNow = MyQItem->ActiveDeliveries == 0;
 159         pthread_mutex_unlock(&ActiveQItemsLock);
 160         return DestructNow;
 161 }
 162 void RemoveQItem(OneQueItem *MyQItem)
 163 {
 164         long len;
 165         const char* Key;
 166         void *VData;
 167         HashPos  *It;
 168
 169         pthread_mutex_lock(&ActiveQItemsLock);
 170         It = GetNewHashPos(ActiveQItems, 0);
 171         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 172                 DeleteEntryFromHash(ActiveQItems, It);
 173         else
 174         {
 175                 SMTPC_syslog(LOG_WARNING,
 176                              "unable to find QItem with ID[%ld]",
 177                              MyQItem->MessageID);
 178                 while (GetNextHashPos(ActiveQItems, It, &len, &Key, &VData))
 179                         SMTPC_syslog(LOG_WARNING,
 180                                      "have_: ID[%ld]",
 181                                      ((OneQueItem *)VData)->MessageID);
 182         }
 183         pthread_mutex_unlock(&ActiveQItemsLock);
 184         DeleteHashPos(&It);
 185 }
 186
 187
 188 void FreeMailQEntry(void *qv)
 189 {
 190         MailQEntry *Q = qv;
 191 /*
 192         SMTPC_syslog(LOG_DEBUG, "---------------%s--------------", __FUNCTION__);
 193         cit_backtrace();
 194 */
 195         FreeStrBuf(&Q->Recipient);
 196         FreeStrBuf(&Q->StatusMessage);
 197
 198         memset(Q, 0, sizeof(MailQEntry));
 199         free(Q);
 200 }
 201 void FreeQueItem(OneQueItem **Item)
 202 {
 203 /*
 204         SMTPC_syslog(LOG_DEBUG, "---------------%s--------------", __FUNCTION__);
 205         cit_backtrace();
 206 */
 207         DeleteHash(&(*Item)->MailQEntries);
 208         FreeStrBuf(&(*Item)->EnvelopeFrom);
 209         FreeStrBuf(&(*Item)->BounceTo);
 210         FreeStrBuf(&(*Item)->SenderRoom);
 211         FreeURL(&(*Item)->URL);
 212         memset(*Item, 0, sizeof(OneQueItem));
 213         free(*Item);
 214         Item = NULL;
 215 }
 216 void HFreeQueItem(void *Item)
 217 {
 218         FreeQueItem((OneQueItem**)&Item);
 219 }
 220
 221 /* inspect recipients with a status of:
 222  * - 0 (no delivery yet attempted)
 223  * - 3/4 (transient errors
 224  *        were experienced and it's time to try again)
 225  */
 226 int CheckQEntryActive(MailQEntry *ThisItem)
 227 {
 228         if ((ThisItem->Status == 0) ||
 229             (ThisItem->Status == 3) ||
 230             (ThisItem->Status == 4))
 231         {
 232                 return 1;
 233         }
 234         else
 235                 return 0;
 236 }
 237 int CheckQEntryIsBounce(MailQEntry *ThisItem)
 238 {
 239         if ((ThisItem->Status == 3) ||
 240             (ThisItem->Status == 4) ||
 241             (ThisItem->Status == 5))
 242         {
 243                 return 1;
 244         }
 245         else
 246                 return 0;
 247 }
 248
 249 int CountActiveQueueEntries(OneQueItem *MyQItem)
 250 {
 251         HashPos  *It;
 252         long len;
 253         long ActiveDeliveries;
 254         const char *Key;
 255         void *vQE;
 256
 257         ActiveDeliveries = 0;
 258         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 259         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 260         {
 261                 MailQEntry *ThisItem = vQE;
 262
 263                 if (CheckQEntryActive(ThisItem))
 264                 {
 265                         ActiveDeliveries++;
 266                         ThisItem->Active = 1;
 267                 }
 268                 else
 269                         ThisItem->Active = 0;
 270         }
 271         DeleteHashPos(&It);
 272         return ActiveDeliveries;
 273 }
 274
 275 OneQueItem *DeserializeQueueItem(StrBuf *RawQItem, long QueMsgID)
 276 {
 277         OneQueItem *Item;
 278         const char *pLine = NULL;
 279         StrBuf *Line;
 280         StrBuf *Token;
 281         void *v;
 282
 283         Item = (OneQueItem*)malloc(sizeof(OneQueItem));
 284         memset(Item, 0, sizeof(OneQueItem));
 285         Item->Retry = SMTP_RETRY_INTERVAL;
 286         Item->MessageID = -1;
 287         Item->QueMsgID = QueMsgID;
 288
 289         Token = NewStrBuf();
 290         Line = NewStrBufPlain(NULL, 128);
 291         while (pLine != StrBufNOTNULL) {
 292                 const char *pItemPart = NULL;
 293                 void *vHandler;
 294
 295                 StrBufExtract_NextToken(Line, RawQItem, &pLine, '\n');
 296                 if (StrLength(Line) == 0) continue;
 297                 StrBufExtract_NextToken(Token, Line, &pItemPart, '|');
 298                 if (GetHash(QItemHandlers, SKEY(Token), &vHandler))
 299                 {
 300                         QItemHandlerStruct *HS;
 301                         HS = (QItemHandlerStruct*) vHandler;
 302                         HS->H(Item, Line, &pItemPart);
 303                 }
 304         }
 305         FreeStrBuf(&Line);
 306         FreeStrBuf(&Token);
 307
 308         if (Item->Retry >= MaxRetry)
 309                 Item->FailNow = 1;
 310
 311         pthread_mutex_lock(&ActiveQItemsLock);
 312         if (GetHash(ActiveQItems,
 313                     LKEY(Item->MessageID),
 314                     &v))
 315         {
 316                 /* WHOOPS. somebody else is already working on this. */
 317                 pthread_mutex_unlock(&ActiveQItemsLock);
 318                 FreeQueItem(&Item);
 319                 return NULL;
 320         }
 321         else {
 322                 /* mark our claim on this. */
 323                 Put(ActiveQItems,
 324                     LKEY(Item->MessageID),
 325                     Item,
 326                     HFreeQueItem);
 327                 pthread_mutex_unlock(&ActiveQItemsLock);
 328         }
 329
 330         return Item;
 331 }
 332
 333 StrBuf *SerializeQueueItem(OneQueItem *MyQItem)
 334 {
 335         StrBuf *QMessage;
 336         HashPos  *It;
 337         const char *Key;
 338         long len;
 339         void *vQE;
 340
 341         QMessage = NewStrBufPlain(NULL, SIZ);
 342         StrBufPrintf(QMessage, "Content-type: %s\n", SPOOLMIME);
 343 //      "attempted|%ld\n"  "retry|%ld\n",, (long)time(NULL), (long)retry );
 344         StrBufAppendBufPlain(QMessage, HKEY("\nmsgid|"), 0);
 345         StrBufAppendPrintf(QMessage, "%ld", MyQItem->MessageID);
 346
 347         StrBufAppendBufPlain(QMessage, HKEY("\nsubmitted|"), 0);
 348         StrBufAppendPrintf(QMessage, "%ld", MyQItem->Submitted);
 349
 350         if (StrLength(MyQItem->BounceTo) > 0) {
 351                 StrBufAppendBufPlain(QMessage, HKEY("\nbounceto|"), 0);
 352                 StrBufAppendBuf(QMessage, MyQItem->BounceTo, 0);
 353         }
 354
 355         if (StrLength(MyQItem->EnvelopeFrom) > 0) {
 356                 StrBufAppendBufPlain(QMessage, HKEY("\nenvelope_from|"), 0);
 357                 StrBufAppendBuf(QMessage, MyQItem->EnvelopeFrom, 0);
 358         }
 359
 360         if (StrLength(MyQItem->SenderRoom) > 0) {
 361                 StrBufAppendBufPlain(QMessage, HKEY("\nsource_room|"), 0);
 362                 StrBufAppendBuf(QMessage, MyQItem->SenderRoom, 0);
 363         }
 364
 365         StrBufAppendBufPlain(QMessage, HKEY("\nretry|"), 0);
 366         StrBufAppendPrintf(QMessage, "%ld",
 367                            MyQItem->Retry);
 368
 369         StrBufAppendBufPlain(QMessage, HKEY("\nattempted|"), 0);
 370         StrBufAppendPrintf(QMessage, "%ld",
 371                            time(NULL) /*ctdl_ev_now()*/ + MyQItem->Retry);
 372
 373         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 374         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 375         {
 376                 MailQEntry *ThisItem = vQE;
 377
 378                 StrBufAppendBufPlain(QMessage, HKEY("\nremote|"), 0);
 379                 StrBufAppendBuf(QMessage, ThisItem->Recipient, 0);
 380                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 381                 StrBufAppendPrintf(QMessage, "%d", ThisItem->Status);
 382                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 383                 StrBufAppendBuf(QMessage, ThisItem->StatusMessage, 0);
 384         }
 385         DeleteHashPos(&It);
 386         StrBufAppendBufPlain(QMessage, HKEY("\n"), 0);
 387         return QMessage;
 388 }
 389
 390
 391
 392
 393
 394 void NewMailQEntry(OneQueItem *Item)
 395 {
 396         Item->Current = (MailQEntry*) malloc(sizeof(MailQEntry));
 397         memset(Item->Current, 0, sizeof(MailQEntry));
 398
 399         if (Item->MailQEntries == NULL)
 400                 Item->MailQEntries = NewHash(1, Flathash);
 401         /* alocate big buffer so we won't get problems reallocating later. */
 402         Item->Current->StatusMessage = NewStrBufPlain(NULL, SIZ);
 403         Item->Current->n = GetCount(Item->MailQEntries);
 404         Put(Item->MailQEntries,
 405             IKEY(Item->Current->n),
 406             Item->Current,
 407             FreeMailQEntry);
 408 }
 409
 410 void QItem_Handle_MsgID(OneQueItem *Item, StrBuf *Line, const char **Pos)
 411 {
 412         Item->MessageID = StrBufExtractNext_long(Line, Pos, '|');
 413 }
 414
 415 void QItem_Handle_EnvelopeFrom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 416 {
 417         if (Item->EnvelopeFrom == NULL)
 418                 Item->EnvelopeFrom = NewStrBufPlain(NULL, StrLength(Line));
 419         StrBufExtract_NextToken(Item->EnvelopeFrom, Line, Pos, '|');
 420 }
 421
 422 void QItem_Handle_BounceTo(OneQueItem *Item, StrBuf *Line, const char **Pos)
 423 {
 424         if (Item->BounceTo == NULL)
 425                 Item->BounceTo = NewStrBufPlain(NULL, StrLength(Line));
 426         StrBufExtract_NextToken(Item->BounceTo, Line, Pos, '|');
 427 }
 428
 429 void QItem_Handle_SenderRoom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 430 {
 431         if (Item->SenderRoom == NULL)
 432                 Item->SenderRoom = NewStrBufPlain(NULL, StrLength(Line));
 433         StrBufExtract_NextToken(Item->SenderRoom, Line, Pos, '|');
 434 }
 435
 436 void QItem_Handle_Recipient(OneQueItem *Item, StrBuf *Line, const char **Pos)
 437 {
 438         if (Item->Current == NULL)
 439                 NewMailQEntry(Item);
 440         if (Item->Current->Recipient == NULL)
 441                 Item->Current->Recipient=NewStrBufPlain(NULL, StrLength(Line));
 442         StrBufExtract_NextToken(Item->Current->Recipient, Line, Pos, '|');
 443         Item->Current->Status = StrBufExtractNext_int(Line, Pos, '|');
 444         StrBufExtract_NextToken(Item->Current->StatusMessage, Line, Pos, '|');
 445         Item->Current = NULL; // TODO: is this always right?
 446 }
 447
 448
 449 void QItem_Handle_retry(OneQueItem *Item, StrBuf *Line, const char **Pos)
 450 {
 451         Item->Retry =
 452                 StrBufExtractNext_int(Line, Pos, '|');
 453         if (Item->Retry == 0)
 454                 Item->Retry = SMTP_RETRY_INTERVAL;
 455         else
 456                 Item->Retry *= 2;
 457 }
 458
 459
 460 void QItem_Handle_Submitted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 461 {
 462         Item->Submitted = atol(*Pos);
 463
 464 }
 465
 466 void QItem_Handle_Attempted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 467 {
 468         Item->ReattemptWhen = StrBufExtractNext_int(Line, Pos, '|');
 469 }
 470
 471
 472
 473 /**
 474  * this one has to have the context for loading the message via the redirect buffer...
 475  */
 476 StrBuf *smtp_load_msg(OneQueItem *MyQItem, int n, char **Author, char **Address)
 477 {
 478         CitContext *CCC=CC;
 479         StrBuf *SendMsg;
 480
 481         CCC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
 482         CtdlOutputMsg(MyQItem->MessageID,
 483                       MT_RFC822, HEADERS_ALL,
 484                       0, 1, NULL,
 485                       (ESC_DOT|SUPPRESS_ENV_TO),
 486                       Author,
 487                       Address);
 488
 489         SendMsg = CCC->redirect_buffer;
 490         CCC->redirect_buffer = NULL;
 491         if ((StrLength(SendMsg) > 0) &&
 492             ChrPtr(SendMsg)[StrLength(SendMsg) - 1] != '\n') {
 493                 SMTPC_syslog(LOG_WARNING,
 494                              "[%d] Possible problem: message did not "
 495                              "correctly terminate. (expecting 0x10, got 0x%02x)\n",
 496                              MsgCount, //yes uncool, but best choice here...
 497                              ChrPtr(SendMsg)[StrLength(SendMsg) - 1] );
 498                 StrBufAppendBufPlain(SendMsg, HKEY("\r\n"), 0);
 499         }
 500         return SendMsg;
 501 }
 502
 503
 504
 505 /*
 506  * smtp_do_bounce() is caled by smtp_do_procmsg() to scan a set of delivery
 507  * instructions for "5" codes (permanent fatal errors) and produce/deliver
 508  * a "bounce" message (delivery status notification).
 509  */
 510 void smtpq_do_bounce(OneQueItem *MyQItem, StrBuf *OMsgTxt)
 511 {
 512         static int seq = 0;
 513
 514         struct CtdlMessage *bmsg = NULL;
 515         StrBuf *boundary;
 516         StrBuf *Msg = NULL;
 517         StrBuf *BounceMB;
 518         struct recptypes *valid;
 519         time_t now;
 520
 521         HashPos *It;
 522         void *vQE;
 523         long len;
 524         const char *Key;
 525
 526         int first_attempt = 0;
 527         int successful_bounce = 0;
 528         int num_bounces = 0;
 529         int give_up = 0;
 530
 531         SMTPCM_syslog(LOG_DEBUG, "smtp_do_bounce() called\n");
 532
 533         if (MyQItem->SendBounceMail == 0)
 534                 return;
 535
 536         now = time (NULL); //ev_time();
 537
 538         if ( (now - MyQItem->Submitted) > SMTP_GIVE_UP ) {
 539                 give_up = 1;
 540         }
 541
 542         if (MyQItem->Retry == SMTP_RETRY_INTERVAL) {
 543                 first_attempt = 1;
 544         }
 545
 546         /*
 547          * Now go through the instructions checking for stuff.
 548          */
 549         Msg = NewStrBufPlain(NULL, 1024);
 550         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 551         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 552         {
 553                 MailQEntry *ThisItem = vQE;
 554                 if ((ThisItem->Active && (ThisItem->Status == 5)) || /* failed now? */
 555                     ((give_up == 1) && (ThisItem->Status != 2)) ||
 556                     ((first_attempt == 1) && (ThisItem->Status != 2)))
 557                         /* giving up after failed attempts... */
 558                 {
 559                         ++num_bounces;
 560
 561                         StrBufAppendBuf(Msg, ThisItem->Recipient, 0);
 562                         StrBufAppendBufPlain(Msg, HKEY(": "), 0);
 563                         StrBufAppendBuf(Msg, ThisItem->StatusMessage, 0);
 564                         StrBufAppendBufPlain(Msg, HKEY("\r\n"), 0);
 565                 }
 566         }
 567         DeleteHashPos(&It);
 568
 569         /* Deliver the bounce if there's anything worth mentioning */
 570         SMTPC_syslog(LOG_DEBUG, "num_bounces = %d\n", num_bounces);
 571
 572         if (num_bounces == 0) {
 573                 FreeStrBuf(&Msg);
 574                 return;
 575         }
 576
 577         boundary = NewStrBufPlain(HKEY("=_Citadel_Multipart_"));
 578         StrBufAppendPrintf(boundary,
 579                            "%s_%04x%04x",
 580                            config.c_fqdn,
 581                            getpid(),
 582                            ++seq);
 583
 584         /* Start building our bounce message; go shopping for memory first. */
 585         BounceMB = NewStrBufPlain(
 586                 NULL,
 587                 1024 + /* mime stuff.... */
 588                 StrLength(Msg) +  /* the bounce information... */
 589                 StrLength(OMsgTxt)); /* the original message */
 590         if (BounceMB == NULL) {
 591                 FreeStrBuf(&boundary);
 592                 SMTPCM_syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 593
 594                 return;
 595         }
 596
 597         bmsg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
 598         if (bmsg == NULL) {
 599                 FreeStrBuf(&boundary);
 600                 FreeStrBuf(&BounceMB);
 601                 SMTPCM_syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 602
 603                 return;
 604         }
 605         memset(bmsg, 0, sizeof(struct CtdlMessage));
 606
 607
 608         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: multipart/mixed; boundary=\""), 0);
 609         StrBufAppendBuf(BounceMB, boundary, 0);
 610         StrBufAppendBufPlain(BounceMB, HKEY("\"\r\n"), 0);
 611         StrBufAppendBufPlain(BounceMB, HKEY("MIME-Version: 1.0\r\n"), 0);
 612         StrBufAppendBufPlain(BounceMB, HKEY("X-Mailer: " CITADEL "\r\n"), 0);
 613         StrBufAppendBufPlain(BounceMB, HKEY("\r\nThis is a multipart message in MIME format.\r\n\r\n"), 0);
 614         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 615         StrBufAppendBuf(BounceMB, boundary, 0);
 616         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 617         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: text/plain\r\n\r\n"), 0);
 618
 619         if (give_up)
 620                 StrBufAppendBufPlain(
 621                         BounceMB,
 622                         HKEY(
 623                                 "A message you sent could not be delivered "
 624                                 "to some or all of its recipients\n"
 625                                 "due to prolonged unavailability "
 626                                 "of its destination(s).\n"
 627                                 "Giving up on the following addresses:\n\n"
 628                                 ), 0);
 629         else
 630                 StrBufAppendBufPlain(
 631                         BounceMB,
 632                         HKEY(
 633                                 "A message you sent could not be delivered "
 634                                 "to some or all of its recipients.\n"
 635                                 "The following addresses "
 636                                 "were undeliverable:\n\n"
 637                                 ), 0);
 638
 639         StrBufAppendBuf(BounceMB, Msg, 0);
 640         FreeStrBuf(&Msg);
 641
 642         if (StrLength(MyQItem->SenderRoom) > 0)
 643         {
 644                 StrBufAppendBufPlain(
 645                         BounceMB,
 646                         HKEY("The message was originaly posted in: "), 0);
 647                 StrBufAppendBuf(BounceMB, MyQItem->SenderRoom, 0);
 648                 StrBufAppendBufPlain(
 649                         BounceMB,
 650                         HKEY("\n"), 0);
 651         }
 652
 653         /* Attach the original message */
 654         StrBufAppendBufPlain(BounceMB, HKEY("\r\n--"), 0);
 655         StrBufAppendBuf(BounceMB, boundary, 0);
 656         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 657         StrBufAppendBufPlain(BounceMB,
 658                              HKEY("Content-type: message/rfc822\r\n"), 0);
 659         StrBufAppendBufPlain(BounceMB,
 660                              HKEY("Content-Transfer-Encoding: 7bit\r\n"), 0);
 661         StrBufAppendBufPlain(BounceMB,
 662                              HKEY("Content-Disposition: inline\r\n"), 0);
 663         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 664         StrBufAppendBuf(BounceMB, OMsgTxt, 0);
 665
 666         /* Close the multipart MIME scope */
 667         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 668         StrBufAppendBuf(BounceMB, boundary, 0);
 669         StrBufAppendBufPlain(BounceMB, HKEY("--\r\n"), 0);
 670
 671         bmsg->cm_magic = CTDLMESSAGE_MAGIC;
 672         bmsg->cm_anon_type = MES_NORMAL;
 673         bmsg->cm_format_type = FMT_RFC822;
 674
 675         bmsg->cm_fields['O'] = strdup(MAILROOM);
 676         bmsg->cm_fields['A'] = strdup("Citadel");
 677         bmsg->cm_fields['N'] = strdup(config.c_nodename);
 678         bmsg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
 679         bmsg->cm_fields['M'] = SmashStrBuf(&BounceMB);
 680
 681         /* First try the user who sent the message */
 682         if (StrLength(MyQItem->BounceTo) == 0) {
 683                 SMTPCM_syslog(LOG_ERR, "No bounce address specified\n");
 684         }
 685         else {
 686                 SMTPC_syslog(LOG_DEBUG, "bounce to user? <%s>\n",
 687                        ChrPtr(MyQItem->BounceTo));
 688         }
 689
 690         /* Can we deliver the bounce to the original sender? */
 691         valid = validate_recipients(ChrPtr(MyQItem->BounceTo), NULL, 0);
 692         if ((valid != NULL) && (valid->num_error == 0)) {
 693                 CtdlSubmitMsg(bmsg, valid, "", QP_EADDR);
 694                 successful_bounce = 1;
 695         }
 696
 697         /* If not, post it in the Aide> room */
 698         if (successful_bounce == 0) {
 699                 CtdlSubmitMsg(bmsg, NULL, config.c_aideroom, QP_EADDR);
 700         }
 701
 702         /* Free up the memory we used */
 703         free_recipients(valid);
 704         FreeStrBuf(&boundary);
 705         CtdlFreeMessage(bmsg);
 706         SMTPCM_syslog(LOG_DEBUG, "Done processing bounces\n");
 707 }
 708
 709 ParsedURL *LoadRelayUrls(OneQueItem *MyQItem,
 710                          char *Author,
 711                          char *Address)
 712 {
 713         int nRelays = 0;
 714         ParsedURL *RelayUrls = NULL;
 715         char mxbuf[SIZ];
 716         ParsedURL **Url = &MyQItem->URL;
 717
 718         nRelays = get_hosts(mxbuf, "fallbackhost");
 719         if (nRelays > 0) {
 720                 StrBuf *All;
 721                 StrBuf *One;
 722                 const char *Pos = NULL;
 723                 All = NewStrBufPlain(mxbuf, -1);
 724                 One = NewStrBufPlain(NULL, StrLength(All) + 1);
 725
 726                 while ((Pos != StrBufNOTNULL) &&
 727                        ((Pos == NULL) ||
 728                         !IsEmptyStr(Pos)))
 729                 {
 730                         StrBufExtract_NextToken(One, All, &Pos, '|');
 731                         if (!ParseURL(Url, One, DefaultMXPort)) {
 732                                 SMTPC_syslog(LOG_DEBUG,
 733                                              "Failed to parse: %s\n",
 734                                              ChrPtr(One));
 735                         }
 736                         else {
 737                                 (*Url)->IsRelay = 1;
 738                                 MyQItem->HaveRelay = 1;
 739                         }
 740                 }
 741                 FreeStrBuf(&All);
 742                 FreeStrBuf(&One);
 743         }
 744         nRelays = get_hosts(mxbuf, "smarthost");
 745         if (nRelays > 0) {
 746                 char *User;
 747                 StrBuf *All;
 748                 StrBuf *One;
 749                 const char *Pos = NULL;
 750                 All = NewStrBufPlain(mxbuf, -1);
 751                 One = NewStrBufPlain(NULL, StrLength(All) + 1);
 752
 753                 while ((Pos != StrBufNOTNULL) &&
 754                        ((Pos == NULL) ||
 755                         !IsEmptyStr(Pos)))
 756                 {
 757                         StrBufExtract_NextToken(One, All, &Pos, '|');
 758                         User = strchr(ChrPtr(One), ' ');
 759                         if (User != NULL) {
 760                                 if (!strcmp(User + 1, Author) ||
 761                                     !strcmp(User + 1, Address))
 762                                         StrBufCutAt(One, 0, User);
 763                                 else
 764                                         continue;
 765
 766                         }
 767                         if (!ParseURL(Url, One, DefaultMXPort)) {
 768                                 SMTPC_syslog(LOG_DEBUG,
 769                                              "Failed to parse: %s\n",
 770                                              ChrPtr(One));
 771                         }
 772                         else {
 773                                 ///if (!Url->IsIP)) // todo dupe me fork ipv6
 774                                 (*Url)->IsRelay = 1;
 775                                 MyQItem->HaveRelay = 1;
 776                         }
 777                 }
 778                 FreeStrBuf(&All);
 779                 FreeStrBuf(&One);
 780         }
 781         return RelayUrls;
 782 }
 783 /*
 784  * smtp_do_procmsg()
 785  *
 786  * Called by smtp_do_queue() to handle an individual message.
 787  */
 788 void smtp_do_procmsg(long msgnum, void *userdata) {
 789         time_t now;
 790         int mynumsessions = num_sessions;
 791         struct CtdlMessage *msg = NULL;
 792         char *Author = NULL;
 793         char *Address = NULL;
 794         char *instr = NULL;
 795         StrBuf *PlainQItem;
 796         OneQueItem *MyQItem;
 797         char *pch;
 798         HashPos  *It;
 799         void *vQE;
 800         long len;
 801         const char *Key;
 802         int HaveBuffers = 0;
 803         StrBuf *Msg =NULL;
 804
 805         if (mynumsessions > max_sessions_for_outbound_smtp) {
 806                 SMTPC_syslog(LOG_DEBUG,
 807                              "skipping because of num jobs %d > %d max_sessions_for_outbound_smtp",
 808                              mynumsessions,
 809                              max_sessions_for_outbound_smtp);
 810         }
 811
 812         SMTPC_syslog(LOG_DEBUG, "smtp_do_procmsg(%ld)\n", msgnum);
 813         ///strcpy(envelope_from, "");
 814
 815         msg = CtdlFetchMessage(msgnum, 1);
 816         if (msg == NULL) {
 817                 SMTPC_syslog(LOG_ERR, "tried %ld but no such message!\n",
 818                        msgnum);
 819                 return;
 820         }
 821
 822         pch = instr = msg->cm_fields['M'];
 823
 824         /* Strip out the headers (no not amd any other non-instruction) line */
 825         while (pch != NULL) {
 826                 pch = strchr(pch, '\n');
 827                 if ((pch != NULL) && (*(pch + 1) == '\n')) {
 828                         instr = pch + 2;
 829                         pch = NULL;
 830                 }
 831         }
 832         PlainQItem = NewStrBufPlain(instr, -1);
 833         CtdlFreeMessage(msg);
 834         MyQItem = DeserializeQueueItem(PlainQItem, msgnum);
 835         FreeStrBuf(&PlainQItem);
 836
 837         if (MyQItem == NULL) {
 838                 SMTPC_syslog(LOG_ERR,
 839                              "Msg No %ld: already in progress!\n",
 840                              msgnum);
 841                 return; /* s.b. else is already processing... */
 842         }
 843
 844         /*
 845          * Postpone delivery if we've already tried recently.
 846          */
 847         now = time(NULL);
 848         if ((MyQItem->ReattemptWhen != 0) &&
 849             (now < MyQItem->ReattemptWhen) &&
 850             (run_queue_now == 0))
 851         {
 852                 SMTPC_syslog(LOG_DEBUG,
 853                              "Retry time not yet reached. %ld seconds left.",
 854                              MyQItem->ReattemptWhen - now);
 855
 856                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 857                 pthread_mutex_lock(&ActiveQItemsLock);
 858                 {
 859                         if (GetHashPosFromKey(ActiveQItems,
 860                                               LKEY(MyQItem->MessageID),
 861                                               It))
 862                         {
 863                                 DeleteEntryFromHash(ActiveQItems, It);
 864                         }
 865                 }
 866                 pthread_mutex_unlock(&ActiveQItemsLock);
 867                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 868                 DeleteHashPos(&It);
 869                 return;
 870         }
 871
 872         /*
 873          * Bail out if there's no actual message associated with this
 874          */
 875         if (MyQItem->MessageID < 0L) {
 876                 SMTPCM_syslog(LOG_ERR, "no 'msgid' directive found!\n");
 877                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 878                 pthread_mutex_lock(&ActiveQItemsLock);
 879                 {
 880                         if (GetHashPosFromKey(ActiveQItems,
 881                                               LKEY(MyQItem->MessageID),
 882                                               It))
 883                         {
 884                                 DeleteEntryFromHash(ActiveQItems, It);
 885                         }
 886                 }
 887                 pthread_mutex_unlock(&ActiveQItemsLock);
 888                 DeleteHashPos(&It);
 889                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 890                 return;
 891         }
 892
 893
 894         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 895         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 896         {
 897                 MailQEntry *ThisItem = vQE;
 898                 SMTPC_syslog(LOG_DEBUG, "SMTP Queue: Task: <%s> %d\n",
 899                              ChrPtr(ThisItem->Recipient),
 900                              ThisItem->Active);
 901         }
 902         DeleteHashPos(&It);
 903
 904         MyQItem->NotYetShutdownDeliveries =
 905         MyQItem->ActiveDeliveries = CountActiveQueueEntries(MyQItem);
 906
 907         /* failsafe against overload:
 908          * will we exceed the limit set?
 909          */
 910         if ((MyQItem->ActiveDeliveries + mynumsessions > max_sessions_for_outbound_smtp) &&
 911             /* if yes, did we reach more than half of the quota? */
 912             ((mynumsessions * 2) > max_sessions_for_outbound_smtp) &&
 913             /* if... would we ever fit into half of the quota?? */
 914             (((MyQItem->ActiveDeliveries * 2)  < max_sessions_for_outbound_smtp)))
 915         {
 916                 /* abort delivery for another time. */
 917                 SMTPC_syslog(LOG_DEBUG,
 918                              "SMTP Queue: skipping because of num jobs %d + %ld > %d max_sessions_for_outbound_smtp",
 919                              mynumsessions,
 920                              MyQItem->ActiveDeliveries,
 921                              max_sessions_for_outbound_smtp);
 922
 923                 FreeQueItem(&MyQItem);
 924
 925                 return;
 926         }
 927
 928
 929         if (MyQItem->ActiveDeliveries > 0)
 930         {
 931                 ParsedURL *RelayUrls = NULL;
 932                 int nActivated = 0;
 933                 int n = MsgCount++;
 934                 int m = MyQItem->ActiveDeliveries;
 935                 int i = 1;
 936
 937                 Msg = smtp_load_msg(MyQItem, n, &Author, &Address);
 938                 RelayUrls = LoadRelayUrls(MyQItem, Author, Address);
 939                 if (Author != NULL) free (Author);
 940                 if (Address != NULL) free (Address);
 941
 942                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 943                 while ((i <= m) &&
 944                        (GetNextHashPos(MyQItem->MailQEntries,
 945                                        It, &len, &Key, &vQE)))
 946                 {
 947                         MailQEntry *ThisItem = vQE;
 948
 949                         if (ThisItem->Active == 1)
 950                         {
 951                                 int KeepBuffers = (i == m);
 952
 953                                 nActivated++;
 954                                 if (nActivated % ndelay_count == 0)
 955                                         usleep(delay_msec);
 956
 957                                 if (i > 1) n = MsgCount++;
 958                                 syslog(LOG_DEBUG,
 959                                        "SMTPC: Trying <%ld> <%s> %d / %d \n",
 960                                        MyQItem->MessageID,
 961                                        ChrPtr(ThisItem->Recipient),
 962                                        i,
 963                                        m);
 964                                 (*((int*) userdata)) ++;
 965                                 smtp_try_one_queue_entry(MyQItem,
 966                                                          ThisItem,
 967                                                          Msg,
 968                                                          KeepBuffers,
 969                                                          n,
 970                                                          RelayUrls);
 971
 972                                 if (KeepBuffers) HaveBuffers = 1;
 973
 974                                 i++;
 975                         }
 976                 }
 977                 DeleteHashPos(&It);
 978         }
 979         else
 980         {
 981                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 982                 pthread_mutex_lock(&ActiveQItemsLock);
 983                 {
 984                         if (GetHashPosFromKey(ActiveQItems,
 985                                               LKEY(MyQItem->MessageID),
 986                                               It))
 987                         {
 988                                 DeleteEntryFromHash(ActiveQItems, It);
 989                         }
 990                         else
 991                         {
 992                                 long len;
 993                                 const char* Key;
 994                                 void *VData;
 995
 996                                 SMTPC_syslog(LOG_WARNING,
 997                                              "unable to find QItem with ID[%ld]",
 998                                              MyQItem->MessageID);
 999                                 while (GetNextHashPos(ActiveQItems,
1000                                                       It,
1001                                                       &len,
1002                                                       &Key,
1003                                                       &VData))
1004                                 {
1005                                         SMTPC_syslog(LOG_WARNING,
1006                                                      "have: ID[%ld]",
1007                                                      ((OneQueItem *)VData)->MessageID);
1008                                 }
1009                         }
1010
1011                 }
1012                 pthread_mutex_unlock(&ActiveQItemsLock);
1013                 DeleteHashPos(&It);
1014                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
1015
1016 // TODO: bounce & delete?
1017
1018         }
1019         if (!HaveBuffers) {
1020                 FreeStrBuf (&Msg);
1021 // TODO : free RelayUrls
1022         }
1023 }
1024
1025
1026
1027 /*
1028  * smtp_queue_thread()
1029  *
1030  * Run through the queue sending out messages.
1031  */
1032 void smtp_do_queue(void) {
1033         int num_processed = 0;
1034         int num_activated = 0;
1035
1036         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
1037         SMTPCM_syslog(LOG_INFO, "processing outbound queue");
1038
1039         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
1040                 SMTPC_syslog(LOG_ERR, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
1041         }
1042         else {
1043                 num_processed = CtdlForEachMessage(MSGS_ALL,
1044                                                    0L,
1045                                                    NULL,
1046                                                    SPOOLMIME,
1047                                                    NULL,
1048                                                    smtp_do_procmsg,
1049                                                    &num_activated);
1050         }
1051         SMTPC_syslog(LOG_INFO,
1052                      "queue run completed; %d messages processed %d activated",
1053                      num_processed, num_activated);
1054
1055 }
1056
1057
1058
1059 /*
1060  * Initialize the SMTP outbound queue
1061  */
1062 void smtp_init_spoolout(void) {
1063         struct ctdlroom qrbuf;
1064
1065         /*
1066          * Create the room.  This will silently fail if the room already
1067          * exists, and that's perfectly ok, because we want it to exist.
1068          */
1069         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
1070
1071         /*
1072          * Make sure it's set to be a "system room" so it doesn't show up
1073          * in the <K>nown rooms list for Aides.
1074          */
1075         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
1076                 qrbuf.QRflags2 |= QR2_SYSTEM;
1077                 CtdlPutRoomLock(&qrbuf);
1078         }
1079 }
1080
1081
1082
1083
1084 /*****************************************************************************/
1085 /*                          SMTP UTILITY COMMANDS                            */
1086 /*****************************************************************************/
1087
1088 void cmd_smtp(char *argbuf) {
1089         char cmd[64];
1090         char node[256];
1091         char buf[1024];
1092         int i;
1093         int num_mxhosts;
1094
1095         if (CtdlAccessCheck(ac_aide)) return;
1096
1097         extract_token(cmd, argbuf, 0, '|', sizeof cmd);
1098
1099         if (!strcasecmp(cmd, "mx")) {
1100                 extract_token(node, argbuf, 1, '|', sizeof node);
1101                 num_mxhosts = getmx(buf, node);
1102                 cprintf("%d %d MX hosts listed for %s\n",
1103                         LISTING_FOLLOWS, num_mxhosts, node);
1104                 for (i=0; i<num_mxhosts; ++i) {
1105                         extract_token(node, buf, i, '|', sizeof node);
1106                         cprintf("%s\n", node);
1107                 }
1108                 cprintf("000\n");
1109                 return;
1110         }
1111
1112         else if (!strcasecmp(cmd, "runqueue")) {
1113                 run_queue_now = 1;
1114                 cprintf("%d All outbound SMTP will be retried now.\n", CIT_OK);
1115                 return;
1116         }
1117
1118         else {
1119                 cprintf("%d Invalid command.\n", ERROR + ILLEGAL_VALUE);
1120         }
1121
1122 }
1123
1124
1125 CTDL_MODULE_INIT(smtp_queu)
1126 {
1127         char *pstr;
1128
1129         if (!threading)
1130         {
1131                 pstr = getenv("CITSERVER_n_session_max");
1132                 if ((pstr != NULL) && (*pstr != '\0'))
1133                         max_sessions_for_outbound_smtp = atol(pstr); /* how many sessions might be active till we stop adding more smtp jobs */
1134
1135                 pstr = getenv("CITSERVER_smtp_n_delay_count");
1136                 if ((pstr != NULL) && (*pstr != '\0'))
1137                         ndelay_count = atol(pstr); /* every n queued messages we will sleep... */
1138
1139                 pstr = getenv("CITSERVER_smtp_delay");
1140                 if ((pstr != NULL) && (*pstr != '\0'))
1141                         delay_msec = atol(pstr) * 1000; /* this many seconds. */
1142
1143
1144
1145
1146                 CtdlFillSystemContext(&smtp_queue_CC, "SMTP_Send");
1147                 ActiveQItems = NewHash(1, lFlathash);
1148                 pthread_mutex_init(&ActiveQItemsLock, NULL);
1149
1150                 QItemHandlers = NewHash(0, NULL);
1151
1152                 RegisterQItemHandler(HKEY("msgid"),             QItem_Handle_MsgID);
1153                 RegisterQItemHandler(HKEY("envelope_from"),     QItem_Handle_EnvelopeFrom);
1154                 RegisterQItemHandler(HKEY("retry"),             QItem_Handle_retry);
1155                 RegisterQItemHandler(HKEY("attempted"),         QItem_Handle_Attempted);
1156                 RegisterQItemHandler(HKEY("remote"),            QItem_Handle_Recipient);
1157                 RegisterQItemHandler(HKEY("bounceto"),          QItem_Handle_BounceTo);
1158                 RegisterQItemHandler(HKEY("source_room"),       QItem_Handle_SenderRoom);
1159                 RegisterQItemHandler(HKEY("submitted"),         QItem_Handle_Submitted);
1160
1161                 smtp_init_spoolout();
1162
1163                 CtdlRegisterEVCleanupHook(smtp_evq_cleanup);
1164
1165                 CtdlRegisterProtoHook(cmd_smtp, "SMTP", "SMTP utility commands");
1166                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER, PRIO_SEND + 10);
1167         }
1168
1169         /* return our Subversion id for the Log */
1170         return "smtpeventclient";
1171 }