citadel/modules/smtp/serv_smtpqueue.c

   1 /*
   2  * This module is an SMTP and ESMTP implementation for the Citadel system.
   3  * It is compliant with all of the following:
   4  *
   5  * RFC  821 - Simple Mail Transfer Protocol
   6  * RFC  876 - Survey of SMTP Implementations
   7  * RFC 1047 - Duplicate messages and SMTP
   8  * RFC 1652 - 8 bit MIME
   9  * RFC 1869 - Extended Simple Mail Transfer Protocol
  10  * RFC 1870 - SMTP Service Extension for Message Size Declaration
  11  * RFC 2033 - Local Mail Transfer Protocol
  12  * RFC 2197 - SMTP Service Extension for Command Pipelining
  13  * RFC 2476 - Message Submission
  14  * RFC 2487 - SMTP Service Extension for Secure SMTP over TLS
  15  * RFC 2554 - SMTP Service Extension for Authentication
  16  * RFC 2821 - Simple Mail Transfer Protocol
  17  * RFC 2822 - Internet Message Format
  18  * RFC 2920 - SMTP Service Extension for Command Pipelining
  19  *
  20  * The VRFY and EXPN commands have been removed from this implementation
  21  * because nobody uses these commands anymore, except for spammers.
  22  *
  23  * Copyright (c) 1998-2009 by the citadel.org team
  24  *
  25  *  This program is free software; you can redistribute it and/or modify
  26  *  it under the terms of the GNU General Public License as published by
  27  *  the Free Software Foundation; either version 3 of the License, or
  28  *  (at your option) any later version.
  29  *
  30  *  This program is distributed in the hope that it will be useful,
  31  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  32  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  33  *  GNU General Public License for more details.
  34  *
  35  *  You should have received a copy of the GNU General Public License
  36  *  along with this program; if not, write to the Free Software
  37  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  38  */
  39
  40 #include "sysdep.h"
  41 #include <stdlib.h>
  42 #include <unistd.h>
  43 #include <stdio.h>
  44 #include <termios.h>
  45 #include <fcntl.h>
  46 #include <signal.h>
  47 #include <pwd.h>
  48 #include <errno.h>
  49 #include <sys/types.h>
  50 #include <syslog.h>
  51
  52 #if TIME_WITH_SYS_TIME
  53 # include <sys/time.h>
  54 # include <time.h>
  55 #else
  56 # if HAVE_SYS_TIME_H
  57 #  include <sys/time.h>
  58 # else
  59 #  include <time.h>
  60 # endif
  61 #endif
  62 #include <sys/wait.h>
  63 #include <ctype.h>
  64 #include <string.h>
  65 #include <limits.h>
  66 #include <sys/socket.h>
  67 #include <netinet/in.h>
  68 #include <arpa/inet.h>
  69 #include <libcitadel.h>
  70 #include "citadel.h"
  71 #include "server.h"
  72 #include "citserver.h"
  73 #include "support.h"
  74 #include "config.h"
  75 #include "control.h"
  76 #include "user_ops.h"
  77 #include "database.h"
  78 #include "msgbase.h"
  79 #include "internet_addressing.h"
  80 #include "genstamp.h"
  81 #include "domain.h"
  82 #include "clientsocket.h"
  83 #include "locate_host.h"
  84 #include "citadel_dirs.h"
  85
  86 #include "ctdl_module.h"
  87
  88 #include "smtpqueue.h"
  89 #include "event_client.h"
  90
  91
  92 struct CitContext smtp_queue_CC;
  93 pthread_mutex_t ActiveQItemsLock;
  94 HashList *ActiveQItems  = NULL;
  95 HashList *QItemHandlers = NULL;
  96
  97 static const long MaxRetry = SMTP_RETRY_INTERVAL * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2;
  98 int MsgCount            = 0;
  99 int run_queue_now       = 0;    /* Set to 1 to ignore SMTP send retry times */
 100
 101 void smtp_try_one_queue_entry(OneQueItem *MyQItem,
 102                               MailQEntry *MyQEntry,
 103                               StrBuf *MsgText,
 104 /* KeepMsgText allows us to use MsgText as ours. */
 105                               int KeepMsgText,
 106                               int MsgCount,
 107                               ParsedURL *RelayUrls);
 108
 109
 110 void smtp_evq_cleanup(void)
 111 {
 112
 113         pthread_mutex_lock(&ActiveQItemsLock);
 114         DeleteHash(&QItemHandlers);
 115         DeleteHash(&ActiveQItems);
 116         pthread_mutex_unlock(&ActiveQItemsLock);
 117         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 118 /*      citthread_mutex_destroy(&ActiveQItemsLock); TODO */
 119 }
 120
 121 int DecreaseQReference(OneQueItem *MyQItem)
 122 {
 123         int IDestructQueItem;
 124
 125         pthread_mutex_lock(&ActiveQItemsLock);
 126         MyQItem->ActiveDeliveries--;
 127         IDestructQueItem = MyQItem->ActiveDeliveries == 0;
 128         pthread_mutex_unlock(&ActiveQItemsLock);
 129         return IDestructQueItem;
 130 }
 131
 132 void RemoveQItem(OneQueItem *MyQItem)
 133 {
 134         long len;
 135         const char* Key;
 136         void *VData;
 137         HashPos  *It;
 138
 139         pthread_mutex_lock(&ActiveQItemsLock);
 140         It = GetNewHashPos(ActiveQItems, 0);
 141         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 142                 DeleteEntryFromHash(ActiveQItems, It);
 143         else
 144         {
 145                 syslog(LOG_WARNING,
 146                        "SMTP cleanup: unable to find QItem with ID[%ld]",
 147                        MyQItem->MessageID);
 148                 while (GetNextHashPos(ActiveQItems, It, &len, &Key, &VData))
 149                         syslog(LOG_WARNING,
 150                                "SMTP cleanup: have_: ID[%ld]",
 151                                ((OneQueItem *)VData)->MessageID);
 152         }
 153         pthread_mutex_unlock(&ActiveQItemsLock);
 154         DeleteHashPos(&It);
 155 }
 156
 157
 158 void FreeMailQEntry(void *qv)
 159 {
 160         MailQEntry *Q = qv;
 161         FreeStrBuf(&Q->Recipient);
 162         FreeStrBuf(&Q->StatusMessage);
 163         free(Q);
 164 }
 165 void FreeQueItem(OneQueItem **Item)
 166 {
 167         DeleteHash(&(*Item)->MailQEntries);
 168         FreeStrBuf(&(*Item)->EnvelopeFrom);
 169         FreeStrBuf(&(*Item)->BounceTo);
 170         FreeURL(&(*Item)->URL);
 171         free(*Item);
 172         Item = NULL;
 173 }
 174 void HFreeQueItem(void *Item)
 175 {
 176         FreeQueItem((OneQueItem**)&Item);
 177 }
 178
 179 /* inspect recipients with a status of:
 180  * - 0 (no delivery yet attempted)
 181  * - 3/4 (transient errors
 182  *        were experienced and it's time to try again)
 183  */
 184 int CountActiveQueueEntries(OneQueItem *MyQItem)
 185 {
 186         HashPos  *It;
 187         long len;
 188         long ActiveDeliveries;
 189         const char *Key;
 190         void *vQE;
 191
 192         ActiveDeliveries = 0;
 193         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 194         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 195         {
 196                 MailQEntry *ThisItem = vQE;
 197                 if ((ThisItem->Status == 0) ||
 198                     (ThisItem->Status == 3) ||
 199                     (ThisItem->Status == 4))
 200                 {
 201                         ActiveDeliveries++;
 202                         ThisItem->Active = 1;
 203                 }
 204                 else
 205                         ThisItem->Active = 0;
 206         }
 207         DeleteHashPos(&It);
 208         return ActiveDeliveries;
 209 }
 210
 211 OneQueItem *DeserializeQueueItem(StrBuf *RawQItem, long QueMsgID)
 212 {
 213         OneQueItem *Item;
 214         const char *pLine = NULL;
 215         StrBuf *Line;
 216         StrBuf *Token;
 217         void *v;
 218
 219         Item = (OneQueItem*)malloc(sizeof(OneQueItem));
 220         memset(Item, 0, sizeof(OneQueItem));
 221         Item->Retry = SMTP_RETRY_INTERVAL;
 222         Item->MessageID = -1;
 223         Item->QueMsgID = QueMsgID;
 224
 225         Token = NewStrBuf();
 226         Line = NewStrBufPlain(NULL, 128);
 227         while (pLine != StrBufNOTNULL) {
 228                 const char *pItemPart = NULL;
 229                 void *vHandler;
 230
 231                 StrBufExtract_NextToken(Line, RawQItem, &pLine, '\n');
 232                 if (StrLength(Line) == 0) continue;
 233                 StrBufExtract_NextToken(Token, Line, &pItemPart, '|');
 234                 if (GetHash(QItemHandlers, SKEY(Token), &vHandler))
 235                 {
 236                         QItemHandler H;
 237                         H = (QItemHandler) vHandler;
 238                         H(Item, Line, &pItemPart);
 239                 }
 240         }
 241         FreeStrBuf(&Line);
 242         FreeStrBuf(&Token);
 243
 244         if (Item->Retry >= MaxRetry)
 245                 Item->FailNow = 1;
 246
 247         pthread_mutex_lock(&ActiveQItemsLock);
 248         if (GetHash(ActiveQItems,
 249                     LKEY(Item->MessageID),
 250                     &v))
 251         {
 252                 /* WHOOPS. somebody else is already working on this. */
 253                 pthread_mutex_unlock(&ActiveQItemsLock);
 254                 FreeQueItem(&Item);
 255                 return NULL;
 256         }
 257         else {
 258                 /* mark our claim on this. */
 259                 Put(ActiveQItems,
 260                     LKEY(Item->MessageID),
 261                     Item,
 262                     HFreeQueItem);
 263                 pthread_mutex_unlock(&ActiveQItemsLock);
 264         }
 265
 266         return Item;
 267 }
 268
 269 StrBuf *SerializeQueueItem(OneQueItem *MyQItem)
 270 {
 271         StrBuf *QMessage;
 272         HashPos  *It;
 273         const char *Key;
 274         long len;
 275         void *vQE;
 276
 277         QMessage = NewStrBufPlain(NULL, SIZ);
 278         StrBufPrintf(QMessage, "Content-type: %s\n", SPOOLMIME);
 279 //      "attempted|%ld\n"  "retry|%ld\n",, (long)time(NULL), (long)retry );
 280         StrBufAppendBufPlain(QMessage, HKEY("\nmsgid|"), 0);
 281         StrBufAppendPrintf(QMessage, "%ld", MyQItem->MessageID);
 282
 283         StrBufAppendBufPlain(QMessage, HKEY("\nsubmitted|"), 0);
 284         StrBufAppendPrintf(QMessage, "%ld", MyQItem->Submitted);
 285
 286         if (StrLength(MyQItem->BounceTo) > 0) {
 287                 StrBufAppendBufPlain(QMessage, HKEY("\nbounceto|"), 0);
 288                 StrBufAppendBuf(QMessage, MyQItem->BounceTo, 0);
 289         }
 290
 291         if (StrLength(MyQItem->EnvelopeFrom) > 0) {
 292                 StrBufAppendBufPlain(QMessage, HKEY("\nenvelope_from|"), 0);
 293                 StrBufAppendBuf(QMessage, MyQItem->EnvelopeFrom, 0);
 294         }
 295
 296         StrBufAppendBufPlain(QMessage, HKEY("\nretry|"), 0);
 297         StrBufAppendPrintf(QMessage, "%ld",
 298                            MyQItem->Retry);
 299
 300         StrBufAppendBufPlain(QMessage, HKEY("\nattempted|"), 0);
 301         StrBufAppendPrintf(QMessage, "%ld",
 302                            MyQItem->ReattemptWhen);
 303
 304         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 305         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 306         {
 307                 MailQEntry *ThisItem = vQE;
 308
 309                 if (!ThisItem->Active)
 310                 {
 311                         /* skip already sent ones from the spoolfile. */
 312                         continue;
 313                 }
 314                 StrBufAppendBufPlain(QMessage, HKEY("\nremote|"), 0);
 315                 StrBufAppendBuf(QMessage, ThisItem->Recipient, 0);
 316                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 317                 StrBufAppendPrintf(QMessage, "%d", ThisItem->Status);
 318                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 319                 StrBufAppendBuf(QMessage, ThisItem->StatusMessage, 0);
 320         }
 321         DeleteHashPos(&It);
 322         StrBufAppendBufPlain(QMessage, HKEY("\n"), 0);
 323         return QMessage;
 324 }
 325
 326
 327
 328
 329
 330 void NewMailQEntry(OneQueItem *Item)
 331 {
 332         Item->Current = (MailQEntry*) malloc(sizeof(MailQEntry));
 333         memset(Item->Current, 0, sizeof(MailQEntry));
 334
 335         if (Item->MailQEntries == NULL)
 336                 Item->MailQEntries = NewHash(1, Flathash);
 337         Item->Current->StatusMessage = NewStrBuf();
 338         Item->Current->n = GetCount(Item->MailQEntries);
 339         Put(Item->MailQEntries,
 340             IKEY(Item->Current->n),
 341             Item->Current,
 342             FreeMailQEntry);
 343 }
 344
 345 void QItem_Handle_MsgID(OneQueItem *Item, StrBuf *Line, const char **Pos)
 346 {
 347         Item->MessageID = StrBufExtractNext_long(Line, Pos, '|');
 348 }
 349
 350 void QItem_Handle_EnvelopeFrom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 351 {
 352         if (Item->EnvelopeFrom == NULL)
 353                 Item->EnvelopeFrom = NewStrBufPlain(NULL, StrLength(Line));
 354         StrBufExtract_NextToken(Item->EnvelopeFrom, Line, Pos, '|');
 355 }
 356
 357 void QItem_Handle_BounceTo(OneQueItem *Item, StrBuf *Line, const char **Pos)
 358 {
 359         if (Item->BounceTo == NULL)
 360                 Item->BounceTo = NewStrBufPlain(NULL, StrLength(Line));
 361         StrBufExtract_NextToken(Item->BounceTo, Line, Pos, '|');
 362 }
 363
 364 void QItem_Handle_Recipient(OneQueItem *Item, StrBuf *Line, const char **Pos)
 365 {
 366         if (Item->Current == NULL)
 367                 NewMailQEntry(Item);
 368         if (Item->Current->Recipient == NULL)
 369                 Item->Current->Recipient=NewStrBufPlain(NULL, StrLength(Line));
 370         StrBufExtract_NextToken(Item->Current->Recipient, Line, Pos, '|');
 371         Item->Current->Status = StrBufExtractNext_int(Line, Pos, '|');
 372         StrBufExtract_NextToken(Item->Current->StatusMessage, Line, Pos, '|');
 373         Item->Current = NULL; // TODO: is this always right?
 374 }
 375
 376
 377 void QItem_Handle_retry(OneQueItem *Item, StrBuf *Line, const char **Pos)
 378 {
 379         Item->Retry =
 380                 StrBufExtractNext_int(Line, Pos, '|');
 381         Item->Retry *= 2;
 382 }
 383
 384
 385 void QItem_Handle_Submitted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 386 {
 387         Item->Submitted = atol(*Pos);
 388
 389 }
 390
 391 void QItem_Handle_Attempted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 392 {
 393         Item->ReattemptWhen = StrBufExtractNext_int(Line, Pos, '|');
 394 }
 395
 396
 397
 398 /**
 399  * this one has to have the context for loading the message via the redirect buffer...
 400  */
 401 StrBuf *smtp_load_msg(OneQueItem *MyQItem, int n)
 402 {
 403         CitContext *CCC=CC;
 404         StrBuf *SendMsg;
 405
 406         CCC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
 407         CtdlOutputMsg(MyQItem->MessageID,
 408                       MT_RFC822, HEADERS_ALL,
 409                       0, 1, NULL,
 410                       (ESC_DOT|SUPPRESS_ENV_TO) );
 411
 412         SendMsg = CCC->redirect_buffer;
 413         CCC->redirect_buffer = NULL;
 414         if ((StrLength(SendMsg) > 0) &&
 415             ChrPtr(SendMsg)[StrLength(SendMsg) - 1] != '\n') {
 416                 syslog(LOG_WARNING,
 417                        "SMTP client[%d]: Possible problem: message did not "
 418                        "correctly terminate. (expecting 0x10, got 0x%02x)\n",
 419                        MsgCount, //yes uncool, but best choice here...
 420                        ChrPtr(SendMsg)[StrLength(SendMsg) - 1] );
 421                 StrBufAppendBufPlain(SendMsg, HKEY("\r\n"), 0);
 422         }
 423         return SendMsg;
 424 }
 425
 426
 427
 428 /*
 429  * smtp_do_bounce() is caled by smtp_do_procmsg() to scan a set of delivery
 430  * instructions for "5" codes (permanent fatal errors) and produce/deliver
 431  * a "bounce" message (delivery status notification).
 432  */
 433 void smtpq_do_bounce(OneQueItem *MyQItem, StrBuf *OMsgTxt)
 434 {
 435         static int seq = 0;
 436
 437         struct CtdlMessage *bmsg = NULL;
 438         StrBuf *boundary;
 439         StrBuf *Msg = NULL;
 440         StrBuf *BounceMB;
 441         struct recptypes *valid;
 442
 443         HashPos *It;
 444         void *vQE;
 445         long len;
 446         const char *Key;
 447
 448         int successful_bounce = 0;
 449         int num_bounces = 0;
 450         int give_up = 0;
 451
 452         syslog(LOG_DEBUG, "smtp_do_bounce() called\n");
 453
 454         if ( (ev_time() - MyQItem->Submitted) > SMTP_GIVE_UP ) {
 455                 give_up = 1;/// TODO: replace time by libevq timer get
 456         }
 457
 458         /*
 459          * Now go through the instructions checking for stuff.
 460          */
 461         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 462         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 463         {
 464                 MailQEntry *ThisItem = vQE;
 465                 if ((ThisItem->Status == 5) || /* failed now? */
 466                     ((give_up == 1) &&
 467                      (ThisItem->Status != 2)))
 468                         /* giving up after failed attempts... */
 469                 {
 470                         if (num_bounces == 0)
 471                                 Msg = NewStrBufPlain(NULL, 1024);
 472                         ++num_bounces;
 473
 474                         StrBufAppendBuf(Msg, ThisItem->Recipient, 0);
 475                         StrBufAppendBufPlain(Msg, HKEY(": "), 0);
 476                         StrBufAppendBuf(Msg, ThisItem->StatusMessage, 0);
 477                         StrBufAppendBufPlain(Msg, HKEY("\r\n"), 0);
 478                 }
 479         }
 480         DeleteHashPos(&It);
 481
 482         /* Deliver the bounce if there's anything worth mentioning */
 483         syslog(LOG_DEBUG, "num_bounces = %d\n", num_bounces);
 484
 485         if (num_bounces == 0) {
 486                 FreeStrBuf(&Msg);
 487                 return;
 488         }
 489
 490         boundary = NewStrBufPlain(HKEY("=_Citadel_Multipart_"));
 491         StrBufAppendPrintf(boundary,
 492                            "%s_%04x%04x",
 493                            config.c_fqdn,
 494                            getpid(),
 495                            ++seq);
 496
 497         /* Start building our bounce message; go shopping for memory first. */
 498         BounceMB = NewStrBufPlain(
 499                 NULL,
 500                 1024 + /* mime stuff.... */
 501                 StrLength(Msg) +  /* the bounce information... */
 502                 StrLength(OMsgTxt)); /* the original message */
 503         if (BounceMB == NULL) {
 504                 FreeStrBuf(&boundary);
 505                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 506
 507                 return;
 508         }
 509
 510         bmsg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
 511         if (bmsg == NULL) {
 512                 FreeStrBuf(&boundary);
 513                 FreeStrBuf(&BounceMB);
 514                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 515
 516                 return;
 517         }
 518         memset(bmsg, 0, sizeof(struct CtdlMessage));
 519
 520
 521         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: multipart/mixed; boundary=\""), 0);
 522         StrBufAppendBuf(BounceMB, boundary, 0);
 523         StrBufAppendBufPlain(BounceMB, HKEY("\"\r\n"), 0);
 524         StrBufAppendBufPlain(BounceMB, HKEY("MIME-Version: 1.0\r\n"), 0);
 525         StrBufAppendBufPlain(BounceMB, HKEY("X-Mailer: " CITADEL "\r\n"), 0);
 526         StrBufAppendBufPlain(BounceMB, HKEY("\r\nThis is a multipart message in MIME format.\r\n\r\n"), 0);
 527         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 528         StrBufAppendBuf(BounceMB, boundary, 0);
 529         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 530         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: text/plain\r\n\r\n"), 0);
 531
 532         if (give_up)
 533                 StrBufAppendBufPlain(
 534                         BounceMB,
 535                         HKEY(
 536                                 "A message you sent could not be delivered "
 537                                 "to some or all of its recipients\n"
 538                                 "due to prolonged unavailability "
 539                                 "of its destination(s).\n"
 540                                 "Giving up on the following addresses:\n\n"
 541                                 ), 0);
 542         else
 543                 StrBufAppendBufPlain(
 544                         BounceMB,
 545                         HKEY(
 546                                 "A message you sent could not be delivered "
 547                                 "to some or all of its recipients.\n"
 548                                 "The following addresses "
 549                                 "were undeliverable:\n\n"
 550                                 ), 0);
 551
 552         StrBufAppendBuf(BounceMB, Msg, 0);
 553         FreeStrBuf(&Msg);
 554
 555         /* Attach the original message */
 556         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 557         StrBufAppendBuf(BounceMB, boundary, 0);
 558         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 559         StrBufAppendBufPlain(BounceMB,
 560                              HKEY("Content-type: message/rfc822\r\n"), 0);
 561         StrBufAppendBufPlain(BounceMB,
 562                              HKEY("Content-Transfer-Encoding: 7bit\r\n"), 0);
 563         StrBufAppendBufPlain(BounceMB,
 564                              HKEY("Content-Disposition: inline\r\n"), 0);
 565         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 566         StrBufAppendBuf(BounceMB, OMsgTxt, 0);
 567
 568         /* Close the multipart MIME scope */
 569         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 570         StrBufAppendBuf(BounceMB, boundary, 0);
 571         StrBufAppendBufPlain(BounceMB, HKEY("--\r\n"), 0);
 572
 573         bmsg->cm_magic = CTDLMESSAGE_MAGIC;
 574         bmsg->cm_anon_type = MES_NORMAL;
 575         bmsg->cm_format_type = FMT_RFC822;
 576
 577         bmsg->cm_fields['O'] = strdup(MAILROOM);
 578         bmsg->cm_fields['A'] = strdup("Citadel");
 579         bmsg->cm_fields['N'] = strdup(config.c_nodename);
 580         bmsg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
 581         bmsg->cm_fields['M'] = SmashStrBuf(&BounceMB);
 582
 583         /* First try the user who sent the message */
 584         if (StrLength(MyQItem->BounceTo) == 0)
 585                 syslog(LOG_ERR, "No bounce address specified\n");
 586         else
 587                 syslog(LOG_DEBUG, "bounce to user? <%s>\n",
 588                        ChrPtr(MyQItem->BounceTo));
 589
 590         /* Can we deliver the bounce to the original sender? */
 591         valid = validate_recipients(ChrPtr(MyQItem->BounceTo), NULL, 0);
 592         if ((valid != NULL) && (valid->num_error == 0)) {
 593                 CtdlSubmitMsg(bmsg, valid, "", QP_EADDR);
 594                 successful_bounce = 1;
 595         }
 596
 597         /* If not, post it in the Aide> room */
 598         if (successful_bounce == 0) {
 599                 CtdlSubmitMsg(bmsg, NULL, config.c_aideroom, QP_EADDR);
 600         }
 601
 602         /* Free up the memory we used */
 603         free_recipients(valid);
 604         FreeStrBuf(&boundary);
 605         CtdlFreeMessage(bmsg);
 606         syslog(LOG_DEBUG, "Done processing bounces\n");
 607 }
 608
 609 /*
 610  * smtp_do_procmsg()
 611  *
 612  * Called by smtp_do_queue() to handle an individual message.
 613  */
 614 void smtp_do_procmsg(long msgnum, void *userdata) {
 615         struct CtdlMessage *msg = NULL;
 616         char *instr = NULL;
 617         StrBuf *PlainQItem;
 618         OneQueItem *MyQItem;
 619         char *pch;
 620         HashPos  *It;
 621         void *vQE;
 622         long len;
 623         const char *Key;
 624         int nRelays = 0;
 625         ParsedURL *RelayUrls = NULL;
 626         int HaveBuffers = 0;
 627         StrBuf *Msg =NULL;
 628
 629         syslog(LOG_DEBUG, "SMTP Queue: smtp_do_procmsg(%ld)\n", msgnum);
 630         ///strcpy(envelope_from, "");
 631
 632         msg = CtdlFetchMessage(msgnum, 1);
 633         if (msg == NULL) {
 634                 syslog(LOG_ERR, "SMTP Queue: tried %ld but no such message!\n",
 635                        msgnum);
 636                 return;
 637         }
 638
 639         pch = instr = msg->cm_fields['M'];
 640
 641         /* Strip out the headers (no not amd any other non-instruction) line */
 642         while (pch != NULL) {
 643                 pch = strchr(pch, '\n');
 644                 if ((pch != NULL) && (*(pch + 1) == '\n')) {
 645                         instr = pch + 2;
 646                         pch = NULL;
 647                 }
 648         }
 649         PlainQItem = NewStrBufPlain(instr, -1);
 650         CtdlFreeMessage(msg);
 651         MyQItem = DeserializeQueueItem(PlainQItem, msgnum);
 652         FreeStrBuf(&PlainQItem);
 653
 654         if (MyQItem == NULL) {
 655                 syslog(LOG_ERR,
 656                        "SMTP Queue: Msg No %ld: already in progress!\n",
 657                        msgnum);
 658                 return; /* s.b. else is already processing... */
 659         }
 660
 661         /*
 662          * Postpone delivery if we've already tried recently.
 663          */
 664         if (((time(NULL) - MyQItem->ReattemptWhen) > 0) &&
 665             (run_queue_now == 0))
 666         {
 667                 syslog(LOG_DEBUG, "SMTP client: Retry time not yet reached.\n");
 668
 669                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 670                 pthread_mutex_lock(&ActiveQItemsLock);
 671                 {
 672                         if (GetHashPosFromKey(ActiveQItems,
 673                                               LKEY(MyQItem->MessageID),
 674                                               It))
 675                         {
 676                                 DeleteEntryFromHash(ActiveQItems, It);
 677                         }
 678                 }
 679                 pthread_mutex_unlock(&ActiveQItemsLock);
 680                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 681                 DeleteHashPos(&It);
 682                 return;
 683         }
 684
 685         /*
 686          * Bail out if there's no actual message associated with this
 687          */
 688         if (MyQItem->MessageID < 0L) {
 689                 syslog(LOG_ERR, "SMTP Queue: no 'msgid' directive found!\n");
 690                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 691                 pthread_mutex_lock(&ActiveQItemsLock);
 692                 {
 693                         if (GetHashPosFromKey(ActiveQItems,
 694                                               LKEY(MyQItem->MessageID),
 695                                               It))
 696                         {
 697                                 DeleteEntryFromHash(ActiveQItems, It);
 698                         }
 699                 }
 700                 pthread_mutex_unlock(&ActiveQItemsLock);
 701                 DeleteHashPos(&It);
 702                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 703                 return;
 704         }
 705
 706         {
 707                 char mxbuf[SIZ];
 708                 ParsedURL **Url = &MyQItem->URL;
 709                 nRelays = get_hosts(mxbuf, "smarthost");
 710                 if (nRelays > 0) {
 711                         StrBuf *All;
 712                         StrBuf *One;
 713                         const char *Pos = NULL;
 714                         All = NewStrBufPlain(mxbuf, -1);
 715                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 716
 717                         while ((Pos != StrBufNOTNULL) &&
 718                                ((Pos == NULL) ||
 719                                 !IsEmptyStr(Pos)))
 720                         {
 721                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 722                                 if (!ParseURL(Url, One, 25))
 723                                         syslog(LOG_DEBUG,
 724                                                "Failed to parse: %s\n",
 725                                                ChrPtr(One));
 726                                 else {
 727                                         ///if (!Url->IsIP)) // todo dupe me fork ipv6
 728                                         Url = &(*Url)->Next;
 729                                 }
 730                         }
 731                         FreeStrBuf(&All);
 732                         FreeStrBuf(&One);
 733                 }
 734
 735                 Url = &MyQItem->FallBackHost;
 736                 nRelays = get_hosts(mxbuf, "fallbackhost");
 737                 if (nRelays > 0) {
 738                         StrBuf *All;
 739                         StrBuf *One;
 740                         const char *Pos = NULL;
 741                         All = NewStrBufPlain(mxbuf, -1);
 742                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 743
 744                         while ((Pos != StrBufNOTNULL) &&
 745                                ((Pos == NULL) ||
 746                                 !IsEmptyStr(Pos)))
 747                         {
 748                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 749                                 if (!ParseURL(Url, One, 25))
 750                                         syslog(LOG_DEBUG,
 751                                                "Failed to parse: %s\n",
 752                                                ChrPtr(One));
 753                                 else
 754                                         Url = &(*Url)->Next;
 755                         }
 756                         FreeStrBuf(&All);
 757                         FreeStrBuf(&One);
 758                 }
 759         }
 760
 761         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 762         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 763         {
 764                 MailQEntry *ThisItem = vQE;
 765                 syslog(LOG_DEBUG, "SMTP Queue: Task: <%s> %d\n",
 766                        ChrPtr(ThisItem->Recipient),
 767                        ThisItem->Active);
 768         }
 769         DeleteHashPos(&It);
 770
 771         MyQItem->ActiveDeliveries = CountActiveQueueEntries(MyQItem);
 772         if (MyQItem->ActiveDeliveries > 0)
 773         {
 774                 int n = MsgCount++;
 775                 int m = MyQItem->ActiveDeliveries;
 776                 int i = 1;
 777                 Msg = smtp_load_msg(MyQItem, n);
 778                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 779                 while ((i <= m) &&
 780                        (GetNextHashPos(MyQItem->MailQEntries,
 781                                        It, &len, &Key, &vQE)))
 782                 {
 783                         MailQEntry *ThisItem = vQE;
 784
 785                         if (ThisItem->Active == 1)
 786                         {
 787                                 int KeepBuffers = (i == m);
 788                                 if (i > 1) n = MsgCount++;
 789                                 syslog(LOG_DEBUG,
 790                                        "SMTPQ: Trying <%ld> <%s> %d / %d \n",
 791                                        MyQItem->MessageID,
 792                                        ChrPtr(ThisItem->Recipient),
 793                                        i,
 794                                        m);
 795                                 smtp_try_one_queue_entry(MyQItem,
 796                                                          ThisItem,
 797                                                          Msg,
 798                                                          KeepBuffers,
 799                                                          n,
 800                                                          RelayUrls);
 801
 802                                 if (KeepBuffers) HaveBuffers = 1;
 803
 804                                 i++;
 805                         }
 806                 }
 807                 DeleteHashPos(&It);
 808         }
 809         else
 810         {
 811                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 812                 pthread_mutex_lock(&ActiveQItemsLock);
 813                 {
 814                         if (GetHashPosFromKey(ActiveQItems,
 815                                               LKEY(MyQItem->MessageID),
 816                                               It))
 817                         {
 818                                 DeleteEntryFromHash(ActiveQItems, It);
 819                         }
 820                         else
 821                         {
 822                                 long len;
 823                                 const char* Key;
 824                                 void *VData;
 825
 826                                 syslog(LOG_WARNING,
 827                                        "SMTP cleanup: unable to find "
 828                                        "QItem with ID[%ld]",
 829                                        MyQItem->MessageID);
 830                                 while (GetNextHashPos(ActiveQItems,
 831                                                       It,
 832                                                       &len,
 833                                                       &Key,
 834                                                       &VData))
 835                                 {
 836                                         syslog(LOG_WARNING,
 837                                                "SMTP cleanup: have: ID[%ld]",
 838                                               ((OneQueItem *)VData)->MessageID);
 839                                 }
 840                         }
 841
 842                 }
 843                 pthread_mutex_unlock(&ActiveQItemsLock);
 844                 DeleteHashPos(&It);
 845                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 846
 847 // TODO: bounce & delete?
 848
 849         }
 850         if (!HaveBuffers) {
 851                 FreeStrBuf (&Msg);
 852 // TODO : free RelayUrls
 853         }
 854 }
 855
 856
 857
 858 /*
 859  * smtp_queue_thread()
 860  *
 861  * Run through the queue sending out messages.
 862  */
 863 void smtp_do_queue(void) {
 864         static int is_running = 0;
 865         int num_processed = 0;
 866
 867         if (is_running)
 868                 return; /* Concurrency check - only one can run */
 869         is_running = 1;
 870
 871         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 872         syslog(LOG_INFO, "SMTP client: processing outbound queue");
 873
 874         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
 875                 syslog(LOG_ERR, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
 876         }
 877         else {
 878                 num_processed = CtdlForEachMessage(MSGS_ALL,
 879                                                    0L,
 880                                                    NULL,
 881                                                    SPOOLMIME,
 882                                                    NULL,
 883                                                    smtp_do_procmsg,
 884                                                    NULL);
 885         }
 886         syslog(LOG_INFO,
 887                "SMTP client: queue run completed; %d messages processed",
 888                num_processed);
 889
 890         run_queue_now = 0;
 891         is_running = 0;
 892 }
 893
 894
 895
 896 /*
 897  * Initialize the SMTP outbound queue
 898  */
 899 void smtp_init_spoolout(void) {
 900         struct ctdlroom qrbuf;
 901
 902         /*
 903          * Create the room.  This will silently fail if the room already
 904          * exists, and that's perfectly ok, because we want it to exist.
 905          */
 906         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_MAILBOX);
 907
 908         /*
 909          * Make sure it's set to be a "system room" so it doesn't show up
 910          * in the <K>nown rooms list for Aides.
 911          */
 912         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
 913                 qrbuf.QRflags2 |= QR2_SYSTEM;
 914                 CtdlPutRoomLock(&qrbuf);
 915         }
 916 }
 917
 918
 919
 920
 921 /*****************************************************************************/
 922 /*                          SMTP UTILITY COMMANDS                            */
 923 /*****************************************************************************/
 924
 925 void cmd_smtp(char *argbuf) {
 926         char cmd[64];
 927         char node[256];
 928         char buf[1024];
 929         int i;
 930         int num_mxhosts;
 931
 932         if (CtdlAccessCheck(ac_aide)) return;
 933
 934         extract_token(cmd, argbuf, 0, '|', sizeof cmd);
 935
 936         if (!strcasecmp(cmd, "mx")) {
 937                 extract_token(node, argbuf, 1, '|', sizeof node);
 938                 num_mxhosts = getmx(buf, node);
 939                 cprintf("%d %d MX hosts listed for %s\n",
 940                         LISTING_FOLLOWS, num_mxhosts, node);
 941                 for (i=0; i<num_mxhosts; ++i) {
 942                         extract_token(node, buf, i, '|', sizeof node);
 943                         cprintf("%s\n", node);
 944                 }
 945                 cprintf("000\n");
 946                 return;
 947         }
 948
 949         else if (!strcasecmp(cmd, "runqueue")) {
 950                 run_queue_now = 1;
 951                 cprintf("%d All outbound SMTP will be retried now.\n", CIT_OK);
 952                 return;
 953         }
 954
 955         else {
 956                 cprintf("%d Invalid command.\n", ERROR + ILLEGAL_VALUE);
 957         }
 958
 959 }
 960
 961
 962 CTDL_MODULE_INIT(smtp_queu)
 963 {
 964         if (!threading)
 965         {
 966                 CtdlFillSystemContext(&smtp_queue_CC, "SMTP_Send");
 967                 ActiveQItems = NewHash(1, lFlathash);
 968                 pthread_mutex_init(&ActiveQItemsLock, NULL);
 969
 970                 QItemHandlers = NewHash(0, NULL);
 971
 972                 Put(QItemHandlers, HKEY("msgid"), QItem_Handle_MsgID, reference_free_handler);
 973                 Put(QItemHandlers, HKEY("envelope_from"), QItem_Handle_EnvelopeFrom, reference_free_handler);
 974                 Put(QItemHandlers, HKEY("retry"), QItem_Handle_retry, reference_free_handler);
 975                 Put(QItemHandlers, HKEY("attempted"), QItem_Handle_Attempted, reference_free_handler);
 976                 Put(QItemHandlers, HKEY("remote"), QItem_Handle_Recipient, reference_free_handler);
 977                 Put(QItemHandlers, HKEY("bounceto"), QItem_Handle_BounceTo, reference_free_handler);
 978                 Put(QItemHandlers, HKEY("submitted"), QItem_Handle_Submitted, reference_free_handler);
 979
 980                 smtp_init_spoolout();
 981
 982                 CtdlRegisterCleanupHook(smtp_evq_cleanup);
 983
 984                 CtdlRegisterProtoHook(cmd_smtp, "SMTP", "SMTP utility commands");
 985                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER);
 986         }
 987
 988         /* return our Subversion id for the Log */
 989         return "smtpeventclient";
 990 }