citadel/modules/smtp/serv_smtpqueue.c

   1 /*
   2  * This module is an SMTP and ESMTP implementation for the Citadel system.
   3  * It is compliant with all of the following:
   4  *
   5  * RFC  821 - Simple Mail Transfer Protocol
   6  * RFC  876 - Survey of SMTP Implementations
   7  * RFC 1047 - Duplicate messages and SMTP
   8  * RFC 1652 - 8 bit MIME
   9  * RFC 1869 - Extended Simple Mail Transfer Protocol
  10  * RFC 1870 - SMTP Service Extension for Message Size Declaration
  11  * RFC 2033 - Local Mail Transfer Protocol
  12  * RFC 2197 - SMTP Service Extension for Command Pipelining
  13  * RFC 2476 - Message Submission
  14  * RFC 2487 - SMTP Service Extension for Secure SMTP over TLS
  15  * RFC 2554 - SMTP Service Extension for Authentication
  16  * RFC 2821 - Simple Mail Transfer Protocol
  17  * RFC 2822 - Internet Message Format
  18  * RFC 2920 - SMTP Service Extension for Command Pipelining
  19  *
  20  * The VRFY and EXPN commands have been removed from this implementation
  21  * because nobody uses these commands anymore, except for spammers.
  22  *
  23  * Copyright (c) 1998-2009 by the citadel.org team
  24  *
  25  *  This program is free software; you can redistribute it and/or modify
  26  *  it under the terms of the GNU General Public License as published by
  27  *  the Free Software Foundation; either version 3 of the License, or
  28  *  (at your option) any later version.
  29  *
  30  *  This program is distributed in the hope that it will be useful,
  31  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  32  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  33  *  GNU General Public License for more details.
  34  *
  35  *  You should have received a copy of the GNU General Public License
  36  *  along with this program; if not, write to the Free Software
  37  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  38  */
  39
  40 #include "sysdep.h"
  41 #include <stdlib.h>
  42 #include <unistd.h>
  43 #include <stdio.h>
  44 #include <termios.h>
  45 #include <fcntl.h>
  46 #include <signal.h>
  47 #include <pwd.h>
  48 #include <errno.h>
  49 #include <sys/types.h>
  50 #include <syslog.h>
  51
  52 #if TIME_WITH_SYS_TIME
  53 # include <sys/time.h>
  54 # include <time.h>
  55 #else
  56 # if HAVE_SYS_TIME_H
  57 #  include <sys/time.h>
  58 # else
  59 #  include <time.h>
  60 # endif
  61 #endif
  62 #include <sys/wait.h>
  63 #include <ctype.h>
  64 #include <string.h>
  65 #include <limits.h>
  66 #include <sys/socket.h>
  67 #include <netinet/in.h>
  68 #include <arpa/inet.h>
  69 #include <libcitadel.h>
  70 #include "citadel.h"
  71 #include "server.h"
  72 #include "citserver.h"
  73 #include "support.h"
  74 #include "config.h"
  75 #include "control.h"
  76 #include "user_ops.h"
  77 #include "database.h"
  78 #include "msgbase.h"
  79 #include "internet_addressing.h"
  80 #include "genstamp.h"
  81 #include "domain.h"
  82 #include "clientsocket.h"
  83 #include "locate_host.h"
  84 #include "citadel_dirs.h"
  85
  86 #include "ctdl_module.h"
  87
  88 #include "smtpqueue.h"
  89 #include "event_client.h"
  90
  91
  92 struct CitContext smtp_queue_CC;
  93 pthread_mutex_t ActiveQItemsLock;
  94 HashList *ActiveQItems  = NULL;
  95 HashList *QItemHandlers = NULL;
  96
  97 int MsgCount            = 0;
  98 int run_queue_now       = 0;    /* Set to 1 to ignore SMTP send retry times */
  99
 100 void smtp_try_one_queue_entry(OneQueItem *MyQItem,
 101                               MailQEntry *MyQEntry,
 102                               StrBuf *MsgText,
 103 /* KeepMsgText allows us to use MsgText as ours. */
 104                               int KeepMsgText,
 105                               int MsgCount,
 106                               ParsedURL *RelayUrls);
 107
 108
 109 void smtp_evq_cleanup(void)
 110 {
 111
 112         pthread_mutex_lock(&ActiveQItemsLock);
 113         DeleteHash(&QItemHandlers);
 114         DeleteHash(&ActiveQItems);
 115         pthread_mutex_unlock(&ActiveQItemsLock);
 116         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 117 /*      citthread_mutex_destroy(&ActiveQItemsLock); TODO */
 118 }
 119
 120 int DecreaseQReference(OneQueItem *MyQItem)
 121 {
 122         int IDestructQueItem;
 123
 124         pthread_mutex_lock(&ActiveQItemsLock);
 125         MyQItem->ActiveDeliveries--;
 126         IDestructQueItem = MyQItem->ActiveDeliveries == 0;
 127         pthread_mutex_unlock(&ActiveQItemsLock);
 128         return IDestructQueItem;
 129 }
 130
 131 void RemoveQItem(OneQueItem *MyQItem)
 132 {
 133         long len;
 134         const char* Key;
 135         void *VData;
 136         HashPos  *It;
 137
 138         pthread_mutex_lock(&ActiveQItemsLock);
 139         It = GetNewHashPos(ActiveQItems, 0);
 140         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 141                 DeleteEntryFromHash(ActiveQItems, It);
 142         else
 143         {
 144                 syslog(LOG_WARNING,
 145                        "SMTP cleanup: unable to find QItem with ID[%ld]",
 146                        MyQItem->MessageID);
 147                 while (GetNextHashPos(ActiveQItems, It, &len, &Key, &VData))
 148                         syslog(LOG_WARNING,
 149                                "SMTP cleanup: have_: ID[%ld]",
 150                                ((OneQueItem *)VData)->MessageID);
 151         }
 152         pthread_mutex_unlock(&ActiveQItemsLock);
 153         DeleteHashPos(&It);
 154 }
 155
 156
 157 void FreeMailQEntry(void *qv)
 158 {
 159         MailQEntry *Q = qv;
 160         FreeStrBuf(&Q->Recipient);
 161         FreeStrBuf(&Q->StatusMessage);
 162         free(Q);
 163 }
 164 void FreeQueItem(OneQueItem **Item)
 165 {
 166         DeleteHash(&(*Item)->MailQEntries);
 167         FreeStrBuf(&(*Item)->EnvelopeFrom);
 168         FreeStrBuf(&(*Item)->BounceTo);
 169         FreeURL(&(*Item)->URL);
 170         free(*Item);
 171         Item = NULL;
 172 }
 173 void HFreeQueItem(void *Item)
 174 {
 175         FreeQueItem((OneQueItem**)&Item);
 176 }
 177
 178 /* inspect recipients with a status of:
 179  * - 0 (no delivery yet attempted)
 180  * - 3/4 (transient errors
 181  *        were experienced and it's time to try again)
 182  */
 183 int CountActiveQueueEntries(OneQueItem *MyQItem)
 184 {
 185         HashPos  *It;
 186         long len;
 187         long ActiveDeliveries;
 188         const char *Key;
 189         void *vQE;
 190
 191         ActiveDeliveries = 0;
 192         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 193         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 194         {
 195                 MailQEntry *ThisItem = vQE;
 196                 if ((ThisItem->Status == 0) ||
 197                     (ThisItem->Status == 3) ||
 198                     (ThisItem->Status == 4))
 199                 {
 200                         ActiveDeliveries++;
 201                         ThisItem->Active = 1;
 202                 }
 203                 else
 204                         ThisItem->Active = 0;
 205         }
 206         DeleteHashPos(&It);
 207         return ActiveDeliveries;
 208 }
 209
 210 OneQueItem *DeserializeQueueItem(StrBuf *RawQItem, long QueMsgID)
 211 {
 212         OneQueItem *Item;
 213         const char *pLine = NULL;
 214         StrBuf *Line;
 215         StrBuf *Token;
 216         void *v;
 217
 218         Item = (OneQueItem*)malloc(sizeof(OneQueItem));
 219         memset(Item, 0, sizeof(OneQueItem));
 220         Item->LastAttempt.retry = SMTP_RETRY_INTERVAL;
 221         Item->MessageID = -1;
 222         Item->QueMsgID = QueMsgID;
 223
 224         Token = NewStrBuf();
 225         Line = NewStrBufPlain(NULL, 128);
 226         while (pLine != StrBufNOTNULL) {
 227                 const char *pItemPart = NULL;
 228                 void *vHandler;
 229
 230                 StrBufExtract_NextToken(Line, RawQItem, &pLine, '\n');
 231                 if (StrLength(Line) == 0) continue;
 232                 StrBufExtract_NextToken(Token, Line, &pItemPart, '|');
 233                 if (GetHash(QItemHandlers, SKEY(Token), &vHandler))
 234                 {
 235                         QItemHandler H;
 236                         H = (QItemHandler) vHandler;
 237                         H(Item, Line, &pItemPart);
 238                 }
 239         }
 240         FreeStrBuf(&Line);
 241         FreeStrBuf(&Token);
 242
 243         pthread_mutex_lock(&ActiveQItemsLock);
 244         if (GetHash(ActiveQItems,
 245                     LKEY(Item->MessageID),
 246                     &v))
 247         {
 248                 /* WHOOPS. somebody else is already working on this. */
 249                 pthread_mutex_unlock(&ActiveQItemsLock);
 250                 FreeQueItem(&Item);
 251                 return NULL;
 252         }
 253         else {
 254                 /* mark our claim on this. */
 255                 Put(ActiveQItems,
 256                     LKEY(Item->MessageID),
 257                     Item,
 258                     HFreeQueItem);
 259                 pthread_mutex_unlock(&ActiveQItemsLock);
 260         }
 261
 262         return Item;
 263 }
 264
 265 StrBuf *SerializeQueueItem(OneQueItem *MyQItem)
 266 {
 267         StrBuf *QMessage;
 268         HashPos  *It;
 269         const char *Key;
 270         long len;
 271         void *vQE;
 272
 273         QMessage = NewStrBufPlain(NULL, SIZ);
 274         StrBufPrintf(QMessage, "Content-type: %s\n", SPOOLMIME);
 275 //      "attempted|%ld\n"  "retry|%ld\n",, (long)time(NULL), (long)retry );
 276         StrBufAppendBufPlain(QMessage, HKEY("\nmsgid|"), 0);
 277         StrBufAppendPrintf(QMessage, "%ld", MyQItem->MessageID);
 278
 279         StrBufAppendBufPlain(QMessage, HKEY("\nsubmitted|"), 0);
 280         StrBufAppendPrintf(QMessage, "%ld", MyQItem->Submitted);
 281
 282         if (StrLength(MyQItem->BounceTo) > 0) {
 283                 StrBufAppendBufPlain(QMessage, HKEY("\nbounceto|"), 0);
 284                 StrBufAppendBuf(QMessage, MyQItem->BounceTo, 0);
 285         }
 286
 287         if (StrLength(MyQItem->EnvelopeFrom) > 0) {
 288                 StrBufAppendBufPlain(QMessage, HKEY("\nenvelope_from|"), 0);
 289                 StrBufAppendBuf(QMessage, MyQItem->EnvelopeFrom, 0);
 290         }
 291
 292         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 293         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 294         {
 295                 MailQEntry *ThisItem = vQE;
 296                 int i;
 297
 298                 if (!ThisItem->Active)
 299                 {
 300                         /* skip already sent ones from the spoolfile. */
 301                         continue;
 302                 }
 303
 304                 for (i=0; i < ThisItem->nAttempts; i++) {
 305                         /* TODO: most probably
 306                          * there is just one retry/attempted per message!
 307                          */
 308                         StrBufAppendBufPlain(QMessage, HKEY("\nretry|"), 0);
 309                         StrBufAppendPrintf(QMessage, "%ld",
 310                                            ThisItem->Attempts[i].retry);
 311
 312                         StrBufAppendBufPlain(QMessage, HKEY("\nattempted|"), 0);
 313                         StrBufAppendPrintf(QMessage, "%ld",
 314                                            ThisItem->Attempts[i].when);
 315                 }
 316                 StrBufAppendBufPlain(QMessage, HKEY("\nremote|"), 0);
 317                 StrBufAppendBuf(QMessage, ThisItem->Recipient, 0);
 318                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 319                 StrBufAppendPrintf(QMessage, "%d", ThisItem->Status);
 320                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 321                 StrBufAppendBuf(QMessage, ThisItem->StatusMessage, 0);
 322         }
 323         DeleteHashPos(&It);
 324         StrBufAppendBufPlain(QMessage, HKEY("\n"), 0);
 325         return QMessage;
 326 }
 327
 328
 329
 330
 331
 332 void NewMailQEntry(OneQueItem *Item)
 333 {
 334         Item->Current = (MailQEntry*) malloc(sizeof(MailQEntry));
 335         memset(Item->Current, 0, sizeof(MailQEntry));
 336
 337         if (Item->MailQEntries == NULL)
 338                 Item->MailQEntries = NewHash(1, Flathash);
 339         Item->Current->StatusMessage = NewStrBuf();
 340         Item->Current->n = GetCount(Item->MailQEntries);
 341         Put(Item->MailQEntries,
 342             IKEY(Item->Current->n),
 343             Item->Current,
 344             FreeMailQEntry);
 345 }
 346
 347 void QItem_Handle_MsgID(OneQueItem *Item, StrBuf *Line, const char **Pos)
 348 {
 349         Item->MessageID = StrBufExtractNext_long(Line, Pos, '|');
 350 }
 351
 352 void QItem_Handle_EnvelopeFrom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 353 {
 354         if (Item->EnvelopeFrom == NULL)
 355                 Item->EnvelopeFrom = NewStrBufPlain(NULL, StrLength(Line));
 356         StrBufExtract_NextToken(Item->EnvelopeFrom, Line, Pos, '|');
 357 }
 358
 359 void QItem_Handle_BounceTo(OneQueItem *Item, StrBuf *Line, const char **Pos)
 360 {
 361         if (Item->BounceTo == NULL)
 362                 Item->BounceTo = NewStrBufPlain(NULL, StrLength(Line));
 363         StrBufExtract_NextToken(Item->BounceTo, Line, Pos, '|');
 364 }
 365
 366 void QItem_Handle_Recipient(OneQueItem *Item, StrBuf *Line, const char **Pos)
 367 {
 368         if (Item->Current == NULL)
 369                 NewMailQEntry(Item);
 370         if (Item->Current->Recipient == NULL)
 371                 Item->Current->Recipient=NewStrBufPlain(NULL, StrLength(Line));
 372         StrBufExtract_NextToken(Item->Current->Recipient, Line, Pos, '|');
 373         Item->Current->Status = StrBufExtractNext_int(Line, Pos, '|');
 374         StrBufExtract_NextToken(Item->Current->StatusMessage, Line, Pos, '|');
 375         Item->Current = NULL; // TODO: is this always right?
 376 }
 377
 378
 379 void QItem_Handle_retry(OneQueItem *Item, StrBuf *Line, const char **Pos)
 380 {
 381         if (Item->Current == NULL)
 382                 NewMailQEntry(Item);
 383         if (Item->Current->Attempts[Item->Current->nAttempts].retry != 0)
 384                 Item->Current->nAttempts++;
 385         if (Item->Current->nAttempts > MaxAttempts) {
 386                 Item->FailNow = 1;
 387                 return;
 388         }
 389         Item->Current->Attempts[Item->Current->nAttempts].retry =
 390                 StrBufExtractNext_int(Line, Pos, '|');
 391 }
 392
 393
 394 void QItem_Handle_Submitted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 395 {
 396         Item->Submitted = atol(*Pos);
 397
 398 }
 399
 400 void QItem_Handle_Attempted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 401 {
 402         if (Item->Current == NULL)
 403                 NewMailQEntry(Item);
 404         if (Item->Current->Attempts[Item->Current->nAttempts].when != 0)
 405                 Item->Current->nAttempts++;
 406         if (Item->Current->nAttempts > MaxAttempts) {
 407                 Item->FailNow = 1;
 408                 return;
 409         }
 410
 411         Item->Current->Attempts[Item->Current->nAttempts].when =
 412                 StrBufExtractNext_int(Line, Pos, '|');
 413         if (Item->Current->Attempts[Item->Current->nAttempts].when >
 414             Item->LastAttempt.when)
 415         {
 416                 Item->LastAttempt.when =
 417                         Item->Current->Attempts[Item->Current->nAttempts].when;
 418
 419                 Item->LastAttempt.retry =
 420                         Item->Current->Attempts[
 421                                 Item->Current->nAttempts
 422                                 ].retry * 2;
 423
 424                 if (Item->LastAttempt.retry > SMTP_RETRY_MAX)
 425                         Item->LastAttempt.retry = SMTP_RETRY_MAX;
 426         }
 427 }
 428
 429
 430
 431 /**
 432  * this one has to have the context for loading the message via the redirect buffer...
 433  */
 434 StrBuf *smtp_load_msg(OneQueItem *MyQItem, int n)
 435 {
 436         CitContext *CCC=CC;
 437         StrBuf *SendMsg;
 438
 439         CCC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
 440         CtdlOutputMsg(MyQItem->MessageID,
 441                       MT_RFC822, HEADERS_ALL,
 442                       0, 1, NULL,
 443                       (ESC_DOT|SUPPRESS_ENV_TO) );
 444
 445         SendMsg = CCC->redirect_buffer;
 446         CCC->redirect_buffer = NULL;
 447         if ((StrLength(SendMsg) > 0) &&
 448             ChrPtr(SendMsg)[StrLength(SendMsg) - 1] != '\n') {
 449                 syslog(LOG_WARNING,
 450                        "SMTP client[%d]: Possible problem: message did not "
 451                        "correctly terminate. (expecting 0x10, got 0x%02x)\n",
 452                        MsgCount, //yes uncool, but best choice here...
 453                        ChrPtr(SendMsg)[StrLength(SendMsg) - 1] );
 454                 StrBufAppendBufPlain(SendMsg, HKEY("\r\n"), 0);
 455         }
 456         return SendMsg;
 457 }
 458
 459
 460
 461 /*
 462  * smtp_do_bounce() is caled by smtp_do_procmsg() to scan a set of delivery
 463  * instructions for "5" codes (permanent fatal errors) and produce/deliver
 464  * a "bounce" message (delivery status notification).
 465  */
 466 void smtpq_do_bounce(OneQueItem *MyQItem, StrBuf *OMsgTxt)
 467 {
 468         static int seq = 0;
 469
 470         struct CtdlMessage *bmsg = NULL;
 471         StrBuf *boundary;
 472         StrBuf *Msg = NULL;
 473         StrBuf *BounceMB;
 474         struct recptypes *valid;
 475
 476         HashPos *It;
 477         void *vQE;
 478         long len;
 479         const char *Key;
 480
 481         int successful_bounce = 0;
 482         int num_bounces = 0;
 483         int give_up = 0;
 484
 485         syslog(LOG_DEBUG, "smtp_do_bounce() called\n");
 486
 487         if ( (ev_time() - MyQItem->Submitted) > SMTP_GIVE_UP ) {
 488                 give_up = 1;/// TODO: replace time by libevq timer get
 489         }
 490
 491         /*
 492          * Now go through the instructions checking for stuff.
 493          */
 494         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 495         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 496         {
 497                 MailQEntry *ThisItem = vQE;
 498                 if ((ThisItem->Status == 5) || /* failed now? */
 499                     ((give_up == 1) &&
 500                      (ThisItem->Status != 2)))
 501                         /* giving up after failed attempts... */
 502                 {
 503                         if (num_bounces == 0)
 504                                 Msg = NewStrBufPlain(NULL, 1024);
 505                         ++num_bounces;
 506
 507                         StrBufAppendBuf(Msg, ThisItem->Recipient, 0);
 508                         StrBufAppendBufPlain(Msg, HKEY(": "), 0);
 509                         StrBufAppendBuf(Msg, ThisItem->StatusMessage, 0);
 510                         StrBufAppendBufPlain(Msg, HKEY("\r\n"), 0);
 511                 }
 512         }
 513         DeleteHashPos(&It);
 514
 515         /* Deliver the bounce if there's anything worth mentioning */
 516         syslog(LOG_DEBUG, "num_bounces = %d\n", num_bounces);
 517
 518         if (num_bounces == 0) {
 519                 FreeStrBuf(&Msg);
 520                 return;
 521         }
 522
 523         boundary = NewStrBufPlain(HKEY("=_Citadel_Multipart_"));
 524         StrBufAppendPrintf(boundary,
 525                            "%s_%04x%04x",
 526                            config.c_fqdn,
 527                            getpid(),
 528                            ++seq);
 529
 530         /* Start building our bounce message; go shopping for memory first. */
 531         BounceMB = NewStrBufPlain(
 532                 NULL,
 533                 1024 + /* mime stuff.... */
 534                 StrLength(Msg) +  /* the bounce information... */
 535                 StrLength(OMsgTxt)); /* the original message */
 536         if (BounceMB == NULL) {
 537                 FreeStrBuf(&boundary);
 538                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 539
 540                 return;
 541         }
 542
 543         bmsg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
 544         if (bmsg == NULL) {
 545                 FreeStrBuf(&boundary);
 546                 FreeStrBuf(&BounceMB);
 547                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 548
 549                 return;
 550         }
 551         memset(bmsg, 0, sizeof(struct CtdlMessage));
 552
 553
 554         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: multipart/mixed; boundary=\""), 0);
 555         StrBufAppendBuf(BounceMB, boundary, 0);
 556         StrBufAppendBufPlain(BounceMB, HKEY("\"\r\n"), 0);
 557         StrBufAppendBufPlain(BounceMB, HKEY("MIME-Version: 1.0\r\n"), 0);
 558         StrBufAppendBufPlain(BounceMB, HKEY("X-Mailer: " CITADEL "\r\n"), 0);
 559         StrBufAppendBufPlain(BounceMB, HKEY("\r\nThis is a multipart message in MIME format.\r\n\r\n"), 0);
 560         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 561         StrBufAppendBuf(BounceMB, boundary, 0);
 562         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 563         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: text/plain\r\n\r\n"), 0);
 564
 565         if (give_up)
 566                 StrBufAppendBufPlain(
 567                         BounceMB,
 568                         HKEY(
 569                                 "A message you sent could not be delivered "
 570                                 "to some or all of its recipients\n"
 571                                 "due to prolonged unavailability "
 572                                 "of its destination(s).\n"
 573                                 "Giving up on the following addresses:\n\n"
 574                                 ), 0);
 575         else
 576                 StrBufAppendBufPlain(
 577                         BounceMB,
 578                         HKEY(
 579                                 "A message you sent could not be delivered "
 580                                 "to some or all of its recipients.\n"
 581                                 "The following addresses "
 582                                 "were undeliverable:\n\n"
 583                                 ), 0);
 584
 585         StrBufAppendBuf(BounceMB, Msg, 0);
 586         FreeStrBuf(&Msg);
 587
 588         /* Attach the original message */
 589         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 590         StrBufAppendBuf(BounceMB, boundary, 0);
 591         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 592         StrBufAppendBufPlain(BounceMB,
 593                              HKEY("Content-type: message/rfc822\r\n"), 0);
 594         StrBufAppendBufPlain(BounceMB,
 595                              HKEY("Content-Transfer-Encoding: 7bit\r\n"), 0);
 596         StrBufAppendBufPlain(BounceMB,
 597                              HKEY("Content-Disposition: inline\r\n"), 0);
 598         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 599         StrBufAppendBuf(BounceMB, OMsgTxt, 0);
 600
 601         /* Close the multipart MIME scope */
 602         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 603         StrBufAppendBuf(BounceMB, boundary, 0);
 604         StrBufAppendBufPlain(BounceMB, HKEY("--\r\n"), 0);
 605
 606         bmsg->cm_magic = CTDLMESSAGE_MAGIC;
 607         bmsg->cm_anon_type = MES_NORMAL;
 608         bmsg->cm_format_type = FMT_RFC822;
 609
 610         bmsg->cm_fields['O'] = strdup(MAILROOM);
 611         bmsg->cm_fields['A'] = strdup("Citadel");
 612         bmsg->cm_fields['N'] = strdup(config.c_nodename);
 613         bmsg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
 614         bmsg->cm_fields['M'] = SmashStrBuf(&BounceMB);
 615
 616         /* First try the user who sent the message */
 617         if (StrLength(MyQItem->BounceTo) == 0)
 618                 syslog(LOG_ERR, "No bounce address specified\n");
 619         else
 620                 syslog(LOG_DEBUG, "bounce to user? <%s>\n",
 621                        ChrPtr(MyQItem->BounceTo));
 622
 623         /* Can we deliver the bounce to the original sender? */
 624         valid = validate_recipients(ChrPtr(MyQItem->BounceTo), NULL, 0);
 625         if ((valid != NULL) && (valid->num_error == 0)) {
 626                 CtdlSubmitMsg(bmsg, valid, "", QP_EADDR);
 627                 successful_bounce = 1;
 628         }
 629
 630         /* If not, post it in the Aide> room */
 631         if (successful_bounce == 0) {
 632                 CtdlSubmitMsg(bmsg, NULL, config.c_aideroom, QP_EADDR);
 633         }
 634
 635         /* Free up the memory we used */
 636         free_recipients(valid);
 637         FreeStrBuf(&boundary);
 638         CtdlFreeMessage(bmsg);
 639         syslog(LOG_DEBUG, "Done processing bounces\n");
 640 }
 641
 642 /*
 643  * smtp_do_procmsg()
 644  *
 645  * Called by smtp_do_queue() to handle an individual message.
 646  */
 647 void smtp_do_procmsg(long msgnum, void *userdata) {
 648         struct CtdlMessage *msg = NULL;
 649         char *instr = NULL;
 650         StrBuf *PlainQItem;
 651         OneQueItem *MyQItem;
 652         char *pch;
 653         HashPos  *It;
 654         void *vQE;
 655         long len;
 656         const char *Key;
 657         int nRelays = 0;
 658         ParsedURL *RelayUrls = NULL;
 659         int HaveBuffers = 0;
 660         StrBuf *Msg =NULL;
 661
 662         syslog(LOG_DEBUG, "SMTP Queue: smtp_do_procmsg(%ld)\n", msgnum);
 663         ///strcpy(envelope_from, "");
 664
 665         msg = CtdlFetchMessage(msgnum, 1);
 666         if (msg == NULL) {
 667                 syslog(LOG_ERR, "SMTP Queue: tried %ld but no such message!\n",
 668                        msgnum);
 669                 return;
 670         }
 671
 672         pch = instr = msg->cm_fields['M'];
 673
 674         /* Strip out the headers (no not amd any other non-instruction) line */
 675         while (pch != NULL) {
 676                 pch = strchr(pch, '\n');
 677                 if ((pch != NULL) && (*(pch + 1) == '\n')) {
 678                         instr = pch + 2;
 679                         pch = NULL;
 680                 }
 681         }
 682         PlainQItem = NewStrBufPlain(instr, -1);
 683         CtdlFreeMessage(msg);
 684         MyQItem = DeserializeQueueItem(PlainQItem, msgnum);
 685         FreeStrBuf(&PlainQItem);
 686
 687         if (MyQItem == NULL) {
 688                 syslog(LOG_ERR,
 689                        "SMTP Queue: Msg No %ld: already in progress!\n",
 690                        msgnum);
 691                 return; /* s.b. else is already processing... */
 692         }
 693
 694         /*
 695          * Postpone delivery if we've already tried recently.
 696          */
 697         if (((time(NULL) - MyQItem->LastAttempt.when) <
 698              MyQItem->LastAttempt.retry) &&
 699             (run_queue_now == 0))
 700         {
 701                 syslog(LOG_DEBUG, "SMTP client: Retry time not yet reached.\n");
 702
 703                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 704                 pthread_mutex_lock(&ActiveQItemsLock);
 705                 {
 706                         if (GetHashPosFromKey(ActiveQItems,
 707                                               LKEY(MyQItem->MessageID),
 708                                               It))
 709                         {
 710                                 DeleteEntryFromHash(ActiveQItems, It);
 711                         }
 712                 }
 713                 pthread_mutex_unlock(&ActiveQItemsLock);
 714                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 715                 DeleteHashPos(&It);
 716                 return;
 717         }
 718
 719         /*
 720          * Bail out if there's no actual message associated with this
 721          */
 722         if (MyQItem->MessageID < 0L) {
 723                 syslog(LOG_ERR, "SMTP Queue: no 'msgid' directive found!\n");
 724                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 725                 pthread_mutex_lock(&ActiveQItemsLock);
 726                 {
 727                         if (GetHashPosFromKey(ActiveQItems,
 728                                               LKEY(MyQItem->MessageID),
 729                                               It))
 730                         {
 731                                 DeleteEntryFromHash(ActiveQItems, It);
 732                         }
 733                 }
 734                 pthread_mutex_unlock(&ActiveQItemsLock);
 735                 DeleteHashPos(&It);
 736                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 737                 return;
 738         }
 739
 740         {
 741                 char mxbuf[SIZ];
 742                 ParsedURL **Url = &MyQItem->URL;
 743                 nRelays = get_hosts(mxbuf, "smarthost");
 744                 if (nRelays > 0) {
 745                         StrBuf *All;
 746                         StrBuf *One;
 747                         const char *Pos = NULL;
 748                         All = NewStrBufPlain(mxbuf, -1);
 749                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 750
 751                         while ((Pos != StrBufNOTNULL) &&
 752                                ((Pos == NULL) ||
 753                                 !IsEmptyStr(Pos)))
 754                         {
 755                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 756                                 if (!ParseURL(Url, One, 25))
 757                                         syslog(LOG_DEBUG,
 758                                                "Failed to parse: %s\n",
 759                                                ChrPtr(One));
 760                                 else {
 761                                         ///if (!Url->IsIP)) // todo dupe me fork ipv6
 762                                         Url = &(*Url)->Next;
 763                                 }
 764                         }
 765                         FreeStrBuf(&All);
 766                         FreeStrBuf(&One);
 767                 }
 768
 769                 Url = &MyQItem->FallBackHost;
 770                 nRelays = get_hosts(mxbuf, "fallbackhost");
 771                 if (nRelays > 0) {
 772                         StrBuf *All;
 773                         StrBuf *One;
 774                         const char *Pos = NULL;
 775                         All = NewStrBufPlain(mxbuf, -1);
 776                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 777
 778                         while ((Pos != StrBufNOTNULL) &&
 779                                ((Pos == NULL) ||
 780                                 !IsEmptyStr(Pos)))
 781                         {
 782                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 783                                 if (!ParseURL(Url, One, 25))
 784                                         syslog(LOG_DEBUG,
 785                                                "Failed to parse: %s\n",
 786                                                ChrPtr(One));
 787                                 else
 788                                         Url = &(*Url)->Next;
 789                         }
 790                         FreeStrBuf(&All);
 791                         FreeStrBuf(&One);
 792                 }
 793         }
 794
 795         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 796         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 797         {
 798                 MailQEntry *ThisItem = vQE;
 799                 syslog(LOG_DEBUG, "SMTP Queue: Task: <%s> %d\n",
 800                        ChrPtr(ThisItem->Recipient),
 801                        ThisItem->Active);
 802         }
 803         DeleteHashPos(&It);
 804
 805         MyQItem->ActiveDeliveries = CountActiveQueueEntries(MyQItem);
 806         if (MyQItem->ActiveDeliveries > 0)
 807         {
 808                 int n = MsgCount++;
 809                 int m = MyQItem->ActiveDeliveries;
 810                 int i = 1;
 811                 Msg = smtp_load_msg(MyQItem, n);
 812                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 813                 while ((i <= m) &&
 814                        (GetNextHashPos(MyQItem->MailQEntries,
 815                                        It, &len, &Key, &vQE)))
 816                 {
 817                         MailQEntry *ThisItem = vQE;
 818
 819                         if (ThisItem->Active == 1)
 820                         {
 821                                 int KeepBuffers = (i == m);
 822                                 if (i > 1) n = MsgCount++;
 823                                 syslog(LOG_DEBUG,
 824                                        "SMTPQ: Trying <%ld> <%s> %d / %d \n",
 825                                        MyQItem->MessageID,
 826                                        ChrPtr(ThisItem->Recipient),
 827                                        i,
 828                                        m);
 829                                 smtp_try_one_queue_entry(MyQItem,
 830                                                          ThisItem,
 831                                                          Msg,
 832                                                          KeepBuffers,
 833                                                          n,
 834                                                          RelayUrls);
 835
 836                                 if (KeepBuffers) HaveBuffers = 1;
 837
 838                                 i++;
 839                         }
 840                 }
 841                 DeleteHashPos(&It);
 842         }
 843         else
 844         {
 845                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 846                 pthread_mutex_lock(&ActiveQItemsLock);
 847                 {
 848                         if (GetHashPosFromKey(ActiveQItems,
 849                                               LKEY(MyQItem->MessageID),
 850                                               It))
 851                         {
 852                                 DeleteEntryFromHash(ActiveQItems, It);
 853                         }
 854                         else
 855                         {
 856                                 long len;
 857                                 const char* Key;
 858                                 void *VData;
 859
 860                                 syslog(LOG_WARNING,
 861                                        "SMTP cleanup: unable to find "
 862                                        "QItem with ID[%ld]",
 863                                        MyQItem->MessageID);
 864                                 while (GetNextHashPos(ActiveQItems,
 865                                                       It,
 866                                                       &len,
 867                                                       &Key,
 868                                                       &VData))
 869                                 {
 870                                         syslog(LOG_WARNING,
 871                                                "SMTP cleanup: have: ID[%ld]",
 872                                               ((OneQueItem *)VData)->MessageID);
 873                                 }
 874                         }
 875
 876                 }
 877                 pthread_mutex_unlock(&ActiveQItemsLock);
 878                 DeleteHashPos(&It);
 879                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 880
 881 // TODO: bounce & delete?
 882
 883         }
 884         if (!HaveBuffers) {
 885                 FreeStrBuf (&Msg);
 886 // TODO : free RelayUrls
 887         }
 888 }
 889
 890
 891
 892 /*
 893  * smtp_queue_thread()
 894  *
 895  * Run through the queue sending out messages.
 896  */
 897 void smtp_do_queue(void) {
 898         static int is_running = 0;
 899         int num_processed = 0;
 900
 901         if (is_running)
 902                 return; /* Concurrency check - only one can run */
 903         is_running = 1;
 904
 905         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 906         syslog(LOG_INFO, "SMTP client: processing outbound queue");
 907
 908         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
 909                 syslog(LOG_ERR, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
 910         }
 911         else {
 912                 num_processed = CtdlForEachMessage(MSGS_ALL,
 913                                                    0L,
 914                                                    NULL,
 915                                                    SPOOLMIME,
 916                                                    NULL,
 917                                                    smtp_do_procmsg,
 918                                                    NULL);
 919         }
 920         syslog(LOG_INFO,
 921                "SMTP client: queue run completed; %d messages processed",
 922                num_processed);
 923
 924         run_queue_now = 0;
 925         is_running = 0;
 926 }
 927
 928
 929
 930 /*
 931  * Initialize the SMTP outbound queue
 932  */
 933 void smtp_init_spoolout(void) {
 934         struct ctdlroom qrbuf;
 935
 936         /*
 937          * Create the room.  This will silently fail if the room already
 938          * exists, and that's perfectly ok, because we want it to exist.
 939          */
 940         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_MAILBOX);
 941
 942         /*
 943          * Make sure it's set to be a "system room" so it doesn't show up
 944          * in the <K>nown rooms list for Aides.
 945          */
 946         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
 947                 qrbuf.QRflags2 |= QR2_SYSTEM;
 948                 CtdlPutRoomLock(&qrbuf);
 949         }
 950 }
 951
 952
 953
 954
 955 /*****************************************************************************/
 956 /*                          SMTP UTILITY COMMANDS                            */
 957 /*****************************************************************************/
 958
 959 void cmd_smtp(char *argbuf) {
 960         char cmd[64];
 961         char node[256];
 962         char buf[1024];
 963         int i;
 964         int num_mxhosts;
 965
 966         if (CtdlAccessCheck(ac_aide)) return;
 967
 968         extract_token(cmd, argbuf, 0, '|', sizeof cmd);
 969
 970         if (!strcasecmp(cmd, "mx")) {
 971                 extract_token(node, argbuf, 1, '|', sizeof node);
 972                 num_mxhosts = getmx(buf, node);
 973                 cprintf("%d %d MX hosts listed for %s\n",
 974                         LISTING_FOLLOWS, num_mxhosts, node);
 975                 for (i=0; i<num_mxhosts; ++i) {
 976                         extract_token(node, buf, i, '|', sizeof node);
 977                         cprintf("%s\n", node);
 978                 }
 979                 cprintf("000\n");
 980                 return;
 981         }
 982
 983         else if (!strcasecmp(cmd, "runqueue")) {
 984                 run_queue_now = 1;
 985                 cprintf("%d All outbound SMTP will be retried now.\n", CIT_OK);
 986                 return;
 987         }
 988
 989         else {
 990                 cprintf("%d Invalid command.\n", ERROR + ILLEGAL_VALUE);
 991         }
 992
 993 }
 994
 995
 996 CTDL_MODULE_INIT(smtp_queu)
 997 {
 998         if (!threading)
 999         {
1000                 CtdlFillSystemContext(&smtp_queue_CC, "SMTP_Send");
1001                 ActiveQItems = NewHash(1, lFlathash);
1002                 pthread_mutex_init(&ActiveQItemsLock, NULL);
1003
1004                 QItemHandlers = NewHash(0, NULL);
1005
1006                 Put(QItemHandlers, HKEY("msgid"), QItem_Handle_MsgID, reference_free_handler);
1007                 Put(QItemHandlers, HKEY("envelope_from"), QItem_Handle_EnvelopeFrom, reference_free_handler);
1008                 Put(QItemHandlers, HKEY("retry"), QItem_Handle_retry, reference_free_handler);
1009                 Put(QItemHandlers, HKEY("attempted"), QItem_Handle_Attempted, reference_free_handler);
1010                 Put(QItemHandlers, HKEY("remote"), QItem_Handle_Recipient, reference_free_handler);
1011                 Put(QItemHandlers, HKEY("bounceto"), QItem_Handle_BounceTo, reference_free_handler);
1012                 Put(QItemHandlers, HKEY("submitted"), QItem_Handle_Submitted, reference_free_handler);
1013
1014                 smtp_init_spoolout();
1015
1016                 CtdlRegisterCleanupHook(smtp_evq_cleanup);
1017
1018                 CtdlRegisterProtoHook(cmd_smtp, "SMTP", "SMTP utility commands");
1019                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER);
1020         }
1021
1022         /* return our Subversion id for the Log */
1023         return "smtpeventclient";
1024 }