citadel/modules/smtp/serv_smtpqueue.c

   1 /*
   2  * This module is an SMTP and ESMTP implementation for the Citadel system.
   3  * It is compliant with all of the following:
   4  *
   5  * RFC  821 - Simple Mail Transfer Protocol
   6  * RFC  876 - Survey of SMTP Implementations
   7  * RFC 1047 - Duplicate messages and SMTP
   8  * RFC 1652 - 8 bit MIME
   9  * RFC 1869 - Extended Simple Mail Transfer Protocol
  10  * RFC 1870 - SMTP Service Extension for Message Size Declaration
  11  * RFC 2033 - Local Mail Transfer Protocol
  12  * RFC 2197 - SMTP Service Extension for Command Pipelining
  13  * RFC 2476 - Message Submission
  14  * RFC 2487 - SMTP Service Extension for Secure SMTP over TLS
  15  * RFC 2554 - SMTP Service Extension for Authentication
  16  * RFC 2821 - Simple Mail Transfer Protocol
  17  * RFC 2822 - Internet Message Format
  18  * RFC 2920 - SMTP Service Extension for Command Pipelining
  19  *
  20  * The VRFY and EXPN commands have been removed from this implementation
  21  * because nobody uses these commands anymore, except for spammers.
  22  *
  23  * Copyright (c) 1998-2009 by the citadel.org team
  24  *
  25  *  This program is free software; you can redistribute it and/or modify
  26  *  it under the terms of the GNU General Public License as published by
  27  *  the Free Software Foundation; either version 3 of the License, or
  28  *  (at your option) any later version.
  29  *
  30  *  This program is distributed in the hope that it will be useful,
  31  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  32  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  33  *  GNU General Public License for more details.
  34  *
  35  *  You should have received a copy of the GNU General Public License
  36  *  along with this program; if not, write to the Free Software
  37  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  38  */
  39
  40 #include "sysdep.h"
  41 #include <stdlib.h>
  42 #include <unistd.h>
  43 #include <stdio.h>
  44 #include <termios.h>
  45 #include <fcntl.h>
  46 #include <signal.h>
  47 #include <pwd.h>
  48 #include <errno.h>
  49 #include <sys/types.h>
  50 #include <syslog.h>
  51
  52 #if TIME_WITH_SYS_TIME
  53 # include <sys/time.h>
  54 # include <time.h>
  55 #else
  56 # if HAVE_SYS_TIME_H
  57 #  include <sys/time.h>
  58 # else
  59 #  include <time.h>
  60 # endif
  61 #endif
  62 #include <sys/wait.h>
  63 #include <ctype.h>
  64 #include <string.h>
  65 #include <limits.h>
  66 #include <sys/socket.h>
  67 #include <netinet/in.h>
  68 #include <arpa/inet.h>
  69 #include <libcitadel.h>
  70 #include "citadel.h"
  71 #include "server.h"
  72 #include "citserver.h"
  73 #include "support.h"
  74 #include "config.h"
  75 #include "control.h"
  76 #include "user_ops.h"
  77 #include "database.h"
  78 #include "msgbase.h"
  79 #include "internet_addressing.h"
  80 #include "genstamp.h"
  81 #include "domain.h"
  82 #include "clientsocket.h"
  83 #include "locate_host.h"
  84 #include "citadel_dirs.h"
  85
  86 #include "ctdl_module.h"
  87
  88 #include "smtpqueue.h"
  89 #include "event_client.h"
  90
  91 HashList *QItemHandlers = NULL;
  92
  93 citthread_mutex_t ActiveQItemsLock;
  94 HashList *ActiveQItems = NULL;
  95
  96 int MsgCount = 0;
  97 int run_queue_now = 0;  /* Set to 1 to ignore SMTP send retry times */
  98
  99 void smtp_try(OneQueItem *MyQItem,
 100               MailQEntry *MyQEntry,
 101               StrBuf *MsgText,
 102               int KeepMsgText, /* KeepMsgText allows us to use MsgText as ours. */
 103               int MsgCount,
 104               ParsedURL *RelayUrls);
 105
 106
 107 void smtp_evq_cleanup(void)
 108 {
 109         citthread_mutex_lock(&ActiveQItemsLock);
 110         DeleteHash(&QItemHandlers);
 111         DeleteHash(&ActiveQItems);
 112         citthread_mutex_unlock(&ActiveQItemsLock);
 113         citthread_mutex_destroy(&ActiveQItemsLock);
 114 }
 115
 116 int DecreaseQReference(OneQueItem *MyQItem)
 117 {
 118         int IDestructQueItem;
 119
 120         MyQItem->ActiveDeliveries--;
 121         IDestructQueItem = MyQItem->ActiveDeliveries == 0;
 122         return IDestructQueItem;
 123 }
 124
 125 void RemoveQItem(OneQueItem *MyQItem)
 126 {
 127         HashPos  *It;
 128
 129         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 130         citthread_mutex_lock(&ActiveQItemsLock);
 131         {
 132                 GetHashPosFromKey(ActiveQItems, IKEY(MyQItem->MessageID), It);
 133                 DeleteEntryFromHash(ActiveQItems, It);
 134         }
 135         citthread_mutex_unlock(&ActiveQItemsLock);
 136         DeleteHashPos(&It);
 137 }
 138
 139 void FreeURL(ParsedURL** Url)
 140 {
 141         if (*Url != NULL) {
 142                 FreeStrBuf(&(*Url)->URL);
 143                 if ((*Url)->Next != NULL)
 144                         FreeURL(&(*Url)->Next);
 145                 free(*Url);
 146                 *Url = NULL;
 147         }
 148 }
 149
 150 void FreeMailQEntry(void *qv)
 151 {
 152         MailQEntry *Q = qv;
 153         FreeStrBuf(&Q->Recipient);
 154         FreeStrBuf(&Q->StatusMessage);
 155         free(Q);
 156 }
 157 void FreeQueItem(OneQueItem **Item)
 158 {
 159         DeleteHash(&(*Item)->MailQEntries);
 160         FreeStrBuf(&(*Item)->EnvelopeFrom);
 161         FreeStrBuf(&(*Item)->BounceTo);
 162         FreeURL(&(*Item)->URL);
 163         free(*Item);
 164         Item = NULL;
 165 }
 166 void HFreeQueItem(void *Item)
 167 {
 168         FreeQueItem((OneQueItem**)&Item);
 169 }
 170
 171 /* inspect recipients with a status of:
 172  * - 0 (no delivery yet attempted)
 173  * - 3/4 (transient errors
 174  *        were experienced and it's time to try again)
 175  */
 176 int CountActiveQueueEntries(OneQueItem *MyQItem)
 177 {
 178         HashPos  *It;
 179         long len;
 180         const char *Key;
 181         void *vQE;
 182
 183         MyQItem->ActiveDeliveries = 0;
 184         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 185         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 186         {
 187                 MailQEntry *ThisItem = vQE;
 188                 if ((ThisItem->Status == 0) ||
 189                     (ThisItem->Status == 3) ||
 190                     (ThisItem->Status == 4))
 191                 {
 192                         MyQItem->ActiveDeliveries++;
 193                         ThisItem->Active = 1;
 194                 }
 195                 else
 196                         ThisItem->Active = 0;
 197         }
 198         DeleteHashPos(&It);
 199         return MyQItem->ActiveDeliveries;
 200 }
 201
 202 OneQueItem *DeserializeQueueItem(StrBuf *RawQItem, long QueMsgID)
 203 {
 204         OneQueItem *Item;
 205         const char *pLine = NULL;
 206         StrBuf *Line;
 207         StrBuf *Token;
 208         void *v;
 209
 210         Item = (OneQueItem*)malloc(sizeof(OneQueItem));
 211         memset(Item, 0, sizeof(OneQueItem));
 212         Item->LastAttempt.retry = SMTP_RETRY_INTERVAL;
 213         Item->MessageID = -1;
 214         Item->QueMsgID = QueMsgID;
 215
 216         citthread_mutex_lock(&ActiveQItemsLock);
 217         if (GetHash(ActiveQItems,
 218                     IKEY(Item->QueMsgID),
 219                     &v))
 220         {
 221                 /* WHOOPS. somebody else is already working on this. */
 222                 citthread_mutex_unlock(&ActiveQItemsLock);
 223                 FreeQueItem(&Item);
 224                 return NULL;
 225         }
 226         else {
 227                 /* mark our claim on this. */
 228                 Put(ActiveQItems,
 229                     IKEY(Item->QueMsgID),
 230                     Item,
 231                     HFreeQueItem);
 232                 citthread_mutex_unlock(&ActiveQItemsLock);
 233         }
 234
 235         Token = NewStrBuf();
 236         Line = NewStrBufPlain(NULL, 128);
 237         while (pLine != StrBufNOTNULL) {
 238                 const char *pItemPart = NULL;
 239                 void *vHandler;
 240
 241                 StrBufExtract_NextToken(Line, RawQItem, &pLine, '\n');
 242                 if (StrLength(Line) == 0) continue;
 243                 StrBufExtract_NextToken(Token, Line, &pItemPart, '|');
 244                 if (GetHash(QItemHandlers, SKEY(Token), &vHandler))
 245                 {
 246                         QItemHandler H;
 247                         H = (QItemHandler) vHandler;
 248                         H(Item, Line, &pItemPart);
 249                 }
 250         }
 251         FreeStrBuf(&Line);
 252         FreeStrBuf(&Token);
 253         return Item;
 254 }
 255
 256 StrBuf *SerializeQueueItem(OneQueItem *MyQItem)
 257 {
 258         StrBuf *QMessage;
 259         HashPos  *It;
 260         const char *Key;
 261         long len;
 262         void *vQE;
 263
 264         QMessage = NewStrBufPlain(NULL, SIZ);
 265         StrBufPrintf(QMessage, "Content-type: %s\n", SPOOLMIME);
 266
 267 //                 "attempted|%ld\n"  "retry|%ld\n",, (long)time(NULL), (long)retry );
 268         StrBufAppendBufPlain(QMessage, HKEY("\nmsgid|"), 0);
 269         StrBufAppendPrintf(QMessage, "%ld", MyQItem->MessageID);
 270
 271         StrBufAppendBufPlain(QMessage, HKEY("\nsubmitted|"), 0);
 272         StrBufAppendPrintf(QMessage, "%ld", MyQItem->Submitted);
 273
 274         if (StrLength(MyQItem->BounceTo) > 0) {
 275                 StrBufAppendBufPlain(QMessage, HKEY("\nbounceto|"), 0);
 276                 StrBufAppendBuf(QMessage, MyQItem->BounceTo, 0);
 277         }
 278
 279         if (StrLength(MyQItem->EnvelopeFrom) > 0) {
 280                 StrBufAppendBufPlain(QMessage, HKEY("\nenvelope_from|"), 0);
 281                 StrBufAppendBuf(QMessage, MyQItem->EnvelopeFrom, 0);
 282         }
 283
 284         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 285         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 286         {
 287                 MailQEntry *ThisItem = vQE;
 288                 int i;
 289
 290                 if (!ThisItem->Active)
 291                         continue; /* skip already sent ones from the spoolfile. */
 292
 293                 for (i=0; i < ThisItem->nAttempts; i++) {
 294                         /* TODO: most probably there is just one retry/attempted per message! */
 295                         StrBufAppendBufPlain(QMessage, HKEY("\nretry|"), 0);
 296                         StrBufAppendPrintf(QMessage, "%ld",
 297                                            ThisItem->Attempts[i].retry);
 298
 299                         StrBufAppendBufPlain(QMessage, HKEY("\nattempted|"), 0);
 300                         StrBufAppendPrintf(QMessage, "%ld",
 301                                            ThisItem->Attempts[i].when);
 302                 }
 303                 StrBufAppendBufPlain(QMessage, HKEY("\nremote|"), 0);
 304                 StrBufAppendBuf(QMessage, ThisItem->Recipient, 0);
 305                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 306                 StrBufAppendPrintf(QMessage, "%d", ThisItem->Status);
 307                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 308                 StrBufAppendBuf(QMessage, ThisItem->StatusMessage, 0);
 309         }
 310         DeleteHashPos(&It);
 311         StrBufAppendBufPlain(QMessage, HKEY("\n"), 0);
 312         return QMessage;
 313 }
 314
 315
 316
 317
 318
 319 void NewMailQEntry(OneQueItem *Item)
 320 {
 321         Item->Current = (MailQEntry*) malloc(sizeof(MailQEntry));
 322         memset(Item->Current, 0, sizeof(MailQEntry));
 323
 324         if (Item->MailQEntries == NULL)
 325                 Item->MailQEntries = NewHash(1, Flathash);
 326         Item->Current->StatusMessage = NewStrBuf();
 327         Item->Current->n = GetCount(Item->MailQEntries);
 328         Put(Item->MailQEntries, IKEY(Item->Current->n), Item->Current, FreeMailQEntry);
 329 }
 330
 331 void QItem_Handle_MsgID(OneQueItem *Item, StrBuf *Line, const char **Pos)
 332 {
 333         Item->MessageID = StrBufExtractNext_int(Line, Pos, '|');
 334 }
 335
 336 void QItem_Handle_EnvelopeFrom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 337 {
 338         if (Item->EnvelopeFrom == NULL)
 339                 Item->EnvelopeFrom = NewStrBufPlain(NULL, StrLength(Line));
 340         StrBufExtract_NextToken(Item->EnvelopeFrom, Line, Pos, '|');
 341 }
 342
 343 void QItem_Handle_BounceTo(OneQueItem *Item, StrBuf *Line, const char **Pos)
 344 {
 345         if (Item->BounceTo == NULL)
 346                 Item->BounceTo = NewStrBufPlain(NULL, StrLength(Line));
 347         StrBufExtract_NextToken(Item->BounceTo, Line, Pos, '|');
 348 }
 349
 350 void QItem_Handle_Recipient(OneQueItem *Item, StrBuf *Line, const char **Pos)
 351 {
 352         if (Item->Current == NULL)
 353                 NewMailQEntry(Item);
 354         if (Item->Current->Recipient == NULL)
 355                 Item->Current->Recipient =  NewStrBufPlain(NULL, StrLength(Line));
 356         StrBufExtract_NextToken(Item->Current->Recipient, Line, Pos, '|');
 357         Item->Current->Status = StrBufExtractNext_int(Line, Pos, '|');
 358         StrBufExtract_NextToken(Item->Current->StatusMessage, Line, Pos, '|');
 359         Item->Current = NULL; // TODO: is this always right?
 360 }
 361
 362
 363 void QItem_Handle_retry(OneQueItem *Item, StrBuf *Line, const char **Pos)
 364 {
 365         if (Item->Current == NULL)
 366                 NewMailQEntry(Item);
 367         if (Item->Current->Attempts[Item->Current->nAttempts].retry != 0)
 368                 Item->Current->nAttempts++;
 369         if (Item->Current->nAttempts > MaxAttempts) {
 370                 Item->FailNow = 1;
 371                 return;
 372         }
 373         Item->Current->Attempts[Item->Current->nAttempts].retry = StrBufExtractNext_int(Line, Pos, '|');
 374 }
 375
 376
 377 void QItem_Handle_Submitted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 378 {
 379         Item->Submitted = atol(*Pos);
 380
 381 }
 382
 383 void QItem_Handle_Attempted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 384 {
 385         if (Item->Current == NULL)
 386                 NewMailQEntry(Item);
 387         if (Item->Current->Attempts[Item->Current->nAttempts].when != 0)
 388                 Item->Current->nAttempts++;
 389         if (Item->Current->nAttempts > MaxAttempts) {
 390                 Item->FailNow = 1;
 391                 return;
 392         }
 393
 394         Item->Current->Attempts[Item->Current->nAttempts].when = StrBufExtractNext_int(Line, Pos, '|');
 395         if (Item->Current->Attempts[Item->Current->nAttempts].when > Item->LastAttempt.when)
 396         {
 397                 Item->LastAttempt.when = Item->Current->Attempts[Item->Current->nAttempts].when;
 398                 Item->LastAttempt.retry = Item->Current->Attempts[Item->Current->nAttempts].retry * 2;
 399                 if (Item->LastAttempt.retry > SMTP_RETRY_MAX)
 400                         Item->LastAttempt.retry = SMTP_RETRY_MAX;
 401         }
 402 }
 403
 404
 405
 406 /**
 407  * this one has to have the context for loading the message via the redirect buffer...
 408  */
 409 StrBuf *smtp_load_msg(OneQueItem *MyQItem, int n)
 410 {
 411         CitContext *CCC=CC;
 412         StrBuf *SendMsg;
 413
 414         CCC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
 415         CtdlOutputMsg(MyQItem->MessageID, MT_RFC822, HEADERS_ALL, 0, 1, NULL, (ESC_DOT|SUPPRESS_ENV_TO) );
 416         SendMsg = CCC->redirect_buffer;
 417         CCC->redirect_buffer = NULL;
 418         if ((StrLength(SendMsg) > 0) &&
 419             ChrPtr(SendMsg)[StrLength(SendMsg) - 1] != '\n') {
 420                 CtdlLogPrintf(CTDL_WARNING,
 421                               "SMTP client[%d]: Possible problem: message did not "
 422                               "correctly terminate. (expecting 0x10, got 0x%02x)\n",
 423                               MsgCount, //yes uncool, but best choice here...
 424                               ChrPtr(SendMsg)[StrLength(SendMsg) - 1] );
 425                 StrBufAppendBufPlain(SendMsg, HKEY("\r\n"), 0);
 426         }
 427         return SendMsg;
 428 }
 429
 430
 431
 432 /*
 433  * smtp_do_bounce() is caled by smtp_do_procmsg() to scan a set of delivery
 434  * instructions for "5" codes (permanent fatal errors) and produce/deliver
 435  * a "bounce" message (delivery status notification).
 436  */
 437 void smtpq_do_bounce(OneQueItem *MyQItem, StrBuf *OMsgTxt)
 438 {
 439         static int seq = 0;
 440
 441         struct CtdlMessage *bmsg = NULL;
 442         StrBuf *boundary;
 443         StrBuf *Msg = NULL;
 444         StrBuf *BounceMB;
 445         struct recptypes *valid;
 446
 447         HashPos *It;
 448         void *vQE;
 449         long len;
 450         const char *Key;
 451
 452         int successful_bounce = 0;
 453         int num_bounces = 0;
 454         int give_up = 0;
 455
 456         CtdlLogPrintf(CTDL_DEBUG, "smtp_do_bounce() called\n");
 457
 458         if ( (ev_time() - MyQItem->Submitted) > SMTP_GIVE_UP ) {
 459                 give_up = 1;/// TODO: replace time by libevq timer get
 460         }
 461
 462         /*
 463          * Now go through the instructions checking for stuff.
 464          */
 465         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 466         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 467         {
 468                 MailQEntry *ThisItem = vQE;
 469                 if ((ThisItem->Status == 5) || /* failed now? */
 470                     ((give_up == 1) && (ThisItem->Status != 2))) /* giving up after failed attempts... */
 471                 {
 472                         if (num_bounces == 0)
 473                                 Msg = NewStrBufPlain(NULL, 1024);
 474                         ++num_bounces;
 475
 476                         StrBufAppendBuf(Msg, ThisItem->Recipient, 0);
 477                         StrBufAppendBufPlain(Msg, HKEY(": "), 0);
 478                         StrBufAppendBuf(Msg, ThisItem->StatusMessage, 0);
 479                         StrBufAppendBufPlain(Msg, HKEY("\r\n"), 0);
 480                 }
 481         }
 482         DeleteHashPos(&It);
 483
 484         /* Deliver the bounce if there's anything worth mentioning */
 485         CtdlLogPrintf(CTDL_DEBUG, "num_bounces = %d\n", num_bounces);
 486
 487         if (num_bounces == 0) {
 488                 FreeStrBuf(&Msg);
 489                 return;
 490         }
 491
 492         boundary = NewStrBufPlain(HKEY("=_Citadel_Multipart_"));
 493         StrBufAppendPrintf(boundary, "%s_%04x%04x", config.c_fqdn, getpid(), ++seq);
 494
 495         /* Start building our bounce message; go shopping for memory first. */
 496         BounceMB = NewStrBufPlain(NULL,
 497                                   1024 + /* mime stuff.... */
 498                                   StrLength(Msg) +  /* the bounce information... */
 499                                   StrLength(OMsgTxt)); /* the original message */
 500         if (BounceMB == NULL) {
 501                 FreeStrBuf(&boundary);
 502                 CtdlLogPrintf(CTDL_ERR, "Failed to alloc() bounce message.\n");
 503
 504                 return;
 505         }
 506
 507         bmsg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
 508         if (bmsg == NULL) {
 509                 FreeStrBuf(&boundary);
 510                 FreeStrBuf(&BounceMB);
 511                 CtdlLogPrintf(CTDL_ERR, "Failed to alloc() bounce message.\n");
 512
 513                 return;
 514         }
 515         memset(bmsg, 0, sizeof(struct CtdlMessage));
 516
 517
 518         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: multipart/mixed; boundary=\""), 0);
 519         StrBufAppendBuf(BounceMB, boundary, 0);
 520         StrBufAppendBufPlain(BounceMB, HKEY("\"\r\n"), 0);
 521         StrBufAppendBufPlain(BounceMB, HKEY("MIME-Version: 1.0\r\n"), 0);
 522         StrBufAppendBufPlain(BounceMB, HKEY("X-Mailer: " CITADEL "\r\n"), 0);
 523         StrBufAppendBufPlain(BounceMB, HKEY("\r\nThis is a multipart message in MIME format.\r\n\r\n"), 0);
 524         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 525         StrBufAppendBuf(BounceMB, boundary, 0);
 526         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 527         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: text/plain\r\n\r\n"), 0);
 528
 529         if (give_up)
 530                 StrBufAppendBufPlain(
 531                         BounceMB,
 532                         HKEY(
 533                                 "A message you sent could not be delivered to some or all of its recipients\n"
 534                                 "due to prolonged unavailability of its destination(s).\n"
 535                                 "Giving up on the following addresses:\n\n"
 536                                 ), 0);
 537         else
 538                 StrBufAppendBufPlain(
 539                         BounceMB,
 540                         HKEY(
 541                                 "A message you sent could not be delivered to some or all of its recipients.\n"
 542                                 "The following addresses were undeliverable:\n\n"
 543                                 ), 0);
 544
 545         StrBufAppendBuf(BounceMB, Msg, 0);
 546         FreeStrBuf(&Msg);
 547
 548         /* Attach the original message */
 549         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 550         StrBufAppendBuf(BounceMB, boundary, 0);
 551         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 552         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: message/rfc822\r\n"), 0);
 553         StrBufAppendBufPlain(BounceMB, HKEY("Content-Transfer-Encoding: 7bit\r\n"), 0);
 554         StrBufAppendBufPlain(BounceMB, HKEY("Content-Disposition: inline\r\n"), 0);
 555         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 556         StrBufAppendBuf(BounceMB, OMsgTxt, 0);
 557
 558         /* Close the multipart MIME scope */
 559         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 560         StrBufAppendBuf(BounceMB, boundary, 0);
 561         StrBufAppendBufPlain(BounceMB, HKEY("--\r\n"), 0);
 562
 563
 564
 565         bmsg->cm_magic = CTDLMESSAGE_MAGIC;
 566         bmsg->cm_anon_type = MES_NORMAL;
 567         bmsg->cm_format_type = FMT_RFC822;
 568
 569         bmsg->cm_fields['O'] = strdup(MAILROOM);
 570         bmsg->cm_fields['A'] = strdup("Citadel");
 571         bmsg->cm_fields['N'] = strdup(config.c_nodename);
 572         bmsg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
 573         bmsg->cm_fields['M'] = SmashStrBuf(&BounceMB);
 574
 575         /* First try the user who sent the message */
 576         if (StrLength(MyQItem->BounceTo) == 0)
 577                 CtdlLogPrintf(CTDL_ERR, "No bounce address specified\n");
 578         else
 579                 CtdlLogPrintf(CTDL_DEBUG, "bounce to user? <%s>\n", ChrPtr(MyQItem->BounceTo));
 580
 581         /* Can we deliver the bounce to the original sender? */
 582         valid = validate_recipients(ChrPtr(MyQItem->BounceTo), NULL, 0);
 583         if ((valid != NULL) && (valid->num_error == 0)) {
 584                 CtdlSubmitMsg(bmsg, valid, "", QP_EADDR);
 585                 successful_bounce = 1;
 586         }
 587
 588         /* If not, post it in the Aide> room */
 589         if (successful_bounce == 0) {
 590                 CtdlSubmitMsg(bmsg, NULL, config.c_aideroom, QP_EADDR);
 591         }
 592
 593         /* Free up the memory we used */
 594         free_recipients(valid);
 595         FreeStrBuf(&boundary);
 596         CtdlFreeMessage(bmsg);
 597         CtdlLogPrintf(CTDL_DEBUG, "Done processing bounces\n");
 598 }
 599
 600
 601
 602 int ParseURL(ParsedURL **Url, StrBuf *UrlStr, short DefaultPort)
 603 {
 604         const char *pch, *pEndHost, *pPort, *pCredEnd, *pUserEnd;
 605         ParsedURL *url = (ParsedURL *)malloc(sizeof(ParsedURL));
 606         memset(url, 0, sizeof(ParsedURL));
 607
 608         url->af = AF_INET;
 609         url->Port =  DefaultPort;
 610         /*
 611          * http://username:passvoid@[ipv6]:port/url
 612          */
 613         url->URL = NewStrBufDup(UrlStr);
 614         url->Host = pch = ChrPtr(url->URL);
 615         url->LocalPart = strchr(pch, '/');
 616         if (url->LocalPart != NULL) {
 617                 if ((*(url->LocalPart + 1) == '/') &&
 618                     (*(url->LocalPart + 2) == ':')) { /* TODO: find default port for this protocol... */
 619                         url->Host = url->LocalPart + 3;
 620                         url->LocalPart = strchr(url->Host, '/');
 621                 }
 622         }
 623         if (url->LocalPart == NULL) {
 624                 url->LocalPart = pch + StrLength(url->URL);
 625         }
 626
 627         pCredEnd = strchr(pch, '@');
 628         if (pCredEnd >= url->LocalPart)
 629                 pCredEnd = NULL;
 630         if (pCredEnd != NULL)
 631         {
 632                 url->User = url->Host;
 633                 url->Host = pCredEnd + 1;
 634                 pUserEnd = strchr(url->User, ':');
 635
 636                 if (pUserEnd > pCredEnd)
 637                         pUserEnd = pCredEnd;
 638                 else {
 639                         url->Pass = pUserEnd + 1;
 640                 }
 641                 StrBufPeek(url->URL, pUserEnd, 0, '\0');
 642                 StrBufPeek(url->URL, pCredEnd, 0, '\0');
 643         }
 644
 645         pPort = NULL;
 646         if (*url->Host == '[') {
 647                 url->Host ++;
 648                 pEndHost = strchr(url->Host, ']');
 649                 if (pEndHost == NULL) {
 650                         FreeStrBuf(&url->URL);
 651                         free(url);
 652                         return 0; /* invalid syntax, no ipv6 */
 653                 }
 654                 StrBufPeek(url->URL, pEndHost, 0, '\0');
 655                 if (*(pEndHost + 1) == ':'){
 656                         StrBufPeek(url->URL, pEndHost + 1, 0, '\0');
 657                         pPort = pEndHost + 2;
 658                 }
 659                 url->af = AF_INET6;
 660         }
 661         else {
 662                 pPort = strchr(url->Host, ':');
 663                 if (pPort != NULL) {
 664                         StrBufPeek(url->URL, pPort, 0, '\0');
 665                         pPort ++;
 666                 }
 667         }
 668         if (pPort != NULL)
 669                 url->Port = atol(pPort);
 670         url->IsIP = inet_pton(url->af, url->Host, &url->Addr);
 671         *Url = url;
 672         return 1;
 673 }
 674 /*
 675 {
 676
 677         if (threadding)
 678                 n_smarthosts =  get_hosts(char *mxbuf, char *rectype);
 679 }
 680 */
 681 /*
 682  * smtp_do_procmsg()
 683  *
 684  * Called by smtp_do_queue() to handle an individual message.
 685  */
 686 void smtp_do_procmsg(long msgnum, void *userdata) {
 687         struct CtdlMessage *msg = NULL;
 688         char *instr = NULL;
 689         StrBuf *PlainQItem;
 690         OneQueItem *MyQItem;
 691         char *pch;
 692         HashPos  *It;
 693         void *vQE;
 694         long len;
 695         const char *Key;
 696         int nRelays = 0;
 697         ParsedURL *RelayUrls = NULL;
 698         int HaveBuffers = 0;
 699         StrBuf *Msg =NULL;
 700
 701         CtdlLogPrintf(CTDL_DEBUG, "SMTP Queue: smtp_do_procmsg(%ld)\n", msgnum);
 702         ///strcpy(envelope_from, "");
 703
 704         msg = CtdlFetchMessage(msgnum, 1);
 705         if (msg == NULL) {
 706                 CtdlLogPrintf(CTDL_ERR, "SMTP Queue: tried %ld but no such message!\n", msgnum);
 707                 return;
 708         }
 709
 710         pch = instr = msg->cm_fields['M'];
 711
 712         /* Strip out the headers (no not amd any other non-instruction) line */
 713         while (pch != NULL) {
 714                 pch = strchr(pch, '\n');
 715                 if ((pch != NULL) && (*(pch + 1) == '\n')) {
 716                         instr = pch + 2;
 717                         pch = NULL;
 718                 }
 719         }
 720         PlainQItem = NewStrBufPlain(instr, -1);
 721         CtdlFreeMessage(msg);
 722         MyQItem = DeserializeQueueItem(PlainQItem, msgnum);
 723         FreeStrBuf(&PlainQItem);
 724
 725         if (MyQItem == NULL) {
 726                 CtdlLogPrintf(CTDL_ERR, "SMTP Queue: Msg No %ld: already in progress!\n", msgnum);
 727                 return; /* s.b. else is already processing... */
 728         }
 729
 730         /*
 731          * Postpone delivery if we've already tried recently.
 732          * /
 733         if (((time(NULL) - MyQItem->LastAttempt.when) < MyQItem->LastAttempt.retry) && (run_queue_now == 0)) {
 734                 CtdlLogPrintf(CTDL_DEBUG, "SMTP client: Retry time not yet reached.\n");
 735
 736                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 737                 citthread_mutex_lock(&ActiveQItemsLock);
 738                 {
 739                         GetHashPosFromKey(ActiveQItems, IKEY(MyQItem->MessageID), It);
 740                         DeleteEntryFromHash(ActiveQItems, It);
 741                 }
 742                 citthread_mutex_unlock(&ActiveQItemsLock);
 743                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 744                 DeleteHashPos(&It);
 745                 return;
 746         }// TODO: reenable me.*/
 747
 748         /*
 749          * Bail out if there's no actual message associated with this
 750          */
 751         if (MyQItem->MessageID < 0L) {
 752                 CtdlLogPrintf(CTDL_ERR, "SMTP Queue: no 'msgid' directive found!\n");
 753                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 754                 citthread_mutex_lock(&ActiveQItemsLock);
 755                 {
 756                         GetHashPosFromKey(ActiveQItems, IKEY(MyQItem->MessageID), It);
 757                         DeleteEntryFromHash(ActiveQItems, It);
 758                 }
 759                 citthread_mutex_unlock(&ActiveQItemsLock);
 760                 DeleteHashPos(&It);
 761                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 762                 return;
 763         }
 764
 765         {
 766                 char mxbuf[SIZ];
 767                 ParsedURL **Url = &MyQItem->URL;
 768                 nRelays = get_hosts(mxbuf, "smarthost");
 769                 if (nRelays > 0) {
 770                         StrBuf *All;
 771                         StrBuf *One;
 772                         const char *Pos = NULL;
 773                         All = NewStrBufPlain(mxbuf, -1);
 774                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 775
 776                         while ((Pos != StrBufNOTNULL) && ((Pos == NULL) || !IsEmptyStr(Pos))) {
 777                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 778                                 if (!ParseURL(Url, One, 25))
 779                                         CtdlLogPrintf(CTDL_DEBUG, "Failed to parse: %s\n", ChrPtr(One));
 780                                 else
 781                                         Url = &(*Url)->Next;
 782                         }
 783                         FreeStrBuf(&All);
 784                         FreeStrBuf(&One);
 785                 }
 786
 787                 Url = &MyQItem->FallBackHost;
 788                 nRelays = get_hosts(mxbuf, "fallbackhost");
 789                 if (nRelays > 0) {
 790                         StrBuf *All;
 791                         StrBuf *One;
 792                         const char *Pos = NULL;
 793                         All = NewStrBufPlain(mxbuf, -1);
 794                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 795
 796                         while ((Pos != StrBufNOTNULL) && ((Pos == NULL) || !IsEmptyStr(Pos))) {
 797                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 798                                 if (!ParseURL(Url, One, 25))
 799                                         CtdlLogPrintf(CTDL_DEBUG, "Failed to parse: %s\n", ChrPtr(One));
 800                                 else
 801                                         Url = &(*Url)->Next;
 802                         }
 803                         FreeStrBuf(&All);
 804                         FreeStrBuf(&One);
 805                 }
 806         }
 807
 808         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 809         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 810         {
 811                 MailQEntry *ThisItem = vQE;
 812                 CtdlLogPrintf(CTDL_DEBUG, "SMTP Queue: Task: <%s> %d\n", ChrPtr(ThisItem->Recipient), ThisItem->Active);
 813         }
 814         DeleteHashPos(&It);
 815
 816         CountActiveQueueEntries(MyQItem);
 817         if (MyQItem->ActiveDeliveries > 0)
 818         {
 819                 int n = MsgCount++;
 820                 int m = MyQItem->ActiveDeliveries;
 821                 int i = 1;
 822                 Msg = smtp_load_msg(MyQItem, n);
 823                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 824                 while ((i <= m) &&
 825                        (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE)))
 826                 {
 827                         MailQEntry *ThisItem = vQE;
 828                         if (ThisItem->Active == 1) {
 829                                 int KeepBuffers = (i == m);
 830                                 if (i > 1) n = MsgCount++;
 831                                 CtdlLogPrintf(CTDL_DEBUG, "SMTP Queue: Trying <%s> %d / %d \n", ChrPtr(ThisItem->Recipient), i, m);
 832                                 smtp_try(MyQItem, ThisItem, Msg, KeepBuffers, n, RelayUrls);
 833                                 if (KeepBuffers) HaveBuffers = 1;
 834                                 i++;
 835                         }
 836                 }
 837                 DeleteHashPos(&It);
 838         }
 839         else
 840         {
 841                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 842                 citthread_mutex_lock(&ActiveQItemsLock);
 843                 {
 844                         GetHashPosFromKey(ActiveQItems, IKEY(MyQItem->MessageID), It);
 845                         DeleteEntryFromHash(ActiveQItems, It);
 846                 }
 847                 citthread_mutex_unlock(&ActiveQItemsLock);
 848                 DeleteHashPos(&It);
 849                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 850
 851 // TODO: bounce & delete?
 852
 853         }
 854         if (!HaveBuffers) {
 855                 FreeStrBuf (&Msg);
 856 // TODO : free RelayUrls
 857         }
 858 }
 859
 860
 861
 862 /*
 863  * smtp_queue_thread()
 864  *
 865  * Run through the queue sending out messages.
 866  */
 867 void *smtp_queue_thread(void *arg) {
 868         int num_processed = 0;
 869         struct CitContext smtp_queue_CC;
 870
 871         CtdlThreadSleep(10);
 872
 873         CtdlFillSystemContext(&smtp_queue_CC, "SMTP Send");
 874         citthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 875         CtdlLogPrintf(CTDL_DEBUG, "smtp_queue_thread() initializing\n");
 876
 877         while (!CtdlThreadCheckStop()) {
 878
 879                 CtdlLogPrintf(CTDL_INFO, "SMTP client: processing outbound queue\n");
 880
 881                 if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
 882                         CtdlLogPrintf(CTDL_ERR, "Cannot find room <%s>\n", SMTP_SPOOLOUT_ROOM);
 883                 }
 884                 else {
 885                         num_processed = CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_do_procmsg, NULL);
 886                 }
 887                 CtdlLogPrintf(CTDL_INFO, "SMTP client: queue run completed; %d messages processed\n", num_processed);
 888                 CtdlThreadSleep(60);
 889         }
 890
 891         CtdlClearSystemContext();
 892         return(NULL);
 893 }
 894
 895
 896
 897 /*
 898  * Initialize the SMTP outbound queue
 899  */
 900 void smtp_init_spoolout(void) {
 901         struct ctdlroom qrbuf;
 902
 903         /*
 904          * Create the room.  This will silently fail if the room already
 905          * exists, and that's perfectly ok, because we want it to exist.
 906          */
 907         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_MAILBOX);
 908
 909         /*
 910          * Make sure it's set to be a "system room" so it doesn't show up
 911          * in the <K>nown rooms list for Aides.
 912          */
 913         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
 914                 qrbuf.QRflags2 |= QR2_SYSTEM;
 915                 CtdlPutRoomLock(&qrbuf);
 916         }
 917 }
 918
 919
 920
 921
 922 /*****************************************************************************/
 923 /*                          SMTP UTILITY COMMANDS                            */
 924 /*****************************************************************************/
 925
 926 void cmd_smtp(char *argbuf) {
 927         char cmd[64];
 928         char node[256];
 929         char buf[1024];
 930         int i;
 931         int num_mxhosts;
 932
 933         if (CtdlAccessCheck(ac_aide)) return;
 934
 935         extract_token(cmd, argbuf, 0, '|', sizeof cmd);
 936
 937         if (!strcasecmp(cmd, "mx")) {
 938                 extract_token(node, argbuf, 1, '|', sizeof node);
 939                 num_mxhosts = getmx(buf, node);
 940                 cprintf("%d %d MX hosts listed for %s\n",
 941                         LISTING_FOLLOWS, num_mxhosts, node);
 942                 for (i=0; i<num_mxhosts; ++i) {
 943                         extract_token(node, buf, i, '|', sizeof node);
 944                         cprintf("%s\n", node);
 945                 }
 946                 cprintf("000\n");
 947                 return;
 948         }
 949
 950         else if (!strcasecmp(cmd, "runqueue")) {
 951                 run_queue_now = 1;
 952                 cprintf("%d All outbound SMTP will be retried now.\n", CIT_OK);
 953                 return;
 954         }
 955
 956         else {
 957                 cprintf("%d Invalid command.\n", ERROR + ILLEGAL_VALUE);
 958         }
 959
 960 }
 961
 962
 963
 964
 965 CTDL_MODULE_INIT(smtp_queu)
 966 {
 967 #ifdef EXPERIMENTAL_SMTP_EVENT_CLIENT
 968         if (!threading)
 969         {
 970                 ActiveQItems = NewHash(1, Flathash);
 971                 citthread_mutex_init(&ActiveQItemsLock, NULL);
 972
 973                 QItemHandlers = NewHash(0, NULL);
 974
 975                 Put(QItemHandlers, HKEY("msgid"), QItem_Handle_MsgID, reference_free_handler);
 976                 Put(QItemHandlers, HKEY("envelope_from"), QItem_Handle_EnvelopeFrom, reference_free_handler);
 977                 Put(QItemHandlers, HKEY("retry"), QItem_Handle_retry, reference_free_handler);
 978                 Put(QItemHandlers, HKEY("attempted"), QItem_Handle_Attempted, reference_free_handler);
 979                 Put(QItemHandlers, HKEY("remote"), QItem_Handle_Recipient, reference_free_handler);
 980                 Put(QItemHandlers, HKEY("bounceto"), QItem_Handle_BounceTo, reference_free_handler);
 981                 Put(QItemHandlers, HKEY("submitted"), QItem_Handle_Submitted, reference_free_handler);
 982 ////TODO: flush qitemhandlers on exit
 983                 smtp_init_spoolout();
 984
 985                 CtdlRegisterCleanupHook(smtp_evq_cleanup);
 986                 CtdlThreadCreate("SMTPEvent Send", CTDLTHREAD_BIGSTACK, smtp_queue_thread, NULL);
 987
 988                 CtdlRegisterProtoHook(cmd_smtp, "SMTP", "SMTP utility commands");
 989         }
 990 #endif
 991
 992         /* return our Subversion id for the Log */
 993         return "smtpeventclient";
 994 }