citadel/modules/smtp/serv_smtpqueue.c

   1 /*
   2  * This module is an SMTP and ESMTP implementation for the Citadel system.
   3  * It is compliant with all of the following:
   4  *
   5  * RFC  821 - Simple Mail Transfer Protocol
   6  * RFC  876 - Survey of SMTP Implementations
   7  * RFC 1047 - Duplicate messages and SMTP
   8  * RFC 1652 - 8 bit MIME
   9  * RFC 1869 - Extended Simple Mail Transfer Protocol
  10  * RFC 1870 - SMTP Service Extension for Message Size Declaration
  11  * RFC 2033 - Local Mail Transfer Protocol
  12  * RFC 2197 - SMTP Service Extension for Command Pipelining
  13  * RFC 2476 - Message Submission
  14  * RFC 2487 - SMTP Service Extension for Secure SMTP over TLS
  15  * RFC 2554 - SMTP Service Extension for Authentication
  16  * RFC 2821 - Simple Mail Transfer Protocol
  17  * RFC 2822 - Internet Message Format
  18  * RFC 2920 - SMTP Service Extension for Command Pipelining
  19  *
  20  * The VRFY and EXPN commands have been removed from this implementation
  21  * because nobody uses these commands anymore, except for spammers.
  22  *
  23  * Copyright (c) 1998-2009 by the citadel.org team
  24  *
  25  *  This program is free software; you can redistribute it and/or modify
  26  *  it under the terms of the GNU General Public License as published by
  27  *  the Free Software Foundation; either version 3 of the License, or
  28  *  (at your option) any later version.
  29  *
  30  *  This program is distributed in the hope that it will be useful,
  31  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  32  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  33  *  GNU General Public License for more details.
  34  *
  35  *  You should have received a copy of the GNU General Public License
  36  *  along with this program; if not, write to the Free Software
  37  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  38  */
  39
  40 #include "sysdep.h"
  41 #include <stdlib.h>
  42 #include <unistd.h>
  43 #include <stdio.h>
  44 #include <termios.h>
  45 #include <fcntl.h>
  46 #include <signal.h>
  47 #include <pwd.h>
  48 #include <errno.h>
  49 #include <sys/types.h>
  50 #include <syslog.h>
  51
  52 #if TIME_WITH_SYS_TIME
  53 # include <sys/time.h>
  54 # include <time.h>
  55 #else
  56 # if HAVE_SYS_TIME_H
  57 #  include <sys/time.h>
  58 # else
  59 #  include <time.h>
  60 # endif
  61 #endif
  62 #include <sys/wait.h>
  63 #include <ctype.h>
  64 #include <string.h>
  65 #include <limits.h>
  66 #include <sys/socket.h>
  67 #include <netinet/in.h>
  68 #include <arpa/inet.h>
  69 #include <libcitadel.h>
  70 #include "citadel.h"
  71 #include "server.h"
  72 #include "citserver.h"
  73 #include "support.h"
  74 #include "config.h"
  75 #include "control.h"
  76 #include "user_ops.h"
  77 #include "database.h"
  78 #include "msgbase.h"
  79 #include "internet_addressing.h"
  80 #include "genstamp.h"
  81 #include "domain.h"
  82 #include "clientsocket.h"
  83 #include "locate_host.h"
  84 #include "citadel_dirs.h"
  85
  86 #include "ctdl_module.h"
  87
  88 #include "smtpqueue.h"
  89 #include "event_client.h"
  90
  91
  92 struct CitContext smtp_queue_CC;
  93 pthread_mutex_t ActiveQItemsLock;
  94 HashList *ActiveQItems  = NULL;
  95 HashList *QItemHandlers = NULL;
  96
  97 int MsgCount            = 0;
  98 int run_queue_now       = 0;    /* Set to 1 to ignore SMTP send retry times */
  99
 100 void smtp_try_one_queue_entry(OneQueItem *MyQItem,
 101                               MailQEntry *MyQEntry,
 102                               StrBuf *MsgText,
 103                               int KeepMsgText, /* KeepMsgText allows us to use MsgText as ours. */
 104                               int MsgCount,
 105                               ParsedURL *RelayUrls);
 106
 107
 108 void smtp_evq_cleanup(void)
 109 {
 110
 111         pthread_mutex_lock(&ActiveQItemsLock);
 112         DeleteHash(&QItemHandlers);
 113         DeleteHash(&ActiveQItems);
 114         pthread_mutex_unlock(&ActiveQItemsLock);
 115         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 116 /*      citthread_mutex_destroy(&ActiveQItemsLock); TODO */
 117 }
 118
 119 int DecreaseQReference(OneQueItem *MyQItem)
 120 {
 121         int IDestructQueItem;
 122
 123         pthread_mutex_lock(&ActiveQItemsLock);
 124         MyQItem->ActiveDeliveries--;
 125         IDestructQueItem = MyQItem->ActiveDeliveries == 0;
 126         pthread_mutex_unlock(&ActiveQItemsLock);
 127         return IDestructQueItem;
 128 }
 129
 130 void RemoveQItem(OneQueItem *MyQItem)
 131 {
 132         long len;
 133         const char* Key;
 134         void *VData;
 135         HashPos  *It;
 136
 137         pthread_mutex_lock(&ActiveQItemsLock);
 138         It = GetNewHashPos(ActiveQItems, 0);
 139         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 140                 DeleteEntryFromHash(ActiveQItems, It);
 141         else
 142         {
 143                 syslog(LOG_WARNING,
 144                        "SMTP cleanup: unable to find QItem with ID[%ld]",
 145                        MyQItem->MessageID);
 146                 while (GetNextHashPos(ActiveQItems, It, &len, &Key, &VData))
 147                         syslog(LOG_WARNING,
 148                                "SMTP cleanup: have_: ID[%ld]",
 149                                ((OneQueItem *)VData)->MessageID);
 150         }
 151         pthread_mutex_unlock(&ActiveQItemsLock);
 152         DeleteHashPos(&It);
 153 }
 154
 155
 156 void FreeMailQEntry(void *qv)
 157 {
 158         MailQEntry *Q = qv;
 159         FreeStrBuf(&Q->Recipient);
 160         FreeStrBuf(&Q->StatusMessage);
 161         free(Q);
 162 }
 163 void FreeQueItem(OneQueItem **Item)
 164 {
 165         DeleteHash(&(*Item)->MailQEntries);
 166         FreeStrBuf(&(*Item)->EnvelopeFrom);
 167         FreeStrBuf(&(*Item)->BounceTo);
 168         FreeURL(&(*Item)->URL);
 169         free(*Item);
 170         Item = NULL;
 171 }
 172 void HFreeQueItem(void *Item)
 173 {
 174         FreeQueItem((OneQueItem**)&Item);
 175 }
 176
 177 /* inspect recipients with a status of:
 178  * - 0 (no delivery yet attempted)
 179  * - 3/4 (transient errors
 180  *        were experienced and it's time to try again)
 181  */
 182 int CountActiveQueueEntries(OneQueItem *MyQItem)
 183 {
 184         HashPos  *It;
 185         long len;
 186         const char *Key;
 187         void *vQE;
 188
 189         MyQItem->ActiveDeliveries = 0;
 190         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 191         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 192         {
 193                 MailQEntry *ThisItem = vQE;
 194                 if ((ThisItem->Status == 0) ||
 195                     (ThisItem->Status == 3) ||
 196                     (ThisItem->Status == 4))
 197                 {
 198                         MyQItem->ActiveDeliveries++;
 199                         ThisItem->Active = 1;
 200                 }
 201                 else
 202                         ThisItem->Active = 0;
 203         }
 204         DeleteHashPos(&It);
 205         return MyQItem->ActiveDeliveries;
 206 }
 207
 208 OneQueItem *DeserializeQueueItem(StrBuf *RawQItem, long QueMsgID)
 209 {
 210         OneQueItem *Item;
 211         const char *pLine = NULL;
 212         StrBuf *Line;
 213         StrBuf *Token;
 214         void *v;
 215
 216         Item = (OneQueItem*)malloc(sizeof(OneQueItem));
 217         memset(Item, 0, sizeof(OneQueItem));
 218         Item->LastAttempt.retry = SMTP_RETRY_INTERVAL;
 219         Item->MessageID = -1;
 220         Item->QueMsgID = QueMsgID;
 221
 222         Token = NewStrBuf();
 223         Line = NewStrBufPlain(NULL, 128);
 224         while (pLine != StrBufNOTNULL) {
 225                 const char *pItemPart = NULL;
 226                 void *vHandler;
 227
 228                 StrBufExtract_NextToken(Line, RawQItem, &pLine, '\n');
 229                 if (StrLength(Line) == 0) continue;
 230                 StrBufExtract_NextToken(Token, Line, &pItemPart, '|');
 231                 if (GetHash(QItemHandlers, SKEY(Token), &vHandler))
 232                 {
 233                         QItemHandler H;
 234                         H = (QItemHandler) vHandler;
 235                         H(Item, Line, &pItemPart);
 236                 }
 237         }
 238         FreeStrBuf(&Line);
 239         FreeStrBuf(&Token);
 240
 241         pthread_mutex_lock(&ActiveQItemsLock);
 242         if (GetHash(ActiveQItems,
 243                     LKEY(Item->MessageID),
 244                     &v))
 245         {
 246                 /* WHOOPS. somebody else is already working on this. */
 247                 pthread_mutex_unlock(&ActiveQItemsLock);
 248                 FreeQueItem(&Item);
 249                 return NULL;
 250         }
 251         else {
 252                 /* mark our claim on this. */
 253                 Put(ActiveQItems,
 254                     LKEY(Item->MessageID),
 255                     Item,
 256                     HFreeQueItem);
 257                 pthread_mutex_unlock(&ActiveQItemsLock);
 258         }
 259
 260         return Item;
 261 }
 262
 263 StrBuf *SerializeQueueItem(OneQueItem *MyQItem)
 264 {
 265         StrBuf *QMessage;
 266         HashPos  *It;
 267         const char *Key;
 268         long len;
 269         void *vQE;
 270
 271         QMessage = NewStrBufPlain(NULL, SIZ);
 272         StrBufPrintf(QMessage, "Content-type: %s\n", SPOOLMIME);
 273
 274 //                 "attempted|%ld\n"  "retry|%ld\n",, (long)time(NULL), (long)retry );
 275         StrBufAppendBufPlain(QMessage, HKEY("\nmsgid|"), 0);
 276         StrBufAppendPrintf(QMessage, "%ld", MyQItem->MessageID);
 277
 278         StrBufAppendBufPlain(QMessage, HKEY("\nsubmitted|"), 0);
 279         StrBufAppendPrintf(QMessage, "%ld", MyQItem->Submitted);
 280
 281         if (StrLength(MyQItem->BounceTo) > 0) {
 282                 StrBufAppendBufPlain(QMessage, HKEY("\nbounceto|"), 0);
 283                 StrBufAppendBuf(QMessage, MyQItem->BounceTo, 0);
 284         }
 285
 286         if (StrLength(MyQItem->EnvelopeFrom) > 0) {
 287                 StrBufAppendBufPlain(QMessage, HKEY("\nenvelope_from|"), 0);
 288                 StrBufAppendBuf(QMessage, MyQItem->EnvelopeFrom, 0);
 289         }
 290
 291         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 292         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 293         {
 294                 MailQEntry *ThisItem = vQE;
 295                 int i;
 296
 297                 if (!ThisItem->Active)
 298                         continue; /* skip already sent ones from the spoolfile. */
 299
 300                 for (i=0; i < ThisItem->nAttempts; i++) {
 301                         /* TODO: most probably there is just one retry/attempted per message! */
 302                         StrBufAppendBufPlain(QMessage, HKEY("\nretry|"), 0);
 303                         StrBufAppendPrintf(QMessage, "%ld",
 304                                            ThisItem->Attempts[i].retry);
 305
 306                         StrBufAppendBufPlain(QMessage, HKEY("\nattempted|"), 0);
 307                         StrBufAppendPrintf(QMessage, "%ld",
 308                                            ThisItem->Attempts[i].when);
 309                 }
 310                 StrBufAppendBufPlain(QMessage, HKEY("\nremote|"), 0);
 311                 StrBufAppendBuf(QMessage, ThisItem->Recipient, 0);
 312                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 313                 StrBufAppendPrintf(QMessage, "%d", ThisItem->Status);
 314                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 315                 StrBufAppendBuf(QMessage, ThisItem->StatusMessage, 0);
 316         }
 317         DeleteHashPos(&It);
 318         StrBufAppendBufPlain(QMessage, HKEY("\n"), 0);
 319         return QMessage;
 320 }
 321
 322
 323
 324
 325
 326 void NewMailQEntry(OneQueItem *Item)
 327 {
 328         Item->Current = (MailQEntry*) malloc(sizeof(MailQEntry));
 329         memset(Item->Current, 0, sizeof(MailQEntry));
 330
 331         if (Item->MailQEntries == NULL)
 332                 Item->MailQEntries = NewHash(1, Flathash);
 333         Item->Current->StatusMessage = NewStrBuf();
 334         Item->Current->n = GetCount(Item->MailQEntries);
 335         Put(Item->MailQEntries, IKEY(Item->Current->n), Item->Current, FreeMailQEntry);
 336 }
 337
 338 void QItem_Handle_MsgID(OneQueItem *Item, StrBuf *Line, const char **Pos)
 339 {
 340         Item->MessageID = StrBufExtractNext_long(Line, Pos, '|');
 341 }
 342
 343 void QItem_Handle_EnvelopeFrom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 344 {
 345         if (Item->EnvelopeFrom == NULL)
 346                 Item->EnvelopeFrom = NewStrBufPlain(NULL, StrLength(Line));
 347         StrBufExtract_NextToken(Item->EnvelopeFrom, Line, Pos, '|');
 348 }
 349
 350 void QItem_Handle_BounceTo(OneQueItem *Item, StrBuf *Line, const char **Pos)
 351 {
 352         if (Item->BounceTo == NULL)
 353                 Item->BounceTo = NewStrBufPlain(NULL, StrLength(Line));
 354         StrBufExtract_NextToken(Item->BounceTo, Line, Pos, '|');
 355 }
 356
 357 void QItem_Handle_Recipient(OneQueItem *Item, StrBuf *Line, const char **Pos)
 358 {
 359         if (Item->Current == NULL)
 360                 NewMailQEntry(Item);
 361         if (Item->Current->Recipient == NULL)
 362                 Item->Current->Recipient =  NewStrBufPlain(NULL, StrLength(Line));
 363         StrBufExtract_NextToken(Item->Current->Recipient, Line, Pos, '|');
 364         Item->Current->Status = StrBufExtractNext_int(Line, Pos, '|');
 365         StrBufExtract_NextToken(Item->Current->StatusMessage, Line, Pos, '|');
 366         Item->Current = NULL; // TODO: is this always right?
 367 }
 368
 369
 370 void QItem_Handle_retry(OneQueItem *Item, StrBuf *Line, const char **Pos)
 371 {
 372         if (Item->Current == NULL)
 373                 NewMailQEntry(Item);
 374         if (Item->Current->Attempts[Item->Current->nAttempts].retry != 0)
 375                 Item->Current->nAttempts++;
 376         if (Item->Current->nAttempts > MaxAttempts) {
 377                 Item->FailNow = 1;
 378                 return;
 379         }
 380         Item->Current->Attempts[Item->Current->nAttempts].retry = StrBufExtractNext_int(Line, Pos, '|');
 381 }
 382
 383
 384 void QItem_Handle_Submitted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 385 {
 386         Item->Submitted = atol(*Pos);
 387
 388 }
 389
 390 void QItem_Handle_Attempted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 391 {
 392         if (Item->Current == NULL)
 393                 NewMailQEntry(Item);
 394         if (Item->Current->Attempts[Item->Current->nAttempts].when != 0)
 395                 Item->Current->nAttempts++;
 396         if (Item->Current->nAttempts > MaxAttempts) {
 397                 Item->FailNow = 1;
 398                 return;
 399         }
 400
 401         Item->Current->Attempts[Item->Current->nAttempts].when = StrBufExtractNext_int(Line, Pos, '|');
 402         if (Item->Current->Attempts[Item->Current->nAttempts].when > Item->LastAttempt.when)
 403         {
 404                 Item->LastAttempt.when = Item->Current->Attempts[Item->Current->nAttempts].when;
 405                 Item->LastAttempt.retry = Item->Current->Attempts[Item->Current->nAttempts].retry * 2;
 406                 if (Item->LastAttempt.retry > SMTP_RETRY_MAX)
 407                         Item->LastAttempt.retry = SMTP_RETRY_MAX;
 408         }
 409 }
 410
 411
 412
 413 /**
 414  * this one has to have the context for loading the message via the redirect buffer...
 415  */
 416 StrBuf *smtp_load_msg(OneQueItem *MyQItem, int n)
 417 {
 418         CitContext *CCC=CC;
 419         StrBuf *SendMsg;
 420
 421         CCC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
 422         CtdlOutputMsg(MyQItem->MessageID, MT_RFC822, HEADERS_ALL, 0, 1, NULL, (ESC_DOT|SUPPRESS_ENV_TO) );
 423         SendMsg = CCC->redirect_buffer;
 424         CCC->redirect_buffer = NULL;
 425         if ((StrLength(SendMsg) > 0) &&
 426             ChrPtr(SendMsg)[StrLength(SendMsg) - 1] != '\n') {
 427                 syslog(LOG_WARNING,
 428                        "SMTP client[%d]: Possible problem: message did not "
 429                        "correctly terminate. (expecting 0x10, got 0x%02x)\n",
 430                        MsgCount, //yes uncool, but best choice here...
 431                        ChrPtr(SendMsg)[StrLength(SendMsg) - 1] );
 432                 StrBufAppendBufPlain(SendMsg, HKEY("\r\n"), 0);
 433         }
 434         return SendMsg;
 435 }
 436
 437
 438
 439 /*
 440  * smtp_do_bounce() is caled by smtp_do_procmsg() to scan a set of delivery
 441  * instructions for "5" codes (permanent fatal errors) and produce/deliver
 442  * a "bounce" message (delivery status notification).
 443  */
 444 void smtpq_do_bounce(OneQueItem *MyQItem, StrBuf *OMsgTxt)
 445 {
 446         static int seq = 0;
 447
 448         struct CtdlMessage *bmsg = NULL;
 449         StrBuf *boundary;
 450         StrBuf *Msg = NULL;
 451         StrBuf *BounceMB;
 452         struct recptypes *valid;
 453
 454         HashPos *It;
 455         void *vQE;
 456         long len;
 457         const char *Key;
 458
 459         int successful_bounce = 0;
 460         int num_bounces = 0;
 461         int give_up = 0;
 462
 463         syslog(LOG_DEBUG, "smtp_do_bounce() called\n");
 464
 465         if ( (ev_time() - MyQItem->Submitted) > SMTP_GIVE_UP ) {
 466                 give_up = 1;/// TODO: replace time by libevq timer get
 467         }
 468
 469         /*
 470          * Now go through the instructions checking for stuff.
 471          */
 472         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 473         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 474         {
 475                 MailQEntry *ThisItem = vQE;
 476                 if ((ThisItem->Status == 5) || /* failed now? */
 477                     ((give_up == 1) && (ThisItem->Status != 2))) /* giving up after failed attempts... */
 478                 {
 479                         if (num_bounces == 0)
 480                                 Msg = NewStrBufPlain(NULL, 1024);
 481                         ++num_bounces;
 482
 483                         StrBufAppendBuf(Msg, ThisItem->Recipient, 0);
 484                         StrBufAppendBufPlain(Msg, HKEY(": "), 0);
 485                         StrBufAppendBuf(Msg, ThisItem->StatusMessage, 0);
 486                         StrBufAppendBufPlain(Msg, HKEY("\r\n"), 0);
 487                 }
 488         }
 489         DeleteHashPos(&It);
 490
 491         /* Deliver the bounce if there's anything worth mentioning */
 492         syslog(LOG_DEBUG, "num_bounces = %d\n", num_bounces);
 493
 494         if (num_bounces == 0) {
 495                 FreeStrBuf(&Msg);
 496                 return;
 497         }
 498
 499         boundary = NewStrBufPlain(HKEY("=_Citadel_Multipart_"));
 500         StrBufAppendPrintf(boundary, "%s_%04x%04x", config.c_fqdn, getpid(), ++seq);
 501
 502         /* Start building our bounce message; go shopping for memory first. */
 503         BounceMB = NewStrBufPlain(NULL,
 504                                   1024 + /* mime stuff.... */
 505                                   StrLength(Msg) +  /* the bounce information... */
 506                                   StrLength(OMsgTxt)); /* the original message */
 507         if (BounceMB == NULL) {
 508                 FreeStrBuf(&boundary);
 509                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 510
 511                 return;
 512         }
 513
 514         bmsg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
 515         if (bmsg == NULL) {
 516                 FreeStrBuf(&boundary);
 517                 FreeStrBuf(&BounceMB);
 518                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 519
 520                 return;
 521         }
 522         memset(bmsg, 0, sizeof(struct CtdlMessage));
 523
 524
 525         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: multipart/mixed; boundary=\""), 0);
 526         StrBufAppendBuf(BounceMB, boundary, 0);
 527         StrBufAppendBufPlain(BounceMB, HKEY("\"\r\n"), 0);
 528         StrBufAppendBufPlain(BounceMB, HKEY("MIME-Version: 1.0\r\n"), 0);
 529         StrBufAppendBufPlain(BounceMB, HKEY("X-Mailer: " CITADEL "\r\n"), 0);
 530         StrBufAppendBufPlain(BounceMB, HKEY("\r\nThis is a multipart message in MIME format.\r\n\r\n"), 0);
 531         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 532         StrBufAppendBuf(BounceMB, boundary, 0);
 533         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 534         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: text/plain\r\n\r\n"), 0);
 535
 536         if (give_up)
 537                 StrBufAppendBufPlain(
 538                         BounceMB,
 539                         HKEY(
 540                                 "A message you sent could not be delivered to some or all of its recipients\n"
 541                                 "due to prolonged unavailability of its destination(s).\n"
 542                                 "Giving up on the following addresses:\n\n"
 543                                 ), 0);
 544         else
 545                 StrBufAppendBufPlain(
 546                         BounceMB,
 547                         HKEY(
 548                                 "A message you sent could not be delivered to some or all of its recipients.\n"
 549                                 "The following addresses were undeliverable:\n\n"
 550                                 ), 0);
 551
 552         StrBufAppendBuf(BounceMB, Msg, 0);
 553         FreeStrBuf(&Msg);
 554
 555         /* Attach the original message */
 556         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 557         StrBufAppendBuf(BounceMB, boundary, 0);
 558         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 559         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: message/rfc822\r\n"), 0);
 560         StrBufAppendBufPlain(BounceMB, HKEY("Content-Transfer-Encoding: 7bit\r\n"), 0);
 561         StrBufAppendBufPlain(BounceMB, HKEY("Content-Disposition: inline\r\n"), 0);
 562         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 563         StrBufAppendBuf(BounceMB, OMsgTxt, 0);
 564
 565         /* Close the multipart MIME scope */
 566         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 567         StrBufAppendBuf(BounceMB, boundary, 0);
 568         StrBufAppendBufPlain(BounceMB, HKEY("--\r\n"), 0);
 569
 570
 571
 572         bmsg->cm_magic = CTDLMESSAGE_MAGIC;
 573         bmsg->cm_anon_type = MES_NORMAL;
 574         bmsg->cm_format_type = FMT_RFC822;
 575
 576         bmsg->cm_fields['O'] = strdup(MAILROOM);
 577         bmsg->cm_fields['A'] = strdup("Citadel");
 578         bmsg->cm_fields['N'] = strdup(config.c_nodename);
 579         bmsg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
 580         bmsg->cm_fields['M'] = SmashStrBuf(&BounceMB);
 581
 582         /* First try the user who sent the message */
 583         if (StrLength(MyQItem->BounceTo) == 0)
 584                 syslog(LOG_ERR, "No bounce address specified\n");
 585         else
 586                 syslog(LOG_DEBUG, "bounce to user? <%s>\n", ChrPtr(MyQItem->BounceTo));
 587
 588         /* Can we deliver the bounce to the original sender? */
 589         valid = validate_recipients(ChrPtr(MyQItem->BounceTo), NULL, 0);
 590         if ((valid != NULL) && (valid->num_error == 0)) {
 591                 CtdlSubmitMsg(bmsg, valid, "", QP_EADDR);
 592                 successful_bounce = 1;
 593         }
 594
 595         /* If not, post it in the Aide> room */
 596         if (successful_bounce == 0) {
 597                 CtdlSubmitMsg(bmsg, NULL, config.c_aideroom, QP_EADDR);
 598         }
 599
 600         /* Free up the memory we used */
 601         free_recipients(valid);
 602         FreeStrBuf(&boundary);
 603         CtdlFreeMessage(bmsg);
 604         syslog(LOG_DEBUG, "Done processing bounces\n");
 605 }
 606
 607
 608
 609 /*
 610 {
 611
 612         if (threadding)
 613                 n_smarthosts =  get_hosts(char *mxbuf, char *rectype);
 614 }
 615 */
 616 /*
 617  * smtp_do_procmsg()
 618  *
 619  * Called by smtp_do_queue() to handle an individual message.
 620  */
 621 void smtp_do_procmsg(long msgnum, void *userdata) {
 622         struct CtdlMessage *msg = NULL;
 623         char *instr = NULL;
 624         StrBuf *PlainQItem;
 625         OneQueItem *MyQItem;
 626         char *pch;
 627         HashPos  *It;
 628         void *vQE;
 629         long len;
 630         const char *Key;
 631         int nRelays = 0;
 632         ParsedURL *RelayUrls = NULL;
 633         int HaveBuffers = 0;
 634         StrBuf *Msg =NULL;
 635
 636         syslog(LOG_DEBUG, "SMTP Queue: smtp_do_procmsg(%ld)\n", msgnum);
 637         ///strcpy(envelope_from, "");
 638
 639         msg = CtdlFetchMessage(msgnum, 1);
 640         if (msg == NULL) {
 641                 syslog(LOG_ERR, "SMTP Queue: tried %ld but no such message!\n", msgnum);
 642                 return;
 643         }
 644
 645         pch = instr = msg->cm_fields['M'];
 646
 647         /* Strip out the headers (no not amd any other non-instruction) line */
 648         while (pch != NULL) {
 649                 pch = strchr(pch, '\n');
 650                 if ((pch != NULL) && (*(pch + 1) == '\n')) {
 651                         instr = pch + 2;
 652                         pch = NULL;
 653                 }
 654         }
 655         PlainQItem = NewStrBufPlain(instr, -1);
 656         CtdlFreeMessage(msg);
 657         MyQItem = DeserializeQueueItem(PlainQItem, msgnum);
 658         FreeStrBuf(&PlainQItem);
 659
 660         if (MyQItem == NULL) {
 661                 syslog(LOG_ERR, "SMTP Queue: Msg No %ld: already in progress!\n", msgnum);
 662                 return; /* s.b. else is already processing... */
 663         }
 664
 665         /*
 666          * Postpone delivery if we've already tried recently.
 667          * /
 668         if (((time(NULL) - MyQItem->LastAttempt.when) < MyQItem->LastAttempt.retry) && (run_queue_now == 0)) {
 669                 syslog(LOG_DEBUG, "SMTP client: Retry time not yet reached.\n");
 670
 671                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 672                 pthread_mutex_lock(&ActiveQItemsLock);
 673                 {
 674                         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 675                                 DeleteEntryFromHash(ActiveQItems, It);
 676                 }
 677                 pthread_mutex_unlock(&ActiveQItemsLock);
 678                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 679                 DeleteHashPos(&It);
 680                 return;
 681         }// TODO: reenable me.*/
 682
 683         /*
 684          * Bail out if there's no actual message associated with this
 685          */
 686         if (MyQItem->MessageID < 0L) {
 687                 syslog(LOG_ERR, "SMTP Queue: no 'msgid' directive found!\n");
 688                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 689                 pthread_mutex_lock(&ActiveQItemsLock);
 690                 {
 691                         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 692                                 DeleteEntryFromHash(ActiveQItems, It);
 693                 }
 694                 pthread_mutex_unlock(&ActiveQItemsLock);
 695                 DeleteHashPos(&It);
 696                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 697                 return;
 698         }
 699
 700         {
 701                 char mxbuf[SIZ];
 702                 ParsedURL **Url = &MyQItem->URL;
 703                 nRelays = get_hosts(mxbuf, "smarthost");
 704                 if (nRelays > 0) {
 705                         StrBuf *All;
 706                         StrBuf *One;
 707                         const char *Pos = NULL;
 708                         All = NewStrBufPlain(mxbuf, -1);
 709                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 710
 711                         while ((Pos != StrBufNOTNULL) && ((Pos == NULL) || !IsEmptyStr(Pos))) {
 712                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 713                                 if (!ParseURL(Url, One, 25))
 714                                         syslog(LOG_DEBUG, "Failed to parse: %s\n", ChrPtr(One));
 715                                 else {
 716                                         ///if (!Url->IsIP)) /// todo dupe me fork ipv6
 717                                         Url = &(*Url)->Next;
 718
 719                                 }
 720                         }
 721                         FreeStrBuf(&All);
 722                         FreeStrBuf(&One);
 723                 }
 724
 725                 Url = &MyQItem->FallBackHost;
 726                 nRelays = get_hosts(mxbuf, "fallbackhost");
 727                 if (nRelays > 0) {
 728                         StrBuf *All;
 729                         StrBuf *One;
 730                         const char *Pos = NULL;
 731                         All = NewStrBufPlain(mxbuf, -1);
 732                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 733
 734                         while ((Pos != StrBufNOTNULL) && ((Pos == NULL) || !IsEmptyStr(Pos))) {
 735                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 736                                 if (!ParseURL(Url, One, 25))
 737                                         syslog(LOG_DEBUG, "Failed to parse: %s\n", ChrPtr(One));
 738                                 else
 739                                         Url = &(*Url)->Next;
 740                         }
 741                         FreeStrBuf(&All);
 742                         FreeStrBuf(&One);
 743                 }
 744         }
 745
 746         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 747         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 748         {
 749                 MailQEntry *ThisItem = vQE;
 750                 syslog(LOG_DEBUG, "SMTP Queue: Task: <%s> %d\n", ChrPtr(ThisItem->Recipient), ThisItem->Active);
 751         }
 752         DeleteHashPos(&It);
 753
 754         CountActiveQueueEntries(MyQItem);
 755         if (MyQItem->ActiveDeliveries > 0)
 756         {
 757                 int n = MsgCount++;
 758                 int m = MyQItem->ActiveDeliveries;
 759                 int i = 1;
 760                 Msg = smtp_load_msg(MyQItem, n);
 761                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 762                 while ((i <= m) &&
 763                        (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE)))
 764                 {
 765                         MailQEntry *ThisItem = vQE;
 766                         if (ThisItem->Active == 1) {
 767                                 int KeepBuffers = (i == m);
 768                                 if (i > 1) n = MsgCount++;
 769                                 syslog(LOG_DEBUG,
 770                                        "SMTP Queue: Trying <%ld> <%s> %d / %d \n",
 771                                        MyQItem->MessageID,
 772                                        ChrPtr(ThisItem->Recipient),
 773                                        i,
 774                                        m);
 775                                 smtp_try_one_queue_entry(MyQItem,
 776                                                          ThisItem,
 777                                                          Msg,
 778                                                          KeepBuffers,
 779                                                          n,
 780                                                          RelayUrls);
 781                                 if (KeepBuffers) HaveBuffers = 1;
 782                                 i++;
 783                         }
 784                 }
 785                 DeleteHashPos(&It);
 786         }
 787         else
 788         {
 789                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 790                 pthread_mutex_lock(&ActiveQItemsLock);
 791                 {
 792                         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 793                                 DeleteEntryFromHash(ActiveQItems, It);
 794                         else
 795                         {
 796                                 long len;
 797                                 const char* Key;
 798                                 void *VData;
 799                                 syslog(LOG_WARNING,
 800                                        "SMTP cleanup: unable to find QItem with ID[%ld]",
 801                                        MyQItem->MessageID);
 802                                 while (GetNextHashPos(ActiveQItems, It, &len, &Key, &VData))
 803                                         syslog(LOG_WARNING,
 804                                                "SMTP cleanup: have: ID[%ld]",
 805                                                ((OneQueItem *)VData)->MessageID);
 806                         }
 807
 808                 }
 809                 pthread_mutex_unlock(&ActiveQItemsLock);
 810                 DeleteHashPos(&It);
 811                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 812
 813 // TODO: bounce & delete?
 814
 815         }
 816         if (!HaveBuffers) {
 817                 FreeStrBuf (&Msg);
 818 // TODO : free RelayUrls
 819         }
 820 }
 821
 822
 823
 824 /*
 825  * smtp_queue_thread()
 826  *
 827  * Run through the queue sending out messages.
 828  */
 829 void smtp_do_queue(void) {
 830         static int is_running = 0;
 831         int num_processed = 0;
 832
 833         if (is_running) return;         /* Concurrency check - only one can run */
 834         is_running = 1;
 835
 836         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 837         syslog(LOG_INFO, "SMTP client: processing outbound queue");
 838
 839         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
 840                 syslog(LOG_ERR, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
 841         }
 842         else {
 843                 num_processed = CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_do_procmsg, NULL);
 844         }
 845         syslog(LOG_INFO, "SMTP client: queue run completed; %d messages processed", num_processed);
 846         run_queue_now = 0;
 847         is_running = 0;
 848 }
 849
 850
 851
 852 /*
 853  * Initialize the SMTP outbound queue
 854  */
 855 void smtp_init_spoolout(void) {
 856         struct ctdlroom qrbuf;
 857
 858         /*
 859          * Create the room.  This will silently fail if the room already
 860          * exists, and that's perfectly ok, because we want it to exist.
 861          */
 862         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_MAILBOX);
 863
 864         /*
 865          * Make sure it's set to be a "system room" so it doesn't show up
 866          * in the <K>nown rooms list for Aides.
 867          */
 868         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
 869                 qrbuf.QRflags2 |= QR2_SYSTEM;
 870                 CtdlPutRoomLock(&qrbuf);
 871         }
 872 }
 873
 874
 875
 876
 877 /*****************************************************************************/
 878 /*                          SMTP UTILITY COMMANDS                            */
 879 /*****************************************************************************/
 880
 881 void cmd_smtp(char *argbuf) {
 882         char cmd[64];
 883         char node[256];
 884         char buf[1024];
 885         int i;
 886         int num_mxhosts;
 887
 888         if (CtdlAccessCheck(ac_aide)) return;
 889
 890         extract_token(cmd, argbuf, 0, '|', sizeof cmd);
 891
 892         if (!strcasecmp(cmd, "mx")) {
 893                 extract_token(node, argbuf, 1, '|', sizeof node);
 894                 num_mxhosts = getmx(buf, node);
 895                 cprintf("%d %d MX hosts listed for %s\n",
 896                         LISTING_FOLLOWS, num_mxhosts, node);
 897                 for (i=0; i<num_mxhosts; ++i) {
 898                         extract_token(node, buf, i, '|', sizeof node);
 899                         cprintf("%s\n", node);
 900                 }
 901                 cprintf("000\n");
 902                 return;
 903         }
 904
 905         else if (!strcasecmp(cmd, "runqueue")) {
 906                 run_queue_now = 1;
 907                 cprintf("%d All outbound SMTP will be retried now.\n", CIT_OK);
 908                 return;
 909         }
 910
 911         else {
 912                 cprintf("%d Invalid command.\n", ERROR + ILLEGAL_VALUE);
 913         }
 914
 915 }
 916
 917
 918 CTDL_MODULE_INIT(smtp_queu)
 919 {
 920         if (!threading)
 921         {
 922                 CtdlFillSystemContext(&smtp_queue_CC, "SMTP_Send");
 923                 ActiveQItems = NewHash(1, lFlathash);
 924                 pthread_mutex_init(&ActiveQItemsLock, NULL);
 925
 926                 QItemHandlers = NewHash(0, NULL);
 927
 928                 Put(QItemHandlers, HKEY("msgid"), QItem_Handle_MsgID, reference_free_handler);
 929                 Put(QItemHandlers, HKEY("envelope_from"), QItem_Handle_EnvelopeFrom, reference_free_handler);
 930                 Put(QItemHandlers, HKEY("retry"), QItem_Handle_retry, reference_free_handler);
 931                 Put(QItemHandlers, HKEY("attempted"), QItem_Handle_Attempted, reference_free_handler);
 932                 Put(QItemHandlers, HKEY("remote"), QItem_Handle_Recipient, reference_free_handler);
 933                 Put(QItemHandlers, HKEY("bounceto"), QItem_Handle_BounceTo, reference_free_handler);
 934                 Put(QItemHandlers, HKEY("submitted"), QItem_Handle_Submitted, reference_free_handler);
 935 ////TODO: flush qitemhandlers on exit
 936                 smtp_init_spoolout();
 937
 938                 CtdlRegisterCleanupHook(smtp_evq_cleanup);
 939
 940                 CtdlRegisterProtoHook(cmd_smtp, "SMTP", "SMTP utility commands");
 941                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER);
 942         }
 943
 944         /* return our Subversion id for the Log */
 945         return "smtpeventclient";
 946 }