citadel/modules/smtp/serv_smtpqueue.c

   1 /*
   2  * This module is an SMTP and ESMTP implementation for the Citadel system.
   3  * It is compliant with all of the following:
   4  *
   5  * RFC  821 - Simple Mail Transfer Protocol
   6  * RFC  876 - Survey of SMTP Implementations
   7  * RFC 1047 - Duplicate messages and SMTP
   8  * RFC 1652 - 8 bit MIME
   9  * RFC 1869 - Extended Simple Mail Transfer Protocol
  10  * RFC 1870 - SMTP Service Extension for Message Size Declaration
  11  * RFC 2033 - Local Mail Transfer Protocol
  12  * RFC 2197 - SMTP Service Extension for Command Pipelining
  13  * RFC 2476 - Message Submission
  14  * RFC 2487 - SMTP Service Extension for Secure SMTP over TLS
  15  * RFC 2554 - SMTP Service Extension for Authentication
  16  * RFC 2821 - Simple Mail Transfer Protocol
  17  * RFC 2822 - Internet Message Format
  18  * RFC 2920 - SMTP Service Extension for Command Pipelining
  19  *
  20  * The VRFY and EXPN commands have been removed from this implementation
  21  * because nobody uses these commands anymore, except for spammers.
  22  *
  23  * Copyright (c) 1998-2009 by the citadel.org team
  24  *
  25  *  This program is free software; you can redistribute it and/or modify
  26  *  it under the terms of the GNU General Public License as published by
  27  *  the Free Software Foundation; either version 3 of the License, or
  28  *  (at your option) any later version.
  29  *
  30  *  This program is distributed in the hope that it will be useful,
  31  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  32  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  33  *  GNU General Public License for more details.
  34  *
  35  *  You should have received a copy of the GNU General Public License
  36  *  along with this program; if not, write to the Free Software
  37  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  38  */
  39
  40 #include "sysdep.h"
  41 #include <stdlib.h>
  42 #include <unistd.h>
  43 #include <stdio.h>
  44 #include <termios.h>
  45 #include <fcntl.h>
  46 #include <signal.h>
  47 #include <pwd.h>
  48 #include <errno.h>
  49 #include <sys/types.h>
  50 #include <syslog.h>
  51
  52 #if TIME_WITH_SYS_TIME
  53 # include <sys/time.h>
  54 # include <time.h>
  55 #else
  56 # if HAVE_SYS_TIME_H
  57 #  include <sys/time.h>
  58 # else
  59 #  include <time.h>
  60 # endif
  61 #endif
  62 #include <sys/wait.h>
  63 #include <ctype.h>
  64 #include <string.h>
  65 #include <limits.h>
  66 #include <sys/socket.h>
  67 #include <netinet/in.h>
  68 #include <arpa/inet.h>
  69 #include <libcitadel.h>
  70 #include "citadel.h"
  71 #include "server.h"
  72 #include "citserver.h"
  73 #include "support.h"
  74 #include "config.h"
  75 #include "control.h"
  76 #include "user_ops.h"
  77 #include "database.h"
  78 #include "msgbase.h"
  79 #include "internet_addressing.h"
  80 #include "genstamp.h"
  81 #include "domain.h"
  82 #include "clientsocket.h"
  83 #include "locate_host.h"
  84 #include "citadel_dirs.h"
  85
  86 #include "ctdl_module.h"
  87
  88 #include "smtpqueue.h"
  89 #include "event_client.h"
  90
  91
  92 struct CitContext smtp_queue_CC;
  93 pthread_mutex_t ActiveQItemsLock;
  94 HashList *ActiveQItems  = NULL;
  95 HashList *QItemHandlers = NULL;
  96
  97 int MsgCount            = 0;
  98 int run_queue_now       = 0;    /* Set to 1 to ignore SMTP send retry times */
  99
 100 void smtp_try_one_queue_entry(OneQueItem *MyQItem,
 101                               MailQEntry *MyQEntry,
 102                               StrBuf *MsgText,
 103                               int KeepMsgText, /* KeepMsgText allows us to use MsgText as ours. */
 104                               int MsgCount,
 105                               ParsedURL *RelayUrls);
 106
 107
 108 void smtp_evq_cleanup(void)
 109 {
 110
 111         pthread_mutex_lock(&ActiveQItemsLock);
 112         DeleteHash(&QItemHandlers);
 113         DeleteHash(&ActiveQItems);
 114         pthread_mutex_unlock(&ActiveQItemsLock);
 115         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 116 /*      citthread_mutex_destroy(&ActiveQItemsLock); TODO */
 117 }
 118
 119 int DecreaseQReference(OneQueItem *MyQItem)
 120 {
 121         int IDestructQueItem;
 122
 123         pthread_mutex_lock(&ActiveQItemsLock);
 124         MyQItem->ActiveDeliveries--;
 125         IDestructQueItem = MyQItem->ActiveDeliveries == 0;
 126         pthread_mutex_unlock(&ActiveQItemsLock);
 127         return IDestructQueItem;
 128 }
 129
 130 void RemoveQItem(OneQueItem *MyQItem)
 131 {
 132         long len;
 133         const char* Key;
 134         void *VData;
 135         HashPos  *It;
 136
 137         pthread_mutex_lock(&ActiveQItemsLock);
 138         It = GetNewHashPos(ActiveQItems, 0);
 139         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 140                 DeleteEntryFromHash(ActiveQItems, It);
 141         else
 142         {
 143                 syslog(LOG_WARNING,
 144                        "SMTP cleanup: unable to find QItem with ID[%ld]",
 145                        MyQItem->MessageID);
 146                 while (GetNextHashPos(ActiveQItems, It, &len, &Key, &VData))
 147                         syslog(LOG_WARNING,
 148                                "SMTP cleanup: have_: ID[%ld]",
 149                                ((OneQueItem *)VData)->MessageID);
 150         }
 151         pthread_mutex_unlock(&ActiveQItemsLock);
 152         DeleteHashPos(&It);
 153 }
 154
 155
 156 void FreeMailQEntry(void *qv)
 157 {
 158         MailQEntry *Q = qv;
 159         FreeStrBuf(&Q->Recipient);
 160         FreeStrBuf(&Q->StatusMessage);
 161         free(Q);
 162 }
 163 void FreeQueItem(OneQueItem **Item)
 164 {
 165         DeleteHash(&(*Item)->MailQEntries);
 166         FreeStrBuf(&(*Item)->EnvelopeFrom);
 167         FreeStrBuf(&(*Item)->BounceTo);
 168         FreeURL(&(*Item)->URL);
 169         free(*Item);
 170         Item = NULL;
 171 }
 172 void HFreeQueItem(void *Item)
 173 {
 174         FreeQueItem((OneQueItem**)&Item);
 175 }
 176
 177 /* inspect recipients with a status of:
 178  * - 0 (no delivery yet attempted)
 179  * - 3/4 (transient errors
 180  *        were experienced and it's time to try again)
 181  */
 182 int CountActiveQueueEntries(OneQueItem *MyQItem)
 183 {
 184         HashPos  *It;
 185         long len;
 186         long ActiveDeliveries;
 187         const char *Key;
 188         void *vQE;
 189
 190         ActiveDeliveries = 0;
 191         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 192         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 193         {
 194                 MailQEntry *ThisItem = vQE;
 195                 if ((ThisItem->Status == 0) ||
 196                     (ThisItem->Status == 3) ||
 197                     (ThisItem->Status == 4))
 198                 {
 199                         ActiveDeliveries++;
 200                         ThisItem->Active = 1;
 201                 }
 202                 else
 203                         ThisItem->Active = 0;
 204         }
 205         DeleteHashPos(&It);
 206         return ActiveDeliveries;
 207 }
 208
 209 OneQueItem *DeserializeQueueItem(StrBuf *RawQItem, long QueMsgID)
 210 {
 211         OneQueItem *Item;
 212         const char *pLine = NULL;
 213         StrBuf *Line;
 214         StrBuf *Token;
 215         void *v;
 216
 217         Item = (OneQueItem*)malloc(sizeof(OneQueItem));
 218         memset(Item, 0, sizeof(OneQueItem));
 219         Item->LastAttempt.retry = SMTP_RETRY_INTERVAL;
 220         Item->MessageID = -1;
 221         Item->QueMsgID = QueMsgID;
 222
 223         Token = NewStrBuf();
 224         Line = NewStrBufPlain(NULL, 128);
 225         while (pLine != StrBufNOTNULL) {
 226                 const char *pItemPart = NULL;
 227                 void *vHandler;
 228
 229                 StrBufExtract_NextToken(Line, RawQItem, &pLine, '\n');
 230                 if (StrLength(Line) == 0) continue;
 231                 StrBufExtract_NextToken(Token, Line, &pItemPart, '|');
 232                 if (GetHash(QItemHandlers, SKEY(Token), &vHandler))
 233                 {
 234                         QItemHandler H;
 235                         H = (QItemHandler) vHandler;
 236                         H(Item, Line, &pItemPart);
 237                 }
 238         }
 239         FreeStrBuf(&Line);
 240         FreeStrBuf(&Token);
 241
 242         pthread_mutex_lock(&ActiveQItemsLock);
 243         if (GetHash(ActiveQItems,
 244                     LKEY(Item->MessageID),
 245                     &v))
 246         {
 247                 /* WHOOPS. somebody else is already working on this. */
 248                 pthread_mutex_unlock(&ActiveQItemsLock);
 249                 FreeQueItem(&Item);
 250                 return NULL;
 251         }
 252         else {
 253                 /* mark our claim on this. */
 254                 Put(ActiveQItems,
 255                     LKEY(Item->MessageID),
 256                     Item,
 257                     HFreeQueItem);
 258                 pthread_mutex_unlock(&ActiveQItemsLock);
 259         }
 260
 261         return Item;
 262 }
 263
 264 StrBuf *SerializeQueueItem(OneQueItem *MyQItem)
 265 {
 266         StrBuf *QMessage;
 267         HashPos  *It;
 268         const char *Key;
 269         long len;
 270         void *vQE;
 271
 272         QMessage = NewStrBufPlain(NULL, SIZ);
 273         StrBufPrintf(QMessage, "Content-type: %s\n", SPOOLMIME);
 274
 275 //                 "attempted|%ld\n"  "retry|%ld\n",, (long)time(NULL), (long)retry );
 276         StrBufAppendBufPlain(QMessage, HKEY("\nmsgid|"), 0);
 277         StrBufAppendPrintf(QMessage, "%ld", MyQItem->MessageID);
 278
 279         StrBufAppendBufPlain(QMessage, HKEY("\nsubmitted|"), 0);
 280         StrBufAppendPrintf(QMessage, "%ld", MyQItem->Submitted);
 281
 282         if (StrLength(MyQItem->BounceTo) > 0) {
 283                 StrBufAppendBufPlain(QMessage, HKEY("\nbounceto|"), 0);
 284                 StrBufAppendBuf(QMessage, MyQItem->BounceTo, 0);
 285         }
 286
 287         if (StrLength(MyQItem->EnvelopeFrom) > 0) {
 288                 StrBufAppendBufPlain(QMessage, HKEY("\nenvelope_from|"), 0);
 289                 StrBufAppendBuf(QMessage, MyQItem->EnvelopeFrom, 0);
 290         }
 291
 292         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 293         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 294         {
 295                 MailQEntry *ThisItem = vQE;
 296                 int i;
 297
 298                 if (!ThisItem->Active)
 299                         continue; /* skip already sent ones from the spoolfile. */
 300
 301                 for (i=0; i < ThisItem->nAttempts; i++) {
 302                         /* TODO: most probably there is just one retry/attempted per message! */
 303                         StrBufAppendBufPlain(QMessage, HKEY("\nretry|"), 0);
 304                         StrBufAppendPrintf(QMessage, "%ld",
 305                                            ThisItem->Attempts[i].retry);
 306
 307                         StrBufAppendBufPlain(QMessage, HKEY("\nattempted|"), 0);
 308                         StrBufAppendPrintf(QMessage, "%ld",
 309                                            ThisItem->Attempts[i].when);
 310                 }
 311                 StrBufAppendBufPlain(QMessage, HKEY("\nremote|"), 0);
 312                 StrBufAppendBuf(QMessage, ThisItem->Recipient, 0);
 313                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 314                 StrBufAppendPrintf(QMessage, "%d", ThisItem->Status);
 315                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 316                 StrBufAppendBuf(QMessage, ThisItem->StatusMessage, 0);
 317         }
 318         DeleteHashPos(&It);
 319         StrBufAppendBufPlain(QMessage, HKEY("\n"), 0);
 320         return QMessage;
 321 }
 322
 323
 324
 325
 326
 327 void NewMailQEntry(OneQueItem *Item)
 328 {
 329         Item->Current = (MailQEntry*) malloc(sizeof(MailQEntry));
 330         memset(Item->Current, 0, sizeof(MailQEntry));
 331
 332         if (Item->MailQEntries == NULL)
 333                 Item->MailQEntries = NewHash(1, Flathash);
 334         Item->Current->StatusMessage = NewStrBuf();
 335         Item->Current->n = GetCount(Item->MailQEntries);
 336         Put(Item->MailQEntries, IKEY(Item->Current->n), Item->Current, FreeMailQEntry);
 337 }
 338
 339 void QItem_Handle_MsgID(OneQueItem *Item, StrBuf *Line, const char **Pos)
 340 {
 341         Item->MessageID = StrBufExtractNext_long(Line, Pos, '|');
 342 }
 343
 344 void QItem_Handle_EnvelopeFrom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 345 {
 346         if (Item->EnvelopeFrom == NULL)
 347                 Item->EnvelopeFrom = NewStrBufPlain(NULL, StrLength(Line));
 348         StrBufExtract_NextToken(Item->EnvelopeFrom, Line, Pos, '|');
 349 }
 350
 351 void QItem_Handle_BounceTo(OneQueItem *Item, StrBuf *Line, const char **Pos)
 352 {
 353         if (Item->BounceTo == NULL)
 354                 Item->BounceTo = NewStrBufPlain(NULL, StrLength(Line));
 355         StrBufExtract_NextToken(Item->BounceTo, Line, Pos, '|');
 356 }
 357
 358 void QItem_Handle_Recipient(OneQueItem *Item, StrBuf *Line, const char **Pos)
 359 {
 360         if (Item->Current == NULL)
 361                 NewMailQEntry(Item);
 362         if (Item->Current->Recipient == NULL)
 363                 Item->Current->Recipient =  NewStrBufPlain(NULL, StrLength(Line));
 364         StrBufExtract_NextToken(Item->Current->Recipient, Line, Pos, '|');
 365         Item->Current->Status = StrBufExtractNext_int(Line, Pos, '|');
 366         StrBufExtract_NextToken(Item->Current->StatusMessage, Line, Pos, '|');
 367         Item->Current = NULL; // TODO: is this always right?
 368 }
 369
 370
 371 void QItem_Handle_retry(OneQueItem *Item, StrBuf *Line, const char **Pos)
 372 {
 373         if (Item->Current == NULL)
 374                 NewMailQEntry(Item);
 375         if (Item->Current->Attempts[Item->Current->nAttempts].retry != 0)
 376                 Item->Current->nAttempts++;
 377         if (Item->Current->nAttempts > MaxAttempts) {
 378                 Item->FailNow = 1;
 379                 return;
 380         }
 381         Item->Current->Attempts[Item->Current->nAttempts].retry = StrBufExtractNext_int(Line, Pos, '|');
 382 }
 383
 384
 385 void QItem_Handle_Submitted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 386 {
 387         Item->Submitted = atol(*Pos);
 388
 389 }
 390
 391 void QItem_Handle_Attempted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 392 {
 393         if (Item->Current == NULL)
 394                 NewMailQEntry(Item);
 395         if (Item->Current->Attempts[Item->Current->nAttempts].when != 0)
 396                 Item->Current->nAttempts++;
 397         if (Item->Current->nAttempts > MaxAttempts) {
 398                 Item->FailNow = 1;
 399                 return;
 400         }
 401
 402         Item->Current->Attempts[Item->Current->nAttempts].when = StrBufExtractNext_int(Line, Pos, '|');
 403         if (Item->Current->Attempts[Item->Current->nAttempts].when > Item->LastAttempt.when)
 404         {
 405                 Item->LastAttempt.when = Item->Current->Attempts[Item->Current->nAttempts].when;
 406                 Item->LastAttempt.retry = Item->Current->Attempts[Item->Current->nAttempts].retry * 2;
 407                 if (Item->LastAttempt.retry > SMTP_RETRY_MAX)
 408                         Item->LastAttempt.retry = SMTP_RETRY_MAX;
 409         }
 410 }
 411
 412
 413
 414 /**
 415  * this one has to have the context for loading the message via the redirect buffer...
 416  */
 417 StrBuf *smtp_load_msg(OneQueItem *MyQItem, int n)
 418 {
 419         CitContext *CCC=CC;
 420         StrBuf *SendMsg;
 421
 422         CCC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
 423         CtdlOutputMsg(MyQItem->MessageID, MT_RFC822, HEADERS_ALL, 0, 1, NULL, (ESC_DOT|SUPPRESS_ENV_TO) );
 424         SendMsg = CCC->redirect_buffer;
 425         CCC->redirect_buffer = NULL;
 426         if ((StrLength(SendMsg) > 0) &&
 427             ChrPtr(SendMsg)[StrLength(SendMsg) - 1] != '\n') {
 428                 syslog(LOG_WARNING,
 429                        "SMTP client[%d]: Possible problem: message did not "
 430                        "correctly terminate. (expecting 0x10, got 0x%02x)\n",
 431                        MsgCount, //yes uncool, but best choice here...
 432                        ChrPtr(SendMsg)[StrLength(SendMsg) - 1] );
 433                 StrBufAppendBufPlain(SendMsg, HKEY("\r\n"), 0);
 434         }
 435         return SendMsg;
 436 }
 437
 438
 439
 440 /*
 441  * smtp_do_bounce() is caled by smtp_do_procmsg() to scan a set of delivery
 442  * instructions for "5" codes (permanent fatal errors) and produce/deliver
 443  * a "bounce" message (delivery status notification).
 444  */
 445 void smtpq_do_bounce(OneQueItem *MyQItem, StrBuf *OMsgTxt)
 446 {
 447         static int seq = 0;
 448
 449         struct CtdlMessage *bmsg = NULL;
 450         StrBuf *boundary;
 451         StrBuf *Msg = NULL;
 452         StrBuf *BounceMB;
 453         struct recptypes *valid;
 454
 455         HashPos *It;
 456         void *vQE;
 457         long len;
 458         const char *Key;
 459
 460         int successful_bounce = 0;
 461         int num_bounces = 0;
 462         int give_up = 0;
 463
 464         syslog(LOG_DEBUG, "smtp_do_bounce() called\n");
 465
 466         if ( (ev_time() - MyQItem->Submitted) > SMTP_GIVE_UP ) {
 467                 give_up = 1;/// TODO: replace time by libevq timer get
 468         }
 469
 470         /*
 471          * Now go through the instructions checking for stuff.
 472          */
 473         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 474         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 475         {
 476                 MailQEntry *ThisItem = vQE;
 477                 if ((ThisItem->Status == 5) || /* failed now? */
 478                     ((give_up == 1) && (ThisItem->Status != 2))) /* giving up after failed attempts... */
 479                 {
 480                         if (num_bounces == 0)
 481                                 Msg = NewStrBufPlain(NULL, 1024);
 482                         ++num_bounces;
 483
 484                         StrBufAppendBuf(Msg, ThisItem->Recipient, 0);
 485                         StrBufAppendBufPlain(Msg, HKEY(": "), 0);
 486                         StrBufAppendBuf(Msg, ThisItem->StatusMessage, 0);
 487                         StrBufAppendBufPlain(Msg, HKEY("\r\n"), 0);
 488                 }
 489         }
 490         DeleteHashPos(&It);
 491
 492         /* Deliver the bounce if there's anything worth mentioning */
 493         syslog(LOG_DEBUG, "num_bounces = %d\n", num_bounces);
 494
 495         if (num_bounces == 0) {
 496                 FreeStrBuf(&Msg);
 497                 return;
 498         }
 499
 500         boundary = NewStrBufPlain(HKEY("=_Citadel_Multipart_"));
 501         StrBufAppendPrintf(boundary, "%s_%04x%04x", config.c_fqdn, getpid(), ++seq);
 502
 503         /* Start building our bounce message; go shopping for memory first. */
 504         BounceMB = NewStrBufPlain(NULL,
 505                                   1024 + /* mime stuff.... */
 506                                   StrLength(Msg) +  /* the bounce information... */
 507                                   StrLength(OMsgTxt)); /* the original message */
 508         if (BounceMB == NULL) {
 509                 FreeStrBuf(&boundary);
 510                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 511
 512                 return;
 513         }
 514
 515         bmsg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
 516         if (bmsg == NULL) {
 517                 FreeStrBuf(&boundary);
 518                 FreeStrBuf(&BounceMB);
 519                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 520
 521                 return;
 522         }
 523         memset(bmsg, 0, sizeof(struct CtdlMessage));
 524
 525
 526         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: multipart/mixed; boundary=\""), 0);
 527         StrBufAppendBuf(BounceMB, boundary, 0);
 528         StrBufAppendBufPlain(BounceMB, HKEY("\"\r\n"), 0);
 529         StrBufAppendBufPlain(BounceMB, HKEY("MIME-Version: 1.0\r\n"), 0);
 530         StrBufAppendBufPlain(BounceMB, HKEY("X-Mailer: " CITADEL "\r\n"), 0);
 531         StrBufAppendBufPlain(BounceMB, HKEY("\r\nThis is a multipart message in MIME format.\r\n\r\n"), 0);
 532         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 533         StrBufAppendBuf(BounceMB, boundary, 0);
 534         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 535         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: text/plain\r\n\r\n"), 0);
 536
 537         if (give_up)
 538                 StrBufAppendBufPlain(
 539                         BounceMB,
 540                         HKEY(
 541                                 "A message you sent could not be delivered to some or all of its recipients\n"
 542                                 "due to prolonged unavailability of its destination(s).\n"
 543                                 "Giving up on the following addresses:\n\n"
 544                                 ), 0);
 545         else
 546                 StrBufAppendBufPlain(
 547                         BounceMB,
 548                         HKEY(
 549                                 "A message you sent could not be delivered to some or all of its recipients.\n"
 550                                 "The following addresses were undeliverable:\n\n"
 551                                 ), 0);
 552
 553         StrBufAppendBuf(BounceMB, Msg, 0);
 554         FreeStrBuf(&Msg);
 555
 556         /* Attach the original message */
 557         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 558         StrBufAppendBuf(BounceMB, boundary, 0);
 559         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 560         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: message/rfc822\r\n"), 0);
 561         StrBufAppendBufPlain(BounceMB, HKEY("Content-Transfer-Encoding: 7bit\r\n"), 0);
 562         StrBufAppendBufPlain(BounceMB, HKEY("Content-Disposition: inline\r\n"), 0);
 563         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 564         StrBufAppendBuf(BounceMB, OMsgTxt, 0);
 565
 566         /* Close the multipart MIME scope */
 567         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 568         StrBufAppendBuf(BounceMB, boundary, 0);
 569         StrBufAppendBufPlain(BounceMB, HKEY("--\r\n"), 0);
 570
 571
 572
 573         bmsg->cm_magic = CTDLMESSAGE_MAGIC;
 574         bmsg->cm_anon_type = MES_NORMAL;
 575         bmsg->cm_format_type = FMT_RFC822;
 576
 577         bmsg->cm_fields['O'] = strdup(MAILROOM);
 578         bmsg->cm_fields['A'] = strdup("Citadel");
 579         bmsg->cm_fields['N'] = strdup(config.c_nodename);
 580         bmsg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
 581         bmsg->cm_fields['M'] = SmashStrBuf(&BounceMB);
 582
 583         /* First try the user who sent the message */
 584         if (StrLength(MyQItem->BounceTo) == 0)
 585                 syslog(LOG_ERR, "No bounce address specified\n");
 586         else
 587                 syslog(LOG_DEBUG, "bounce to user? <%s>\n", ChrPtr(MyQItem->BounceTo));
 588
 589         /* Can we deliver the bounce to the original sender? */
 590         valid = validate_recipients(ChrPtr(MyQItem->BounceTo), NULL, 0);
 591         if ((valid != NULL) && (valid->num_error == 0)) {
 592                 CtdlSubmitMsg(bmsg, valid, "", QP_EADDR);
 593                 successful_bounce = 1;
 594         }
 595
 596         /* If not, post it in the Aide> room */
 597         if (successful_bounce == 0) {
 598                 CtdlSubmitMsg(bmsg, NULL, config.c_aideroom, QP_EADDR);
 599         }
 600
 601         /* Free up the memory we used */
 602         free_recipients(valid);
 603         FreeStrBuf(&boundary);
 604         CtdlFreeMessage(bmsg);
 605         syslog(LOG_DEBUG, "Done processing bounces\n");
 606 }
 607
 608
 609
 610 /*
 611 {
 612
 613         if (threadding)
 614                 n_smarthosts =  get_hosts(char *mxbuf, char *rectype);
 615 }
 616 */
 617 /*
 618  * smtp_do_procmsg()
 619  *
 620  * Called by smtp_do_queue() to handle an individual message.
 621  */
 622 void smtp_do_procmsg(long msgnum, void *userdata) {
 623         struct CtdlMessage *msg = NULL;
 624         char *instr = NULL;
 625         StrBuf *PlainQItem;
 626         OneQueItem *MyQItem;
 627         char *pch;
 628         HashPos  *It;
 629         void *vQE;
 630         long len;
 631         const char *Key;
 632         int nRelays = 0;
 633         ParsedURL *RelayUrls = NULL;
 634         int HaveBuffers = 0;
 635         StrBuf *Msg =NULL;
 636
 637         syslog(LOG_DEBUG, "SMTP Queue: smtp_do_procmsg(%ld)\n", msgnum);
 638         ///strcpy(envelope_from, "");
 639
 640         msg = CtdlFetchMessage(msgnum, 1);
 641         if (msg == NULL) {
 642                 syslog(LOG_ERR, "SMTP Queue: tried %ld but no such message!\n", msgnum);
 643                 return;
 644         }
 645
 646         pch = instr = msg->cm_fields['M'];
 647
 648         /* Strip out the headers (no not amd any other non-instruction) line */
 649         while (pch != NULL) {
 650                 pch = strchr(pch, '\n');
 651                 if ((pch != NULL) && (*(pch + 1) == '\n')) {
 652                         instr = pch + 2;
 653                         pch = NULL;
 654                 }
 655         }
 656         PlainQItem = NewStrBufPlain(instr, -1);
 657         CtdlFreeMessage(msg);
 658         MyQItem = DeserializeQueueItem(PlainQItem, msgnum);
 659         FreeStrBuf(&PlainQItem);
 660
 661         if (MyQItem == NULL) {
 662                 syslog(LOG_ERR, "SMTP Queue: Msg No %ld: already in progress!\n", msgnum);
 663                 return; /* s.b. else is already processing... */
 664         }
 665
 666         /*
 667          * Postpone delivery if we've already tried recently.
 668          * /
 669         if (((time(NULL) - MyQItem->LastAttempt.when) < MyQItem->LastAttempt.retry) && (run_queue_now == 0)) {
 670                 syslog(LOG_DEBUG, "SMTP client: Retry time not yet reached.\n");
 671
 672                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 673                 pthread_mutex_lock(&ActiveQItemsLock);
 674                 {
 675                         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 676                                 DeleteEntryFromHash(ActiveQItems, It);
 677                 }
 678                 pthread_mutex_unlock(&ActiveQItemsLock);
 679                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 680                 DeleteHashPos(&It);
 681                 return;
 682         }// TODO: reenable me.*/
 683
 684         /*
 685          * Bail out if there's no actual message associated with this
 686          */
 687         if (MyQItem->MessageID < 0L) {
 688                 syslog(LOG_ERR, "SMTP Queue: no 'msgid' directive found!\n");
 689                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 690                 pthread_mutex_lock(&ActiveQItemsLock);
 691                 {
 692                         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 693                                 DeleteEntryFromHash(ActiveQItems, It);
 694                 }
 695                 pthread_mutex_unlock(&ActiveQItemsLock);
 696                 DeleteHashPos(&It);
 697                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 698                 return;
 699         }
 700
 701         {
 702                 char mxbuf[SIZ];
 703                 ParsedURL **Url = &MyQItem->URL;
 704                 nRelays = get_hosts(mxbuf, "smarthost");
 705                 if (nRelays > 0) {
 706                         StrBuf *All;
 707                         StrBuf *One;
 708                         const char *Pos = NULL;
 709                         All = NewStrBufPlain(mxbuf, -1);
 710                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 711
 712                         while ((Pos != StrBufNOTNULL) && ((Pos == NULL) || !IsEmptyStr(Pos))) {
 713                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 714                                 if (!ParseURL(Url, One, 25))
 715                                         syslog(LOG_DEBUG, "Failed to parse: %s\n", ChrPtr(One));
 716                                 else {
 717                                         ///if (!Url->IsIP)) /// todo dupe me fork ipv6
 718                                         Url = &(*Url)->Next;
 719
 720                                 }
 721                         }
 722                         FreeStrBuf(&All);
 723                         FreeStrBuf(&One);
 724                 }
 725
 726                 Url = &MyQItem->FallBackHost;
 727                 nRelays = get_hosts(mxbuf, "fallbackhost");
 728                 if (nRelays > 0) {
 729                         StrBuf *All;
 730                         StrBuf *One;
 731                         const char *Pos = NULL;
 732                         All = NewStrBufPlain(mxbuf, -1);
 733                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 734
 735                         while ((Pos != StrBufNOTNULL) && ((Pos == NULL) || !IsEmptyStr(Pos))) {
 736                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 737                                 if (!ParseURL(Url, One, 25))
 738                                         syslog(LOG_DEBUG, "Failed to parse: %s\n", ChrPtr(One));
 739                                 else
 740                                         Url = &(*Url)->Next;
 741                         }
 742                         FreeStrBuf(&All);
 743                         FreeStrBuf(&One);
 744                 }
 745         }
 746
 747         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 748         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 749         {
 750                 MailQEntry *ThisItem = vQE;
 751                 syslog(LOG_DEBUG, "SMTP Queue: Task: <%s> %d\n",
 752                        ChrPtr(ThisItem->Recipient),
 753                        ThisItem->Active);
 754         }
 755         DeleteHashPos(&It);
 756
 757         MyQItem->ActiveDeliveries = CountActiveQueueEntries(MyQItem);
 758         if (MyQItem->ActiveDeliveries > 0)
 759         {
 760                 int n = MsgCount++;
 761                 int m = MyQItem->ActiveDeliveries;
 762                 int i = 1;
 763                 Msg = smtp_load_msg(MyQItem, n);
 764                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 765                 while ((i <= m) &&
 766                        (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE)))
 767                 {
 768                         MailQEntry *ThisItem = vQE;
 769                         if (ThisItem->Active == 1) {
 770                                 int KeepBuffers = (i == m);
 771                                 if (i > 1) n = MsgCount++;
 772                                 syslog(LOG_DEBUG,
 773                                        "SMTP Queue: Trying <%ld> <%s> %d / %d \n",
 774                                        MyQItem->MessageID,
 775                                        ChrPtr(ThisItem->Recipient),
 776                                        i,
 777                                        m);
 778                                 smtp_try_one_queue_entry(MyQItem,
 779                                                          ThisItem,
 780                                                          Msg,
 781                                                          KeepBuffers,
 782                                                          n,
 783                                                          RelayUrls);
 784                                 if (KeepBuffers) HaveBuffers = 1;
 785                                 i++;
 786                         }
 787                 }
 788                 DeleteHashPos(&It);
 789         }
 790         else
 791         {
 792                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 793                 pthread_mutex_lock(&ActiveQItemsLock);
 794                 {
 795                         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
 796                                 DeleteEntryFromHash(ActiveQItems, It);
 797                         else
 798                         {
 799                                 long len;
 800                                 const char* Key;
 801                                 void *VData;
 802                                 syslog(LOG_WARNING,
 803                                        "SMTP cleanup: unable to find QItem with ID[%ld]",
 804                                        MyQItem->MessageID);
 805                                 while (GetNextHashPos(ActiveQItems, It, &len, &Key, &VData))
 806                                         syslog(LOG_WARNING,
 807                                                "SMTP cleanup: have: ID[%ld]",
 808                                                ((OneQueItem *)VData)->MessageID);
 809                         }
 810
 811                 }
 812                 pthread_mutex_unlock(&ActiveQItemsLock);
 813                 DeleteHashPos(&It);
 814                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 815
 816 // TODO: bounce & delete?
 817
 818         }
 819         if (!HaveBuffers) {
 820                 FreeStrBuf (&Msg);
 821 // TODO : free RelayUrls
 822         }
 823 }
 824
 825
 826
 827 /*
 828  * smtp_queue_thread()
 829  *
 830  * Run through the queue sending out messages.
 831  */
 832 void smtp_do_queue(void) {
 833         static int is_running = 0;
 834         int num_processed = 0;
 835
 836         if (is_running) return;         /* Concurrency check - only one can run */
 837         is_running = 1;
 838
 839         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 840         syslog(LOG_INFO, "SMTP client: processing outbound queue");
 841
 842         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
 843                 syslog(LOG_ERR, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
 844         }
 845         else {
 846                 num_processed = CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_do_procmsg, NULL);
 847         }
 848         syslog(LOG_INFO, "SMTP client: queue run completed; %d messages processed", num_processed);
 849         run_queue_now = 0;
 850         is_running = 0;
 851 }
 852
 853
 854
 855 /*
 856  * Initialize the SMTP outbound queue
 857  */
 858 void smtp_init_spoolout(void) {
 859         struct ctdlroom qrbuf;
 860
 861         /*
 862          * Create the room.  This will silently fail if the room already
 863          * exists, and that's perfectly ok, because we want it to exist.
 864          */
 865         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_MAILBOX);
 866
 867         /*
 868          * Make sure it's set to be a "system room" so it doesn't show up
 869          * in the <K>nown rooms list for Aides.
 870          */
 871         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
 872                 qrbuf.QRflags2 |= QR2_SYSTEM;
 873                 CtdlPutRoomLock(&qrbuf);
 874         }
 875 }
 876
 877
 878
 879
 880 /*****************************************************************************/
 881 /*                          SMTP UTILITY COMMANDS                            */
 882 /*****************************************************************************/
 883
 884 void cmd_smtp(char *argbuf) {
 885         char cmd[64];
 886         char node[256];
 887         char buf[1024];
 888         int i;
 889         int num_mxhosts;
 890
 891         if (CtdlAccessCheck(ac_aide)) return;
 892
 893         extract_token(cmd, argbuf, 0, '|', sizeof cmd);
 894
 895         if (!strcasecmp(cmd, "mx")) {
 896                 extract_token(node, argbuf, 1, '|', sizeof node);
 897                 num_mxhosts = getmx(buf, node);
 898                 cprintf("%d %d MX hosts listed for %s\n",
 899                         LISTING_FOLLOWS, num_mxhosts, node);
 900                 for (i=0; i<num_mxhosts; ++i) {
 901                         extract_token(node, buf, i, '|', sizeof node);
 902                         cprintf("%s\n", node);
 903                 }
 904                 cprintf("000\n");
 905                 return;
 906         }
 907
 908         else if (!strcasecmp(cmd, "runqueue")) {
 909                 run_queue_now = 1;
 910                 cprintf("%d All outbound SMTP will be retried now.\n", CIT_OK);
 911                 return;
 912         }
 913
 914         else {
 915                 cprintf("%d Invalid command.\n", ERROR + ILLEGAL_VALUE);
 916         }
 917
 918 }
 919
 920
 921 CTDL_MODULE_INIT(smtp_queu)
 922 {
 923         if (!threading)
 924         {
 925                 CtdlFillSystemContext(&smtp_queue_CC, "SMTP_Send");
 926                 ActiveQItems = NewHash(1, lFlathash);
 927                 pthread_mutex_init(&ActiveQItemsLock, NULL);
 928
 929                 QItemHandlers = NewHash(0, NULL);
 930
 931                 Put(QItemHandlers, HKEY("msgid"), QItem_Handle_MsgID, reference_free_handler);
 932                 Put(QItemHandlers, HKEY("envelope_from"), QItem_Handle_EnvelopeFrom, reference_free_handler);
 933                 Put(QItemHandlers, HKEY("retry"), QItem_Handle_retry, reference_free_handler);
 934                 Put(QItemHandlers, HKEY("attempted"), QItem_Handle_Attempted, reference_free_handler);
 935                 Put(QItemHandlers, HKEY("remote"), QItem_Handle_Recipient, reference_free_handler);
 936                 Put(QItemHandlers, HKEY("bounceto"), QItem_Handle_BounceTo, reference_free_handler);
 937                 Put(QItemHandlers, HKEY("submitted"), QItem_Handle_Submitted, reference_free_handler);
 938 ////TODO: flush qitemhandlers on exit
 939                 smtp_init_spoolout();
 940
 941                 CtdlRegisterCleanupHook(smtp_evq_cleanup);
 942
 943                 CtdlRegisterProtoHook(cmd_smtp, "SMTP", "SMTP utility commands");
 944                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER);
 945         }
 946
 947         /* return our Subversion id for the Log */
 948         return "smtpeventclient";
 949 }