citadel/modules/smtp/serv_smtpqueue.c

   1 /*
   2  * This module is an SMTP and ESMTP implementation for the Citadel system.
   3  * It is compliant with all of the following:
   4  *
   5  * RFC  821 - Simple Mail Transfer Protocol
   6  * RFC  876 - Survey of SMTP Implementations
   7  * RFC 1047 - Duplicate messages and SMTP
   8  * RFC 1652 - 8 bit MIME
   9  * RFC 1869 - Extended Simple Mail Transfer Protocol
  10  * RFC 1870 - SMTP Service Extension for Message Size Declaration
  11  * RFC 2033 - Local Mail Transfer Protocol
  12  * RFC 2197 - SMTP Service Extension for Command Pipelining
  13  * RFC 2476 - Message Submission
  14  * RFC 2487 - SMTP Service Extension for Secure SMTP over TLS
  15  * RFC 2554 - SMTP Service Extension for Authentication
  16  * RFC 2821 - Simple Mail Transfer Protocol
  17  * RFC 2822 - Internet Message Format
  18  * RFC 2920 - SMTP Service Extension for Command Pipelining
  19  *
  20  * The VRFY and EXPN commands have been removed from this implementation
  21  * because nobody uses these commands anymore, except for spammers.
  22  *
  23  * Copyright (c) 1998-2009 by the citadel.org team
  24  *
  25  *  This program is free software; you can redistribute it and/or modify
  26  *  it under the terms of the GNU General Public License as published by
  27  *  the Free Software Foundation; either version 3 of the License, or
  28  *  (at your option) any later version.
  29  *
  30  *  This program is distributed in the hope that it will be useful,
  31  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  32  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  33  *  GNU General Public License for more details.
  34  *
  35  *  You should have received a copy of the GNU General Public License
  36  *  along with this program; if not, write to the Free Software
  37  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  38  */
  39
  40 #include "sysdep.h"
  41 #include <stdlib.h>
  42 #include <unistd.h>
  43 #include <stdio.h>
  44 #include <termios.h>
  45 #include <fcntl.h>
  46 #include <signal.h>
  47 #include <pwd.h>
  48 #include <errno.h>
  49 #include <sys/types.h>
  50 #include <syslog.h>
  51
  52 #if TIME_WITH_SYS_TIME
  53 # include <sys/time.h>
  54 # include <time.h>
  55 #else
  56 # if HAVE_SYS_TIME_H
  57 #  include <sys/time.h>
  58 # else
  59 #  include <time.h>
  60 # endif
  61 #endif
  62 #include <sys/wait.h>
  63 #include <ctype.h>
  64 #include <string.h>
  65 #include <limits.h>
  66 #include <sys/socket.h>
  67 #include <netinet/in.h>
  68 #include <arpa/inet.h>
  69 #include <libcitadel.h>
  70 #include "citadel.h"
  71 #include "server.h"
  72 #include "citserver.h"
  73 #include "support.h"
  74 #include "config.h"
  75 #include "control.h"
  76 #include "user_ops.h"
  77 #include "database.h"
  78 #include "msgbase.h"
  79 #include "internet_addressing.h"
  80 #include "genstamp.h"
  81 #include "domain.h"
  82 #include "clientsocket.h"
  83 #include "locate_host.h"
  84 #include "citadel_dirs.h"
  85
  86 #include "ctdl_module.h"
  87
  88 #include "smtpqueue.h"
  89 #include "event_client.h"
  90
  91
  92 struct CitContext smtp_queue_CC;
  93 pthread_mutex_t ActiveQItemsLock;
  94 HashList *ActiveQItems  = NULL;
  95 HashList *QItemHandlers = NULL;
  96
  97 int MsgCount            = 0;
  98 int run_queue_now       = 0;    /* Set to 1 to ignore SMTP send retry times */
  99
 100 void smtp_try_one_queue_entry(OneQueItem *MyQItem,
 101                               MailQEntry *MyQEntry,
 102                               StrBuf *MsgText,
 103                               int KeepMsgText, /* KeepMsgText allows us to use MsgText as ours. */
 104                               int MsgCount,
 105                               ParsedURL *RelayUrls);
 106
 107
 108 void smtp_evq_cleanup(void)
 109 {
 110
 111         pthread_mutex_lock(&ActiveQItemsLock);
 112         DeleteHash(&QItemHandlers);
 113         DeleteHash(&ActiveQItems);
 114         pthread_mutex_unlock(&ActiveQItemsLock);
 115         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 116 /*      citthread_mutex_destroy(&ActiveQItemsLock); TODO */
 117 }
 118
 119 int DecreaseQReference(OneQueItem *MyQItem)
 120 {
 121         int IDestructQueItem;
 122
 123         pthread_mutex_lock(&ActiveQItemsLock);
 124         MyQItem->ActiveDeliveries--;
 125         IDestructQueItem = MyQItem->ActiveDeliveries == 0;
 126         pthread_mutex_unlock(&ActiveQItemsLock);
 127         return IDestructQueItem;
 128 }
 129
 130 void RemoveQItem(OneQueItem *MyQItem)
 131 {
 132         HashPos  *It;
 133
 134         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 135         pthread_mutex_lock(&ActiveQItemsLock);
 136         {
 137                 GetHashPosFromKey(ActiveQItems, IKEY(MyQItem->MessageID), It);
 138                 DeleteEntryFromHash(ActiveQItems, It);
 139         }
 140         pthread_mutex_unlock(&ActiveQItemsLock);
 141         DeleteHashPos(&It);
 142 }
 143
 144
 145 void FreeMailQEntry(void *qv)
 146 {
 147         MailQEntry *Q = qv;
 148         FreeStrBuf(&Q->Recipient);
 149         FreeStrBuf(&Q->StatusMessage);
 150         free(Q);
 151 }
 152 void FreeQueItem(OneQueItem **Item)
 153 {
 154         DeleteHash(&(*Item)->MailQEntries);
 155         FreeStrBuf(&(*Item)->EnvelopeFrom);
 156         FreeStrBuf(&(*Item)->BounceTo);
 157         FreeURL(&(*Item)->URL);
 158         free(*Item);
 159         Item = NULL;
 160 }
 161 void HFreeQueItem(void *Item)
 162 {
 163         FreeQueItem((OneQueItem**)&Item);
 164 }
 165
 166 /* inspect recipients with a status of:
 167  * - 0 (no delivery yet attempted)
 168  * - 3/4 (transient errors
 169  *        were experienced and it's time to try again)
 170  */
 171 int CountActiveQueueEntries(OneQueItem *MyQItem)
 172 {
 173         HashPos  *It;
 174         long len;
 175         const char *Key;
 176         void *vQE;
 177
 178         MyQItem->ActiveDeliveries = 0;
 179         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 180         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 181         {
 182                 MailQEntry *ThisItem = vQE;
 183                 if ((ThisItem->Status == 0) ||
 184                     (ThisItem->Status == 3) ||
 185                     (ThisItem->Status == 4))
 186                 {
 187                         MyQItem->ActiveDeliveries++;
 188                         ThisItem->Active = 1;
 189                 }
 190                 else
 191                         ThisItem->Active = 0;
 192         }
 193         DeleteHashPos(&It);
 194         return MyQItem->ActiveDeliveries;
 195 }
 196
 197 OneQueItem *DeserializeQueueItem(StrBuf *RawQItem, long QueMsgID)
 198 {
 199         OneQueItem *Item;
 200         const char *pLine = NULL;
 201         StrBuf *Line;
 202         StrBuf *Token;
 203         void *v;
 204
 205         Item = (OneQueItem*)malloc(sizeof(OneQueItem));
 206         memset(Item, 0, sizeof(OneQueItem));
 207         Item->LastAttempt.retry = SMTP_RETRY_INTERVAL;
 208         Item->MessageID = -1;
 209         Item->QueMsgID = QueMsgID;
 210
 211         pthread_mutex_lock(&ActiveQItemsLock);
 212         if (GetHash(ActiveQItems,
 213                     IKEY(QueMsgID),
 214                     &v))
 215         {
 216                 /* WHOOPS. somebody else is already working on this. */
 217                 pthread_mutex_unlock(&ActiveQItemsLock);
 218                 FreeQueItem(&Item);
 219                 return NULL;
 220         }
 221         else {
 222                 /* mark our claim on this. */
 223                 Put(ActiveQItems,
 224                     IKEY(Item->QueMsgID),
 225                     Item,
 226                     HFreeQueItem);
 227                 pthread_mutex_unlock(&ActiveQItemsLock);
 228         }
 229
 230         Token = NewStrBuf();
 231         Line = NewStrBufPlain(NULL, 128);
 232         while (pLine != StrBufNOTNULL) {
 233                 const char *pItemPart = NULL;
 234                 void *vHandler;
 235
 236                 StrBufExtract_NextToken(Line, RawQItem, &pLine, '\n');
 237                 if (StrLength(Line) == 0) continue;
 238                 StrBufExtract_NextToken(Token, Line, &pItemPart, '|');
 239                 if (GetHash(QItemHandlers, SKEY(Token), &vHandler))
 240                 {
 241                         QItemHandler H;
 242                         H = (QItemHandler) vHandler;
 243                         H(Item, Line, &pItemPart);
 244                 }
 245         }
 246         FreeStrBuf(&Line);
 247         FreeStrBuf(&Token);
 248         return Item;
 249 }
 250
 251 StrBuf *SerializeQueueItem(OneQueItem *MyQItem)
 252 {
 253         StrBuf *QMessage;
 254         HashPos  *It;
 255         const char *Key;
 256         long len;
 257         void *vQE;
 258
 259         QMessage = NewStrBufPlain(NULL, SIZ);
 260         StrBufPrintf(QMessage, "Content-type: %s\n", SPOOLMIME);
 261
 262 //                 "attempted|%ld\n"  "retry|%ld\n",, (long)time(NULL), (long)retry );
 263         StrBufAppendBufPlain(QMessage, HKEY("\nmsgid|"), 0);
 264         StrBufAppendPrintf(QMessage, "%ld", MyQItem->MessageID);
 265
 266         StrBufAppendBufPlain(QMessage, HKEY("\nsubmitted|"), 0);
 267         StrBufAppendPrintf(QMessage, "%ld", MyQItem->Submitted);
 268
 269         if (StrLength(MyQItem->BounceTo) > 0) {
 270                 StrBufAppendBufPlain(QMessage, HKEY("\nbounceto|"), 0);
 271                 StrBufAppendBuf(QMessage, MyQItem->BounceTo, 0);
 272         }
 273
 274         if (StrLength(MyQItem->EnvelopeFrom) > 0) {
 275                 StrBufAppendBufPlain(QMessage, HKEY("\nenvelope_from|"), 0);
 276                 StrBufAppendBuf(QMessage, MyQItem->EnvelopeFrom, 0);
 277         }
 278
 279         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 280         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 281         {
 282                 MailQEntry *ThisItem = vQE;
 283                 int i;
 284
 285                 if (!ThisItem->Active)
 286                         continue; /* skip already sent ones from the spoolfile. */
 287
 288                 for (i=0; i < ThisItem->nAttempts; i++) {
 289                         /* TODO: most probably there is just one retry/attempted per message! */
 290                         StrBufAppendBufPlain(QMessage, HKEY("\nretry|"), 0);
 291                         StrBufAppendPrintf(QMessage, "%ld",
 292                                            ThisItem->Attempts[i].retry);
 293
 294                         StrBufAppendBufPlain(QMessage, HKEY("\nattempted|"), 0);
 295                         StrBufAppendPrintf(QMessage, "%ld",
 296                                            ThisItem->Attempts[i].when);
 297                 }
 298                 StrBufAppendBufPlain(QMessage, HKEY("\nremote|"), 0);
 299                 StrBufAppendBuf(QMessage, ThisItem->Recipient, 0);
 300                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 301                 StrBufAppendPrintf(QMessage, "%d", ThisItem->Status);
 302                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
 303                 StrBufAppendBuf(QMessage, ThisItem->StatusMessage, 0);
 304         }
 305         DeleteHashPos(&It);
 306         StrBufAppendBufPlain(QMessage, HKEY("\n"), 0);
 307         return QMessage;
 308 }
 309
 310
 311
 312
 313
 314 void NewMailQEntry(OneQueItem *Item)
 315 {
 316         Item->Current = (MailQEntry*) malloc(sizeof(MailQEntry));
 317         memset(Item->Current, 0, sizeof(MailQEntry));
 318
 319         if (Item->MailQEntries == NULL)
 320                 Item->MailQEntries = NewHash(1, Flathash);
 321         Item->Current->StatusMessage = NewStrBuf();
 322         Item->Current->n = GetCount(Item->MailQEntries);
 323         Put(Item->MailQEntries, IKEY(Item->Current->n), Item->Current, FreeMailQEntry);
 324 }
 325
 326 void QItem_Handle_MsgID(OneQueItem *Item, StrBuf *Line, const char **Pos)
 327 {
 328         Item->MessageID = StrBufExtractNext_int(Line, Pos, '|');
 329 }
 330
 331 void QItem_Handle_EnvelopeFrom(OneQueItem *Item, StrBuf *Line, const char **Pos)
 332 {
 333         if (Item->EnvelopeFrom == NULL)
 334                 Item->EnvelopeFrom = NewStrBufPlain(NULL, StrLength(Line));
 335         StrBufExtract_NextToken(Item->EnvelopeFrom, Line, Pos, '|');
 336 }
 337
 338 void QItem_Handle_BounceTo(OneQueItem *Item, StrBuf *Line, const char **Pos)
 339 {
 340         if (Item->BounceTo == NULL)
 341                 Item->BounceTo = NewStrBufPlain(NULL, StrLength(Line));
 342         StrBufExtract_NextToken(Item->BounceTo, Line, Pos, '|');
 343 }
 344
 345 void QItem_Handle_Recipient(OneQueItem *Item, StrBuf *Line, const char **Pos)
 346 {
 347         if (Item->Current == NULL)
 348                 NewMailQEntry(Item);
 349         if (Item->Current->Recipient == NULL)
 350                 Item->Current->Recipient =  NewStrBufPlain(NULL, StrLength(Line));
 351         StrBufExtract_NextToken(Item->Current->Recipient, Line, Pos, '|');
 352         Item->Current->Status = StrBufExtractNext_int(Line, Pos, '|');
 353         StrBufExtract_NextToken(Item->Current->StatusMessage, Line, Pos, '|');
 354         Item->Current = NULL; // TODO: is this always right?
 355 }
 356
 357
 358 void QItem_Handle_retry(OneQueItem *Item, StrBuf *Line, const char **Pos)
 359 {
 360         if (Item->Current == NULL)
 361                 NewMailQEntry(Item);
 362         if (Item->Current->Attempts[Item->Current->nAttempts].retry != 0)
 363                 Item->Current->nAttempts++;
 364         if (Item->Current->nAttempts > MaxAttempts) {
 365                 Item->FailNow = 1;
 366                 return;
 367         }
 368         Item->Current->Attempts[Item->Current->nAttempts].retry = StrBufExtractNext_int(Line, Pos, '|');
 369 }
 370
 371
 372 void QItem_Handle_Submitted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 373 {
 374         Item->Submitted = atol(*Pos);
 375
 376 }
 377
 378 void QItem_Handle_Attempted(OneQueItem *Item, StrBuf *Line, const char **Pos)
 379 {
 380         if (Item->Current == NULL)
 381                 NewMailQEntry(Item);
 382         if (Item->Current->Attempts[Item->Current->nAttempts].when != 0)
 383                 Item->Current->nAttempts++;
 384         if (Item->Current->nAttempts > MaxAttempts) {
 385                 Item->FailNow = 1;
 386                 return;
 387         }
 388
 389         Item->Current->Attempts[Item->Current->nAttempts].when = StrBufExtractNext_int(Line, Pos, '|');
 390         if (Item->Current->Attempts[Item->Current->nAttempts].when > Item->LastAttempt.when)
 391         {
 392                 Item->LastAttempt.when = Item->Current->Attempts[Item->Current->nAttempts].when;
 393                 Item->LastAttempt.retry = Item->Current->Attempts[Item->Current->nAttempts].retry * 2;
 394                 if (Item->LastAttempt.retry > SMTP_RETRY_MAX)
 395                         Item->LastAttempt.retry = SMTP_RETRY_MAX;
 396         }
 397 }
 398
 399
 400
 401 /**
 402  * this one has to have the context for loading the message via the redirect buffer...
 403  */
 404 StrBuf *smtp_load_msg(OneQueItem *MyQItem, int n)
 405 {
 406         CitContext *CCC=CC;
 407         StrBuf *SendMsg;
 408
 409         CCC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
 410         CtdlOutputMsg(MyQItem->MessageID, MT_RFC822, HEADERS_ALL, 0, 1, NULL, (ESC_DOT|SUPPRESS_ENV_TO) );
 411         SendMsg = CCC->redirect_buffer;
 412         CCC->redirect_buffer = NULL;
 413         if ((StrLength(SendMsg) > 0) &&
 414             ChrPtr(SendMsg)[StrLength(SendMsg) - 1] != '\n') {
 415                 syslog(LOG_WARNING,
 416                        "SMTP client[%d]: Possible problem: message did not "
 417                        "correctly terminate. (expecting 0x10, got 0x%02x)\n",
 418                        MsgCount, //yes uncool, but best choice here...
 419                        ChrPtr(SendMsg)[StrLength(SendMsg) - 1] );
 420                 StrBufAppendBufPlain(SendMsg, HKEY("\r\n"), 0);
 421         }
 422         return SendMsg;
 423 }
 424
 425
 426
 427 /*
 428  * smtp_do_bounce() is caled by smtp_do_procmsg() to scan a set of delivery
 429  * instructions for "5" codes (permanent fatal errors) and produce/deliver
 430  * a "bounce" message (delivery status notification).
 431  */
 432 void smtpq_do_bounce(OneQueItem *MyQItem, StrBuf *OMsgTxt)
 433 {
 434         static int seq = 0;
 435
 436         struct CtdlMessage *bmsg = NULL;
 437         StrBuf *boundary;
 438         StrBuf *Msg = NULL;
 439         StrBuf *BounceMB;
 440         struct recptypes *valid;
 441
 442         HashPos *It;
 443         void *vQE;
 444         long len;
 445         const char *Key;
 446
 447         int successful_bounce = 0;
 448         int num_bounces = 0;
 449         int give_up = 0;
 450
 451         syslog(LOG_DEBUG, "smtp_do_bounce() called\n");
 452
 453         if ( (ev_time() - MyQItem->Submitted) > SMTP_GIVE_UP ) {
 454                 give_up = 1;/// TODO: replace time by libevq timer get
 455         }
 456
 457         /*
 458          * Now go through the instructions checking for stuff.
 459          */
 460         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 461         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 462         {
 463                 MailQEntry *ThisItem = vQE;
 464                 if ((ThisItem->Status == 5) || /* failed now? */
 465                     ((give_up == 1) && (ThisItem->Status != 2))) /* giving up after failed attempts... */
 466                 {
 467                         if (num_bounces == 0)
 468                                 Msg = NewStrBufPlain(NULL, 1024);
 469                         ++num_bounces;
 470
 471                         StrBufAppendBuf(Msg, ThisItem->Recipient, 0);
 472                         StrBufAppendBufPlain(Msg, HKEY(": "), 0);
 473                         StrBufAppendBuf(Msg, ThisItem->StatusMessage, 0);
 474                         StrBufAppendBufPlain(Msg, HKEY("\r\n"), 0);
 475                 }
 476         }
 477         DeleteHashPos(&It);
 478
 479         /* Deliver the bounce if there's anything worth mentioning */
 480         syslog(LOG_DEBUG, "num_bounces = %d\n", num_bounces);
 481
 482         if (num_bounces == 0) {
 483                 FreeStrBuf(&Msg);
 484                 return;
 485         }
 486
 487         boundary = NewStrBufPlain(HKEY("=_Citadel_Multipart_"));
 488         StrBufAppendPrintf(boundary, "%s_%04x%04x", config.c_fqdn, getpid(), ++seq);
 489
 490         /* Start building our bounce message; go shopping for memory first. */
 491         BounceMB = NewStrBufPlain(NULL,
 492                                   1024 + /* mime stuff.... */
 493                                   StrLength(Msg) +  /* the bounce information... */
 494                                   StrLength(OMsgTxt)); /* the original message */
 495         if (BounceMB == NULL) {
 496                 FreeStrBuf(&boundary);
 497                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 498
 499                 return;
 500         }
 501
 502         bmsg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
 503         if (bmsg == NULL) {
 504                 FreeStrBuf(&boundary);
 505                 FreeStrBuf(&BounceMB);
 506                 syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
 507
 508                 return;
 509         }
 510         memset(bmsg, 0, sizeof(struct CtdlMessage));
 511
 512
 513         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: multipart/mixed; boundary=\""), 0);
 514         StrBufAppendBuf(BounceMB, boundary, 0);
 515         StrBufAppendBufPlain(BounceMB, HKEY("\"\r\n"), 0);
 516         StrBufAppendBufPlain(BounceMB, HKEY("MIME-Version: 1.0\r\n"), 0);
 517         StrBufAppendBufPlain(BounceMB, HKEY("X-Mailer: " CITADEL "\r\n"), 0);
 518         StrBufAppendBufPlain(BounceMB, HKEY("\r\nThis is a multipart message in MIME format.\r\n\r\n"), 0);
 519         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 520         StrBufAppendBuf(BounceMB, boundary, 0);
 521         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 522         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: text/plain\r\n\r\n"), 0);
 523
 524         if (give_up)
 525                 StrBufAppendBufPlain(
 526                         BounceMB,
 527                         HKEY(
 528                                 "A message you sent could not be delivered to some or all of its recipients\n"
 529                                 "due to prolonged unavailability of its destination(s).\n"
 530                                 "Giving up on the following addresses:\n\n"
 531                                 ), 0);
 532         else
 533                 StrBufAppendBufPlain(
 534                         BounceMB,
 535                         HKEY(
 536                                 "A message you sent could not be delivered to some or all of its recipients.\n"
 537                                 "The following addresses were undeliverable:\n\n"
 538                                 ), 0);
 539
 540         StrBufAppendBuf(BounceMB, Msg, 0);
 541         FreeStrBuf(&Msg);
 542
 543         /* Attach the original message */
 544         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 545         StrBufAppendBuf(BounceMB, boundary, 0);
 546         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 547         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: message/rfc822\r\n"), 0);
 548         StrBufAppendBufPlain(BounceMB, HKEY("Content-Transfer-Encoding: 7bit\r\n"), 0);
 549         StrBufAppendBufPlain(BounceMB, HKEY("Content-Disposition: inline\r\n"), 0);
 550         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
 551         StrBufAppendBuf(BounceMB, OMsgTxt, 0);
 552
 553         /* Close the multipart MIME scope */
 554         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
 555         StrBufAppendBuf(BounceMB, boundary, 0);
 556         StrBufAppendBufPlain(BounceMB, HKEY("--\r\n"), 0);
 557
 558
 559
 560         bmsg->cm_magic = CTDLMESSAGE_MAGIC;
 561         bmsg->cm_anon_type = MES_NORMAL;
 562         bmsg->cm_format_type = FMT_RFC822;
 563
 564         bmsg->cm_fields['O'] = strdup(MAILROOM);
 565         bmsg->cm_fields['A'] = strdup("Citadel");
 566         bmsg->cm_fields['N'] = strdup(config.c_nodename);
 567         bmsg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
 568         bmsg->cm_fields['M'] = SmashStrBuf(&BounceMB);
 569
 570         /* First try the user who sent the message */
 571         if (StrLength(MyQItem->BounceTo) == 0)
 572                 syslog(LOG_ERR, "No bounce address specified\n");
 573         else
 574                 syslog(LOG_DEBUG, "bounce to user? <%s>\n", ChrPtr(MyQItem->BounceTo));
 575
 576         /* Can we deliver the bounce to the original sender? */
 577         valid = validate_recipients(ChrPtr(MyQItem->BounceTo), NULL, 0);
 578         if ((valid != NULL) && (valid->num_error == 0)) {
 579                 CtdlSubmitMsg(bmsg, valid, "", QP_EADDR);
 580                 successful_bounce = 1;
 581         }
 582
 583         /* If not, post it in the Aide> room */
 584         if (successful_bounce == 0) {
 585                 CtdlSubmitMsg(bmsg, NULL, config.c_aideroom, QP_EADDR);
 586         }
 587
 588         /* Free up the memory we used */
 589         free_recipients(valid);
 590         FreeStrBuf(&boundary);
 591         CtdlFreeMessage(bmsg);
 592         syslog(LOG_DEBUG, "Done processing bounces\n");
 593 }
 594
 595
 596
 597 /*
 598 {
 599
 600         if (threadding)
 601                 n_smarthosts =  get_hosts(char *mxbuf, char *rectype);
 602 }
 603 */
 604 /*
 605  * smtp_do_procmsg()
 606  *
 607  * Called by smtp_do_queue() to handle an individual message.
 608  */
 609 void smtp_do_procmsg(long msgnum, void *userdata) {
 610         struct CtdlMessage *msg = NULL;
 611         char *instr = NULL;
 612         StrBuf *PlainQItem;
 613         OneQueItem *MyQItem;
 614         char *pch;
 615         HashPos  *It;
 616         void *vQE;
 617         long len;
 618         const char *Key;
 619         int nRelays = 0;
 620         ParsedURL *RelayUrls = NULL;
 621         int HaveBuffers = 0;
 622         StrBuf *Msg =NULL;
 623
 624         syslog(LOG_DEBUG, "SMTP Queue: smtp_do_procmsg(%ld)\n", msgnum);
 625         ///strcpy(envelope_from, "");
 626
 627         msg = CtdlFetchMessage(msgnum, 1);
 628         if (msg == NULL) {
 629                 syslog(LOG_ERR, "SMTP Queue: tried %ld but no such message!\n", msgnum);
 630                 return;
 631         }
 632
 633         pch = instr = msg->cm_fields['M'];
 634
 635         /* Strip out the headers (no not amd any other non-instruction) line */
 636         while (pch != NULL) {
 637                 pch = strchr(pch, '\n');
 638                 if ((pch != NULL) && (*(pch + 1) == '\n')) {
 639                         instr = pch + 2;
 640                         pch = NULL;
 641                 }
 642         }
 643         PlainQItem = NewStrBufPlain(instr, -1);
 644         CtdlFreeMessage(msg);
 645         MyQItem = DeserializeQueueItem(PlainQItem, msgnum);
 646         FreeStrBuf(&PlainQItem);
 647
 648         if (MyQItem == NULL) {
 649                 syslog(LOG_ERR, "SMTP Queue: Msg No %ld: already in progress!\n", msgnum);
 650                 return; /* s.b. else is already processing... */
 651         }
 652
 653         /*
 654          * Postpone delivery if we've already tried recently.
 655          * /
 656         if (((time(NULL) - MyQItem->LastAttempt.when) < MyQItem->LastAttempt.retry) && (run_queue_now == 0)) {
 657                 syslog(LOG_DEBUG, "SMTP client: Retry time not yet reached.\n");
 658
 659                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 660                 pthread_mutex_lock(&ActiveQItemsLock);
 661                 {
 662                         GetHashPosFromKey(ActiveQItems, IKEY(MyQItem->MessageID), It);
 663                         DeleteEntryFromHash(ActiveQItems, It);
 664                 }
 665                 pthread_mutex_unlock(&ActiveQItemsLock);
 666                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 667                 DeleteHashPos(&It);
 668                 return;
 669         }// TODO: reenable me.*/
 670
 671         /*
 672          * Bail out if there's no actual message associated with this
 673          */
 674         if (MyQItem->MessageID < 0L) {
 675                 syslog(LOG_ERR, "SMTP Queue: no 'msgid' directive found!\n");
 676                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 677                 pthread_mutex_lock(&ActiveQItemsLock);
 678                 {
 679                         GetHashPosFromKey(ActiveQItems, IKEY(MyQItem->MessageID), It);
 680                         DeleteEntryFromHash(ActiveQItems, It);
 681                 }
 682                 pthread_mutex_unlock(&ActiveQItemsLock);
 683                 DeleteHashPos(&It);
 684                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 685                 return;
 686         }
 687
 688         {
 689                 char mxbuf[SIZ];
 690                 ParsedURL **Url = &MyQItem->URL;
 691                 nRelays = get_hosts(mxbuf, "smarthost");
 692                 if (nRelays > 0) {
 693                         StrBuf *All;
 694                         StrBuf *One;
 695                         const char *Pos = NULL;
 696                         All = NewStrBufPlain(mxbuf, -1);
 697                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 698
 699                         while ((Pos != StrBufNOTNULL) && ((Pos == NULL) || !IsEmptyStr(Pos))) {
 700                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 701                                 if (!ParseURL(Url, One, 25))
 702                                         syslog(LOG_DEBUG, "Failed to parse: %s\n", ChrPtr(One));
 703                                 else {
 704                                         ///if (!Url->IsIP)) /// todo dupe me fork ipv6
 705                                         Url = &(*Url)->Next;
 706
 707                                 }
 708                         }
 709                         FreeStrBuf(&All);
 710                         FreeStrBuf(&One);
 711                 }
 712
 713                 Url = &MyQItem->FallBackHost;
 714                 nRelays = get_hosts(mxbuf, "fallbackhost");
 715                 if (nRelays > 0) {
 716                         StrBuf *All;
 717                         StrBuf *One;
 718                         const char *Pos = NULL;
 719                         All = NewStrBufPlain(mxbuf, -1);
 720                         One = NewStrBufPlain(NULL, StrLength(All) + 1);
 721
 722                         while ((Pos != StrBufNOTNULL) && ((Pos == NULL) || !IsEmptyStr(Pos))) {
 723                                 StrBufExtract_NextToken(One, All, &Pos, '|');
 724                                 if (!ParseURL(Url, One, 25))
 725                                         syslog(LOG_DEBUG, "Failed to parse: %s\n", ChrPtr(One));
 726                                 else
 727                                         Url = &(*Url)->Next;
 728                         }
 729                         FreeStrBuf(&All);
 730                         FreeStrBuf(&One);
 731                 }
 732         }
 733
 734         It = GetNewHashPos(MyQItem->MailQEntries, 0);
 735         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
 736         {
 737                 MailQEntry *ThisItem = vQE;
 738                 syslog(LOG_DEBUG, "SMTP Queue: Task: <%s> %d\n", ChrPtr(ThisItem->Recipient), ThisItem->Active);
 739         }
 740         DeleteHashPos(&It);
 741
 742         CountActiveQueueEntries(MyQItem);
 743         if (MyQItem->ActiveDeliveries > 0)
 744         {
 745                 int n = MsgCount++;
 746                 int m = MyQItem->ActiveDeliveries;
 747                 int i = 1;
 748                 Msg = smtp_load_msg(MyQItem, n);
 749                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 750                 while ((i <= m) &&
 751                        (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE)))
 752                 {
 753                         MailQEntry *ThisItem = vQE;
 754                         if (ThisItem->Active == 1) {
 755                                 int KeepBuffers = (i == m);
 756                                 if (i > 1) n = MsgCount++;
 757                                 syslog(LOG_DEBUG,
 758                                        "SMTP Queue: Trying <%s> %d / %d \n",
 759                                        ChrPtr(ThisItem->Recipient),
 760                                        i,
 761                                        m);
 762                                 smtp_try_one_queue_entry(MyQItem,
 763                                                          ThisItem,
 764                                                          Msg,
 765                                                          KeepBuffers,
 766                                                          n,
 767                                                          RelayUrls);
 768                                 if (KeepBuffers) HaveBuffers = 1;
 769                                 i++;
 770                         }
 771                 }
 772                 DeleteHashPos(&It);
 773         }
 774         else
 775         {
 776                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
 777                 pthread_mutex_lock(&ActiveQItemsLock);
 778                 {
 779                         GetHashPosFromKey(ActiveQItems, IKEY(MyQItem->MessageID), It);
 780                         DeleteEntryFromHash(ActiveQItems, It);
 781                 }
 782                 pthread_mutex_unlock(&ActiveQItemsLock);
 783                 DeleteHashPos(&It);
 784                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
 785
 786 // TODO: bounce & delete?
 787
 788         }
 789         if (!HaveBuffers) {
 790                 FreeStrBuf (&Msg);
 791 // TODO : free RelayUrls
 792         }
 793 }
 794
 795
 796
 797 /*
 798  * smtp_queue_thread()
 799  *
 800  * Run through the queue sending out messages.
 801  */
 802 void smtp_do_queue(void) {
 803         static int is_running = 0;
 804         int num_processed = 0;
 805
 806         if (is_running) return;         /* Concurrency check - only one can run */
 807         is_running = 1;
 808
 809         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
 810         syslog(LOG_INFO, "SMTP client: processing outbound queue");
 811
 812         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
 813                 syslog(LOG_ERR, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
 814         }
 815         else {
 816                 num_processed = CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_do_procmsg, NULL);
 817         }
 818         syslog(LOG_INFO, "SMTP client: queue run completed; %d messages processed", num_processed);
 819         run_queue_now = 0;
 820         is_running = 0;
 821 }
 822
 823
 824
 825 /*
 826  * Initialize the SMTP outbound queue
 827  */
 828 void smtp_init_spoolout(void) {
 829         struct ctdlroom qrbuf;
 830
 831         /*
 832          * Create the room.  This will silently fail if the room already
 833          * exists, and that's perfectly ok, because we want it to exist.
 834          */
 835         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_MAILBOX);
 836
 837         /*
 838          * Make sure it's set to be a "system room" so it doesn't show up
 839          * in the <K>nown rooms list for Aides.
 840          */
 841         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
 842                 qrbuf.QRflags2 |= QR2_SYSTEM;
 843                 CtdlPutRoomLock(&qrbuf);
 844         }
 845 }
 846
 847
 848
 849
 850 /*****************************************************************************/
 851 /*                          SMTP UTILITY COMMANDS                            */
 852 /*****************************************************************************/
 853
 854 void cmd_smtp(char *argbuf) {
 855         char cmd[64];
 856         char node[256];
 857         char buf[1024];
 858         int i;
 859         int num_mxhosts;
 860
 861         if (CtdlAccessCheck(ac_aide)) return;
 862
 863         extract_token(cmd, argbuf, 0, '|', sizeof cmd);
 864
 865         if (!strcasecmp(cmd, "mx")) {
 866                 extract_token(node, argbuf, 1, '|', sizeof node);
 867                 num_mxhosts = getmx(buf, node);
 868                 cprintf("%d %d MX hosts listed for %s\n",
 869                         LISTING_FOLLOWS, num_mxhosts, node);
 870                 for (i=0; i<num_mxhosts; ++i) {
 871                         extract_token(node, buf, i, '|', sizeof node);
 872                         cprintf("%s\n", node);
 873                 }
 874                 cprintf("000\n");
 875                 return;
 876         }
 877
 878         else if (!strcasecmp(cmd, "runqueue")) {
 879                 run_queue_now = 1;
 880                 cprintf("%d All outbound SMTP will be retried now.\n", CIT_OK);
 881                 return;
 882         }
 883
 884         else {
 885                 cprintf("%d Invalid command.\n", ERROR + ILLEGAL_VALUE);
 886         }
 887
 888 }
 889
 890
 891 CTDL_MODULE_INIT(smtp_queu)
 892 {
 893         if (!threading)
 894         {
 895                 CtdlFillSystemContext(&smtp_queue_CC, "SMTP_Send");
 896                 ActiveQItems = NewHash(1, lFlathash);
 897                 pthread_mutex_init(&ActiveQItemsLock, NULL);
 898
 899                 QItemHandlers = NewHash(0, NULL);
 900
 901                 Put(QItemHandlers, HKEY("msgid"), QItem_Handle_MsgID, reference_free_handler);
 902                 Put(QItemHandlers, HKEY("envelope_from"), QItem_Handle_EnvelopeFrom, reference_free_handler);
 903                 Put(QItemHandlers, HKEY("retry"), QItem_Handle_retry, reference_free_handler);
 904                 Put(QItemHandlers, HKEY("attempted"), QItem_Handle_Attempted, reference_free_handler);
 905                 Put(QItemHandlers, HKEY("remote"), QItem_Handle_Recipient, reference_free_handler);
 906                 Put(QItemHandlers, HKEY("bounceto"), QItem_Handle_BounceTo, reference_free_handler);
 907                 Put(QItemHandlers, HKEY("submitted"), QItem_Handle_Submitted, reference_free_handler);
 908 ////TODO: flush qitemhandlers on exit
 909                 smtp_init_spoolout();
 910
 911                 CtdlRegisterCleanupHook(smtp_evq_cleanup);
 912
 913                 CtdlRegisterProtoHook(cmd_smtp, "SMTP", "SMTP utility commands");
 914         }
 915
 916         /* return our Subversion id for the Log */
 917         return "smtpeventclient";
 918 }