SMTP-Client: add the stati of all delivery attempts to the status messages.
[citadel.git] / citadel / modules / smtp / serv_smtpqueue.c
1 /*
2  * This module is an SMTP and ESMTP implementation for the Citadel system.
3  * It is compliant with all of the following:
4  *
5  * RFC  821 - Simple Mail Transfer Protocol
6  * RFC  876 - Survey of SMTP Implementations
7  * RFC 1047 - Duplicate messages and SMTP
8  * RFC 1652 - 8 bit MIME
9  * RFC 1869 - Extended Simple Mail Transfer Protocol
10  * RFC 1870 - SMTP Service Extension for Message Size Declaration
11  * RFC 2033 - Local Mail Transfer Protocol
12  * RFC 2197 - SMTP Service Extension for Command Pipelining
13  * RFC 2476 - Message Submission
14  * RFC 2487 - SMTP Service Extension for Secure SMTP over TLS
15  * RFC 2554 - SMTP Service Extension for Authentication
16  * RFC 2821 - Simple Mail Transfer Protocol
17  * RFC 2822 - Internet Message Format
18  * RFC 2920 - SMTP Service Extension for Command Pipelining
19  *  
20  * The VRFY and EXPN commands have been removed from this implementation
21  * because nobody uses these commands anymore, except for spammers.
22  *
23  * Copyright (c) 1998-2012 by the citadel.org team
24  *
25  *  This program is open source software; you can redistribute it and/or modify
26  *  it under the terms of the GNU General Public License version 3.
27  *  
28  *  
29  *
30  *  This program is distributed in the hope that it will be useful,
31  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
32  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
33  *  GNU General Public License for more details.
34  *
35  *  
36  *  
37  *  
38  */
39
40 #include "sysdep.h"
41 #include <stdlib.h>
42 #include <unistd.h>
43 #include <stdio.h>
44 #include <termios.h>
45 #include <fcntl.h>
46 #include <signal.h>
47 #include <pwd.h>
48 #include <errno.h>
49 #include <sys/types.h>
50 #include <syslog.h>
51
52 #if TIME_WITH_SYS_TIME
53 # include <sys/time.h>
54 # include <time.h>
55 #else
56 # if HAVE_SYS_TIME_H
57 #  include <sys/time.h>
58 # else
59 #  include <time.h>
60 # endif
61 #endif
62 #include <sys/wait.h>
63 #include <ctype.h>
64 #include <string.h>
65 #include <limits.h>
66 #include <sys/socket.h>
67 #include <netinet/in.h>
68 #include <arpa/inet.h>
69 #include <libcitadel.h>
70 #include "citadel.h"
71 #include "server.h"
72 #include "citserver.h"
73 #include "support.h"
74 #include "config.h"
75 #include "control.h"
76 #include "user_ops.h"
77 #include "database.h"
78 #include "msgbase.h"
79 #include "internet_addressing.h"
80 #include "genstamp.h"
81 #include "domain.h"
82 #include "clientsocket.h"
83 #include "locate_host.h"
84 #include "citadel_dirs.h"
85
86 #include "ctdl_module.h"
87
88 #include "smtpqueue.h"
89 #include "smtp_clienthandlers.h"
90 #include "event_client.h"
91
92
93 struct CitContext smtp_queue_CC;
94 pthread_mutex_t ActiveQItemsLock;
95 HashList *ActiveQItems  = NULL;
96 HashList *QItemHandlers = NULL;
97 const unsigned short DefaultMXPort = 25;
98 int max_sessions_for_outbound_smtp = 500; /* how many sessions might be active till we stop adding more smtp jobs */
99 int ndelay_count = 50; /* every n queued messages we will sleep... */
100 int delay_msec = 5000; /* this many seconds. */
101
102 static const long MaxRetry = SMTP_RETRY_INTERVAL * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2 * 2;
103 int MsgCount            = 0;
104 int run_queue_now       = 0;    /* Set to 1 to ignore SMTP send retry times */
105
106 void RegisterQItemHandler(const char *Key, long Len, QItemHandler H)
107 {
108         QItemHandlerStruct *HS = (QItemHandlerStruct*)malloc(sizeof(QItemHandlerStruct));
109         HS->H = H;
110         Put(QItemHandlers, Key, Len, HS, NULL);
111 }
112
113
114
115 void smtp_try_one_queue_entry(OneQueItem *MyQItem,
116                               MailQEntry *MyQEntry,
117                               StrBuf *MsgText,
118 /* KeepMsgText allows us to use MsgText as ours. */
119                               int KeepMsgText,
120                               int MsgCount,
121                               ParsedURL *RelayUrls);
122
123
124 void smtp_evq_cleanup(void)
125 {
126
127         pthread_mutex_lock(&ActiveQItemsLock);
128         DeleteHash(&QItemHandlers);
129         DeleteHash(&ActiveQItems);
130         pthread_mutex_unlock(&ActiveQItemsLock);
131         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
132 /*      citthread_mutex_destroy(&ActiveQItemsLock); TODO */
133 }
134
135 int DecreaseQReference(OneQueItem *MyQItem)
136 {
137         int IDestructQueItem;
138
139         pthread_mutex_lock(&ActiveQItemsLock);
140         MyQItem->ActiveDeliveries--;
141         IDestructQueItem = MyQItem->ActiveDeliveries == 0;
142         pthread_mutex_unlock(&ActiveQItemsLock);
143         return IDestructQueItem;
144 }
145
146 void DecreaseShutdownDeliveries(OneQueItem *MyQItem)
147 {
148         pthread_mutex_lock(&ActiveQItemsLock);
149         MyQItem->NotYetShutdownDeliveries--;
150         pthread_mutex_unlock(&ActiveQItemsLock);
151 }
152
153 int GetShutdownDeliveries(OneQueItem *MyQItem)
154 {
155         int DestructNow;
156
157         pthread_mutex_lock(&ActiveQItemsLock);
158         DestructNow = MyQItem->ActiveDeliveries == 0;
159         pthread_mutex_unlock(&ActiveQItemsLock);
160         return DestructNow;
161 }
162 void RemoveQItem(OneQueItem *MyQItem)
163 {
164         long len;
165         const char* Key;
166         void *VData;
167         HashPos  *It;
168
169         pthread_mutex_lock(&ActiveQItemsLock);
170         It = GetNewHashPos(ActiveQItems, 0);
171         if (GetHashPosFromKey(ActiveQItems, LKEY(MyQItem->MessageID), It))
172                 DeleteEntryFromHash(ActiveQItems, It);
173         else
174         {
175                 SMTPC_syslog(LOG_WARNING,
176                              "unable to find QItem with ID[%ld]",
177                              MyQItem->MessageID);
178                 while (GetNextHashPos(ActiveQItems, It, &len, &Key, &VData))
179                         SMTPC_syslog(LOG_WARNING,
180                                      "have_: ID[%ld]",
181                                      ((OneQueItem *)VData)->MessageID);
182         }
183         pthread_mutex_unlock(&ActiveQItemsLock);
184         DeleteHashPos(&It);
185 }
186
187
188 void FreeMailQEntry(void *qv)
189 {
190         MailQEntry *Q = qv;
191 /*
192         SMTPC_syslog(LOG_DEBUG, "---------------%s--------------", __FUNCTION__);
193         cit_backtrace();
194 */
195         FreeStrBuf(&Q->Recipient);
196         FreeStrBuf(&Q->StatusMessage);
197         FreeStrBuf(&Q->AllStatusMessages);
198         memset(Q, 0, sizeof(MailQEntry));
199         free(Q);
200 }
201 void FreeQueItem(OneQueItem **Item)
202 {
203 /*
204         SMTPC_syslog(LOG_DEBUG, "---------------%s--------------", __FUNCTION__);
205         cit_backtrace();
206 */
207         DeleteHash(&(*Item)->MailQEntries);
208         FreeStrBuf(&(*Item)->EnvelopeFrom);
209         FreeStrBuf(&(*Item)->BounceTo);
210         FreeStrBuf(&(*Item)->SenderRoom);
211         FreeURL(&(*Item)->URL);
212         memset(*Item, 0, sizeof(OneQueItem));
213         free(*Item);
214         Item = NULL;
215 }
216 void HFreeQueItem(void *Item)
217 {
218         FreeQueItem((OneQueItem**)&Item);
219 }
220
221 /* inspect recipients with a status of:
222  * - 0 (no delivery yet attempted)
223  * - 3/4 (transient errors
224  *        were experienced and it's time to try again)
225  */
226 int CheckQEntryActive(MailQEntry *ThisItem)
227 {
228         if ((ThisItem->Status == 0) ||
229             (ThisItem->Status == 3) ||
230             (ThisItem->Status == 4))
231         {
232                 return 1;
233         }
234         else
235                 return 0;
236 }
237 int CheckQEntryIsBounce(MailQEntry *ThisItem)
238 {
239         if ((ThisItem->Status == 3) ||
240             (ThisItem->Status == 4) ||
241             (ThisItem->Status == 5))
242         {
243                 return 1;
244         }
245         else
246                 return 0;
247 }       
248
249 int CountActiveQueueEntries(OneQueItem *MyQItem, int before)
250 {
251         HashPos  *It;
252         long len;
253         long ActiveDeliveries;
254         const char *Key;
255         void *vQE;
256
257         ActiveDeliveries = 0;
258         It = GetNewHashPos(MyQItem->MailQEntries, 0);
259         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
260         {
261                 int Active;
262                 MailQEntry *ThisItem = vQE;
263
264                 if (CheckQEntryActive(ThisItem))
265                 {
266                         ActiveDeliveries++;
267                         Active = 1;
268                 }
269                 else
270                         Active = 0;
271                 if (before)
272                         ThisItem->Active = Active;
273                 else
274                         ThisItem->StillActive = Active;
275         }
276         DeleteHashPos(&It);
277         return ActiveDeliveries;
278 }
279
280 OneQueItem *DeserializeQueueItem(StrBuf *RawQItem, long QueMsgID)
281 {
282         OneQueItem *Item;
283         const char *pLine = NULL;
284         StrBuf *Line;
285         StrBuf *Token;
286         void *v;
287
288         Item = (OneQueItem*)malloc(sizeof(OneQueItem));
289         memset(Item, 0, sizeof(OneQueItem));
290         Item->Retry = SMTP_RETRY_INTERVAL;
291         Item->MessageID = -1;
292         Item->QueMsgID = QueMsgID;
293
294         Token = NewStrBuf();
295         Line = NewStrBufPlain(NULL, 128);
296         while (pLine != StrBufNOTNULL) {
297                 const char *pItemPart = NULL;
298                 void *vHandler;
299
300                 StrBufExtract_NextToken(Line, RawQItem, &pLine, '\n');
301                 if (StrLength(Line) == 0) continue;
302                 StrBufExtract_NextToken(Token, Line, &pItemPart, '|');
303                 if (GetHash(QItemHandlers, SKEY(Token), &vHandler))
304                 {
305                         QItemHandlerStruct *HS;
306                         HS = (QItemHandlerStruct*) vHandler;
307                         HS->H(Item, Line, &pItemPart);
308                 }
309         }
310         FreeStrBuf(&Line);
311         FreeStrBuf(&Token);
312
313         if (Item->Retry >= MaxRetry)
314                 Item->FailNow = 1;
315
316         pthread_mutex_lock(&ActiveQItemsLock);
317         if (GetHash(ActiveQItems,
318                     LKEY(Item->MessageID),
319                     &v))
320         {
321                 /* WHOOPS. somebody else is already working on this. */
322                 pthread_mutex_unlock(&ActiveQItemsLock);
323                 FreeQueItem(&Item);
324                 return NULL;
325         }
326         else {
327                 /* mark our claim on this. */
328                 Put(ActiveQItems,
329                     LKEY(Item->MessageID),
330                     Item,
331                     HFreeQueItem);
332                 pthread_mutex_unlock(&ActiveQItemsLock);
333         }
334
335         return Item;
336 }
337
338 StrBuf *SerializeQueueItem(OneQueItem *MyQItem)
339 {
340         StrBuf *QMessage;
341         HashPos  *It;
342         const char *Key;
343         long len;
344         void *vQE;
345
346         QMessage = NewStrBufPlain(NULL, SIZ);
347         StrBufPrintf(QMessage, "Content-type: %s\n", SPOOLMIME);
348 //      "attempted|%ld\n"  "retry|%ld\n",, (long)time(NULL), (long)retry );
349         StrBufAppendBufPlain(QMessage, HKEY("\nmsgid|"), 0);
350         StrBufAppendPrintf(QMessage, "%ld", MyQItem->MessageID);
351
352         StrBufAppendBufPlain(QMessage, HKEY("\nsubmitted|"), 0);
353         StrBufAppendPrintf(QMessage, "%ld", MyQItem->Submitted);
354
355         if (StrLength(MyQItem->BounceTo) > 0) {
356                 StrBufAppendBufPlain(QMessage, HKEY("\nbounceto|"), 0);
357                 StrBufAppendBuf(QMessage, MyQItem->BounceTo, 0);
358         }
359
360         if (StrLength(MyQItem->EnvelopeFrom) > 0) {
361                 StrBufAppendBufPlain(QMessage, HKEY("\nenvelope_from|"), 0);
362                 StrBufAppendBuf(QMessage, MyQItem->EnvelopeFrom, 0);
363         }
364
365         if (StrLength(MyQItem->SenderRoom) > 0) {
366                 StrBufAppendBufPlain(QMessage, HKEY("\nsource_room|"), 0);
367                 StrBufAppendBuf(QMessage, MyQItem->SenderRoom, 0);
368         }
369
370         StrBufAppendBufPlain(QMessage, HKEY("\nretry|"), 0);
371         StrBufAppendPrintf(QMessage, "%ld",
372                            MyQItem->Retry);
373
374         StrBufAppendBufPlain(QMessage, HKEY("\nattempted|"), 0);
375         StrBufAppendPrintf(QMessage, "%ld",
376                            time(NULL) /*ctdl_ev_now()*/ + MyQItem->Retry);
377
378         It = GetNewHashPos(MyQItem->MailQEntries, 0);
379         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
380         {
381                 MailQEntry *ThisItem = vQE;
382
383                 StrBufAppendBufPlain(QMessage, HKEY("\nremote|"), 0);
384                 StrBufAppendBuf(QMessage, ThisItem->Recipient, 0);
385                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
386                 StrBufAppendPrintf(QMessage, "%d", ThisItem->Status);
387                 StrBufAppendBufPlain(QMessage, HKEY("|"), 0);
388                 if (ThisItem->AllStatusMessages != NULL)
389                         StrBufAppendBuf(QMessage, ThisItem->AllStatusMessages, 0);
390                 else
391                         StrBufAppendBuf(QMessage, ThisItem->StatusMessage, 0);
392         }
393         DeleteHashPos(&It);
394         StrBufAppendBufPlain(QMessage, HKEY("\n"), 0);
395         return QMessage;
396 }
397
398
399
400
401
402 void NewMailQEntry(OneQueItem *Item)
403 {
404         Item->Current = (MailQEntry*) malloc(sizeof(MailQEntry));
405         memset(Item->Current, 0, sizeof(MailQEntry));
406
407         if (Item->MailQEntries == NULL)
408                 Item->MailQEntries = NewHash(1, Flathash);
409         /* alocate big buffer so we won't get problems reallocating later. */
410         Item->Current->StatusMessage = NewStrBufPlain(NULL, SIZ);
411         Item->Current->n = GetCount(Item->MailQEntries);
412         Put(Item->MailQEntries,
413             IKEY(Item->Current->n),
414             Item->Current,
415             FreeMailQEntry);
416 }
417
418 void QItem_Handle_MsgID(OneQueItem *Item, StrBuf *Line, const char **Pos)
419 {
420         Item->MessageID = StrBufExtractNext_long(Line, Pos, '|');
421 }
422
423 void QItem_Handle_EnvelopeFrom(OneQueItem *Item, StrBuf *Line, const char **Pos)
424 {
425         if (Item->EnvelopeFrom == NULL)
426                 Item->EnvelopeFrom = NewStrBufPlain(NULL, StrLength(Line));
427         StrBufExtract_NextToken(Item->EnvelopeFrom, Line, Pos, '|');
428 }
429
430 void QItem_Handle_BounceTo(OneQueItem *Item, StrBuf *Line, const char **Pos)
431 {
432         if (Item->BounceTo == NULL)
433                 Item->BounceTo = NewStrBufPlain(NULL, StrLength(Line));
434         StrBufExtract_NextToken(Item->BounceTo, Line, Pos, '|');
435 }
436
437 void QItem_Handle_SenderRoom(OneQueItem *Item, StrBuf *Line, const char **Pos)
438 {
439         if (Item->SenderRoom == NULL)
440                 Item->SenderRoom = NewStrBufPlain(NULL, StrLength(Line));
441         StrBufExtract_NextToken(Item->SenderRoom, Line, Pos, '|');
442 }
443
444 void QItem_Handle_Recipient(OneQueItem *Item, StrBuf *Line, const char **Pos)
445 {
446         if (Item->Current == NULL)
447                 NewMailQEntry(Item);
448         if (Item->Current->Recipient == NULL)
449                 Item->Current->Recipient=NewStrBufPlain(NULL, StrLength(Line));
450         StrBufExtract_NextToken(Item->Current->Recipient, Line, Pos, '|');
451         Item->Current->Status = StrBufExtractNext_int(Line, Pos, '|');
452         StrBufExtract_NextToken(Item->Current->StatusMessage, Line, Pos, '|');
453         Item->Current = NULL; // TODO: is this always right?
454 }
455
456
457 void QItem_Handle_retry(OneQueItem *Item, StrBuf *Line, const char **Pos)
458 {
459         Item->Retry =
460                 StrBufExtractNext_int(Line, Pos, '|');
461         if (Item->Retry == 0)
462                 Item->Retry = SMTP_RETRY_INTERVAL;
463         else
464                 Item->Retry *= 2;
465 }
466
467
468 void QItem_Handle_Submitted(OneQueItem *Item, StrBuf *Line, const char **Pos)
469 {
470         Item->Submitted = atol(*Pos);
471
472 }
473
474 void QItem_Handle_Attempted(OneQueItem *Item, StrBuf *Line, const char **Pos)
475 {
476         Item->ReattemptWhen = StrBufExtractNext_int(Line, Pos, '|');
477 }
478
479
480
481 /**
482  * this one has to have the context for loading the message via the redirect buffer...
483  */
484 StrBuf *smtp_load_msg(OneQueItem *MyQItem, int n, char **Author, char **Address)
485 {
486         CitContext *CCC=CC;
487         StrBuf *SendMsg;
488
489         CCC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
490         CtdlOutputMsg(MyQItem->MessageID,
491                       MT_RFC822, HEADERS_ALL,
492                       0, 1, NULL,
493                       (ESC_DOT|SUPPRESS_ENV_TO),
494                       Author,
495                       Address);
496
497         SendMsg = CCC->redirect_buffer;
498         CCC->redirect_buffer = NULL;
499         if ((StrLength(SendMsg) > 0) &&
500             ChrPtr(SendMsg)[StrLength(SendMsg) - 1] != '\n') {
501                 SMTPC_syslog(LOG_WARNING,
502                              "[%d] Possible problem: message did not "
503                              "correctly terminate. (expecting 0x10, got 0x%02x)\n",
504                              MsgCount, //yes uncool, but best choice here...
505                              ChrPtr(SendMsg)[StrLength(SendMsg) - 1] );
506                 StrBufAppendBufPlain(SendMsg, HKEY("\r\n"), 0);
507         }
508         return SendMsg;
509 }
510
511
512
513 /*
514  * smtp_do_bounce() is caled by smtp_do_procmsg() to scan a set of delivery
515  * instructions for "5" codes (permanent fatal errors) and produce/deliver
516  * a "bounce" message (delivery status notification).
517  */
518 void smtpq_do_bounce(OneQueItem *MyQItem, StrBuf *OMsgTxt, ParsedURL *Relay)
519 {
520         static int seq = 0;
521
522         struct CtdlMessage *bmsg = NULL;
523         StrBuf *boundary;
524         StrBuf *Msg = NULL;
525         StrBuf *BounceMB;
526         struct recptypes *valid;
527         time_t now;
528
529         HashPos *It;
530         void *vQE;
531         long len;
532         const char *Key;
533
534         int first_attempt = 0;
535         int successful_bounce = 0;
536         int num_bounces = 0;
537         int give_up = 0;
538
539         SMTPCM_syslog(LOG_DEBUG, "smtp_do_bounce() called\n");
540
541         if (MyQItem->SendBounceMail == 0)
542                 return;
543
544         now = time (NULL); //ev_time();
545
546         if ( (now - MyQItem->Submitted) > SMTP_GIVE_UP ) {
547                 give_up = 1;
548         }
549
550         if (MyQItem->Retry == SMTP_RETRY_INTERVAL) {
551                 first_attempt = 1;
552         }
553
554         /*
555          * Now go through the instructions checking for stuff.
556          */
557         Msg = NewStrBufPlain(NULL, 1024);
558         It = GetNewHashPos(MyQItem->MailQEntries, 0);
559         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
560         {
561                 MailQEntry *ThisItem = vQE;
562                 if ((ThisItem->Active && (ThisItem->Status == 5)) || /* failed now? */
563                     ((give_up == 1) && (ThisItem->Status != 2)) ||
564                     ((first_attempt == 1) && (ThisItem->Status != 2)))
565                         /* giving up after failed attempts... */
566                 {
567                         ++num_bounces;
568
569                         StrBufAppendBufPlain(Msg, HKEY(" "), 0);
570                         StrBufAppendBuf(Msg, ThisItem->Recipient, 0);
571                         StrBufAppendBufPlain(Msg, HKEY(": "), 0);
572                         StrBufAppendBuf(Msg, ThisItem->StatusMessage, 0);
573                         StrBufAppendBufPlain(Msg, HKEY("\r\n"), 0);
574                 }
575         }
576         DeleteHashPos(&It);
577
578         /* Deliver the bounce if there's anything worth mentioning */
579         SMTPC_syslog(LOG_DEBUG, "num_bounces = %d\n", num_bounces);
580
581         if (num_bounces == 0) {
582                 FreeStrBuf(&Msg);
583                 return;
584         }
585
586         if ((StrLength(MyQItem->SenderRoom) == 0) && MyQItem->HaveRelay) {
587                 const char *RelayUrlStr = "[not found]";
588                 /* one message that relaying is broken is enough; no extra room error message. */
589                 StrBuf *RelayDetails = NewStrBuf();
590
591                 if (Relay != NULL)
592                         RelayUrlStr = ChrPtr(Relay->URL);
593
594                 StrBufPrintf(RelayDetails,
595                              "Relaying via %s failed permanently. \n Reason:\n%s\n Revalidate your relay configuration.",
596                              RelayUrlStr,
597                              ChrPtr(Msg));
598                 CtdlAideMessage(ChrPtr(RelayDetails), "Relaying Failed");
599                 FreeStrBuf(&RelayDetails);
600         }
601
602         boundary = NewStrBufPlain(HKEY("=_Citadel_Multipart_"));
603         StrBufAppendPrintf(boundary,
604                            "%s_%04x%04x",
605                            config.c_fqdn,
606                            getpid(),
607                            ++seq);
608
609         /* Start building our bounce message; go shopping for memory first. */
610         BounceMB = NewStrBufPlain(
611                 NULL,
612                 1024 + /* mime stuff.... */
613                 StrLength(Msg) +  /* the bounce information... */
614                 StrLength(OMsgTxt)); /* the original message */
615         if (BounceMB == NULL) {
616                 FreeStrBuf(&boundary);
617                 SMTPCM_syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
618
619                 return;
620         }
621
622         bmsg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
623         if (bmsg == NULL) {
624                 FreeStrBuf(&boundary);
625                 FreeStrBuf(&BounceMB);
626                 SMTPCM_syslog(LOG_ERR, "Failed to alloc() bounce message.\n");
627
628                 return;
629         }
630         memset(bmsg, 0, sizeof(struct CtdlMessage));
631
632
633         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: multipart/mixed; boundary=\""), 0);
634         StrBufAppendBuf(BounceMB, boundary, 0);
635         StrBufAppendBufPlain(BounceMB, HKEY("\"\r\n"), 0);
636         StrBufAppendBufPlain(BounceMB, HKEY("MIME-Version: 1.0\r\n"), 0);
637         StrBufAppendBufPlain(BounceMB, HKEY("X-Mailer: " CITADEL "\r\n"), 0);
638         StrBufAppendBufPlain(BounceMB, HKEY("\r\nThis is a multipart message in MIME format.\r\n\r\n"), 0);
639         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
640         StrBufAppendBuf(BounceMB, boundary, 0);
641         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
642         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: text/plain\r\n\r\n"), 0);
643
644         if (give_up)
645                 StrBufAppendBufPlain(
646                         BounceMB,
647                         HKEY(
648                                 "A message you sent could not be delivered "
649                                 "to some or all of its recipients\n"
650                                 "due to prolonged unavailability "
651                                 "of its destination(s).\n"
652                                 "Giving up on the following addresses:\n\n"
653                                 ), 0);
654         else
655                 StrBufAppendBufPlain(
656                         BounceMB,
657                         HKEY(
658                                 "A message you sent could not be delivered "
659                                 "to some or all of its recipients.\n"
660                                 "The following addresses "
661                                 "were undeliverable:\n\n"
662                                 ), 0);
663
664         StrBufAppendBuf(BounceMB, Msg, 0);
665         FreeStrBuf(&Msg);
666
667         if (StrLength(MyQItem->SenderRoom) > 0)
668         {
669                 StrBufAppendBufPlain(
670                         BounceMB,
671                         HKEY("The message was originaly posted in: "), 0);
672                 StrBufAppendBuf(BounceMB, MyQItem->SenderRoom, 0);
673                 StrBufAppendBufPlain(
674                         BounceMB,
675                         HKEY("\n"), 0);
676         }
677
678         /* Attach the original message */
679         StrBufAppendBufPlain(BounceMB, HKEY("\r\n--"), 0);
680         StrBufAppendBuf(BounceMB, boundary, 0);
681         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
682         StrBufAppendBufPlain(BounceMB,
683                              HKEY("Content-type: message/rfc822\r\n"), 0);
684         StrBufAppendBufPlain(BounceMB,
685                              HKEY("Content-Transfer-Encoding: 7bit\r\n"), 0);
686         StrBufAppendBufPlain(BounceMB,
687                              HKEY("Content-Disposition: inline\r\n"), 0);
688         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
689         StrBufAppendBuf(BounceMB, OMsgTxt, 0);
690
691         /* Close the multipart MIME scope */
692         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
693         StrBufAppendBuf(BounceMB, boundary, 0);
694         StrBufAppendBufPlain(BounceMB, HKEY("--\r\n"), 0);
695
696         bmsg->cm_magic = CTDLMESSAGE_MAGIC;
697         bmsg->cm_anon_type = MES_NORMAL;
698         bmsg->cm_format_type = FMT_RFC822;
699
700         bmsg->cm_fields['O'] = strdup(MAILROOM);
701         bmsg->cm_fields['A'] = strdup("Citadel");
702         bmsg->cm_fields['N'] = strdup(config.c_nodename);
703         bmsg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
704         bmsg->cm_fields['M'] = SmashStrBuf(&BounceMB);
705
706         /* First try the user who sent the message */
707         if (StrLength(MyQItem->BounceTo) == 0) {
708                 SMTPCM_syslog(LOG_ERR, "No bounce address specified\n");
709         }
710         else {
711                 SMTPC_syslog(LOG_DEBUG, "bounce to user? <%s>\n",
712                        ChrPtr(MyQItem->BounceTo));
713         }
714
715         /* Can we deliver the bounce to the original sender? */
716         valid = validate_recipients(ChrPtr(MyQItem->BounceTo), NULL, 0);
717         if ((valid != NULL) && (valid->num_error == 0)) {
718                 CtdlSubmitMsg(bmsg, valid, "", QP_EADDR);
719                 successful_bounce = 1;
720         }
721
722         /* If not, post it in the Aide> room */
723         if (successful_bounce == 0) {
724                 CtdlSubmitMsg(bmsg, NULL, config.c_aideroom, QP_EADDR);
725         }
726
727         /* Free up the memory we used */
728         free_recipients(valid);
729         FreeStrBuf(&boundary);
730         CtdlFreeMessage(bmsg);
731         SMTPCM_syslog(LOG_DEBUG, "Done processing bounces\n");
732 }
733
734 ParsedURL *LoadRelayUrls(OneQueItem *MyQItem,
735                          char *Author,
736                          char *Address)
737 {
738         int nRelays = 0;
739         ParsedURL *RelayUrls = NULL;
740         char mxbuf[SIZ];
741         ParsedURL **Url = &MyQItem->URL;
742
743         nRelays = get_hosts(mxbuf, "fallbackhost");
744         if (nRelays > 0) {
745                 StrBuf *All;
746                 StrBuf *One;
747                 const char *Pos = NULL;
748                 All = NewStrBufPlain(mxbuf, -1);
749                 One = NewStrBufPlain(NULL, StrLength(All) + 1);
750                 
751                 while ((Pos != StrBufNOTNULL) &&
752                        ((Pos == NULL) ||
753                         !IsEmptyStr(Pos)))
754                 {
755                         StrBufExtract_NextToken(One, All, &Pos, '|');
756                         if (!ParseURL(Url, One, DefaultMXPort)) {
757                                 SMTPC_syslog(LOG_DEBUG,
758                                              "Failed to parse: %s\n",
759                                              ChrPtr(One));
760                         }
761                         else {
762                                 (*Url)->IsRelay = 1;
763                                 MyQItem->HaveRelay = 1;
764                         }
765                 }
766                 FreeStrBuf(&All);
767                 FreeStrBuf(&One);
768         }
769         nRelays = get_hosts(mxbuf, "smarthost");
770         if (nRelays > 0) {
771                 char *User;
772                 StrBuf *All;
773                 StrBuf *One;
774                 const char *Pos = NULL;
775                 All = NewStrBufPlain(mxbuf, -1);
776                 One = NewStrBufPlain(NULL, StrLength(All) + 1);
777                 
778                 while ((Pos != StrBufNOTNULL) &&
779                        ((Pos == NULL) ||
780                         !IsEmptyStr(Pos)))
781                 {
782                         StrBufExtract_NextToken(One, All, &Pos, '|');
783                         User = strchr(ChrPtr(One), ' ');
784                         if (User != NULL) {
785                                 if (!strcmp(User + 1, Author) ||
786                                     !strcmp(User + 1, Address))
787                                         StrBufCutAt(One, 0, User);
788                                 else {
789                                         MyQItem->HaveRelay = 1;
790                                         continue;
791                                 }
792                         }
793                         if (!ParseURL(Url, One, DefaultMXPort)) {
794                                 SMTPC_syslog(LOG_DEBUG,
795                                              "Failed to parse: %s\n",
796                                              ChrPtr(One));
797                         }
798                         else {
799                                 ///if (!Url->IsIP)) // todo dupe me fork ipv6
800                                 (*Url)->IsRelay = 1;
801                                 MyQItem->HaveRelay = 1;
802                         }
803                 }
804                 FreeStrBuf(&All);
805                 FreeStrBuf(&One);
806         }
807         return RelayUrls;
808 }
809 /*
810  * smtp_do_procmsg()
811  *
812  * Called by smtp_do_queue() to handle an individual message.
813  */
814 void smtp_do_procmsg(long msgnum, void *userdata) {
815         time_t now;
816         int mynumsessions = num_sessions;
817         struct CtdlMessage *msg = NULL;
818         char *Author = NULL;
819         char *Address = NULL;
820         char *instr = NULL;
821         StrBuf *PlainQItem;
822         OneQueItem *MyQItem;
823         char *pch;
824         HashPos  *It;
825         void *vQE;
826         long len;
827         const char *Key;
828         int HaveBuffers = 0;
829         StrBuf *Msg =NULL;
830
831         if (mynumsessions > max_sessions_for_outbound_smtp) {
832                 SMTPC_syslog(LOG_DEBUG,
833                              "skipping because of num jobs %d > %d max_sessions_for_outbound_smtp",
834                              mynumsessions,
835                              max_sessions_for_outbound_smtp);
836         }
837
838         SMTPC_syslog(LOG_DEBUG, "smtp_do_procmsg(%ld)\n", msgnum);
839         ///strcpy(envelope_from, "");
840
841         msg = CtdlFetchMessage(msgnum, 1);
842         if (msg == NULL) {
843                 SMTPC_syslog(LOG_ERR, "tried %ld but no such message!\n",
844                        msgnum);
845                 return;
846         }
847
848         pch = instr = msg->cm_fields['M'];
849
850         /* Strip out the headers (no not amd any other non-instruction) line */
851         while (pch != NULL) {
852                 pch = strchr(pch, '\n');
853                 if ((pch != NULL) && (*(pch + 1) == '\n')) {
854                         instr = pch + 2;
855                         pch = NULL;
856                 }
857         }
858         PlainQItem = NewStrBufPlain(instr, -1);
859         CtdlFreeMessage(msg);
860         MyQItem = DeserializeQueueItem(PlainQItem, msgnum);
861         FreeStrBuf(&PlainQItem);
862
863         if (MyQItem == NULL) {
864                 SMTPC_syslog(LOG_ERR,
865                              "Msg No %ld: already in progress!\n",
866                              msgnum);
867                 return; /* s.b. else is already processing... */
868         }
869
870         /*
871          * Postpone delivery if we've already tried recently.
872          */
873         now = time(NULL);
874         if ((MyQItem->ReattemptWhen != 0) && 
875             (now < MyQItem->ReattemptWhen) &&
876             (run_queue_now == 0))
877         {
878                 SMTPC_syslog(LOG_DEBUG, 
879                              "Retry time not yet reached. %ld seconds left.",
880                              MyQItem->ReattemptWhen - now);
881
882                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
883                 pthread_mutex_lock(&ActiveQItemsLock);
884                 {
885                         if (GetHashPosFromKey(ActiveQItems,
886                                               LKEY(MyQItem->MessageID),
887                                               It))
888                         {
889                                 DeleteEntryFromHash(ActiveQItems, It);
890                         }
891                 }
892                 pthread_mutex_unlock(&ActiveQItemsLock);
893                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
894                 DeleteHashPos(&It);
895                 return;
896         }
897
898         /*
899          * Bail out if there's no actual message associated with this
900          */
901         if (MyQItem->MessageID < 0L) {
902                 SMTPCM_syslog(LOG_ERR, "no 'msgid' directive found!\n");
903                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
904                 pthread_mutex_lock(&ActiveQItemsLock);
905                 {
906                         if (GetHashPosFromKey(ActiveQItems,
907                                               LKEY(MyQItem->MessageID),
908                                               It))
909                         {
910                                 DeleteEntryFromHash(ActiveQItems, It);
911                         }
912                 }
913                 pthread_mutex_unlock(&ActiveQItemsLock);
914                 DeleteHashPos(&It);
915                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
916                 return;
917         }
918
919
920         It = GetNewHashPos(MyQItem->MailQEntries, 0);
921         while (GetNextHashPos(MyQItem->MailQEntries, It, &len, &Key, &vQE))
922         {
923                 MailQEntry *ThisItem = vQE;
924                 SMTPC_syslog(LOG_DEBUG, "SMTP Queue: Task: <%s> %d\n",
925                              ChrPtr(ThisItem->Recipient),
926                              ThisItem->Active);
927         }
928         DeleteHashPos(&It);
929
930         MyQItem->NotYetShutdownDeliveries = 
931                 MyQItem->ActiveDeliveries = CountActiveQueueEntries(MyQItem, 1);
932
933         /* failsafe against overload: 
934          * will we exceed the limit set? 
935          */
936         if ((MyQItem->ActiveDeliveries + mynumsessions > max_sessions_for_outbound_smtp) && 
937             /* if yes, did we reach more than half of the quota? */
938             ((mynumsessions * 2) > max_sessions_for_outbound_smtp) && 
939             /* if... would we ever fit into half of the quota?? */
940             (((MyQItem->ActiveDeliveries * 2)  < max_sessions_for_outbound_smtp)))
941         {
942                 /* abort delivery for another time. */
943                 SMTPC_syslog(LOG_DEBUG,
944                              "SMTP Queue: skipping because of num jobs %d + %ld > %d max_sessions_for_outbound_smtp",
945                              mynumsessions,
946                              MyQItem->ActiveDeliveries,
947                              max_sessions_for_outbound_smtp);
948
949                 FreeQueItem(&MyQItem);
950
951                 return;
952         }
953
954
955         if (MyQItem->ActiveDeliveries > 0)
956         {
957                 ParsedURL *RelayUrls = NULL;
958                 int nActivated = 0;
959                 int n = MsgCount++;
960                 int m = MyQItem->ActiveDeliveries;
961                 int i = 1;
962
963                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
964
965                 Msg = smtp_load_msg(MyQItem, n, &Author, &Address);
966                 RelayUrls = LoadRelayUrls(MyQItem, Author, Address);
967                 if ((RelayUrls == NULL) && MyQItem->HaveRelay) {
968
969                         while ((i <= m) &&
970                                (GetNextHashPos(MyQItem->MailQEntries,
971                                                It, &len, &Key, &vQE)))
972                         {
973                                 int KeepBuffers = (i == m);
974                                 MailQEntry *ThisItem = vQE;
975                                 StrBufPrintf(ThisItem->StatusMessage,
976                                              "No relay configured matching %s / %s", 
977                                              (Author != NULL)? Author : "",
978                                              (Address != NULL)? Address : "");
979                                 ThisItem->Status = 5;
980
981                                 nActivated++;
982
983                                 if (i > 1) n = MsgCount++;
984                                 syslog(LOG_INFO,
985                                        "SMTPC: giving up on <%ld> <%s> %d / %d \n",
986                                        MyQItem->MessageID,
987                                        ChrPtr(ThisItem->Recipient),
988                                        i,
989                                        m);
990                                 (*((int*) userdata)) ++;
991                                 smtp_try_one_queue_entry(MyQItem,
992                                                          ThisItem,
993                                                          Msg,
994                                                          KeepBuffers,
995                                                          n,
996                                                          RelayUrls);
997
998                                 if (KeepBuffers) HaveBuffers = 1;
999
1000                                 i++;
1001                         }
1002                         if (Author != NULL) free (Author);
1003                         if (Address != NULL) free (Address);
1004                         DeleteHashPos(&It);
1005
1006                         return;
1007                 }
1008                 if (Author != NULL) free (Author);
1009                 if (Address != NULL) free (Address);
1010
1011                 while ((i <= m) &&
1012                        (GetNextHashPos(MyQItem->MailQEntries,
1013                                        It, &len, &Key, &vQE)))
1014                 {
1015                         MailQEntry *ThisItem = vQE;
1016
1017                         if (ThisItem->Active == 1)
1018                         {
1019                                 int KeepBuffers = (i == m);
1020
1021                                 nActivated++;
1022                                 if (nActivated % ndelay_count == 0)
1023                                         usleep(delay_msec);
1024
1025                                 if (i > 1) n = MsgCount++;
1026                                 syslog(LOG_DEBUG,
1027                                        "SMTPC: Trying <%ld> <%s> %d / %d \n",
1028                                        MyQItem->MessageID,
1029                                        ChrPtr(ThisItem->Recipient),
1030                                        i,
1031                                        m);
1032                                 (*((int*) userdata)) ++;
1033                                 smtp_try_one_queue_entry(MyQItem,
1034                                                          ThisItem,
1035                                                          Msg,
1036                                                          KeepBuffers,
1037                                                          n,
1038                                                          RelayUrls);
1039
1040                                 if (KeepBuffers) HaveBuffers = 1;
1041
1042                                 i++;
1043                         }
1044                 }
1045                 DeleteHashPos(&It);
1046         }
1047         else
1048         {
1049                 It = GetNewHashPos(MyQItem->MailQEntries, 0);
1050                 pthread_mutex_lock(&ActiveQItemsLock);
1051                 {
1052                         if (GetHashPosFromKey(ActiveQItems,
1053                                               LKEY(MyQItem->MessageID),
1054                                               It))
1055                         {
1056                                 DeleteEntryFromHash(ActiveQItems, It);
1057                         }
1058                         else
1059                         {
1060                                 long len;
1061                                 const char* Key;
1062                                 void *VData;
1063
1064                                 SMTPC_syslog(LOG_WARNING,
1065                                              "unable to find QItem with ID[%ld]",
1066                                              MyQItem->MessageID);
1067                                 while (GetNextHashPos(ActiveQItems,
1068                                                       It,
1069                                                       &len,
1070                                                       &Key,
1071                                                       &VData))
1072                                 {
1073                                         SMTPC_syslog(LOG_WARNING,
1074                                                      "have: ID[%ld]",
1075                                                      ((OneQueItem *)VData)->MessageID);
1076                                 }
1077                         }
1078
1079                 }
1080                 pthread_mutex_unlock(&ActiveQItemsLock);
1081                 DeleteHashPos(&It);
1082                 ////FreeQueItem(&MyQItem); TODO: DeleteEntryFromHash frees this?
1083
1084 // TODO: bounce & delete?
1085
1086         }
1087         if (!HaveBuffers) {
1088                 FreeStrBuf (&Msg);
1089 // TODO : free RelayUrls
1090         }
1091 }
1092
1093
1094
1095 /*
1096  * smtp_queue_thread()
1097  *
1098  * Run through the queue sending out messages.
1099  */
1100 void smtp_do_queue(void) {
1101         int num_processed = 0;
1102         int num_activated = 0;
1103
1104         pthread_setspecific(MyConKey, (void *)&smtp_queue_CC);
1105         SMTPCM_syslog(LOG_INFO, "processing outbound queue");
1106
1107         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
1108                 SMTPC_syslog(LOG_ERR, "Cannot find room <%s>", SMTP_SPOOLOUT_ROOM);
1109         }
1110         else {
1111                 num_processed = CtdlForEachMessage(MSGS_ALL,
1112                                                    0L,
1113                                                    NULL,
1114                                                    SPOOLMIME,
1115                                                    NULL,
1116                                                    smtp_do_procmsg,
1117                                                    &num_activated);
1118         }
1119         SMTPC_syslog(LOG_INFO,
1120                      "queue run completed; %d messages processed %d activated",
1121                      num_processed, num_activated);
1122
1123 }
1124
1125
1126
1127 /*
1128  * Initialize the SMTP outbound queue
1129  */
1130 void smtp_init_spoolout(void) {
1131         struct ctdlroom qrbuf;
1132
1133         /*
1134          * Create the room.  This will silently fail if the room already
1135          * exists, and that's perfectly ok, because we want it to exist.
1136          */
1137         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_QUEUE);
1138
1139         /*
1140          * Make sure it's set to be a "system room" so it doesn't show up
1141          * in the <K>nown rooms list for Aides.
1142          */
1143         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
1144                 qrbuf.QRflags2 |= QR2_SYSTEM;
1145                 CtdlPutRoomLock(&qrbuf);
1146         }
1147 }
1148
1149
1150
1151
1152 /*****************************************************************************/
1153 /*                          SMTP UTILITY COMMANDS                            */
1154 /*****************************************************************************/
1155
1156 void cmd_smtp(char *argbuf) {
1157         char cmd[64];
1158         char node[256];
1159         char buf[1024];
1160         int i;
1161         int num_mxhosts;
1162
1163         if (CtdlAccessCheck(ac_aide)) return;
1164
1165         extract_token(cmd, argbuf, 0, '|', sizeof cmd);
1166
1167         if (!strcasecmp(cmd, "mx")) {
1168                 extract_token(node, argbuf, 1, '|', sizeof node);
1169                 num_mxhosts = getmx(buf, node);
1170                 cprintf("%d %d MX hosts listed for %s\n",
1171                         LISTING_FOLLOWS, num_mxhosts, node);
1172                 for (i=0; i<num_mxhosts; ++i) {
1173                         extract_token(node, buf, i, '|', sizeof node);
1174                         cprintf("%s\n", node);
1175                 }
1176                 cprintf("000\n");
1177                 return;
1178         }
1179
1180         else if (!strcasecmp(cmd, "runqueue")) {
1181                 run_queue_now = 1;
1182                 cprintf("%d All outbound SMTP will be retried now.\n", CIT_OK);
1183                 return;
1184         }
1185
1186         else {
1187                 cprintf("%d Invalid command.\n", ERROR + ILLEGAL_VALUE);
1188         }
1189
1190 }
1191
1192
1193 CTDL_MODULE_INIT(smtp_queu)
1194 {
1195         char *pstr;
1196
1197         if (!threading)
1198         {
1199                 pstr = getenv("CITSERVER_n_session_max");
1200                 if ((pstr != NULL) && (*pstr != '\0'))
1201                         max_sessions_for_outbound_smtp = atol(pstr); /* how many sessions might be active till we stop adding more smtp jobs */
1202
1203                 pstr = getenv("CITSERVER_smtp_n_delay_count");
1204                 if ((pstr != NULL) && (*pstr != '\0'))
1205                         ndelay_count = atol(pstr); /* every n queued messages we will sleep... */
1206
1207                 pstr = getenv("CITSERVER_smtp_delay");
1208                 if ((pstr != NULL) && (*pstr != '\0'))
1209                         delay_msec = atol(pstr) * 1000; /* this many seconds. */
1210
1211
1212
1213
1214                 CtdlFillSystemContext(&smtp_queue_CC, "SMTP_Send");
1215                 ActiveQItems = NewHash(1, lFlathash);
1216                 pthread_mutex_init(&ActiveQItemsLock, NULL);
1217
1218                 QItemHandlers = NewHash(0, NULL);
1219
1220                 RegisterQItemHandler(HKEY("msgid"),             QItem_Handle_MsgID);
1221                 RegisterQItemHandler(HKEY("envelope_from"),     QItem_Handle_EnvelopeFrom);
1222                 RegisterQItemHandler(HKEY("retry"),             QItem_Handle_retry);
1223                 RegisterQItemHandler(HKEY("attempted"),         QItem_Handle_Attempted);
1224                 RegisterQItemHandler(HKEY("remote"),            QItem_Handle_Recipient);
1225                 RegisterQItemHandler(HKEY("bounceto"),          QItem_Handle_BounceTo);
1226                 RegisterQItemHandler(HKEY("source_room"),       QItem_Handle_SenderRoom);
1227                 RegisterQItemHandler(HKEY("submitted"),         QItem_Handle_Submitted);
1228
1229                 smtp_init_spoolout();
1230
1231                 CtdlRegisterEVCleanupHook(smtp_evq_cleanup);
1232
1233                 CtdlRegisterProtoHook(cmd_smtp, "SMTP", "SMTP utility commands");
1234                 CtdlRegisterSessionHook(smtp_do_queue, EVT_TIMER, PRIO_SEND + 10);
1235         }
1236
1237         /* return our Subversion id for the Log */
1238         return "smtpeventclient";
1239 }