b0572f5a1f7b96f347a4328169d56941e0447970
[citadel.git] / citadel / modules / smtp / serv_smtp.c
1 /*
2  * $Id$
3  *
4  * This module is an SMTP and ESMTP implementation for the Citadel system.
5  * It is compliant with all of the following:
6  *
7  * RFC  821 - Simple Mail Transfer Protocol
8  * RFC  876 - Survey of SMTP Implementations
9  * RFC 1047 - Duplicate messages and SMTP
10  * RFC 1652 - 8 bit MIME
11  * RFC 1869 - Extended Simple Mail Transfer Protocol
12  * RFC 1870 - SMTP Service Extension for Message Size Declaration
13  * RFC 2033 - Local Mail Transfer Protocol
14  * RFC 2197 - SMTP Service Extension for Command Pipelining
15  * RFC 2476 - Message Submission
16  * RFC 2487 - SMTP Service Extension for Secure SMTP over TLS
17  * RFC 2554 - SMTP Service Extension for Authentication
18  * RFC 2821 - Simple Mail Transfer Protocol
19  * RFC 2822 - Internet Message Format
20  * RFC 2920 - SMTP Service Extension for Command Pipelining
21  *  
22  * The VRFY and EXPN commands have been removed from this implementation
23  * because nobody uses these commands anymore, except for spammers.
24  *
25  * Copyright (c) 1998-2009 by the citadel.org team
26  *
27  *  This program is free software; you can redistribute it and/or modify
28  *  it under the terms of the GNU General Public License as published by
29  *  the Free Software Foundation; either version 3 of the License, or
30  *  (at your option) any later version.
31  *
32  *  This program is distributed in the hope that it will be useful,
33  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
34  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
35  *  GNU General Public License for more details.
36  *
37  *  You should have received a copy of the GNU General Public License
38  *  along with this program; if not, write to the Free Software
39  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
40  */
41
42 #include "sysdep.h"
43 #include <stdlib.h>
44 #include <unistd.h>
45 #include <stdio.h>
46 #include <termios.h>
47 #include <fcntl.h>
48 #include <signal.h>
49 #include <pwd.h>
50 #include <errno.h>
51 #include <sys/types.h>
52 #include <syslog.h>
53
54 #if TIME_WITH_SYS_TIME
55 # include <sys/time.h>
56 # include <time.h>
57 #else
58 # if HAVE_SYS_TIME_H
59 #  include <sys/time.h>
60 # else
61 #  include <time.h>
62 # endif
63 #endif
64
65 #include <sys/wait.h>
66 #include <ctype.h>
67 #include <string.h>
68 #include <limits.h>
69 #include <sys/socket.h>
70 #include <netinet/in.h>
71 #include <arpa/inet.h>
72 #include <libcitadel.h>
73 #include "citadel.h"
74 #include "server.h"
75 #include "citserver.h"
76 #include "support.h"
77 #include "config.h"
78 #include "control.h"
79 #include "user_ops.h"
80 #include "database.h"
81 #include "msgbase.h"
82 #include "internet_addressing.h"
83 #include "genstamp.h"
84 #include "domain.h"
85 #include "clientsocket.h"
86 #include "locate_host.h"
87 #include "citadel_dirs.h"
88
89
90
91 #ifndef HAVE_SNPRINTF
92 #include "snprintf.h"
93 #endif
94
95
96 #include "ctdl_module.h"
97
98
99
100 typedef struct _citsmtp {               /* Information about the current session */
101         int command_state;
102         char helo_node[SIZ];
103         char from[SIZ];
104         char recipients[SIZ];
105         int number_of_recipients;
106         int delivery_mode;
107         int message_originated_locally;
108         int is_lmtp;
109         int is_unfiltered;
110         int is_msa;
111 }citsmtp;
112
113 enum {                          /* Command states for login authentication */
114         smtp_command,
115         smtp_user,
116         smtp_password,
117         smtp_plain
118 };
119
120 #define SMTP            ((citsmtp *)CC->session_specific_data)
121
122
123 int run_queue_now = 0;  /* Set to 1 to ignore SMTP send retry times */
124
125 citthread_mutex_t smtp_send_lock;
126
127
128 /*****************************************************************************/
129 /*                      SMTP SERVER (INBOUND) STUFF                          */
130 /*****************************************************************************/
131
132
133 /*
134  * Here's where our SMTP session begins its happy day.
135  */
136 void smtp_greeting(int is_msa)
137 {
138         citsmtp *sSMTP;
139         char message_to_spammer[1024];
140
141         strcpy(CC->cs_clientname, "SMTP session");
142         CC->internal_pgm = 1;
143         CC->cs_flags |= CS_STEALTH;
144         CC->session_specific_data = malloc(sizeof(citsmtp));
145         memset(SMTP, 0, sizeof(citsmtp));
146         sSMTP = SMTP;
147         sSMTP->is_msa = is_msa;
148
149         /* If this config option is set, reject connections from problem
150          * addresses immediately instead of after they execute a RCPT
151          */
152         if ( (config.c_rbl_at_greeting) && (sSMTP->is_msa == 0) ) {
153                 if (rbl_check(message_to_spammer)) {
154                         if (CtdlThreadCheckStop())
155                                 cprintf("421 %s\r\n", message_to_spammer);
156                         else
157                                 cprintf("550 %s\r\n", message_to_spammer);
158                         CC->kill_me = 1;
159                         /* no need to free_recipients(valid), it's not allocated yet */
160                         return;
161                 }
162         }
163
164         /* Otherwise we're either clean or we check later. */
165
166         if (CC->nologin==1) {
167                 cprintf("500 Too many users are already online (maximum is %d)\r\n",
168                         config.c_maxsessions
169                 );
170                 CC->kill_me = 1;
171                 /* no need to free_recipients(valid), it's not allocated yet */
172                 return;
173         }
174
175         /* Note: the FQDN *must* appear as the first thing after the 220 code.
176          * Some clients (including citmail.c) depend on it being there.
177          */
178         cprintf("220 %s ESMTP Citadel server ready.\r\n", config.c_fqdn);
179 }
180
181
182 /*
183  * SMTPS is just like SMTP, except it goes crypto right away.
184  */
185 void smtps_greeting(void) {
186         CtdlModuleStartCryptoMsgs(NULL, NULL, NULL);
187 #ifdef HAVE_OPENSSL
188         if (!CC->redirect_ssl) CC->kill_me = 1;         /* kill session if no crypto */
189 #endif
190         smtp_greeting(0);
191 }
192
193
194 /*
195  * SMTP MSA port requires authentication.
196  */
197 void smtp_msa_greeting(void) {
198         smtp_greeting(1);
199 }
200
201
202 /*
203  * LMTP is like SMTP but with some extra bonus footage added.
204  */
205 void lmtp_greeting(void) {
206         citsmtp *sSMTP;
207
208         smtp_greeting(0);
209         sSMTP = SMTP;
210         SMTP->is_lmtp = 1;
211 }
212
213
214 /* 
215  * Generic SMTP MTA greeting
216  */
217 void smtp_mta_greeting(void) {
218         smtp_greeting(0);
219 }
220
221
222 /*
223  * We also have an unfiltered LMTP socket that bypasses spam filters.
224  */
225 void lmtp_unfiltered_greeting(void) {
226         citsmtp *sSMTP;
227
228         smtp_greeting(0);
229         sSMTP = SMTP;
230         sSMTP->is_lmtp = 1;
231         sSMTP->is_unfiltered = 1;
232 }
233
234
235 /*
236  * Login greeting common to all auth methods
237  */
238 void smtp_auth_greeting(void) {
239                 cprintf("235 Hello, %s\r\n", CC->user.fullname);
240                 CtdlLogPrintf(CTDL_NOTICE, "SMTP authenticated %s\n", CC->user.fullname);
241                 CC->internal_pgm = 0;
242                 CC->cs_flags &= ~CS_STEALTH;
243 }
244
245
246 /*
247  * Implement HELO and EHLO commands.
248  *
249  * which_command:  0=HELO, 1=EHLO, 2=LHLO
250  */
251 void smtp_hello(char *argbuf, int which_command) {
252         citsmtp *sSMTP = SMTP;
253
254         safestrncpy(sSMTP->helo_node, argbuf, sizeof sSMTP->helo_node);
255
256         if ( (which_command != 2) && (sSMTP->is_lmtp) ) {
257                 cprintf("500 Only LHLO is allowed when running LMTP\r\n");
258                 return;
259         }
260
261         if ( (which_command == 2) && (sSMTP->is_lmtp == 0) ) {
262                 cprintf("500 LHLO is only allowed when running LMTP\r\n");
263                 return;
264         }
265
266         if (which_command == 0) {
267                 cprintf("250 Hello %s (%s [%s])\r\n",
268                         sSMTP->helo_node,
269                         CC->cs_host,
270                         CC->cs_addr
271                 );
272         }
273         else {
274                 if (which_command == 1) {
275                         cprintf("250-Hello %s (%s [%s])\r\n",
276                                 sSMTP->helo_node,
277                                 CC->cs_host,
278                                 CC->cs_addr
279                         );
280                 }
281                 else {
282                         cprintf("250-Greetings and joyous salutations.\r\n");
283                 }
284                 cprintf("250-HELP\r\n");
285                 cprintf("250-SIZE %ld\r\n", config.c_maxmsglen);
286
287 #ifdef HAVE_OPENSSL
288                 /*
289                  * Offer TLS, but only if TLS is not already active.
290                  * Furthermore, only offer TLS when running on
291                  * the SMTP-MSA port, not on the SMTP-MTA port, due to
292                  * questionable reliability of TLS in certain sending MTA's.
293                  */
294                 if ( (!CC->redirect_ssl) && (sSMTP->is_msa) ) {
295                         cprintf("250-STARTTLS\r\n");
296                 }
297 #endif  /* HAVE_OPENSSL */
298
299                 cprintf("250-AUTH LOGIN PLAIN\r\n"
300                         "250-AUTH=LOGIN PLAIN\r\n"
301                         "250 8BITMIME\r\n"
302                 );
303         }
304 }
305
306
307
308 /*
309  * Implement HELP command.
310  */
311 void smtp_help(void) {
312         cprintf("214 RTFM http://www.ietf.org/rfc/rfc2821.txt\r\n");
313 }
314
315
316 /*
317  *
318  */
319 void smtp_get_user(char *argbuf) {
320         char buf[SIZ];
321         char username[SIZ];
322         citsmtp *sSMTP = SMTP;
323
324         CtdlDecodeBase64(username, argbuf, SIZ);
325         /* CtdlLogPrintf(CTDL_DEBUG, "Trying <%s>\n", username); */
326         if (CtdlLoginExistingUser(NULL, username) == login_ok) {
327                 CtdlEncodeBase64(buf, "Password:", 9, 0);
328                 cprintf("334 %s\r\n", buf);
329                 sSMTP->command_state = smtp_password;
330         }
331         else {
332                 cprintf("500 No such user.\r\n");
333                 sSMTP->command_state = smtp_command;
334         }
335 }
336
337
338 /*
339  *
340  */
341 void smtp_get_pass(char *argbuf) {
342         char password[SIZ];
343         long len;
344
345         memset(password, 0, sizeof(password));  
346         len = CtdlDecodeBase64(password, argbuf, SIZ);
347         /* CtdlLogPrintf(CTDL_DEBUG, "Trying <%s>\n", password); */
348         if (CtdlTryPassword(password, len) == pass_ok) {
349                 smtp_auth_greeting();
350         }
351         else {
352                 cprintf("535 Authentication failed.\r\n");
353         }
354         SMTP->command_state = smtp_command;
355 }
356
357
358 /*
359  * Back end for PLAIN auth method (either inline or multistate)
360  */
361 void smtp_try_plain(char *encoded_authstring) {
362         char decoded_authstring[1024];
363         char ident[256];
364         char user[256];
365         char pass[256];
366         int result;
367         long len;
368
369         CtdlDecodeBase64(decoded_authstring, encoded_authstring, strlen(encoded_authstring) );
370         safestrncpy(ident, decoded_authstring, sizeof ident);
371         safestrncpy(user, &decoded_authstring[strlen(ident) + 1], sizeof user);
372         len = safestrncpy(pass, &decoded_authstring[strlen(ident) + strlen(user) + 2], sizeof pass);
373         if (len == -1)
374                 len = sizeof(pass) - 1;
375
376         SMTP->command_state = smtp_command;
377
378         if (!IsEmptyStr(ident)) {
379                 result = CtdlLoginExistingUser(user, ident);
380         }
381         else {
382                 result = CtdlLoginExistingUser(NULL, user);
383         }
384
385         if (result == login_ok) {
386                 if (CtdlTryPassword(pass, len) == pass_ok) {
387                         smtp_auth_greeting();
388                         return;
389                 }
390         }
391         cprintf("504 Authentication failed.\r\n");
392 }
393
394
395 /*
396  * Attempt to perform authenticated SMTP
397  */
398 void smtp_auth(char *argbuf) {
399         char username_prompt[64];
400         char method[64];
401         char encoded_authstring[1024];
402
403         if (CC->logged_in) {
404                 cprintf("504 Already logged in.\r\n");
405                 return;
406         }
407
408         extract_token(method, argbuf, 0, ' ', sizeof method);
409
410         if (!strncasecmp(method, "login", 5) ) {
411                 if (strlen(argbuf) >= 7) {
412                         smtp_get_user(&argbuf[6]);
413                 }
414                 else {
415                         CtdlEncodeBase64(username_prompt, "Username:", 9, 0);
416                         cprintf("334 %s\r\n", username_prompt);
417                         SMTP->command_state = smtp_user;
418                 }
419                 return;
420         }
421
422         if (!strncasecmp(method, "plain", 5) ) {
423                 if (num_tokens(argbuf, ' ') < 2) {
424                         cprintf("334 \r\n");
425                         SMTP->command_state = smtp_plain;
426                         return;
427                 }
428
429                 extract_token(encoded_authstring, argbuf, 1, ' ', sizeof encoded_authstring);
430
431                 smtp_try_plain(encoded_authstring);
432                 return;
433         }
434
435         if (strncasecmp(method, "login", 5) ) {
436                 cprintf("504 Unknown authentication method.\r\n");
437                 return;
438         }
439
440 }
441
442
443 /*
444  * Implements the RSET (reset state) command.
445  * Currently this just zeroes out the state buffer.  If pointers to data
446  * allocated with malloc() are ever placed in the state buffer, we have to
447  * be sure to free() them first!
448  *
449  * Set do_response to nonzero to output the SMTP RSET response code.
450  */
451 void smtp_rset(int do_response) {
452         int is_lmtp;
453         int is_unfiltered;
454         citsmtp *sSMTP = SMTP;
455
456         /*
457          * Our entire SMTP state is discarded when a RSET command is issued,
458          * but we need to preserve this one little piece of information, so
459          * we save it for later.
460          */
461         is_lmtp = sSMTP->is_lmtp;
462         is_unfiltered = sSMTP->is_unfiltered;
463
464         memset(sSMTP, 0, sizeof(citsmtp));
465
466         /*
467          * It is somewhat ambiguous whether we want to log out when a RSET
468          * command is issued.  Here's the code to do it.  It is commented out
469          * because some clients (such as Pine) issue RSET commands before
470          * each message, but still expect to be logged in.
471          *
472          * if (CC->logged_in) {
473          *      logout(CC);
474          * }
475          */
476
477         /*
478          * Reinstate this little piece of information we saved (see above).
479          */
480         sSMTP->is_lmtp = is_lmtp;
481         sSMTP->is_unfiltered = is_unfiltered;
482
483         if (do_response) {
484                 cprintf("250 Zap!\r\n");
485         }
486 }
487
488 /*
489  * Clear out the portions of the state buffer that need to be cleared out
490  * after the DATA command finishes.
491  */
492 void smtp_data_clear(void) {
493         citsmtp *sSMTP = SMTP;
494
495         strcpy(sSMTP->from, "");
496         strcpy(sSMTP->recipients, "");
497         sSMTP->number_of_recipients = 0;
498         sSMTP->delivery_mode = 0;
499         sSMTP->message_originated_locally = 0;
500 }
501
502 const char *smtp_get_Recipients(void)
503 {
504         citsmtp *sSMTP = SMTP;
505
506         if (sSMTP == NULL)
507                 return NULL;
508         else return sSMTP->from;
509 }
510
511 /*
512  * Implements the "MAIL FROM:" command
513  */
514 void smtp_mail(char *argbuf) {
515         char user[SIZ];
516         char node[SIZ];
517         char name[SIZ];
518         citsmtp *sSMTP = SMTP;
519
520         if (!IsEmptyStr(sSMTP->from)) {
521                 cprintf("503 Only one sender permitted\r\n");
522                 return;
523         }
524
525         if (strncasecmp(argbuf, "From:", 5)) {
526                 cprintf("501 Syntax error\r\n");
527                 return;
528         }
529
530         strcpy(sSMTP->from, &argbuf[5]);
531         striplt(sSMTP->from);
532         if (haschar(sSMTP->from, '<') > 0) {
533                 stripallbut(sSMTP->from, '<', '>');
534         }
535
536         /* We used to reject empty sender names, until it was brought to our
537          * attention that RFC1123 5.2.9 requires that this be allowed.  So now
538          * we allow it, but replace the empty string with a fake
539          * address so we don't have to contend with the empty string causing
540          * other code to fail when it's expecting something there.
541          */
542         if (IsEmptyStr(sSMTP->from)) {
543                 strcpy(sSMTP->from, "someone@example.com");
544         }
545
546         /* If this SMTP connection is from a logged-in user, force the 'from'
547          * to be the user's Internet e-mail address as Citadel knows it.
548          */
549         if (CC->logged_in) {
550                 safestrncpy(sSMTP->from, CC->cs_inet_email, sizeof sSMTP->from);
551                 cprintf("250 Sender ok <%s>\r\n", sSMTP->from);
552                 sSMTP->message_originated_locally = 1;
553                 return;
554         }
555
556         else if (sSMTP->is_lmtp) {
557                 /* Bypass forgery checking for LMTP */
558         }
559
560         /* Otherwise, make sure outsiders aren't trying to forge mail from
561          * this system (unless, of course, c_allow_spoofing is enabled)
562          */
563         else if (config.c_allow_spoofing == 0) {
564                 process_rfc822_addr(sSMTP->from, user, node, name);
565                 if (CtdlHostAlias(node) != hostalias_nomatch) {
566                         cprintf("550 You must log in to send mail from %s\r\n", node);
567                         strcpy(sSMTP->from, "");
568                         return;
569                 }
570         }
571
572         cprintf("250 Sender ok\r\n");
573 }
574
575
576
577 /*
578  * Implements the "RCPT To:" command
579  */
580 void smtp_rcpt(char *argbuf) {
581         char recp[1024];
582         char message_to_spammer[SIZ];
583         struct recptypes *valid = NULL;
584         citsmtp *sSMTP = SMTP;
585
586         if (IsEmptyStr(sSMTP->from)) {
587                 cprintf("503 Need MAIL before RCPT\r\n");
588                 return;
589         }
590
591         if (strncasecmp(argbuf, "To:", 3)) {
592                 cprintf("501 Syntax error\r\n");
593                 return;
594         }
595
596         if ( (sSMTP->is_msa) && (!CC->logged_in) ) {
597                 cprintf("550 You must log in to send mail on this port.\r\n");
598                 strcpy(sSMTP->from, "");
599                 return;
600         }
601
602         safestrncpy(recp, &argbuf[3], sizeof recp);
603         striplt(recp);
604         stripallbut(recp, '<', '>');
605
606         if ( (strlen(recp) + strlen(sSMTP->recipients) + 1 ) >= SIZ) {
607                 cprintf("452 Too many recipients\r\n");
608                 return;
609         }
610
611         /* RBL check */
612         if ( (!CC->logged_in)   /* Don't RBL authenticated users */
613            && (!sSMTP->is_lmtp) ) {     /* Don't RBL LMTP clients */
614                 if (config.c_rbl_at_greeting == 0) {    /* Don't RBL again if we already did it */
615                         if (rbl_check(message_to_spammer)) {
616                                 if (CtdlThreadCheckStop())
617                                         cprintf("421 %s\r\n", message_to_spammer);
618                                 else
619                                         cprintf("550 %s\r\n", message_to_spammer);
620                                 /* no need to free_recipients(valid), it's not allocated yet */
621                                 return;
622                         }
623                 }
624         }
625
626         valid = validate_recipients(recp, 
627                                     smtp_get_Recipients (),
628                                     (sSMTP->is_lmtp)? POST_LMTP:
629                                        (CC->logged_in)? POST_LOGGED_IN:
630                                                         POST_EXTERNAL);
631         if (valid->num_error != 0) {
632                 cprintf("550 %s\r\n", valid->errormsg);
633                 free_recipients(valid);
634                 return;
635         }
636
637         if (valid->num_internet > 0) {
638                 if (CC->logged_in) {
639                         if (CtdlCheckInternetMailPermission(&CC->user)==0) {
640                                 cprintf("551 <%s> - you do not have permission to send Internet mail\r\n", recp);
641                                 free_recipients(valid);
642                                 return;
643                         }
644                 }
645         }
646
647         if (valid->num_internet > 0) {
648                 if ( (sSMTP->message_originated_locally == 0)
649                    && (sSMTP->is_lmtp == 0) ) {
650                         cprintf("551 <%s> - relaying denied\r\n", recp);
651                         free_recipients(valid);
652                         return;
653                 }
654         }
655
656         cprintf("250 RCPT ok <%s>\r\n", recp);
657         if (!IsEmptyStr(sSMTP->recipients)) {
658                 strcat(sSMTP->recipients, ",");
659         }
660         strcat(sSMTP->recipients, recp);
661         sSMTP->number_of_recipients += 1;
662         if (valid != NULL)  {
663                 free_recipients(valid);
664         }
665 }
666
667
668
669
670 /*
671  * Implements the DATA command
672  */
673 void smtp_data(void) {
674         StrBuf *body;
675         char *defbody; //TODO: remove me
676         struct CtdlMessage *msg = NULL;
677         long msgnum = (-1L);
678         char nowstamp[SIZ];
679         struct recptypes *valid;
680         int scan_errors;
681         int i;
682         char result[SIZ];
683         citsmtp *sSMTP = SMTP;
684
685         if (IsEmptyStr(sSMTP->from)) {
686                 cprintf("503 Need MAIL command first.\r\n");
687                 return;
688         }
689
690         if (sSMTP->number_of_recipients < 1) {
691                 cprintf("503 Need RCPT command first.\r\n");
692                 return;
693         }
694
695         cprintf("354 Transmit message now - terminate with '.' by itself\r\n");
696         
697         datestring(nowstamp, sizeof nowstamp, time(NULL), DATESTRING_RFC822);
698         defbody = malloc(4096);
699
700         if (defbody != NULL) {
701                 if (sSMTP->is_lmtp && (CC->cs_UDSclientUID != -1)) {
702                         snprintf(defbody, 4096,
703                                "Received: from %s (Citadel from userid %ld)\n"
704                                "        by %s; %s\n",
705                                sSMTP->helo_node,
706                                (long int) CC->cs_UDSclientUID,
707                                config.c_fqdn,
708                                nowstamp);
709                 }
710                 else {
711                         snprintf(defbody, 4096,
712                                  "Received: from %s (%s [%s])\n"
713                                  "      by %s; %s\n",
714                                  sSMTP->helo_node,
715                                  CC->cs_host,
716                                  CC->cs_addr,
717                                  config.c_fqdn,
718                                  nowstamp);
719                 }
720         }
721         body = CtdlReadMessageBodyBuf(HKEY("."), config.c_maxmsglen, defbody, 1, NULL);
722         if (body == NULL) {
723                 cprintf("550 Unable to save message: internal error.\r\n");
724                 return;
725         }
726
727         CtdlLogPrintf(CTDL_DEBUG, "Converting message...\n");
728         msg = convert_internet_message_buf(&body);
729
730         /* If the user is locally authenticated, FORCE the From: header to
731          * show up as the real sender.  Yes, this violates the RFC standard,
732          * but IT MAKES SENSE.  If you prefer strict RFC adherence over
733          * common sense, you can disable this in the configuration.
734          *
735          * We also set the "message room name" ('O' field) to MAILROOM
736          * (which is Mail> on most systems) to prevent it from getting set
737          * to something ugly like "0000058008.Sent Items>" when the message
738          * is read with a Citadel client.
739          */
740         if ( (CC->logged_in) && (config.c_rfc822_strict_from == 0) ) {
741                 if (msg->cm_fields['A'] != NULL) free(msg->cm_fields['A']);
742                 if (msg->cm_fields['N'] != NULL) free(msg->cm_fields['N']);
743                 if (msg->cm_fields['H'] != NULL) free(msg->cm_fields['H']);
744                 if (msg->cm_fields['F'] != NULL) free(msg->cm_fields['F']);
745                 if (msg->cm_fields['O'] != NULL) free(msg->cm_fields['O']);
746                 msg->cm_fields['A'] = strdup(CC->user.fullname);
747                 msg->cm_fields['N'] = strdup(config.c_nodename);
748                 msg->cm_fields['H'] = strdup(config.c_humannode);
749                 msg->cm_fields['F'] = strdup(CC->cs_inet_email);
750                 msg->cm_fields['O'] = strdup(MAILROOM);
751         }
752
753         /* Set the "envelope from" address */
754         if (msg->cm_fields['P'] != NULL) {
755                 free(msg->cm_fields['P']);
756         }
757         msg->cm_fields['P'] = strdup(sSMTP->from);
758
759         /* Set the "envelope to" address */
760         if (msg->cm_fields['V'] != NULL) {
761                 free(msg->cm_fields['V']);
762         }
763         msg->cm_fields['V'] = strdup(sSMTP->recipients);
764
765         /* Submit the message into the Citadel system. */
766         valid = validate_recipients(sSMTP->recipients, 
767                                     smtp_get_Recipients (),
768                                     (sSMTP->is_lmtp)? POST_LMTP:
769                                        (CC->logged_in)? POST_LOGGED_IN:
770                                                         POST_EXTERNAL);
771
772         /* If there are modules that want to scan this message before final
773          * submission (such as virus checkers or spam filters), call them now
774          * and give them an opportunity to reject the message.
775          */
776         if (sSMTP->is_unfiltered) {
777                 scan_errors = 0;
778         }
779         else {
780                 scan_errors = PerformMessageHooks(msg, EVT_SMTPSCAN);
781         }
782
783         if (scan_errors > 0) {  /* We don't want this message! */
784
785                 if (msg->cm_fields['0'] == NULL) {
786                         msg->cm_fields['0'] = strdup("Message rejected by filter");
787                 }
788
789                 sprintf(result, "550 %s\r\n", msg->cm_fields['0']);
790         }
791         
792         else {                  /* Ok, we'll accept this message. */
793                 msgnum = CtdlSubmitMsg(msg, valid, "", 0);
794                 if (msgnum > 0L) {
795                         sprintf(result, "250 Message accepted.\r\n");
796                 }
797                 else {
798                         sprintf(result, "550 Internal delivery error\r\n");
799                 }
800         }
801
802         /* For SMTP and ESTMP, just print the result message.  For LMTP, we
803          * have to print one result message for each recipient.  Since there
804          * is nothing in Citadel which would cause different recipients to
805          * have different results, we can get away with just spitting out the
806          * same message once for each recipient.
807          */
808         if (sSMTP->is_lmtp) {
809                 for (i=0; i<sSMTP->number_of_recipients; ++i) {
810                         cprintf("%s", result);
811                 }
812         }
813         else {
814                 cprintf("%s", result);
815         }
816
817         /* Write something to the syslog (which may or may not be where the
818          * rest of the Citadel logs are going; some sysadmins want LOG_MAIL).
819          */
820         if (enable_syslog) {
821                 syslog((LOG_MAIL | LOG_INFO),
822                         "%ld: from=<%s>, nrcpts=%d, relay=%s [%s], stat=%s",
823                         msgnum,
824                         sSMTP->from,
825                         sSMTP->number_of_recipients,
826                         CC->cs_host,
827                         CC->cs_addr,
828                         result
829                 );
830         }
831
832         /* Clean up */
833         CtdlFreeMessage(msg);
834         free_recipients(valid);
835         smtp_data_clear();      /* clear out the buffers now */
836 }
837
838
839 /*
840  * implements the STARTTLS command (Citadel API version)
841  */
842 void smtp_starttls(void)
843 {
844         char ok_response[SIZ];
845         char nosup_response[SIZ];
846         char error_response[SIZ];
847
848         sprintf(ok_response,
849                 "220 Begin TLS negotiation now\r\n");
850         sprintf(nosup_response,
851                 "554 TLS not supported here\r\n");
852         sprintf(error_response,
853                 "554 Internal error\r\n");
854         CtdlModuleStartCryptoMsgs(ok_response, nosup_response, error_response);
855         smtp_rset(0);
856 }
857
858
859
860 /* 
861  * Main command loop for SMTP sessions.
862  */
863 void smtp_command_loop(void) {
864         char cmdbuf[SIZ];
865         citsmtp *sSMTP = SMTP;
866
867         if (sSMTP == NULL) {
868                 CtdlLogPrintf(CTDL_EMERG, "Session SMTP data is null.  WTF?  We will crash now.\n");
869         }
870
871         time(&CC->lastcmd);
872         memset(cmdbuf, 0, sizeof cmdbuf); /* Clear it, just in case */
873         if (client_getln(cmdbuf, sizeof cmdbuf) < 1) {
874                 CtdlLogPrintf(CTDL_CRIT, "Client disconnected: ending session.\n");
875                 CC->kill_me = 1;
876                 return;
877         }
878         CtdlLogPrintf(CTDL_INFO, "SMTP server: %s\n", cmdbuf);
879         while (strlen(cmdbuf) < 5) strcat(cmdbuf, " ");
880
881         if (sSMTP->command_state == smtp_user) {
882                 smtp_get_user(cmdbuf);
883         }
884
885         else if (sSMTP->command_state == smtp_password) {
886                 smtp_get_pass(cmdbuf);
887         }
888
889         else if (sSMTP->command_state == smtp_plain) {
890                 smtp_try_plain(cmdbuf);
891         }
892
893         else if (!strncasecmp(cmdbuf, "AUTH", 4)) {
894                 smtp_auth(&cmdbuf[5]);
895         }
896
897         else if (!strncasecmp(cmdbuf, "DATA", 4)) {
898                 smtp_data();
899         }
900
901         else if (!strncasecmp(cmdbuf, "HELO", 4)) {
902                 smtp_hello(&cmdbuf[5], 0);
903         }
904
905         else if (!strncasecmp(cmdbuf, "EHLO", 4)) {
906                 smtp_hello(&cmdbuf[5], 1);
907         }
908
909         else if (!strncasecmp(cmdbuf, "LHLO", 4)) {
910                 smtp_hello(&cmdbuf[5], 2);
911         }
912
913         else if (!strncasecmp(cmdbuf, "HELP", 4)) {
914                 smtp_help();
915         }
916
917         else if (!strncasecmp(cmdbuf, "MAIL", 4)) {
918                 smtp_mail(&cmdbuf[5]);
919         }
920
921         else if (!strncasecmp(cmdbuf, "NOOP", 4)) {
922                 cprintf("250 NOOP\r\n");
923         }
924
925         else if (!strncasecmp(cmdbuf, "QUIT", 4)) {
926                 cprintf("221 Goodbye...\r\n");
927                 CC->kill_me = 1;
928                 return;
929         }
930
931         else if (!strncasecmp(cmdbuf, "RCPT", 4)) {
932                 smtp_rcpt(&cmdbuf[5]);
933         }
934
935         else if (!strncasecmp(cmdbuf, "RSET", 4)) {
936                 smtp_rset(1);
937         }
938 #ifdef HAVE_OPENSSL
939         else if (!strcasecmp(cmdbuf, "STARTTLS")) {
940                 smtp_starttls();
941         }
942 #endif
943         else {
944                 cprintf("502 I'm afraid I can't do that.\r\n");
945         }
946
947
948 }
949
950
951
952
953 /*****************************************************************************/
954 /*               SMTP CLIENT (OUTBOUND PROCESSING) STUFF                     */
955 /*****************************************************************************/
956
957
958
959 /*
960  * smtp_try()
961  *
962  * Called by smtp_do_procmsg() to attempt delivery to one SMTP host
963  *
964  */
965 void smtp_try(const char *key, const char *addr, int *status,
966               char *dsn, size_t n, long msgnum, char *envelope_from)
967 {
968         int sock = (-1);
969         char mxhosts[1024];
970         int num_mxhosts;
971         int mx;
972         int i;
973         char user[1024], node[1024], name[1024];
974         char buf[1024];
975         char mailfrom[1024];
976         char mx_user[256];
977         char mx_pass[256];
978         char mx_host[256];
979         char mx_port[256];
980         int lp, rp;
981         char *msgtext;
982         const char *ptr;
983         size_t msg_size;
984         int scan_done;
985         CitContext *CCC=CC;
986         
987         
988         /* Parse out the host portion of the recipient address */
989         process_rfc822_addr(addr, user, node, name);
990
991         CtdlLogPrintf(CTDL_DEBUG, "SMTP client: Attempting delivery to <%s> @ <%s> (%s)\n",
992                 user, node, name);
993
994         /* Load the message out of the database */
995         CCC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
996         CtdlOutputMsg(msgnum, MT_RFC822, HEADERS_ALL, 0, 1, NULL, ESC_DOT);
997         msg_size = StrLength(CC->redirect_buffer);
998         msgtext = SmashStrBuf(&CC->redirect_buffer);
999
1000         /* If no envelope_from is supplied, extract one from the message */
1001         if ( (envelope_from == NULL) || (IsEmptyStr(envelope_from)) ) {
1002                 strcpy(mailfrom, "");
1003                 scan_done = 0;
1004                 ptr = msgtext;
1005                 do {
1006                         if (ptr = cmemreadline(ptr, buf, sizeof buf), *ptr == 0) {
1007                                 scan_done = 1;
1008                         }
1009                         if (!strncasecmp(buf, "From:", 5)) {
1010                                 safestrncpy(mailfrom, &buf[5], sizeof mailfrom);
1011                                 striplt(mailfrom);
1012                                 for (i=0; mailfrom[i]; ++i) {
1013                                         if (!isprint(mailfrom[i])) {
1014                                                 strcpy(&mailfrom[i], &mailfrom[i+1]);
1015                                                 i=0;
1016                                         }
1017                                 }
1018         
1019                                 /* Strip out parenthesized names */
1020                                 lp = (-1);
1021                                 rp = (-1);
1022                                 for (i=0; mailfrom[i]; ++i) {
1023                                         if (mailfrom[i] == '(') lp = i;
1024                                         if (mailfrom[i] == ')') rp = i;
1025                                 }
1026                                 if ((lp>0)&&(rp>lp)) {
1027                                         strcpy(&mailfrom[lp-1], &mailfrom[rp+1]);
1028                                 }
1029         
1030                                 /* Prefer brokketized names */
1031                                 lp = (-1);
1032                                 rp = (-1);
1033                                 for (i=0; mailfrom[i]; ++i) {
1034                                         if (mailfrom[i] == '<') lp = i;
1035                                         if (mailfrom[i] == '>') rp = i;
1036                                 }
1037                                 if ( (lp>=0) && (rp>lp) ) {
1038                                         mailfrom[rp] = 0;
1039                                         strcpy(mailfrom, &mailfrom[lp]);
1040                                 }
1041         
1042                                 scan_done = 1;
1043                         }
1044                 } while (scan_done == 0);
1045                 if (IsEmptyStr(mailfrom)) strcpy(mailfrom, "someone@somewhere.org");
1046                 stripallbut(mailfrom, '<', '>');
1047                 envelope_from = mailfrom;
1048         }
1049
1050         /* Figure out what mail exchanger host we have to connect to */
1051         num_mxhosts = getmx(mxhosts, node);
1052         CtdlLogPrintf(CTDL_DEBUG, "Number of MX hosts for <%s> is %d [%s]\n", node, num_mxhosts, mxhosts);
1053         if (num_mxhosts < 1) {
1054                 *status = 5;
1055                 snprintf(dsn, SIZ, "No MX hosts found for <%s>", node);
1056                 return;
1057         }
1058
1059         sock = (-1);
1060         for (mx=0; (mx<num_mxhosts && sock < 0); ++mx) {
1061                 char *endpart;
1062                 extract_token(buf, mxhosts, mx, '|', sizeof buf);
1063                 strcpy(mx_user, "");
1064                 strcpy(mx_pass, "");
1065                 if (num_tokens(buf, '@') > 1) {
1066                         strcpy (mx_user, buf);
1067                         endpart = strrchr(mx_user, '@');
1068                         *endpart = '\0';
1069                         strcpy (mx_host, endpart + 1);
1070                         endpart = strrchr(mx_user, ':');
1071                         if (endpart != NULL) {
1072                                 strcpy(mx_pass, endpart+1);
1073                                 *endpart = '\0';
1074                         }
1075                 }
1076                 else
1077                         strcpy (mx_host, buf);
1078                 endpart = strrchr(mx_host, ':');
1079                 if (endpart != 0){
1080                         *endpart = '\0';
1081                         strcpy(mx_port, endpart + 1);
1082                 }               
1083                 else {
1084                         strcpy(mx_port, "25");
1085                 }
1086                 CtdlLogPrintf(CTDL_DEBUG, "SMTP client: connecting to %s : %s ...\n", mx_host, mx_port);
1087                 sock = sock_connect(mx_host, mx_port, "tcp");
1088                 snprintf(dsn, SIZ, "Could not connect: %s", strerror(errno));
1089                 if (sock >= 0) 
1090                 {
1091                         CtdlLogPrintf(CTDL_DEBUG, "SMTP client: connected!\n");
1092                                 int fdflags; 
1093                                 fdflags = fcntl(sock, F_GETFL);
1094                                 if (fdflags < 0)
1095                                         CtdlLogPrintf(CTDL_DEBUG,
1096                                                       "unable to get SMTP-Client socket flags! %s \n",
1097                                                       strerror(errno));
1098                                 fdflags = fdflags | O_NONBLOCK;
1099                                 if (fcntl(sock, F_SETFL, fdflags) < 0)
1100                                         CtdlLogPrintf(CTDL_DEBUG,
1101                                                       "unable to set SMTP-Client socket nonblocking flags! %s \n",
1102                                                       strerror(errno));
1103                 }
1104                 if (sock < 0) {
1105                         if (errno > 0) {
1106                                 snprintf(dsn, SIZ, "%s", strerror(errno));
1107                         }
1108                         else {
1109                                 snprintf(dsn, SIZ, "Unable to connect to %s : %s\n", mx_host, mx_port);
1110                         }
1111                 }
1112         }
1113
1114         if (sock < 0) {
1115                 *status = 4;    /* dsn is already filled in */
1116                 return;
1117         }
1118
1119         CCC->sReadBuf = NewStrBuf();
1120         CCC->sMigrateBuf = NewStrBuf();
1121         CCC->sPos = NULL;
1122
1123         /* Process the SMTP greeting from the server */
1124         if (ml_sock_gets(&sock, buf, 90) < 0) {
1125                 *status = 4;
1126                 strcpy(dsn, "Connection broken during SMTP conversation");
1127                 goto bail;
1128         }
1129         CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf);
1130         if (buf[0] != '2') {
1131                 if (buf[0] == '4') {
1132                         *status = 4;
1133                         safestrncpy(dsn, &buf[4], 1023);
1134                         goto bail;
1135                 }
1136                 else {
1137                         *status = 5;
1138                         safestrncpy(dsn, &buf[4], 1023);
1139                         goto bail;
1140                 }
1141         }
1142
1143         /* At this point we know we are talking to a real SMTP server */
1144
1145         /* Do a EHLO command.  If it fails, try the HELO command. */
1146         snprintf(buf, sizeof buf, "EHLO %s\r\n", config.c_fqdn);
1147         CtdlLogPrintf(CTDL_DEBUG, ">%s", buf);
1148         sock_write(&sock, buf, strlen(buf));
1149         if (ml_sock_gets(&sock, buf, 30) < 0) {
1150                 *status = 4;
1151                 strcpy(dsn, "Connection broken during SMTP HELO");
1152                 goto bail;
1153         }
1154         CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf);
1155         if (buf[0] != '2') {
1156                 snprintf(buf, sizeof buf, "HELO %s\r\n", config.c_fqdn);
1157                 CtdlLogPrintf(CTDL_DEBUG, ">%s", buf);
1158                 sock_write(&sock, buf, strlen(buf));
1159                 if (ml_sock_gets(&sock, buf, 30) < 0) {
1160                         *status = 4;
1161                         strcpy(dsn, "Connection broken during SMTP HELO");
1162                         goto bail;
1163                 }
1164         }
1165         if (buf[0] != '2') {
1166                 if (buf[0] == '4') {
1167                         *status = 4;
1168                         safestrncpy(dsn, &buf[4], 1023);
1169                         goto bail;
1170                 }
1171                 else {
1172                         *status = 5;
1173                         safestrncpy(dsn, &buf[4], 1023);
1174                         goto bail;
1175                 }
1176         }
1177
1178         /* Do an AUTH command if necessary */
1179         if (!IsEmptyStr(mx_user)) {
1180                 char encoded[1024];
1181                 sprintf(buf, "%s%c%s%c%s", mx_user, '\0', mx_user, '\0', mx_pass);
1182                 CtdlEncodeBase64(encoded, buf, strlen(mx_user) + strlen(mx_user) + strlen(mx_pass) + 2, 0);
1183                 snprintf(buf, sizeof buf, "AUTH PLAIN %s\r\n", encoded);
1184                 CtdlLogPrintf(CTDL_DEBUG, ">%s", buf);
1185                 sock_write(&sock, buf, strlen(buf));
1186                 if (ml_sock_gets(&sock, buf, 30) < 0) {
1187                         *status = 4;
1188                         strcpy(dsn, "Connection broken during SMTP AUTH");
1189                         goto bail;
1190                 }
1191                 CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf);
1192                 if (buf[0] != '2') {
1193                         if (buf[0] == '4') {
1194                                 *status = 4;
1195                                 safestrncpy(dsn, &buf[4], 1023);
1196                                 goto bail;
1197                         }
1198                         else {
1199                                 *status = 5;
1200                                 safestrncpy(dsn, &buf[4], 1023);
1201                                 goto bail;
1202                         }
1203                 }
1204         }
1205
1206         /* previous command succeeded, now try the MAIL FROM: command */
1207         snprintf(buf, sizeof buf, "MAIL FROM:<%s>\r\n", envelope_from);
1208         CtdlLogPrintf(CTDL_DEBUG, ">%s", buf);
1209         sock_write(&sock, buf, strlen(buf));
1210         if (ml_sock_gets(&sock, buf, 30) < 0) {
1211                 *status = 4;
1212                 strcpy(dsn, "Connection broken during SMTP MAIL");
1213                 goto bail;
1214         }
1215         CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf);
1216         if (buf[0] != '2') {
1217                 if (buf[0] == '4') {
1218                         *status = 4;
1219                         safestrncpy(dsn, &buf[4], 1023);
1220                         goto bail;
1221                 }
1222                 else {
1223                         *status = 5;
1224                         safestrncpy(dsn, &buf[4], 1023);
1225                         goto bail;
1226                 }
1227         }
1228
1229         /* MAIL succeeded, now try the RCPT To: command */
1230         snprintf(buf, sizeof buf, "RCPT TO:<%s@%s>\r\n", user, node);
1231         CtdlLogPrintf(CTDL_DEBUG, ">%s", buf);
1232         sock_write(&sock, buf, strlen(buf));
1233         if (ml_sock_gets(&sock, buf, 30) < 0) {
1234                 *status = 4;
1235                 strcpy(dsn, "Connection broken during SMTP RCPT");
1236                 goto bail;
1237         }
1238         CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf);
1239         if (buf[0] != '2') {
1240                 if (buf[0] == '4') {
1241                         *status = 4;
1242                         safestrncpy(dsn, &buf[4], 1023);
1243                         goto bail;
1244                 }
1245                 else {
1246                         *status = 5;
1247                         safestrncpy(dsn, &buf[4], 1023);
1248                         goto bail;
1249                 }
1250         }
1251
1252         /* RCPT succeeded, now try the DATA command */
1253         CtdlLogPrintf(CTDL_DEBUG, ">DATA\n");
1254         sock_write(&sock, "DATA\r\n", 6);
1255         if (ml_sock_gets(&sock, buf, 30) < 0) {
1256                 *status = 4;
1257                 strcpy(dsn, "Connection broken during SMTP DATA");
1258                 goto bail;
1259         }
1260         CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf);
1261         if (buf[0] != '3') {
1262                 if (buf[0] == '4') {
1263                         *status = 3;
1264                         safestrncpy(dsn, &buf[4], 1023);
1265                         goto bail;
1266                 }
1267                 else {
1268                         *status = 5;
1269                         safestrncpy(dsn, &buf[4], 1023);
1270                         goto bail;
1271                 }
1272         }
1273
1274         /* If we reach this point, the server is expecting data.*/
1275         sock_write(&sock, msgtext, msg_size);
1276         if (msgtext[msg_size-1] != 10) {
1277                 CtdlLogPrintf(CTDL_WARNING, "Possible problem: message did not "
1278                         "correctly terminate. (expecting 0x10, got 0x%02x)\n",
1279                                 buf[msg_size-1]);
1280                 sock_write(&sock, "\r\n", 2);
1281         }
1282
1283         sock_write(&sock, ".\r\n", 3);
1284         tcdrain(sock);
1285         if (ml_sock_gets(&sock, buf, 90) < 0) {
1286                 *status = 4;
1287                 strcpy(dsn, "Connection broken during SMTP message transmit");
1288                 goto bail;
1289         }
1290         CtdlLogPrintf(CTDL_DEBUG, "%s\n", buf);
1291         if (buf[0] != '2') {
1292                 if (buf[0] == '4') {
1293                         *status = 4;
1294                         safestrncpy(dsn, &buf[4], 1023);
1295                         goto bail;
1296                 }
1297                 else {
1298                         *status = 5;
1299                         safestrncpy(dsn, &buf[4], 1023);
1300                         goto bail;
1301                 }
1302         }
1303
1304         /* We did it! */
1305         safestrncpy(dsn, &buf[4], 1023);
1306         *status = 2;
1307
1308         CtdlLogPrintf(CTDL_DEBUG, ">QUIT\n");
1309         sock_write(&sock, "QUIT\r\n", 6);
1310         ml_sock_gets(&sock, buf, 30);
1311         CtdlLogPrintf(CTDL_DEBUG, "<%s\n", buf);
1312         CtdlLogPrintf(CTDL_INFO, "SMTP client: delivery to <%s> @ <%s> (%s) succeeded\n",
1313                 user, node, name);
1314
1315 bail:   free(msgtext);
1316         FreeStrBuf(&CCC->sReadBuf);
1317         FreeStrBuf(&CCC->sMigrateBuf);
1318         if (sock != -1)
1319                 sock_close(sock);
1320
1321         /* Write something to the syslog (which may or may not be where the
1322          * rest of the Citadel logs are going; some sysadmins want LOG_MAIL).
1323          */
1324         if (enable_syslog) {
1325                 syslog((LOG_MAIL | LOG_INFO),
1326                         "%ld: to=<%s>, relay=%s, stat=%s",
1327                         msgnum,
1328                         addr,
1329                         mx_host,
1330                         dsn
1331                 );
1332         }
1333
1334         return;
1335 }
1336
1337
1338
1339 /*
1340  * smtp_do_bounce() is caled by smtp_do_procmsg() to scan a set of delivery
1341  * instructions for "5" codes (permanent fatal errors) and produce/deliver
1342  * a "bounce" message (delivery status notification).
1343  */
1344 void smtp_do_bounce(char *instr) {
1345         int i;
1346         int lines;
1347         int status;
1348         char buf[1024];
1349         char key[1024];
1350         char addr[1024];
1351         char dsn[1024];
1352         char bounceto[1024];
1353         StrBuf *boundary;
1354         int num_bounces = 0;
1355         int bounce_this = 0;
1356         long bounce_msgid = (-1);
1357         time_t submitted = 0L;
1358         struct CtdlMessage *bmsg = NULL;
1359         int give_up = 0;
1360         struct recptypes *valid;
1361         int successful_bounce = 0;
1362         static int seq = 0;
1363         StrBuf *BounceMB;
1364         long omsgid = (-1);
1365
1366         CtdlLogPrintf(CTDL_DEBUG, "smtp_do_bounce() called\n");
1367         strcpy(bounceto, "");
1368         boundary = NewStrBufPlain(HKEY("=_Citadel_Multipart_"));
1369         StrBufAppendPrintf(boundary, "%s_%04x%04x", config.c_fqdn, getpid(), ++seq);
1370         lines = num_tokens(instr, '\n');
1371
1372         /* See if it's time to give up on delivery of this message */
1373         for (i=0; i<lines; ++i) {
1374                 extract_token(buf, instr, i, '\n', sizeof buf);
1375                 extract_token(key, buf, 0, '|', sizeof key);
1376                 extract_token(addr, buf, 1, '|', sizeof addr);
1377                 if (!strcasecmp(key, "submitted")) {
1378                         submitted = atol(addr);
1379                 }
1380         }
1381
1382         if ( (time(NULL) - submitted) > SMTP_GIVE_UP ) {
1383                 give_up = 1;
1384         }
1385
1386         /* Start building our bounce message */
1387
1388         bmsg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
1389         if (bmsg == NULL) return;
1390         memset(bmsg, 0, sizeof(struct CtdlMessage));
1391         BounceMB = NewStrBufPlain(NULL, 1024);
1392
1393         bmsg->cm_magic = CTDLMESSAGE_MAGIC;
1394         bmsg->cm_anon_type = MES_NORMAL;
1395         bmsg->cm_format_type = FMT_RFC822;
1396         bmsg->cm_fields['A'] = strdup("Citadel");
1397         bmsg->cm_fields['O'] = strdup(MAILROOM);
1398         bmsg->cm_fields['N'] = strdup(config.c_nodename);
1399         bmsg->cm_fields['U'] = strdup("Delivery Status Notification (Failure)");
1400         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: multipart/mixed; boundary=\""), 0);
1401         StrBufAppendBuf(BounceMB, boundary, 0);
1402         StrBufAppendBufPlain(BounceMB, HKEY("\"\r\n"), 0);
1403         StrBufAppendBufPlain(BounceMB, HKEY("MIME-Version: 1.0\r\n"), 0);
1404         StrBufAppendBufPlain(BounceMB, HKEY("X-Mailer: " CITADEL "\r\n"), 0);
1405         StrBufAppendBufPlain(BounceMB, HKEY("\r\nThis is a multipart message in MIME format.\r\n\r\n"), 0);
1406         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
1407         StrBufAppendBuf(BounceMB, boundary, 0);
1408         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
1409         StrBufAppendBufPlain(BounceMB, HKEY("Content-type: text/plain\r\n\r\n"), 0);
1410
1411         if (give_up) StrBufAppendBufPlain(BounceMB, HKEY(
1412 "A message you sent could not be delivered to some or all of its recipients\n"
1413 "due to prolonged unavailability of its destination(s).\n"
1414 "Giving up on the following addresses:\n\n"
1415                                                   ), 0);
1416
1417         else StrBufAppendBufPlain(BounceMB, HKEY(
1418 "A message you sent could not be delivered to some or all of its recipients.\n"
1419 "The following addresses were undeliverable:\n\n"
1420                                           ), 0);
1421
1422         /*
1423          * Now go through the instructions checking for stuff.
1424          */
1425         for (i=0; i<lines; ++i) {
1426                 long addrlen;
1427                 long dsnlen;
1428                 extract_token(buf, instr, i, '\n', sizeof buf);
1429                 extract_token(key, buf, 0, '|', sizeof key);
1430                 addrlen = extract_token(addr, buf, 1, '|', sizeof addr);
1431                 status = extract_int(buf, 2);
1432                 dsnlen = extract_token(dsn, buf, 3, '|', sizeof dsn);
1433                 bounce_this = 0;
1434
1435                 CtdlLogPrintf(CTDL_DEBUG, "key=<%s> addr=<%s> status=%d dsn=<%s>\n",
1436                         key, addr, status, dsn);
1437
1438                 if (!strcasecmp(key, "bounceto")) {
1439                         strcpy(bounceto, addr);
1440                 }
1441
1442                 if (!strcasecmp(key, "msgid")) {
1443                         omsgid = atol(addr);
1444                 }
1445
1446                 if (!strcasecmp(key, "remote")) {
1447                         if (status == 5) bounce_this = 1;
1448                         if (give_up) bounce_this = 1;
1449                 }
1450
1451                 if (bounce_this) {
1452                         ++num_bounces;
1453
1454                         StrBufAppendBufPlain(BounceMB, addr, addrlen, 0);
1455                         StrBufAppendBufPlain(BounceMB, HKEY(": "), 0);
1456                         StrBufAppendBufPlain(BounceMB, dsn, dsnlen, 0);
1457                         StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
1458
1459                         remove_token(instr, i, '\n');
1460                         --i;
1461                         --lines;
1462                 }
1463         }
1464
1465         /* Attach the original message */
1466         if (omsgid >= 0) {
1467                 StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
1468                 StrBufAppendBuf(BounceMB, boundary, 0);
1469                 StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
1470                 StrBufAppendBufPlain(BounceMB, HKEY("Content-type: message/rfc822\r\n"), 0);
1471                 StrBufAppendBufPlain(BounceMB, HKEY("Content-Transfer-Encoding: 7bit\r\n"), 0);
1472                 StrBufAppendBufPlain(BounceMB, HKEY("Content-Disposition: inline\r\n"), 0);
1473                 StrBufAppendBufPlain(BounceMB, HKEY("\r\n"), 0);
1474         
1475                 CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
1476                 CtdlOutputMsg(omsgid, MT_RFC822, HEADERS_ALL, 0, 1, NULL, 0);
1477                 StrBufAppendBuf(BounceMB, CC->redirect_buffer, 0);
1478                 FreeStrBuf(&CC->redirect_buffer);
1479         }
1480
1481         /* Close the multipart MIME scope */
1482         StrBufAppendBufPlain(BounceMB, HKEY("--"), 0);
1483         StrBufAppendBuf(BounceMB, boundary, 0);
1484         StrBufAppendBufPlain(BounceMB, HKEY("--\r\n"), 0);
1485         if (bmsg->cm_fields['A'] != NULL)
1486                 free(bmsg->cm_fields['A']);
1487         bmsg->cm_fields['A'] = SmashStrBuf(&BounceMB);
1488         /* Deliver the bounce if there's anything worth mentioning */
1489         CtdlLogPrintf(CTDL_DEBUG, "num_bounces = %d\n", num_bounces);
1490         if (num_bounces > 0) {
1491
1492                 /* First try the user who sent the message */
1493                 CtdlLogPrintf(CTDL_DEBUG, "bounce to user? <%s>\n", bounceto);
1494                 if (IsEmptyStr(bounceto)) {
1495                         CtdlLogPrintf(CTDL_ERR, "No bounce address specified\n");
1496                         bounce_msgid = (-1L);
1497                 }
1498
1499                 /* Can we deliver the bounce to the original sender? */
1500                 valid = validate_recipients(bounceto, smtp_get_Recipients (), 0);
1501                 if (valid != NULL) {
1502                         if (valid->num_error == 0) {
1503                                 CtdlSubmitMsg(bmsg, valid, "", QP_EADDR);
1504                                 successful_bounce = 1;
1505                         }
1506                 }
1507
1508                 /* If not, post it in the Aide> room */
1509                 if (successful_bounce == 0) {
1510                         CtdlSubmitMsg(bmsg, NULL, config.c_aideroom, QP_EADDR);
1511                 }
1512
1513                 /* Free up the memory we used */
1514                 if (valid != NULL) {
1515                         free_recipients(valid);
1516                 }
1517         }
1518         FreeStrBuf(&boundary);
1519         CtdlFreeMessage(bmsg);
1520         CtdlLogPrintf(CTDL_DEBUG, "Done processing bounces\n");
1521 }
1522
1523
1524 /*
1525  * smtp_purge_completed_deliveries() is caled by smtp_do_procmsg() to scan a
1526  * set of delivery instructions for completed deliveries and remove them.
1527  *
1528  * It returns the number of incomplete deliveries remaining.
1529  */
1530 int smtp_purge_completed_deliveries(char *instr) {
1531         int i;
1532         int lines;
1533         int status;
1534         char buf[1024];
1535         char key[1024];
1536         char addr[1024];
1537         char dsn[1024];
1538         int completed;
1539         int incomplete = 0;
1540
1541         lines = num_tokens(instr, '\n');
1542         for (i=0; i<lines; ++i) {
1543                 extract_token(buf, instr, i, '\n', sizeof buf);
1544                 extract_token(key, buf, 0, '|', sizeof key);
1545                 extract_token(addr, buf, 1, '|', sizeof addr);
1546                 status = extract_int(buf, 2);
1547                 extract_token(dsn, buf, 3, '|', sizeof dsn);
1548
1549                 completed = 0;
1550
1551                 if (!strcasecmp(key, "remote")) {
1552                         if (status == 2) completed = 1;
1553                         else ++incomplete;
1554                 }
1555
1556                 if (completed) {
1557                         remove_token(instr, i, '\n');
1558                         --i;
1559                         --lines;
1560                 }
1561         }
1562
1563         return(incomplete);
1564 }
1565
1566
1567 /*
1568  * smtp_do_procmsg()
1569  *
1570  * Called by smtp_do_queue() to handle an individual message.
1571  */
1572 void smtp_do_procmsg(long msgnum, void *userdata) {
1573         struct CtdlMessage *msg = NULL;
1574         char *instr = NULL;
1575         char *results = NULL;
1576         int i;
1577         int lines;
1578         int status;
1579         char buf[1024];
1580         char key[1024];
1581         char addr[1024];
1582         char dsn[1024];
1583         char envelope_from[1024];
1584         long text_msgid = (-1);
1585         int incomplete_deliveries_remaining;
1586         time_t attempted = 0L;
1587         time_t last_attempted = 0L;
1588         time_t retry = SMTP_RETRY_INTERVAL;
1589
1590         CtdlLogPrintf(CTDL_DEBUG, "SMTP client: smtp_do_procmsg(%ld)\n", msgnum);
1591         strcpy(envelope_from, "");
1592
1593         msg = CtdlFetchMessage(msgnum, 1);
1594         if (msg == NULL) {
1595                 CtdlLogPrintf(CTDL_ERR, "SMTP client: tried %ld but no such message!\n", msgnum);
1596                 return;
1597         }
1598
1599         instr = strdup(msg->cm_fields['M']);
1600         CtdlFreeMessage(msg);
1601
1602         /* Strip out the headers amd any other non-instruction line */
1603         lines = num_tokens(instr, '\n');
1604         for (i=0; i<lines; ++i) {
1605                 extract_token(buf, instr, i, '\n', sizeof buf);
1606                 if (num_tokens(buf, '|') < 2) {
1607                         remove_token(instr, i, '\n');
1608                         --lines;
1609                         --i;
1610                 }
1611         }
1612
1613         /* Learn the message ID and find out about recent delivery attempts */
1614         lines = num_tokens(instr, '\n');
1615         for (i=0; i<lines; ++i) {
1616                 extract_token(buf, instr, i, '\n', sizeof buf);
1617                 extract_token(key, buf, 0, '|', sizeof key);
1618                 if (!strcasecmp(key, "msgid")) {
1619                         text_msgid = extract_long(buf, 1);
1620                 }
1621                 if (!strcasecmp(key, "envelope_from")) {
1622                         extract_token(envelope_from, buf, 1, '|', sizeof envelope_from);
1623                 }
1624                 if (!strcasecmp(key, "retry")) {
1625                         /* double the retry interval after each attempt */
1626                         retry = extract_long(buf, 1) * 2L;
1627                         if (retry > SMTP_RETRY_MAX) {
1628                                 retry = SMTP_RETRY_MAX;
1629                         }
1630                         remove_token(instr, i, '\n');
1631                 }
1632                 if (!strcasecmp(key, "attempted")) {
1633                         attempted = extract_long(buf, 1);
1634                         if (attempted > last_attempted)
1635                                 last_attempted = attempted;
1636                 }
1637         }
1638
1639         /*
1640          * Postpone delivery if we've already tried recently.
1641          */
1642         if (((time(NULL) - last_attempted) < retry) && (run_queue_now == 0)) {
1643                 CtdlLogPrintf(CTDL_DEBUG, "SMTP client: Retry time not yet reached.\n");
1644                 free(instr);
1645                 return;
1646         }
1647
1648
1649         /*
1650          * Bail out if there's no actual message associated with this
1651          */
1652         if (text_msgid < 0L) {
1653                 CtdlLogPrintf(CTDL_ERR, "SMTP client: no 'msgid' directive found!\n");
1654                 free(instr);
1655                 return;
1656         }
1657
1658         /* Plow through the instructions looking for 'remote' directives and
1659          * a status of 0 (no delivery yet attempted) or 3/4 (transient errors
1660          * were experienced and it's time to try again)
1661          */
1662         lines = num_tokens(instr, '\n');
1663         for (i=0; i<lines; ++i) {
1664                 extract_token(buf, instr, i, '\n', sizeof buf);
1665                 extract_token(key, buf, 0, '|', sizeof key);
1666                 extract_token(addr, buf, 1, '|', sizeof addr);
1667                 status = extract_int(buf, 2);
1668                 extract_token(dsn, buf, 3, '|', sizeof dsn);
1669                 if ( (!strcasecmp(key, "remote"))
1670                    && ((status==0)||(status==3)||(status==4)) ) {
1671
1672                         /* Remove this "remote" instruction from the set,
1673                          * but replace the set's final newline if
1674                          * remove_token() stripped it.  It has to be there.
1675                          */
1676                         remove_token(instr, i, '\n');
1677                         if (instr[strlen(instr)-1] != '\n') {
1678                                 strcat(instr, "\n");
1679                         }
1680
1681                         --i;
1682                         --lines;
1683                         CtdlLogPrintf(CTDL_DEBUG, "SMTP client: Trying <%s>\n", addr);
1684                         smtp_try(key, addr, &status, dsn, sizeof dsn, text_msgid, envelope_from);
1685                         if (status != 2) {
1686                                 if (results == NULL) {
1687                                         results = malloc(1024);
1688                                         memset(results, 0, 1024);
1689                                 }
1690                                 else {
1691                                         results = realloc(results, strlen(results) + 1024);
1692                                 }
1693                                 snprintf(&results[strlen(results)], 1024,
1694                                         "%s|%s|%d|%s\n",
1695                                         key, addr, status, dsn);
1696                         }
1697                 }
1698         }
1699
1700         if (results != NULL) {
1701                 instr = realloc(instr, strlen(instr) + strlen(results) + 2);
1702                 strcat(instr, results);
1703                 free(results);
1704         }
1705
1706
1707         /* Generate 'bounce' messages */
1708         smtp_do_bounce(instr);
1709
1710         /* Go through the delivery list, deleting completed deliveries */
1711         incomplete_deliveries_remaining = 
1712                 smtp_purge_completed_deliveries(instr);
1713
1714
1715         /*
1716          * No delivery instructions remain, so delete both the instructions
1717          * message and the message message.
1718          */
1719         if (incomplete_deliveries_remaining <= 0) {
1720                 long delmsgs[2];
1721                 delmsgs[0] = msgnum;
1722                 delmsgs[1] = text_msgid;
1723                 CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, delmsgs, 2, "");
1724         }
1725
1726         /*
1727          * Uncompleted delivery instructions remain, so delete the old
1728          * instructions and replace with the updated ones.
1729          */
1730         if (incomplete_deliveries_remaining > 0) {
1731                 CtdlDeleteMessages(SMTP_SPOOLOUT_ROOM, &msgnum, 1, "");
1732                 msg = malloc(sizeof(struct CtdlMessage));
1733                 memset(msg, 0, sizeof(struct CtdlMessage));
1734                 msg->cm_magic = CTDLMESSAGE_MAGIC;
1735                 msg->cm_anon_type = MES_NORMAL;
1736                 msg->cm_format_type = FMT_RFC822;
1737                 msg->cm_fields['M'] = malloc(strlen(instr)+SIZ);
1738                 snprintf(msg->cm_fields['M'],
1739                         strlen(instr)+SIZ,
1740                         "Content-type: %s\n\n%s\n"
1741                         "attempted|%ld\n"
1742                         "retry|%ld\n",
1743                         SPOOLMIME, instr, (long)time(NULL), (long)retry );
1744                 CtdlSubmitMsg(msg, NULL, SMTP_SPOOLOUT_ROOM, QP_EADDR);
1745                 CtdlFreeMessage(msg);
1746         }
1747
1748         free(instr);
1749 }
1750
1751
1752
1753
1754 /*
1755  * smtp_do_queue()
1756  * 
1757  * Run through the queue sending out messages.
1758  */
1759 void *smtp_do_queue(void *arg) {
1760         int num_processed = 0;
1761         struct CitContext smtp_queue_CC;
1762
1763         CtdlFillSystemContext(&smtp_queue_CC, "SMTP Send");
1764         citthread_setspecific(MyConKey, (void *)&smtp_queue_CC );
1765         CtdlLogPrintf(CTDL_INFO, "SMTP client: processing outbound queue\n");
1766
1767         if (CtdlGetRoom(&CC->room, SMTP_SPOOLOUT_ROOM) != 0) {
1768                 CtdlLogPrintf(CTDL_ERR, "Cannot find room <%s>\n", SMTP_SPOOLOUT_ROOM);
1769         }
1770         else {
1771                 num_processed = CtdlForEachMessage(MSGS_ALL, 0L, NULL, SPOOLMIME, NULL, smtp_do_procmsg, NULL);
1772         }
1773
1774         citthread_mutex_unlock (&smtp_send_lock);
1775         CtdlLogPrintf(CTDL_INFO, "SMTP client: queue run completed; %d messages processed\n", num_processed);
1776
1777         CtdlClearSystemContext();
1778         return(NULL);
1779 }
1780
1781
1782
1783 /*
1784  * smtp_queue_thread
1785  *
1786  * Create a thread to run the SMTP queue
1787  *
1788  * This was created as a response to a situation seen on Uncensored where a bad remote was holding
1789  * up SMTP sending for long times.
1790  * Converting to a thread does not fix the problem caused by the bad remote but it does prevent
1791  * the SMTP sending from stopping housekeeping and the EVT_TIMER event system which in turn prevented
1792  * other things from happening.
1793  */
1794 void smtp_queue_thread (void)
1795 {
1796         if (citthread_mutex_trylock (&smtp_send_lock)) {
1797                 CtdlLogPrintf(CTDL_DEBUG, "SMTP queue run already in progress\n");
1798         }
1799         else {
1800                 CtdlThreadCreate("SMTP Send", CTDLTHREAD_BIGSTACK, smtp_do_queue, NULL);
1801         }
1802 }
1803
1804
1805
1806 void smtp_server_going_down (void)
1807 {
1808         CtdlLogPrintf(CTDL_DEBUG, "SMTP module clean up for shutdown.\n");
1809
1810         citthread_mutex_destroy (&smtp_send_lock);
1811 }
1812
1813
1814
1815 /*****************************************************************************/
1816 /*                          SMTP UTILITY COMMANDS                            */
1817 /*****************************************************************************/
1818
1819 void cmd_smtp(char *argbuf) {
1820         char cmd[64];
1821         char node[256];
1822         char buf[1024];
1823         int i;
1824         int num_mxhosts;
1825
1826         if (CtdlAccessCheck(ac_aide)) return;
1827
1828         extract_token(cmd, argbuf, 0, '|', sizeof cmd);
1829
1830         if (!strcasecmp(cmd, "mx")) {
1831                 extract_token(node, argbuf, 1, '|', sizeof node);
1832                 num_mxhosts = getmx(buf, node);
1833                 cprintf("%d %d MX hosts listed for %s\n",
1834                         LISTING_FOLLOWS, num_mxhosts, node);
1835                 for (i=0; i<num_mxhosts; ++i) {
1836                         extract_token(node, buf, i, '|', sizeof node);
1837                         cprintf("%s\n", node);
1838                 }
1839                 cprintf("000\n");
1840                 return;
1841         }
1842
1843         else if (!strcasecmp(cmd, "runqueue")) {
1844                 run_queue_now = 1;
1845                 cprintf("%d All outbound SMTP will be retried now.\n", CIT_OK);
1846                 return;
1847         }
1848
1849         else {
1850                 cprintf("%d Invalid command.\n", ERROR + ILLEGAL_VALUE);
1851         }
1852
1853 }
1854
1855
1856 /*
1857  * Initialize the SMTP outbound queue
1858  */
1859 void smtp_init_spoolout(void) {
1860         struct ctdlroom qrbuf;
1861
1862         /*
1863          * Create the room.  This will silently fail if the room already
1864          * exists, and that's perfectly ok, because we want it to exist.
1865          */
1866         CtdlCreateRoom(SMTP_SPOOLOUT_ROOM, 3, "", 0, 1, 0, VIEW_MAILBOX);
1867
1868         /*
1869          * Make sure it's set to be a "system room" so it doesn't show up
1870          * in the <K>nown rooms list for Aides.
1871          */
1872         if (CtdlGetRoomLock(&qrbuf, SMTP_SPOOLOUT_ROOM) == 0) {
1873                 qrbuf.QRflags2 |= QR2_SYSTEM;
1874                 CtdlPutRoomLock(&qrbuf);
1875         }
1876 }
1877
1878
1879
1880
1881 /*****************************************************************************/
1882 /*                      MODULE INITIALIZATION STUFF                          */
1883 /*****************************************************************************/
1884 /*
1885  * This cleanup function blows away the temporary memory used by
1886  * the SMTP server.
1887  */
1888 void smtp_cleanup_function(void) {
1889
1890         /* Don't do this stuff if this is not an SMTP session! */
1891         if (CC->h_command_function != smtp_command_loop) return;
1892
1893         CtdlLogPrintf(CTDL_DEBUG, "Performing SMTP cleanup hook\n");
1894         free(SMTP);
1895 }
1896
1897
1898
1899 const char *CitadelServiceSMTP_MTA="SMTP-MTA";
1900 const char *CitadelServiceSMTPS_MTA="SMTPs-MTA";
1901 const char *CitadelServiceSMTP_MSA="SMTP-MSA";
1902 const char *CitadelServiceSMTP_LMTP="LMTP";
1903 const char *CitadelServiceSMTP_LMTP_UNF="LMTP-UnF";
1904
1905 CTDL_MODULE_INIT(smtp)
1906 {
1907         if (!threading)
1908         {
1909                 CtdlRegisterServiceHook(config.c_smtp_port,     /* SMTP MTA */
1910                                         NULL,
1911                                         smtp_mta_greeting,
1912                                         smtp_command_loop,
1913                                         NULL, 
1914                                         CitadelServiceSMTP_MTA);
1915
1916 #ifdef HAVE_OPENSSL
1917                 CtdlRegisterServiceHook(config.c_smtps_port,
1918                                         NULL,
1919                                         smtps_greeting,
1920                                         smtp_command_loop,
1921                                         NULL,
1922                                         CitadelServiceSMTPS_MTA);
1923 #endif
1924
1925                 CtdlRegisterServiceHook(config.c_msa_port,      /* SMTP MSA */
1926                                         NULL,
1927                                         smtp_msa_greeting,
1928                                         smtp_command_loop,
1929                                         NULL,
1930                                         CitadelServiceSMTP_MSA);
1931
1932                 CtdlRegisterServiceHook(0,                      /* local LMTP */
1933                                         file_lmtp_socket,
1934                                         lmtp_greeting,
1935                                         smtp_command_loop,
1936                                         NULL,
1937                                         CitadelServiceSMTP_LMTP);
1938
1939                 CtdlRegisterServiceHook(0,                      /* local LMTP */
1940                                         file_lmtp_unfiltered_socket,
1941                                         lmtp_unfiltered_greeting,
1942                                         smtp_command_loop,
1943                                         NULL,
1944                                         CitadelServiceSMTP_LMTP_UNF);
1945
1946                 smtp_init_spoolout();
1947                 CtdlRegisterSessionHook(smtp_queue_thread, EVT_TIMER);
1948                 CtdlRegisterSessionHook(smtp_cleanup_function, EVT_STOP);
1949                 CtdlRegisterProtoHook(cmd_smtp, "SMTP", "SMTP utility commands");
1950                 CtdlRegisterCleanupHook (smtp_server_going_down);
1951                 citthread_mutex_init (&smtp_send_lock, NULL);
1952         }
1953         
1954         /* return our Subversion id for the Log */
1955         return "$Id$";
1956 }