NETSPOOL: combine collecting of email recipients into one function; participates...
[citadel.git] / citadel / modules / network / serv_netmail.c
1 /*
2  * This module handles shared rooms, inter-Citadel mail, and outbound
3  * mailing list processing.
4  *
5  * Copyright (c) 2000-2012 by the citadel.org team
6  *
7  *  This program is open source software; you can redistribute it and/or modify
8  *  it under the terms of the GNU General Public License, version 3.
9  *
10  *  This program is distributed in the hope that it will be useful,
11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  *  GNU General Public License for more details.
14  *
15  * ** NOTE **   A word on the S_NETCONFIGS semaphore:
16  * This is a fairly high-level type of critical section.  It ensures that no
17  * two threads work on the netconfigs files at the same time.  Since we do
18  * so many things inside these, here are the rules:
19  *  1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others.
20  *  2. Do *not* perform any I/O with the client during these sections.
21  *
22  */
23
24 /*
25  * Duration of time (in seconds) after which pending list subscribe/unsubscribe
26  * requests that have not been confirmed will be deleted.
27  */
28 #define EXP     259200  /* three days */
29
30 #include "sysdep.h"
31 #include <stdlib.h>
32 #include <unistd.h>
33 #include <stdio.h>
34 #include <fcntl.h>
35 #include <ctype.h>
36 #include <signal.h>
37 #include <pwd.h>
38 #include <errno.h>
39 #include <sys/stat.h>
40 #include <sys/types.h>
41 #include <dirent.h>
42 #if TIME_WITH_SYS_TIME
43 # include <sys/time.h>
44 # include <time.h>
45 #else
46 # if HAVE_SYS_TIME_H
47 #  include <sys/time.h>
48 # else
49 #  include <time.h>
50 # endif
51 #endif
52 #ifdef HAVE_SYSCALL_H
53 # include <syscall.h>
54 #else
55 # if HAVE_SYS_SYSCALL_H
56 #  include <sys/syscall.h>
57 # endif
58 #endif
59
60 #include <sys/wait.h>
61 #include <string.h>
62 #include <limits.h>
63 #include <libcitadel.h>
64 #include "citadel.h"
65 #include "server.h"
66 #include "citserver.h"
67 #include "support.h"
68 #include "config.h"
69 #include "user_ops.h"
70 #include "database.h"
71 #include "msgbase.h"
72 #include "internet_addressing.h"
73 #include "serv_network.h"
74 #include "clientsocket.h"
75 #include "file_ops.h"
76 #include "citadel_dirs.h"
77 #include "threads.h"
78 #include "context.h"
79 #include "ctdl_module.h"
80 #include "netspool.h"
81 #include "netmail.h"
82
83 void network_deliver_list(struct CtdlMessage *msg, SpoolControl *sc, const char *RoomName);
84
85 void aggregate_recipients(StrBuf **recps, RoomNetCfg Which, OneRoomNetCfg *OneRNCfg)
86 {
87         size_t recps_len = 0;
88         RoomNetCfgLine *nptr;
89         struct CitContext *CCC = CC;
90
91         *recps = NULL;
92         /*
93          * Figure out how big a buffer we need to allocate
94          */
95         for (nptr = OneRNCfg->NetConfigs[Which]; nptr != NULL; nptr = nptr->next) {
96                 recps_len = recps_len + StrLength(nptr->Value[0]) + 2;
97         }
98
99         /* Nothing todo... */
100         if (recps_len == 0)
101                 return;
102
103         *recps = NewStrBufPlain(NULL, recps_len);
104
105         if (*recps == NULL) {
106                 QN_syslog(LOG_EMERG,
107                           "Cannot allocate %ld bytes for recps...\n",
108                           (long)recps_len);
109                 abort();
110         }
111
112         /* Each recipient */
113         for (nptr = OneRNCfg->NetConfigs[Which]; nptr != NULL; nptr = nptr->next) {
114                 if (nptr != OneRNCfg->NetConfigs[Which]) {
115                         StrBufAppendBufPlain(*recps, HKEY(","), 0);
116                 }
117                 StrBufAppendBuf(*recps, nptr->Value[0], 0);
118         }
119 }
120
121 /*
122  * Deliver digest messages
123  */
124 void network_deliver_digest(SpoolControl *sc)
125 {
126         char buf[SIZ];
127         int i;
128         struct CtdlMessage *msg = NULL;
129         long msglen;
130         StrBuf *recps = NULL;
131         char *precps;
132         struct recptypes *valid;
133         char bounce_to[256];
134
135         if (sc->num_msgs_spooled < 1) {
136                 fclose(sc->digestfp);
137                 sc->digestfp = NULL;
138                 return;
139         }
140
141         msg = malloc(sizeof(struct CtdlMessage));
142         memset(msg, 0, sizeof(struct CtdlMessage));
143         msg->cm_magic = CTDLMESSAGE_MAGIC;
144         msg->cm_format_type = FMT_RFC822;
145         msg->cm_anon_type = MES_NORMAL;
146
147         sprintf(buf, "%ld", time(NULL));
148         msg->cm_fields['T'] = strdup(buf);
149         msg->cm_fields['A'] = strdup(CC->room.QRname);
150         snprintf(buf, sizeof buf, "[%s]", CC->room.QRname);
151         msg->cm_fields['U'] = strdup(buf);
152         sprintf(buf, "room_%s@%s", CC->room.QRname, config.c_fqdn);
153         for (i=0; buf[i]; ++i) {
154                 if (isspace(buf[i])) buf[i]='_';
155                 buf[i] = tolower(buf[i]);
156         }
157         msg->cm_fields['F'] = strdup(buf);
158         msg->cm_fields['R'] = strdup(buf);
159
160         /* Set the 'List-ID' header */
161         msg->cm_fields['L'] = malloc(1024);
162         snprintf(msg->cm_fields['L'], 1024,
163                  "%s <%ld.list-id.%s>",
164                  CC->room.QRname,
165                  CC->room.QRnumber,
166                  config.c_fqdn
167                 );
168
169         /*
170          * Go fetch the contents of the digest
171          */
172         fseek(sc->digestfp, 0L, SEEK_END);
173         msglen = ftell(sc->digestfp);
174
175         msg->cm_fields['M'] = malloc(msglen + 1);
176         fseek(sc->digestfp, 0L, SEEK_SET);
177         fread(msg->cm_fields['M'], (size_t)msglen, 1, sc->digestfp);
178         msg->cm_fields['M'][msglen] = '\0';
179
180         fclose(sc->digestfp);
181         sc->digestfp = NULL;
182
183         /* Now generate the delivery instructions */
184         aggregate_recipients(&recps, digestrecp, sc->RNCfg);
185
186         /* Where do we want bounces and other noise to be heard?
187          *Surely not the list members! */
188         snprintf(bounce_to, sizeof bounce_to, "room_aide@%s", config.c_fqdn);
189
190         /* Now submit the message */
191         precps = SmashStrBuf(&recps);
192         valid = validate_recipients(precps, NULL, 0);
193         free(precps);
194         if (valid != NULL) {
195                 valid->bounce_to = strdup(bounce_to);
196                 valid->envelope_from = strdup(bounce_to);
197                 CtdlSubmitMsg(msg, valid, NULL, 0);
198         }
199         CtdlFreeMessage(msg);
200         free_recipients(valid);
201 }
202
203
204 void network_process_digest(SpoolControl *sc, struct CtdlMessage *omsg, long *delete_after_send)
205 {
206
207         struct CtdlMessage *msg = NULL;
208
209         /*
210          * Process digest recipients
211          */
212         if ((sc->RNCfg->NetConfigs[digestrecp] == NULL) || 
213             (sc->digestfp == NULL))
214                 return;
215
216         msg = CtdlDuplicateMessage(omsg);
217         if (msg != NULL) {
218                 fprintf(sc->digestfp,
219                         " -----------------------------------"
220                         "------------------------------------"
221                         "-------\n");
222                 fprintf(sc->digestfp, "From: ");
223                 if (msg->cm_fields['A'] != NULL) {
224                         fprintf(sc->digestfp,
225                                 "%s ",
226                                 msg->cm_fields['A']);
227                 }
228                 if (msg->cm_fields['F'] != NULL) {
229                         fprintf(sc->digestfp,
230                                 "<%s> ",
231                                 msg->cm_fields['F']);
232                 }
233                 else if (msg->cm_fields['N'] != NULL) {
234                         fprintf(sc->digestfp,
235                                 "@%s ",
236                                 msg->cm_fields['N']);
237                 }
238                 fprintf(sc->digestfp, "\n");
239                 if (msg->cm_fields['U'] != NULL) {
240                         fprintf(sc->digestfp,
241                                 "Subject: %s\n",
242                                 msg->cm_fields['U']);
243                 }
244
245                 CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
246
247                 safestrncpy(CC->preferred_formats,
248                             "text/plain",
249                             sizeof CC->preferred_formats);
250
251                 CtdlOutputPreLoadedMsg(msg,
252                                        MT_CITADEL,
253                                        HEADERS_NONE,
254                                        0, 0, 0);
255
256                 StrBufTrim(CC->redirect_buffer);
257                 fwrite(HKEY("\n"), 1, sc->digestfp);
258                 fwrite(SKEY(CC->redirect_buffer), 1, sc->digestfp);
259                 fwrite(HKEY("\n"), 1, sc->digestfp);
260
261                 FreeStrBuf(&CC->redirect_buffer);
262
263                 sc->num_msgs_spooled += 1;
264                 CtdlFreeMessage(msg);
265         }
266 }
267
268
269 void network_process_list(SpoolControl *sc, struct CtdlMessage *omsg, long *delete_after_send)
270 {
271         int rlen;
272         char *pCh;
273         StrBuf *Subject, *FlatSubject;
274         struct CtdlMessage *msg = NULL;
275         int i;
276
277         /*
278          * Process mailing list recipients
279          */
280         if (sc->RNCfg->NetConfigs[listrecp] == NULL) 
281                 return;
282
283         /* create our own copy of the message.
284          *  We're going to need to modify it
285          * in order to insert the [list name] in it, etc.
286          */
287
288         msg = CtdlDuplicateMessage(omsg);
289
290         if (msg->cm_fields['K'] != NULL)
291                 free(msg->cm_fields['K']);
292         if (msg->cm_fields['V'] == NULL){
293                 /* local message, no enVelope */
294                 StrBuf *Buf;
295                 Buf = NewStrBuf();
296                 StrBufAppendBufPlain(Buf,
297                                      msg->cm_fields['O']
298                                      , -1, 0);
299                 StrBufAppendBufPlain(Buf, HKEY("@"), 0);
300                 StrBufAppendBufPlain(Buf, config.c_fqdn, -1, 0);
301
302                 msg->cm_fields['K'] = SmashStrBuf(&Buf);
303         }
304         else {
305                 msg->cm_fields['K'] =
306                         strdup (msg->cm_fields['V']);
307         }
308         /* Set the 'List-ID' header */
309         if (msg->cm_fields['L'] != NULL) {
310                 free(msg->cm_fields['L']);
311         }
312         msg->cm_fields['L'] = malloc(1024);
313         snprintf(msg->cm_fields['L'], 1024,
314                  "%s <%ld.list-id.%s>",
315                  CC->room.QRname,
316                  CC->room.QRnumber,
317                  config.c_fqdn
318                 );
319
320         /* Prepend "[List name]" to the subject */
321         if (msg->cm_fields['U'] == NULL) {
322                 Subject = NewStrBufPlain(HKEY("(no subject)"));
323         }
324         else {
325                 Subject = NewStrBufPlain(
326                         msg->cm_fields['U'], -1);
327         }
328         FlatSubject = NewStrBufPlain(NULL, StrLength(Subject));
329         StrBuf_RFC822_to_Utf8(FlatSubject, Subject, NULL, NULL);
330
331         rlen = strlen(CC->room.QRname);
332         pCh  = strstr(ChrPtr(FlatSubject), CC->room.QRname);
333         if ((pCh == NULL) ||
334             (*(pCh + rlen) != ']') ||
335             (pCh == ChrPtr(FlatSubject)) ||
336             (*(pCh - 1) != '[')
337                 )
338         {
339                 StrBuf *tmp;
340                 StrBufPlain(Subject, HKEY("["));
341                 StrBufAppendBufPlain(Subject,
342                                      CC->room.QRname,
343                                      rlen, 0);
344                 StrBufAppendBufPlain(Subject, HKEY("] "), 0);
345                 StrBufAppendBuf(Subject, FlatSubject, 0);
346                 /* so we can free the right one swap them */
347                 tmp = Subject;
348                 Subject = FlatSubject;
349                 FlatSubject = tmp;
350                 StrBufRFC2047encode(&Subject, FlatSubject);
351         }
352
353         if (msg->cm_fields['U'] != NULL)
354                 free (msg->cm_fields['U']);
355         msg->cm_fields['U'] = SmashStrBuf(&Subject);
356
357         FreeStrBuf(&FlatSubject);
358
359         /* else we won't modify the buffer, since the
360          * roomname is already here.
361          */
362
363         /* if there is no other recipient, Set the recipient
364          * of the list message to the email address of the
365          * room itself.
366          */
367         if ((msg->cm_fields['R'] == NULL) ||
368             IsEmptyStr(msg->cm_fields['R']))
369         {
370                 if (msg->cm_fields['R'] != NULL)
371                         free(msg->cm_fields['R']);
372
373                 msg->cm_fields['R'] = malloc(256);
374                 snprintf(msg->cm_fields['R'], 256,
375                          "room_%s@%s", CC->room.QRname,
376                          config.c_fqdn);
377                 for (i=0; msg->cm_fields['R'][i]; ++i) {
378                         if (isspace(msg->cm_fields['R'][i])) {
379                                 msg->cm_fields['R'][i] = '_';
380                         }
381                 }
382         }
383
384         /* Handle delivery */
385         network_deliver_list(msg, sc, CC->room.QRname);
386         CtdlFreeMessage(msg);
387 }
388
389 /*
390  * Deliver list messages to everyone on the list ... efficiently
391  */
392 void network_deliver_list(struct CtdlMessage *msg, SpoolControl *sc, const char *RoomName)
393 {
394         StrBuf *recps = NULL;
395         char *precps = NULL;
396         struct recptypes *valid;
397         char bounce_to[256];
398
399         /* Don't do this if there were no recipients! */
400         if (sc->RNCfg->NetConfigs[listrecp] == NULL) return;
401
402         /* Now generate the delivery instructions */
403
404         /*
405          * Figure out how big a buffer we need to allocate
406          */
407         aggregate_recipients(&recps, listrecp, sc->RNCfg);
408
409         /* Where do we want bounces and other noise to be heard?
410          *  Surely not the list members! */
411         snprintf(bounce_to, sizeof bounce_to, "room_aide@%s", config.c_fqdn);
412
413         /* Now submit the message */
414         precps = SmashStrBuf(&recps);
415         valid = validate_recipients(precps, NULL, 0);
416         free(precps);
417         if (valid != NULL) {
418                 valid->bounce_to = strdup(bounce_to);
419                 valid->envelope_from = strdup(bounce_to);
420                 valid->sending_room = strdup(RoomName);
421                 CtdlSubmitMsg(msg, valid, NULL, 0);
422                 free_recipients(valid);
423         }
424         /* Do not call CtdlFreeMessage(msg) here; the caller will free it. */
425 }
426
427
428 void network_process_participate(SpoolControl *sc, struct CtdlMessage *omsg, long *delete_after_send)
429 {
430         struct CtdlMessage *msg = NULL;
431         int i;
432         int ok_to_participate = 0;
433         StrBuf *Buf = NULL;
434         struct recptypes *valid;
435
436         /*
437          * Process client-side list participations for this room
438          */
439         if (sc->RNCfg->NetConfigs[participate] == NULL)
440                 return;
441
442         msg = CtdlDuplicateMessage(omsg);
443
444         /* Only send messages which originated on our own
445          * Citadel network, otherwise we'll end up sending the
446          * remote mailing list's messages back to it, which
447          * is rude...
448          */
449         ok_to_participate = 0;
450         if (msg->cm_fields['N'] != NULL) {
451                 if (!strcasecmp(msg->cm_fields['N'],
452                                 config.c_nodename)) {
453                         ok_to_participate = 1;
454                 }
455                 
456                 Buf = NewStrBufPlain(msg->cm_fields['N'], -1);
457                 if (CtdlIsValidNode(NULL,
458                                     NULL,
459                                     Buf,
460                                     sc->working_ignetcfg,
461                                     sc->the_netmap) == 0)
462                 {
463                         ok_to_participate = 1;
464                 }
465         }
466         if (ok_to_participate)
467         {
468                 StrBuf *recps = NULL;
469                 char *precps;
470
471                 if (msg->cm_fields['F'] != NULL) {
472                         free(msg->cm_fields['F']);
473                 }
474                 msg->cm_fields['F'] = malloc(SIZ);
475                 /* Replace the Internet email address of the
476                  * actual author with the email address of the
477                  * room itself, so the remote listserv doesn't
478                  * reject us.
479                  * FIXME  I want to be able to pick any address
480                  */
481                 snprintf(msg->cm_fields['F'], SIZ,
482                          "room_%s@%s", CC->room.QRname,
483                          config.c_fqdn);
484                 for (i=0; msg->cm_fields['F'][i]; ++i) {
485                         if (isspace(msg->cm_fields['F'][i])) {
486                                 msg->cm_fields['F'][i] = '_';
487                         }
488                 }
489
490                 aggregate_recipients(&recps, participate, sc->RNCfg);
491                 precps = SmashStrBuf(&recps);
492                 valid = validate_recipients(precps, NULL, 0);
493
494                 if (msg->cm_fields['R'] != NULL) {
495                         free(msg->cm_fields['R']);
496                 }/* TODO: check whether 'R' is set appropriate later. */
497
498                 CtdlSubmitMsg(msg, valid, "", 0);
499                 free_recipients(valid);
500         }
501         FreeStrBuf(&Buf);
502         CtdlFreeMessage(msg);
503 }
504
505 void network_process_ignetpush(SpoolControl *sc, struct CtdlMessage *omsg, long *delete_after_send)
506 {
507         struct CtdlMessage *msg = NULL;
508         struct CitContext *CCC = CC;
509         struct ser_ret sermsg;
510         char buf[SIZ];
511         char filename[PATH_MAX];
512         FILE *fp;
513         size_t newpath_len;
514         char *newpath = NULL;
515         RoomNetCfgLine* mptr;
516         StrBuf *Buf = NULL;
517         int i;
518         int bang = 0;
519         int send = 1;
520
521         if (sc->RNCfg->NetConfigs[ignet_push_share] == NULL)
522                 return;
523         /*
524          * Process IGnet push shares
525          */
526         msg = CtdlDuplicateMessage(omsg);
527
528         /* Prepend our node name to the Path field whenever
529          * sending a message to another IGnet node
530          */
531         if (msg->cm_fields['P'] == NULL)
532         {
533                 msg->cm_fields['P'] = strdup("username");
534         }
535         newpath_len = strlen(msg->cm_fields['P']) +
536                 strlen(config.c_nodename) + 2;
537         newpath = malloc(newpath_len);
538         snprintf(newpath, newpath_len, "%s!%s",
539                  config.c_nodename, msg->cm_fields['P']);
540         free(msg->cm_fields['P']);
541         msg->cm_fields['P'] = newpath;
542         
543         /*
544          * Determine if this message is set to be deleted
545          * after sending out on the network
546          */
547         if (msg->cm_fields['S'] != NULL) {
548                 if (!strcasecmp(msg->cm_fields['S'], "CANCEL")) {
549                         *delete_after_send = 1;
550                 }
551         }
552
553         /* Now send it to every node */
554         for (mptr = sc->RNCfg->NetConfigs[ignet_push_share];
555              mptr != NULL;
556              mptr = mptr->next)
557         {
558                 send = 1;
559                 NewStrBufDupAppendFlush(&Buf, mptr->Value[0], NULL, 1);
560                         
561                 /* Check for valid node name */
562                 if (CtdlIsValidNode(NULL,
563                                     NULL,
564                                     Buf,
565                                     sc->working_ignetcfg,
566                                     sc->the_netmap) != 0)
567                 {
568                         QN_syslog(LOG_ERR,
569                                   "Invalid node <%s>\n",
570                                   ChrPtr(mptr->Value[0]));
571                         
572                         send = 0;
573                 }
574                 
575                 /* Check for split horizon */
576                 QN_syslog(LOG_DEBUG, "Path is %s\n", msg->cm_fields['P']);
577                 bang = num_tokens(msg->cm_fields['P'], '!');
578                 if (bang > 1) {
579                         for (i=0; i<(bang-1); ++i) {
580                                 extract_token(buf,
581                                               msg->cm_fields['P'],
582                                               i, '!',
583                                               sizeof buf);
584
585                                 QN_syslog(LOG_DEBUG, "Compare <%s> to <%s>\n",
586                                           buf, ChrPtr(mptr->Value[0])) ;
587                                 if (!strcasecmp(buf, ChrPtr(mptr->Value[0]))) {
588                                         send = 0;
589                                         break;
590                                 }
591                         }
592                         
593                         QN_syslog(LOG_INFO,
594                                   "%sSending to %s\n",
595                                   (send)?"":"Not ",
596                                   ChrPtr(mptr->Value[0]));
597                 }
598                 
599                 /* Send the message */
600                 if (send == 1)
601                 {
602                         /*
603                          * Force the message to appear in the correct
604                          * room on the far end by setting the C field
605                          * correctly
606                          */
607                         if (msg->cm_fields['C'] != NULL) {
608                                 free(msg->cm_fields['C']);
609                         }
610                         if (StrLength(mptr->Value[0]) > 0) {
611                                 msg->cm_fields['C'] =
612                                         strdup(ChrPtr(mptr->Value[0]));
613                         }
614                         else {
615                                 msg->cm_fields['C'] =
616                                         strdup(CC->room.QRname);
617                         }
618                         
619                         /* serialize it for transmission */
620                         serialize_message(&sermsg, msg);
621                         if (sermsg.len > 0) {
622                                 
623                                 /* write it to a spool file */
624                                 snprintf(filename,
625                                          sizeof(filename),
626                                          "%s/%s@%lx%x",
627                                          ctdl_netout_dir,
628                                          ChrPtr(mptr->Value[0]),
629                                          time(NULL),
630                                          rand()
631                                         );
632                                         
633                                 QN_syslog(LOG_DEBUG,
634                                           "Appending to %s\n",
635                                           filename);
636                                 
637                                 fp = fopen(filename, "ab");
638                                 if (fp != NULL) {
639                                         fwrite(sermsg.ser,
640                                                sermsg.len, 1, fp);
641                                         fclose(fp);
642                                 }
643                                 else {
644                                         QN_syslog(LOG_ERR,
645                                                   "%s: %s\n",
646                                                   filename,
647                                                   strerror(errno));
648                                 }
649
650                                 /* free the serialized version */
651                                 free(sermsg.ser);
652                         }
653                 }
654         }
655         FreeStrBuf(&Buf);
656         CtdlFreeMessage(msg);
657 }
658
659
660 /*
661  * Spools out one message from the list.
662  */
663 void network_spool_msg(long msgnum,
664                        void *userdata)
665 {
666         struct CtdlMessage *msg = NULL;
667         long delete_after_send = 0;     /* Set to 1 to delete after spooling */
668         SpoolControl *sc;
669
670         sc = (SpoolControl *)userdata;
671
672         msg = CtdlFetchMessage(msgnum, 1);
673
674         network_process_list(sc, msg, &delete_after_send);
675         network_process_digest(sc, msg, &delete_after_send);
676         network_process_participate(sc, msg, &delete_after_send);
677         network_process_ignetpush(sc, msg, &delete_after_send);
678         
679         CtdlFreeMessage(msg);
680
681         /* update lastsent */
682         sc->lastsent = msgnum;
683
684         /* Delete this message if delete-after-send is set */
685         if (delete_after_send) {
686                 CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, "");
687         }
688 }