ca44ec690f584a59e62f8a3ddb03433847dd3dca
[citadel.git] / citadel / modules / network / serv_netmail.c
1 /*
2  * This module handles shared rooms, inter-Citadel mail, and outbound
3  * mailing list processing.
4  *
5  * Copyright (c) 2000-2012 by the citadel.org team
6  *
7  *  This program is open source software; you can redistribute it and/or modify
8  *  it under the terms of the GNU General Public License, version 3.
9  *
10  *  This program is distributed in the hope that it will be useful,
11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  *  GNU General Public License for more details.
14  *
15  * ** NOTE **   A word on the S_NETCONFIGS semaphore:
16  * This is a fairly high-level type of critical section.  It ensures that no
17  * two threads work on the netconfigs files at the same time.  Since we do
18  * so many things inside these, here are the rules:
19  *  1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others.
20  *  2. Do *not* perform any I/O with the client during these sections.
21  *
22  */
23
24 /*
25  * Duration of time (in seconds) after which pending list subscribe/unsubscribe
26  * requests that have not been confirmed will be deleted.
27  */
28 #define EXP     259200  /* three days */
29
30 #include "sysdep.h"
31 #include <stdlib.h>
32 #include <unistd.h>
33 #include <stdio.h>
34 #include <fcntl.h>
35 #include <ctype.h>
36 #include <signal.h>
37 #include <pwd.h>
38 #include <errno.h>
39 #include <sys/stat.h>
40 #include <sys/types.h>
41 #include <dirent.h>
42 #if TIME_WITH_SYS_TIME
43 # include <sys/time.h>
44 # include <time.h>
45 #else
46 # if HAVE_SYS_TIME_H
47 #  include <sys/time.h>
48 # else
49 #  include <time.h>
50 # endif
51 #endif
52 #ifdef HAVE_SYSCALL_H
53 # include <syscall.h>
54 #else
55 # if HAVE_SYS_SYSCALL_H
56 #  include <sys/syscall.h>
57 # endif
58 #endif
59
60 #include <sys/wait.h>
61 #include <string.h>
62 #include <limits.h>
63 #include <libcitadel.h>
64 #include "citadel.h"
65 #include "server.h"
66 #include "citserver.h"
67 #include "support.h"
68 #include "config.h"
69 #include "user_ops.h"
70 #include "database.h"
71 #include "msgbase.h"
72 #include "internet_addressing.h"
73 #include "serv_network.h"
74 #include "clientsocket.h"
75 #include "file_ops.h"
76 #include "citadel_dirs.h"
77 #include "threads.h"
78 #include "context.h"
79 #include "ctdl_module.h"
80 #include "netspool.h"
81 #include "netmail.h"
82
83
84 /*
85  * Deliver digest messages
86  */
87 void network_deliver_digest(SpoolControl *sc) {
88         struct CitContext *CCC = CC;
89         char buf[SIZ];
90         int i;
91         struct CtdlMessage *msg = NULL;
92         long msglen;
93         StrBuf *recps = NULL;
94         char *precps;
95         size_t recps_len = SIZ;
96         struct recptypes *valid;
97         RoomNetCfgLine *nptr;
98         char bounce_to[256];
99
100         if (sc->num_msgs_spooled < 1) {
101                 fclose(sc->digestfp);
102                 sc->digestfp = NULL;
103                 return;
104         }
105
106         msg = malloc(sizeof(struct CtdlMessage));
107         memset(msg, 0, sizeof(struct CtdlMessage));
108         msg->cm_magic = CTDLMESSAGE_MAGIC;
109         msg->cm_format_type = FMT_RFC822;
110         msg->cm_anon_type = MES_NORMAL;
111
112         sprintf(buf, "%ld", time(NULL));
113         msg->cm_fields['T'] = strdup(buf);
114         msg->cm_fields['A'] = strdup(CC->room.QRname);
115         snprintf(buf, sizeof buf, "[%s]", CC->room.QRname);
116         msg->cm_fields['U'] = strdup(buf);
117         sprintf(buf, "room_%s@%s", CC->room.QRname, config.c_fqdn);
118         for (i=0; buf[i]; ++i) {
119                 if (isspace(buf[i])) buf[i]='_';
120                 buf[i] = tolower(buf[i]);
121         }
122         msg->cm_fields['F'] = strdup(buf);
123         msg->cm_fields['R'] = strdup(buf);
124
125         /* Set the 'List-ID' header */
126         msg->cm_fields['L'] = malloc(1024);
127         snprintf(msg->cm_fields['L'], 1024,
128                 "%s <%ld.list-id.%s>",
129                 CC->room.QRname,
130                 CC->room.QRnumber,
131                 config.c_fqdn
132         );
133
134         /*
135          * Go fetch the contents of the digest
136          */
137         fseek(sc->digestfp, 0L, SEEK_END);
138         msglen = ftell(sc->digestfp);
139
140         msg->cm_fields['M'] = malloc(msglen + 1);
141         fseek(sc->digestfp, 0L, SEEK_SET);
142         fread(msg->cm_fields['M'], (size_t)msglen, 1, sc->digestfp);
143         msg->cm_fields['M'][msglen] = '\0';
144
145         fclose(sc->digestfp);
146         sc->digestfp = NULL;
147
148         /* Now generate the delivery instructions */
149
150         /*
151          * Figure out how big a buffer we need to allocate
152          */
153         for (nptr = sc->RNCfg->NetConfigs[digestrecp]; nptr != NULL; nptr = nptr->next) {
154                 recps_len = recps_len + StrLength(nptr->Value) + 2;
155         }
156
157         recps = NewStrBufPlain(NULL, recps_len);
158
159         if (recps == NULL) {
160                 QN_syslog(LOG_EMERG,
161                           "Cannot allocate %ld bytes for recps...\n",
162                           (long)recps_len);
163                 abort();
164         }
165
166         /* Each recipient */
167         for (nptr = sc->RNCfg->NetConfigs[digestrecp]; nptr != NULL; nptr = nptr->next) {
168                 if (nptr != sc->RNCfg->NetConfigs[digestrecp]) {
169                         StrBufAppendBufPlain(recps, HKEY(","), 0);
170                 }
171                 StrBufAppendBuf(recps, nptr->Value, 0);
172         }
173
174         /* Where do we want bounces and other noise to be heard?
175          *Surely not the list members! */
176         snprintf(bounce_to, sizeof bounce_to, "room_aide@%s", config.c_fqdn);
177
178         /* Now submit the message */
179         precps = SmashStrBuf(&recps);
180         valid = validate_recipients(precps, NULL, 0);
181         free(precps);
182         if (valid != NULL) {
183                 valid->bounce_to = strdup(bounce_to);
184                 valid->envelope_from = strdup(bounce_to);
185                 CtdlSubmitMsg(msg, valid, NULL, 0);
186         }
187         CtdlFreeMessage(msg);
188         free_recipients(valid);
189 }
190
191
192 /*
193  * Deliver list messages to everyone on the list ... efficiently
194  */
195 void network_deliver_list(struct CtdlMessage *msg, SpoolControl *sc, const char *RoomName)
196 {
197         struct CitContext *CCC = CC;
198         StrBuf *recps = NULL;
199         char *precps = NULL;
200         size_t recps_len = SIZ;
201         struct recptypes *valid;
202         RoomNetCfgLine *nptr;
203         char bounce_to[256];
204
205         /* Don't do this if there were no recipients! */
206         if (sc->RNCfg->NetConfigs[listrecp] == NULL) return;
207
208         /* Now generate the delivery instructions */
209
210         /*
211          * Figure out how big a buffer we need to allocate
212          */
213         for (nptr = sc->RNCfg->NetConfigs[listrecp]; nptr != NULL; nptr = nptr->next) {
214                 recps_len = recps_len + StrLength(nptr->Value) + 2;
215         }
216
217         recps = NewStrBufPlain(NULL, recps_len);
218
219         if (recps == NULL) {
220                 QN_syslog(LOG_EMERG,
221                           "Cannot allocate %ld bytes for recps...\n",
222                           (long)recps_len);
223                 abort();
224         }
225
226         /* Each recipient */
227         for (nptr = sc->RNCfg->NetConfigs[listrecp]; nptr != NULL; nptr = nptr->next) {
228                 if (nptr != sc->RNCfg->NetConfigs[listrecp]) {
229                         StrBufAppendBufPlain(recps, HKEY(","), 0);
230                 }
231                 StrBufAppendBuf(recps, nptr->Value, 0);
232         }
233
234         /* Where do we want bounces and other noise to be heard?
235          *  Surely not the list members! */
236         snprintf(bounce_to, sizeof bounce_to, "room_aide@%s", config.c_fqdn);
237
238         /* Now submit the message */
239         precps = SmashStrBuf(&recps);
240         valid = validate_recipients(precps, NULL, 0);
241         free(precps);
242         if (valid != NULL) {
243                 valid->bounce_to = strdup(bounce_to);
244                 valid->envelope_from = strdup(bounce_to);
245                 valid->sending_room = strdup(RoomName);
246                 CtdlSubmitMsg(msg, valid, NULL, 0);
247                 free_recipients(valid);
248         }
249         /* Do not call CtdlFreeMessage(msg) here; the caller will free it. */
250 }
251
252
253 /*
254  * Spools out one message from the list.
255  */
256 void network_spool_msg(long msgnum,
257                        void *userdata)
258 {
259         struct CitContext *CCC = CC;
260         StrBuf *Buf = NULL;
261         SpoolControl *sc;
262         int i;
263         char *newpath = NULL;
264         struct CtdlMessage *msg = NULL;
265         RoomNetCfgLine *nptr;
266         MapList *mptr;
267         struct ser_ret sermsg;
268         FILE *fp;
269         char filename[PATH_MAX];
270         char buf[SIZ];
271         int bang = 0;
272         int send = 1;
273         int delete_after_send = 0;      /* Set to 1 to delete after spooling */
274         int ok_to_participate = 0;
275         struct recptypes *valid;
276
277         sc = (SpoolControl *)userdata;
278
279         /*
280          * Process mailing list recipients
281          */
282         if (sc->RNCfg->NetConfigs[listrecp] != NULL) {
283                 /* Fetch the message.  We're going to need to modify it
284                  * in order to insert the [list name] in it, etc.
285                  */
286                 msg = CtdlFetchMessage(msgnum, 1);
287                 if (msg != NULL) {
288                         int rlen;
289                         char *pCh;
290                         StrBuf *Subject, *FlatSubject;
291
292                         if (msg->cm_fields['K'] != NULL)
293                                 free(msg->cm_fields['K']);
294                         if (msg->cm_fields['V'] == NULL){
295                                 /* local message, no enVelope */
296                                 StrBuf *Buf;
297                                 Buf = NewStrBuf();
298                                 StrBufAppendBufPlain(Buf,
299                                                      msg->cm_fields['O']
300                                                      , -1, 0);
301                                 StrBufAppendBufPlain(Buf, HKEY("@"), 0);
302                                 StrBufAppendBufPlain(Buf, config.c_fqdn, -1, 0);
303
304                                 msg->cm_fields['K'] = SmashStrBuf(&Buf);
305                         }
306                         else {
307                                 msg->cm_fields['K'] =
308                                         strdup (msg->cm_fields['V']);
309                         }
310                         /* Set the 'List-ID' header */
311                         if (msg->cm_fields['L'] != NULL) {
312                                 free(msg->cm_fields['L']);
313                         }
314                         msg->cm_fields['L'] = malloc(1024);
315                         snprintf(msg->cm_fields['L'], 1024,
316                                 "%s <%ld.list-id.%s>",
317                                 CC->room.QRname,
318                                 CC->room.QRnumber,
319                                 config.c_fqdn
320                         );
321
322                         /* Prepend "[List name]" to the subject */
323                         if (msg->cm_fields['U'] == NULL) {
324                                 Subject = NewStrBufPlain(HKEY("(no subject)"));
325                         }
326                         else {
327                                 Subject = NewStrBufPlain(
328                                         msg->cm_fields['U'], -1);
329                         }
330                         FlatSubject = NewStrBufPlain(NULL, StrLength(Subject));
331                         StrBuf_RFC822_to_Utf8(FlatSubject, Subject, NULL, NULL);
332
333                         rlen = strlen(CC->room.QRname);
334                         pCh  = strstr(ChrPtr(FlatSubject), CC->room.QRname);
335                         if ((pCh == NULL) ||
336                             (*(pCh + rlen) != ']') ||
337                             (pCh == ChrPtr(FlatSubject)) ||
338                             (*(pCh - 1) != '[')
339                                 )
340                         {
341                                 StrBuf *tmp;
342                                 StrBufPlain(Subject, HKEY("["));
343                                 StrBufAppendBufPlain(Subject,
344                                                      CC->room.QRname,
345                                                      rlen, 0);
346                                 StrBufAppendBufPlain(Subject, HKEY("] "), 0);
347                                 StrBufAppendBuf(Subject, FlatSubject, 0);
348                                  /* so we can free the right one swap them */
349                                 tmp = Subject;
350                                 Subject = FlatSubject;
351                                 FlatSubject = tmp;
352                                 StrBufRFC2047encode(&Subject, FlatSubject);
353                         }
354
355                         if (msg->cm_fields['U'] != NULL)
356                                 free (msg->cm_fields['U']);
357                         msg->cm_fields['U'] = SmashStrBuf(&Subject);
358
359                         FreeStrBuf(&FlatSubject);
360
361                         /* else we won't modify the buffer, since the
362                          * roomname is already here.
363                          */
364
365                         /* if there is no other recipient, Set the recipient
366                          * of the list message to the email address of the
367                          * room itself.
368                          */
369                         if ((msg->cm_fields['R'] == NULL) ||
370                             IsEmptyStr(msg->cm_fields['R']))
371                         {
372                                 if (msg->cm_fields['R'] != NULL)
373                                         free(msg->cm_fields['R']);
374
375                                 msg->cm_fields['R'] = malloc(256);
376                                 snprintf(msg->cm_fields['R'], 256,
377                                          "room_%s@%s", CC->room.QRname,
378                                          config.c_fqdn);
379                                 for (i=0; msg->cm_fields['R'][i]; ++i) {
380                                         if (isspace(msg->cm_fields['R'][i])) {
381                                                 msg->cm_fields['R'][i] = '_';
382                                         }
383                                 }
384                         }
385
386                         /* Handle delivery */
387                         network_deliver_list(msg, sc, CC->room.QRname);
388                         CtdlFreeMessage(msg);
389                 }
390         }
391
392         /*
393          * Process digest recipients
394          */
395         if ((sc->RNCfg->NetConfigs[digestrecp] != NULL) && (sc->digestfp != NULL)) {
396                 msg = CtdlFetchMessage(msgnum, 1);
397                 if (msg != NULL) {
398                         fprintf(sc->digestfp,
399                                 " -----------------------------------"
400                                 "------------------------------------"
401                                 "-------\n");
402                         fprintf(sc->digestfp, "From: ");
403                         if (msg->cm_fields['A'] != NULL) {
404                                 fprintf(sc->digestfp,
405                                         "%s ",
406                                         msg->cm_fields['A']);
407                         }
408                         if (msg->cm_fields['F'] != NULL) {
409                                 fprintf(sc->digestfp,
410                                         "<%s> ",
411                                         msg->cm_fields['F']);
412                         }
413                         else if (msg->cm_fields['N'] != NULL) {
414                                 fprintf(sc->digestfp,
415                                         "@%s ",
416                                         msg->cm_fields['N']);
417                         }
418                         fprintf(sc->digestfp, "\n");
419                         if (msg->cm_fields['U'] != NULL) {
420                                 fprintf(sc->digestfp,
421                                         "Subject: %s\n",
422                                         msg->cm_fields['U']);
423                         }
424
425                         CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
426
427                         safestrncpy(CC->preferred_formats,
428                                     "text/plain",
429                                     sizeof CC->preferred_formats);
430
431                         CtdlOutputPreLoadedMsg(msg,
432                                                MT_CITADEL,
433                                                HEADERS_NONE,
434                                                0, 0, 0);
435
436                         StrBufTrim(CC->redirect_buffer);
437                         fwrite(HKEY("\n"), 1, sc->digestfp);
438                         fwrite(SKEY(CC->redirect_buffer), 1, sc->digestfp);
439                         fwrite(HKEY("\n"), 1, sc->digestfp);
440
441                         FreeStrBuf(&CC->redirect_buffer);
442
443                         sc->num_msgs_spooled += 1;
444                         CtdlFreeMessage(msg);
445                 }
446         }
447
448         /*
449          * Process client-side list participations for this room
450          */
451         if (sc->RNCfg->NetConfigs[participate] != NULL) {
452                 msg = CtdlFetchMessage(msgnum, 1);
453                 if (msg != NULL) {
454
455                         /* Only send messages which originated on our own
456                          * Citadel network, otherwise we'll end up sending the
457                          * remote mailing list's messages back to it, which
458                          * is rude...
459                          */
460                         ok_to_participate = 0;
461                         if (msg->cm_fields['N'] != NULL) {
462                                 if (!strcasecmp(msg->cm_fields['N'],
463                                                 config.c_nodename)) {
464                                         ok_to_participate = 1;
465                                 }
466
467                                 Buf = NewStrBufPlain(msg->cm_fields['N'], -1);
468                                 if (CtdlIsValidNode(NULL,
469                                                     NULL,
470                                                     Buf,
471                                                     sc->working_ignetcfg,
472                                                     sc->the_netmap) == 0)
473                                 {
474                                         ok_to_participate = 1;
475                                 }
476                         }
477                         if (ok_to_participate) {
478                                 if (msg->cm_fields['F'] != NULL) {
479                                         free(msg->cm_fields['F']);
480                                 }
481                                 msg->cm_fields['F'] = malloc(SIZ);
482                                 /* Replace the Internet email address of the
483                                  * actual author with the email address of the
484                                  * room itself, so the remote listserv doesn't
485                                  * reject us.
486                                  * FIXME  I want to be able to pick any address
487                                 */
488                                 snprintf(msg->cm_fields['F'], SIZ,
489                                         "room_%s@%s", CC->room.QRname,
490                                         config.c_fqdn);
491                                 for (i=0; msg->cm_fields['F'][i]; ++i) {
492                                         if (isspace(msg->cm_fields['F'][i])) {
493                                                 msg->cm_fields['F'][i] = '_';
494                                         }
495                                 }
496
497                                 /*
498                                  * Figure out how big a buffer we need to alloc
499                                  */
500                                 for (nptr = sc->RNCfg->NetConfigs[participate];
501                                      nptr != NULL;
502                                      nptr = nptr->next)
503                                 {
504                                         if (msg->cm_fields['R'] != NULL) {
505                                                 free(msg->cm_fields['R']);
506                                         }
507                                         msg->cm_fields['R'] =
508                                                 strdup(ChrPtr(nptr->Value));
509
510                                         valid = validate_recipients(msg->cm_fields['R'],
511                                                                     NULL, 0);
512
513                                         CtdlSubmitMsg(msg, valid, "", 0);
514                                         free_recipients(valid);
515                                 }
516                         }
517                         CtdlFreeMessage(msg);
518                 }
519         }
520
521         /*
522          * Process IGnet push shares
523          */
524         msg = CtdlFetchMessage(msgnum, 1);
525         if (msg != NULL) {
526                 size_t newpath_len;
527
528                 /* Prepend our node name to the Path field whenever
529                  * sending a message to another IGnet node
530                  */
531                 if (msg->cm_fields['P'] == NULL) {
532                         msg->cm_fields['P'] = strdup("username");
533                 }
534                 newpath_len = strlen(msg->cm_fields['P']) +
535                          strlen(config.c_nodename) + 2;
536                 newpath = malloc(newpath_len);
537                 snprintf(newpath, newpath_len, "%s!%s",
538                          config.c_nodename, msg->cm_fields['P']);
539                 free(msg->cm_fields['P']);
540                 msg->cm_fields['P'] = newpath;
541
542                 /*
543                  * Determine if this message is set to be deleted
544                  * after sending out on the network
545                  */
546                 if (msg->cm_fields['S'] != NULL) {
547                         if (!strcasecmp(msg->cm_fields['S'], "CANCEL")) {
548                                 delete_after_send = 1;
549                         }
550                 }
551
552                 /* Now send it to every node */
553                 if (sc->RNCfg->NetConfigs[ignet_push_share] != NULL)
554                 for (mptr = (MapList*)sc->RNCfg->NetConfigs[ignet_push_share]; mptr != NULL;
555                     mptr = mptr->next) {
556
557                         send = 1;
558                         NewStrBufDupAppendFlush(&Buf, mptr->remote_nodename, NULL, 1);
559
560                         /* Check for valid node name */
561                         if (CtdlIsValidNode(NULL,
562                                             NULL,
563                                             Buf,
564                                             sc->working_ignetcfg,
565                                             sc->the_netmap) != 0)
566                         {
567                                 QN_syslog(LOG_ERR,
568                                           "Invalid node <%s>\n",
569                                           ChrPtr(mptr->remote_nodename));
570
571                                 send = 0;
572                         }
573
574                         /* Check for split horizon */
575                         QN_syslog(LOG_DEBUG, "Path is %s\n", msg->cm_fields['P']);
576                         bang = num_tokens(msg->cm_fields['P'], '!');
577                         if (bang > 1) {
578                                 for (i=0; i<(bang-1); ++i) {
579                                         extract_token(buf,
580                                                       msg->cm_fields['P'],
581                                                       i, '!',
582                                                       sizeof buf);
583                                         
584                                         QN_syslog(LOG_DEBUG, "Compare <%s> to <%s>\n",
585                                                   buf, ChrPtr(mptr->remote_nodename)) ;
586                                         if (!strcasecmp(buf, ChrPtr(mptr->remote_nodename))) {
587                                                 send = 0;
588                                                 break;
589                                         }
590                                 }
591
592                                 QN_syslog(LOG_INFO,
593                                           "%sSending to %s\n",
594                                           (send)?"":"Not ",
595                                           ChrPtr(mptr->remote_nodename));
596                         }
597
598                         /* Send the message */
599                         if (send == 1)
600                         {
601                                 /*
602                                  * Force the message to appear in the correct
603                                  * room on the far end by setting the C field
604                                  * correctly
605                                  */
606                                 if (msg->cm_fields['C'] != NULL) {
607                                         free(msg->cm_fields['C']);
608                                 }
609                                 if (StrLength(mptr->remote_roomname) > 0) {
610                                         msg->cm_fields['C'] =
611                                                 strdup(ChrPtr(mptr->remote_roomname));
612                                 }
613                                 else {
614                                         msg->cm_fields['C'] =
615                                                 strdup(CC->room.QRname);
616                                 }
617
618                                 /* serialize it for transmission */
619                                 serialize_message(&sermsg, msg);
620                                 if (sermsg.len > 0) {
621
622                                         /* write it to a spool file */
623                                         snprintf(filename,
624                                                  sizeof(filename),
625                                                  "%s/%s@%lx%x",
626                                                  ctdl_netout_dir,
627                                                  ChrPtr(mptr->remote_nodename),
628                                                  time(NULL),
629                                                  rand()
630                                         );
631
632                                         QN_syslog(LOG_DEBUG,
633                                                   "Appending to %s\n",
634                                                   filename);
635
636                                         fp = fopen(filename, "ab");
637                                         if (fp != NULL) {
638                                                 fwrite(sermsg.ser,
639                                                         sermsg.len, 1, fp);
640                                                 fclose(fp);
641                                         }
642                                         else {
643                                                 QN_syslog(LOG_ERR,
644                                                           "%s: %s\n",
645                                                           filename,
646                                                           strerror(errno));
647                                         }
648
649                                         /* free the serialized version */
650                                         free(sermsg.ser);
651                                 }
652
653                         }
654                 }
655                 CtdlFreeMessage(msg);
656         }
657
658         /* update lastsent */
659         sc->RNCfg->lastsent = msgnum;
660
661         /* Delete this message if delete-after-send is set */
662         if (delete_after_send) {
663                 CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, "");
664         }
665         FreeStrBuf(&Buf);
666 }