Netconfigs: start abstracting handling of network config files
[citadel.git] / citadel / modules / network / serv_netmail.c
1 /*
2  * This module handles shared rooms, inter-Citadel mail, and outbound
3  * mailing list processing.
4  *
5  * Copyright (c) 2000-2012 by the citadel.org team
6  *
7  *  This program is open source software; you can redistribute it and/or modify
8  *  it under the terms of the GNU General Public License, version 3.
9  *
10  *  This program is distributed in the hope that it will be useful,
11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  *  GNU General Public License for more details.
14  *
15  * ** NOTE **   A word on the S_NETCONFIGS semaphore:
16  * This is a fairly high-level type of critical section.  It ensures that no
17  * two threads work on the netconfigs files at the same time.  Since we do
18  * so many things inside these, here are the rules:
19  *  1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others.
20  *  2. Do *not* perform any I/O with the client during these sections.
21  *
22  */
23
24 /*
25  * Duration of time (in seconds) after which pending list subscribe/unsubscribe
26  * requests that have not been confirmed will be deleted.
27  */
28 #define EXP     259200  /* three days */
29
30 #include "sysdep.h"
31 #include <stdlib.h>
32 #include <unistd.h>
33 #include <stdio.h>
34 #include <fcntl.h>
35 #include <ctype.h>
36 #include <signal.h>
37 #include <pwd.h>
38 #include <errno.h>
39 #include <sys/stat.h>
40 #include <sys/types.h>
41 #include <dirent.h>
42 #if TIME_WITH_SYS_TIME
43 # include <sys/time.h>
44 # include <time.h>
45 #else
46 # if HAVE_SYS_TIME_H
47 #  include <sys/time.h>
48 # else
49 #  include <time.h>
50 # endif
51 #endif
52 #ifdef HAVE_SYSCALL_H
53 # include <syscall.h>
54 #else
55 # if HAVE_SYS_SYSCALL_H
56 #  include <sys/syscall.h>
57 # endif
58 #endif
59
60 #include <sys/wait.h>
61 #include <string.h>
62 #include <limits.h>
63 #include <libcitadel.h>
64 #include "citadel.h"
65 #include "server.h"
66 #include "citserver.h"
67 #include "support.h"
68 #include "config.h"
69 #include "user_ops.h"
70 #include "database.h"
71 #include "msgbase.h"
72 #include "internet_addressing.h"
73 #include "serv_network.h"
74 #include "clientsocket.h"
75 #include "file_ops.h"
76 #include "citadel_dirs.h"
77 #include "threads.h"
78 #include "context.h"
79 #include "netconfig.h"
80 #include "netspool.h"
81 #include "netmail.h"
82 #include "ctdl_module.h"
83
84
85 /*
86  * Deliver digest messages
87  */
88 void network_deliver_digest(SpoolControl *sc) {
89         struct CitContext *CCC = CC;
90         char buf[SIZ];
91         int i;
92         struct CtdlMessage *msg = NULL;
93         long msglen;
94         StrBuf *recps = NULL;
95         char *precps;
96         size_t recps_len = SIZ;
97         struct recptypes *valid;
98         namelist *nptr;
99         char bounce_to[256];
100
101         if (sc->num_msgs_spooled < 1) {
102                 fclose(sc->digestfp);
103                 sc->digestfp = NULL;
104                 return;
105         }
106
107         msg = malloc(sizeof(struct CtdlMessage));
108         memset(msg, 0, sizeof(struct CtdlMessage));
109         msg->cm_magic = CTDLMESSAGE_MAGIC;
110         msg->cm_format_type = FMT_RFC822;
111         msg->cm_anon_type = MES_NORMAL;
112
113         sprintf(buf, "%ld", time(NULL));
114         msg->cm_fields['T'] = strdup(buf);
115         msg->cm_fields['A'] = strdup(CC->room.QRname);
116         snprintf(buf, sizeof buf, "[%s]", CC->room.QRname);
117         msg->cm_fields['U'] = strdup(buf);
118         sprintf(buf, "room_%s@%s", CC->room.QRname, config.c_fqdn);
119         for (i=0; buf[i]; ++i) {
120                 if (isspace(buf[i])) buf[i]='_';
121                 buf[i] = tolower(buf[i]);
122         }
123         msg->cm_fields['F'] = strdup(buf);
124         msg->cm_fields['R'] = strdup(buf);
125
126         /* Set the 'List-ID' header */
127         msg->cm_fields['L'] = malloc(1024);
128         snprintf(msg->cm_fields['L'], 1024,
129                 "%s <%ld.list-id.%s>",
130                 CC->room.QRname,
131                 CC->room.QRnumber,
132                 config.c_fqdn
133         );
134
135         /*
136          * Go fetch the contents of the digest
137          */
138         fseek(sc->digestfp, 0L, SEEK_END);
139         msglen = ftell(sc->digestfp);
140
141         msg->cm_fields['M'] = malloc(msglen + 1);
142         fseek(sc->digestfp, 0L, SEEK_SET);
143         fread(msg->cm_fields['M'], (size_t)msglen, 1, sc->digestfp);
144         msg->cm_fields['M'][msglen] = '\0';
145
146         fclose(sc->digestfp);
147         sc->digestfp = NULL;
148
149         /* Now generate the delivery instructions */
150
151         /*
152          * Figure out how big a buffer we need to allocate
153          */
154         for (nptr = sc->NetConfigs[digestrecp]; nptr != NULL; nptr = nptr->next) {
155                 recps_len = recps_len + StrLength(nptr->Value) + 2;
156         }
157
158         recps = NewStrBufPlain(NULL, recps_len);
159
160         if (recps == NULL) {
161                 QN_syslog(LOG_EMERG,
162                           "Cannot allocate %ld bytes for recps...\n",
163                           (long)recps_len);
164                 abort();
165         }
166
167         /* Each recipient */
168         for (nptr = sc->NetConfigs[digestrecp]; nptr != NULL; nptr = nptr->next) {
169                 if (nptr != sc->NetConfigs[digestrecp]) {
170                         StrBufAppendBufPlain(recps, HKEY(","), 0);
171                 }
172                 StrBufAppendBuf(recps, nptr->Value, 0);
173         }
174
175         /* Where do we want bounces and other noise to be heard?
176          *Surely not the list members! */
177         snprintf(bounce_to, sizeof bounce_to, "room_aide@%s", config.c_fqdn);
178
179         /* Now submit the message */
180         precps = SmashStrBuf(&recps);
181         valid = validate_recipients(precps, NULL, 0);
182         free(precps);
183         if (valid != NULL) {
184                 valid->bounce_to = strdup(bounce_to);
185                 valid->envelope_from = strdup(bounce_to);
186                 CtdlSubmitMsg(msg, valid, NULL, 0);
187         }
188         CtdlFreeMessage(msg);
189         free_recipients(valid);
190 }
191
192
193 /*
194  * Deliver list messages to everyone on the list ... efficiently
195  */
196 void network_deliver_list(struct CtdlMessage *msg, SpoolControl *sc, const char *RoomName)
197 {
198         struct CitContext *CCC = CC;
199         StrBuf *recps = NULL;
200         char *precps = NULL;
201         size_t recps_len = SIZ;
202         struct recptypes *valid;
203         namelist *nptr;
204         char bounce_to[256];
205
206         /* Don't do this if there were no recipients! */
207         if (sc->NetConfigs[listrecp] == NULL) return;
208
209         /* Now generate the delivery instructions */
210
211         /*
212          * Figure out how big a buffer we need to allocate
213          */
214         for (nptr = sc->NetConfigs[listrecp]; nptr != NULL; nptr = nptr->next) {
215                 recps_len = recps_len + StrLength(nptr->Value) + 2;
216         }
217
218         recps = NewStrBufPlain(NULL, recps_len);
219
220         if (recps == NULL) {
221                 QN_syslog(LOG_EMERG,
222                           "Cannot allocate %ld bytes for recps...\n",
223                           (long)recps_len);
224                 abort();
225         }
226
227         /* Each recipient */
228         for (nptr = sc->NetConfigs[listrecp]; nptr != NULL; nptr = nptr->next) {
229                 if (nptr != sc->NetConfigs[listrecp]) {
230                         StrBufAppendBufPlain(recps, HKEY(","), 0);
231                 }
232                 StrBufAppendBuf(recps, nptr->Value, 0);
233         }
234
235         /* Where do we want bounces and other noise to be heard?
236          *  Surely not the list members! */
237         snprintf(bounce_to, sizeof bounce_to, "room_aide@%s", config.c_fqdn);
238
239         /* Now submit the message */
240         precps = SmashStrBuf(&recps);
241         valid = validate_recipients(precps, NULL, 0);
242         free(precps);
243         if (valid != NULL) {
244                 valid->bounce_to = strdup(bounce_to);
245                 valid->envelope_from = strdup(bounce_to);
246                 valid->sending_room = strdup(RoomName);
247                 CtdlSubmitMsg(msg, valid, NULL, 0);
248                 free_recipients(valid);
249         }
250         /* Do not call CtdlFreeMessage(msg) here; the caller will free it. */
251 }
252
253
254 /*
255  * Spools out one message from the list.
256  */
257 void network_spool_msg(long msgnum,
258                        void *userdata)
259 {
260         struct CitContext *CCC = CC;
261         StrBuf *Buf = NULL;
262         SpoolControl *sc;
263         int i;
264         char *newpath = NULL;
265         struct CtdlMessage *msg = NULL;
266         namelist *nptr;
267         maplist *mptr;
268         struct ser_ret sermsg;
269         FILE *fp;
270         char filename[PATH_MAX];
271         char buf[SIZ];
272         int bang = 0;
273         int send = 1;
274         int delete_after_send = 0;      /* Set to 1 to delete after spooling */
275         int ok_to_participate = 0;
276         struct recptypes *valid;
277
278         sc = (SpoolControl *)userdata;
279
280         /*
281          * Process mailing list recipients
282          */
283         if (sc->NetConfigs[listrecp] != NULL) {
284                 /* Fetch the message.  We're going to need to modify it
285                  * in order to insert the [list name] in it, etc.
286                  */
287                 msg = CtdlFetchMessage(msgnum, 1);
288                 if (msg != NULL) {
289                         int rlen;
290                         char *pCh;
291                         StrBuf *Subject, *FlatSubject;
292
293                         if (msg->cm_fields['K'] != NULL)
294                                 free(msg->cm_fields['K']);
295                         if (msg->cm_fields['V'] == NULL){
296                                 /* local message, no enVelope */
297                                 StrBuf *Buf;
298                                 Buf = NewStrBuf();
299                                 StrBufAppendBufPlain(Buf,
300                                                      msg->cm_fields['O']
301                                                      , -1, 0);
302                                 StrBufAppendBufPlain(Buf, HKEY("@"), 0);
303                                 StrBufAppendBufPlain(Buf, config.c_fqdn, -1, 0);
304
305                                 msg->cm_fields['K'] = SmashStrBuf(&Buf);
306                         }
307                         else {
308                                 msg->cm_fields['K'] =
309                                         strdup (msg->cm_fields['V']);
310                         }
311                         /* Set the 'List-ID' header */
312                         if (msg->cm_fields['L'] != NULL) {
313                                 free(msg->cm_fields['L']);
314                         }
315                         msg->cm_fields['L'] = malloc(1024);
316                         snprintf(msg->cm_fields['L'], 1024,
317                                 "%s <%ld.list-id.%s>",
318                                 CC->room.QRname,
319                                 CC->room.QRnumber,
320                                 config.c_fqdn
321                         );
322
323                         /* Prepend "[List name]" to the subject */
324                         if (msg->cm_fields['U'] == NULL) {
325                                 Subject = NewStrBufPlain(HKEY("(no subject)"));
326                         }
327                         else {
328                                 Subject = NewStrBufPlain(
329                                         msg->cm_fields['U'], -1);
330                         }
331                         FlatSubject = NewStrBufPlain(NULL, StrLength(Subject));
332                         StrBuf_RFC822_to_Utf8(FlatSubject, Subject, NULL, NULL);
333
334                         rlen = strlen(CC->room.QRname);
335                         pCh  = strstr(ChrPtr(FlatSubject), CC->room.QRname);
336                         if ((pCh == NULL) ||
337                             (*(pCh + rlen) != ']') ||
338                             (pCh == ChrPtr(FlatSubject)) ||
339                             (*(pCh - 1) != '[')
340                                 )
341                         {
342                                 StrBuf *tmp;
343                                 StrBufPlain(Subject, HKEY("["));
344                                 StrBufAppendBufPlain(Subject,
345                                                      CC->room.QRname,
346                                                      rlen, 0);
347                                 StrBufAppendBufPlain(Subject, HKEY("] "), 0);
348                                 StrBufAppendBuf(Subject, FlatSubject, 0);
349                                  /* so we can free the right one swap them */
350                                 tmp = Subject;
351                                 Subject = FlatSubject;
352                                 FlatSubject = tmp;
353                                 StrBufRFC2047encode(&Subject, FlatSubject);
354                         }
355
356                         if (msg->cm_fields['U'] != NULL)
357                                 free (msg->cm_fields['U']);
358                         msg->cm_fields['U'] = SmashStrBuf(&Subject);
359
360                         FreeStrBuf(&FlatSubject);
361
362                         /* else we won't modify the buffer, since the
363                          * roomname is already here.
364                          */
365
366                         /* if there is no other recipient, Set the recipient
367                          * of the list message to the email address of the
368                          * room itself.
369                          */
370                         if ((msg->cm_fields['R'] == NULL) ||
371                             IsEmptyStr(msg->cm_fields['R']))
372                         {
373                                 if (msg->cm_fields['R'] != NULL)
374                                         free(msg->cm_fields['R']);
375
376                                 msg->cm_fields['R'] = malloc(256);
377                                 snprintf(msg->cm_fields['R'], 256,
378                                          "room_%s@%s", CC->room.QRname,
379                                          config.c_fqdn);
380                                 for (i=0; msg->cm_fields['R'][i]; ++i) {
381                                         if (isspace(msg->cm_fields['R'][i])) {
382                                                 msg->cm_fields['R'][i] = '_';
383                                         }
384                                 }
385                         }
386
387                         /* Handle delivery */
388                         network_deliver_list(msg, sc, CC->room.QRname);
389                         CtdlFreeMessage(msg);
390                 }
391         }
392
393         /*
394          * Process digest recipients
395          */
396         if ((sc->NetConfigs[digestrecp] != NULL) && (sc->digestfp != NULL)) {
397                 msg = CtdlFetchMessage(msgnum, 1);
398                 if (msg != NULL) {
399                         fprintf(sc->digestfp,
400                                 " -----------------------------------"
401                                 "------------------------------------"
402                                 "-------\n");
403                         fprintf(sc->digestfp, "From: ");
404                         if (msg->cm_fields['A'] != NULL) {
405                                 fprintf(sc->digestfp,
406                                         "%s ",
407                                         msg->cm_fields['A']);
408                         }
409                         if (msg->cm_fields['F'] != NULL) {
410                                 fprintf(sc->digestfp,
411                                         "<%s> ",
412                                         msg->cm_fields['F']);
413                         }
414                         else if (msg->cm_fields['N'] != NULL) {
415                                 fprintf(sc->digestfp,
416                                         "@%s ",
417                                         msg->cm_fields['N']);
418                         }
419                         fprintf(sc->digestfp, "\n");
420                         if (msg->cm_fields['U'] != NULL) {
421                                 fprintf(sc->digestfp,
422                                         "Subject: %s\n",
423                                         msg->cm_fields['U']);
424                         }
425
426                         CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
427
428                         safestrncpy(CC->preferred_formats,
429                                     "text/plain",
430                                     sizeof CC->preferred_formats);
431
432                         CtdlOutputPreLoadedMsg(msg,
433                                                MT_CITADEL,
434                                                HEADERS_NONE,
435                                                0, 0, 0);
436
437                         StrBufTrim(CC->redirect_buffer);
438                         fwrite(HKEY("\n"), 1, sc->digestfp);
439                         fwrite(SKEY(CC->redirect_buffer), 1, sc->digestfp);
440                         fwrite(HKEY("\n"), 1, sc->digestfp);
441
442                         FreeStrBuf(&CC->redirect_buffer);
443
444                         sc->num_msgs_spooled += 1;
445                         CtdlFreeMessage(msg);
446                 }
447         }
448
449         /*
450          * Process client-side list participations for this room
451          */
452         if (sc->NetConfigs[participate] != NULL) {
453                 msg = CtdlFetchMessage(msgnum, 1);
454                 if (msg != NULL) {
455
456                         /* Only send messages which originated on our own
457                          * Citadel network, otherwise we'll end up sending the
458                          * remote mailing list's messages back to it, which
459                          * is rude...
460                          */
461                         ok_to_participate = 0;
462                         if (msg->cm_fields['N'] != NULL) {
463                                 if (!strcasecmp(msg->cm_fields['N'],
464                                                 config.c_nodename)) {
465                                         ok_to_participate = 1;
466                                 }
467
468                                 Buf = NewStrBufPlain(msg->cm_fields['N'], -1);
469                                 if (is_valid_node(NULL,
470                                                   NULL,
471                                                   Buf,
472                                                   sc->working_ignetcfg,
473                                                   sc->the_netmap) == 0)
474                                 {
475                                         ok_to_participate = 1;
476                                 }
477                         }
478                         if (ok_to_participate) {
479                                 if (msg->cm_fields['F'] != NULL) {
480                                         free(msg->cm_fields['F']);
481                                 }
482                                 msg->cm_fields['F'] = malloc(SIZ);
483                                 /* Replace the Internet email address of the
484                                  * actual author with the email address of the
485                                  * room itself, so the remote listserv doesn't
486                                  * reject us.
487                                  * FIXME  I want to be able to pick any address
488                                 */
489                                 snprintf(msg->cm_fields['F'], SIZ,
490                                         "room_%s@%s", CC->room.QRname,
491                                         config.c_fqdn);
492                                 for (i=0; msg->cm_fields['F'][i]; ++i) {
493                                         if (isspace(msg->cm_fields['F'][i])) {
494                                                 msg->cm_fields['F'][i] = '_';
495                                         }
496                                 }
497
498                                 /*
499                                  * Figure out how big a buffer we need to alloc
500                                  */
501                                 for (nptr = sc->NetConfigs[participate];
502                                      nptr != NULL;
503                                      nptr = nptr->next)
504                                 {
505                                         if (msg->cm_fields['R'] != NULL) {
506                                                 free(msg->cm_fields['R']);
507                                         }
508                                         msg->cm_fields['R'] =
509                                                 strdup(ChrPtr(nptr->Value));
510
511                                         valid = validate_recipients(msg->cm_fields['R'],
512                                                                     NULL, 0);
513
514                                         CtdlSubmitMsg(msg, valid, "", 0);
515                                         free_recipients(valid);
516                                 }
517                         }
518                         CtdlFreeMessage(msg);
519                 }
520         }
521
522         /*
523          * Process IGnet push shares
524          */
525         msg = CtdlFetchMessage(msgnum, 1);
526         if (msg != NULL) {
527                 size_t newpath_len;
528
529                 /* Prepend our node name to the Path field whenever
530                  * sending a message to another IGnet node
531                  */
532                 if (msg->cm_fields['P'] == NULL) {
533                         msg->cm_fields['P'] = strdup("username");
534                 }
535                 newpath_len = strlen(msg->cm_fields['P']) +
536                          strlen(config.c_nodename) + 2;
537                 newpath = malloc(newpath_len);
538                 snprintf(newpath, newpath_len, "%s!%s",
539                          config.c_nodename, msg->cm_fields['P']);
540                 free(msg->cm_fields['P']);
541                 msg->cm_fields['P'] = newpath;
542
543                 /*
544                  * Determine if this message is set to be deleted
545                  * after sending out on the network
546                  */
547                 if (msg->cm_fields['S'] != NULL) {
548                         if (!strcasecmp(msg->cm_fields['S'], "CANCEL")) {
549                                 delete_after_send = 1;
550                         }
551                 }
552
553                 /* Now send it to every node */
554                 if (sc->NetConfigs[ignet_push_share] != NULL)
555                 for (mptr = (maplist*)sc->NetConfigs[ignet_push_share]; mptr != NULL;
556                     mptr = mptr->next) {
557
558                         send = 1;
559                         NewStrBufDupAppendFlush(&Buf, mptr->remote_nodename, NULL, 1);
560
561                         /* Check for valid node name */
562                         if (is_valid_node(NULL,
563                                           NULL,
564                                           Buf,
565                                           sc->working_ignetcfg,
566                                           sc->the_netmap) != 0)
567                         {
568                                 QN_syslog(LOG_ERR,
569                                           "Invalid node <%s>\n",
570                                           ChrPtr(mptr->remote_nodename));
571
572                                 send = 0;
573                         }
574
575                         /* Check for split horizon */
576                         QN_syslog(LOG_DEBUG, "Path is %s\n", msg->cm_fields['P']);
577                         bang = num_tokens(msg->cm_fields['P'], '!');
578                         if (bang > 1) {
579                                 for (i=0; i<(bang-1); ++i) {
580                                         extract_token(buf,
581                                                       msg->cm_fields['P'],
582                                                       i, '!',
583                                                       sizeof buf);
584                                         
585                                         QN_syslog(LOG_DEBUG, "Compare <%s> to <%s>\n",
586                                                   buf, ChrPtr(mptr->remote_nodename)) ;
587                                         if (!strcasecmp(buf, ChrPtr(mptr->remote_nodename))) {
588                                                 send = 0;
589                                                 break;
590                                         }
591                                 }
592
593                                 QN_syslog(LOG_INFO,
594                                           "%sSending to %s\n",
595                                           (send)?"":"Not ",
596                                           ChrPtr(mptr->remote_nodename));
597                         }
598
599                         /* Send the message */
600                         if (send == 1)
601                         {
602                                 /*
603                                  * Force the message to appear in the correct
604                                  * room on the far end by setting the C field
605                                  * correctly
606                                  */
607                                 if (msg->cm_fields['C'] != NULL) {
608                                         free(msg->cm_fields['C']);
609                                 }
610                                 if (StrLength(mptr->remote_roomname) > 0) {
611                                         msg->cm_fields['C'] =
612                                                 strdup(ChrPtr(mptr->remote_roomname));
613                                 }
614                                 else {
615                                         msg->cm_fields['C'] =
616                                                 strdup(CC->room.QRname);
617                                 }
618
619                                 /* serialize it for transmission */
620                                 serialize_message(&sermsg, msg);
621                                 if (sermsg.len > 0) {
622
623                                         /* write it to a spool file */
624                                         snprintf(filename,
625                                                  sizeof(filename),
626                                                  "%s/%s@%lx%x",
627                                                  ctdl_netout_dir,
628                                                  ChrPtr(mptr->remote_nodename),
629                                                  time(NULL),
630                                                  rand()
631                                         );
632
633                                         QN_syslog(LOG_DEBUG,
634                                                   "Appending to %s\n",
635                                                   filename);
636
637                                         fp = fopen(filename, "ab");
638                                         if (fp != NULL) {
639                                                 fwrite(sermsg.ser,
640                                                         sermsg.len, 1, fp);
641                                                 fclose(fp);
642                                         }
643                                         else {
644                                                 QN_syslog(LOG_ERR,
645                                                           "%s: %s\n",
646                                                           filename,
647                                                           strerror(errno));
648                                         }
649
650                                         /* free the serialized version */
651                                         free(sermsg.ser);
652                                 }
653
654                         }
655                 }
656                 CtdlFreeMessage(msg);
657         }
658
659         /* update lastsent */
660         sc->lastsent = msgnum;
661
662         /* Delete this message if delete-after-send is set */
663         if (delete_after_send) {
664                 CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, "");
665         }
666         FreeStrBuf(&Buf);
667 }