NetQueue: make logging runtime configurable; add context.
[citadel.git] / citadel / modules / network / serv_netmail.c
1 /*
2  * This module handles shared rooms, inter-Citadel mail, and outbound
3  * mailing list processing.
4  *
5  * Copyright (c) 2000-2012 by the citadel.org team
6  *
7  *  This program is open source software; you can redistribute it and/or modify
8  *  it under the terms of the GNU General Public License as published by
9  *  the Free Software Foundation; either version 3 of the License, or
10  *  (at your option) any later version.
11  *
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  *  GNU General Public License for more details.
16  *
17  *  You should have received a copy of the GNU General Public License
18  *  along with this program; if not, write to the Free Software
19  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20  *
21  * ** NOTE **   A word on the S_NETCONFIGS semaphore:
22  * This is a fairly high-level type of critical section.  It ensures that no
23  * two threads work on the netconfigs files at the same time.  Since we do
24  * so many things inside these, here are the rules:
25  *  1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others.
26  *  2. Do *not* perform any I/O with the client during these sections.
27  *
28  */
29
30 /*
31  * Duration of time (in seconds) after which pending list subscribe/unsubscribe
32  * requests that have not been confirmed will be deleted.
33  */
34 #define EXP     259200  /* three days */
35
36 #include "sysdep.h"
37 #include <stdlib.h>
38 #include <unistd.h>
39 #include <stdio.h>
40 #include <fcntl.h>
41 #include <ctype.h>
42 #include <signal.h>
43 #include <pwd.h>
44 #include <errno.h>
45 #include <sys/stat.h>
46 #include <sys/types.h>
47 #include <dirent.h>
48 #if TIME_WITH_SYS_TIME
49 # include <sys/time.h>
50 # include <time.h>
51 #else
52 # if HAVE_SYS_TIME_H
53 #  include <sys/time.h>
54 # else
55 #  include <time.h>
56 # endif
57 #endif
58 #ifdef HAVE_SYSCALL_H
59 # include <syscall.h>
60 #else
61 # if HAVE_SYS_SYSCALL_H
62 #  include <sys/syscall.h>
63 # endif
64 #endif
65
66 #include <sys/wait.h>
67 #include <string.h>
68 #include <limits.h>
69 #include <libcitadel.h>
70 #include "citadel.h"
71 #include "server.h"
72 #include "citserver.h"
73 #include "support.h"
74 #include "config.h"
75 #include "user_ops.h"
76 #include "database.h"
77 #include "msgbase.h"
78 #include "internet_addressing.h"
79 #include "serv_network.h"
80 #include "clientsocket.h"
81 #include "file_ops.h"
82 #include "citadel_dirs.h"
83 #include "threads.h"
84
85 #ifndef HAVE_SNPRINTF
86 #include "snprintf.h"
87 #endif
88
89 #include "context.h"
90 #include "netconfig.h"
91 #include "netspool.h"
92 #include "netmail.h"
93 #include "ctdl_module.h"
94
95
96 /*
97  * Deliver digest messages
98  */
99 void network_deliver_digest(SpoolControl *sc) {
100         struct CitContext *CCC = CC;
101         char buf[SIZ];
102         int i;
103         struct CtdlMessage *msg = NULL;
104         long msglen;
105         char *recps = NULL;
106         size_t recps_len = SIZ;
107         struct recptypes *valid;
108         namelist *nptr;
109         char bounce_to[256];
110
111         if (sc->num_msgs_spooled < 1) {
112                 fclose(sc->digestfp);
113                 sc->digestfp = NULL;
114                 return;
115         }
116
117         msg = malloc(sizeof(struct CtdlMessage));
118         memset(msg, 0, sizeof(struct CtdlMessage));
119         msg->cm_magic = CTDLMESSAGE_MAGIC;
120         msg->cm_format_type = FMT_RFC822;
121         msg->cm_anon_type = MES_NORMAL;
122
123         sprintf(buf, "%ld", time(NULL));
124         msg->cm_fields['T'] = strdup(buf);
125         msg->cm_fields['A'] = strdup(CC->room.QRname);
126         snprintf(buf, sizeof buf, "[%s]", CC->room.QRname);
127         msg->cm_fields['U'] = strdup(buf);
128         sprintf(buf, "room_%s@%s", CC->room.QRname, config.c_fqdn);
129         for (i=0; buf[i]; ++i) {
130                 if (isspace(buf[i])) buf[i]='_';
131                 buf[i] = tolower(buf[i]);
132         }
133         msg->cm_fields['F'] = strdup(buf);
134         msg->cm_fields['R'] = strdup(buf);
135
136         /* Set the 'List-ID' header */
137         msg->cm_fields['L'] = malloc(1024);
138         snprintf(msg->cm_fields['L'], 1024,
139                 "%s <%ld.list-id.%s>",
140                 CC->room.QRname,
141                 CC->room.QRnumber,
142                 config.c_fqdn
143         );
144
145         /*
146          * Go fetch the contents of the digest
147          */
148         fseek(sc->digestfp, 0L, SEEK_END);
149         msglen = ftell(sc->digestfp);
150
151         msg->cm_fields['M'] = malloc(msglen + 1);
152         fseek(sc->digestfp, 0L, SEEK_SET);
153         fread(msg->cm_fields['M'], (size_t)msglen, 1, sc->digestfp);
154         msg->cm_fields['M'][msglen] = '\0';
155
156         fclose(sc->digestfp);
157         sc->digestfp = NULL;
158
159         /* Now generate the delivery instructions */
160
161         /*
162          * Figure out how big a buffer we need to allocate
163          */
164         for (nptr = sc->digestrecps; nptr != NULL; nptr = nptr->next) {
165                 recps_len = recps_len + strlen(nptr->name) + 2;
166         }
167
168         recps = malloc(recps_len);
169
170         if (recps == NULL) {
171                 QN_syslog(LOG_EMERG,
172                           "Cannot allocate %ld bytes for recps...\n",
173                           (long)recps_len);
174                 abort();
175         }
176
177         strcpy(recps, "");
178
179         /* Each recipient */
180         for (nptr = sc->digestrecps; nptr != NULL; nptr = nptr->next) {
181                 if (nptr != sc->digestrecps) {
182                         strcat(recps, ",");
183                 }
184                 strcat(recps, nptr->name);
185         }
186
187         /* Where do we want bounces and other noise to be heard?
188          *Surely not the list members! */
189         snprintf(bounce_to, sizeof bounce_to, "room_aide@%s", config.c_fqdn);
190
191         /* Now submit the message */
192         valid = validate_recipients(recps, NULL, 0);
193         free(recps);
194         if (valid != NULL) {
195                 valid->bounce_to = strdup(bounce_to);
196                 valid->envelope_from = strdup(bounce_to);
197                 CtdlSubmitMsg(msg, valid, NULL, 0);
198         }
199         CtdlFreeMessage(msg);
200         free_recipients(valid);
201 }
202
203
204 /*
205  * Deliver list messages to everyone on the list ... efficiently
206  */
207 void network_deliver_list(struct CtdlMessage *msg, SpoolControl *sc, const char *RoomName)
208 {
209         struct CitContext *CCC = CC;
210         char *recps = NULL;
211         size_t recps_len = SIZ;
212         struct recptypes *valid;
213         namelist *nptr;
214         char bounce_to[256];
215
216         /* Don't do this if there were no recipients! */
217         if (sc->listrecps == NULL) return;
218
219         /* Now generate the delivery instructions */
220
221         /*
222          * Figure out how big a buffer we need to allocate
223          */
224         for (nptr = sc->listrecps; nptr != NULL; nptr = nptr->next) {
225                 recps_len = recps_len + strlen(nptr->name) + 2;
226         }
227
228         recps = malloc(recps_len);
229
230         if (recps == NULL) {
231                 QN_syslog(LOG_EMERG,
232                           "Cannot allocate %ld bytes for recps...\n",
233                           (long)recps_len);
234                 abort();
235         }
236
237         strcpy(recps, "");
238
239         /* Each recipient */
240         for (nptr = sc->listrecps; nptr != NULL; nptr = nptr->next) {
241                 if (nptr != sc->listrecps) {
242                         strcat(recps, ",");
243                 }
244                 strcat(recps, nptr->name);
245         }
246
247         /* Where do we want bounces and other noise to be heard?
248          *  Surely not the list members! */
249         snprintf(bounce_to, sizeof bounce_to, "room_aide@%s", config.c_fqdn);
250
251         /* Now submit the message */
252         valid = validate_recipients(recps, NULL, 0);
253         free(recps);
254         if (valid != NULL) {
255                 valid->bounce_to = strdup(bounce_to);
256                 valid->envelope_from = strdup(bounce_to);
257                 valid->sending_room = strdup(RoomName);
258                 CtdlSubmitMsg(msg, valid, NULL, 0);
259                 free_recipients(valid);
260         }
261         /* Do not call CtdlFreeMessage(msg) here; the caller will free it. */
262 }
263
264
265 /*
266  * Spools out one message from the list.
267  */
268 void network_spool_msg(long msgnum,
269                        void *userdata)
270 {
271         struct CitContext *CCC = CC;
272         StrBuf *Buf = NULL;
273         SpoolControl *sc;
274         int i;
275         char *newpath = NULL;
276         struct CtdlMessage *msg = NULL;
277         namelist *nptr;
278         maplist *mptr;
279         struct ser_ret sermsg;
280         FILE *fp;
281         char filename[PATH_MAX];
282         char buf[SIZ];
283         int bang = 0;
284         int send = 1;
285         int delete_after_send = 0;      /* Set to 1 to delete after spooling */
286         int ok_to_participate = 0;
287         struct recptypes *valid;
288
289         sc = (SpoolControl *)userdata;
290
291         /*
292          * Process mailing list recipients
293          */
294         if (sc->listrecps != NULL) {
295                 /* Fetch the message.  We're going to need to modify it
296                  * in order to insert the [list name] in it, etc.
297                  */
298                 msg = CtdlFetchMessage(msgnum, 1);
299                 if (msg != NULL) {
300                         int rlen;
301                         char *pCh;
302                         StrBuf *Subject, *FlatSubject;
303
304                         if (msg->cm_fields['K'] != NULL)
305                                 free(msg->cm_fields['K']);
306                         if (msg->cm_fields['V'] == NULL){
307                                 /* local message, no enVelope */
308                                 StrBuf *Buf;
309                                 Buf = NewStrBuf();
310                                 StrBufAppendBufPlain(Buf,
311                                                      msg->cm_fields['O']
312                                                      , -1, 0);
313                                 StrBufAppendBufPlain(Buf, HKEY("@"), 0);
314                                 StrBufAppendBufPlain(Buf, config.c_fqdn, -1, 0);
315
316                                 msg->cm_fields['K'] = SmashStrBuf(&Buf);
317                         }
318                         else {
319                                 msg->cm_fields['K'] =
320                                         strdup (msg->cm_fields['V']);
321                         }
322                         /* Set the 'List-ID' header */
323                         if (msg->cm_fields['L'] != NULL) {
324                                 free(msg->cm_fields['L']);
325                         }
326                         msg->cm_fields['L'] = malloc(1024);
327                         snprintf(msg->cm_fields['L'], 1024,
328                                 "%s <%ld.list-id.%s>",
329                                 CC->room.QRname,
330                                 CC->room.QRnumber,
331                                 config.c_fqdn
332                         );
333
334                         /* Prepend "[List name]" to the subject */
335                         if (msg->cm_fields['U'] == NULL) {
336                                 Subject = NewStrBufPlain(HKEY("(no subject)"));
337                         }
338                         else {
339                                 Subject = NewStrBufPlain(
340                                         msg->cm_fields['U'], -1);
341                         }
342                         FlatSubject = NewStrBufPlain(NULL, StrLength(Subject));
343                         StrBuf_RFC822_to_Utf8(FlatSubject, Subject, NULL, NULL);
344
345                         rlen = strlen(CC->room.QRname);
346                         pCh  = strstr(ChrPtr(FlatSubject), CC->room.QRname);
347                         if ((pCh == NULL) ||
348                             (*(pCh + rlen) != ']') ||
349                             (pCh == ChrPtr(FlatSubject)) ||
350                             (*(pCh - 1) != '[')
351                                 )
352                         {
353                                 StrBuf *tmp;
354                                 StrBufPlain(Subject, HKEY("["));
355                                 StrBufAppendBufPlain(Subject,
356                                                      CC->room.QRname,
357                                                      rlen, 0);
358                                 StrBufAppendBufPlain(Subject, HKEY("] "), 0);
359                                 StrBufAppendBuf(Subject, FlatSubject, 0);
360                                  /* so we can free the right one swap them */
361                                 tmp = Subject;
362                                 Subject = FlatSubject;
363                                 FlatSubject = tmp;
364                                 StrBufRFC2047encode(&Subject, FlatSubject);
365                         }
366
367                         if (msg->cm_fields['U'] != NULL)
368                                 free (msg->cm_fields['U']);
369                         msg->cm_fields['U'] = SmashStrBuf(&Subject);
370
371                         FreeStrBuf(&FlatSubject);
372
373                         /* else we won't modify the buffer, since the
374                          * roomname is already here.
375                          */
376
377                         /* if there is no other recipient, Set the recipient
378                          * of the list message to the email address of the
379                          * room itself.
380                          */
381                         if ((msg->cm_fields['R'] == NULL) ||
382                             IsEmptyStr(msg->cm_fields['R']))
383                         {
384                                 if (msg->cm_fields['R'] != NULL)
385                                         free(msg->cm_fields['R']);
386
387                                 msg->cm_fields['R'] = malloc(256);
388                                 snprintf(msg->cm_fields['R'], 256,
389                                          "room_%s@%s", CC->room.QRname,
390                                          config.c_fqdn);
391                                 for (i=0; msg->cm_fields['R'][i]; ++i) {
392                                         if (isspace(msg->cm_fields['R'][i])) {
393                                                 msg->cm_fields['R'][i] = '_';
394                                         }
395                                 }
396                         }
397
398                         /* Handle delivery */
399                         network_deliver_list(msg, sc, CC->room.QRname);
400                         CtdlFreeMessage(msg);
401                 }
402         }
403
404         /*
405          * Process digest recipients
406          */
407         if ((sc->digestrecps != NULL) && (sc->digestfp != NULL)) {
408                 msg = CtdlFetchMessage(msgnum, 1);
409                 if (msg != NULL) {
410                         fprintf(sc->digestfp,
411                                 " -----------------------------------"
412                                 "------------------------------------"
413                                 "-------\n");
414                         fprintf(sc->digestfp, "From: ");
415                         if (msg->cm_fields['A'] != NULL) {
416                                 fprintf(sc->digestfp,
417                                         "%s ",
418                                         msg->cm_fields['A']);
419                         }
420                         if (msg->cm_fields['F'] != NULL) {
421                                 fprintf(sc->digestfp,
422                                         "<%s> ",
423                                         msg->cm_fields['F']);
424                         }
425                         else if (msg->cm_fields['N'] != NULL) {
426                                 fprintf(sc->digestfp,
427                                         "@%s ",
428                                         msg->cm_fields['N']);
429                         }
430                         fprintf(sc->digestfp, "\n");
431                         if (msg->cm_fields['U'] != NULL) {
432                                 fprintf(sc->digestfp,
433                                         "Subject: %s\n",
434                                         msg->cm_fields['U']);
435                         }
436
437                         CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
438
439                         safestrncpy(CC->preferred_formats,
440                                     "text/plain",
441                                     sizeof CC->preferred_formats);
442
443                         CtdlOutputPreLoadedMsg(msg,
444                                                MT_CITADEL,
445                                                HEADERS_NONE,
446                                                0, 0, 0);
447
448                         StrBufTrim(CC->redirect_buffer);
449                         fwrite(HKEY("\n"), 1, sc->digestfp);
450                         fwrite(SKEY(CC->redirect_buffer), 1, sc->digestfp);
451                         fwrite(HKEY("\n"), 1, sc->digestfp);
452
453                         FreeStrBuf(&CC->redirect_buffer);
454
455                         sc->num_msgs_spooled += 1;
456                         CtdlFreeMessage(msg);
457                 }
458         }
459
460         /*
461          * Process client-side list participations for this room
462          */
463         if (sc->participates != NULL) {
464                 msg = CtdlFetchMessage(msgnum, 1);
465                 if (msg != NULL) {
466
467                         /* Only send messages which originated on our own
468                          * Citadel network, otherwise we'll end up sending the
469                          * remote mailing list's messages back to it, which
470                          * is rude...
471                          */
472                         ok_to_participate = 0;
473                         if (msg->cm_fields['N'] != NULL) {
474                                 if (!strcasecmp(msg->cm_fields['N'],
475                                                 config.c_nodename)) {
476                                         ok_to_participate = 1;
477                                 }
478
479                                 Buf = NewStrBufPlain(msg->cm_fields['N'], -1);
480                                 if (is_valid_node(NULL,
481                                                   NULL,
482                                                   Buf,
483                                                   sc->working_ignetcfg,
484                                                   sc->the_netmap) == 0)
485                                 {
486                                         ok_to_participate = 1;
487                                 }
488                         }
489                         if (ok_to_participate) {
490                                 if (msg->cm_fields['F'] != NULL) {
491                                         free(msg->cm_fields['F']);
492                                 }
493                                 msg->cm_fields['F'] = malloc(SIZ);
494                                 /* Replace the Internet email address of the
495                                  * actual author with the email address of the
496                                  * room itself, so the remote listserv doesn't
497                                  * reject us.
498                                  * FIXME  I want to be able to pick any address
499                                 */
500                                 snprintf(msg->cm_fields['F'], SIZ,
501                                         "room_%s@%s", CC->room.QRname,
502                                         config.c_fqdn);
503                                 for (i=0; msg->cm_fields['F'][i]; ++i) {
504                                         if (isspace(msg->cm_fields['F'][i])) {
505                                                 msg->cm_fields['F'][i] = '_';
506                                         }
507                                 }
508
509                                 /*
510                                  * Figure out how big a buffer we need to alloc
511                                  */
512                                 for (nptr = sc->participates;
513                                      nptr != NULL;
514                                      nptr = nptr->next)
515                                 {
516                                         if (msg->cm_fields['R'] != NULL) {
517                                                 free(msg->cm_fields['R']);
518                                         }
519                                         msg->cm_fields['R'] =
520                                                 strdup(nptr->name);
521
522                                         valid = validate_recipients(nptr->name,
523                                                                     NULL, 0);
524
525                                         CtdlSubmitMsg(msg, valid, "", 0);
526                                         free_recipients(valid);
527                                 }
528                         }
529                         CtdlFreeMessage(msg);
530                 }
531         }
532
533         /*
534          * Process IGnet push shares
535          */
536         msg = CtdlFetchMessage(msgnum, 1);
537         if (msg != NULL) {
538                 size_t newpath_len;
539
540                 /* Prepend our node name to the Path field whenever
541                  * sending a message to another IGnet node
542                  */
543                 if (msg->cm_fields['P'] == NULL) {
544                         msg->cm_fields['P'] = strdup("username");
545                 }
546                 newpath_len = strlen(msg->cm_fields['P']) +
547                          strlen(config.c_nodename) + 2;
548                 newpath = malloc(newpath_len);
549                 snprintf(newpath, newpath_len, "%s!%s",
550                          config.c_nodename, msg->cm_fields['P']);
551                 free(msg->cm_fields['P']);
552                 msg->cm_fields['P'] = newpath;
553
554                 /*
555                  * Determine if this message is set to be deleted
556                  * after sending out on the network
557                  */
558                 if (msg->cm_fields['S'] != NULL) {
559                         if (!strcasecmp(msg->cm_fields['S'], "CANCEL")) {
560                                 delete_after_send = 1;
561                         }
562                 }
563
564                 /* Now send it to every node */
565                 if (sc->ignet_push_shares != NULL)
566                   for (mptr = sc->ignet_push_shares; mptr != NULL;
567                     mptr = mptr->next) {
568
569                         send = 1;
570                         if (Buf == NULL)
571                                 Buf = NewStrBufPlain(mptr->remote_nodename, -1);
572                         else
573                                 StrBufPlain(Buf, mptr->remote_nodename, -1);
574                         /* Check for valid node name */
575                         if (is_valid_node(NULL,
576                                           NULL,
577                                           Buf,
578                                           sc->working_ignetcfg,
579                                           sc->the_netmap) != 0)
580                         {
581                                 QN_syslog(LOG_ERR,
582                                           "Invalid node <%s>\n",
583                                           mptr->remote_nodename);
584
585                                 send = 0;
586                         }
587
588                         /* Check for split horizon */
589                         QN_syslog(LOG_DEBUG, "Path is %s\n", msg->cm_fields['P']);
590                         bang = num_tokens(msg->cm_fields['P'], '!');
591                         if (bang > 1) {
592                                 for (i=0; i<(bang-1); ++i) {
593                                         extract_token(buf,
594                                                       msg->cm_fields['P'],
595                                                       i, '!',
596                                                       sizeof buf);
597                                         
598                                         QN_syslog(LOG_DEBUG, "Compare <%s> to <%s>\n",
599                                                   buf, mptr->remote_nodename) ;
600                                         if (!strcasecmp(buf, mptr->remote_nodename)) {
601                                                 send = 0;
602                                                 break;
603                                         }
604                                 }
605
606                                 QN_syslog(LOG_INFO,
607                                           "%sSending to %s\n",
608                                           (send)?"":"Not ",
609                                           mptr->remote_nodename);
610                         }
611
612                         /* Send the message */
613                         if (send == 1)
614                         {
615                                 /*
616                                  * Force the message to appear in the correct
617                                  * room on the far end by setting the C field
618                                  * correctly
619                                  */
620                                 if (msg->cm_fields['C'] != NULL) {
621                                         free(msg->cm_fields['C']);
622                                 }
623                                 if (!IsEmptyStr(mptr->remote_roomname)) {
624                                         msg->cm_fields['C'] =
625                                                 strdup(mptr->remote_roomname);
626                                 }
627                                 else {
628                                         msg->cm_fields['C'] =
629                                                 strdup(CC->room.QRname);
630                                 }
631
632                                 /* serialize it for transmission */
633                                 serialize_message(&sermsg, msg);
634                                 if (sermsg.len > 0) {
635
636                                         /* write it to a spool file */
637                                         snprintf(filename,
638                                                  sizeof(filename),
639                                                  "%s/%s@%lx%x",
640                                                  ctdl_netout_dir,
641                                                  mptr->remote_nodename,
642                                                  time(NULL),
643                                                  rand()
644                                         );
645
646                                         QN_syslog(LOG_DEBUG,
647                                                   "Appending to %s\n",
648                                                   filename);
649
650                                         fp = fopen(filename, "ab");
651                                         if (fp != NULL) {
652                                                 fwrite(sermsg.ser,
653                                                         sermsg.len, 1, fp);
654                                                 fclose(fp);
655                                         }
656                                         else {
657                                                 QN_syslog(LOG_ERR,
658                                                           "%s: %s\n",
659                                                           filename,
660                                                           strerror(errno));
661                                         }
662
663                                         /* free the serialized version */
664                                         free(sermsg.ser);
665                                 }
666
667                         }
668                 }
669                 CtdlFreeMessage(msg);
670         }
671
672         /* update lastsent */
673         sc->lastsent = msgnum;
674
675         /* Delete this message if delete-after-send is set */
676         if (delete_after_send) {
677                 CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, "");
678         }
679         FreeStrBuf(&Buf);
680 }