Rework networker
[citadel.git] / citadel / modules / network / serv_netmail.c
1 /*
2  * This module handles shared rooms, inter-Citadel mail, and outbound
3  * mailing list processing.
4  *
5  * Copyright (c) 2000-2012 by the citadel.org team
6  *
7  *  This program is open source software; you can redistribute it and/or modify
8  *  it under the terms of the GNU General Public License as published by
9  *  the Free Software Foundation; either version 3 of the License, or
10  *  (at your option) any later version.
11  *
12  *  This program is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  *  GNU General Public License for more details.
16  *
17  *  You should have received a copy of the GNU General Public License
18  *  along with this program; if not, write to the Free Software
19  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20  *
21  * ** NOTE **   A word on the S_NETCONFIGS semaphore:
22  * This is a fairly high-level type of critical section.  It ensures that no
23  * two threads work on the netconfigs files at the same time.  Since we do
24  * so many things inside these, here are the rules:
25  *  1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others.
26  *  2. Do *not* perform any I/O with the client during these sections.
27  *
28  */
29
30 /*
31  * Duration of time (in seconds) after which pending list subscribe/unsubscribe
32  * requests that have not been confirmed will be deleted.
33  */
34 #define EXP     259200  /* three days */
35
36 #include "sysdep.h"
37 #include <stdlib.h>
38 #include <unistd.h>
39 #include <stdio.h>
40 #include <fcntl.h>
41 #include <ctype.h>
42 #include <signal.h>
43 #include <pwd.h>
44 #include <errno.h>
45 #include <sys/stat.h>
46 #include <sys/types.h>
47 #include <dirent.h>
48 #if TIME_WITH_SYS_TIME
49 # include <sys/time.h>
50 # include <time.h>
51 #else
52 # if HAVE_SYS_TIME_H
53 #  include <sys/time.h>
54 # else
55 #  include <time.h>
56 # endif
57 #endif
58 #ifdef HAVE_SYSCALL_H
59 # include <syscall.h>
60 #else
61 # if HAVE_SYS_SYSCALL_H
62 #  include <sys/syscall.h>
63 # endif
64 #endif
65
66 #include <sys/wait.h>
67 #include <string.h>
68 #include <limits.h>
69 #include <libcitadel.h>
70 #include "citadel.h"
71 #include "server.h"
72 #include "citserver.h"
73 #include "support.h"
74 #include "config.h"
75 #include "user_ops.h"
76 #include "database.h"
77 #include "msgbase.h"
78 #include "internet_addressing.h"
79 #include "serv_network.h"
80 #include "clientsocket.h"
81 #include "file_ops.h"
82 #include "citadel_dirs.h"
83 #include "threads.h"
84
85 #ifndef HAVE_SNPRINTF
86 #include "snprintf.h"
87 #endif
88
89 #include "context.h"
90 #include "netconfig.h"
91 #include "netspool.h"
92 #include "netmail.h"
93 #include "ctdl_module.h"
94
95
96 /*
97  * Deliver digest messages
98  */
99 void network_deliver_digest(SpoolControl *sc) {
100         char buf[SIZ];
101         int i;
102         struct CtdlMessage *msg = NULL;
103         long msglen;
104         char *recps = NULL;
105         size_t recps_len = SIZ;
106         struct recptypes *valid;
107         namelist *nptr;
108         char bounce_to[256];
109
110         if (sc->num_msgs_spooled < 1) {
111                 fclose(sc->digestfp);
112                 sc->digestfp = NULL;
113                 return;
114         }
115
116         msg = malloc(sizeof(struct CtdlMessage));
117         memset(msg, 0, sizeof(struct CtdlMessage));
118         msg->cm_magic = CTDLMESSAGE_MAGIC;
119         msg->cm_format_type = FMT_RFC822;
120         msg->cm_anon_type = MES_NORMAL;
121
122         sprintf(buf, "%ld", time(NULL));
123         msg->cm_fields['T'] = strdup(buf);
124         msg->cm_fields['A'] = strdup(CC->room.QRname);
125         snprintf(buf, sizeof buf, "[%s]", CC->room.QRname);
126         msg->cm_fields['U'] = strdup(buf);
127         sprintf(buf, "room_%s@%s", CC->room.QRname, config.c_fqdn);
128         for (i=0; buf[i]; ++i) {
129                 if (isspace(buf[i])) buf[i]='_';
130                 buf[i] = tolower(buf[i]);
131         }
132         msg->cm_fields['F'] = strdup(buf);
133         msg->cm_fields['R'] = strdup(buf);
134
135         /* Set the 'List-ID' header */
136         msg->cm_fields['L'] = malloc(1024);
137         snprintf(msg->cm_fields['L'], 1024,
138                 "%s <%ld.list-id.%s>",
139                 CC->room.QRname,
140                 CC->room.QRnumber,
141                 config.c_fqdn
142         );
143
144         /*
145          * Go fetch the contents of the digest
146          */
147         fseek(sc->digestfp, 0L, SEEK_END);
148         msglen = ftell(sc->digestfp);
149
150         msg->cm_fields['M'] = malloc(msglen + 1);
151         fseek(sc->digestfp, 0L, SEEK_SET);
152         fread(msg->cm_fields['M'], (size_t)msglen, 1, sc->digestfp);
153         msg->cm_fields['M'][msglen] = '\0';
154
155         fclose(sc->digestfp);
156         sc->digestfp = NULL;
157
158         /* Now generate the delivery instructions */
159
160         /*
161          * Figure out how big a buffer we need to allocate
162          */
163         for (nptr = sc->digestrecps; nptr != NULL; nptr = nptr->next) {
164                 recps_len = recps_len + strlen(nptr->name) + 2;
165         }
166
167         recps = malloc(recps_len);
168
169         if (recps == NULL) {
170                 syslog(LOG_EMERG,
171                        "Cannot allocate %ld bytes for recps...\n",
172                        (long)recps_len);
173                 abort();
174         }
175
176         strcpy(recps, "");
177
178         /* Each recipient */
179         for (nptr = sc->digestrecps; nptr != NULL; nptr = nptr->next) {
180                 if (nptr != sc->digestrecps) {
181                         strcat(recps, ",");
182                 }
183                 strcat(recps, nptr->name);
184         }
185
186         /* Where do we want bounces and other noise to be heard?
187          *Surely not the list members! */
188         snprintf(bounce_to, sizeof bounce_to, "room_aide@%s", config.c_fqdn);
189
190         /* Now submit the message */
191         valid = validate_recipients(recps, NULL, 0);
192         free(recps);
193         if (valid != NULL) {
194                 valid->bounce_to = strdup(bounce_to);
195                 valid->envelope_from = strdup(bounce_to);
196                 CtdlSubmitMsg(msg, valid, NULL, 0);
197         }
198         CtdlFreeMessage(msg);
199         free_recipients(valid);
200 }
201
202
203 /*
204  * Deliver list messages to everyone on the list ... efficiently
205  */
206 void network_deliver_list(struct CtdlMessage *msg, SpoolControl *sc, const char *RoomName) {
207         char *recps = NULL;
208         size_t recps_len = SIZ;
209         struct recptypes *valid;
210         namelist *nptr;
211         char bounce_to[256];
212
213         /* Don't do this if there were no recipients! */
214         if (sc->listrecps == NULL) return;
215
216         /* Now generate the delivery instructions */
217
218         /*
219          * Figure out how big a buffer we need to allocate
220          */
221         for (nptr = sc->listrecps; nptr != NULL; nptr = nptr->next) {
222                 recps_len = recps_len + strlen(nptr->name) + 2;
223         }
224
225         recps = malloc(recps_len);
226
227         if (recps == NULL) {
228                 syslog(LOG_EMERG,
229                        "Cannot allocate %ld bytes for recps...\n",
230                        (long)recps_len);
231                 abort();
232         }
233
234         strcpy(recps, "");
235
236         /* Each recipient */
237         for (nptr = sc->listrecps; nptr != NULL; nptr = nptr->next) {
238                 if (nptr != sc->listrecps) {
239                         strcat(recps, ",");
240                 }
241                 strcat(recps, nptr->name);
242         }
243
244         /* Where do we want bounces and other noise to be heard?
245          *  Surely not the list members! */
246         snprintf(bounce_to, sizeof bounce_to, "room_aide@%s", config.c_fqdn);
247
248         /* Now submit the message */
249         valid = validate_recipients(recps, NULL, 0);
250         free(recps);
251         if (valid != NULL) {
252                 valid->bounce_to = strdup(bounce_to);
253                 valid->envelope_from = strdup(bounce_to);
254                 valid->sending_room = strdup(RoomName);
255                 CtdlSubmitMsg(msg, valid, NULL, 0);
256                 free_recipients(valid);
257         }
258         /* Do not call CtdlFreeMessage(msg) here; the caller will free it. */
259 }
260
261
262 /*
263  * Spools out one message from the list.
264  */
265 void network_spool_msg(long msgnum,
266                        void *userdata)
267 {
268         StrBuf *Buf = NULL;
269         SpoolControl *sc;
270         int i;
271         char *newpath = NULL;
272         struct CtdlMessage *msg = NULL;
273         namelist *nptr;
274         maplist *mptr;
275         struct ser_ret sermsg;
276         FILE *fp;
277         char filename[PATH_MAX];
278         char buf[SIZ];
279         int bang = 0;
280         int send = 1;
281         int delete_after_send = 0;      /* Set to 1 to delete after spooling */
282         int ok_to_participate = 0;
283         struct recptypes *valid;
284
285         sc = (SpoolControl *)userdata;
286
287         /*
288          * Process mailing list recipients
289          */
290         if (sc->listrecps != NULL) {
291                 /* Fetch the message.  We're going to need to modify it
292                  * in order to insert the [list name] in it, etc.
293                  */
294                 msg = CtdlFetchMessage(msgnum, 1);
295                 if (msg != NULL) {
296                         int rlen;
297                         char *pCh;
298                         StrBuf *Subject, *FlatSubject;
299
300                         if (msg->cm_fields['K'] != NULL)
301                                 free(msg->cm_fields['K']);
302                         if (msg->cm_fields['V'] == NULL){
303                                 /* local message, no enVelope */
304                                 StrBuf *Buf;
305                                 Buf = NewStrBuf();
306                                 StrBufAppendBufPlain(Buf,
307                                                      msg->cm_fields['O']
308                                                      , -1, 0);
309                                 StrBufAppendBufPlain(Buf, HKEY("@"), 0);
310                                 StrBufAppendBufPlain(Buf, config.c_fqdn, -1, 0);
311
312                                 msg->cm_fields['K'] = SmashStrBuf(&Buf);
313                         }
314                         else {
315                                 msg->cm_fields['K'] =
316                                         strdup (msg->cm_fields['V']);
317                         }
318                         /* Set the 'List-ID' header */
319                         if (msg->cm_fields['L'] != NULL) {
320                                 free(msg->cm_fields['L']);
321                         }
322                         msg->cm_fields['L'] = malloc(1024);
323                         snprintf(msg->cm_fields['L'], 1024,
324                                 "%s <%ld.list-id.%s>",
325                                 CC->room.QRname,
326                                 CC->room.QRnumber,
327                                 config.c_fqdn
328                         );
329
330                         /* Prepend "[List name]" to the subject */
331                         if (msg->cm_fields['U'] == NULL) {
332                                 Subject = NewStrBufPlain(HKEY("(no subject)"));
333                         }
334                         else {
335                                 Subject = NewStrBufPlain(
336                                         msg->cm_fields['U'], -1);
337                         }
338                         FlatSubject = NewStrBufPlain(NULL, StrLength(Subject));
339                         StrBuf_RFC822_to_Utf8(FlatSubject, Subject, NULL, NULL);
340
341                         rlen = strlen(CC->room.QRname);
342                         pCh  = strstr(ChrPtr(FlatSubject), CC->room.QRname);
343                         if ((pCh == NULL) ||
344                             (*(pCh + rlen) != ']') ||
345                             (pCh == ChrPtr(FlatSubject)) ||
346                             (*(pCh - 1) != '[')
347                                 )
348                         {
349                                 StrBuf *tmp;
350                                 StrBufPlain(Subject, HKEY("["));
351                                 StrBufAppendBufPlain(Subject,
352                                                      CC->room.QRname,
353                                                      rlen, 0);
354                                 StrBufAppendBufPlain(Subject, HKEY("] "), 0);
355                                 StrBufAppendBuf(Subject, FlatSubject, 0);
356                                  /* so we can free the right one swap them */
357                                 tmp = Subject;
358                                 Subject = FlatSubject;
359                                 FlatSubject = tmp;
360                                 StrBufRFC2047encode(&Subject, FlatSubject);
361                         }
362
363                         if (msg->cm_fields['U'] != NULL)
364                                 free (msg->cm_fields['U']);
365                         msg->cm_fields['U'] = SmashStrBuf(&Subject);
366
367                         FreeStrBuf(&FlatSubject);
368
369                         /* else we won't modify the buffer, since the
370                          * roomname is already here.
371                          */
372
373                         /* if there is no other recipient, Set the recipient
374                          * of the list message to the email address of the
375                          * room itself.
376                          */
377                         if ((msg->cm_fields['R'] == NULL) ||
378                             IsEmptyStr(msg->cm_fields['R']))
379                         {
380                                 if (msg->cm_fields['R'] != NULL)
381                                         free(msg->cm_fields['R']);
382
383                                 msg->cm_fields['R'] = malloc(256);
384                                 snprintf(msg->cm_fields['R'], 256,
385                                          "room_%s@%s", CC->room.QRname,
386                                          config.c_fqdn);
387                                 for (i=0; msg->cm_fields['R'][i]; ++i) {
388                                         if (isspace(msg->cm_fields['R'][i])) {
389                                                 msg->cm_fields['R'][i] = '_';
390                                         }
391                                 }
392                         }
393
394                         /* Handle delivery */
395                         network_deliver_list(msg, sc, CC->room.QRname);
396                         CtdlFreeMessage(msg);
397                 }
398         }
399
400         /*
401          * Process digest recipients
402          */
403         if ((sc->digestrecps != NULL) && (sc->digestfp != NULL)) {
404                 msg = CtdlFetchMessage(msgnum, 1);
405                 if (msg != NULL) {
406                         fprintf(sc->digestfp,
407                                 " -----------------------------------"
408                                 "------------------------------------"
409                                 "-------\n");
410                         fprintf(sc->digestfp, "From: ");
411                         if (msg->cm_fields['A'] != NULL) {
412                                 fprintf(sc->digestfp,
413                                         "%s ",
414                                         msg->cm_fields['A']);
415                         }
416                         if (msg->cm_fields['F'] != NULL) {
417                                 fprintf(sc->digestfp,
418                                         "<%s> ",
419                                         msg->cm_fields['F']);
420                         }
421                         else if (msg->cm_fields['N'] != NULL) {
422                                 fprintf(sc->digestfp,
423                                         "@%s ",
424                                         msg->cm_fields['N']);
425                         }
426                         fprintf(sc->digestfp, "\n");
427                         if (msg->cm_fields['U'] != NULL) {
428                                 fprintf(sc->digestfp,
429                                         "Subject: %s\n",
430                                         msg->cm_fields['U']);
431                         }
432
433                         CC->redirect_buffer = NewStrBufPlain(NULL, SIZ);
434
435                         safestrncpy(CC->preferred_formats,
436                                     "text/plain",
437                                     sizeof CC->preferred_formats);
438
439                         CtdlOutputPreLoadedMsg(msg,
440                                                MT_CITADEL,
441                                                HEADERS_NONE,
442                                                0, 0, 0);
443
444                         StrBufTrim(CC->redirect_buffer);
445                         fwrite(HKEY("\n"), 1, sc->digestfp);
446                         fwrite(SKEY(CC->redirect_buffer), 1, sc->digestfp);
447                         fwrite(HKEY("\n"), 1, sc->digestfp);
448
449                         FreeStrBuf(&CC->redirect_buffer);
450
451                         sc->num_msgs_spooled += 1;
452                         CtdlFreeMessage(msg);
453                 }
454         }
455
456         /*
457          * Process client-side list participations for this room
458          */
459         if (sc->participates != NULL) {
460                 msg = CtdlFetchMessage(msgnum, 1);
461                 if (msg != NULL) {
462
463                         /* Only send messages which originated on our own
464                          * Citadel network, otherwise we'll end up sending the
465                          * remote mailing list's messages back to it, which
466                          * is rude...
467                          */
468                         ok_to_participate = 0;
469                         if (msg->cm_fields['N'] != NULL) {
470                                 if (!strcasecmp(msg->cm_fields['N'],
471                                                 config.c_nodename)) {
472                                         ok_to_participate = 1;
473                                 }
474
475                                 Buf = NewStrBufPlain(msg->cm_fields['N'], -1);
476                                 if (is_valid_node(NULL,
477                                                   NULL,
478                                                   Buf,
479                                                   sc->working_ignetcfg,
480                                                   sc->the_netmap) == 0)
481                                 {
482                                         ok_to_participate = 1;
483                                 }
484                         }
485                         if (ok_to_participate) {
486                                 if (msg->cm_fields['F'] != NULL) {
487                                         free(msg->cm_fields['F']);
488                                 }
489                                 msg->cm_fields['F'] = malloc(SIZ);
490                                 /* Replace the Internet email address of the
491                                  * actual author with the email address of the
492                                  * room itself, so the remote listserv doesn't
493                                  * reject us.
494                                  * FIXME  I want to be able to pick any address
495                                 */
496                                 snprintf(msg->cm_fields['F'], SIZ,
497                                         "room_%s@%s", CC->room.QRname,
498                                         config.c_fqdn);
499                                 for (i=0; msg->cm_fields['F'][i]; ++i) {
500                                         if (isspace(msg->cm_fields['F'][i])) {
501                                                 msg->cm_fields['F'][i] = '_';
502                                         }
503                                 }
504
505                                 /*
506                                  * Figure out how big a buffer we need to alloc
507                                  */
508                                 for (nptr = sc->participates;
509                                      nptr != NULL;
510                                      nptr = nptr->next)
511                                 {
512                                         if (msg->cm_fields['R'] != NULL) {
513                                                 free(msg->cm_fields['R']);
514                                         }
515                                         msg->cm_fields['R'] =
516                                                 strdup(nptr->name);
517
518                                         valid = validate_recipients(nptr->name,
519                                                                     NULL, 0);
520
521                                         CtdlSubmitMsg(msg, valid, "", 0);
522                                         free_recipients(valid);
523                                 }
524                         }
525                         CtdlFreeMessage(msg);
526                 }
527         }
528
529         /*
530          * Process IGnet push shares
531          */
532         msg = CtdlFetchMessage(msgnum, 1);
533         if (msg != NULL) {
534                 size_t newpath_len;
535
536                 /* Prepend our node name to the Path field whenever
537                  * sending a message to another IGnet node
538                  */
539                 if (msg->cm_fields['P'] == NULL) {
540                         msg->cm_fields['P'] = strdup("username");
541                 }
542                 newpath_len = strlen(msg->cm_fields['P']) +
543                          strlen(config.c_nodename) + 2;
544                 newpath = malloc(newpath_len);
545                 snprintf(newpath, newpath_len, "%s!%s",
546                          config.c_nodename, msg->cm_fields['P']);
547                 free(msg->cm_fields['P']);
548                 msg->cm_fields['P'] = newpath;
549
550                 /*
551                  * Determine if this message is set to be deleted
552                  * after sending out on the network
553                  */
554                 if (msg->cm_fields['S'] != NULL) {
555                         if (!strcasecmp(msg->cm_fields['S'], "CANCEL")) {
556                                 delete_after_send = 1;
557                         }
558                 }
559
560                 /* Now send it to every node */
561                 if (sc->ignet_push_shares != NULL)
562                   for (mptr = sc->ignet_push_shares; mptr != NULL;
563                     mptr = mptr->next) {
564
565                         send = 1;
566                         if (Buf == NULL)
567                                 Buf = NewStrBufPlain(mptr->remote_nodename, -1);
568                         else
569                                 StrBufPlain(Buf, mptr->remote_nodename, -1);
570                         /* Check for valid node name */
571                         if (is_valid_node(NULL,
572                                           NULL,
573                                           Buf,
574                                           sc->working_ignetcfg,
575                                           sc->the_netmap) != 0)
576                         {
577                                 syslog(LOG_ERR,
578                                        "Invalid node <%s>\n",
579                                        mptr->remote_nodename);
580
581                                 send = 0;
582                         }
583
584                         /* Check for split horizon */
585                         syslog(LOG_DEBUG, "Path is %s\n", msg->cm_fields['P']);
586                         bang = num_tokens(msg->cm_fields['P'], '!');
587                         if (bang > 1) for (i=0; i<(bang-1); ++i) {
588                                 extract_token(buf,
589                                               msg->cm_fields['P'],
590                                               i, '!',
591                                               sizeof buf);
592
593                                 syslog(LOG_DEBUG, "Compare <%s> to <%s>\n",
594                                         buf, mptr->remote_nodename) ;
595                                 if (!strcasecmp(buf, mptr->remote_nodename)) {
596                                         send = 0;
597                                         syslog(LOG_DEBUG, "Not sending to %s\n",
598                                                 mptr->remote_nodename);
599                                 }
600                                 else {
601                                         syslog(LOG_DEBUG,
602                                                "Sending to %s\n",
603                                                mptr->remote_nodename);
604                                 }
605                         }
606
607                         /* Send the message */
608                         if (send == 1)
609                         {
610                                 /*
611                                  * Force the message to appear in the correct
612                                  * room on the far end by setting the C field
613                                  * correctly
614                                  */
615                                 if (msg->cm_fields['C'] != NULL) {
616                                         free(msg->cm_fields['C']);
617                                 }
618                                 if (!IsEmptyStr(mptr->remote_roomname)) {
619                                         msg->cm_fields['C'] =
620                                                 strdup(mptr->remote_roomname);
621                                 }
622                                 else {
623                                         msg->cm_fields['C'] =
624                                                 strdup(CC->room.QRname);
625                                 }
626
627                                 /* serialize it for transmission */
628                                 serialize_message(&sermsg, msg);
629                                 if (sermsg.len > 0) {
630
631                                         /* write it to a spool file */
632                                         snprintf(filename,
633                                                  sizeof(filename),
634                                                  "%s/%s@%lx%x",
635                                                  ctdl_netout_dir,
636                                                  mptr->remote_nodename,
637                                                  time(NULL),
638                                                  rand()
639                                         );
640
641                                         syslog(LOG_DEBUG,
642                                                "Appending to %s\n",
643                                                filename);
644
645                                         fp = fopen(filename, "ab");
646                                         if (fp != NULL) {
647                                                 fwrite(sermsg.ser,
648                                                         sermsg.len, 1, fp);
649                                                 fclose(fp);
650                                         }
651                                         else {
652                                                 syslog(LOG_ERR,
653                                                        "%s: %s\n",
654                                                        filename,
655                                                        strerror(errno));
656                                         }
657
658                                         /* free the serialized version */
659                                         free(sermsg.ser);
660                                 }
661
662                         }
663                 }
664                 CtdlFreeMessage(msg);
665         }
666
667         /* update lastsent */
668         sc->lastsent = msgnum;
669
670         /* Delete this message if delete-after-send is set */
671         if (delete_after_send) {
672                 CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, "");
673         }
674         FreeStrBuf(&Buf);
675 }