2 * This module handles shared rooms, inter-Citadel mail, and outbound
3 * mailing list processing.
5 * Copyright (c) 2000-2012 by the citadel.org team
7 * This program is open source software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License, version 3.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * ** NOTE ** A word on the S_NETCONFIGS semaphore:
16 * This is a fairly high-level type of critical section. It ensures that no
17 * two threads work on the netconfigs files at the same time. Since we do
18 * so many things inside these, here are the rules:
19 * 1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others.
20 * 2. Do *not* perform any I/O with the client during these sections.
25 * Duration of time (in seconds) after which pending list subscribe/unsubscribe
26 * requests that have not been confirmed will be deleted.
28 #define EXP 259200 /* three days */
40 #include <sys/types.h>
42 #if TIME_WITH_SYS_TIME
43 # include <sys/time.h>
47 # include <sys/time.h>
55 # if HAVE_SYS_SYSCALL_H
56 # include <sys/syscall.h>
63 #include <libcitadel.h>
66 #include "citserver.h"
72 #include "internet_addressing.h"
73 #include "serv_network.h"
74 #include "clientsocket.h"
76 #include "citadel_dirs.h"
80 #include "ctdl_module.h"
82 #include "netconfig.h"
89 void ParseLastSent(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *rncfg)
91 rncfg->lastsent = extract_long(LinePos, 0);
93 void ParseIgnetPushShare(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *rncfg)
96 extract_token(nodename, LinePos, 0, '|', sizeof nodename);
97 extract_token(roomname, LinePos, 1, '|', sizeof roomname);
98 mptr = (maplist *) malloc(sizeof(maplist));
99 mptr->next = rncfg->RNCfg->ignet_push_shares;
100 strcpy(mptr->remote_nodename, nodename);
101 strcpy(mptr->remote_roomname, roomname);
102 rncfg->RNCfg->ignet_push_shares = mptr;
106 void ParseRoomAlias(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *rncfg)
109 if (rncfg->RNCfg->sender != NULL)
110 continue; / * just one alowed... * /
111 extract_token(nptr->name, buf, 1, '|', sizeof nptr->name);
112 rncfg->RNCfg->sender = nptr;
116 void ParseSubPendingLine(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *rncfg)
119 if (time(NULL) - extract_long(LinePos, 3) > EXP) {
127 void ParseUnSubPendingLine(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *rncfg)
129 ///int skipthisline = 0;
130 if (time(NULL) - extract_long(LinePos, 2) > EXP) {
137 void SerializeLastSent(const CfgLineType *ThisOne, StrBuf *OutputBuffer, OneRoomNetCfg *RNCfg, RoomNetCfgLine *data)
139 StrBufAppendBufPlain(OutputBuffer, CKEY(ThisOne->Str), 0);
140 StrBufAppendPrintf(OutputBuffer, "|%ld\n", RNCfg->lastsent);
143 void SerializeIgnetPushShare(const CfgLineType *ThisOne, StrBuf *OutputBuffer, OneRoomNetCfg *RNCfg, RoomNetCfgLine *data)
145 StrBufAppendBufPlain(OutputBuffer, CKEY(ThisOne->Str), 0);
147 StrBufAppendPrintf(Cfg, "ignet_push_share|%s", RNCfg->ignet_push_shares->remote_nodename);
148 if (!IsEmptyStr(RNCfg->ignet_push_shares->remote_roomname)) {
149 StrBufAppendPrintf(Cfg, "|%s", RNCfg->ignet_push_shares->remote_roomname);
151 StrBufAppendPrintf(Cfg, "\n");
152 mptr = RNCfg->ignet_push_shares->next;
153 free(RNCfg->ignet_push_shares);
154 RNCfg->ignet_push_shares = mptr;
156 StrBufAppendBuf(OutputBuffer, data->Value, 0);
157 StrBufAppendBufPlain(OutputBuffer, HKEY("\n"), 0);
162 * Batch up and send all outbound traffic from the current room
164 void network_spoolout_room(RoomProcList *room_to_spool,
165 HashList *working_ignetcfg,
166 HashList *the_netmap)
169 char filename[PATH_MAX];
174 * If the room doesn't exist, don't try to perform its networking tasks.
175 * Normally this should never happen, but once in a while maybe a room gets
176 * queued for networking and then deleted before it can happen.
178 if (CtdlGetRoom(&CC->room, room_to_spool->name) != 0) {
179 syslog(LOG_CRIT, "ERROR: cannot load <%s>\n", room_to_spool->name);
183 assoc_file_name(filename, sizeof filename, &CC->room, ctdl_netcfg_dir);
184 begin_critical_section(S_NETCONFIGS);
186 /* Only do net processing for rooms that have netconfigs */
187 if (!read_spoolcontrol_file(&sc, filename))
189 end_critical_section(S_NETCONFIGS);
192 syslog(LOG_INFO, "Networking started for <%s>\n", CC->room.QRname);
194 sc->working_ignetcfg = working_ignetcfg;
195 sc->the_netmap = the_netmap;
197 /* If there are digest recipients, we have to build a digest */
198 if (sc->RNCfg->NetConfigs[digestrecp] != NULL) {
199 sc->digestfp = tmpfile();
200 fprintf(sc->digestfp, "Content-type: text/plain\n\n");
203 /* Do something useful */
204 CtdlForEachMessage(MSGS_GT, sc->RNCfg->lastsent, NULL, NULL, NULL,
205 network_spool_msg, sc);
207 /* If we wrote a digest, deliver it and then close it */
208 snprintf(buf, sizeof buf, "room_%s@%s",
209 CC->room.QRname, config.c_fqdn);
210 for (i=0; buf[i]; ++i) {
211 buf[i] = tolower(buf[i]);
212 if (isspace(buf[i])) buf[i] = '_';
214 if (sc->digestfp != NULL) {
215 fprintf(sc->digestfp, " -----------------------------------"
216 "------------------------------------"
218 "You are subscribed to the '%s' "
220 "To post to the list: %s\n",
223 network_deliver_digest(sc); /* deliver and close */
226 /* Now rewrite the config file */
227 //// todo writenfree_spoolcontrol_file(&sc, filename);
228 end_critical_section(S_NETCONFIGS);
232 * Process a buffer containing a single message from a single file
233 * from the inbound queue
235 void network_process_buffer(char *buffer, long size, HashList *working_ignetcfg, HashList *the_netmap, int *netmap_changed)
237 struct CitContext *CCC = CC;
239 struct CtdlMessage *msg = NULL;
242 struct recptypes *recp = NULL;
243 char target_room[ROOMNAMELEN];
244 struct ser_ret sermsg;
245 char *oldpath = NULL;
246 char filename[PATH_MAX];
248 const StrBuf *nexthop = NULL;
249 unsigned char firstbyte;
250 unsigned char lastbyte;
252 QN_syslog(LOG_DEBUG, "network_process_buffer() processing %ld bytes\n", size);
254 /* Validate just a little bit. First byte should be FF and * last byte should be 00. */
255 firstbyte = buffer[0];
256 lastbyte = buffer[size-1];
257 if ( (firstbyte != 255) || (lastbyte != 0) ) {
258 QN_syslog(LOG_ERR, "Corrupt message ignored. Length=%ld, firstbyte = %d, lastbyte = %d\n",
259 size, firstbyte, lastbyte);
263 /* Set default target room to trash */
264 strcpy(target_room, TWITROOM);
266 /* Load the message into memory */
267 msg = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
268 memset(msg, 0, sizeof(struct CtdlMessage));
269 msg->cm_magic = CTDLMESSAGE_MAGIC;
270 msg->cm_anon_type = buffer[1];
271 msg->cm_format_type = buffer[2];
273 for (pos = 3; pos < size; ++pos) {
275 msg->cm_fields[field] = strdup(&buffer[pos+1]);
276 pos = pos + strlen(&buffer[(int)pos]);
279 /* Check for message routing */
280 if (msg->cm_fields['D'] != NULL) {
281 if (strcasecmp(msg->cm_fields['D'], config.c_nodename)) {
283 /* route the message */
284 Buf = NewStrBufPlain(msg->cm_fields['D'], -1);
285 if (CtdlIsValidNode(&nexthop,
291 /* prepend our node to the path */
292 if (msg->cm_fields['P'] != NULL) {
293 oldpath = msg->cm_fields['P'];
294 msg->cm_fields['P'] = NULL;
297 oldpath = strdup("unknown_user");
299 size = strlen(oldpath) + SIZ;
300 msg->cm_fields['P'] = malloc(size);
301 snprintf(msg->cm_fields['P'], size, "%s!%s",
302 config.c_nodename, oldpath);
305 /* serialize the message */
306 serialize_message(&sermsg, msg);
309 if (StrLength(nexthop) == 0) {
320 QN_syslog(LOG_DEBUG, "Appending to %s\n", filename);
321 fp = fopen(filename, "ab");
323 fwrite(sermsg.ser, sermsg.len, 1, fp);
327 QN_syslog(LOG_ERR, "%s: %s\n", filename, strerror(errno));
330 CtdlFreeMessage(msg);
335 else { /* invalid destination node name */
339 "A message you sent could not be delivered due to an invalid destination node"
340 " name. Please check the address and try sending the message again.\n");
349 * Check to see if we already have a copy of this message, and
350 * abort its processing if so. (We used to post a warning to Aide>
351 * every time this happened, but the network is now so densely
352 * connected that it's inevitable.)
354 if (network_usetable(msg) != 0) {
355 CtdlFreeMessage(msg);
359 /* Learn network topology from the path */
360 if ((msg->cm_fields['N'] != NULL) && (msg->cm_fields['P'] != NULL)) {
361 NetworkLearnTopology(msg->cm_fields['N'],
367 /* Is the sending node giving us a very persuasive suggestion about
368 * which room this message should be saved in? If so, go with that.
370 if (msg->cm_fields['C'] != NULL) {
371 safestrncpy(target_room, msg->cm_fields['C'], sizeof target_room);
374 /* Otherwise, does it have a recipient? If so, validate it... */
375 else if (msg->cm_fields['R'] != NULL) {
376 recp = validate_recipients(msg->cm_fields['R'], NULL, 0);
377 if (recp != NULL) if (recp->num_error != 0) {
379 "A message you sent could not be delivered due to an invalid address.\n"
380 "Please check the address and try sending the message again.\n");
382 free_recipients(recp);
383 QNM_syslog(LOG_DEBUG, "Bouncing message due to invalid recipient address.\n");
386 strcpy(target_room, ""); /* no target room if mail */
389 /* Our last shot at finding a home for this message is to see if
390 * it has the O field (Originating room) set.
392 else if (msg->cm_fields['O'] != NULL) {
393 safestrncpy(target_room, msg->cm_fields['O'], sizeof target_room);
396 /* Strip out fields that are only relevant during transit */
397 if (msg->cm_fields['D'] != NULL) {
398 free(msg->cm_fields['D']);
399 msg->cm_fields['D'] = NULL;
401 if (msg->cm_fields['C'] != NULL) {
402 free(msg->cm_fields['C']);
403 msg->cm_fields['C'] = NULL;
406 /* save the message into a room */
407 if (PerformNetprocHooks(msg, target_room) == 0) {
408 msg->cm_flags = CM_SKIP_HOOKS;
409 CtdlSubmitMsg(msg, recp, target_room, 0);
411 CtdlFreeMessage(msg);
412 free_recipients(recp);
417 * Process a single message from a single file from the inbound queue
419 void network_process_message(FILE *fp,
422 HashList *working_ignetcfg,
423 HashList *the_netmap,
430 hold_pos = ftell(fp);
431 size = msgend - msgstart + 1;
432 buffer = malloc(size);
433 if (buffer != NULL) {
434 fseek(fp, msgstart, SEEK_SET);
435 if (fread(buffer, size, 1, fp) > 0) {
436 network_process_buffer(buffer,
445 fseek(fp, hold_pos, SEEK_SET);
450 * Process a single file from the inbound queue
452 void network_process_file(char *filename,
453 HashList *working_ignetcfg,
454 HashList *the_netmap,
457 struct CitContext *CCC = CC;
459 long msgstart = (-1L);
465 fp = fopen(filename, "rb");
467 QN_syslog(LOG_CRIT, "Error opening %s: %s\n", filename, strerror(errno));
471 fseek(fp, 0L, SEEK_END);
472 QN_syslog(LOG_INFO, "network: processing %ld bytes from %s\n", ftell(fp), filename);
475 /* Look for messages in the data stream and break them out */
476 while (ch = getc(fp), ch >= 0) {
479 if (msgstart >= 0L) {
481 network_process_message(fp,
495 if (msgstart >= 0L) {
496 network_process_message(fp,
510 * Process anything in the inbound queue
512 void network_do_spoolin(HashList *working_ignetcfg, HashList *the_netmap, int *netmap_changed)
514 struct CitContext *CCC = CC;
518 char filename[PATH_MAX];
519 static time_t last_spoolin_mtime = 0L;
522 * Check the spoolin directory's modification time. If it hasn't
523 * been touched, we don't need to scan it.
525 if (stat(ctdl_netin_dir, &statbuf)) return;
526 if (statbuf.st_mtime == last_spoolin_mtime) {
527 QNM_syslog(LOG_DEBUG, "network: nothing in inbound queue\n");
530 last_spoolin_mtime = statbuf.st_mtime;
531 QNM_syslog(LOG_DEBUG, "network: processing inbound queue\n");
534 * Ok, there's something interesting in there, so scan it.
536 dp = opendir(ctdl_netin_dir);
537 if (dp == NULL) return;
539 while (d = readdir(dp), d != NULL) {
540 if ((strcmp(d->d_name, ".")) && (strcmp(d->d_name, ".."))) {
547 network_process_file(filename,
558 * Step 1: consolidate files in the outbound queue into one file per neighbor node
559 * Step 2: delete any files in the outbound queue that were for neighbors who no longer exist.
561 void network_consolidate_spoolout(HashList *working_ignetcfg, HashList *the_netmap)
563 struct CitContext *CCC = CC;
569 struct dirent *filedir_entry;
571 char spooloutfilename[PATH_MAX];
572 char filename[PATH_MAX];
573 const StrBuf *nexthop;
579 /* Step 1: consolidate files in the outbound queue into one file per neighbor node */
580 d = (struct dirent *)malloc(offsetof(struct dirent, d_name) + PATH_MAX + 1);
581 if (d == NULL) return;
583 dp = opendir(ctdl_netout_dir);
589 NextHop = NewStrBuf();
590 memset(&IOB, 0, sizeof(IOBuffer));
591 memset(&FDIO, 0, sizeof(FDIOBuffer));
594 while ((readdir_r(dp, d, &filedir_entry) == 0) &&
595 (filedir_entry != NULL))
597 #ifdef _DIRENT_HAVE_D_NAMELEN
598 d_namelen = filedir_entry->d_namelen;
607 #define IFTODT(mode) (((mode) & 0170000) >> 12)
608 #define DTTOIF(dirtype) ((dirtype) << 12)
610 d_namelen = strlen(filedir_entry->d_name);
612 if ((d_namelen > 1) && filedir_entry->d_name[d_namelen - 1] == '~')
613 continue; /* Ignore backup files... */
615 if ((d_namelen == 1) &&
616 (filedir_entry->d_name[0] == '.'))
619 if ((d_namelen == 2) &&
620 (filedir_entry->d_name[0] == '.') &&
621 (filedir_entry->d_name[1] == '.'))
624 pch = strchr(filedir_entry->d_name, '@');
632 filedir_entry->d_name);
635 filedir_entry->d_name,
636 pch - filedir_entry->d_name);
638 snprintf(spooloutfilename,
639 sizeof spooloutfilename,
644 QN_syslog(LOG_DEBUG, "Consolidate %s to %s\n", filename, ChrPtr(NextHop));
645 if (CtdlNetworkTalkingTo(SKEY(NextHop), NTT_CHECK)) {
648 "Currently online with %s - skipping for now\n",
656 const char *err = NULL;
657 CtdlNetworkTalkingTo(SKEY(NextHop), NTT_ADD);
659 infd = open(filename, O_RDONLY);
663 "failed to open %s for reading due to %s; skipping.\n",
664 filename, strerror(errno)
666 CtdlNetworkTalkingTo(SKEY(NextHop), NTT_REMOVE);
670 outfd = open(spooloutfilename,
671 O_EXCL|O_CREAT|O_NONBLOCK|O_WRONLY,
675 outfd = open(spooloutfilename,
676 O_EXCL|O_NONBLOCK|O_WRONLY,
682 "failed to open %s for reading due to %s; skipping.\n",
683 spooloutfilename, strerror(errno)
686 CtdlNetworkTalkingTo(SKEY(NextHop), NTT_REMOVE);
690 dsize = lseek(outfd, 0, SEEK_END);
691 lseek(outfd, -dsize, SEEK_SET);
693 fstat(infd, &statbuf);
694 fsize = statbuf.st_size;
696 fsize = lseek(infd, 0, SEEK_END);
699 FDIOBufferInit(&FDIO, &IOB, outfd, fsize + dsize);
700 FDIO.ChunkSendRemain = fsize;
701 FDIO.TotalSentAlready = dsize;
704 do {} while ((FileMoveChunked(&FDIO, &err) > 0) && (err == NULL));
711 "failed to append to %s [%s]; rolling back..\n",
712 spooloutfilename, strerror(errno)
714 /* whoops partial append?? truncate spooloutfilename again! */
715 ftruncate(outfd, dsize);
717 FDIOBufferDelete(&FDIO);
720 CtdlNetworkTalkingTo(SKEY(NextHop), NTT_REMOVE);
726 FreeStrBuf(&NextHop);
728 "skipping Spoolcleanup because of %d files unprocessed.\n",
735 /* Step 2: delete any files in the outbound queue that were for neighbors who no longer exist */
736 dp = opendir(ctdl_netout_dir);
738 FreeStrBuf(&NextHop);
743 while ((readdir_r(dp, d, &filedir_entry) == 0) &&
744 (filedir_entry != NULL))
746 #ifdef _DIRENT_HAVE_D_NAMELEN
747 d_namelen = filedir_entry->d_namelen;
748 d_type = filedir_entry->d_type;
757 #define IFTODT(mode) (((mode) & 0170000) >> 12)
758 #define DTTOIF(dirtype) ((dirtype) << 12)
760 d_namelen = strlen(filedir_entry->d_name);
762 if ((d_namelen == 1) &&
763 (filedir_entry->d_name[0] == '.'))
766 if ((d_namelen == 2) &&
767 (filedir_entry->d_name[0] == '.') &&
768 (filedir_entry->d_name[1] == '.'))
771 pch = strchr(filedir_entry->d_name, '@');
772 if (pch == NULL) /* no @ in name? consolidated file. */
776 filedir_entry->d_name,
777 pch - filedir_entry->d_name);
783 filedir_entry->d_name
786 i = CtdlIsValidNode(&nexthop,
792 if ( (i != 0) || (StrLength(nexthop) > 0) ) {
796 FreeStrBuf(&NextHop);
805 * It's ok if these directories already exist. Just fail silently.
807 void create_spool_dirs(void) {
808 if ((mkdir(ctdl_spool_dir, 0700) != 0) && (errno != EEXIST))
809 syslog(LOG_EMERG, "unable to create directory [%s]: %s", ctdl_spool_dir, strerror(errno));
810 if (chown(ctdl_spool_dir, CTDLUID, (-1)) != 0)
811 syslog(LOG_EMERG, "unable to set the access rights for [%s]: %s", ctdl_spool_dir, strerror(errno));
812 if ((mkdir(ctdl_netin_dir, 0700) != 0) && (errno != EEXIST))
813 syslog(LOG_EMERG, "unable to create directory [%s]: %s", ctdl_netin_dir, strerror(errno));
814 if (chown(ctdl_netin_dir, CTDLUID, (-1)) != 0)
815 syslog(LOG_EMERG, "unable to set the access rights for [%s]: %s", ctdl_netin_dir, strerror(errno));
816 if ((mkdir(ctdl_nettmp_dir, 0700) != 0) && (errno != EEXIST))
817 syslog(LOG_EMERG, "unable to create directory [%s]: %s", ctdl_nettmp_dir, strerror(errno));
818 if (chown(ctdl_nettmp_dir, CTDLUID, (-1)) != 0)
819 syslog(LOG_EMERG, "unable to set the access rights for [%s]: %s", ctdl_nettmp_dir, strerror(errno));
820 if ((mkdir(ctdl_netout_dir, 0700) != 0) && (errno != EEXIST))
821 syslog(LOG_EMERG, "unable to create directory [%s]: %s", ctdl_netout_dir, strerror(errno));
822 if (chown(ctdl_netout_dir, CTDLUID, (-1)) != 0)
823 syslog(LOG_EMERG, "unable to set the access rights for [%s]: %s", ctdl_netout_dir, strerror(errno));
829 CTDL_MODULE_INIT(network_spool)
833 // CtdlREGISTERRoomCfgType(subpending, ParseSubPendingLine, 0, SerializeSubPendingLine, DeleteSubPendingLine); /// todo: move this to mailinglist manager
834 // CtdlREGISTERRoomCfgType(unsubpending, ParseUnSubPendingLine0, SerializeUnSubPendingLine, DeleteUnSubPendingLine); /// todo: move this to mailinglist manager
835 // CtdlREGISTERRoomCfgType(lastsent, ParseLastSent, 1, SerializeLastSent, DeleteLastSent);
836 /// CtdlREGISTERRoomCfgType(ignet_push_share, ParseIgnetPushShare, 0, SerializeIgnetPushShare, DeleteIgnetPushShare); // todo: move this to the ignet client
837 CtdlREGISTERRoomCfgType(listrecp, ParseGeneric, 0, SerializeGeneric, DeleteGenericCfgLine);
838 CtdlREGISTERRoomCfgType(digestrecp, ParseGeneric, 0, SerializeGeneric, DeleteGenericCfgLine);
839 CtdlREGISTERRoomCfgType(participate, ParseGeneric, 0, SerializeGeneric, DeleteGenericCfgLine);
840 CtdlREGISTERRoomCfgType(roommailalias, ParseRoomAlias, 0, SerializeGeneric, DeleteGenericCfgLine);
843 //////todo CtdlRegisterCleanupHook(destroy_network_queue_room);
845 return "network_spool";