Silence!
[citadel.git] / citadel / modules / network / serv_netspool.c
index f0128f0168fffdd5135000912a40f47ba69d8786..0edb1f11efa0e240bcf7c7c3d79ead11d21b3934 100644 (file)
@@ -2,22 +2,16 @@
  * This module handles shared rooms, inter-Citadel mail, and outbound
  * mailing list processing.
  *
- * Copyright (c) 2000-2011 by the citadel.org team
+ * Copyright (c) 2000-2012 by the citadel.org team
  *
  *  This program is open source software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 3 of the License, or
- *  (at your option) any later version.
+ *  it under the terms of the GNU General Public License, version 3.
  *
  *  This program is distributed in the hope that it will be useful,
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  *  GNU General Public License for more details.
  *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- *
  * ** NOTE **   A word on the S_NETCONFIGS semaphore:
  * This is a fairly high-level type of critical section.  It ensures that no
  * two threads work on the netconfigs files at the same time.  Since we do
 #include "internet_addressing.h"
 #include "serv_network.h"
 #include "clientsocket.h"
-#include "file_ops.h"
 #include "citadel_dirs.h"
 #include "threads.h"
+#include "context.h"
 
-#ifndef HAVE_SNPRINTF
-#include "snprintf.h"
-#endif
+#include "ctdl_module.h"
 
-#include "context.h"
-#include "netconfig.h"
 #include "netspool.h"
 #include "netmail.h"
-#include "ctdl_module.h"
 
-/*
- * Learn topology from path fields
- */
-void network_learn_topology(char *node, char *path, NetMap *the_netmap, int *netmap_changed) {
-       char nexthop[256];
-       NetMap *nmptr;
-
-       strcpy(nexthop, "");
-
-       if (num_tokens(path, '!') < 3) return;
-       for (nmptr = the_netmap; nmptr != NULL; nmptr = nmptr->next) {
-               if (!strcasecmp(nmptr->nodename, node)) {
-                       extract_token(nmptr->nexthop, path, 0, '!', sizeof nmptr->nexthop);
-                       nmptr->lastcontact = time(NULL);
-                       (*netmap_changed) ++;
-                       return;
-               }
-       }
 
-       /* If we got here then it's not in the map, so add it. */
-       nmptr = (NetMap *) malloc(sizeof (NetMap));
-       strcpy(nmptr->nodename, node);
-       nmptr->lastcontact = time(NULL);
-       extract_token(nmptr->nexthop, path, 0, '!', sizeof nmptr->nexthop);
-       nmptr->next = the_netmap;
-       the_netmap = nmptr;
-       (*netmap_changed) ++;
+#ifndef DT_UNKNOWN
+#define DT_UNKNOWN     0
+#define DT_DIR         4
+#define DT_REG         8
+#define DT_LNK         10
+
+#define IFTODT(mode)   (((mode) & 0170000) >> 12)
+#define DTTOIF(dirtype)        ((dirtype) << 12)
+#endif
+
+
+void ParseLastSent(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *OneRNCFG)
+{
+       RoomNetCfgLine *nptr;
+       nptr = (RoomNetCfgLine *)
+               malloc(sizeof(RoomNetCfgLine));
+       memset(nptr, 0, sizeof(RoomNetCfgLine));
+       OneRNCFG->lastsent = extract_long(LinePos, 0);
+       OneRNCFG->NetConfigs[ThisOne->C] = nptr;
 }
 
+void ParseRoomAlias(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *rncfg)
+{
+       if (rncfg->Sender != NULL)
+               return;
 
-       
+       ParseGeneric(ThisOne, Line, LinePos, rncfg);
+       rncfg->Sender = NewStrBufDup(rncfg->NetConfigs[roommailalias]->Value[0]);
+}
 
-int read_spoolcontrol_file(SpoolControl **scc, char *filename)
+void ParseSubPendingLine(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *OneRNCFG)
 {
-       FILE *fp;
-       char instr[SIZ];
-       char buf[SIZ];
-       char nodename[256];
-       char roomname[ROOMNAMELEN];
-       size_t miscsize = 0;
-       size_t linesize = 0;
-       int skipthisline = 0;
-       namelist *nptr = NULL;
-       maplist *mptr = NULL;
-       SpoolControl *sc;
+       if (time(NULL) - extract_long(LinePos, 3) > EXP) 
+               return; /* expired subscription... */
 
-       fp = fopen(filename, "r");
-       if (fp == NULL) {
-               return 0;
-       }
-       sc = malloc(sizeof(SpoolControl));
-       memset(sc, 0, sizeof(SpoolControl));
-       *scc = sc;
+       ParseGeneric(ThisOne, Line, LinePos, OneRNCFG);
+}
+void ParseUnSubPendingLine(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *OneRNCFG)
+{
+       if (time(NULL) - extract_long(LinePos, 2) > EXP)
+               return; /* expired subscription... */
 
-       while (fgets(buf, sizeof buf, fp) != NULL) {
-               buf[strlen(buf)-1] = 0;
+       ParseGeneric(ThisOne, Line, LinePos, OneRNCFG);
+}
 
-               extract_token(instr, buf, 0, '|', sizeof instr);
-               if (!strcasecmp(instr, strof(lastsent))) {
-                       sc->lastsent = extract_long(buf, 1);
-               }
-               else if (!strcasecmp(instr, strof(listrecp))) {
-                       nptr = (namelist *)
-                               malloc(sizeof(namelist));
-                       nptr->next = sc->listrecps;
-                       extract_token(nptr->name, buf, 1, '|', sizeof nptr->name);
-                       sc->listrecps = nptr;
-               }
-               else if (!strcasecmp(instr, strof(participate))) {
-                       nptr = (namelist *)
-                               malloc(sizeof(namelist));
-                       nptr->next = sc->participates;
-                       extract_token(nptr->name, buf, 1, '|', sizeof nptr->name);
-                       sc->participates = nptr;
-               }
-               else if (!strcasecmp(instr, strof(digestrecp))) {
-                       nptr = (namelist *)
-                               malloc(sizeof(namelist));
-                       nptr->next = sc->digestrecps;
-                       extract_token(nptr->name, buf, 1, '|', sizeof nptr->name);
-                       sc->digestrecps = nptr;
-               }
-               else if (!strcasecmp(instr, strof(ignet_push_share))) {
-                       extract_token(nodename, buf, 1, '|', sizeof nodename);
-                       extract_token(roomname, buf, 2, '|', sizeof roomname);
-                       mptr = (maplist *) malloc(sizeof(maplist));
-                       mptr->next = sc->ignet_push_shares;
-                       strcpy(mptr->remote_nodename, nodename);
-                       strcpy(mptr->remote_roomname, roomname);
-                       sc->ignet_push_shares = mptr;
-               }
-               else {
-                       /* Preserve 'other' lines ... *unless* they happen to
-                        * be subscribe/unsubscribe pendings with expired
-                        * timestamps.
-                        */
-                       skipthisline = 0;
-                       if (!strncasecmp(buf, strof(subpending)"|", 11)) {
-                               if (time(NULL) - extract_long(buf, 4) > EXP) {
-                                       skipthisline = 1;
-                               }
-                       }
-                       if (!strncasecmp(buf, strof(unsubpending)"|", 13)) {
-                               if (time(NULL) - extract_long(buf, 3) > EXP) {
-                                       skipthisline = 1;
-                               }
-                       }
 
-                       if (skipthisline == 0) {
-                               linesize = strlen(buf);
-                               sc->misc = realloc(sc->misc,
-                                       (miscsize + linesize + 2) );
-                               sprintf(&sc->misc[miscsize], "%s\n", buf);
-                               miscsize = miscsize + linesize + 1;
-                       }
-               }
+void SerializeLastSent(const CfgLineType *ThisOne, StrBuf *OutputBuffer, OneRoomNetCfg *RNCfg, RoomNetCfgLine *data)
+{
+       StrBufAppendBufPlain(OutputBuffer, CKEY(ThisOne->Str), 0);
+       StrBufAppendPrintf(OutputBuffer, "|%ld\n", RNCfg->lastsent);
+}
 
+void DeleteLastSent(const CfgLineType *ThisOne, RoomNetCfgLine **data)
+{
+       free(*data);
+       *data = NULL;
+}
+
+static const RoomNetCfg SpoolCfgs [4] = {
+       listrecp,
+       digestrecp,
+       participate,
+       ignet_push_share
+};
+
+static const long SpoolCfgsCopyN [4] = {
+       1, 1, 1, 2
+};
+
+int HaveSpoolConfig(OneRoomNetCfg* RNCfg)
+{
+       int i;
+       int interested = 0;
+       for (i=0; i < 4; i++) if (RNCfg->NetConfigs[SpoolCfgs[i]] == NULL) interested = 1;
+       return interested;
+}
 
+void Netmap_AddMe(struct CtdlMessage *msg, const char *defl, long defllen)
+{
+       long node_len;
+       char buf[SIZ];
+
+       /* prepend our node to the path */
+       if (CM_IsEmpty(msg, eMessagePath)) {
+               CM_SetField(msg, eMessagePath, defl, defllen);
        }
-       fclose(fp);
-       return 1;
+       node_len = configlen.c_nodename;
+       if (node_len >= SIZ) 
+               node_len = SIZ - 1;
+       memcpy(buf, config.c_nodename, node_len);
+       buf[node_len] = '!';
+       buf[node_len + 1] = '\0';
+       CM_PrependToField(msg, eMessagePath, buf, node_len + 1);
 }
 
-void free_spoolcontrol_struct(SpoolControl **scc)
+void InspectQueuedRoom(SpoolControl **pSC,
+                      RoomProcList *room_to_spool,     
+                      HashList *working_ignetcfg,
+                      HashList *the_netmap)
 {
+       struct CitContext *CCC = CC;
        SpoolControl *sc;
-       namelist *nptr = NULL;
-       maplist *mptr = NULL;
-
-       sc = *scc;
-       while (sc->listrecps != NULL) {
-               nptr = sc->listrecps->next;
-               free(sc->listrecps);
-               sc->listrecps = nptr;
-       }
-       /* Do the same for digestrecps */
-       while (sc->digestrecps != NULL) {
-               nptr = sc->digestrecps->next;
-               free(sc->digestrecps);
-               sc->digestrecps = nptr;
-       }
-       /* Do the same for participates */
-       while (sc->participates != NULL) {
-               nptr = sc->participates->next;
-               free(sc->participates);
-               sc->participates = nptr;
-       }
-       while (sc->ignet_push_shares != NULL) {
-               mptr = sc->ignet_push_shares->next;
-               free(sc->ignet_push_shares);
-               sc->ignet_push_shares = mptr;
-       }
-       free(sc->misc);
-       free(sc);
-       *scc=NULL;
+       int i = 0;
+
+       sc = (SpoolControl*)malloc(sizeof(SpoolControl));
+       memset(sc, 0, sizeof(SpoolControl));
+       sc->RNCfg = room_to_spool->OneRNCfg;
+       sc->lastsent = room_to_spool->lastsent;
+       sc->working_ignetcfg = working_ignetcfg;
+       sc->the_netmap = the_netmap;
+
+       /*
+        * If the room doesn't exist, don't try to perform its networking tasks.
+        * Normally this should never happen, but once in a while maybe a room gets
+        * queued for networking and then deleted before it can happen.
+        */
+       if (CtdlGetRoom(&sc->room, room_to_spool->name) != 0) {
+               syslog(LOG_CRIT, "ERROR: cannot load <%s>\n", room_to_spool->name);
+               free(sc);
+               return;
+       }
+       if (sc->room.QRhighest <= sc->lastsent)
+       {
+               QN_syslog(LOG_DEBUG, "nothing to do for <%s>\n", room_to_spool->name);
+               free(sc);
+               return;
+       }
+
+       begin_critical_section(S_NETCONFIGS);
+       if (sc->RNCfg == NULL)
+               sc->RNCfg = CtdlGetNetCfgForRoom(sc->room.QRnumber);
+
+       if (!HaveSpoolConfig(sc->RNCfg))
+       {
+               end_critical_section(S_NETCONFIGS);
+               free(sc);
+               /* nothing to do for this room... */
+               return;
+       }
+
+       /* Now lets remember whats needed for the actual work... */
+
+       for (i=0; i < 4; i++)
+       {
+               aggregate_recipients(&sc->Users[SpoolCfgs[i]],
+                                    SpoolCfgs[i],
+                                    sc->RNCfg,
+                                    SpoolCfgsCopyN[i]);
+       }
+       
+       if (StrLength(sc->RNCfg->Sender) > 0)
+               sc->Users[roommailalias] = NewStrBufDup(sc->RNCfg->Sender);
+       end_critical_section(S_NETCONFIGS);
+
+       sc->next = *pSC;
+       *pSC = sc;
+
 }
 
-int writenfree_spoolcontrol_file(SpoolControl **scc, char *filename)
+void CalcListID(SpoolControl *sc)
 {
-       char tempfilename[PATH_MAX];
-       int TmpFD;
-       SpoolControl *sc;
-       namelist *nptr = NULL;
-       maplist *mptr = NULL;
-       long len;
-       time_t unixtime;
-       struct timeval tv;
-       long reltid; /* if we don't have SYS_gettid, use "random" value */
-       StrBuf *Cfg;
-       int rc;
+       StrBuf *RoomName;
+       const char *err;
+       int fd;
+       struct CitContext *CCC = CC;
+       char filename[PATH_MAX];
+#define MAX_LISTIDLENGTH 150
 
-       len = strlen(filename);
-       memcpy(tempfilename, filename, len + 1);
+       assoc_file_name(filename, sizeof filename, &sc->room, ctdl_info_dir);
+       fd = open(filename, 0);
 
+       if (fd > 0) {
+               struct stat stbuf;
 
-#if defined(HAVE_SYSCALL_H) && defined (SYS_gettid)
-       reltid = syscall(SYS_gettid);
-#endif
-       gettimeofday(&tv, NULL);
-       /* Promote to time_t; types differ on some OSes (like darwin) */
-       unixtime = tv.tv_sec;
-
-       sprintf(tempfilename + len, ".%ld-%ld", reltid, unixtime);
-       sc = *scc;
-       errno = 0;
-       TmpFD = open(tempfilename, O_CREAT|O_EXCL|O_RDWR, S_IRUSR|S_IWUSR);
-       Cfg = NewStrBuf();
-       if ((TmpFD < 0) || (errno != 0)) {
-               syslog(LOG_CRIT, "ERROR: cannot open %s: %s\n",
-                       filename, strerror(errno));
-               free_spoolcontrol_struct(scc);
-               unlink(tempfilename);
-       }
-       else {
-               fchown(TmpFD, config.c_ctdluid, 0);
-               StrBufAppendPrintf(Cfg, "lastsent|%ld\n", sc->lastsent);
-               
-               /* Write out the listrecps while freeing from memory at the
-                * same time.  Am I clever or what?  :)
-                */
-               while (sc->listrecps != NULL) {
-                   StrBufAppendPrintf(Cfg, "listrecp|%s\n", sc->listrecps->name);
-                       nptr = sc->listrecps->next;
-                       free(sc->listrecps);
-                       sc->listrecps = nptr;
-               }
-               /* Do the same for digestrecps */
-               while (sc->digestrecps != NULL) {
-                       StrBufAppendPrintf(Cfg, "digestrecp|%s\n", sc->digestrecps->name);
-                       nptr = sc->digestrecps->next;
-                       free(sc->digestrecps);
-                       sc->digestrecps = nptr;
-               }
-               /* Do the same for participates */
-               while (sc->participates != NULL) {
-                       StrBufAppendPrintf(Cfg, "participate|%s\n", sc->participates->name);
-                       nptr = sc->participates->next;
-                       free(sc->participates);
-                       sc->participates = nptr;
-               }
-               while (sc->ignet_push_shares != NULL) {
-                       StrBufAppendPrintf(Cfg, "ignet_push_share|%s", sc->ignet_push_shares->remote_nodename);
-                       if (!IsEmptyStr(sc->ignet_push_shares->remote_roomname)) {
-                               StrBufAppendPrintf(Cfg, "|%s", sc->ignet_push_shares->remote_roomname);
-                       }
-                       StrBufAppendPrintf(Cfg, "\n");
-                       mptr = sc->ignet_push_shares->next;
-                       free(sc->ignet_push_shares);
-                       sc->ignet_push_shares = mptr;
-               }
-               if (sc->misc != NULL) {
-                       StrBufAppendBufPlain(Cfg, sc->misc, -1, 0);
+               if ((fstat(fd, &stbuf) == 0) &&
+                   (stbuf.st_size > 0))
+               {
+                       sc->RoomInfo = NewStrBufPlain(NULL, stbuf.st_size + 1);
+                       StrBufReadBLOB(sc->RoomInfo, &fd, 0, stbuf.st_size, &err);
                }
-               free(sc->misc);
+               close(fd);
+       }
+
+       sc->ListID = NewStrBufPlain(NULL, 1024);
+       if (StrLength(sc->RoomInfo) > 0)
+       {
+               const char *Pos = NULL;
+               StrBufSipLine(sc->ListID, sc->RoomInfo, &Pos);
 
-               rc = write(TmpFD, ChrPtr(Cfg), StrLength(Cfg));
-               if ((rc >=0 ) && (rc == StrLength(Cfg))) 
+               if (StrLength(sc->ListID) > MAX_LISTIDLENGTH)
                {
-                       close(TmpFD);
-                       rename(tempfilename, filename);
+                       StrBufCutAt(sc->ListID, MAX_LISTIDLENGTH, NULL);
+                       StrBufAppendBufPlain(sc->ListID, HKEY("..."), 0);
                }
-               else {
-                       syslog(LOG_EMERG, 
-                                     "unable to write %s; [%s]; not enough space on the disk?\n", 
-                                     tempfilename, 
-                                     strerror(errno));
-                       close(TmpFD);
-                       unlink(tempfilename);
+               StrBufAsciify(sc->ListID, ' ');
+       }
+       else
+       {
+               StrBufAppendBufPlain(sc->ListID, CCC->room.QRname, -1, 0);
+       }
+
+       StrBufAppendBufPlain(sc->ListID, HKEY("<"), 0);
+       RoomName = NewStrBufPlain (sc->room.QRname, -1);
+       StrBufAsciify(RoomName, '_');
+       StrBufReplaceChars(RoomName, ' ', '_');
+
+       if (StrLength(sc->Users[roommailalias]) > 0)
+       {
+               long Pos;
+               const char *AtPos;
+
+               Pos = StrLength(sc->ListID);
+               StrBufAppendBuf(sc->ListID, sc->Users[roommailalias], 0);
+               AtPos = strchr(ChrPtr(sc->ListID) + Pos, '@');
+
+               if (AtPos != NULL)
+               {
+                       StrBufPeek(sc->ListID, AtPos, 0, '.');
                }
-               FreeStrBuf(&Cfg);
-               free(sc);
-               *scc=NULL;
        }
-       return 1;
-}
-int is_recipient(SpoolControl *sc, const char *Name)
-{
-       namelist *nptr;
-       size_t len;
-
-       len = strlen(Name);
-       nptr = sc->listrecps;
-       while (nptr != NULL) {
-               if (strncmp(Name, nptr->name, len)==0)
-                       return 1;
-               nptr = nptr->next;
-       }
-       /* Do the same for digestrecps */
-       nptr = sc->digestrecps;
-       while (nptr != NULL) {
-               if (strncmp(Name, nptr->name, len)==0)
-                       return 1;
-               nptr = nptr->next;
-       }
-       /* Do the same for participates */
-       nptr = sc->participates;
-       while (nptr != NULL) {
-               if (strncmp(Name, nptr->name, len)==0)
-                       return 1;
-               nptr = nptr->next;
-       }
-       return 0;
+       else
+       {
+               StrBufAppendBufPlain(sc->ListID, HKEY("room_"), 0);
+               StrBufAppendBuf(sc->ListID, RoomName, 0);
+               StrBufAppendBufPlain(sc->ListID, HKEY("."), 0);
+               StrBufAppendBufPlain(sc->ListID, config.c_fqdn, -1, 0);
+               /*
+                * this used to be:
+                * roomname <Room-Number.list-id.fqdn>
+                * according to rfc2919.txt it only has to be a uniq identifier
+                * under the domain of the system; 
+                * in general MUAs use it to calculate the reply address nowadays.
+                */
+       }
+       StrBufAppendBufPlain(sc->ListID, HKEY(">"), 0);
+
+       if (StrLength(sc->Users[roommailalias]) == 0)
+       {
+               sc->Users[roommailalias] = NewStrBuf();
+               
+               StrBufAppendBufPlain(sc->Users[roommailalias], HKEY("room_"), 0);
+               StrBufAppendBuf(sc->Users[roommailalias], RoomName, 0);
+               StrBufAppendBufPlain(sc->Users[roommailalias], HKEY("@"), 0);
+               StrBufAppendBufPlain(sc->Users[roommailalias], config.c_fqdn, -1, 0);
+
+               StrBufLowerCase(sc->Users[roommailalias]);
+       }
+
+       FreeStrBuf(&RoomName);
 }
 
+static time_t last_digest_delivery = 0;
 
 /*
  * Batch up and send all outbound traffic from the current room
  */
-void network_spoolout_room(char *room_to_spool,                       
-                          char *working_ignetcfg,
-                          NetMap *the_netmap)
+void network_spoolout_room(SpoolControl *sc)
 {
+       struct CitContext *CCC = CC;
        char buf[SIZ];
-       char filename[PATH_MAX];
-       SpoolControl *sc;
        int i;
+       long lastsent;
 
        /*
         * If the room doesn't exist, don't try to perform its networking tasks.
         * Normally this should never happen, but once in a while maybe a room gets
         * queued for networking and then deleted before it can happen.
         */
-       if (CtdlGetRoom(&CC->room, room_to_spool) != 0) {
-               syslog(LOG_CRIT, "ERROR: cannot load <%s>\n", room_to_spool);
-               return;
-       }
+       memcpy (&CCC->room, &sc->room, sizeof(ctdlroom));
 
-       assoc_file_name(filename, sizeof filename, &CC->room, ctdl_netcfg_dir);
-       begin_critical_section(S_NETCONFIGS);
-
-       /* Only do net processing for rooms that have netconfigs */
-       if (!read_spoolcontrol_file(&sc, filename))
-       {
-               end_critical_section(S_NETCONFIGS);
-               return;
-       }
-       syslog(LOG_INFO, "Networking started for <%s>\n", CC->room.QRname);
+       syslog(LOG_INFO, "Networking started for <%s>\n", CCC->room.QRname);
 
-       sc->working_ignetcfg = working_ignetcfg;
-       sc->the_netmap = the_netmap;
+       CalcListID(sc);
 
-       /* If there are digest recipients, we have to build a digest */
-       if (sc->digestrecps != NULL) {
-               sc->digestfp = tmpfile();
-               fprintf(sc->digestfp, "Content-type: text/plain\n\n");
-       }
+       /* remember where we started... */
+       lastsent = sc->lastsent;
 
-       /* Do something useful */
+       /* Fetch the messages we ought to send & prepare them. */
        CtdlForEachMessage(MSGS_GT, sc->lastsent, NULL, NULL, NULL,
                network_spool_msg, sc);
 
-       /* If we wrote a digest, deliver it and then close it */
-       snprintf(buf, sizeof buf, "room_%s@%s",
-               CC->room.QRname, config.c_fqdn);
+       if (StrLength(sc->Users[roommailalias]) > 0)
+       {
+               long len;
+               len = StrLength(sc->Users[roommailalias]);
+               if (len + 1 > sizeof(buf))
+                       len = sizeof(buf) - 1;
+               memcpy(buf, ChrPtr(sc->Users[roommailalias]), len);
+               buf[len] = '\0';
+       }
+       else
+       {
+               snprintf(buf, sizeof buf, "room_%s@%s",
+                        CCC->room.QRname, config.c_fqdn);
+       }
+
        for (i=0; buf[i]; ++i) {
                buf[i] = tolower(buf[i]);
                if (isspace(buf[i])) buf[i] = '_';
        }
-       if (sc->digestfp != NULL) {
-               fprintf(sc->digestfp,   " -----------------------------------"
+
+
+       /* If we wrote a digest, deliver it and then close it */
+       if (sc->Users[digestrecp] != NULL) {
+               time_t now = time(NULL);
+               time_t secs_today = now % (24 * 60 * 60);
+               long delta = 0;
+
+               if (last_digest_delivery != 0) {
+                       delta = now - last_digest_delivery;
+                       delta = (24 * 60 * 60) - delta;
+               }
+
+               if ((secs_today < 300) && 
+                   (delta < 300))
+               {
+                       if (sc->digestfp == NULL) {
+                               sc->digestfp = create_digest_file(&sc->room, 0);
+                       }
+                       if (sc->digestfp != NULL) {
+                               last_digest_delivery = now;
+                               fprintf(sc->digestfp,
+                                       " -----------------------------------"
                                        "------------------------------------"
                                        "-------\n"
                                        "You are subscribed to the '%s' "
                                        "list.\n"
                                        "To post to the list: %s\n",
-                                       CC->room.QRname, buf
-               );
-               network_deliver_digest(sc);     /* deliver and close */
+                                       CCC->room.QRname, buf
+                                       );
+                               network_deliver_digest(sc);     /* deliver */
+                               remove_digest_file(&sc->room);
+                       }
+               }
+       }
+       if (sc->digestfp != NULL) {
+               fclose(sc->digestfp);
+               sc->digestfp = NULL;
        }
 
        /* Now rewrite the config file */
-       writenfree_spoolcontrol_file(&sc, filename);
-       end_critical_section(S_NETCONFIGS);
+       if (sc->lastsent != lastsent)
+       {
+               begin_critical_section(S_NETCONFIGS);
+               sc->RNCfg = CtdlGetNetCfgForRoom(sc->room.QRnumber);
+
+               sc->RNCfg->lastsent = sc->lastsent;
+               sc->RNCfg->changed = 1;
+               end_critical_section(S_NETCONFIGS);
+       }
 }
 
 /*
  * Process a buffer containing a single message from a single file
  * from the inbound queue 
  */
-void network_process_buffer(char *buffer, long size, char *working_ignetcfg, NetMap *the_netmap, int *netmap_changed)
+void network_process_buffer(char *buffer, long size, HashList *working_ignetcfg, HashList *the_netmap, int *netmap_changed)
 {
+       long len;
+       struct CitContext *CCC = CC;
+       StrBuf *Buf = NULL;
        struct CtdlMessage *msg = NULL;
        long pos;
        int field;
-       struct recptypes *recp = NULL;
+       recptypes *recp = NULL;
        char target_room[ROOMNAMELEN];
        struct ser_ret sermsg;
-       char *oldpath = NULL;
        char filename[PATH_MAX];
        FILE *fp;
-       char nexthop[SIZ];
+       const StrBuf *nexthop = NULL;
        unsigned char firstbyte;
        unsigned char lastbyte;
 
-       syslog(LOG_DEBUG, "network_process_buffer() processing %ld bytes\n", size);
+       QN_syslog(LOG_DEBUG, "network_process_buffer() processing %ld bytes\n", size);
 
        /* Validate just a little bit.  First byte should be FF and * last byte should be 00. */
        firstbyte = buffer[0];
        lastbyte = buffer[size-1];
        if ( (firstbyte != 255) || (lastbyte != 0) ) {
-               syslog(LOG_ERR, "Corrupt message ignored.  Length=%ld, firstbyte = %d, lastbyte = %d\n",
-                       size, firstbyte, lastbyte);
+               QN_syslog(LOG_ERR, "Corrupt message ignored.  Length=%ld, firstbyte = %d, lastbyte = %d\n",
+                         size, firstbyte, lastbyte);
                return;
        }
 
@@ -489,66 +478,57 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net
 
        for (pos = 3; pos < size; ++pos) {
                field = buffer[pos];
-               msg->cm_fields[field] = strdup(&buffer[pos+1]);
-               pos = pos + strlen(&buffer[(int)pos]);
+               len = strlen(buffer + pos + 1);
+               CM_SetField(msg, field, buffer + pos + 1, len);
+               pos = pos + len + 1;
        }
 
        /* Check for message routing */
-       if (msg->cm_fields['D'] != NULL) {
-               if (strcasecmp(msg->cm_fields['D'], config.c_nodename)) {
+       if (!CM_IsEmpty(msg, eDestination)) {
+               if (strcasecmp(msg->cm_fields[eDestination], config.c_nodename)) {
 
                        /* route the message */
-                       strcpy(nexthop, "");
-                       if (is_valid_node(nexthop, 
-                                         NULL, 
-                                         msg->cm_fields['D']
-                                         working_ignetcfg, 
-                                         the_netmap) == 0) 
+                       Buf = NewStrBufPlain(CM_KEY(msg,eDestination));
+                       if (CtdlIsValidNode(&nexthop, 
+                                           NULL, 
+                                           Buf
+                                           working_ignetcfg, 
+                                           the_netmap) == 0) 
                        {
-                               /* prepend our node to the path */
-                               if (msg->cm_fields['P'] != NULL) {
-                                       oldpath = msg->cm_fields['P'];
-                                       msg->cm_fields['P'] = NULL;
-                               }
-                               else {
-                                       oldpath = strdup("unknown_user");
-                               }
-                               size = strlen(oldpath) + SIZ;
-                               msg->cm_fields['P'] = malloc(size);
-                               snprintf(msg->cm_fields['P'], size, "%s!%s",
-                                       config.c_nodename, oldpath);
-                               free(oldpath);
+                               Netmap_AddMe(msg, HKEY("unknown_user"));
 
                                /* serialize the message */
-                               serialize_message(&sermsg, msg);
+                               CtdlSerializeMessage(&sermsg, msg);
 
                                /* now send it */
-                               if (IsEmptyStr(nexthop)) {
-                                       strcpy(nexthop, msg->cm_fields['D']);
+                               if (StrLength(nexthop) == 0) {
+                                       nexthop = Buf;
                                }
-                               snprintf(filename, 
-                                       sizeof filename,
-                                       "%s/%s@%lx%x",
-                                       ctdl_netout_dir,
-                                       nexthop,
-                                       time(NULL),
-                                       rand()
+                               snprintf(filename,
+                                        sizeof filename,
+                                        "%s/%s@%lx%x",
+                                        ctdl_netout_dir,
+                                        ChrPtr(nexthop),
+                                        time(NULL),
+                                        rand()
                                );
-                               syslog(LOG_DEBUG, "Appending to %s\n", filename);
+                               QN_syslog(LOG_DEBUG, "Appending to %s\n", filename);
                                fp = fopen(filename, "ab");
                                if (fp != NULL) {
                                        fwrite(sermsg.ser, sermsg.len, 1, fp);
                                        fclose(fp);
                                }
                                else {
-                                       syslog(LOG_ERR, "%s: %s\n", filename, strerror(errno));
+                                       QN_syslog(LOG_ERR, "%s: %s\n", filename, strerror(errno));
                                }
                                free(sermsg.ser);
-                               CtdlFreeMessage(msg);
+                               CM_Free(msg);
+                               FreeStrBuf(&Buf);
                                return;
                        }
                        
                        else {  /* invalid destination node name */
+                               FreeStrBuf(&Buf);
 
                                network_bounce(msg,
 "A message you sent could not be delivered due to an invalid destination node"
@@ -567,63 +547,57 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net
         * connected that it's inevitable.)
         */
        if (network_usetable(msg) != 0) {
-               CtdlFreeMessage(msg);
+               CM_Free(msg);
                return;
        }
 
        /* Learn network topology from the path */
-       if ((msg->cm_fields['N'] != NULL) && (msg->cm_fields['P'] != NULL)) {
-               network_learn_topology(msg->cm_fields['N'], 
-                                      msg->cm_fields['P'], 
-                                      the_netmap, 
-                                      netmap_changed);
+       if (!CM_IsEmpty(msg, eNodeName) && !CM_IsEmpty(msg, eMessagePath)) {
+               NetworkLearnTopology(msg->cm_fields[eNodeName], 
+                                    msg->cm_fields[eMessagePath], 
+                                    the_netmap, 
+                                    netmap_changed);
        }
 
        /* Is the sending node giving us a very persuasive suggestion about
         * which room this message should be saved in?  If so, go with that.
         */
-       if (msg->cm_fields['C'] != NULL) {
-               safestrncpy(target_room, msg->cm_fields['C'], sizeof target_room);
+       if (!CM_IsEmpty(msg, eRemoteRoom)) {
+               safestrncpy(target_room, msg->cm_fields[eRemoteRoom], sizeof target_room);
        }
 
        /* Otherwise, does it have a recipient?  If so, validate it... */
-       else if (msg->cm_fields['R'] != NULL) {
-               recp = validate_recipients(msg->cm_fields['R'], NULL, 0);
+       else if (!CM_IsEmpty(msg, eRecipient)) {
+               recp = validate_recipients(msg->cm_fields[eRecipient], NULL, 0);
                if (recp != NULL) if (recp->num_error != 0) {
                        network_bounce(msg,
                                "A message you sent could not be delivered due to an invalid address.\n"
                                "Please check the address and try sending the message again.\n");
                        msg = NULL;
                        free_recipients(recp);
-                       syslog(LOG_DEBUG, "Bouncing message due to invalid recipient address.\n");
+                       QNM_syslog(LOG_DEBUG, "Bouncing message due to invalid recipient address.\n");
                        return;
                }
                strcpy(target_room, "");        /* no target room if mail */
        }
 
        /* Our last shot at finding a home for this message is to see if
-        * it has the O field (Originating room) set.
+        * it has the eOriginalRoom (O) field (Originating room) set.
         */
-       else if (msg->cm_fields['O'] != NULL) {
-               safestrncpy(target_room, msg->cm_fields['O'], sizeof target_room);
+       else if (!CM_IsEmpty(msg, eOriginalRoom)) {
+               safestrncpy(target_room, msg->cm_fields[eOriginalRoom], sizeof target_room);
        }
 
        /* Strip out fields that are only relevant during transit */
-       if (msg->cm_fields['D'] != NULL) {
-               free(msg->cm_fields['D']);
-               msg->cm_fields['D'] = NULL;
-       }
-       if (msg->cm_fields['C'] != NULL) {
-               free(msg->cm_fields['C']);
-               msg->cm_fields['C'] = NULL;
-       }
+       CM_FlushField(msg, eDestination);
+       CM_FlushField(msg, eRemoteRoom);
 
        /* save the message into a room */
        if (PerformNetprocHooks(msg, target_room) == 0) {
                msg->cm_flags = CM_SKIP_HOOKS;
                CtdlSubmitMsg(msg, recp, target_room, 0);
        }
-       CtdlFreeMessage(msg);
+       CM_Free(msg);
        free_recipients(recp);
 }
 
@@ -634,8 +608,8 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net
 void network_process_message(FILE *fp, 
                             long msgstart, 
                             long msgend,
-                            char *working_ignetcfg,
-                            NetMap *the_netmap, 
+                            HashList *working_ignetcfg,
+                            HashList *the_netmap, 
                             int *netmap_changed)
 {
        long hold_pos;
@@ -665,25 +639,26 @@ void network_process_message(FILE *fp,
  * Process a single file from the inbound queue 
  */
 void network_process_file(char *filename,
-                         char *working_ignetcfg,
-                         NetMap *the_netmap, 
+                         HashList *working_ignetcfg,
+                         HashList *the_netmap, 
                          int *netmap_changed)
 {
+       struct CitContext *CCC = CC;
        FILE *fp;
        long msgstart = (-1L);
        long msgend = (-1L);
        long msgcur = 0L;
        int ch;
-
+       int nMessages = 0;
 
        fp = fopen(filename, "rb");
        if (fp == NULL) {
-               syslog(LOG_CRIT, "Error opening %s: %s\n", filename, strerror(errno));
+               QN_syslog(LOG_CRIT, "Error opening %s: %s\n", filename, strerror(errno));
                return;
        }
 
        fseek(fp, 0L, SEEK_END);
-       syslog(LOG_INFO, "network: processing %ld bytes from %s\n", ftell(fp), filename);
+       QN_syslog(LOG_INFO, "network: processing %ld bytes from %s\n", ftell(fp), filename);
        rewind(fp);
 
        /* Look for messages in the data stream and break them out */
@@ -703,6 +678,7 @@ void network_process_file(char *filename,
                }
 
                ++msgcur;
+               nMessages ++;
        }
 
        msgend = msgcur - 1;
@@ -713,8 +689,15 @@ void network_process_file(char *filename,
                                        working_ignetcfg,
                                        the_netmap,
                                        netmap_changed);
+               nMessages ++;
        }
 
+       if (nMessages > 0)
+               QN_syslog(LOG_INFO,
+                         "network: processed %d messages in %s\n",
+                         nMessages,
+                         filename);
+
        fclose(fp);
        unlink(filename);
 }
@@ -723,13 +706,17 @@ void network_process_file(char *filename,
 /*
  * Process anything in the inbound queue
  */
-void network_do_spoolin(char *working_ignetcfg, NetMap *the_netmap, int *netmap_changed)
+void network_do_spoolin(HashList *working_ignetcfg, HashList *the_netmap, int *netmap_changed)
 {
+       struct CitContext *CCC = CC;
        DIR *dp;
        struct dirent *d;
+       struct dirent *filedir_entry;
        struct stat statbuf;
        char filename[PATH_MAX];
        static time_t last_spoolin_mtime = 0L;
+       int d_type = 0;
+        int d_namelen;
 
        /*
         * Check the spoolin directory's modification time.  If it hasn't
@@ -737,11 +724,11 @@ void network_do_spoolin(char *working_ignetcfg, NetMap *the_netmap, int *netmap_
         */
        if (stat(ctdl_netin_dir, &statbuf)) return;
        if (statbuf.st_mtime == last_spoolin_mtime) {
-               syslog(LOG_DEBUG, "network: nothing in inbound queue\n");
+               QNM_syslog(LOG_DEBUG, "network: nothing in inbound queue\n");
                return;
        }
        last_spoolin_mtime = statbuf.st_mtime;
-       syslog(LOG_DEBUG, "network: processing inbound queue\n");
+       QNM_syslog(LOG_DEBUG, "network: processing inbound queue\n");
 
        /*
         * Ok, there's something interesting in there, so scan it.
@@ -749,8 +736,60 @@ void network_do_spoolin(char *working_ignetcfg, NetMap *the_netmap, int *netmap_
        dp = opendir(ctdl_netin_dir);
        if (dp == NULL) return;
 
-       while (d = readdir(dp), d != NULL) {
-               if ((strcmp(d->d_name, ".")) && (strcmp(d->d_name, ".."))) {
+       d = (struct dirent *)malloc(offsetof(struct dirent, d_name) + PATH_MAX + 1);
+       if (d == NULL) {
+               closedir(dp);
+               return;
+       }
+
+       while ((readdir_r(dp, d, &filedir_entry) == 0) &&
+              (filedir_entry != NULL))
+       {
+#ifdef _DIRENT_HAVE_D_NAMLEN
+               d_namelen = filedir_entry->d_namlen;
+
+#else
+               d_namelen = strlen(filedir_entry->d_name);
+#endif
+
+#ifdef _DIRENT_HAVE_D_TYPE
+               d_type = filedir_entry->d_type;
+#else
+               d_type = DT_UNKNOWN;
+#endif
+               if ((d_namelen > 1) && filedir_entry->d_name[d_namelen - 1] == '~')
+                       continue; /* Ignore backup files... */
+
+               if ((d_namelen == 1) && 
+                   (filedir_entry->d_name[0] == '.'))
+                       continue;
+
+               if ((d_namelen == 2) && 
+                   (filedir_entry->d_name[0] == '.') &&
+                   (filedir_entry->d_name[1] == '.'))
+                       continue;
+
+               if (d_type == DT_UNKNOWN) {
+                       struct stat s;
+                       char path[PATH_MAX];
+
+                       snprintf(path,
+                                PATH_MAX,
+                                "%s/%s", 
+                                ctdl_netin_dir,
+                                filedir_entry->d_name);
+
+                       if (lstat(path, &s) == 0) {
+                               d_type = IFTODT(s.st_mode);
+                       }
+               }
+
+               switch (d_type)
+               {
+               case DT_DIR:
+                       break;
+               case DT_LNK: /* TODO: check whether its a file or a directory */
+               case DT_REG:
                        snprintf(filename, 
                                sizeof filename,
                                "%s/%s",
@@ -765,97 +804,274 @@ void network_do_spoolin(char *working_ignetcfg, NetMap *the_netmap, int *netmap_
        }
 
        closedir(dp);
+       free(d);
 }
 
 /*
  * Step 1: consolidate files in the outbound queue into one file per neighbor node
  * Step 2: delete any files in the outbound queue that were for neighbors who no longer exist.
  */
-void network_consolidate_spoolout(char *working_ignetcfg, NetMap *the_netmap)
+void network_consolidate_spoolout(HashList *working_ignetcfg, HashList *the_netmap)
 {
+       struct CitContext *CCC = CC;
+       IOBuffer IOB;
+       FDIOBuffer FDIO;
+        int d_namelen;
        DIR *dp;
        struct dirent *d;
+       struct dirent *filedir_entry;
+       const char *pch;
+       char spooloutfilename[PATH_MAX];
        char filename[PATH_MAX];
-       char cmd[PATH_MAX];
-       char nexthop[256];
+       const StrBuf *nexthop;
+       StrBuf *NextHop;
        int i;
-       char *ptr;
+       struct stat statbuf;
+       int nFailed = 0;
+       int d_type = 0;
+
 
        /* Step 1: consolidate files in the outbound queue into one file per neighbor node */
+       d = (struct dirent *)malloc(offsetof(struct dirent, d_name) + PATH_MAX + 1);
+       if (d == NULL)  return;
+
        dp = opendir(ctdl_netout_dir);
-       if (dp == NULL) return;
-       while (d = readdir(dp), d != NULL) {
-               if (
-                       (strcmp(d->d_name, "."))
-                       && (strcmp(d->d_name, ".."))
-                       && (strchr(d->d_name, '@') != NULL)
-               ) {
-                       safestrncpy(nexthop, d->d_name, sizeof nexthop);
-                       ptr = strchr(nexthop, '@');
-                       if (ptr) *ptr = 0;
-       
-                       snprintf(filename, 
-                               sizeof filename,
-                               "%s/%s",
-                               ctdl_netout_dir,
-                               d->d_name
-                       );
-       
-                       syslog(LOG_DEBUG, "Consolidate %s to %s\n", filename, nexthop);
-                       if (network_talking_to(nexthop, NTT_CHECK)) {
-                               syslog(LOG_DEBUG,
-                                       "Currently online with %s - skipping for now\n",
-                                       nexthop
+       if (dp == NULL) {
+               free(d);
+               return;
+       }
+
+       NextHop = NewStrBuf();
+       memset(&IOB, 0, sizeof(IOBuffer));
+       memset(&FDIO, 0, sizeof(FDIOBuffer));
+       FDIO.IOB = &IOB;
+
+       while ((readdir_r(dp, d, &filedir_entry) == 0) &&
+              (filedir_entry != NULL))
+       {
+#ifdef _DIRENT_HAVE_D_NAMLEN
+               d_namelen = filedir_entry->d_namlen;
+
+#else
+               d_namelen = strlen(filedir_entry->d_name);
+#endif
+
+#ifdef _DIRENT_HAVE_D_TYPE
+               d_type = filedir_entry->d_type;
+#else
+               d_type = DT_UNKNOWN;
+#endif
+               if (d_type == DT_DIR)
+                       continue;
+
+               if ((d_namelen > 1) && filedir_entry->d_name[d_namelen - 1] == '~')
+                       continue; /* Ignore backup files... */
+
+               if ((d_namelen == 1) && 
+                   (filedir_entry->d_name[0] == '.'))
+                       continue;
+
+               if ((d_namelen == 2) && 
+                   (filedir_entry->d_name[0] == '.') &&
+                   (filedir_entry->d_name[1] == '.'))
+                       continue;
+
+               pch = strchr(filedir_entry->d_name, '@');
+               if (pch == NULL)
+                       continue;
+
+               snprintf(filename, 
+                        sizeof filename,
+                        "%s/%s",
+                        ctdl_netout_dir,
+                        filedir_entry->d_name);
+
+               StrBufPlain(NextHop,
+                           filedir_entry->d_name,
+                           pch - filedir_entry->d_name);
+
+               snprintf(spooloutfilename,
+                        sizeof spooloutfilename,
+                        "%s/%s",
+                        ctdl_netout_dir,
+                        ChrPtr(NextHop));
+
+               QN_syslog(LOG_DEBUG, "Consolidate %s to %s\n", filename, ChrPtr(NextHop));
+               if (CtdlNetworkTalkingTo(SKEY(NextHop), NTT_CHECK)) {
+                       nFailed++;
+                       QN_syslog(LOG_DEBUG,
+                                 "Currently online with %s - skipping for now\n",
+                                 ChrPtr(NextHop)
                                );
+               }
+               else {
+                       size_t dsize;
+                       size_t fsize;
+                       int infd, outfd;
+                       const char *err = NULL;
+                       CtdlNetworkTalkingTo(SKEY(NextHop), NTT_ADD);
+
+                       infd = open(filename, O_RDONLY);
+                       if (infd == -1) {
+                               nFailed++;
+                               QN_syslog(LOG_ERR,
+                                         "failed to open %s for reading due to %s; skipping.\n",
+                                         filename, strerror(errno)
+                                       );
+                               CtdlNetworkTalkingTo(SKEY(NextHop), NTT_REMOVE);
+                               continue;                               
+                       }
+                       
+                       outfd = open(spooloutfilename,
+                                 O_EXCL|O_CREAT|O_NONBLOCK|O_WRONLY, 
+                                 S_IRUSR|S_IWUSR);
+                       if (outfd == -1)
+                       {
+                               outfd = open(spooloutfilename,
+                                            O_EXCL|O_NONBLOCK|O_WRONLY, 
+                                            S_IRUSR | S_IWUSR);
+                       }
+                       if (outfd == -1) {
+                               nFailed++;
+                               QN_syslog(LOG_ERR,
+                                         "failed to open %s for reading due to %s; skipping.\n",
+                                         spooloutfilename, strerror(errno)
+                                       );
+                               close(infd);
+                               CtdlNetworkTalkingTo(SKEY(NextHop), NTT_REMOVE);
+                               continue;
+                       }
+
+                       dsize = lseek(outfd, 0, SEEK_END);
+                       lseek(outfd, -dsize, SEEK_SET);
+
+                       fstat(infd, &statbuf);
+                       fsize = statbuf.st_size;
+/*
+                       fsize = lseek(infd, 0, SEEK_END);
+*/                     
+                       IOB.fd = infd;
+                       FDIOBufferInit(&FDIO, &IOB, outfd, fsize + dsize);
+                       FDIO.ChunkSendRemain = fsize;
+                       FDIO.TotalSentAlready = dsize;
+                       err = NULL;
+                       errno = 0;
+                       do {} while ((FileMoveChunked(&FDIO, &err) > 0) && (err == NULL));
+                       if (err == NULL) {
+                               unlink(filename);
+                               QN_syslog(LOG_DEBUG,
+                                         "Spoolfile %s now "SIZE_T_FMT" k\n",
+                                         spooloutfilename,
+                                         (dsize + fsize)/1024
+                                       );                              
                        }
                        else {
-                               network_talking_to(nexthop, NTT_ADD);
-                               snprintf(cmd, sizeof cmd, "/bin/cat %s >>%s/%s && /bin/rm -f %s",
-                                       filename,
-                                       ctdl_netout_dir, nexthop,
-                                       filename
-                               );
-                               system(cmd);
-                               network_talking_to(nexthop, NTT_REMOVE);
+                               nFailed++;
+                               QN_syslog(LOG_ERR,
+                                         "failed to append to %s [%s]; rolling back..\n",
+                                         spooloutfilename, strerror(errno)
+                                       );
+                               /* whoops partial append?? truncate spooloutfilename again! */
+                               ftruncate(outfd, dsize);
                        }
+                       FDIOBufferDelete(&FDIO);
+                       close(infd);
+                       close(outfd);
+                       CtdlNetworkTalkingTo(SKEY(NextHop), NTT_REMOVE);
                }
        }
        closedir(dp);
 
-       /* Step 2: delete any files in the outbound queue that were for neighbors who no longer exist */
+       if (nFailed > 0) {
+               FreeStrBuf(&NextHop);
+               QN_syslog(LOG_INFO,
+                         "skipping Spoolcleanup because of %d files unprocessed.\n",
+                         nFailed
+                       );
 
+               return;
+       }
+
+       /* Step 2: delete any files in the outbound queue that were for neighbors who no longer exist */
        dp = opendir(ctdl_netout_dir);
-       if (dp == NULL) return;
+       if (dp == NULL) {
+               FreeStrBuf(&NextHop);
+               free(d);
+               return;
+       }
+
+       while ((readdir_r(dp, d, &filedir_entry) == 0) &&
+              (filedir_entry != NULL))
+       {
+#ifdef _DIRENT_HAVE_D_NAMLEN
+               d_namelen = filedir_entry->d_namlen;
+
+#else
+               d_namelen = strlen(filedir_entry->d_name);
+#endif
 
-       while (d = readdir(dp), d != NULL) {
-               if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, ".."))
+#ifdef _DIRENT_HAVE_D_TYPE
+               d_type = filedir_entry->d_type;
+#else
+               d_type = DT_UNKNOWN;
+#endif
+               if (d_type == DT_DIR)
                        continue;
-               ptr = strchr(d->d_name, '@');
-               if (d != NULL)
+
+               if ((d_namelen == 1) && 
+                   (filedir_entry->d_name[0] == '.'))
+                       continue;
+
+               if ((d_namelen == 2) && 
+                   (filedir_entry->d_name[0] == '.') &&
+                   (filedir_entry->d_name[1] == '.'))
+                       continue;
+
+               pch = strchr(filedir_entry->d_name, '@');
+               if (pch == NULL) /* no @ in name? consolidated file. */
                        continue;
+
+               StrBufPlain(NextHop,
+                           filedir_entry->d_name,
+                           pch - filedir_entry->d_name);
+
                snprintf(filename, 
                        sizeof filename,
                        "%s/%s",
                        ctdl_netout_dir,
-                       d->d_name
+                       filedir_entry->d_name
                );
 
-               strcpy(nexthop, "");
-               i = is_valid_node(nexthop,
-                                 NULL,
-                                 d->d_name,
-                                 working_ignetcfg,
-                                 the_netmap);
+               i = CtdlIsValidNode(&nexthop,
+                                   NULL,
+                                   NextHop,
+                                   working_ignetcfg,
+                                   the_netmap);
        
-               if ( (i != 0) || !IsEmptyStr(nexthop) ) {
+               if ( (i != 0) || (StrLength(nexthop) > 0) ) {
                        unlink(filename);
                }
        }
-
-
+       FreeStrBuf(&NextHop);
+       free(d);
        closedir(dp);
 }
 
+void free_spoolcontrol_struct(SpoolControl **sc)
+{
+       free_spoolcontrol_struct_members(*sc);
+       free(*sc);
+       *sc = NULL;
+}
+
+void free_spoolcontrol_struct_members(SpoolControl *sc)
+{
+       int i;
+       FreeStrBuf(&sc->RoomInfo);
+       FreeStrBuf(&sc->ListID);
+       for (i = 0; i < maxRoomNetCfg; i++)
+               FreeStrBuf(&sc->Users[i]);
+}
 
 
 
@@ -888,8 +1104,17 @@ CTDL_MODULE_INIT(network_spool)
 {
        if (!threading)
        {
+               CtdlREGISTERRoomCfgType(subpending,       ParseSubPendingLine,   0, 5, SerializeGeneric,  DeleteGenericCfgLine); /// todo: move this to mailinglist manager
+               CtdlREGISTERRoomCfgType(unsubpending,     ParseUnSubPendingLine, 0, 4, SerializeGeneric,  DeleteGenericCfgLine); /// todo: move this to mailinglist manager
+               CtdlREGISTERRoomCfgType(lastsent,         ParseLastSent,         1, 1, SerializeLastSent, DeleteLastSent);
+               CtdlREGISTERRoomCfgType(ignet_push_share, ParseGeneric,          0, 2, SerializeGeneric,  DeleteGenericCfgLine); // [remotenode|remoteroomname (optional)]// todo: move this to the ignet client
+               CtdlREGISTERRoomCfgType(listrecp,         ParseGeneric,          0, 1, SerializeGeneric,  DeleteGenericCfgLine);
+               CtdlREGISTERRoomCfgType(digestrecp,       ParseGeneric,          0, 1, SerializeGeneric,  DeleteGenericCfgLine);
+               CtdlREGISTERRoomCfgType(participate,      ParseGeneric,          0, 1, SerializeGeneric,  DeleteGenericCfgLine);
+               CtdlREGISTERRoomCfgType(roommailalias,    ParseRoomAlias,        0, 1, SerializeGeneric,  DeleteGenericCfgLine);
+
                create_spool_dirs();
-               CtdlRegisterCleanupHook(destroy_network_queue_room);
+//////todo             CtdlRegisterCleanupHook(destroy_network_queue_room);
        }
        return "network_spool";
 }