Rework networker
authorWilfried Goesgens <dothebart@citadel.org>
Sun, 10 Jun 2012 10:38:30 +0000 (12:38 +0200)
committerWilfried Goesgens <dothebart@citadel.org>
Sun, 10 Jun 2012 10:38:30 +0000 (12:38 +0200)
  - keep netconfigs in hash; create the usual deserialize/delete/... functions
  - keep the netmap in hash; create the usual de/serialize/delete/... functions
  - change sequence: serialize & save netmap; combine spoolfiles; free netmap.
  - use strbufs splice() wrappers to combine the message-spoolfiles into host-spoolfiles instead of forking a shell & running cat + rm
  - when appending to host-spoolfile fails, remove partial adds to avoid creating invalid spool control files
  - only cleanup spool directory if we successfully combined all files; else we might loose messages if i.e. a networker is running for that node.

citadel/modules/network/netconfig.h
citadel/modules/network/netspool.h
citadel/modules/network/serv_netconfig.c
citadel/modules/network/serv_netmail.c
citadel/modules/network/serv_netspool.c
citadel/modules/network/serv_network.c

index 9d71d9fb4dbf1c3c8266fc8006fc2d9ff7ef785d..6168f0402487eda1fc21d11be02b8b575b591463 100644 (file)
@@ -1,13 +1,54 @@
-typedef struct NetMap NetMap;
+/*
+ * This module handles shared rooms, inter-Citadel mail, and outbound
+ * mailing list processing.
+ *
+ * Copyright (c) 2000-2012 by the citadel.org team
+ *
+ *  This program is open source software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ * ** NOTE **   A word on the S_NETCONFIGS semaphore:
+ * This is a fairly high-level type of critical section.  It ensures that no
+ * two threads work on the netconfigs files at the same time.  Since we do
+ * so many things inside these, here are the rules:
+ *  1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others.
+ *  2. Do *not* perform any I/O with the client during these sections.
+ *
+ */
 
-struct  NetMap {
-       NetMap *next;
-       char nodename[SIZ];
+typedef struct _nodeconf {
+       int DeleteMe;
+       StrBuf *NodeName;
+       StrBuf *Secret;
+       StrBuf *Host;
+       StrBuf *Port;
+}NodeConf;
+
+typedef struct __NetMap {
+       StrBuf *NodeName;
        time_t lastcontact;
-       char nexthop[SIZ];
-};
+       StrBuf *NextHop;
+}NetMap;
+
+HashList* load_ignetcfg(void);
+
+HashList* read_network_map(void);
+StrBuf *SerializeNetworkMap(HashList *Map);
+void network_learn_topology(char *node, char *path, HashList *the_netmap, int *netmap_changed);
 
-char* load_working_ignetcfg(void);
-NetMap *read_network_map(void);
-void write_and_free_network_map(NetMap **the_netmap, int netmap_changed);
-int is_valid_node(char *nexthop, char *secret, char *node, char *working_ignetcfg, NetMap *the_netmap);
+int is_valid_node(const StrBuf **nexthop,
+                 const StrBuf **secret,
+                 StrBuf *node,
+                 HashList *IgnetCfg,
+                 HashList *the_netmap);
index 0c0364ce34c8c8fae9bbec34d045b054088f1d0e..e52829bce1e3b4fd639f412c81585668b9e7a2ae 100644 (file)
@@ -1,3 +1,32 @@
+/*
+ * This module handles shared rooms, inter-Citadel mail, and outbound
+ * mailing list processing.
+ *
+ * Copyright (c) 2000-2012 by the citadel.org team
+ *
+ *  This program is open source software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ * ** NOTE **   A word on the S_NETCONFIGS semaphore:
+ * This is a fairly high-level type of critical section.  It ensures that no
+ * two threads work on the netconfigs files at the same time.  Since we do
+ * so many things inside these, here are the rules:
+ *  1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others.
+ *  2. Do *not* perform any I/O with the client during these sections.
+ *
+ */
+
 typedef struct maplist maplist;
 
 struct maplist {
@@ -19,16 +48,16 @@ struct SpoolControl {
        FILE *digestfp;
        int num_msgs_spooled;
 
-       char *working_ignetcfg;
-       NetMap *the_netmap;
+       HashList *working_ignetcfg;
+       HashList *the_netmap;
 };
 
 
 void network_spoolout_room(RoomProcList *room_to_spool,                       
-                          char *working_ignetcfg,
-                          NetMap *the_netmap);
-void network_do_spoolin(char *working_ignetcfg, NetMap **the_netmap, int *netmap_changed);
-void network_consolidate_spoolout(char *working_ignetcfg, NetMap *the_netmap);
+                          HashList *working_ignetcfg,
+                          HashList *the_netmap);
+void network_do_spoolin(HashList *working_ignetcfg, HashList *the_netmap, int *netmap_changed);
+void network_consolidate_spoolout(HashList *working_ignetcfg, HashList *the_netmap);
 void free_spoolcontrol_struct(SpoolControl **scc);
 int writenfree_spoolcontrol_file(SpoolControl **scc, char *filename);
 int read_spoolcontrol_file(SpoolControl **scc, char *filename);
index 9d1e6bc742bad69962b743a5eeb8f5488aa6cc53..cfcc20b6f84ca3fee5b678a3eb2387adfa6dadbe 100644 (file)
@@ -2,7 +2,7 @@
  * This module handles shared rooms, inter-Citadel mail, and outbound
  * mailing list processing.
  *
- * Copyright (c) 2000-2011 by the citadel.org team
+ * Copyright (c) 2000-2012 by the citadel.org team
  *
  *  This program is open source software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
 #include "ctdl_module.h"
 
 
+
+void DeleteNodeConf(void *vNode)
+{
+       NodeConf *Node = (NodeConf*) vNode;
+       FreeStrBuf(&Node->NodeName);
+       FreeStrBuf(&Node->Secret);
+       FreeStrBuf(&Node->Host);
+       FreeStrBuf(&Node->Port);
+       free(Node);
+}
+
+NodeConf *NewNode(StrBuf *SerializedNode)
+{
+       const char *Pos = NULL;
+       NodeConf *Node;
+
+       /* we need at least 4 pipes and some other text so its invalid. */
+       if (StrLength(SerializedNode) < 8)
+               return NULL;
+       Node = (NodeConf *) malloc(sizeof(NodeConf));
+
+       Node->DeleteMe = 0;
+
+       Node->NodeName=NewStrBuf();
+       StrBufExtract_NextToken(Node->NodeName, SerializedNode, &Pos, '|');
+
+       Node->Secret=NewStrBuf();
+       StrBufExtract_NextToken(Node->Secret, SerializedNode, &Pos, '|');
+
+       Node->Host=NewStrBuf();
+       StrBufExtract_NextToken(Node->Host, SerializedNode, &Pos, '|');
+
+       Node->Port=NewStrBuf();
+       StrBufExtract_NextToken(Node->Port, SerializedNode, &Pos, '|');
+       return Node;
+}
+
+
 /*
  * Load or refresh the Citadel network (IGnet) configuration for this node.
  */
-char* load_working_ignetcfg(void) {
-       return CtdlGetSysConfig(IGNETCFG);
+HashList* load_ignetcfg(void)
+{
+       const char *LinePos;
+       char       *Cfg;
+       StrBuf     *Buf;
+       StrBuf     *LineBuf;
+       HashList   *Hash;
+       NodeConf   *Node;
+
+       Cfg =  CtdlGetSysConfig(IGNETCFG);
+       if ((Cfg == NULL) || IsEmptyStr(Cfg)) {
+               if (Cfg != NULL)
+                       free(Cfg);
+               return NULL;
+       }
+
+       Hash = NewHash(1, NULL);
+       Buf = NewStrBufPlain(Cfg, -1);
+       free(Cfg);
+       LineBuf = NewStrBufPlain(NULL, StrLength(Buf));
+       LinePos = NULL;
+       do
+       {
+               StrBufSipLine(LineBuf, Buf, &LinePos);
+               if (StrLength(LineBuf) != 0) {
+                       Node = NewNode(LineBuf);
+                       if (Node != NULL) {
+                               Put(Hash, SKEY(Node->NodeName), Node, DeleteNodeConf);
+                       }
+               }
+       } while (LinePos != StrBufNOTNULL);
+       FreeStrBuf(&Buf);
+       FreeStrBuf(&LineBuf);
+       return Hash;
 }
 
+void DeleteNetMap(void *vNetMap)
+{
+       NetMap *TheNetMap = (NetMap*) vNetMap;
+       FreeStrBuf(&TheNetMap->NodeName);
+       FreeStrBuf(&TheNetMap->NextHop);
+       free(TheNetMap);
+}
 
-/* 
- * Read the network map from its configuration file into memory.
- */
-NetMap *read_network_map(void) {
-       char *serialized_map = NULL;
-       int i;
-       char buf[SIZ];
-       NetMap *nmptr, *the_netmap;
+NetMap *NewNetMap(StrBuf *SerializedNetMap)
+{
+       const char *Pos = NULL;
+       NetMap *NM;
 
-       the_netmap = NULL;
-       serialized_map = CtdlGetSysConfig(IGNETMAP);
-       if (serialized_map == NULL) return NULL;        /* if null, no entries */
+       /* we need at least 3 pipes and some other text so its invalid. */
+       if (StrLength(SerializedNetMap) < 6)
+               return NULL;
+       NM = (NetMap *) malloc(sizeof(NetMap));
 
-       /* Use the string tokenizer to grab one line at a time */
-       for (i=0; i<num_tokens(serialized_map, '\n'); ++i) {
-               extract_token(buf, serialized_map, i, '\n', sizeof buf);
+       NM->NodeName=NewStrBuf();
+       StrBufExtract_NextToken(NM->NodeName, SerializedNetMap, &Pos, '|');
 
-               nmptr = (NetMap *) malloc(sizeof(NetMap));
+       NM->lastcontact = StrBufExtractNext_long(SerializedNetMap, &Pos, '|');
 
-               extract_token(nmptr->nodename, buf, 0, '|', sizeof nmptr->nodename);
-               nmptr->lastcontact = extract_long(buf, 1);
-               extract_token(nmptr->nexthop, buf, 2, '|', sizeof nmptr->nexthop);
+       NM->NextHop=NewStrBuf();
+       StrBufExtract_NextToken(NM->NextHop, SerializedNetMap, &Pos, '|');
 
-               nmptr->next = the_netmap;
-               the_netmap = nmptr;
+       return NM;
+}
+
+HashList* read_network_map(void)
+{
+       const char *LinePos;
+       char       *Cfg;
+       StrBuf     *Buf;
+       StrBuf     *LineBuf;
+       HashList   *Hash;
+       NetMap     *TheNetMap;
+
+       Cfg =  CtdlGetSysConfig(IGNETMAP);
+       if ((Cfg == NULL) || IsEmptyStr(Cfg)) {
+               if (Cfg != NULL)
+                       free(Cfg);
+               return NULL;
        }
 
-       free(serialized_map);
-       return the_netmap;
+       Hash = NewHash(1, NULL);
+       Buf = NewStrBufPlain(Cfg, -1);
+       free(Cfg);
+       LineBuf = NewStrBufPlain(NULL, StrLength(Buf));
+       LinePos = NULL;
+       while (StrBufSipLine(Buf, LineBuf, &LinePos))
+       {
+               TheNetMap = NewNetMap(LineBuf);
+               if (TheNetMap != NULL) { /* TODO: is the NodeName Uniq? */
+                       Put(Hash, SKEY(TheNetMap->NodeName), TheNetMap, DeleteNetMap);
+               }
+       }
+       FreeStrBuf(&Buf);
+       FreeStrBuf(&LineBuf);
+       return Hash;
+}
+
+StrBuf *SerializeNetworkMap(HashList *Map)
+{
+       void *vMap;
+       const char *key;
+       long len;
+       StrBuf *Ret = NewStrBuf();
+       HashPos *Pos = GetNewHashPos(Map, 0);
+
+       while (GetNextHashPos(Map, Pos, &len, &key, &vMap))
+       {
+               NetMap *pMap = (NetMap*) vMap;
+               StrBufAppendBuf(Ret, pMap->NodeName, 0);
+               StrBufAppendBufPlain(Ret, HKEY("|"), 0);
+
+               StrBufAppendPrintf(Ret, "%ld", pMap->lastcontact, 0);
+               StrBufAppendBufPlain(Ret, HKEY("|"), 0);
+
+               StrBufAppendBuf(Ret, pMap->NextHop, 0);
+               StrBufAppendBufPlain(Ret, HKEY("\n"), 0);
+       }
+       DeleteHashPos(&Pos);
+       return Ret;
 }
 
 
 /*
- * Write the network map from memory back to the configuration file.
+ * Learn topology from path fields
  */
-void write_and_free_network_map(NetMap **the_netmap, int netmap_changed)
+void network_learn_topology(char *node, char *path, HashList *the_netmap, int *netmap_changed)
 {
-       char *serialized_map = NULL;
+       NetMap *pNM = NULL;
+       void *vptr;
+       char nexthop[256];
        NetMap *nmptr;
 
-       if (netmap_changed) {
-               serialized_map = strdup("");
-       
-               if (*the_netmap != NULL) {
-                       for (nmptr = *the_netmap; nmptr != NULL; nmptr = nmptr->next) {
-                               serialized_map = realloc(serialized_map,
-                                                       (strlen(serialized_map)+SIZ) );
-                               if (!IsEmptyStr(nmptr->nodename)) {
-                                       snprintf(&serialized_map[strlen(serialized_map)],
-                                               SIZ,
-                                               "%s|%ld|%s\n",
-                                               nmptr->nodename,
-                                               (long)nmptr->lastcontact,
-                                               nmptr->nexthop);
-                               }
-                       }
+       if (GetHash(the_netmap, node, strlen(node), &vptr) && 
+           (vptr != NULL))/* TODO: is the NodeName Uniq? */
+       {
+               pNM = (NetMap*)vptr;
+               extract_token(nexthop, path, 0, '!', sizeof nexthop);
+               if (!strcmp(nexthop, ChrPtr(pNM->NextHop))) {
+                       pNM->lastcontact = time(NULL);
+                       (*netmap_changed) ++;
+                       return;
                }
-
-               CtdlPutSysConfig(IGNETMAP, serialized_map);
-               free(serialized_map);
        }
 
-       /* Now free the list */
-       while (*the_netmap != NULL) {
-               nmptr = (*the_netmap)->next;
-               free(*the_netmap);
-               *the_netmap = nmptr;
-       }
+       /* If we got here then it's not in the map, so add it. */
+       nmptr = (NetMap *) malloc(sizeof (NetMap));
+       nmptr->NodeName = NewStrBufPlain(node, -1);
+       nmptr->lastcontact = time(NULL);
+       nmptr->NextHop = NewStrBuf ();
+       StrBufExtract_tokenFromStr(nmptr->NextHop, path, strlen(path), 0, '!');
+       /* TODO: is the NodeName Uniq? */
+       Put(the_netmap, SKEY(nmptr->NodeName), nmptr, DeleteNetMap);
+       (*netmap_changed) ++;
 }
 
 
-
-/* 
+/*
  * Check the network map and determine whether the supplied node name is
  * valid.  If it is not a neighbor node, supply the name of a neighbor node
  * which is the next hop.  If it *is* a neighbor node, we also fill in the
  * shared secret.
  */
-int is_valid_node(char *nexthop, 
-                 char *secret, 
-                 char *node, 
-                 char *working_ignetcfg, 
-                 NetMap *the_netmap)
+int is_valid_node(const StrBuf **nexthop,
+                 const StrBuf **secret,
+                 StrBuf *node,
+                 HashList *IgnetCfg,
+                 HashList *the_netmap)
 {
-       int i;
-       char linebuf[SIZ];
-       char buf[SIZ];
-       int retval;
-       NetMap *nmptr;
+       void *vNetMap;
+       void *vNodeConf;
+       NodeConf *TheNode;
+       NetMap *TheNetMap;
 
-       if (node == NULL) {
+       if (StrLength(node) == 0) {
                return(-1);
        }
 
        /*
         * First try the neighbor nodes
         */
-       if ((working_ignetcfg == NULL) || (*working_ignetcfg == '\0')) {
-               syslog(LOG_ERR, "working_ignetcfg is empty!\n");
+       if (GetCount(IgnetCfg) == 0) {
+               syslog(LOG_INFO, "IgnetCfg is empty!\n");
                if (nexthop != NULL) {
-                       strcpy(nexthop, "");
+                       *nexthop = NULL;
                }
                return(-1);
        }
 
-       retval = (-1);
-       if (nexthop != NULL) {
-               strcpy(nexthop, "");
-       }
-
-       /* Use the string tokenizer to grab one line at a time */
-       for (i=0; i<num_tokens(working_ignetcfg, '\n'); ++i) {
-               extract_token(linebuf, working_ignetcfg, i, '\n', sizeof linebuf);
-               extract_token(buf, linebuf, 0, '|', sizeof buf);
-               if (!strcasecmp(buf, node)) {
-                       if (nexthop != NULL) {
-                               strcpy(nexthop, "");
-                       }
-                       if (secret != NULL) {
-                               extract_token(secret, linebuf, 1, '|', 256);
-                       }
-                       retval = 0;
-               }
-       }
-
-       if (retval == 0) {
-               return(retval);         /* yup, it's a direct neighbor */
+       /* try to find a neigbour with the name 'node' */
+       if (GetHash(IgnetCfg, SKEY(node), &vNodeConf) && 
+           (vNodeConf != NULL))
+       {
+               TheNode = (NodeConf*)vNodeConf;
+               if (secret != NULL)
+                       *secret = TheNode->Secret;
+               return 0;               /* yup, it's a direct neighbor */
        }
 
-       /*      
+       /*
         * If we get to this point we have to see if we know the next hop
-        */
-       if (the_netmap != NULL) {
-               for (nmptr = the_netmap; nmptr != NULL; nmptr = nmptr->next) {
-                       if (!strcasecmp(nmptr->nodename, node)) {
-                               if (nexthop != NULL) {
-                                       strcpy(nexthop, nmptr->nexthop);
-                               }
-                               return(0);
-                       }
-               }
+        *//* TODO: is the NodeName Uniq? */
+       if ((GetCount(the_netmap) > 0) &&
+           (GetHash(the_netmap, SKEY(node), &vNetMap)))
+       {
+               TheNetMap = (NetMap*)vNetMap;
+               if (nexthop != NULL)
+                       *nexthop = TheNetMap->NextHop;
+               return(0);
        }
 
        /*
         * If we get to this point, the supplied node name is bogus.
         */
-       syslog(LOG_ERR, "Invalid node name <%s>\n", node);
+       syslog(LOG_ERR, "Invalid node name <%s>\n", ChrPtr(node));
        return(-1);
 }
 
 
 
-void cmd_gnet(char *argbuf) {
+void cmd_gnet(char *argbuf)
+{
        char filename[PATH_MAX];
        char buf[SIZ];
        FILE *fp;
@@ -365,24 +467,24 @@ void cmd_snet(char *argbuf) {
  */
 void cmd_netp(char *cmdbuf)
 {
-       char *working_ignetcfg;
+       HashList *working_ignetcfg;
        char node[256];
+       StrBuf *NodeStr;
        long nodelen;
        char pass[256];
        int v;
 
-       char secret[256] = "";
-       char nexthop[256] = "";
+       const StrBuf *secret = NULL;
+       const StrBuf *nexthop = NULL;
        char err_buf[SIZ] = "";
 
        /* Authenticate */
        nodelen = extract_token(node, cmdbuf, 0, '|', sizeof node);
        extract_token(pass, cmdbuf, 1, '|', sizeof pass);
-
+       NodeStr = NewStrBufPlain(node, nodelen);
        /* load the IGnet Configuration to check node validity */
-       working_ignetcfg = load_working_ignetcfg();
-       v = is_valid_node(nexthop, secret, node, working_ignetcfg, NULL); //// TODO do we need the netmap?
-
+       working_ignetcfg = load_ignetcfg();
+       v = is_valid_node(&nexthop, &secret, NodeStr, working_ignetcfg, NULL);
        if (v != 0) {
                snprintf(err_buf, sizeof err_buf,
                        "An unknown Citadel server called \"%s\" attempted to connect from %s [%s].\n",
@@ -391,11 +493,12 @@ void cmd_netp(char *cmdbuf)
                syslog(LOG_WARNING, "%s", err_buf);
                cprintf("%d authentication failed\n", ERROR + PASSWORD_REQUIRED);
                CtdlAideMessage(err_buf, "IGNet Networking.");
-               free(working_ignetcfg);
+               DeleteHash(&working_ignetcfg);
+               FreeStrBuf(&NodeStr);
                return;
        }
 
-       if (strcasecmp(pass, secret)) {
+       if (strcasecmp(pass, ChrPtr(secret))) {
                snprintf(err_buf, sizeof err_buf,
                        "A Citadel server at %s [%s] failed to authenticate as network node \"%s\".\n",
                        CC->cs_host, CC->cs_addr, node
@@ -403,14 +506,16 @@ void cmd_netp(char *cmdbuf)
                syslog(LOG_WARNING, "%s", err_buf);
                cprintf("%d authentication failed\n", ERROR + PASSWORD_REQUIRED);
                CtdlAideMessage(err_buf, "IGNet Networking.");
-               free(working_ignetcfg);
+               DeleteHash(&working_ignetcfg);
+               FreeStrBuf(&NodeStr);
                return;
        }
 
        if (network_talking_to(node, nodelen, NTT_CHECK)) {
                syslog(LOG_WARNING, "Duplicate session for network node <%s>", node);
                cprintf("%d Already talking to %s right now\n", ERROR + RESOURCE_BUSY, node);
-               free(working_ignetcfg);
+               DeleteHash(&working_ignetcfg);
+               FreeStrBuf(&NodeStr);
                return;
        }
 
@@ -420,7 +525,8 @@ void cmd_netp(char *cmdbuf)
                CC->net_node, CC->cs_host, CC->cs_addr
        );
        cprintf("%d authenticated as network node '%s'\n", CIT_OK, CC->net_node);
-       free(working_ignetcfg);
+       DeleteHash(&working_ignetcfg);
+       FreeStrBuf(&NodeStr);
 }
 
 int netconfig_check_roomaccess(
index 7dcda8db61a9c01044032541599dfedb0dbe9bc1..5b2a02524a5645ed36fa5ea6d2012c47445cfc04 100644 (file)
@@ -2,7 +2,7 @@
  * This module handles shared rooms, inter-Citadel mail, and outbound
  * mailing list processing.
  *
- * Copyright (c) 2000-2011 by the citadel.org team
+ * Copyright (c) 2000-2012 by the citadel.org team
  *
  *  This program is open source software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -265,6 +265,7 @@ void network_deliver_list(struct CtdlMessage *msg, SpoolControl *sc, const char
 void network_spool_msg(long msgnum,
                       void *userdata)
 {
+       StrBuf *Buf = NULL;
        SpoolControl *sc;
        int i;
        char *newpath = NULL;
@@ -470,9 +471,11 @@ void network_spool_msg(long msgnum,
                                                config.c_nodename)) {
                                        ok_to_participate = 1;
                                }
+
+                               Buf = NewStrBufPlain(msg->cm_fields['N'], -1);
                                if (is_valid_node(NULL,
                                                  NULL,
-                                                 msg->cm_fields['N'],
+                                                 Buf,
                                                  sc->working_ignetcfg,
                                                  sc->the_netmap) == 0)
                                {
@@ -560,11 +563,14 @@ void network_spool_msg(long msgnum,
                    mptr = mptr->next) {
 
                        send = 1;
-
+                       if (Buf == NULL)
+                               Buf = NewStrBufPlain(mptr->remote_nodename, -1);
+                       else
+                               StrBufPlain(Buf, mptr->remote_nodename, -1);
                        /* Check for valid node name */
                        if (is_valid_node(NULL,
                                          NULL,
-                                         mptr->remote_nodename,
+                                         Buf,
                                          sc->working_ignetcfg,
                                          sc->the_netmap) != 0)
                        {
@@ -665,5 +671,5 @@ void network_spool_msg(long msgnum,
        if (delete_after_send) {
                CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, "");
        }
-
+       FreeStrBuf(&Buf);
 }
index de98a3cf1ff68193193cd79f507f4a806ceb5a8a..479b303de7562ecdfcb63f2dc384035778d4ead1 100644 (file)
@@ -2,7 +2,7 @@
  * This module handles shared rooms, inter-Citadel mail, and outbound
  * mailing list processing.
  *
- * Copyright (c) 2000-2011 by the citadel.org team
+ * Copyright (c) 2000-2012 by the citadel.org team
  *
  *  This program is open source software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
 #include "netmail.h"
 #include "ctdl_module.h"
 
-/*
- * Learn topology from path fields
- */
-static void network_learn_topology(char *node, char *path, NetMap **the_netmap, int *netmap_changed)
-{
-       char nexthop[256];
-       NetMap *nmptr;
-
-       *nexthop = '\0';
-
-       if (num_tokens(path, '!') < 3) return;
-       for (nmptr = *the_netmap; nmptr != NULL; nmptr = nmptr->next) {
-               if (!strcasecmp(nmptr->nodename, node)) {
-                       extract_token(nmptr->nexthop, path, 0, '!', sizeof nmptr->nexthop);
-                       nmptr->lastcontact = time(NULL);
-                       (*netmap_changed) ++;
-                       return;
-               }
-       }
-
-       /* If we got here then it's not in the map, so add it. */
-       nmptr = (NetMap *) malloc(sizeof (NetMap));
-       strcpy(nmptr->nodename, node);
-       nmptr->lastcontact = time(NULL);
-       extract_token(nmptr->nexthop, path, 0, '!', sizeof nmptr->nexthop);
-       nmptr->next = *the_netmap;
-       *the_netmap = nmptr;
-       (*netmap_changed) ++;
-}
-
 
        
 
@@ -382,8 +352,8 @@ int is_recipient(SpoolControl *sc, const char *Name)
  * Batch up and send all outbound traffic from the current room
  */
 void network_spoolout_room(RoomProcList *room_to_spool,                       
-                          char *working_ignetcfg,
-                          NetMap *the_netmap)
+                          HashList *working_ignetcfg,
+                          HashList *the_netmap)
 {
        char buf[SIZ];
        char filename[PATH_MAX];
@@ -452,8 +422,9 @@ void network_spoolout_room(RoomProcList *room_to_spool,
  * Process a buffer containing a single message from a single file
  * from the inbound queue 
  */
-void network_process_buffer(char *buffer, long size, char *working_ignetcfg, NetMap **the_netmap, int *netmap_changed)
+void network_process_buffer(char *buffer, long size, HashList *working_ignetcfg, HashList *the_netmap, int *netmap_changed)
 {
+       StrBuf *Buf = NULL;
        struct CtdlMessage *msg = NULL;
        long pos;
        int field;
@@ -463,7 +434,7 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net
        char *oldpath = NULL;
        char filename[PATH_MAX];
        FILE *fp;
-       char nexthop[SIZ];
+       const StrBuf *nexthop = NULL;
        unsigned char firstbyte;
        unsigned char lastbyte;
 
@@ -499,12 +470,12 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net
                if (strcasecmp(msg->cm_fields['D'], config.c_nodename)) {
 
                        /* route the message */
-                       strcpy(nexthop, "");
-                       if (is_valid_node(nexthop, 
+                       Buf = NewStrBufPlain(msg->cm_fields['D'], -1);
+                       if (is_valid_node(&nexthop, 
                                          NULL, 
-                                         msg->cm_fields['D']
+                                         Buf
                                          working_ignetcfg, 
-                                         *the_netmap) == 0) 
+                                         the_netmap) == 0) 
                        {
                                /* prepend our node to the path */
                                if (msg->cm_fields['P'] != NULL) {
@@ -524,16 +495,16 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net
                                serialize_message(&sermsg, msg);
 
                                /* now send it */
-                               if (IsEmptyStr(nexthop)) {
-                                       strcpy(nexthop, msg->cm_fields['D']);
+                               if (StrLength(nexthop) == 0) {
+                                       nexthop = Buf;
                                }
-                               snprintf(filename, 
-                                       sizeof filename,
-                                       "%s/%s@%lx%x",
-                                       ctdl_netout_dir,
-                                       nexthop,
-                                       time(NULL),
-                                       rand()
+                               snprintf(filename,
+                                        sizeof filename,
+                                        "%s/%s@%lx%x",
+                                        ctdl_netout_dir,
+                                        ChrPtr(nexthop),
+                                        time(NULL),
+                                        rand()
                                );
                                syslog(LOG_DEBUG, "Appending to %s\n", filename);
                                fp = fopen(filename, "ab");
@@ -546,10 +517,12 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net
                                }
                                free(sermsg.ser);
                                CtdlFreeMessage(msg);
+                               FreeStrBuf(&Buf);
                                return;
                        }
                        
                        else {  /* invalid destination node name */
+                               FreeStrBuf(&Buf);
 
                                network_bounce(msg,
 "A message you sent could not be delivered due to an invalid destination node"
@@ -635,8 +608,8 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net
 void network_process_message(FILE *fp, 
                             long msgstart, 
                             long msgend,
-                            char *working_ignetcfg,
-                            NetMap **the_netmap, 
+                            HashList *working_ignetcfg,
+                            HashList *the_netmap, 
                             int *netmap_changed)
 {
        long hold_pos;
@@ -666,8 +639,8 @@ void network_process_message(FILE *fp,
  * Process a single file from the inbound queue 
  */
 void network_process_file(char *filename,
-                         char *working_ignetcfg,
-                         NetMap **the_netmap, 
+                         HashList *working_ignetcfg,
+                         HashList *the_netmap, 
                          int *netmap_changed)
 {
        FILE *fp;
@@ -724,7 +697,7 @@ void network_process_file(char *filename,
 /*
  * Process anything in the inbound queue
  */
-void network_do_spoolin(char *working_ignetcfg, NetMap **the_netmap, int *netmap_changed)
+void network_do_spoolin(HashList *working_ignetcfg, HashList *the_netmap, int *netmap_changed)
 {
        DIR *dp;
        struct dirent *d;
@@ -772,90 +745,233 @@ void network_do_spoolin(char *working_ignetcfg, NetMap **the_netmap, int *netmap
  * Step 1: consolidate files in the outbound queue into one file per neighbor node
  * Step 2: delete any files in the outbound queue that were for neighbors who no longer exist.
  */
-void network_consolidate_spoolout(char *working_ignetcfg, NetMap *the_netmap)
+void network_consolidate_spoolout(HashList *working_ignetcfg, HashList *the_netmap)
 {
+       IOBuffer IOB;
+       FDIOBuffer FDIO;
+        int d_namelen;
        DIR *dp;
        struct dirent *d;
+       struct dirent *filedir_entry;
+       const char *pch;
+       char spooloutfilename[PATH_MAX];
        char filename[PATH_MAX];
-       char cmd[PATH_MAX];
-       char nexthop[256];
-       long nexthoplen;
+       const StrBuf *nexthop;
+       StrBuf *NextHop;
        int i;
-       char *ptr;
+       int nFailed = 0;
 
        /* Step 1: consolidate files in the outbound queue into one file per neighbor node */
+       d = (struct dirent *)malloc(offsetof(struct dirent, d_name) + PATH_MAX + 1);
+       if (d == NULL)  return;
+
        dp = opendir(ctdl_netout_dir);
-       if (dp == NULL) return;
-       while (d = readdir(dp), d != NULL) {
-               if (
-                       (strcmp(d->d_name, "."))
-                       && (strcmp(d->d_name, ".."))
-                       && (strchr(d->d_name, '@') != NULL)
-               ) {
-                       nexthoplen = safestrncpy(nexthop, d->d_name, sizeof nexthop);
-                       ptr = strchr(nexthop, '@');
-                       if (ptr) {
-                               *ptr = 0;
-                               nexthoplen = ptr - nexthop;
-                       }                               
-       
-                       snprintf(filename, 
-                               sizeof filename,
-                               "%s/%s",
-                               ctdl_netout_dir,
-                               d->d_name
-                       );
-       
-                       syslog(LOG_DEBUG, "Consolidate %s to %s\n", filename, nexthop);
-                       if (network_talking_to(nexthop, nexthoplen, NTT_CHECK)) {
-                               syslog(LOG_DEBUG,
-                                       "Currently online with %s - skipping for now\n",
-                                       nexthop
+       if (dp == NULL) {
+               free(d);
+               return;
+       }
+
+       NextHop = NewStrBuf();
+       memset(&IOB, 0, sizeof(IOBuffer));
+       memset(&FDIO, 0, sizeof(FDIOBuffer));
+       FDIO.IOB = &IOB;
+
+       while ((readdir_r(dp, d, &filedir_entry) == 0) &&
+              (filedir_entry != NULL))
+       {
+#ifdef _DIRENT_HAVE_D_NAMELEN
+               d_namelen = filedir_entry->d_namelen;
+#else
+
+#ifndef DT_UNKNOWN
+#define DT_UNKNOWN     0
+#define DT_DIR         4
+#define DT_REG         8
+#define DT_LNK         10
+
+#define IFTODT(mode)   (((mode) & 0170000) >> 12)
+#define DTTOIF(dirtype)        ((dirtype) << 12)
+#endif
+               d_namelen = strlen(filedir_entry->d_name);
+#endif
+               if ((d_namelen > 1) && filedir_entry->d_name[d_namelen - 1] == '~')
+                       continue; /* Ignore backup files... */
+
+               if ((d_namelen == 1) && 
+                   (filedir_entry->d_name[0] == '.'))
+                       continue;
+
+               if ((d_namelen == 2) && 
+                   (filedir_entry->d_name[0] == '.') &&
+                   (filedir_entry->d_name[1] == '.'))
+                       continue;
+
+               pch = strchr(filedir_entry->d_name, '@');
+               if (pch == NULL)
+                       continue;
+
+               snprintf(filename, 
+                        sizeof filename,
+                        "%s/%s",
+                        ctdl_netout_dir,
+                        filedir_entry->d_name);
+
+               StrBufPlain(NextHop,
+                           filedir_entry->d_name,
+                           pch - filedir_entry->d_name);
+
+               snprintf(spooloutfilename,
+                        sizeof spooloutfilename,
+                        "%s/%s",
+                        ctdl_netout_dir,
+                        ChrPtr(NextHop));
+
+               syslog(LOG_DEBUG, "Consolidate %s to %s\n", filename, ChrPtr(NextHop));
+               if (network_talking_to(SKEY(NextHop), NTT_CHECK)) {
+                       nFailed++;
+                       syslog(LOG_DEBUG,
+                              "Currently online with %s - skipping for now\n",
+                              ChrPtr(NextHop)
+                               );
+               }
+               else {
+                       size_t dsize;
+                       size_t fsize;
+                       int fd;
+                       const char *err = NULL;
+                       network_talking_to(SKEY(NextHop), NTT_ADD);
+
+                       IOB.fd = open(filename, O_RDONLY);
+                       if (IOB.fd == -1) {
+                               nFailed++;
+                               syslog(LOG_ERR,
+                                      "failed to open %s for reading due to %s; skipping.\n",
+                                      filename, strerror(errno)
+                               );
+                               network_talking_to(SKEY(NextHop), NTT_REMOVE);
+                               continue;                               
+                       }
+                       
+                       fd = open(spooloutfilename,
+                                 O_EXCL|O_CREAT|O_NONBLOCK|O_WRONLY, 
+                                 S_IRUSR|S_IWUSR);
+                       if (fd == -1)
+                       {
+                               fd = open(spooloutfilename,
+                                         O_EXCL|O_NONBLOCK|O_WRONLY, 
+                                         S_IRUSR | S_IWUSR);
+                       }
+                       if (fd == -1) {
+                               nFailed++;
+                               syslog(LOG_ERR,
+                                      "failed to open %s for reading due to %s; skipping.\n",
+                                      spooloutfilename, strerror(errno)
                                );
+                               close(IOB.fd);
+                               network_talking_to(SKEY(NextHop), NTT_REMOVE);
+                               continue;
+                       }
+                       dsize = lseek(fd, 0, SEEK_END);
+                       fsize = lseek(IOB.fd, 0, SEEK_END);
+                       
+                       FDIOBufferInit(&FDIO, &IOB, fd, fsize + dsize);
+                       FDIO.ChunkSendRemain = fsize;
+                       FDIO.TotalSentAlready = dsize;
+
+                       do {} while (FileRecvChunked(&FDIO, &err) > 0);
+                       if (err == NULL) {
+                               unlink(filename);
                        }
                        else {
-                               network_talking_to(nexthop, nexthoplen, NTT_ADD);
-                               snprintf(cmd, sizeof cmd, "/bin/cat %s >>%s/%s && /bin/rm -f %s",
-                                       filename,
-                                       ctdl_netout_dir, nexthop,
-                                       filename
+                               nFailed++;
+                               syslog(LOG_ERR,
+                                      "failed to append to %s [%s]; rolling back..\n",
+                                      spooloutfilename, strerror(errno)
                                );
-                               system(cmd);
-                               network_talking_to(nexthop, nexthoplen, NTT_REMOVE);
+                               /* whoops partial append?? truncate spooloutfilename again! */
+                               ftruncate(fd, dsize);
                        }
+                       FDIOBufferDelete(&FDIO);
+                       close(IOB.fd);
+                       close(fd);
+                       network_talking_to(SKEY(NextHop), NTT_REMOVE);
                }
        }
        closedir(dp);
 
-       /* Step 2: delete any files in the outbound queue that were for neighbors who no longer exist */
+       if (nFailed > 0) {
+               FreeStrBuf(&NextHop);
+               syslog(LOG_INFO,
+                      "skipping Spoolcleanup because of %d files unprocessed.\n",
+                      nFailed
+                       );
+
+               return;
+       }
 
+       /* Step 2: delete any files in the outbound queue that were for neighbors who no longer exist */
        dp = opendir(ctdl_netout_dir);
-       if (dp == NULL) return;
+       if (dp == NULL) {
+               FreeStrBuf(&NextHop);
+               free(d);
+               return;
+       }
 
-       while (d = readdir(dp), d != NULL) {
-               if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, ".."))
+       while ((readdir_r(dp, d, &filedir_entry) == 0) &&
+              (filedir_entry != NULL))
+       {
+#ifdef _DIRENT_HAVE_D_NAMELEN
+               d_namelen = filedir_entry->d_namelen;
+               d_type = filedir_entry->d_type;
+#else
+
+#ifndef DT_UNKNOWN
+#define DT_UNKNOWN     0
+#define DT_DIR         4
+#define DT_REG         8
+#define DT_LNK         10
+
+#define IFTODT(mode)   (((mode) & 0170000) >> 12)
+#define DTTOIF(dirtype)        ((dirtype) << 12)
+#endif
+               d_namelen = strlen(filedir_entry->d_name);
+#endif
+               if ((d_namelen == 1) && 
+                   (filedir_entry->d_name[0] == '.'))
                        continue;
 
+               if ((d_namelen == 2) && 
+                   (filedir_entry->d_name[0] == '.') &&
+                   (filedir_entry->d_name[1] == '.'))
+                       continue;
+
+               pch = strchr(filedir_entry->d_name, '@');
+               if (pch == NULL) /* no @ in name? consolidated file. */
+                       continue;
+
+               StrBufPlain(NextHop,
+                           filedir_entry->d_name,
+                           pch - filedir_entry->d_name);
+
                snprintf(filename, 
                        sizeof filename,
                        "%s/%s",
                        ctdl_netout_dir,
-                       d->d_name
+                       filedir_entry->d_name
                );
 
-               strcpy(nexthop, "");
-               i = is_valid_node(nexthop,
+               i = is_valid_node(&nexthop,
                                  NULL,
-                                 d->d_name,
+                                 NextHop,
                                  working_ignetcfg,
                                  the_netmap);
        
-               if ( (i != 0) || !IsEmptyStr(nexthop) ) {
+               if ( (i != 0) || (StrLength(nexthop) > 0) ) {
                        unlink(filename);
                }
        }
-
-
+       FreeStrBuf(&NextHop);
+       free(d);
        closedir(dp);
 }
 
index 7c8c47047991de20dffb8cafa34f92e56b636bf7..a0e821557d8a90afd843c55450b77d5ffd36cf3a 100644 (file)
@@ -2,7 +2,7 @@
  * This module handles shared rooms, inter-Citadel mail, and outbound
  * mailing list processing.
  *
- * Copyright (c) 2000-2011 by the citadel.org team
+ * Copyright (c) 2000-2012 by the citadel.org team
  *
  *  This program is open source software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -495,8 +495,8 @@ void network_do_queue(void) {
        static int doing_queue = 0;
        static time_t last_run = 0L;
        int full_processing = 1;
-       char *working_ignetcfg;
-       NetMap *the_netmap = NULL;
+       HashList *working_ignetcfg;
+       HashList *the_netmap = NULL;
        int netmap_changed = 0;
        roomlists RL;
 
@@ -537,7 +537,7 @@ void network_do_queue(void) {
                        return;
        }
        /* Load the IGnet Configuration into memory */
-       working_ignetcfg = load_working_ignetcfg();
+       working_ignetcfg = load_ignetcfg();
 
        /*
         * Load the network map and filter list into memory.
@@ -586,19 +586,27 @@ void network_do_queue(void) {
        /* If there is anything in the inbound queue, process it */
        if (!server_shutting_down) {
                network_do_spoolin(working_ignetcfg, 
-                                  &the_netmap,
+                                  the_netmap,
                                   &netmap_changed);
        }
 
        /* Free the filter list in memory */
        free_netfilter_list();
 
+       /* Save the network map back to disk */
+       if (netmap_changed) {
+               StrBuf *MapStr = SerializeNetworkMap(the_netmap);
+               CtdlPutSysConfig(IGNETMAP, SmashStrBuf(&MapStr));
+       }
+
+       /* combine singe message files into one spool entry per remote node. */
        network_consolidate_spoolout(working_ignetcfg, the_netmap);
 
-       /* Save the network map back to disk */
-       write_and_free_network_map(&the_netmap, netmap_changed);
+       /* shut down. */
+
+       DeleteHash(&the_netmap);
 
-       free(working_ignetcfg);
+       DeleteHash(&working_ignetcfg);
 
        syslog(LOG_DEBUG, "network: queue run completed\n");