From b35ed2c5958f57f2113dec3adc66274419af3066 Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Sun, 10 Jun 2012 12:38:30 +0200 Subject: [PATCH] Rework networker - keep netconfigs in hash; create the usual deserialize/delete/... functions - keep the netmap in hash; create the usual de/serialize/delete/... functions - change sequence: serialize & save netmap; combine spoolfiles; free netmap. - use strbufs splice() wrappers to combine the message-spoolfiles into host-spoolfiles instead of forking a shell & running cat + rm - when appending to host-spoolfile fails, remove partial adds to avoid creating invalid spool control files - only cleanup spool directory if we successfully combined all files; else we might loose messages if i.e. a networker is running for that node. --- citadel/modules/network/netconfig.h | 61 +++- citadel/modules/network/netspool.h | 41 ++- citadel/modules/network/serv_netconfig.c | 338 +++++++++++++++-------- citadel/modules/network/serv_netmail.c | 16 +- citadel/modules/network/serv_netspool.c | 320 ++++++++++++++------- citadel/modules/network/serv_network.c | 24 +- 6 files changed, 553 insertions(+), 247 deletions(-) diff --git a/citadel/modules/network/netconfig.h b/citadel/modules/network/netconfig.h index 9d71d9fb4..6168f0402 100644 --- a/citadel/modules/network/netconfig.h +++ b/citadel/modules/network/netconfig.h @@ -1,13 +1,54 @@ -typedef struct NetMap NetMap; +/* + * This module handles shared rooms, inter-Citadel mail, and outbound + * mailing list processing. + * + * Copyright (c) 2000-2012 by the citadel.org team + * + * This program is open source software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * ** NOTE ** A word on the S_NETCONFIGS semaphore: + * This is a fairly high-level type of critical section. It ensures that no + * two threads work on the netconfigs files at the same time. Since we do + * so many things inside these, here are the rules: + * 1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others. + * 2. Do *not* perform any I/O with the client during these sections. + * + */ -struct NetMap { - NetMap *next; - char nodename[SIZ]; +typedef struct _nodeconf { + int DeleteMe; + StrBuf *NodeName; + StrBuf *Secret; + StrBuf *Host; + StrBuf *Port; +}NodeConf; + +typedef struct __NetMap { + StrBuf *NodeName; time_t lastcontact; - char nexthop[SIZ]; -}; + StrBuf *NextHop; +}NetMap; + +HashList* load_ignetcfg(void); + +HashList* read_network_map(void); +StrBuf *SerializeNetworkMap(HashList *Map); +void network_learn_topology(char *node, char *path, HashList *the_netmap, int *netmap_changed); -char* load_working_ignetcfg(void); -NetMap *read_network_map(void); -void write_and_free_network_map(NetMap **the_netmap, int netmap_changed); -int is_valid_node(char *nexthop, char *secret, char *node, char *working_ignetcfg, NetMap *the_netmap); +int is_valid_node(const StrBuf **nexthop, + const StrBuf **secret, + StrBuf *node, + HashList *IgnetCfg, + HashList *the_netmap); diff --git a/citadel/modules/network/netspool.h b/citadel/modules/network/netspool.h index 0c0364ce3..e52829bce 100644 --- a/citadel/modules/network/netspool.h +++ b/citadel/modules/network/netspool.h @@ -1,3 +1,32 @@ +/* + * This module handles shared rooms, inter-Citadel mail, and outbound + * mailing list processing. + * + * Copyright (c) 2000-2012 by the citadel.org team + * + * This program is open source software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * ** NOTE ** A word on the S_NETCONFIGS semaphore: + * This is a fairly high-level type of critical section. It ensures that no + * two threads work on the netconfigs files at the same time. Since we do + * so many things inside these, here are the rules: + * 1. begin_critical_section(S_NETCONFIGS) *before* begin_ any others. + * 2. Do *not* perform any I/O with the client during these sections. + * + */ + typedef struct maplist maplist; struct maplist { @@ -19,16 +48,16 @@ struct SpoolControl { FILE *digestfp; int num_msgs_spooled; - char *working_ignetcfg; - NetMap *the_netmap; + HashList *working_ignetcfg; + HashList *the_netmap; }; void network_spoolout_room(RoomProcList *room_to_spool, - char *working_ignetcfg, - NetMap *the_netmap); -void network_do_spoolin(char *working_ignetcfg, NetMap **the_netmap, int *netmap_changed); -void network_consolidate_spoolout(char *working_ignetcfg, NetMap *the_netmap); + HashList *working_ignetcfg, + HashList *the_netmap); +void network_do_spoolin(HashList *working_ignetcfg, HashList *the_netmap, int *netmap_changed); +void network_consolidate_spoolout(HashList *working_ignetcfg, HashList *the_netmap); void free_spoolcontrol_struct(SpoolControl **scc); int writenfree_spoolcontrol_file(SpoolControl **scc, char *filename); int read_spoolcontrol_file(SpoolControl **scc, char *filename); diff --git a/citadel/modules/network/serv_netconfig.c b/citadel/modules/network/serv_netconfig.c index 9d1e6bc74..cfcc20b6f 100644 --- a/citadel/modules/network/serv_netconfig.c +++ b/citadel/modules/network/serv_netconfig.c @@ -2,7 +2,7 @@ * This module handles shared rooms, inter-Citadel mail, and outbound * mailing list processing. * - * Copyright (c) 2000-2011 by the citadel.org team + * Copyright (c) 2000-2012 by the citadel.org team * * This program is open source software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -92,167 +92,269 @@ #include "ctdl_module.h" + +void DeleteNodeConf(void *vNode) +{ + NodeConf *Node = (NodeConf*) vNode; + FreeStrBuf(&Node->NodeName); + FreeStrBuf(&Node->Secret); + FreeStrBuf(&Node->Host); + FreeStrBuf(&Node->Port); + free(Node); +} + +NodeConf *NewNode(StrBuf *SerializedNode) +{ + const char *Pos = NULL; + NodeConf *Node; + + /* we need at least 4 pipes and some other text so its invalid. */ + if (StrLength(SerializedNode) < 8) + return NULL; + Node = (NodeConf *) malloc(sizeof(NodeConf)); + + Node->DeleteMe = 0; + + Node->NodeName=NewStrBuf(); + StrBufExtract_NextToken(Node->NodeName, SerializedNode, &Pos, '|'); + + Node->Secret=NewStrBuf(); + StrBufExtract_NextToken(Node->Secret, SerializedNode, &Pos, '|'); + + Node->Host=NewStrBuf(); + StrBufExtract_NextToken(Node->Host, SerializedNode, &Pos, '|'); + + Node->Port=NewStrBuf(); + StrBufExtract_NextToken(Node->Port, SerializedNode, &Pos, '|'); + return Node; +} + + /* * Load or refresh the Citadel network (IGnet) configuration for this node. */ -char* load_working_ignetcfg(void) { - return CtdlGetSysConfig(IGNETCFG); +HashList* load_ignetcfg(void) +{ + const char *LinePos; + char *Cfg; + StrBuf *Buf; + StrBuf *LineBuf; + HashList *Hash; + NodeConf *Node; + + Cfg = CtdlGetSysConfig(IGNETCFG); + if ((Cfg == NULL) || IsEmptyStr(Cfg)) { + if (Cfg != NULL) + free(Cfg); + return NULL; + } + + Hash = NewHash(1, NULL); + Buf = NewStrBufPlain(Cfg, -1); + free(Cfg); + LineBuf = NewStrBufPlain(NULL, StrLength(Buf)); + LinePos = NULL; + do + { + StrBufSipLine(LineBuf, Buf, &LinePos); + if (StrLength(LineBuf) != 0) { + Node = NewNode(LineBuf); + if (Node != NULL) { + Put(Hash, SKEY(Node->NodeName), Node, DeleteNodeConf); + } + } + } while (LinePos != StrBufNOTNULL); + FreeStrBuf(&Buf); + FreeStrBuf(&LineBuf); + return Hash; } +void DeleteNetMap(void *vNetMap) +{ + NetMap *TheNetMap = (NetMap*) vNetMap; + FreeStrBuf(&TheNetMap->NodeName); + FreeStrBuf(&TheNetMap->NextHop); + free(TheNetMap); +} -/* - * Read the network map from its configuration file into memory. - */ -NetMap *read_network_map(void) { - char *serialized_map = NULL; - int i; - char buf[SIZ]; - NetMap *nmptr, *the_netmap; +NetMap *NewNetMap(StrBuf *SerializedNetMap) +{ + const char *Pos = NULL; + NetMap *NM; - the_netmap = NULL; - serialized_map = CtdlGetSysConfig(IGNETMAP); - if (serialized_map == NULL) return NULL; /* if null, no entries */ + /* we need at least 3 pipes and some other text so its invalid. */ + if (StrLength(SerializedNetMap) < 6) + return NULL; + NM = (NetMap *) malloc(sizeof(NetMap)); - /* Use the string tokenizer to grab one line at a time */ - for (i=0; iNodeName=NewStrBuf(); + StrBufExtract_NextToken(NM->NodeName, SerializedNetMap, &Pos, '|'); - nmptr = (NetMap *) malloc(sizeof(NetMap)); + NM->lastcontact = StrBufExtractNext_long(SerializedNetMap, &Pos, '|'); - extract_token(nmptr->nodename, buf, 0, '|', sizeof nmptr->nodename); - nmptr->lastcontact = extract_long(buf, 1); - extract_token(nmptr->nexthop, buf, 2, '|', sizeof nmptr->nexthop); + NM->NextHop=NewStrBuf(); + StrBufExtract_NextToken(NM->NextHop, SerializedNetMap, &Pos, '|'); - nmptr->next = the_netmap; - the_netmap = nmptr; + return NM; +} + +HashList* read_network_map(void) +{ + const char *LinePos; + char *Cfg; + StrBuf *Buf; + StrBuf *LineBuf; + HashList *Hash; + NetMap *TheNetMap; + + Cfg = CtdlGetSysConfig(IGNETMAP); + if ((Cfg == NULL) || IsEmptyStr(Cfg)) { + if (Cfg != NULL) + free(Cfg); + return NULL; } - free(serialized_map); - return the_netmap; + Hash = NewHash(1, NULL); + Buf = NewStrBufPlain(Cfg, -1); + free(Cfg); + LineBuf = NewStrBufPlain(NULL, StrLength(Buf)); + LinePos = NULL; + while (StrBufSipLine(Buf, LineBuf, &LinePos)) + { + TheNetMap = NewNetMap(LineBuf); + if (TheNetMap != NULL) { /* TODO: is the NodeName Uniq? */ + Put(Hash, SKEY(TheNetMap->NodeName), TheNetMap, DeleteNetMap); + } + } + FreeStrBuf(&Buf); + FreeStrBuf(&LineBuf); + return Hash; +} + +StrBuf *SerializeNetworkMap(HashList *Map) +{ + void *vMap; + const char *key; + long len; + StrBuf *Ret = NewStrBuf(); + HashPos *Pos = GetNewHashPos(Map, 0); + + while (GetNextHashPos(Map, Pos, &len, &key, &vMap)) + { + NetMap *pMap = (NetMap*) vMap; + StrBufAppendBuf(Ret, pMap->NodeName, 0); + StrBufAppendBufPlain(Ret, HKEY("|"), 0); + + StrBufAppendPrintf(Ret, "%ld", pMap->lastcontact, 0); + StrBufAppendBufPlain(Ret, HKEY("|"), 0); + + StrBufAppendBuf(Ret, pMap->NextHop, 0); + StrBufAppendBufPlain(Ret, HKEY("\n"), 0); + } + DeleteHashPos(&Pos); + return Ret; } /* - * Write the network map from memory back to the configuration file. + * Learn topology from path fields */ -void write_and_free_network_map(NetMap **the_netmap, int netmap_changed) +void network_learn_topology(char *node, char *path, HashList *the_netmap, int *netmap_changed) { - char *serialized_map = NULL; + NetMap *pNM = NULL; + void *vptr; + char nexthop[256]; NetMap *nmptr; - if (netmap_changed) { - serialized_map = strdup(""); - - if (*the_netmap != NULL) { - for (nmptr = *the_netmap; nmptr != NULL; nmptr = nmptr->next) { - serialized_map = realloc(serialized_map, - (strlen(serialized_map)+SIZ) ); - if (!IsEmptyStr(nmptr->nodename)) { - snprintf(&serialized_map[strlen(serialized_map)], - SIZ, - "%s|%ld|%s\n", - nmptr->nodename, - (long)nmptr->lastcontact, - nmptr->nexthop); - } - } + if (GetHash(the_netmap, node, strlen(node), &vptr) && + (vptr != NULL))/* TODO: is the NodeName Uniq? */ + { + pNM = (NetMap*)vptr; + extract_token(nexthop, path, 0, '!', sizeof nexthop); + if (!strcmp(nexthop, ChrPtr(pNM->NextHop))) { + pNM->lastcontact = time(NULL); + (*netmap_changed) ++; + return; } - - CtdlPutSysConfig(IGNETMAP, serialized_map); - free(serialized_map); } - /* Now free the list */ - while (*the_netmap != NULL) { - nmptr = (*the_netmap)->next; - free(*the_netmap); - *the_netmap = nmptr; - } + /* If we got here then it's not in the map, so add it. */ + nmptr = (NetMap *) malloc(sizeof (NetMap)); + nmptr->NodeName = NewStrBufPlain(node, -1); + nmptr->lastcontact = time(NULL); + nmptr->NextHop = NewStrBuf (); + StrBufExtract_tokenFromStr(nmptr->NextHop, path, strlen(path), 0, '!'); + /* TODO: is the NodeName Uniq? */ + Put(the_netmap, SKEY(nmptr->NodeName), nmptr, DeleteNetMap); + (*netmap_changed) ++; } - -/* +/* * Check the network map and determine whether the supplied node name is * valid. If it is not a neighbor node, supply the name of a neighbor node * which is the next hop. If it *is* a neighbor node, we also fill in the * shared secret. */ -int is_valid_node(char *nexthop, - char *secret, - char *node, - char *working_ignetcfg, - NetMap *the_netmap) +int is_valid_node(const StrBuf **nexthop, + const StrBuf **secret, + StrBuf *node, + HashList *IgnetCfg, + HashList *the_netmap) { - int i; - char linebuf[SIZ]; - char buf[SIZ]; - int retval; - NetMap *nmptr; + void *vNetMap; + void *vNodeConf; + NodeConf *TheNode; + NetMap *TheNetMap; - if (node == NULL) { + if (StrLength(node) == 0) { return(-1); } /* * First try the neighbor nodes */ - if ((working_ignetcfg == NULL) || (*working_ignetcfg == '\0')) { - syslog(LOG_ERR, "working_ignetcfg is empty!\n"); + if (GetCount(IgnetCfg) == 0) { + syslog(LOG_INFO, "IgnetCfg is empty!\n"); if (nexthop != NULL) { - strcpy(nexthop, ""); + *nexthop = NULL; } return(-1); } - retval = (-1); - if (nexthop != NULL) { - strcpy(nexthop, ""); - } - - /* Use the string tokenizer to grab one line at a time */ - for (i=0; iSecret; + return 0; /* yup, it's a direct neighbor */ } - /* + /* * If we get to this point we have to see if we know the next hop - */ - if (the_netmap != NULL) { - for (nmptr = the_netmap; nmptr != NULL; nmptr = nmptr->next) { - if (!strcasecmp(nmptr->nodename, node)) { - if (nexthop != NULL) { - strcpy(nexthop, nmptr->nexthop); - } - return(0); - } - } + *//* TODO: is the NodeName Uniq? */ + if ((GetCount(the_netmap) > 0) && + (GetHash(the_netmap, SKEY(node), &vNetMap))) + { + TheNetMap = (NetMap*)vNetMap; + if (nexthop != NULL) + *nexthop = TheNetMap->NextHop; + return(0); } /* * If we get to this point, the supplied node name is bogus. */ - syslog(LOG_ERR, "Invalid node name <%s>\n", node); + syslog(LOG_ERR, "Invalid node name <%s>\n", ChrPtr(node)); return(-1); } -void cmd_gnet(char *argbuf) { +void cmd_gnet(char *argbuf) +{ char filename[PATH_MAX]; char buf[SIZ]; FILE *fp; @@ -365,24 +467,24 @@ void cmd_snet(char *argbuf) { */ void cmd_netp(char *cmdbuf) { - char *working_ignetcfg; + HashList *working_ignetcfg; char node[256]; + StrBuf *NodeStr; long nodelen; char pass[256]; int v; - char secret[256] = ""; - char nexthop[256] = ""; + const StrBuf *secret = NULL; + const StrBuf *nexthop = NULL; char err_buf[SIZ] = ""; /* Authenticate */ nodelen = extract_token(node, cmdbuf, 0, '|', sizeof node); extract_token(pass, cmdbuf, 1, '|', sizeof pass); - + NodeStr = NewStrBufPlain(node, nodelen); /* load the IGnet Configuration to check node validity */ - working_ignetcfg = load_working_ignetcfg(); - v = is_valid_node(nexthop, secret, node, working_ignetcfg, NULL); //// TODO do we need the netmap? - + working_ignetcfg = load_ignetcfg(); + v = is_valid_node(&nexthop, &secret, NodeStr, working_ignetcfg, NULL); if (v != 0) { snprintf(err_buf, sizeof err_buf, "An unknown Citadel server called \"%s\" attempted to connect from %s [%s].\n", @@ -391,11 +493,12 @@ void cmd_netp(char *cmdbuf) syslog(LOG_WARNING, "%s", err_buf); cprintf("%d authentication failed\n", ERROR + PASSWORD_REQUIRED); CtdlAideMessage(err_buf, "IGNet Networking."); - free(working_ignetcfg); + DeleteHash(&working_ignetcfg); + FreeStrBuf(&NodeStr); return; } - if (strcasecmp(pass, secret)) { + if (strcasecmp(pass, ChrPtr(secret))) { snprintf(err_buf, sizeof err_buf, "A Citadel server at %s [%s] failed to authenticate as network node \"%s\".\n", CC->cs_host, CC->cs_addr, node @@ -403,14 +506,16 @@ void cmd_netp(char *cmdbuf) syslog(LOG_WARNING, "%s", err_buf); cprintf("%d authentication failed\n", ERROR + PASSWORD_REQUIRED); CtdlAideMessage(err_buf, "IGNet Networking."); - free(working_ignetcfg); + DeleteHash(&working_ignetcfg); + FreeStrBuf(&NodeStr); return; } if (network_talking_to(node, nodelen, NTT_CHECK)) { syslog(LOG_WARNING, "Duplicate session for network node <%s>", node); cprintf("%d Already talking to %s right now\n", ERROR + RESOURCE_BUSY, node); - free(working_ignetcfg); + DeleteHash(&working_ignetcfg); + FreeStrBuf(&NodeStr); return; } @@ -420,7 +525,8 @@ void cmd_netp(char *cmdbuf) CC->net_node, CC->cs_host, CC->cs_addr ); cprintf("%d authenticated as network node '%s'\n", CIT_OK, CC->net_node); - free(working_ignetcfg); + DeleteHash(&working_ignetcfg); + FreeStrBuf(&NodeStr); } int netconfig_check_roomaccess( diff --git a/citadel/modules/network/serv_netmail.c b/citadel/modules/network/serv_netmail.c index 7dcda8db6..5b2a02524 100644 --- a/citadel/modules/network/serv_netmail.c +++ b/citadel/modules/network/serv_netmail.c @@ -2,7 +2,7 @@ * This module handles shared rooms, inter-Citadel mail, and outbound * mailing list processing. * - * Copyright (c) 2000-2011 by the citadel.org team + * Copyright (c) 2000-2012 by the citadel.org team * * This program is open source software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -265,6 +265,7 @@ void network_deliver_list(struct CtdlMessage *msg, SpoolControl *sc, const char void network_spool_msg(long msgnum, void *userdata) { + StrBuf *Buf = NULL; SpoolControl *sc; int i; char *newpath = NULL; @@ -470,9 +471,11 @@ void network_spool_msg(long msgnum, config.c_nodename)) { ok_to_participate = 1; } + + Buf = NewStrBufPlain(msg->cm_fields['N'], -1); if (is_valid_node(NULL, NULL, - msg->cm_fields['N'], + Buf, sc->working_ignetcfg, sc->the_netmap) == 0) { @@ -560,11 +563,14 @@ void network_spool_msg(long msgnum, mptr = mptr->next) { send = 1; - + if (Buf == NULL) + Buf = NewStrBufPlain(mptr->remote_nodename, -1); + else + StrBufPlain(Buf, mptr->remote_nodename, -1); /* Check for valid node name */ if (is_valid_node(NULL, NULL, - mptr->remote_nodename, + Buf, sc->working_ignetcfg, sc->the_netmap) != 0) { @@ -665,5 +671,5 @@ void network_spool_msg(long msgnum, if (delete_after_send) { CtdlDeleteMessages(CC->room.QRname, &msgnum, 1, ""); } - + FreeStrBuf(&Buf); } diff --git a/citadel/modules/network/serv_netspool.c b/citadel/modules/network/serv_netspool.c index de98a3cf1..479b303de 100644 --- a/citadel/modules/network/serv_netspool.c +++ b/citadel/modules/network/serv_netspool.c @@ -2,7 +2,7 @@ * This module handles shared rooms, inter-Citadel mail, and outbound * mailing list processing. * - * Copyright (c) 2000-2011 by the citadel.org team + * Copyright (c) 2000-2012 by the citadel.org team * * This program is open source software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -92,36 +92,6 @@ #include "netmail.h" #include "ctdl_module.h" -/* - * Learn topology from path fields - */ -static void network_learn_topology(char *node, char *path, NetMap **the_netmap, int *netmap_changed) -{ - char nexthop[256]; - NetMap *nmptr; - - *nexthop = '\0'; - - if (num_tokens(path, '!') < 3) return; - for (nmptr = *the_netmap; nmptr != NULL; nmptr = nmptr->next) { - if (!strcasecmp(nmptr->nodename, node)) { - extract_token(nmptr->nexthop, path, 0, '!', sizeof nmptr->nexthop); - nmptr->lastcontact = time(NULL); - (*netmap_changed) ++; - return; - } - } - - /* If we got here then it's not in the map, so add it. */ - nmptr = (NetMap *) malloc(sizeof (NetMap)); - strcpy(nmptr->nodename, node); - nmptr->lastcontact = time(NULL); - extract_token(nmptr->nexthop, path, 0, '!', sizeof nmptr->nexthop); - nmptr->next = *the_netmap; - *the_netmap = nmptr; - (*netmap_changed) ++; -} - @@ -382,8 +352,8 @@ int is_recipient(SpoolControl *sc, const char *Name) * Batch up and send all outbound traffic from the current room */ void network_spoolout_room(RoomProcList *room_to_spool, - char *working_ignetcfg, - NetMap *the_netmap) + HashList *working_ignetcfg, + HashList *the_netmap) { char buf[SIZ]; char filename[PATH_MAX]; @@ -452,8 +422,9 @@ void network_spoolout_room(RoomProcList *room_to_spool, * Process a buffer containing a single message from a single file * from the inbound queue */ -void network_process_buffer(char *buffer, long size, char *working_ignetcfg, NetMap **the_netmap, int *netmap_changed) +void network_process_buffer(char *buffer, long size, HashList *working_ignetcfg, HashList *the_netmap, int *netmap_changed) { + StrBuf *Buf = NULL; struct CtdlMessage *msg = NULL; long pos; int field; @@ -463,7 +434,7 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net char *oldpath = NULL; char filename[PATH_MAX]; FILE *fp; - char nexthop[SIZ]; + const StrBuf *nexthop = NULL; unsigned char firstbyte; unsigned char lastbyte; @@ -499,12 +470,12 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net if (strcasecmp(msg->cm_fields['D'], config.c_nodename)) { /* route the message */ - strcpy(nexthop, ""); - if (is_valid_node(nexthop, + Buf = NewStrBufPlain(msg->cm_fields['D'], -1); + if (is_valid_node(&nexthop, NULL, - msg->cm_fields['D'], + Buf, working_ignetcfg, - *the_netmap) == 0) + the_netmap) == 0) { /* prepend our node to the path */ if (msg->cm_fields['P'] != NULL) { @@ -524,16 +495,16 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net serialize_message(&sermsg, msg); /* now send it */ - if (IsEmptyStr(nexthop)) { - strcpy(nexthop, msg->cm_fields['D']); + if (StrLength(nexthop) == 0) { + nexthop = Buf; } - snprintf(filename, - sizeof filename, - "%s/%s@%lx%x", - ctdl_netout_dir, - nexthop, - time(NULL), - rand() + snprintf(filename, + sizeof filename, + "%s/%s@%lx%x", + ctdl_netout_dir, + ChrPtr(nexthop), + time(NULL), + rand() ); syslog(LOG_DEBUG, "Appending to %s\n", filename); fp = fopen(filename, "ab"); @@ -546,10 +517,12 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net } free(sermsg.ser); CtdlFreeMessage(msg); + FreeStrBuf(&Buf); return; } else { /* invalid destination node name */ + FreeStrBuf(&Buf); network_bounce(msg, "A message you sent could not be delivered due to an invalid destination node" @@ -635,8 +608,8 @@ void network_process_buffer(char *buffer, long size, char *working_ignetcfg, Net void network_process_message(FILE *fp, long msgstart, long msgend, - char *working_ignetcfg, - NetMap **the_netmap, + HashList *working_ignetcfg, + HashList *the_netmap, int *netmap_changed) { long hold_pos; @@ -666,8 +639,8 @@ void network_process_message(FILE *fp, * Process a single file from the inbound queue */ void network_process_file(char *filename, - char *working_ignetcfg, - NetMap **the_netmap, + HashList *working_ignetcfg, + HashList *the_netmap, int *netmap_changed) { FILE *fp; @@ -724,7 +697,7 @@ void network_process_file(char *filename, /* * Process anything in the inbound queue */ -void network_do_spoolin(char *working_ignetcfg, NetMap **the_netmap, int *netmap_changed) +void network_do_spoolin(HashList *working_ignetcfg, HashList *the_netmap, int *netmap_changed) { DIR *dp; struct dirent *d; @@ -772,90 +745,233 @@ void network_do_spoolin(char *working_ignetcfg, NetMap **the_netmap, int *netmap * Step 1: consolidate files in the outbound queue into one file per neighbor node * Step 2: delete any files in the outbound queue that were for neighbors who no longer exist. */ -void network_consolidate_spoolout(char *working_ignetcfg, NetMap *the_netmap) +void network_consolidate_spoolout(HashList *working_ignetcfg, HashList *the_netmap) { + IOBuffer IOB; + FDIOBuffer FDIO; + int d_namelen; DIR *dp; struct dirent *d; + struct dirent *filedir_entry; + const char *pch; + char spooloutfilename[PATH_MAX]; char filename[PATH_MAX]; - char cmd[PATH_MAX]; - char nexthop[256]; - long nexthoplen; + const StrBuf *nexthop; + StrBuf *NextHop; int i; - char *ptr; + int nFailed = 0; /* Step 1: consolidate files in the outbound queue into one file per neighbor node */ + d = (struct dirent *)malloc(offsetof(struct dirent, d_name) + PATH_MAX + 1); + if (d == NULL) return; + dp = opendir(ctdl_netout_dir); - if (dp == NULL) return; - while (d = readdir(dp), d != NULL) { - if ( - (strcmp(d->d_name, ".")) - && (strcmp(d->d_name, "..")) - && (strchr(d->d_name, '@') != NULL) - ) { - nexthoplen = safestrncpy(nexthop, d->d_name, sizeof nexthop); - ptr = strchr(nexthop, '@'); - if (ptr) { - *ptr = 0; - nexthoplen = ptr - nexthop; - } - - snprintf(filename, - sizeof filename, - "%s/%s", - ctdl_netout_dir, - d->d_name - ); - - syslog(LOG_DEBUG, "Consolidate %s to %s\n", filename, nexthop); - if (network_talking_to(nexthop, nexthoplen, NTT_CHECK)) { - syslog(LOG_DEBUG, - "Currently online with %s - skipping for now\n", - nexthop + if (dp == NULL) { + free(d); + return; + } + + NextHop = NewStrBuf(); + memset(&IOB, 0, sizeof(IOBuffer)); + memset(&FDIO, 0, sizeof(FDIOBuffer)); + FDIO.IOB = &IOB; + + while ((readdir_r(dp, d, &filedir_entry) == 0) && + (filedir_entry != NULL)) + { +#ifdef _DIRENT_HAVE_D_NAMELEN + d_namelen = filedir_entry->d_namelen; +#else + +#ifndef DT_UNKNOWN +#define DT_UNKNOWN 0 +#define DT_DIR 4 +#define DT_REG 8 +#define DT_LNK 10 + +#define IFTODT(mode) (((mode) & 0170000) >> 12) +#define DTTOIF(dirtype) ((dirtype) << 12) +#endif + d_namelen = strlen(filedir_entry->d_name); +#endif + if ((d_namelen > 1) && filedir_entry->d_name[d_namelen - 1] == '~') + continue; /* Ignore backup files... */ + + if ((d_namelen == 1) && + (filedir_entry->d_name[0] == '.')) + continue; + + if ((d_namelen == 2) && + (filedir_entry->d_name[0] == '.') && + (filedir_entry->d_name[1] == '.')) + continue; + + pch = strchr(filedir_entry->d_name, '@'); + if (pch == NULL) + continue; + + snprintf(filename, + sizeof filename, + "%s/%s", + ctdl_netout_dir, + filedir_entry->d_name); + + StrBufPlain(NextHop, + filedir_entry->d_name, + pch - filedir_entry->d_name); + + snprintf(spooloutfilename, + sizeof spooloutfilename, + "%s/%s", + ctdl_netout_dir, + ChrPtr(NextHop)); + + syslog(LOG_DEBUG, "Consolidate %s to %s\n", filename, ChrPtr(NextHop)); + if (network_talking_to(SKEY(NextHop), NTT_CHECK)) { + nFailed++; + syslog(LOG_DEBUG, + "Currently online with %s - skipping for now\n", + ChrPtr(NextHop) + ); + } + else { + size_t dsize; + size_t fsize; + int fd; + const char *err = NULL; + network_talking_to(SKEY(NextHop), NTT_ADD); + + IOB.fd = open(filename, O_RDONLY); + if (IOB.fd == -1) { + nFailed++; + syslog(LOG_ERR, + "failed to open %s for reading due to %s; skipping.\n", + filename, strerror(errno) + ); + network_talking_to(SKEY(NextHop), NTT_REMOVE); + continue; + } + + fd = open(spooloutfilename, + O_EXCL|O_CREAT|O_NONBLOCK|O_WRONLY, + S_IRUSR|S_IWUSR); + if (fd == -1) + { + fd = open(spooloutfilename, + O_EXCL|O_NONBLOCK|O_WRONLY, + S_IRUSR | S_IWUSR); + } + if (fd == -1) { + nFailed++; + syslog(LOG_ERR, + "failed to open %s for reading due to %s; skipping.\n", + spooloutfilename, strerror(errno) ); + close(IOB.fd); + network_talking_to(SKEY(NextHop), NTT_REMOVE); + continue; + } + dsize = lseek(fd, 0, SEEK_END); + fsize = lseek(IOB.fd, 0, SEEK_END); + + FDIOBufferInit(&FDIO, &IOB, fd, fsize + dsize); + FDIO.ChunkSendRemain = fsize; + FDIO.TotalSentAlready = dsize; + + do {} while (FileRecvChunked(&FDIO, &err) > 0); + if (err == NULL) { + unlink(filename); } else { - network_talking_to(nexthop, nexthoplen, NTT_ADD); - snprintf(cmd, sizeof cmd, "/bin/cat %s >>%s/%s && /bin/rm -f %s", - filename, - ctdl_netout_dir, nexthop, - filename + nFailed++; + syslog(LOG_ERR, + "failed to append to %s [%s]; rolling back..\n", + spooloutfilename, strerror(errno) ); - system(cmd); - network_talking_to(nexthop, nexthoplen, NTT_REMOVE); + /* whoops partial append?? truncate spooloutfilename again! */ + ftruncate(fd, dsize); } + FDIOBufferDelete(&FDIO); + close(IOB.fd); + close(fd); + network_talking_to(SKEY(NextHop), NTT_REMOVE); } } closedir(dp); - /* Step 2: delete any files in the outbound queue that were for neighbors who no longer exist */ + if (nFailed > 0) { + FreeStrBuf(&NextHop); + syslog(LOG_INFO, + "skipping Spoolcleanup because of %d files unprocessed.\n", + nFailed + ); + + return; + } + /* Step 2: delete any files in the outbound queue that were for neighbors who no longer exist */ dp = opendir(ctdl_netout_dir); - if (dp == NULL) return; + if (dp == NULL) { + FreeStrBuf(&NextHop); + free(d); + return; + } - while (d = readdir(dp), d != NULL) { - if (!strcmp(d->d_name, ".") || !strcmp(d->d_name, "..")) + while ((readdir_r(dp, d, &filedir_entry) == 0) && + (filedir_entry != NULL)) + { +#ifdef _DIRENT_HAVE_D_NAMELEN + d_namelen = filedir_entry->d_namelen; + d_type = filedir_entry->d_type; +#else + +#ifndef DT_UNKNOWN +#define DT_UNKNOWN 0 +#define DT_DIR 4 +#define DT_REG 8 +#define DT_LNK 10 + +#define IFTODT(mode) (((mode) & 0170000) >> 12) +#define DTTOIF(dirtype) ((dirtype) << 12) +#endif + d_namelen = strlen(filedir_entry->d_name); +#endif + if ((d_namelen == 1) && + (filedir_entry->d_name[0] == '.')) continue; + if ((d_namelen == 2) && + (filedir_entry->d_name[0] == '.') && + (filedir_entry->d_name[1] == '.')) + continue; + + pch = strchr(filedir_entry->d_name, '@'); + if (pch == NULL) /* no @ in name? consolidated file. */ + continue; + + StrBufPlain(NextHop, + filedir_entry->d_name, + pch - filedir_entry->d_name); + snprintf(filename, sizeof filename, "%s/%s", ctdl_netout_dir, - d->d_name + filedir_entry->d_name ); - strcpy(nexthop, ""); - i = is_valid_node(nexthop, + i = is_valid_node(&nexthop, NULL, - d->d_name, + NextHop, working_ignetcfg, the_netmap); - if ( (i != 0) || !IsEmptyStr(nexthop) ) { + if ( (i != 0) || (StrLength(nexthop) > 0) ) { unlink(filename); } } - - + FreeStrBuf(&NextHop); + free(d); closedir(dp); } diff --git a/citadel/modules/network/serv_network.c b/citadel/modules/network/serv_network.c index 7c8c47047..a0e821557 100644 --- a/citadel/modules/network/serv_network.c +++ b/citadel/modules/network/serv_network.c @@ -2,7 +2,7 @@ * This module handles shared rooms, inter-Citadel mail, and outbound * mailing list processing. * - * Copyright (c) 2000-2011 by the citadel.org team + * Copyright (c) 2000-2012 by the citadel.org team * * This program is open source software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -495,8 +495,8 @@ void network_do_queue(void) { static int doing_queue = 0; static time_t last_run = 0L; int full_processing = 1; - char *working_ignetcfg; - NetMap *the_netmap = NULL; + HashList *working_ignetcfg; + HashList *the_netmap = NULL; int netmap_changed = 0; roomlists RL; @@ -537,7 +537,7 @@ void network_do_queue(void) { return; } /* Load the IGnet Configuration into memory */ - working_ignetcfg = load_working_ignetcfg(); + working_ignetcfg = load_ignetcfg(); /* * Load the network map and filter list into memory. @@ -586,19 +586,27 @@ void network_do_queue(void) { /* If there is anything in the inbound queue, process it */ if (!server_shutting_down) { network_do_spoolin(working_ignetcfg, - &the_netmap, + the_netmap, &netmap_changed); } /* Free the filter list in memory */ free_netfilter_list(); + /* Save the network map back to disk */ + if (netmap_changed) { + StrBuf *MapStr = SerializeNetworkMap(the_netmap); + CtdlPutSysConfig(IGNETMAP, SmashStrBuf(&MapStr)); + } + + /* combine singe message files into one spool entry per remote node. */ network_consolidate_spoolout(working_ignetcfg, the_netmap); - /* Save the network map back to disk */ - write_and_free_network_map(&the_netmap, netmap_changed); + /* shut down. */ + + DeleteHash(&the_netmap); - free(working_ignetcfg); + DeleteHash(&working_ignetcfg); syslog(LOG_DEBUG, "network: queue run completed\n"); -- 2.30.2