* Began implementation of a networker use table that doesn't chew up oodles

  of CPU time.  (It uses a cdb instead.)

diff --git a/citadel/ChangeLog b/citadel/ChangeLog
index ad83d1f9874133e8a320a8788268f5978ffd47ea..40b29ca725fc8c27a5b4b4e5718a2a3650a7babf 100644 (file)
--- a/citadel/ChangeLog
+++ b/citadel/ChangeLog
@@ -1,4 +1,8 @@
  $Log$
+ Revision 590.114  2002/02/15 04:05:08  ajc
+ * Began implementation of a networker use table that doesn't chew up oodles
+   of CPU time.  (It uses a cdb instead.)
+
  Revision 590.113  2002/02/15 03:40:06  ajc
  * Stu's changes (which he checked in without making any ChangeLog comments,
    bad Stu!) didn't build properly without curses.  Added #ifdef's.
@@ -3344,3 +3348,4 @@ Sat Jul 11 00:20:48 EDT 1998 Nathan Bryant <bryant@cs.usm.maine.edu>
 
 Fri Jul 10 1998 Art Cancro <ajc@uncensored.citadel.org>
        * Initial CVS import 
+

diff --git a/citadel/database.c b/citadel/database.c
index 6e87a51bcb895a2fe3ef150c89af6caa62569376..7ea71305133c5c68e9ab51fa2cb8b350a59a4354 100644 (file)
--- a/citadel/database.c
+++ b/citadel/database.c
@@ -101,6 +101,10 @@ void defrag_databases(void)
        begin_critical_section(S_DIRECTORY);
        gdbm_reorganize(gdbms[CDB_DIRECTORY]);
        end_critical_section(S_DIRECTORY);
+
+       /* defrag the use table */
+       lprintf(7, "Defragmenting the use table\n");
+       gdbm_reorganize(gdbms[CDB_USETABLE]);
 }
 
 
@@ -174,6 +178,14 @@ void open_databases(void)
                        gdbm_strerror(gdbm_errno));
                exit(1);
        }
+       gdbms[CDB_USETABLE] = gdbm_open("data/usetable.gdbm", 0,
+                                       GDBM_WRCREAT, 0600, NULL);
+       if (gdbms[CDB_USETABLE] == NULL) {
+               lprintf(2, "Cannot open use table: %s\n",
+                       gdbm_strerror(gdbm_errno));
+               exit(1);
+       }
+
        /*
           end_critical_section(S_DATABASE);
         */

diff --git a/citadel/serv_network.c b/citadel/serv_network.c
index d0400d13db1a0624ccd560f29484a5aa1b769fe2..717c95815117d1cabe1c8b59c835daa2787e71d2 100644 (file)
--- a/citadel/serv_network.c
+++ b/citadel/serv_network.c
@@ -76,138 +76,51 @@ struct NetMap *the_netmap = NULL;
 
 
 /*
- * Manage the use table.  This is a list of messages which have recently
+ * Check the use table.  This is a list of messages which have recently
  * arrived on the system.  It is maintained and queried to prevent the same
  * message from being entered into the database multiple times if it happens
  * to arrive multiple times by accident.
  */
-int network_usetable(int operation, struct CtdlMessage *msg) {
+int network_usetable(struct CtdlMessage *msg) {
 
-       static struct UseTable *ut = NULL;
-       struct UseTable *uptr = NULL;
-       char *serialized_table = NULL;
-       char *ptr;
        char msgid[SIZ];
-       char buf[SIZ];
-       int i;
-       size_t stlen = 0;
-
-       switch(operation) {
-       
-               case UT_LOAD:
-                       serialized_table = CtdlGetSysConfig(USETABLE);
-                       if (serialized_table == NULL) return(0);
-
-                       ptr = serialized_table;
-                       i = 0;
-                       buf[0] = 0;
-                       while (ptr[0] != 0) {
-                               buf[i] = *ptr;
-               
-                               if (buf[i]=='\n') {
-                                       buf[i] = 0;
-                                       if (strlen(buf) > 0) {
-
-                                               uptr = (struct UseTable *)
-                                                       mallok(sizeof(struct UseTable));
-                                               if (uptr != NULL) {
-                                                       uptr->next = ut;
-                                                       extract(msgid, buf, 0);
-                                                       uptr->message_id = strdoop(msgid);
-                                                       uptr->timestamp = extract_long(buf, 1);
-                                                       ut = uptr;
-                                               }
-                                       }
-
-                                       i = 0;
-                                       buf[0] = 0;
-                               }
-                               else {
-                                       ++i;
-                               }
-                               ++ptr;
-                       }
+       struct cdbdata *cdbut;
+       time_t timestamp;
 
-                       phree(serialized_table);
-                       serialized_table = NULL;
-                       return(0);
-
-               case UT_INSERT:
-                       /* Bail out if we can't generate a message ID */
-                       if (msg == NULL) {
-                               return(0);
-                       }
-                       if (msg->cm_fields['I'] == NULL) {
-                               return(0);
-                       }
-                       if (strlen(msg->cm_fields['I']) == 0) {
-                               return(0);
-                       }
-
-                       /* Generate the message ID */
-                       strcpy(msgid, msg->cm_fields['I']);
-                       if (haschar(msgid, '@') == 0) {
-                               strcat(msgid, "@");
-                               if (msg->cm_fields['N'] != NULL) {
-                                       strcat(msgid, msg->cm_fields['N']);
-                               }
-                               else {
-                                       return(0);
-                               }
-                       }
-
-                       /* Compare to the existing list */
-                       for (uptr = ut; uptr != NULL; uptr = uptr->next) {
-                               if (!strcasecmp(msgid, uptr->message_id)) {
-                                       return(1);
-                               }
-                       }
-
-                       /* If we got to this point, it's unique: add it. */
-                       uptr = (struct UseTable *)
-                               mallok(sizeof(struct UseTable));
-                       if (uptr == NULL) return(0);
-                       uptr->next = ut;
-                       uptr->message_id = strdoop(msgid);
-                       uptr->timestamp = time(NULL);
-                       ut = uptr;
-                       return(0);
-
-               case UT_SAVE:
-                       /* Figure out how big the serialized buffer should be */
-                       stlen = 0;
-                       for (uptr = ut; uptr != NULL; uptr = uptr->next) {
-                               stlen = stlen + strlen(uptr->message_id) + 20;
-                       }
-                       serialized_table = mallok(stlen);
-                       memset(serialized_table, 0, stlen);
-
-                       while (ut != NULL) {
-                               if ( (serialized_table != NULL) 
-                                  && ( (ut->timestamp - time(NULL)) <
-                                     USETABLE_RETAIN) ) {
-                                       sprintf(&serialized_table[strlen(
-                                         serialized_table)], "%s|%ld\n",
-                                           ut->message_id,
-                                           (long)ut->timestamp);
-                               }
-
-                               /* Now free the memory */
-                               uptr = ut;
-                               ut = ut->next;
-                               phree(uptr->message_id);
-                               phree(uptr);
-                       }
+       /* Bail out if we can't generate a message ID */
+       if (msg == NULL) {
+               return(0);
+       }
+       if (msg->cm_fields['I'] == NULL) {
+               return(0);
+       }
+       if (strlen(msg->cm_fields['I']) == 0) {
+               return(0);
+       }
 
-                       /* Write to disk */
-                       CtdlPutSysConfig(USETABLE, serialized_table);
-                       phree(serialized_table);
+       /* Generate the message ID */
+       strcpy(msgid, msg->cm_fields['I']);
+       if (haschar(msgid, '@') == 0) {
+               strcat(msgid, "@");
+               if (msg->cm_fields['N'] != NULL) {
+                       strcat(msgid, msg->cm_fields['N']);
+               }
+               else {
                        return(0);
+               }
+       }
 
+       cdbut = cdb_fetch(CDB_USETABLE, msgid, strlen(msgid));
+       if (cdbut != NULL) {
+               cdb_free(cdbut);
+               return(1);
        }
 
-       /* should never get here unless illegal operation specified */
-       return(2);
+       /* If we got to this point, it's unique: add it. */
+       timestamp = time(NULL);
+       cdb_store(CDB_USETABLE, msgid, strlen(msgid),
+               &timestamp, sizeof(timestamp) );
+       return(0);
 }
 
 
@@ -920,7 +833,7 @@ void network_process_buffer(char *buffer, long size) {
        /*
         * Check to see if we already have a copy of this message
         */
-       if (network_usetable(UT_INSERT, msg) != 0) {
+       if (network_usetable(msg) != 0) {
                sprintf(buf,
                        "Loopzapper rejected message <%s> "
                        "from <%s> in <%s> @ <%s>\n",
@@ -1328,10 +1241,9 @@ void network_do_queue(void) {
        network_poll_other_citadel_nodes();
 
        /*
-        * Load the network map and use table into memory.
+        * Load the network map into memory.
         */
        read_network_map();
-       network_usetable(UT_LOAD, NULL);
 
        /* 
         * Go ahead and run the queue
@@ -1350,8 +1262,7 @@ void network_do_queue(void) {
        lprintf(7, "network: processing inbound queue\n");
        network_do_spoolin();
 
-       /* Save the usetable and network map back to disk */
-       network_usetable(UT_SAVE, NULL);
+       /* Save the network map back to disk */
        write_network_map();
 
        lprintf(7, "network: queue run completed\n");

diff --git a/citadel/serv_network.h b/citadel/serv_network.h
index f3c9bbdb896d39ac989e92eb5b9c7e854df0d023..0c01df98c8dde255893a8ad75a4df72b3d82f550 100644 (file)
--- a/citadel/serv_network.h
+++ b/citadel/serv_network.h
@@ -21,17 +21,3 @@ struct NetMap {
        char nexthop[SIZ];
 };
 
-struct UseTable {
-       struct UseTable *next;
-       char *message_id;
-       time_t timestamp;
-};
-
-/*
- * Operations which can be performed by the network_usetable() function
- */
-enum {
-       UT_INSERT,
-       UT_LOAD,
-       UT_SAVE
-};

diff --git a/citadel/server.h b/citadel/server.h
index 583f55a2ccf4ad5f5d0fce97ee0b39c3b4815b1c..519535c13621129a001089554599b89a574f3720 100644 (file)
--- a/citadel/server.h
+++ b/citadel/server.h
@@ -250,6 +250,7 @@ enum {
        CDB_MSGLISTS,           /* room message lists            */
        CDB_VISIT,              /* user/room relationships       */
        CDB_DIRECTORY,          /* address book directory        */
+       CDB_USETABLE,           /* network use table             */
        MAXCDB                  /* total number of CDB's defined */
 };