From 5257cde78179f04a2dfaca87f5bee950eebb57e9 Mon Sep 17 00:00:00 2001 From: Art Cancro Date: Thu, 6 Oct 2005 03:36:05 +0000 Subject: [PATCH] * Changed the format of the euidindex record to contain the record's key. This will allow us to auto-purge stale records later. --- citadel/ChangeLog | 5 ++++ citadel/citadel.h | 4 +-- citadel/euidindex.c | 46 +++++++++++++++++++++++++--------- citadel/msgbase.c | 17 +++++++------ citadel/serv_upgrade.c | 2 +- citadel/techdoc/hack.txt | 54 +++++++++++++++++++++++++++++++--------- 6 files changed, 93 insertions(+), 35 deletions(-) diff --git a/citadel/ChangeLog b/citadel/ChangeLog index f2d05f784..4df65e6ba 100644 --- a/citadel/ChangeLog +++ b/citadel/ChangeLog @@ -1,4 +1,8 @@ $Log$ +Revision 655.21 2005/10/06 03:36:05 ajc +* Changed the format of the euidindex record to contain the record's key. + This will allow us to auto-purge stale records later. + Revision 655.20 2005/10/04 16:38:17 ajc * CtdlOutputPreLoadedMsg() calling syntax has changed. It no longer needs the message number, because it is being supplied a preloaded message. @@ -7197,3 +7201,4 @@ Sat Jul 11 00:20:48 EDT 1998 Nathan Bryant Fri Jul 10 1998 Art Cancro * Initial CVS import + diff --git a/citadel/citadel.h b/citadel/citadel.h index a8f97fe47..7f9f65a17 100644 --- a/citadel/citadel.h +++ b/citadel/citadel.h @@ -33,7 +33,7 @@ extern "C" { /* * Text description of this software */ -#define CITADEL "Citadel 6.57" +#define CITADEL "Citadel 6.58" /* * REV_LEVEL is the current version number (multiplied by 100 to avoid having @@ -45,7 +45,7 @@ extern "C" { * usually more strict because you're not really supposed to dump/load and * upgrade at the same time. */ -#define REV_LEVEL 657 /* This version */ +#define REV_LEVEL 658 /* This version */ #define REV_MIN 591 /* Oldest compatible database */ #define EXPORT_REV_MIN 655 /* Oldest compatible export files */ diff --git a/citadel/euidindex.c b/citadel/euidindex.c index 62bc53c12..9a8d59843 100644 --- a/citadel/euidindex.c +++ b/citadel/euidindex.c @@ -45,6 +45,19 @@ #include "tools.h" #include "euidindex.h" +/* + * The structure of an EUID *index* is: + * + * |----room_number----|----------EUID-------------| + * (sizeof long) (actual length of euid) + * + * + * The structure of an EUID *record* is: + * + * |-----msg_number----|----room_number----|----------EUID-------------| + * (sizeof long) (sizeof long) (actual length of euid) + * + */ long locate_message_by_euid(char *euid, struct ctdlroom *qrbuf) { char *key; @@ -63,21 +76,24 @@ long locate_message_by_euid(char *euid, struct ctdlroom *qrbuf) { free(key); if (cdb_euid == NULL) { - return(-1L); + msgnum = (-1L); } - - if (cdb_euid->len == sizeof(long)) { - msgnum = *(long *)cdb_euid->ptr; + else { + /* The first (sizeof long) of the record is what we're + * looking for. Throw away the rest. + */ + memcpy(&msgnum, cdb_euid->ptr, sizeof(long)); + cdb_free(cdb_euid); } - - cdb_free(cdb_euid); - lprintf(CTDL_DEBUG, "returning msgnum==%ld\n", msgnum); + lprintf(CTDL_DEBUG, "returning msgnum = %ld\n", msgnum); return(msgnum); } void index_message_by_euid(char *euid, struct ctdlroom *qrbuf, long msgnum) { char *key; int key_len; + char *data; + int data_len; lprintf(CTDL_DEBUG, "Indexing message #%ld <%s> in <%s>\n", msgnum, euid, qrbuf->QRname); @@ -86,8 +102,15 @@ void index_message_by_euid(char *euid, struct ctdlroom *qrbuf, long msgnum) { memcpy(key, &qrbuf->QRnumber, sizeof(long)); strcpy(&key[sizeof(long)], euid); - cdb_store(CDB_EUIDINDEX, key, key_len, &msgnum, sizeof(long)); + data_len = sizeof(long) + key_len; + data = malloc(data_len); + + memcpy(data, &msgnum, sizeof(long)); + memcpy(&data[sizeof(long)], key, key_len); + + cdb_store(CDB_EUIDINDEX, key, key_len, data, data_len); free(key); + free(data); } @@ -144,8 +167,7 @@ void rebuild_euid_index_for_room(struct ctdlroom *qrbuf, void *data) { * Globally rebuild the EUID indices in every room. */ void rebuild_euid_index(void) { - cdb_trunc(CDB_EUIDINDEX); - - ForEachRoom(rebuild_euid_index_for_room, NULL); - rebuild_euid_index_for_room(NULL, NULL); + cdb_trunc(CDB_EUIDINDEX); /* delete the old indices */ + ForEachRoom(rebuild_euid_index_for_room, NULL); /* enumerate the room names */ + rebuild_euid_index_for_room(NULL, NULL); /* now do indexing on them */ } diff --git a/citadel/msgbase.c b/citadel/msgbase.c index 36eda40de..ffddf5c73 100644 --- a/citadel/msgbase.c +++ b/citadel/msgbase.c @@ -1865,6 +1865,13 @@ int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int do_repl_check, /* Free up the memory we used. */ free(msglist); + /* If the message has an Exclusive ID, index that... */ + if (msg != NULL) { + if (msg->cm_fields['E'] != NULL) { + index_message_by_euid(msg->cm_fields['E'], &CC->room, msgid); + } + } + /* Update the highest-message pointer and unlock the room. */ CC->room.QRhighest = highest_msg; lputroom(&CC->room); @@ -1873,13 +1880,6 @@ int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int do_repl_check, /* Bump the reference count for this message. */ AdjRefCount(msgid, +1); - /* If the message has an Exclusive ID, index that... */ - if (msg != NULL) { - if (msg->cm_fields['E'] != NULL) { - index_message_by_euid(msg->cm_fields['E'], &CC->room, msgid); - } - } - /* Return success. */ if ( (msg != NULL) && (msg != supplied_msg) ) CtdlFreeMessage(msg); return (0); @@ -2019,7 +2019,8 @@ void ReplicationChecks(struct CtdlMessage *msg) { if (msg == NULL) return; if (msg->cm_fields['E'] == NULL) return; if (strlen(msg->cm_fields['E']) == 0) return; - lprintf(CTDL_DEBUG, "Exclusive ID: <%s>\n", msg->cm_fields['E']); + lprintf(CTDL_DEBUG, "Exclusive ID: <%s> for room <%s>\n", + msg->cm_fields['E'], CC->room.QRname); old_msgnum = locate_message_by_euid(msg->cm_fields['E'], &CC->room); if (old_msgnum > 0L) { diff --git a/citadel/serv_upgrade.c b/citadel/serv_upgrade.c index 19aff117f..f9fa6fab7 100644 --- a/citadel/serv_upgrade.c +++ b/citadel/serv_upgrade.c @@ -217,7 +217,7 @@ void check_server_upgrades(void) { if ((CitControl.version > 000) && (CitControl.version < 608)) { convert_ctdluid_to_minusone(); } - if ((CitControl.version > 000) && (CitControl.version < 657)) { + if ((CitControl.version > 000) && (CitControl.version < 658)) { rebuild_euid_index(); } diff --git a/citadel/techdoc/hack.txt b/citadel/techdoc/hack.txt index cfa5838f6..e1616b3d7 100644 --- a/citadel/techdoc/hack.txt +++ b/citadel/techdoc/hack.txt @@ -8,8 +8,8 @@ system works internally. For details you're going to have to dig through the code, but this'll get you started. - Database tables - + DATABASE TABLES + --------------- As you probably already know by now, Citadel uses a group of tables stored with a record manager (usually Berkeley DB). Since we're using a record @@ -310,16 +310,43 @@ place. The H field looks better when it is placed immediately after the N field. + EUID (EXCLUSIVE MESSAGE ID'S) + ----------------------------- + + This is where the groupware magic happens. Any message in any room may have +a field called the Exclusive message ID, or EUID. We keep an index in the +table CDB_EUIDINDEX which knows the message number of any item that has an +EUID. This allows us to do two things: + + 1. If a subsequent message arrives with the same EUID, it automatically +*deletes* the existing one, because the new one is considered a replacement +for the existing one. + 2. If we know the EUID of the item we're looking for, we can fetch it by EUID +and get the most up-to-date version, even if it's been updated several times. + This functionality is made more useful by server-side hooks. For example, +when we save a vCard to an address book room, or an iCalendar item to a +calendar room, our server modules detect this condition, and automatically set +the EUID of the message to the UUID of the vCard or iCalendar item. Therefore +when you save an updated version of an address book entry or a calendar item, +the old one is automatically deleted. - Networking + + NETWORKING (REPLICATION) + ------------------------ Citadel nodes network by sharing one or more rooms. Any Citadel node can choose to share messages with any other Citadel node, through the sending of spool files. The sending system takes all messages it hasn't sent yet, and spools them to the recieving system, which posts them in the rooms. +The EUID discussion above is extremely relevant, because EUID is carried over +the network as well, and the replacement rules are followed over the network +as well. Therefore, when a message containing an EUID is saved in a networked +room, it replaces any existing message with the same EUID *on every node in +the network*. + Complexities arise primarily from the possibility of densely connected networks: one does not wish to accumulate multiple copies of a given message, which can easily happen. Nor does one want to see old messages @@ -333,9 +360,12 @@ message. If no path field is present, it generates one. With the path present, all the networker has to do to assure that it doesn't send another system a message it's already received is check the

ath field for that system's name somewhere in the bang path. If it's present, the system -has already seen the message, so we don't send it. (Note that the current -implementation does not allow for "loops" in the network -- if you build your -net this way you will see lots of duplicate messages.) +has already seen the message, so we don't send it. + +We also keep a small database, called the "use table," containing the ID's of +all messages we've seen recently. If the same message arrives a second or +subsequent time, we will find its ID in the use table, indicating that we +already have a copy of that message. It will therefore be discarded. The above discussion should make the function of the fields reasonably clear: @@ -349,8 +379,8 @@ distribution includes serv_network.c, which is basically a database replicator; please see network.txt on its operation and functionality (if any). - - Portability issues + PORTABILITY ISSUES + ------------------ Citadel is 64-bit clean, architecture-independent, and Year 2000 compliant. The software should compile on any POSIX compliant system with @@ -362,9 +392,8 @@ build ok on non-POSIX systems with porting libraries (such as Cygwin). - - - SUPPORTING PRIVATE MAIL + SUPPORTING PRIVATE MAIL + ----------------------- Can one have an elegant kludge? This must come pretty close. @@ -385,7 +414,8 @@ regular. - PASSWORDS AND NAME VALIDATION + PASSWORDS AND NAME VALIDATION + ----------------------------- This has changed a couple of times over the course of Citadel's history. At this point it's very simple, again due to the fact that record managers are -- 2.30.2