From: Art Cancro Date: Sun, 2 Oct 2005 04:40:58 +0000 (+0000) Subject: * The EUID index is now built, and replication checks are being performed X-Git-Tag: v7.86~4592 X-Git-Url: https://code.citadel.org/?p=citadel.git;a=commitdiff_plain;h=83543fd906ac428c01815a04d78d6a17e23eea31 * The EUID index is now built, and replication checks are being performed using it. It is much faster now because we don't have to scan the entire room anymore. We still need to do two things: 1. Write a server command to fetch messages by EUID instead of msgnum 2. Find a way to purge stale EUID index records. --- diff --git a/citadel/ChangeLog b/citadel/ChangeLog index 68f8d76be..fac7b5233 100644 --- a/citadel/ChangeLog +++ b/citadel/ChangeLog @@ -1,4 +1,11 @@ $Log$ +Revision 655.19 2005/10/02 04:40:58 ajc +* The EUID index is now built, and replication checks are being performed + using it. It is much faster now because we don't have to scan the entire + room anymore. We still need to do two things: + 1. Write a server command to fetch messages by EUID instead of msgnum + 2. Find a way to purge stale EUID index records. + Revision 655.18 2005/10/01 05:18:57 ajc * Began writing code to index messages by euid per room @@ -7183,3 +7190,4 @@ Sat Jul 11 00:20:48 EDT 1998 Nathan Bryant Fri Jul 10 1998 Art Cancro * Initial CVS import + diff --git a/citadel/citadel.h b/citadel/citadel.h index 78eb2a4cf..a8f97fe47 100644 --- a/citadel/citadel.h +++ b/citadel/citadel.h @@ -33,7 +33,7 @@ extern "C" { /* * Text description of this software */ -#define CITADEL "Citadel 6.55" +#define CITADEL "Citadel 6.57" /* * REV_LEVEL is the current version number (multiplied by 100 to avoid having @@ -45,7 +45,7 @@ extern "C" { * usually more strict because you're not really supposed to dump/load and * upgrade at the same time. */ -#define REV_LEVEL 656 /* This version */ +#define REV_LEVEL 657 /* This version */ #define REV_MIN 591 /* Oldest compatible database */ #define EXPORT_REV_MIN 655 /* Oldest compatible export files */ diff --git a/citadel/database_cleanup.sh b/citadel/database_cleanup.sh index 850dc181d..4a37bba76 100755 --- a/citadel/database_cleanup.sh +++ b/citadel/database_cleanup.sh @@ -37,6 +37,10 @@ WARNING #3: Please try "db_recover -c" from the data/ directory first. Use this tool only if that one fails to fix your problem. +WARNING #4: + You must have an amount of free space on your disk that is at least twice + the size of your database. + ! echo -n "Do you want to continue? " @@ -52,7 +56,7 @@ case "$yesno" in exit esac -for x in 00 01 02 03 04 05 06 07 08 09 +for x in 00 01 02 03 04 05 06 07 08 09 0a do filename=cdb.$x echo Dumping $filename diff --git a/citadel/euidindex.c b/citadel/euidindex.c index 04c14b91a..62bc53c12 100644 --- a/citadel/euidindex.c +++ b/citadel/euidindex.c @@ -47,7 +47,32 @@ long locate_message_by_euid(char *euid, struct ctdlroom *qrbuf) { - return(0); + char *key; + int key_len; + struct cdbdata *cdb_euid; + long msgnum = (-1L); + + lprintf(CTDL_DEBUG, "Searching for EUID <%s> in <%s>\n", euid, qrbuf->QRname); + + key_len = strlen(euid) + sizeof(long) + 1; + key = malloc(key_len); + memcpy(key, &qrbuf->QRnumber, sizeof(long)); + strcpy(&key[sizeof(long)], euid); + + cdb_euid = cdb_fetch(CDB_EUIDINDEX, key, key_len); + free(key); + + if (cdb_euid == NULL) { + return(-1L); + } + + if (cdb_euid->len == sizeof(long)) { + msgnum = *(long *)cdb_euid->ptr; + } + + cdb_free(cdb_euid); + lprintf(CTDL_DEBUG, "returning msgnum==%ld\n", msgnum); + return(msgnum); } void index_message_by_euid(char *euid, struct ctdlroom *qrbuf, long msgnum) { @@ -62,6 +87,7 @@ void index_message_by_euid(char *euid, struct ctdlroom *qrbuf, long msgnum) { strcpy(&key[sizeof(long)], euid); cdb_store(CDB_EUIDINDEX, key, key_len, &msgnum, sizeof(long)); + free(key); } diff --git a/citadel/msgbase.c b/citadel/msgbase.c index 3bca31e77..71a769022 100644 --- a/citadel/msgbase.c +++ b/citadel/msgbase.c @@ -1772,8 +1772,14 @@ void cmd_opna(char *cmdbuf) * Save a message pointer into a specified room * (Returns 0 for success, nonzero for failure) * roomname may be NULL to use the current room + * + * Note that the 'supplied_msg' field may be set to NULL, in which case + * the message will be fetched from disk, by number, if we need to perform + * replication checks. This adds an additional database read, so if the + * caller already has the message in memory then it should be supplied. */ -int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int flags) { +int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int do_repl_check, + struct CtdlMessage *supplied_msg) { int i; char hold_rm[ROOMNAMELEN]; struct cdbdata *cdbfr; @@ -1782,37 +1788,34 @@ int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int flags) { long highest_msg = 0L; struct CtdlMessage *msg = NULL; - lprintf(CTDL_DEBUG, "CtdlSaveMsgPointerInRoom(%s, %ld, %d)\n", - roomname, msgid, flags); + lprintf(CTDL_DEBUG, "CtdlSaveMsgPointerInRoom(roomname=%s, msgid=%ld, do_repl_check=%d)\n", + roomname, msgid, do_repl_check); strcpy(hold_rm, CC->room.QRname); /* We may need to check to see if this message is real */ - if ( (flags & SM_VERIFY_GOODNESS) - || (flags & SM_DO_REPL_CHECK) - ) { - msg = CtdlFetchMessage(msgid, 1); + if (do_repl_check) { + if (supplied_msg != NULL) { + msg = supplied_msg; + } + else { + msg = CtdlFetchMessage(msgid, 0); + } if (msg == NULL) return(ERROR + ILLEGAL_VALUE); } /* Perform replication checks if necessary */ - if ( (flags & SM_DO_REPL_CHECK) && (msg != NULL) ) { + if ( (do_repl_check) && (msg != NULL) ) { if (getroom(&CC->room, ((roomname != NULL) ? roomname : CC->room.QRname) ) != 0) { lprintf(CTDL_ERR, "No such room <%s>\n", roomname); - if (msg != NULL) CtdlFreeMessage(msg); + if ( (msg != NULL) && (msg != supplied_msg) ) CtdlFreeMessage(msg); return(ERROR + ROOM_NOT_FOUND); } - if (ReplicationChecks(msg) != 0) { - getroom(&CC->room, hold_rm); - if (msg != NULL) CtdlFreeMessage(msg); - lprintf(CTDL_DEBUG, - "Did replication, and newer exists\n"); - return(0); - } + ReplicationChecks(msg); } /* Now the regular stuff */ @@ -1820,7 +1823,7 @@ int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int flags) { ((roomname != NULL) ? roomname : CC->room.QRname) ) != 0) { lprintf(CTDL_ERR, "No such room <%s>\n", roomname); - if (msg != NULL) CtdlFreeMessage(msg); + if ( (msg != NULL) && (msg != supplied_msg) ) CtdlFreeMessage(msg); return(ERROR + ROOM_NOT_FOUND); } @@ -1846,7 +1849,7 @@ int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int flags) { if (msglist[i] == msgid) { lputroom(&CC->room); /* unlock the room */ getroom(&CC->room, hold_rm); - if (msg != NULL) CtdlFreeMessage(msg); + if ( (msg != NULL) && (msg != supplied_msg) ) CtdlFreeMessage(msg); free(msglist); return(ERROR + ALREADY_EXISTS); } @@ -1880,12 +1883,17 @@ int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int flags) { getroom(&CC->room, hold_rm); /* Bump the reference count for this message. */ - if ((flags & SM_DONT_BUMP_REF)==0) { - AdjRefCount(msgid, +1); + AdjRefCount(msgid, +1); + + /* If the message has an Exclusive ID, index that... */ + if (msg != NULL) { + if (msg->cm_fields['E'] != NULL) { + index_message_by_euid(msg->cm_fields['E'], &CC->room, msgid); + } } /* Return success. */ - if (msg != NULL) CtdlFreeMessage(msg); + if ( (msg != NULL) && (msg != supplied_msg) ) CtdlFreeMessage(msg); return (0); } @@ -2013,37 +2021,23 @@ void serialize_message(struct ser_ret *ret, /* return values */ /* - * Back end for the ReplicationChecks() function + * Check to see if any messages already exist in the current room which + * carry the same Exclusive ID as this one. If any are found, delete them. */ -void check_repl(long msgnum, void *userdata) { - lprintf(CTDL_DEBUG, "check_repl() replacing message %ld\n", msgnum); - CtdlDeleteMessages(CC->room.QRname, msgnum, "", 0); -} - - -/* - * Check to see if any messages already exist which carry the same Exclusive ID - * as this one. If any are found, delete them. - * - */ -int ReplicationChecks(struct CtdlMessage *msg) { - struct CtdlMessage *template; - int abort_this = 0; +void ReplicationChecks(struct CtdlMessage *msg) { + long old_msgnum = (-1L); /* No exclusive id? Don't do anything. */ - if (msg->cm_fields['E'] == NULL) return 0; - if (strlen(msg->cm_fields['E']) == 0) return 0; + if (msg == NULL) return; + if (msg->cm_fields['E'] == NULL) return; + if (strlen(msg->cm_fields['E']) == 0) return; lprintf(CTDL_DEBUG, "Exclusive ID: <%s>\n", msg->cm_fields['E']); - template = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage)); - memset(template, 0, sizeof(struct CtdlMessage)); - template->cm_fields['E'] = strdup(msg->cm_fields['E']); - - CtdlForEachMessage(MSGS_ALL, 0L, NULL, template, check_repl, NULL); - - CtdlFreeMessage(template); - lprintf(CTDL_DEBUG, "ReplicationChecks() returning %d\n", abort_this); - return(abort_this); + old_msgnum = locate_message_by_euid(msg->cm_fields['E'], &CC->room); + if (old_msgnum > 0L) { + lprintf(CTDL_DEBUG, "ReplicationChecks() replacing message %ld\n", old_msgnum); + CtdlDeleteMessages(CC->room.QRname, old_msgnum, "", 0); + } } @@ -2186,7 +2180,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg, /* message to save */ /* If this message has an Exclusive ID, perform replication checks */ lprintf(CTDL_DEBUG, "Performing replication checks\n"); - if (ReplicationChecks(msg) > 0) return(-4); + ReplicationChecks(msg); /* Save it to disk */ lprintf(CTDL_DEBUG, "Saving to disk\n"); @@ -2235,17 +2229,16 @@ long CtdlSubmitMsg(struct CtdlMessage *msg, /* message to save */ * is no local sender; it would otherwise go to the Trashcan). */ if ((!CC->internal_pgm) || (recps == NULL)) { - if (CtdlSaveMsgPointerInRoom(actual_rm, newmsgid, 0) != 0) { + if (CtdlSaveMsgPointerInRoom(actual_rm, newmsgid, 1, msg) != 0) { lprintf(CTDL_ERR, "ERROR saving message pointer!\n"); - CtdlSaveMsgPointerInRoom(config.c_aideroom, - newmsgid, 0); + CtdlSaveMsgPointerInRoom(config.c_aideroom, newmsgid, 0, msg); } } /* For internet mail, drop a copy in the outbound queue room */ if (recps != NULL) if (recps->num_internet > 0) { - CtdlSaveMsgPointerInRoom(SMTP_SPOOLOUT_ROOM, newmsgid, 0); + CtdlSaveMsgPointerInRoom(SMTP_SPOOLOUT_ROOM, newmsgid, 0, msg); } /* If other rooms are specified, drop them there too. */ @@ -2255,7 +2248,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg, /* message to save */ extract_token(recipient, recps->recp_room, i, '|', sizeof recipient); lprintf(CTDL_DEBUG, "Delivering to room <%s>\n", recipient); - CtdlSaveMsgPointerInRoom(recipient, newmsgid, 0); + CtdlSaveMsgPointerInRoom(recipient, newmsgid, 0, msg); } /* Bump this user's messages posted counter. */ @@ -2277,13 +2270,12 @@ long CtdlSubmitMsg(struct CtdlMessage *msg, /* message to save */ if (getuser(&userbuf, recipient) == 0) { MailboxName(actual_rm, sizeof actual_rm, &userbuf, MAILROOM); - CtdlSaveMsgPointerInRoom(actual_rm, newmsgid, 0); + CtdlSaveMsgPointerInRoom(actual_rm, newmsgid, 0, msg); BumpNewMailCounter(userbuf.usernum); } else { lprintf(CTDL_DEBUG, "No user <%s>\n", recipient); - CtdlSaveMsgPointerInRoom(config.c_aideroom, - newmsgid, 0); + CtdlSaveMsgPointerInRoom(config.c_aideroom, newmsgid, 0, msg); } } @@ -3258,8 +3250,7 @@ void cmd_dele(char *delstr) int CtdlCopyMsgToRoom(long msgnum, char *dest) { int err; - err = CtdlSaveMsgPointerInRoom(dest, msgnum, - (SM_VERIFY_GOODNESS | SM_DO_REPL_CHECK) ); + err = CtdlSaveMsgPointerInRoom(dest, msgnum, 1, NULL); if (err != 0) return(err); return(0); diff --git a/citadel/msgbase.h b/citadel/msgbase.h index ad232cf05..ead3bbf2d 100644 --- a/citadel/msgbase.h +++ b/citadel/msgbase.h @@ -10,15 +10,6 @@ #define MSGS_GT 5 #define MSGS_EQ 6 -/* - * Flags which may be passed to CtdlSaveMsgPointerInRoom() - */ -#define SM_VERIFY_GOODNESS 1 /* Verify this is a real msg number */ -#define SM_DO_REPL_CHECK 2 /* Perform replication checks */ -#define SM_DONT_BUMP_REF 4 /* Don't bump reference count - (use with extreme care!!!!!!) */ - - /* * Possible return codes from CtdlOutputMsg() */ @@ -117,8 +108,8 @@ struct CtdlMessage *CtdlFetchMessage(long msgnum, int with_body); void CtdlFreeMessage(struct CtdlMessage *msg); void serialize_message(struct ser_ret *, struct CtdlMessage *); int is_valid_message(struct CtdlMessage *); -int ReplicationChecks(struct CtdlMessage *); -int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int flags); +void ReplicationChecks(struct CtdlMessage *); +int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int do_repl_check, struct CtdlMessage *msg); char *CtdlReadMessageBody(char *terminator, size_t maxlen, char *exist, int crlf); char *CtdlGetSysConfig(char *sysconfname); void CtdlPutSysConfig(char *sysconfname, char *sysconfdata); diff --git a/citadel/serv_chat.c b/citadel/serv_chat.c index b323dab7a..ce6f7324b 100644 --- a/citadel/serv_chat.c +++ b/citadel/serv_chat.c @@ -639,7 +639,7 @@ int send_instant_message(char *lun, char *x_user, char *x_msg) /* Now save a copy in the global log room, if configured */ if (strlen(config.c_logpages) > 0) { create_room(config.c_logpages, 3, "", 0, 1, 1, VIEW_BBS); - CtdlSaveMsgPointerInRoom(config.c_logpages, msgnum, 0); + CtdlSaveMsgPointerInRoom(config.c_logpages, msgnum, 0, NULL); } /* Save a copy in each recipient's log room, creating those @@ -649,7 +649,7 @@ int send_instant_message(char *lun, char *x_user, char *x_msg) */ while (sl != NULL) { create_room(sl->roomname, 5, "", 0, 1, 1, VIEW_BBS); - CtdlSaveMsgPointerInRoom(sl->roomname, msgnum, 0); + CtdlSaveMsgPointerInRoom(sl->roomname, msgnum, 0, NULL); sptr = sl->next; free(sl); sl = sptr; diff --git a/citadel/serv_upgrade.c b/citadel/serv_upgrade.c index db92fbf2d..19aff117f 100644 --- a/citadel/serv_upgrade.c +++ b/citadel/serv_upgrade.c @@ -217,6 +217,9 @@ void check_server_upgrades(void) { if ((CitControl.version > 000) && (CitControl.version < 608)) { convert_ctdluid_to_minusone(); } + if ((CitControl.version > 000) && (CitControl.version < 657)) { + rebuild_euid_index(); + } CitControl.version = REV_LEVEL; put_control(); @@ -226,6 +229,5 @@ void check_server_upgrades(void) { char *serv_upgrade_init(void) { check_server_upgrades(); - rebuild_euid_index(); /* FIXME */ return "$Id$"; } diff --git a/citadel/serv_vandelay.c b/citadel/serv_vandelay.c index 5f4de4aca..49f5ed9de 100644 --- a/citadel/serv_vandelay.c +++ b/citadel/serv_vandelay.c @@ -43,6 +43,7 @@ #include "user_ops.h" #include "room_ops.h" #include "control.h" +#include "euidindex.h" #define END_OF_MESSAGE "---eom---dbd---" @@ -460,7 +461,7 @@ void artv_import_room(void) { * one per line terminated by a 0. */ while (client_getln(buf, sizeof buf), msgnum = atol(buf), msgnum > 0) { - CtdlSaveMsgPointerInRoom(qrbuf.QRname, msgnum, 0); + CtdlSaveMsgPointerInRoom(qrbuf.QRname, msgnum, 0, NULL); ++msgcount; } lprintf(CTDL_INFO, "(%d messages)\n", msgcount); @@ -590,6 +591,7 @@ void artv_do_import(void) { } lprintf(CTDL_INFO, "Invalid keyword <%s>. Flushing input.\n", buf); while (client_getln(buf, sizeof buf), strcmp(buf, "000")) ;; + rebuild_euid_index(); } diff --git a/citadel/serv_vcard.c b/citadel/serv_vcard.c index 86e5afd3d..0c35d8c18 100644 --- a/citadel/serv_vcard.c +++ b/citadel/serv_vcard.c @@ -455,8 +455,7 @@ int vcard_upload_aftersave(struct CtdlMessage *msg) { vcard_free(v); /* Put it in the Global Address Book room... */ - CtdlSaveMsgPointerInRoom(ADDRESS_BOOK_ROOM, I, - (SM_VERIFY_GOODNESS | SM_DO_REPL_CHECK) ); + CtdlSaveMsgPointerInRoom(ADDRESS_BOOK_ROOM, I, 1, msg); /* ...and also in the directory database. */ vcard_add_to_directory(I, NULL);