From 15486ebab6558878b3f290d5c478e1b41b9eb981 Mon Sep 17 00:00:00 2001 From: Art Cancro Date: Tue, 15 Jun 2004 20:42:43 +0000 Subject: [PATCH] * Store the body of any large (>1K) message in a separate database. This will allow fast headers-only retrieval later. --- citadel/ChangeLog | 5 +++- citadel/database_cleanup.sh | 4 +-- citadel/msgbase.c | 56 +++++++++++++++++++++++++++---------- citadel/msgbase.h | 2 +- citadel/server.h | 1 + citadel/sysconfig.h | 6 ++++ citadel/techdoc/hack.txt | 4 +++ 7 files changed, 59 insertions(+), 19 deletions(-) diff --git a/citadel/ChangeLog b/citadel/ChangeLog index 93a975b61..d4215549d 100644 --- a/citadel/ChangeLog +++ b/citadel/ChangeLog @@ -1,4 +1,8 @@ $Log$ + Revision 621.14 2004/06/15 20:42:42 ajc + * Store the body of any large (>1K) message in a separate database. This + will allow fast headers-only retrieval later. + Revision 621.13 2004/06/15 03:05:01 ajc * Bumped internal version number to 6.22 * Added a new faster headers-only mode that excludes enumeration of @@ -5843,4 +5847,3 @@ Sat Jul 11 00:20:48 EDT 1998 Nathan Bryant Fri Jul 10 1998 Art Cancro * Initial CVS import - diff --git a/citadel/database_cleanup.sh b/citadel/database_cleanup.sh index 466c8ee15..faf8a560a 100755 --- a/citadel/database_cleanup.sh +++ b/citadel/database_cleanup.sh @@ -30,7 +30,7 @@ case "$yesno" in exit esac -for x in 00 01 02 03 04 05 06 07 +for x in 00 01 02 03 04 05 06 07 08 do filename=cdb.$x echo Dumping $filename @@ -41,7 +41,7 @@ done echo Removing log files rm -f ./data/* -for x in 00 01 02 03 04 05 06 07 +for x in 00 01 02 03 04 05 06 07 08 do filename=cdb.$x echo Loading $filename diff --git a/citadel/msgbase.c b/citadel/msgbase.c index e1f3041cd..16b4267bb 100644 --- a/citadel/msgbase.c +++ b/citadel/msgbase.c @@ -784,6 +784,7 @@ void mime_download(char *name, char *filename, char *partnum, char *disp, struct CtdlMessage *CtdlFetchMessage(long msgnum) { struct cdbdata *dmsgtext; + struct cdbdata *dbigmsg; struct CtdlMessage *ret = NULL; char *mptr; cit_uint8_t ch; @@ -833,9 +834,22 @@ struct CtdlMessage *CtdlFetchMessage(long msgnum) cdb_free(dmsgtext); - /* Always make sure there's something in the msg text field */ - if (ret->cm_fields['M'] == NULL) - ret->cm_fields['M'] = strdup("\n"); + /* Always make sure there's something in the msg text field. If + * it's NULL, the message text is most likely stored separately, + * so go ahead and fetch that. Failing that, just set a dummy + * body so other code doesn't barf. + */ + if (ret->cm_fields['M'] == NULL) { + + dbigmsg = cdb_fetch(CDB_BIGMSGS, &msgnum, sizeof(long)); + if (dmsgtext == NULL) { + ret->cm_fields['M'] = strdup("\n"); + } + else { + ret->cm_fields['M'] = strdup(dbigmsg->ptr); + cdb_free(dbigmsg); + } + } /* Perform "before read" hooks (aborting if any return nonzero) */ if (PerformMessageHooks(ret, EVT_BEFOREREAD) > 0) { @@ -1721,13 +1735,13 @@ int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int flags) { * called by server-side modules. * */ -long send_message(struct CtdlMessage *msg, /* pointer to buffer */ - FILE *save_a_copy) /* save a copy to disk? */ -{ +long send_message(struct CtdlMessage *msg) { long newmsgid; long retval; char msgidbuf[SIZ]; struct ser_ret smr; + int is_bigmsg = 0; + char *holdM = NULL; /* Get a new message number */ newmsgid = get_new_message_number(); @@ -1737,9 +1751,23 @@ long send_message(struct CtdlMessage *msg, /* pointer to buffer */ if (msg->cm_fields['I']==NULL) { msg->cm_fields['I'] = strdup(msgidbuf); } - + + /* If the message is big, set its body aside for storage elsewhere */ + if (msg->cm_fields['M'] != NULL) { + if (strlen(msg->cm_fields['M']) > BIGMSG) { + is_bigmsg = 1; + holdM = msg->cm_fields['M']; + msg->cm_fields['M'] = NULL; + } + } + + /* Serialize our data structure for storage in the database */ serialize_message(&smr, msg); + if (is_bigmsg) { + msg->cm_fields['M'] = holdM; + } + if (smr.len == 0) { cprintf("%d Unable to serialize message\n", ERROR + INTERNAL_ERROR); @@ -1752,16 +1780,13 @@ long send_message(struct CtdlMessage *msg, /* pointer to buffer */ lprintf(CTDL_ERR, "Can't store message\n"); retval = 0L; } else { + if (is_bigmsg) { + cdb_store(CDB_BIGMSGS, &newmsgid, sizeof(long), + holdM, strlen(holdM) ); + } retval = newmsgid; } - /* If the caller specified that a copy should be saved to a particular - * file handle, do that now too. - */ - if (save_a_copy != NULL) { - fwrite(smr.ser, smr.len, 1, save_a_copy); - } - /* Free the memory we used for the serialized message */ free(smr.ser); @@ -2031,7 +2056,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg, /* message to save */ /* Save it to disk */ lprintf(CTDL_DEBUG, "Saving to disk\n"); - newmsgid = send_message(msg, NULL); + newmsgid = send_message(msg); if (newmsgid <= 0L) return(-1); /* Write a supplemental message info record. This doesn't have to @@ -3077,6 +3102,7 @@ void AdjRefCount(long msgnum, int incr) lprintf(CTDL_DEBUG, "Deleting message <%ld>\n", msgnum); delnum = msgnum; cdb_delete(CDB_MSGMAIN, &delnum, sizeof(long)); + cdb_delete(CDB_BIGMSGS, &delnum, sizeof(long)); /* We have to delete the metadata record too! */ delnum = (0L - msgnum); diff --git a/citadel/msgbase.h b/citadel/msgbase.h index 1d1874e45..f009909ff 100644 --- a/citadel/msgbase.h +++ b/citadel/msgbase.h @@ -82,7 +82,7 @@ void cmd_msg3 (char *cmdbuf); void cmd_msg4 (char *cmdbuf); void cmd_msgp (char *cmdbuf); void cmd_opna (char *cmdbuf); -long send_message (struct CtdlMessage *, FILE *); +long send_message (struct CtdlMessage *); void loadtroom (void); long CtdlSubmitMsg(struct CtdlMessage *, struct recptypes *, char *); void quickie_message (char *, char *, char *, char *, int, char *); diff --git a/citadel/server.h b/citadel/server.h index 06ed9b846..a8f072d37 100644 --- a/citadel/server.h +++ b/citadel/server.h @@ -267,6 +267,7 @@ enum { CDB_VISIT, /* user/room relationships */ CDB_DIRECTORY, /* address book directory */ CDB_USETABLE, /* network use table */ + CDB_BIGMSGS, /* larger message bodies */ MAXCDB /* total number of CDB's defined */ }; diff --git a/citadel/sysconfig.h b/citadel/sysconfig.h index 7ebdec079..c06f8f5e5 100644 --- a/citadel/sysconfig.h +++ b/citadel/sysconfig.h @@ -64,6 +64,12 @@ */ #define SIZ 4096 +/* + * If the body of a message is beyond this size, it will be stored in + * a separate table. + */ +#define BIGMSG 1024 + /* * SMTP delivery retry rules (all values are in seconds) * diff --git a/citadel/techdoc/hack.txt b/citadel/techdoc/hack.txt index 5cceb84bb..59c9fbf8d 100644 --- a/citadel/techdoc/hack.txt +++ b/citadel/techdoc/hack.txt @@ -239,6 +239,10 @@ all software should be written to IGNORE fields not currently defined. BYTE Mnemonic Comments A Author Name of originator of message. +B Big message This is a flag which indicates that the message is + big, and Citadel is storing the body in a separate + record. You will never see this field because the + internal API handles it. D Destination Contains name of the system this message should be sent to, for mail routing (private mail only). E Extended ID A persistent alphanumeric Message ID used for -- 2.30.2