* Store the body of any large (>1K) message in a separate database. This

  will allow fast headers-only retrieval later.

diff --git a/citadel/ChangeLog b/citadel/ChangeLog
index 93a975b61dcc451a81a63cc41f42fc5e54c4241b..d4215549dd5db298df955ce584d7d23f58bb63b7 100644 (file)
--- a/citadel/ChangeLog
+++ b/citadel/ChangeLog
@@ -1,4 +1,8 @@
  $Log$
+ Revision 621.14  2004/06/15 20:42:42  ajc
+ * Store the body of any large (>1K) message in a separate database.  This
+   will allow fast headers-only retrieval later.
+
  Revision 621.13  2004/06/15 03:05:01  ajc
  * Bumped internal version number to 6.22
  * Added a new faster headers-only mode that excludes enumeration of
@@ -5843,4 +5847,3 @@ Sat Jul 11 00:20:48 EDT 1998 Nathan Bryant <bryant@cs.usm.maine.edu>
 
 Fri Jul 10 1998 Art Cancro <ajc@uncensored.citadel.org>
        * Initial CVS import
-

diff --git a/citadel/database_cleanup.sh b/citadel/database_cleanup.sh
index 466c8ee15f0fa623450c744c357d2062f03ed1c9..faf8a560a33a88c4fe5b4de25366047cb02beb71 100755 (executable)
--- a/citadel/database_cleanup.sh
+++ b/citadel/database_cleanup.sh
@@ -30,7 +30,7 @@ case "$yesno" in
                exit
 esac
 
-for x in 00 01 02 03 04 05 06 07
+for x in 00 01 02 03 04 05 06 07 08
 do
        filename=cdb.$x
        echo Dumping $filename
@@ -41,7 +41,7 @@ done
 echo Removing log files
 rm -f ./data/*
 
-for x in 00 01 02 03 04 05 06 07
+for x in 00 01 02 03 04 05 06 07 08
 do
        filename=cdb.$x
        echo Loading $filename

diff --git a/citadel/msgbase.c b/citadel/msgbase.c
index e1f3041cde89028d1ad8e93176ff004eac9dd6a3..16b4267bbf3f2c4e429dfec870e28dbb42e4893f 100644 (file)
--- a/citadel/msgbase.c
+++ b/citadel/msgbase.c
@@ -784,6 +784,7 @@ void mime_download(char *name, char *filename, char *partnum, char *disp,
 struct CtdlMessage *CtdlFetchMessage(long msgnum)
 {
        struct cdbdata *dmsgtext;
+       struct cdbdata *dbigmsg;
        struct CtdlMessage *ret = NULL;
        char *mptr;
        cit_uint8_t ch;
@@ -833,9 +834,22 @@ struct CtdlMessage *CtdlFetchMessage(long msgnum)
 
        cdb_free(dmsgtext);
 
-       /* Always make sure there's something in the msg text field */
-       if (ret->cm_fields['M'] == NULL)
-               ret->cm_fields['M'] = strdup("<no text>\n");
+       /* Always make sure there's something in the msg text field.  If
+        * it's NULL, the message text is most likely stored separately,
+        * so go ahead and fetch that.  Failing that, just set a dummy
+        * body so other code doesn't barf.
+        */
+       if (ret->cm_fields['M'] == NULL) {
+
+               dbigmsg = cdb_fetch(CDB_BIGMSGS, &msgnum, sizeof(long));
+               if (dmsgtext == NULL) {
+                       ret->cm_fields['M'] = strdup("<no text>\n");
+               }
+               else {
+                       ret->cm_fields['M'] = strdup(dbigmsg->ptr);
+                       cdb_free(dbigmsg);
+               }
+       }
 
        /* Perform "before read" hooks (aborting if any return nonzero) */
        if (PerformMessageHooks(ret, EVT_BEFOREREAD) > 0) {
@@ -1721,13 +1735,13 @@ int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int flags) {
  * called by server-side modules.
  *
  */
-long send_message(struct CtdlMessage *msg,     /* pointer to buffer */
-               FILE *save_a_copy)              /* save a copy to disk? */
-{
+long send_message(struct CtdlMessage *msg) {
        long newmsgid;
        long retval;
        char msgidbuf[SIZ];
         struct ser_ret smr;
+       int is_bigmsg = 0;
+       char *holdM = NULL;
 
        /* Get a new message number */
        newmsgid = get_new_message_number();
@@ -1737,9 +1751,23 @@ long send_message(struct CtdlMessage *msg,       /* pointer to buffer */
        if (msg->cm_fields['I']==NULL) {
                msg->cm_fields['I'] = strdup(msgidbuf);
        }
-       
+
+       /* If the message is big, set its body aside for storage elsewhere */
+       if (msg->cm_fields['M'] != NULL) {
+               if (strlen(msg->cm_fields['M']) > BIGMSG) {
+                       is_bigmsg = 1;
+                       holdM = msg->cm_fields['M'];
+                       msg->cm_fields['M'] = NULL;
+               }
+       }
+
+       /* Serialize our data structure for storage in the database */  
         serialize_message(&smr, msg);
 
+       if (is_bigmsg) {
+               msg->cm_fields['M'] = holdM;
+       }
+
         if (smr.len == 0) {
                 cprintf("%d Unable to serialize message\n",
                         ERROR + INTERNAL_ERROR);
@@ -1752,16 +1780,13 @@ long send_message(struct CtdlMessage *msg,      /* pointer to buffer */
                lprintf(CTDL_ERR, "Can't store message\n");
                retval = 0L;
        } else {
+               if (is_bigmsg) {
+                       cdb_store(CDB_BIGMSGS, &newmsgid, sizeof(long),
+                               holdM, strlen(holdM) );
+               }
                retval = newmsgid;
        }
 
-       /* If the caller specified that a copy should be saved to a particular
-        * file handle, do that now too.
-        */
-       if (save_a_copy != NULL) {
-               fwrite(smr.ser, smr.len, 1, save_a_copy);
-       }
-
        /* Free the memory we used for the serialized message */
         free(smr.ser);
 
@@ -2031,7 +2056,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
 
        /* Save it to disk */
        lprintf(CTDL_DEBUG, "Saving to disk\n");
-       newmsgid = send_message(msg, NULL);
+       newmsgid = send_message(msg);
        if (newmsgid <= 0L) return(-1);
 
        /* Write a supplemental message info record.  This doesn't have to
@@ -3077,6 +3102,7 @@ void AdjRefCount(long msgnum, int incr)
                lprintf(CTDL_DEBUG, "Deleting message <%ld>\n", msgnum);
                delnum = msgnum;
                cdb_delete(CDB_MSGMAIN, &delnum, sizeof(long));
+               cdb_delete(CDB_BIGMSGS, &delnum, sizeof(long));
 
                /* We have to delete the metadata record too! */
                delnum = (0L - msgnum);

diff --git a/citadel/msgbase.h b/citadel/msgbase.h
index 1d1874e45754c2e0fdec3b152c6809f2952dce9a..f009909ff8116341e153d77878bba0170bcaea08 100644 (file)
--- a/citadel/msgbase.h
+++ b/citadel/msgbase.h
@@ -82,7 +82,7 @@ void cmd_msg3 (char *cmdbuf);
 void cmd_msg4 (char *cmdbuf);
 void cmd_msgp (char *cmdbuf);
 void cmd_opna (char *cmdbuf);
-long send_message (struct CtdlMessage *, FILE *);
+long send_message (struct CtdlMessage *);
 void loadtroom (void);
 long CtdlSubmitMsg(struct CtdlMessage *, struct recptypes *, char *);
 void quickie_message (char *, char *, char *, char *, int, char *);

diff --git a/citadel/server.h b/citadel/server.h
index 06ed9b84660d88b76125c087f6555d11ac7a362a..a8f072d37a6bf5c6c5b28991d6e9800c99688082 100644 (file)
--- a/citadel/server.h
+++ b/citadel/server.h
@@ -267,6 +267,7 @@ enum {
        CDB_VISIT,              /* user/room relationships       */
        CDB_DIRECTORY,          /* address book directory        */
        CDB_USETABLE,           /* network use table             */
+       CDB_BIGMSGS,            /* larger message bodies         */
        MAXCDB                  /* total number of CDB's defined */
 };
 

diff --git a/citadel/sysconfig.h b/citadel/sysconfig.h
index 7ebdec079bf3edb2b8c18053a98a9235edbd1454..c06f8f5e5c0dcf548c40e9b3a28159ceaa6235ec 100644 (file)
--- a/citadel/sysconfig.h
+++ b/citadel/sysconfig.h
@@ -64,6 +64,12 @@
  */
 #define SIZ            4096
 
+/*
+ * If the body of a message is beyond this size, it will be stored in
+ * a separate table.
+ */
+#define BIGMSG         1024
+
 /*
  * SMTP delivery retry rules (all values are in seconds)
  *

diff --git a/citadel/techdoc/hack.txt b/citadel/techdoc/hack.txt
index 5cceb84bb2dd498b0662512d93e8d0ebab4d51b7..59c9fbf8dfe8d0c907045823f0027562027bbe72 100644 (file)
--- a/citadel/techdoc/hack.txt
+++ b/citadel/techdoc/hack.txt
@@ -239,6 +239,10 @@ all software should be written to IGNORE fields not currently defined.
 BYTE   Mnemonic        Comments
 
 A      Author          Name of originator of message.
+B      Big message     This is a flag which indicates that the message is
+                        big, and Citadel is storing the body in a separate
+                        record.  You will never see this field because the
+                        internal API handles it.
 D      Destination     Contains name of the system this message should
                        be sent to, for mail routing (private mail only).
 E      Extended ID     A persistent alphanumeric Message ID used for