* More complex cache handling for IMAP fetch operations -- now we can

  fetch/cache "just the headers" and remember whether we did so, so we can
  burn the cache if the client then comes around and requests something
  that requires the body.  Still needs some testing and tuning.

diff --git a/citadel/ChangeLog b/citadel/ChangeLog
index 7e430d81367d0377994a5f3a53f6f5aaa481989f..5abb31730dcd1619ba5082d768574c52d3cf4cb4 100644 (file)
--- a/citadel/ChangeLog
+++ b/citadel/ChangeLog
@@ -1,4 +1,10 @@
  $Log$
+ Revision 645.15  2005/04/29 20:47:45  ajc
+ * More complex cache handling for IMAP fetch operations -- now we can
+   fetch/cache "just the headers" and remember whether we did so, so we can
+   burn the cache if the client then comes around and requests something
+   that requires the body.  Still needs some testing and tuning.
+
  Revision 645.14  2005/04/29 16:50:03  ajc
  * Significantly reduced the memory footprint of struct CitContext.
 
@@ -6640,4 +6646,3 @@ Sat Jul 11 00:20:48 EDT 1998 Nathan Bryant <bryant@cs.usm.maine.edu>
 
 Fri Jul 10 1998 Art Cancro <ajc@uncensored.citadel.org>
        * Initial CVS import
-

diff --git a/citadel/imap_fetch.c b/citadel/imap_fetch.c
index 3d59567f39c011e408533365e543fa293013e8c1..17c634cd62f3e7f9930054a0c387fadf3ab1adc3 100644 (file)
--- a/citadel/imap_fetch.c
+++ b/citadel/imap_fetch.c
@@ -120,6 +120,16 @@ void imap_fetch_rfc822(long msgnum, char *whichfmt) {
        size_t bytes_to_send = 0;
        struct MetaData smi;
        int need_to_rewrite_metadata = 0;
+       int need_body = 0;
+
+       /* Determine whether this particular fetch operation requires
+        * us to fetch the message body from disk.  If not, we can save
+        * on some disk operations...
+        */
+       if ( (!strcasecmp(whichfmt, "RFC822"))
+          || (!strcasecmp(whichfmt, "RFC822.TEXT")) ) {
+               need_body = 1;
+       }
 
        /* If this is an RFC822.SIZE fetch, first look in the message's
         * metadata record to see if we've saved that information.
@@ -131,14 +141,18 @@ void imap_fetch_rfc822(long msgnum, char *whichfmt) {
                        return;
                }
                need_to_rewrite_metadata = 1;
+               need_body = 1;
        }
        
        /* Cache the most recent RFC822 FETCH because some clients like to
         * fetch in pieces, and we don't want to have to go back to the
-        * message store for each piece.
+        * message store for each piece.  We also burn the cache if the
+        * client requests something that involves reading the message
+        * body, but we haven't fetched the body yet.
         */
        if ((IMAP->cached_rfc822_data != NULL)
-          && (IMAP->cached_rfc822_msgnum == msgnum)) {
+          && (IMAP->cached_rfc822_msgnum == msgnum)
+          && (IMAP->cached_rfc822_withbody || (!need_body)) ) {
                /* Good to go! */
        }
        else if (IMAP->cached_rfc822_data != NULL) {
@@ -159,14 +173,18 @@ void imap_fetch_rfc822(long msgnum, char *whichfmt) {
                CC->redirect_buffer = malloc(SIZ);
                CC->redirect_len = 0;
                CC->redirect_alloc = SIZ;
-               CtdlOutputMsg(msgnum, MT_RFC822, HEADERS_ALL, 0, 1);
+               CtdlOutputMsg(msgnum, MT_RFC822,
+                       (need_body ? HEADERS_ALL : HEADERS_ONLY),
+                       0, 1);
+               if (!need_body) printf("\r\n"); /* extra trailing newline */
                IMAP->cached_rfc822_data = CC->redirect_buffer;
                IMAP->cached_rfc822_len = CC->redirect_len;
                IMAP->cached_rfc822_msgnum = msgnum;
+               IMAP->cached_rfc822_withbody = need_body;
                CC->redirect_buffer = NULL;
                CC->redirect_len = 0;
                CC->redirect_alloc = 0;
-               if (need_to_rewrite_metadata) {
+               if ( (need_to_rewrite_metadata) && (IMAP->cached_rfc822_len > 0) ) {
                        smi.meta_rfc822_length = (long)IMAP->cached_rfc822_len;
                        PutMetaData(&smi);
                }
@@ -180,19 +198,26 @@ void imap_fetch_rfc822(long msgnum, char *whichfmt) {
        text_size = 0;
        total_size = 0;
 
-       ptr = IMAP->cached_rfc822_data;
-       do {
-               ptr = memreadline(ptr, buf, sizeof buf);
-               if (*ptr != 0) {
-                       striplt(buf);
-                       if (strlen(buf) == 0) {
-                               headers_size = ptr - IMAP->cached_rfc822_data;
+       if (need_body) {
+               ptr = IMAP->cached_rfc822_data;
+               do {
+                       ptr = memreadline(ptr, buf, sizeof buf);
+                       if (*ptr != 0) {
+                               striplt(buf);
+                               if (strlen(buf) == 0) {
+                                       headers_size = ptr - IMAP->cached_rfc822_data;
+                               }
                        }
-               }
-       } while ( (headers_size == 0) && (*ptr != 0) );
+               } while ( (headers_size == 0) && (*ptr != 0) );
 
-       total_size = IMAP->cached_rfc822_len;
-       text_size = total_size - headers_size;
+               total_size = IMAP->cached_rfc822_len;
+               text_size = total_size - headers_size;
+       }
+       else {
+               headers_size = IMAP->cached_rfc822_len;
+               total_size = IMAP->cached_rfc822_len;
+               text_size = 0;
+       }
 
        lprintf(CTDL_DEBUG, "RFC822: headers=%d, text=%d, total=%d\n",
                headers_size, text_size, total_size);
@@ -447,7 +472,6 @@ void imap_fetch_envelope(long msgnum, struct CtdlMessage *msg) {
        cprintf(")");
 }
 
-
 /*
  * This function is called only when CC->redirect_buffer contains a set of
  * RFC822 headers with no body attached.  Its job is to strip that set of
@@ -539,6 +563,7 @@ void imap_fetch_body(long msgnum, char *item, int is_peek) {
        int is_partial = 0;
        size_t pstart, pbytes;
        int loading_body_now = 0;
+       int need_body = 1;
 
        /* extract section */
        safestrncpy(section, item, sizeof section);
@@ -546,12 +571,16 @@ void imap_fetch_body(long msgnum, char *item, int is_peek) {
                stripallbut(section, '[', ']');
        }
        /* lprintf(CTDL_DEBUG, "Section is: %s%s\n", section, ((strlen(section)==0) ? "(empty)" : "") ); */
+       if (!strncasecmp(section, "HEADER", 6)) {
+               need_body = 0;
+       }
 
        /* Burn the cache if we don't have the same section of the 
         * same message again.
         */
        if (IMAP->cached_body != NULL) {
                if ((IMAP->cached_bodymsgnum != msgnum)
+                  || ( (IMAP->cached_body_withbody) || (!need_body) )
                   || (strcasecmp(IMAP->cached_bodypart, section)) ) {
                        free(IMAP->cached_body);
                        IMAP->cached_body_len = 0;
@@ -575,7 +604,7 @@ void imap_fetch_body(long msgnum, char *item, int is_peek) {
                CC->redirect_len = 0;
                CC->redirect_alloc = SIZ;
                loading_body_now = 1;
-               msg = CtdlFetchMessage(msgnum, 1);
+               msg = CtdlFetchMessage(msgnum, (need_body ? 1 : 0));
        }
 
        /* Now figure out what the client wants, and get it */
@@ -623,6 +652,7 @@ void imap_fetch_body(long msgnum, char *item, int is_peek) {
                IMAP->cached_body = CC->redirect_buffer;
                IMAP->cached_body_len = CC->redirect_len;
                IMAP->cached_bodymsgnum = msgnum;
+               IMAP->cached_body_withbody = need_body;
                strcpy(IMAP->cached_bodypart, section);
                CC->redirect_buffer = NULL;
                CC->redirect_len = 0;

diff --git a/citadel/msgbase.c b/citadel/msgbase.c
index 8d34b662a563cb439ac02cdd681c0e62393d7559..4fddd330ec2711a749bb3f7bd6d049c9eccfbd52 100644 (file)
--- a/citadel/msgbase.c
+++ b/citadel/msgbase.c
@@ -878,6 +878,7 @@ struct CtdlMessage *CtdlFetchMessage(long msgnum, int with_body)
         * body so other code doesn't barf.
         */
        if ( (ret->cm_fields['M'] == NULL) && (with_body) ) {
+               lprintf(CTDL_DEBUG, "** FETCHING BIG MESSAGE **\n"); /* FIXME */
                dmsgtext = cdb_fetch(CDB_BIGMSGS, &msgnum, sizeof(long));
                if (dmsgtext != NULL) {
                        ret->cm_fields['M'] = strdup(dmsgtext->ptr);
@@ -1130,11 +1131,11 @@ int CtdlOutputMsg(long msg_num,         /* message number (local) to fetch */
        /* FIXME: check message id against msglist for this room */
 
        /*
-        * Fetch the message from disk.  If we're in sooper-fast headers
+        * Fetch the message from disk.  If we're in any sort of headers
         * only mode, request that we don't even bother loading the body
         * into memory.
         */
-       if (headers_only == HEADERS_FAST) {
+       if ( (headers_only == HEADERS_FAST) || (headers_only == HEADERS_ONLY) ) {
                TheMessage = CtdlFetchMessage(msg_num, 0);
        }
        else {

diff --git a/citadel/serv_imap.c b/citadel/serv_imap.c
index c56ee75e5fc269d60443f44702c7f20b1b3b9a58..e3e1acd0cc2daa66f8363778d5f48e9a37dd83fc 100644 (file)
--- a/citadel/serv_imap.c
+++ b/citadel/serv_imap.c
@@ -91,6 +91,7 @@ void imap_free_msgids(void)
                free(IMAP->flags);
                IMAP->flags = NULL;
        }
+       IMAP->last_mtime = (-1);
 }
 
 
@@ -250,6 +251,12 @@ void imap_rescan_msgids(void)
                return;
        }
 
+       /* Check to see if the room's contents have changed.  If not, we can avoid rescan */
+       getroom(&CC->room, CC->room.QRname);
+       if (IMAP->last_mtime == CC->room.QRmtime) {     /* No changes! */
+               return;
+       }
+
        /* Load the *current* message list from disk, so we can compare it
         * to what we have in memory.
         */
@@ -339,8 +346,10 @@ void imap_rescan_msgids(void)
                cprintf("* %d RECENT\r\n", num_recent);
        }
 
-       if (num_msgs != 0)
+       if (num_msgs != 0) {
                free(msglist);
+       }
+       IMAP->last_mtime = CC->room.QRmtime;
 }
 
 
@@ -373,6 +382,7 @@ void imap_cleanup_function(void)
                free(IMAP->cached_rfc822_data);
                IMAP->cached_rfc822_data = NULL;
                IMAP->cached_rfc822_msgnum = (-1);
+               IMAP->cached_rfc822_withbody = 0;
        }
 
        if (IMAP->cached_body != NULL) {
@@ -400,6 +410,7 @@ void imap_greeting(void)
        IMAP->authstate = imap_as_normal;
        IMAP->cached_rfc822_data = NULL;
        IMAP->cached_rfc822_msgnum = (-1);
+       IMAP->cached_rfc822_withbody = 0;
 
        cprintf("* OK %s Citadel IMAP4rev1 server ready\r\n",
                config.c_fqdn);
@@ -612,6 +623,7 @@ void imap_select(int num_parms, char *parms[])
        }
 
        imap_load_msgids();
+       IMAP->last_mtime = CC->room.QRmtime;
 
        cprintf("* %d EXISTS\r\n", msgs);
        cprintf("* %d RECENT\r\n", new);

diff --git a/citadel/serv_imap.h b/citadel/serv_imap.h
index 591ac4505b0e8fe537941767231d178c71e5b8ba..ddd28a717961c0f01b079f78fb7baaf6c0856b78 100644 (file)
--- a/citadel/serv_imap.h
+++ b/citadel/serv_imap.h
@@ -17,6 +17,7 @@ struct citimap {
        int readonly;                   /* mailbox is open read only */
        int num_msgs;                   /* Number of messages being mapped */
        int num_alloc;                  /* Number of messages for which we've allocated space */
+       time_t last_mtime;              /* For checking whether the room was modified... */
        long *msgids;
        unsigned int *flags;
        char *transmitted_message;      /* for APPEND command... */
@@ -26,12 +27,14 @@ struct citimap {
        char *cached_rfc822_data;
        long cached_rfc822_msgnum;
        size_t cached_rfc822_len;
+       char cached_rfc822_withbody;    /* 1 = body cached; 0 = only headers cached */
 
        /* Cache most recent BODY FETCH because client might load in pieces */
        char *cached_body;
        size_t cached_body_len;
        char cached_bodypart[SIZ];
        long cached_bodymsgnum;
+       char cached_body_withbody;      /* 1 = body cached; 0 = only headers cached */
 };
 
 /*