Moved to new module init structure.
[citadel.git] / citadel / imap_fetch.c
index 913f5f4c08abdff48750d2b6c08dcea79c67e535..66d5b44addce658164315f1c2e25e011b42760c0 100644 (file)
@@ -38,7 +38,6 @@
 #include "citserver.h"
 #include "support.h"
 #include "config.h"
-#include "serv_extensions.h"
 #include "room_ops.h"
 #include "user_ops.h"
 #include "policy.h"
 
 
 
-struct imap_fetch_part {
-       char desired_section[SIZ];
-       FILE *output_fp;
-};
-
 /*
  * Individual field functions for imap_do_fetch_msg() ...
  */
@@ -97,6 +91,7 @@ void imap_fetch_internaldate(struct CtdlMessage *msg) {
        char buf[SIZ];
        time_t msgdate;
 
+       if (!msg) return;
        if (msg->cm_fields['T'] != NULL) {
                msgdate = atol(msg->cm_fields['T']);
        }
@@ -120,16 +115,44 @@ void imap_fetch_internaldate(struct CtdlMessage *msg) {
  */
 void imap_fetch_rfc822(long msgnum, char *whichfmt) {
        char buf[SIZ];
-       char *ptr;
+       char *ptr = NULL;
        size_t headers_size, text_size, total_size;
-       size_t bytes_to_send;
+       size_t bytes_to_send = 0;
+       struct MetaData smi;
+       int need_to_rewrite_metadata = 0;
+       int need_body = 0;
+
+       /* Determine whether this particular fetch operation requires
+        * us to fetch the message body from disk.  If not, we can save
+        * on some disk operations...
+        */
+       if ( (!strcasecmp(whichfmt, "RFC822"))
+          || (!strcasecmp(whichfmt, "RFC822.TEXT")) ) {
+               need_body = 1;
+       }
 
+       /* If this is an RFC822.SIZE fetch, first look in the message's
+        * metadata record to see if we've saved that information.
+        */
+       if (!strcasecmp(whichfmt, "RFC822.SIZE")) {
+               GetMetaData(&smi, msgnum);
+               if (smi.meta_rfc822_length > 0L) {
+                       cprintf("RFC822.SIZE %ld", smi.meta_rfc822_length);
+                       return;
+               }
+               need_to_rewrite_metadata = 1;
+               need_body = 1;
+       }
+       
        /* Cache the most recent RFC822 FETCH because some clients like to
         * fetch in pieces, and we don't want to have to go back to the
-        * message store for each piece.
+        * message store for each piece.  We also burn the cache if the
+        * client requests something that involves reading the message
+        * body, but we haven't fetched the body yet.
         */
        if ((IMAP->cached_rfc822_data != NULL)
-          && (IMAP->cached_rfc822_msgnum == msgnum)) {
+          && (IMAP->cached_rfc822_msgnum == msgnum)
+          && (IMAP->cached_rfc822_withbody || (!need_body)) ) {
                /* Good to go! */
        }
        else if (IMAP->cached_rfc822_data != NULL) {
@@ -150,13 +173,21 @@ void imap_fetch_rfc822(long msgnum, char *whichfmt) {
                CC->redirect_buffer = malloc(SIZ);
                CC->redirect_len = 0;
                CC->redirect_alloc = SIZ;
-               CtdlOutputMsg(msgnum, MT_RFC822, HEADERS_ALL, 0, 1);
+               CtdlOutputMsg(msgnum, MT_RFC822,
+                       (need_body ? HEADERS_ALL : HEADERS_ONLY),
+                       0, 1, NULL);
+               if (!need_body) cprintf("\r\n");        /* extra trailing newline */
                IMAP->cached_rfc822_data = CC->redirect_buffer;
                IMAP->cached_rfc822_len = CC->redirect_len;
                IMAP->cached_rfc822_msgnum = msgnum;
+               IMAP->cached_rfc822_withbody = need_body;
                CC->redirect_buffer = NULL;
                CC->redirect_len = 0;
                CC->redirect_alloc = 0;
+               if ( (need_to_rewrite_metadata) && (IMAP->cached_rfc822_len > 0) ) {
+                       smi.meta_rfc822_length = (long)IMAP->cached_rfc822_len;
+                       PutMetaData(&smi);
+               }
        }
 
        /*
@@ -167,19 +198,26 @@ void imap_fetch_rfc822(long msgnum, char *whichfmt) {
        text_size = 0;
        total_size = 0;
 
-       ptr = IMAP->cached_rfc822_data;
-       do {
-               ptr = memreadline(ptr, buf, sizeof buf);
-               if (*ptr != 0) {
-                       striplt(buf);
-                       if (strlen(buf) == 0) {
-                               headers_size = ptr - IMAP->cached_rfc822_data;
+       if (need_body) {
+               ptr = IMAP->cached_rfc822_data;
+               do {
+                       ptr = memreadline(ptr, buf, sizeof buf);
+                       if (*ptr != 0) {
+                               striplt(buf);
+                               if (strlen(buf) == 0) {
+                                       headers_size = ptr - IMAP->cached_rfc822_data;
+                               }
                        }
-               }
-       } while ( (headers_size == 0) && (*ptr != 0) );
+               } while ( (headers_size == 0) && (*ptr != 0) );
 
-       total_size = IMAP->cached_rfc822_len;
-       text_size = total_size - headers_size;
+               total_size = IMAP->cached_rfc822_len;
+               text_size = total_size - headers_size;
+       }
+       else {
+               headers_size = IMAP->cached_rfc822_len;
+               total_size = IMAP->cached_rfc822_len;
+               text_size = 0;
+       }
 
        lprintf(CTDL_DEBUG, "RFC822: headers=%d, text=%d, total=%d\n",
                headers_size, text_size, total_size);
@@ -213,46 +251,44 @@ void imap_fetch_rfc822(long msgnum, char *whichfmt) {
 /*
  * Load a specific part of a message into the temp file to be output to a
  * client.  FIXME we can handle parts like "2" and "2.1" and even "2.MIME"
- * but we still can't handle "2.HEADER" (which might not be a problem, because
- * we currently don't have the ability to break out nested RFC822's anyway).
+ * but we still can't handle "2.HEADER" (which might not be a problem).
  *
  * Note: mime_parser() was called with dont_decode set to 1, so we have the
  * luxury of simply spewing without having to re-encode.
  */
 void imap_load_part(char *name, char *filename, char *partnum, char *disp,
-                   void *content, char *cbtype, size_t length, char *encoding,
+                   void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
                    void *cbuserdata)
 {
-       struct imap_fetch_part *imfp;
        char mbuf2[SIZ];
+       char *desired_section;
 
-       imfp = (struct imap_fetch_part *)cbuserdata;
+       desired_section = (char *)cbuserdata;
 
-       if (!strcasecmp(partnum, imfp->desired_section)) {
-               fwrite(content, length, 1, imfp->output_fp);
+       if (!strcasecmp(partnum, desired_section)) {
+               client_write(content, length);
        }
 
        snprintf(mbuf2, sizeof mbuf2, "%s.MIME", partnum);
 
-       if (!strcasecmp(imfp->desired_section, mbuf2)) {
-               fprintf(imfp->output_fp, "Content-type: %s", cbtype);
+       if (!strcasecmp(desired_section, mbuf2)) {
+               cprintf("Content-type: %s", cbtype);
+               if (strlen(cbcharset) > 0)
+                       cprintf("; charset=\"%s\"", cbcharset);
                if (strlen(name) > 0)
-                       fprintf(imfp->output_fp, "; name=\"%s\"", name);
-               fprintf(imfp->output_fp, "\r\n");
+                       cprintf("; name=\"%s\"", name);
+               cprintf("\r\n");
                if (strlen(encoding) > 0)
-                       fprintf(imfp->output_fp,
-                               "Content-Transfer-Encoding: %s\r\n", encoding);
+                       cprintf("Content-Transfer-Encoding: %s\r\n", encoding);
                if (strlen(encoding) > 0) {
-                       fprintf(imfp->output_fp, "Content-Disposition: %s",
-                                       disp);
+                       cprintf("Content-Disposition: %s", disp);
                        if (strlen(filename) > 0) {
-                               fprintf(imfp->output_fp, "; filename=\"%s\"",
-                                       filename);
+                               cprintf("; filename=\"%s\"", filename);
                        }
-                       fprintf(imfp->output_fp, "\r\n");
+                       cprintf("\r\n");
                }
-               fprintf(imfp->output_fp, "Content-Length: %ld\r\n", (long)length);
-               fprintf(imfp->output_fp, "\r\n");
+               cprintf("Content-Length: %ld\r\n", (long)length);
+               cprintf("\r\n");
        }
                        
 
@@ -267,6 +303,8 @@ void imap_load_part(char *name, char *filename, char *partnum, char *disp,
 void imap_output_envelope_from(struct CtdlMessage *msg) {
        char user[SIZ], node[SIZ], name[SIZ];
 
+       if (!msg) return;
+
        /* For anonymous messages, it's so easy! */
        if (!is_room_aide() && (msg->cm_anon_type == MES_ANONONLY)) {
                cprintf("((\"----\" NIL \"x\" \"x.org\")) ");
@@ -312,12 +350,12 @@ void imap_output_envelope_from(struct CtdlMessage *msg) {
  * fields.  But we can use it for "To" and possibly others.
  */
 void imap_output_envelope_addr(char *addr) {
-       char individual_addr[SIZ];
+       char individual_addr[256];
        int num_addrs;
        int i;
-       char user[SIZ];
-       char node[SIZ];
-       char name[SIZ];
+       char user[256];
+       char node[256];
+       char name[256];
 
        if (addr == NULL) {
                cprintf("NIL ");
@@ -336,7 +374,7 @@ void imap_output_envelope_addr(char *addr) {
 
        /* Output them one by one. */
        for (i=0; i<num_addrs; ++i) {
-               extract_token(individual_addr, addr, i, ',');
+               extract_token(individual_addr, addr, i, ',', sizeof individual_addr);
                striplt(individual_addr);
                process_rfc822_addr(individual_addr, user, node, name);
                cprintf("(");
@@ -359,11 +397,13 @@ void imap_output_envelope_addr(char *addr) {
  * Note that the imap_strout() function can cleverly output NULL fields as NIL,
  * so we don't have to check for that condition like we do elsewhere.
  */
-void imap_fetch_envelope(long msgnum, struct CtdlMessage *msg) {
+void imap_fetch_envelope(struct CtdlMessage *msg) {
        char datestringbuf[SIZ];
        time_t msgdate;
        char *fieldptr = NULL;
 
+       if (!msg) return;
+
        /* Parse the message date into an IMAP-format date string */
        if (msg->cm_fields['T'] != NULL) {
                msgdate = atol(msg->cm_fields['T']);
@@ -415,10 +455,16 @@ void imap_fetch_envelope(long msgnum, struct CtdlMessage *msg) {
        /* To */
        imap_output_envelope_addr(msg->cm_fields['R']);
 
-       /* Cc */
-       fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "Cc");
-       imap_output_envelope_addr(fieldptr);
-       if (fieldptr != NULL) free(fieldptr);
+       /* Cc (we do it this way because there might be a legacy non-Citadel Cc: field present) */
+       fieldptr = msg->cm_fields['Y'];
+       if (fieldptr != NULL) {
+               imap_output_envelope_addr(fieldptr);
+       }
+       else {
+               fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "Cc");
+               imap_output_envelope_addr(fieldptr);
+               if (fieldptr != NULL) free(fieldptr);
+       }
 
        /* Bcc */
        fieldptr = rfc822_fetch_field(msg->cm_fields['M'], "Bcc");
@@ -437,12 +483,12 @@ void imap_fetch_envelope(long msgnum, struct CtdlMessage *msg) {
        cprintf(")");
 }
 
-
 /*
- * Strip any non header information out of a chunk of RFC822 data on disk,
- * then boil it down to just the fields we want.
+ * This function is called only when CC->redirect_buffer contains a set of
+ * RFC822 headers with no body attached.  Its job is to strip that set of
+ * headers down to *only* the ones we're interested in.
  */
-void imap_strip_headers(FILE *fp, char *section) {
+void imap_strip_headers(char *section) {
        char buf[SIZ];
        char *which_fields = NULL;
        int doing_headers = 0;
@@ -453,6 +499,9 @@ void imap_strip_headers(FILE *fp, char *section) {
        char *boiled_headers = NULL;
        int ok = 0;
        int done_headers = 0;
+       char *ptr = NULL;
+
+       if (CC->redirect_buffer == NULL) return;
 
        which_fields = strdup(section);
 
@@ -471,13 +520,12 @@ void imap_strip_headers(FILE *fp, char *section) {
        }
        num_parms = imap_parameterize(parms, which_fields);
 
-       fseek(fp, 0L, SEEK_END);
-       boiled_headers = malloc((size_t)(ftell(fp) + 256L));
+       boiled_headers = malloc(CC->redirect_alloc);
        strcpy(boiled_headers, "");
 
-       rewind(fp);
+       ptr = CC->redirect_buffer;
        ok = 0;
-       while ( (done_headers == 0) && (fgets(buf, sizeof buf, fp) != NULL) ) {
+       while ( (done_headers == 0) && (ptr = memreadline(ptr, buf, sizeof buf), *ptr != 0) ) {
                if (!isspace(buf[0])) {
                        ok = 0;
                        if (doing_headers == 0) ok = 1;
@@ -497,6 +545,7 @@ void imap_strip_headers(FILE *fp, char *section) {
 
                if (ok) {
                        strcat(boiled_headers, buf);
+                       strcat(boiled_headers, "\r\n");
                }
 
                if (strlen(buf) == 0) done_headers = 1;
@@ -506,13 +555,10 @@ void imap_strip_headers(FILE *fp, char *section) {
 
        strcat(boiled_headers, "\r\n");
 
-       /* Now write it back */
-       rewind(fp);
-       fwrite(boiled_headers, strlen(boiled_headers), 1, fp);
-       fflush(fp);
-       ftruncate(fileno(fp), ftell(fp));
-       fflush(fp);
-       rewind(fp);
+       /* Now save it back (it'll always be smaller) */
+       strcpy(CC->redirect_buffer, boiled_headers);
+       CC->redirect_len = strlen(boiled_headers);
+
        free(which_fields);
        free(boiled_headers);
 }
@@ -526,27 +572,38 @@ void imap_fetch_body(long msgnum, char *item, int is_peek) {
        char section[SIZ];
        char partial[SIZ];
        int is_partial = 0;
-       char buf[SIZ];
-       FILE *tmp = NULL;
-       long bytes_remaining = 0;
-       long blocksize;
-       long pstart, pbytes;
-       struct imap_fetch_part imfp;
+       size_t pstart, pbytes;
+       int loading_body_now = 0;
+       int need_body = 1;
+       int burn_the_cache = 0;
 
        /* extract section */
        safestrncpy(section, item, sizeof section);
        if (strchr(section, '[') != NULL) {
                stripallbut(section, '[', ']');
        }
-       /* lprintf(CTDL_DEBUG, "Section is: %s%s\n", section, ((strlen(section)==0) ? "(empty)" : "") ); */
+       lprintf(CTDL_DEBUG, "Section is: %s%s\n", section, ((strlen(section)==0) ? "(empty)" : "") );
+       if (!strncasecmp(section, "HEADER", 6)) {
+               need_body = 0;
+       }
 
        /* Burn the cache if we don't have the same section of the 
         * same message again.
         */
        if (IMAP->cached_body != NULL) {
-               if ((IMAP->cached_bodymsgnum != msgnum)
-                  || (strcasecmp(IMAP->cached_bodypart, section)) ) {
-                       fclose(IMAP->cached_body);
+               if (IMAP->cached_bodymsgnum != msgnum) {
+                       burn_the_cache = 1;
+               }
+               else if ( (!IMAP->cached_body_withbody) && (need_body) ) {
+                       burn_the_cache = 1;
+               }
+               else if (strcasecmp(IMAP->cached_bodypart, section)) {
+                       burn_the_cache = 1;
+               }
+               if (burn_the_cache) {
+                       /* Yup, go ahead and burn the cache. */
+                       free(IMAP->cached_body);
+                       IMAP->cached_body_len = 0;
                        IMAP->cached_body = NULL;
                        IMAP->cached_bodymsgnum = (-1);
                        strcpy(IMAP->cached_bodypart, "");
@@ -563,30 +620,25 @@ void imap_fetch_body(long msgnum, char *item, int is_peek) {
        /* if (strlen(partial) > 0) lprintf(CTDL_DEBUG, "Partial is %s\n", partial); */
 
        if (IMAP->cached_body == NULL) {
-               tmp = tmpfile();
-               if (tmp == NULL) {
-                       lprintf(CTDL_CRIT, "Cannot open temp file: %s\n", strerror(errno));
-                       return;
-               }
-               msg = CtdlFetchMessage(msgnum, 1);
+               CC->redirect_buffer = malloc(SIZ);
+               CC->redirect_len = 0;
+               CC->redirect_alloc = SIZ;
+               loading_body_now = 1;
+               msg = CtdlFetchMessage(msgnum, (need_body ? 1 : 0));
        }
 
        /* Now figure out what the client wants, and get it */
 
-       if (IMAP->cached_body != NULL) {
-               tmp = IMAP->cached_body;
+       if (!loading_body_now) {
+               /* What we want is already in memory */
        }
+
        else if ( (!strcmp(section, "1")) && (msg->cm_format_type != 4) ) {
-               CtdlRedirectOutput(tmp);
-               CtdlOutputPreLoadedMsg(msg, msgnum, MT_RFC822,
-                                               HEADERS_NONE, 0, 1);
-               CtdlRedirectOutput(NULL);
+               CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_NONE, 0, 1);
        }
 
        else if (!strcmp(section, "")) {
-               CtdlRedirectOutput(tmp);
-               CtdlOutputPreLoadedMsg(msg, msgnum, MT_RFC822, HEADERS_ALL, 0, 1);
-               CtdlRedirectOutput(NULL);
+               CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1);
        }
 
        /*
@@ -594,19 +646,15 @@ void imap_fetch_body(long msgnum, char *item, int is_peek) {
         * fields, strip it down.
         */
        else if (!strncasecmp(section, "HEADER", 6)) {
-               CtdlRedirectOutput(tmp);
-               CtdlOutputPreLoadedMsg(msg, msgnum, MT_RFC822, HEADERS_ONLY, 0, 1);
-               CtdlRedirectOutput(NULL);
-               imap_strip_headers(tmp, section);
+               CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ONLY, 0, 1);
+               imap_strip_headers(section);
        }
 
        /*
         * Strip it down if the client asked for everything _except_ headers.
         */
        else if (!strncasecmp(section, "TEXT", 4)) {
-               CtdlRedirectOutput(tmp);
-               CtdlOutputPreLoadedMsg(msg, msgnum, MT_RFC822, HEADERS_NONE, 0, 1);
-               CtdlRedirectOutput(NULL);
+               CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_NONE, 0, 1);
        }
 
        /*
@@ -614,46 +662,38 @@ void imap_fetch_body(long msgnum, char *item, int is_peek) {
         * (Note value of 1 passed as 'dont_decode' so client gets it encoded)
         */
        else {
-               safestrncpy(imfp.desired_section, section, sizeof(imfp.desired_section));
-               imfp.output_fp = tmp;
                mime_parser(msg->cm_fields['M'], NULL,
                                *imap_load_part, NULL, NULL,
-                               (void *)&imfp,
+                               section,
                                1);
        }
 
-
-       fseek(tmp, 0L, SEEK_END);
-       bytes_remaining = ftell(tmp);
+       if (loading_body_now) {
+               IMAP->cached_body = CC->redirect_buffer;
+               IMAP->cached_body_len = CC->redirect_len;
+               IMAP->cached_bodymsgnum = msgnum;
+               IMAP->cached_body_withbody = need_body;
+               strcpy(IMAP->cached_bodypart, section);
+               CC->redirect_buffer = NULL;
+               CC->redirect_len = 0;
+               CC->redirect_alloc = 0;
+       }
 
        if (is_partial == 0) {
-               rewind(tmp);
-               cprintf("BODY[%s] {%ld}\r\n", section, bytes_remaining);
+               cprintf("BODY[%s] {%d}\r\n", section, IMAP->cached_body_len);
+               pstart = 0;
+               pbytes = IMAP->cached_body_len;
        }
        else {
-               sscanf(partial, "%ld.%ld", &pstart, &pbytes);
-               if ((bytes_remaining - pstart) < pbytes) {
-                       pbytes = bytes_remaining - pstart;
+               sscanf(partial, "%d.%d", &pstart, &pbytes);
+               if (pbytes > (IMAP->cached_body_len - pstart)) {
+                       pbytes = IMAP->cached_body_len - pstart;
                }
-               fseek(tmp, pstart, SEEK_SET);
-               bytes_remaining = pbytes;
-               cprintf("BODY[%s]<%ld> {%ld}\r\n",
-                       section, pstart, bytes_remaining);
-       }
-
-       blocksize = (long)sizeof(buf);
-       while (bytes_remaining > 0L) {
-               if (blocksize > bytes_remaining) blocksize = bytes_remaining;
-               fread(buf, blocksize, 1, tmp);
-               client_write(buf, (int)blocksize);
-               bytes_remaining = bytes_remaining - blocksize;
+               cprintf("BODY[%s]<%d> {%d}\r\n", section, pstart, pbytes);
        }
 
-       /* Don't close it ... cache it! */
-       /* fclose(tmp); */
-       IMAP->cached_body = tmp;
-       IMAP->cached_bodymsgnum = msgnum;
-       strcpy(IMAP->cached_bodypart, section);
+       /* Here we go -- output it */
+       client_write(&IMAP->cached_body[pstart], pbytes);
 
        if (msg != NULL) {
                CtdlFreeMessage(msg);
@@ -661,7 +701,7 @@ void imap_fetch_body(long msgnum, char *item, int is_peek) {
 
        /* Mark this message as "seen" *unless* this is a "peek" operation */
        if (is_peek == 0) {
-               CtdlSetSeen(msgnum, 1, ctdlsetseen_seen);
+               CtdlSetSeen(&msgnum, 1, 1, ctdlsetseen_seen, NULL, NULL);
        }
 }
 
@@ -670,7 +710,7 @@ void imap_fetch_body(long msgnum, char *item, int is_peek) {
  */
 void imap_fetch_bodystructure_pre(
                char *name, char *filename, char *partnum, char *disp,
-               void *content, char *cbtype, size_t length, char *encoding,
+               void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
                void *cbuserdata
                ) {
 
@@ -684,16 +724,16 @@ void imap_fetch_bodystructure_pre(
  */
 void imap_fetch_bodystructure_post(
                char *name, char *filename, char *partnum, char *disp,
-               void *content, char *cbtype, size_t length, char *encoding,
+               void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
                void *cbuserdata
                ) {
 
-       char subtype[SIZ];
+       char subtype[128];
 
        cprintf(" ");
 
        /* disposition */
-       extract_token(subtype, cbtype, 1, '/');
+       extract_token(subtype, cbtype, 1, '/', sizeof subtype);
        imap_strout(subtype);
 
        /* body language */
@@ -710,7 +750,7 @@ void imap_fetch_bodystructure_post(
  */
 void imap_fetch_bodystructure_part(
                char *name, char *filename, char *partnum, char *disp,
-               void *content, char *cbtype, size_t length, char *encoding,
+               void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
                void *cbuserdata
                ) {
 
@@ -718,13 +758,13 @@ void imap_fetch_bodystructure_part(
        int have_encoding = 0;
        int lines = 0;
        size_t i;
-       char cbmaintype[SIZ];
-       char cbsubtype[SIZ];
+       char cbmaintype[128];
+       char cbsubtype[128];
 
        if (cbtype != NULL) if (strlen(cbtype)>0) have_cbtype = 1;
        if (have_cbtype) {
-               extract_token(cbmaintype, cbtype, 0, '/');
-               extract_token(cbsubtype, cbtype, 1, '/');
+               extract_token(cbmaintype, cbtype, 0, '/', sizeof cbmaintype);
+               extract_token(cbsubtype, cbtype, 1, '/', sizeof cbsubtype);
        }
        else {
                strcpy(cbmaintype, "TEXT");
@@ -737,7 +777,16 @@ void imap_fetch_bodystructure_part(
        imap_strout(cbsubtype);
        cprintf(" ");
 
-       cprintf("(\"CHARSET\" \"US-ASCII\"");
+       if (cbcharset == NULL) {
+               cprintf("(\"CHARSET\" \"US-ASCII\"");
+       }
+       else if (strlen(cbcharset) == 0) {
+               cprintf("(\"CHARSET\" \"US-ASCII\"");
+       }
+       else {
+               cprintf("(\"CHARSET\" ");
+               imap_strout(cbcharset);
+       }
 
        if (name != NULL) if (strlen(name)>0) {
                cprintf(" \"NAME\" ");
@@ -824,6 +873,14 @@ void imap_fetch_bodystructure (long msgnum, char *item,
        char buf[SIZ];
        int lines = 0;
 
+       /* Handle NULL message gracefully */
+       if (msg == NULL) {
+               cprintf("BODYSTRUCTURE (\"TEXT\" \"PLAIN\" "
+                       "(\"CHARSET\" \"US-ASCII\") NIL NIL "
+                       "\"7BIT\" 0 0)");
+               return;
+       }
+
        /* For non-RFC822 (ordinary Citadel) messages, this is short and
         * sweet...
         */
@@ -836,7 +893,7 @@ void imap_fetch_bodystructure (long msgnum, char *item,
                CC->redirect_buffer = malloc(SIZ);
                CC->redirect_len = 0;
                CC->redirect_alloc = SIZ;
-               CtdlOutputPreLoadedMsg(msg, msgnum, MT_RFC822, 0, 0, 1);
+               CtdlOutputPreLoadedMsg(msg, MT_RFC822, 0, 0, 1);
                rfc822 = CC->redirect_buffer;
                rfc822_len = CC->redirect_len;
                CC->redirect_buffer = NULL;
@@ -881,7 +938,13 @@ void imap_fetch_bodystructure (long msgnum, char *item,
 void imap_do_fetch_msg(int seq, int num_items, char **itemlist) {
        int i;
        struct CtdlMessage *msg = NULL;
+       int body_loaded = 0;
 
+       /* Don't attempt to fetch bogus messages or UID's */
+       if (seq < 1) return;
+       if (IMAP->msgids[seq-1] < 1L) return;
+
+       buffer_output();
        cprintf("* %d FETCH (", seq);
 
        for (i=0; i<num_items; ++i) {
@@ -921,16 +984,29 @@ void imap_do_fetch_msg(int seq, int num_items, char **itemlist) {
                /* Otherwise, load the message into memory.
                 */
                else if (!strcasecmp(itemlist[i], "BODYSTRUCTURE")) {
-                       if (msg == NULL) msg = CtdlFetchMessage(IMAP->msgids[seq-1], 1);
+                       if ((msg != NULL) && (!body_loaded)) {
+                               CtdlFreeMessage(msg);   /* need the whole thing */
+                               msg = NULL;
+                       }
+                       if (msg == NULL) {
+                               msg = CtdlFetchMessage(IMAP->msgids[seq-1], 1);
+                               body_loaded = 1;
+                       }
                        imap_fetch_bodystructure(IMAP->msgids[seq-1],
                                        itemlist[i], msg);
                }
                else if (!strcasecmp(itemlist[i], "ENVELOPE")) {
-                       if (msg == NULL) msg = CtdlFetchMessage(IMAP->msgids[seq-1], 1);
-                       imap_fetch_envelope(IMAP->msgids[seq-1], msg);
+                       if (msg == NULL) {
+                               msg = CtdlFetchMessage(IMAP->msgids[seq-1], 0);
+                               body_loaded = 0;
+                       }
+                       imap_fetch_envelope(msg);
                }
                else if (!strcasecmp(itemlist[i], "INTERNALDATE")) {
-                       if (msg == NULL) msg = CtdlFetchMessage(IMAP->msgids[seq-1], 1);
+                       if (msg == NULL) {
+                               msg = CtdlFetchMessage(IMAP->msgids[seq-1], 0);
+                               body_loaded = 0;
+                       }
                        imap_fetch_internaldate(msg);
                }
 
@@ -938,6 +1014,7 @@ void imap_do_fetch_msg(int seq, int num_items, char **itemlist) {
        }
 
        cprintf(")\r\n");
+       unbuffer_output();
        if (msg != NULL) {
                CtdlFreeMessage(msg);
        }
@@ -954,6 +1031,15 @@ void imap_do_fetch(int num_items, char **itemlist) {
 
        if (IMAP->num_msgs > 0) {
                for (i = 0; i < IMAP->num_msgs; ++i) {
+
+                       /* Abort the fetch loop if the session breaks.
+                        * This is important for users who keep mailboxes
+                        * that are too big *and* are too impatient to
+                        * let them finish loading.  :)
+                        */
+                       if (CC->kill_me) return;
+
+                       /* Get any message marked for fetch. */
                        if (IMAP->flags[i] & IMAP_SELECTED) {
                                imap_do_fetch_msg(i+1, num_items, itemlist);
                        }
@@ -1132,15 +1218,15 @@ void imap_pick_range(char *supplied_range, int is_uid) {
         */
        num_sets = num_tokens(actual_range, ',');
        for (s=0; s<num_sets; ++s) {
-               extract_token(setstr, actual_range, s, ',');
+               extract_token(setstr, actual_range, s, ',', sizeof setstr);
 
-               extract_token(lostr, setstr, 0, ':');
+               extract_token(lostr, setstr, 0, ':', sizeof lostr);
                if (num_tokens(setstr, ':') >= 2) {
-                       extract_token(histr, setstr, 1, ':');
+                       extract_token(histr, setstr, 1, ':', sizeof histr);
                        if (!strcmp(histr, "*")) snprintf(histr, sizeof histr, "%ld", LONG_MAX);
                } 
                else {
-                       strcpy(histr, lostr);
+                       safestrncpy(histr, lostr, sizeof histr);
                }
                lo = atol(lostr);
                hi = atol(histr);
@@ -1150,14 +1236,12 @@ void imap_pick_range(char *supplied_range, int is_uid) {
                        if (is_uid) {   /* fetch by sequence number */
                                if ( (IMAP->msgids[i-1]>=lo)
                                   && (IMAP->msgids[i-1]<=hi)) {
-                                       IMAP->flags[i-1] =
-                                               IMAP->flags[i-1] | IMAP_SELECTED;
+                                       IMAP->flags[i-1] |= IMAP_SELECTED;
                                }
                        }
                        else {          /* fetch by uid */
                                if ( (i>=lo) && (i<=hi)) {
-                                       IMAP->flags[i-1] =
-                                               IMAP->flags[i-1] | IMAP_SELECTED;
+                                       IMAP->flags[i-1] |= IMAP_SELECTED;
                                }
                        }
                }
@@ -1172,7 +1256,7 @@ void imap_pick_range(char *supplied_range, int is_uid) {
  */
 void imap_fetch(int num_parms, char *parms[]) {
        char items[SIZ];
-       char *itemlist[SIZ];
+       char *itemlist[512];
        int num_items;
        int i;
 
@@ -1204,7 +1288,7 @@ void imap_fetch(int num_parms, char *parms[]) {
  */
 void imap_uidfetch(int num_parms, char *parms[]) {
        char items[SIZ];
-       char *itemlist[SIZ];
+       char *itemlist[512];
        int num_items;
        int i;
        int have_uid_item = 0;
@@ -1229,12 +1313,16 @@ void imap_uidfetch(int num_parms, char *parms[]) {
        }
 
        /* If the "UID" item was not included, we include it implicitly
-        * because this is a UID FETCH command
+        * (at the beginning) because this is a UID FETCH command
         */
        for (i=0; i<num_items; ++i) {
                if (!strcasecmp(itemlist[i], "UID")) ++have_uid_item;
        }
-       if (have_uid_item == 0) itemlist[num_items++] = "UID";
+       if (have_uid_item == 0) {
+               memmove(&itemlist[1], &itemlist[0], (sizeof(itemlist[0]) * num_items));
+               ++num_items;
+               itemlist[0] = "UID";
+       }
 
        imap_do_fetch(num_items, itemlist);
        cprintf("%s OK UID FETCH completed\r\n", parms[0]);