]> code.citadel.org Git - citadel.git/blobdiff - citadel/msgbase.c
* make QP encoding of senders optional, so we can flag it if we need it.
[citadel.git] / citadel / msgbase.c
index e8e51eb935e8ace1ce68590a8707425f63259758..28601e5bebd4b122e4353fac637fd15b78012c26 100644 (file)
@@ -5,10 +5,6 @@
  *
  */
 
-#ifdef DLL_EXPORT
-#define IN_LIBCIT
-#endif
-
 #include "sysdep.h"
 #include <stdlib.h>
 #include <unistd.h>
@@ -33,6 +29,9 @@
 #include <errno.h>
 #include <stdarg.h>
 #include <sys/stat.h>
+#include <sys/types.h>
+#include <regex.h>
+#include <libcitadel.h>
 #include "citadel.h"
 #include "server.h"
 #include "serv_extensions.h"
 #include "room_ops.h"
 #include "user_ops.h"
 #include "file_ops.h"
+#include "config.h"
 #include "control.h"
-#include "tools.h"
-#include "mime_parser.h"
-#include "html.h"
 #include "genstamp.h"
 #include "internet_addressing.h"
+#include "euidindex.h"
+#include "journaling.h"
+#include "citadel_dirs.h"
+#include "clientsocket.h"
+#include "serv_network.h"
+#include "threads.h"
 
-#define desired_section ((char *)CtdlGetUserData(SYM_DESIRED_SECTION))
-#define ma ((struct ma_info *)CtdlGetUserData(SYM_MA_INFO))
-#define msg_repl ((struct repl *)CtdlGetUserData(SYM_REPL))
-
-extern struct config config;
 long config_msgnum;
+struct addresses_to_be_filed *atbf = NULL;
 
+/* This temp file holds the queue of operations for AdjRefCount() */
+static FILE *arcfp = NULL;
 
 /* 
  * This really belongs in serv_network.c, but I don't know how to export
@@ -87,7 +88,8 @@ char *msgkeys[] = {
        NULL, 
        "hnod",
        "msgn",
-       NULL, NULL, NULL,
+       "jrnl",
+       NULL, NULL,
        "text",
        "node",
        "room",
@@ -98,9 +100,9 @@ char *msgkeys[] = {
        "time",
        "subj",
        NULL,
+       "wefw",
        NULL,
-       NULL,
-       NULL,
+       "cccc",
        NULL
 };
 
@@ -138,14 +140,18 @@ int alias(char *name)
        char *ignetcfg = NULL;
        char *ignetmap = NULL;
        int at = 0;
-       char node[SIZ];
-       char testnode[SIZ];
+       char node[64];
+       char testnode[64];
        char buf[SIZ];
 
+       char original_name[256];
+       safestrncpy(original_name, name, sizeof original_name);
+
        striplt(name);
        remove_any_whitespace_to_the_left_or_right_of_at_symbol(name);
+       stripallbut(name, '<', '>');
 
-       fp = fopen("network/mail.aliases", "r");
+       fp = fopen(file_mail_aliases, "r");
        if (fp == NULL) {
                fp = fopen("/dev/null", "r");
        }
@@ -171,18 +177,20 @@ int alias(char *name)
        fclose(fp);
 
        /* Hit the Global Address Book */
-       if (CtdlDirectoryLookup(aaa, name) == 0) {
+       if (CtdlDirectoryLookup(aaa, name, sizeof aaa) == 0) {
                strcpy(name, aaa);
        }
 
-       lprintf(CTDL_INFO, "Mail is being forwarded to %s\n", name);
+       if (strcasecmp(original_name, name)) {
+               CtdlLogPrintf(CTDL_INFO, "%s is being forwarded to %s\n", original_name, name);
+       }
 
        /* Change "user @ xxx" to "user" if xxx is an alias for this host */
        for (a=0; a<strlen(name); ++a) {
                if (name[a] == '@') {
                        if (CtdlHostAlias(&name[a+1]) == hostalias_localhost) {
                                name[a] = 0;
-                               lprintf(CTDL_INFO, "Changed to <%s>\n", name);
+                               CtdlLogPrintf(CTDL_INFO, "Changed to <%s>\n", name);
                        }
                }
        }
@@ -194,7 +202,7 @@ int alias(char *name)
        remove_any_whitespace_to_the_left_or_right_of_at_symbol(name);
 
        /* figure out the delivery mode */
-       extract_token(node, name, 1, '@');
+       extract_token(node, name, 1, '@', sizeof node);
 
        /* If there are one or more dots in the nodename, we assume that it
         * is an FQDN and will attempt SMTP delivery to the Internet.
@@ -208,8 +216,8 @@ int alias(char *name)
         */
        ignetcfg = CtdlGetSysConfig(IGNETCFG);
        for (i=0; i<num_tokens(ignetcfg, '\n'); ++i) {
-               extract_token(buf, ignetcfg, i, '\n');
-               extract_token(testnode, buf, 0, '|');
+               extract_token(buf, ignetcfg, i, '\n', sizeof buf);
+               extract_token(testnode, buf, 0, '|', sizeof testnode);
                if (!strcasecmp(node, testnode)) {
                        free(ignetcfg);
                        return(MES_IGNET);
@@ -222,8 +230,8 @@ int alias(char *name)
         */
        ignetmap = CtdlGetSysConfig(IGNETMAP);
        for (i=0; i<num_tokens(ignetmap, '\n'); ++i) {
-               extract_token(buf, ignetmap, i, '\n');
-               extract_token(testnode, buf, 0, '|');
+               extract_token(buf, ignetmap, i, '\n', sizeof buf);
+               extract_token(testnode, buf, 0, '|', sizeof testnode);
                if (!strcasecmp(node, testnode)) {
                        free(ignetmap);
                        return(MES_IGNET);
@@ -236,25 +244,38 @@ int alias(char *name)
 }
 
 
-void get_mm(void)
+/*
+ * Back end for the MSGS command: output message number only.
+ */
+void simple_listing(long msgnum, void *userdata)
 {
-       FILE *fp;
-
-       fp = fopen("citadel.control", "r");
-       if (fp == NULL) {
-               lprintf(CTDL_CRIT, "Cannot open citadel.control: %s\n",
-                       strerror(errno));
-               exit(errno);
-       }
-       fread((char *) &CitControl, sizeof(struct CitControl), 1, fp);
-       fclose(fp);
+       cprintf("%ld\n", msgnum);
 }
 
 
 
-void simple_listing(long msgnum, void *userdata)
+/*
+ * Back end for the MSGS command: output header summary.
+ */
+void headers_listing(long msgnum, void *userdata)
 {
-       cprintf("%ld\n", msgnum);
+       struct CtdlMessage *msg;
+
+       msg = CtdlFetchMessage(msgnum, 0);
+       if (msg == NULL) {
+               cprintf("%ld|0|||||\n", msgnum);
+               return;
+       }
+
+       cprintf("%ld|%s|%s|%s|%s|%s|\n",
+               msgnum,
+               (msg->cm_fields['T'] ? msg->cm_fields['T'] : "0"),
+               (msg->cm_fields['A'] ? msg->cm_fields['A'] : ""),
+               (msg->cm_fields['N'] ? msg->cm_fields['N'] : ""),
+               (msg->cm_fields['F'] ? msg->cm_fields['F'] : ""),
+               (msg->cm_fields['U'] ? msg->cm_fields['U'] : "")
+       );
+       CtdlFreeMessage(msg);
 }
 
 
@@ -309,78 +330,145 @@ void CtdlGetSeen(char *buf, int which_set) {
 /*
  * Manipulate the "seen msgs" string (or other message set strings)
  */
-void CtdlSetSeen(long target_msgnum, int target_setting, int which_set) {
-       char newseen[SIZ];
+void CtdlSetSeen(long *target_msgnums, int num_target_msgnums,
+               int target_setting, int which_set,
+               struct ctdluser *which_user, struct ctdlroom *which_room) {
        struct cdbdata *cdbfr;
-       int i;
+       int i, j, k;
        int is_seen = 0;
-       int was_seen = 1;
+       int was_seen = 0;
        long lo = (-1L);
        long hi = (-1L);
+       long t = (-1L);
+       int trimming = 0;
        struct visit vbuf;
        long *msglist;
        int num_msgs = 0;
        char vset[SIZ];
+       char *is_set;   /* actually an array of booleans */
+       int num_sets;
+       int s;
+       char setstr[SIZ], lostr[SIZ], histr[SIZ];
+       size_t tmp;
+
+       /* Don't bother doing *anything* if we were passed a list of zero messages */
+       if (num_target_msgnums < 1) {
+               return;
+       }
 
-       lprintf(CTDL_DEBUG, "CtdlSetSeen(%ld, %d, %d)\n",
-               target_msgnum, target_setting, which_set);
+       CtdlLogPrintf(CTDL_DEBUG, "CtdlSetSeen(%d msgs starting with %ld, %d, %d)\n",
+               num_target_msgnums, target_msgnums[0],
+               target_setting, which_set);
 
        /* Learn about the user and room in question */
-       CtdlGetRelationship(&vbuf, &CC->user, &CC->room);
+       CtdlGetRelationship(&vbuf,
+               ((which_user != NULL) ? which_user : &CC->user),
+               ((which_room != NULL) ? which_room : &CC->room)
+       );
 
        /* Load the message list */
        cdbfr = cdb_fetch(CDB_MSGLISTS, &CC->room.QRnumber, sizeof(long));
        if (cdbfr != NULL) {
-               msglist = malloc(cdbfr->len);
-               memcpy(msglist, cdbfr->ptr, cdbfr->len);
+               msglist = (long *) cdbfr->ptr;
+               cdbfr->ptr = NULL;      /* CtdlSetSeen() now owns this memory */
                num_msgs = cdbfr->len / sizeof(long);
                cdb_free(cdbfr);
        } else {
                return; /* No messages at all?  No further action. */
        }
 
+       is_set = malloc(num_msgs * sizeof(char));
+       memset(is_set, 0, (num_msgs * sizeof(char)) );
+
        /* Decide which message set we're manipulating */
-       if (which_set == ctdlsetseen_seen) strcpy(vset, vbuf.v_seen);
-       if (which_set == ctdlsetseen_answered) strcpy(vset, vbuf.v_answered);
+       switch(which_set) {
+               case ctdlsetseen_seen:
+                       safestrncpy(vset, vbuf.v_seen, sizeof vset);
+                       break;
+               case ctdlsetseen_answered:
+                       safestrncpy(vset, vbuf.v_answered, sizeof vset);
+                       break;
+       }
 
-       lprintf(CTDL_DEBUG, "before optimize: %s\n", vset);
-       strcpy(newseen, "");
+       /* CtdlLogPrintf(CTDL_DEBUG, "before optimize: %s\n", vset); */
 
-       for (i=0; i<num_msgs; ++i) {
-               is_seen = 0;
+       /* Translate the existing sequence set into an array of booleans */
+       num_sets = num_tokens(vset, ',');
+       for (s=0; s<num_sets; ++s) {
+               extract_token(setstr, vset, s, ',', sizeof setstr);
 
-               if (msglist[i] == target_msgnum) {
-                       is_seen = target_setting;
+               extract_token(lostr, setstr, 0, ':', sizeof lostr);
+               if (num_tokens(setstr, ':') >= 2) {
+                       extract_token(histr, setstr, 1, ':', sizeof histr);
+                       if (!strcmp(histr, "*")) {
+                               snprintf(histr, sizeof histr, "%ld", LONG_MAX);
+                       }
                }
                else {
-                       if (is_msg_in_mset(vset, msglist[i])) {
-                               is_seen = 1;
+                       strcpy(histr, lostr);
+               }
+               lo = atol(lostr);
+               hi = atol(histr);
+
+               for (i = 0; i < num_msgs; ++i) {
+                       if ((msglist[i] >= lo) && (msglist[i] <= hi)) {
+                               is_set[i] = 1;
+                       }
+               }
+       }
+
+       /* Now translate the array of booleans back into a sequence set */
+       strcpy(vset, "");
+       lo = (-1L);
+       hi = (-1L);
+
+       for (i=0; i<num_msgs; ++i) {
+
+               is_seen = is_set[i];    /* Default to existing setting */
+
+               for (k=0; k<num_target_msgnums; ++k) {
+                       if (msglist[i] == target_msgnums[k]) {
+                               is_seen = target_setting;
                        }
                }
 
-               if (is_seen == 1) {
+               if (is_seen) {
                        if (lo < 0L) lo = msglist[i];
                        hi = msglist[i];
                }
+
                if (  ((is_seen == 0) && (was_seen == 1))
                   || ((is_seen == 1) && (i == num_msgs-1)) ) {
-                       size_t tmp;
 
-                       if ( (strlen(newseen) + 20) > SIZ) {
-                               strcpy(newseen, &newseen[20]);
-                               newseen[0] = '*';
+                       /* begin trim-o-matic code */
+                       j=9;
+                       trimming = 0;
+                       while ( (strlen(vset) + 20) > sizeof vset) {
+                               remove_token(vset, 0, ',');
+                               trimming = 1;
+                               if (j--) break; /* loop no more than 9 times */
                        }
-                       tmp = strlen(newseen);
+                       if ( (trimming) && (which_set == ctdlsetseen_seen) ) {
+                               t = atol(vset);
+                               if (t<2) t=2;
+                               --t;
+                               snprintf(lostr, sizeof lostr,
+                                       "1:%ld,%s", t, vset);
+                               safestrncpy(vset, lostr, sizeof vset);
+                       }
+                       /* end trim-o-matic code */
+
+                       tmp = strlen(vset);
                        if (tmp > 0) {
-                               strcat(newseen, ",");
-                               tmp++;
+                               strcat(vset, ",");
+                               ++tmp;
                        }
                        if (lo == hi) {
-                               snprintf(&newseen[tmp], sizeof newseen - tmp,
+                               snprintf(&vset[tmp], (sizeof vset) - tmp,
                                         "%ld", lo);
                        }
                        else {
-                               snprintf(&newseen[tmp], sizeof newseen - tmp,
+                               snprintf(&vset[tmp], (sizeof vset) - tmp,
                                         "%ld:%ld", lo, hi);
                        }
                        lo = (-1L);
@@ -390,12 +478,23 @@ void CtdlSetSeen(long target_msgnum, int target_setting, int which_set) {
        }
 
        /* Decide which message set we're manipulating */
-       if (which_set == ctdlsetseen_seen) strcpy(vbuf.v_seen, newseen);
-       if (which_set == ctdlsetseen_answered) strcpy(vbuf.v_answered, newseen);
+       switch (which_set) {
+               case ctdlsetseen_seen:
+                       safestrncpy(vbuf.v_seen, vset, sizeof vbuf.v_seen);
+                       break;
+               case ctdlsetseen_answered:
+                       safestrncpy(vbuf.v_answered, vset,
+                                               sizeof vbuf.v_answered);
+                       break;
+       }
+       free(is_set);
 
-       lprintf(CTDL_DEBUG, " after optimize: %s\n", newseen);
+       /* CtdlLogPrintf(CTDL_DEBUG, " after optimize: %s\n", vset); */
        free(msglist);
-       CtdlSetRelationship(&vbuf, &CC->user, &CC->room);
+       CtdlSetRelationship(&vbuf,
+               ((which_user != NULL) ? which_user : &CC->user),
+               ((which_room != NULL) ? which_room : &CC->room)
+       );
 }
 
 
@@ -403,14 +502,14 @@ void CtdlSetSeen(long target_msgnum, int target_setting, int which_set) {
  * API function to perform an operation for each qualifying message in the
  * current room.  (Returns the number of messages processed.)
  */
-int CtdlForEachMessage(int mode, long ref,
+int CtdlForEachMessage(int mode, long ref, char *search_string,
                        char *content_type,
                        struct CtdlMessage *compare,
                        void (*CallBack) (long, void *),
                        void *userdata)
 {
 
-       int a;
+       int a, i, j;
        struct visit vbuf;
        struct cdbdata *cdbfr;
        long *msglist = NULL;
@@ -418,24 +517,32 @@ int CtdlForEachMessage(int mode, long ref,
        int num_processed = 0;
        long thismsg;
        struct MetaData smi;
-       struct CtdlMessage *msg;
-       int is_seen;
+       struct CtdlMessage *msg = NULL;
+       int is_seen = 0;
        long lastold = 0L;
        int printed_lastold = 0;
+       int num_search_msgs = 0;
+       long *search_msgs = NULL;
+       regex_t re;
+       int need_to_free_re = 0;
+       regmatch_t pm;
+
+       if (content_type) if (!IsEmptyStr(content_type)) {
+               regcomp(&re, content_type, 0);
+               need_to_free_re = 1;
+       }
 
        /* Learn about the user and room in question */
-       get_mm();
        getuser(&CC->user, CC->curr_user);
        CtdlGetRelationship(&vbuf, &CC->user, &CC->room);
 
        /* Load the message list */
        cdbfr = cdb_fetch(CDB_MSGLISTS, &CC->room.QRnumber, sizeof(long));
        if (cdbfr != NULL) {
-               msglist = malloc(cdbfr->len);
-               memcpy(msglist, cdbfr->ptr, cdbfr->len);
+               msglist = (long *) cdbfr->ptr;
                num_msgs = cdbfr->len / sizeof(long);
-               cdb_free(cdbfr);
        } else {
+               if (need_to_free_re) regfree(&re);
                return 0;       /* No messages at all?  No further action. */
        }
 
@@ -448,7 +555,7 @@ int CtdlForEachMessage(int mode, long ref,
                /* If the caller is looking for a specific MIME type, filter
                 * out all messages which are not of the type requested.
                 */
-               if (content_type != NULL) if (strlen(content_type) > 0) {
+               if (content_type != NULL) if (!IsEmptyStr(content_type)) {
 
                        /* This call to GetMetaData() sits inside this loop
                         * so that we only do the extra database read per msg
@@ -461,7 +568,8 @@ int CtdlForEachMessage(int mode, long ref,
                         */
                        GetMetaData(&smi, msglist[a]);
 
-                       if (strcasecmp(smi.meta_content_type, content_type)) {
+                       /* if (strcasecmp(smi.meta_content_type, content_type)) { old non-regex way */
+                       if (regexec(&re, smi.meta_content_type, 1, &pm, 0) != 0) {
                                msglist[a] = 0L;
                        }
                }
@@ -486,7 +594,51 @@ int CtdlForEachMessage(int mode, long ref,
                }
        }
 
+       /* If a search string was specified, get a message list from
+        * the full text index and remove messages which aren't on both
+        * lists.
+        *
+        * How this works:
+        * Since the lists are sorted and strictly ascending, and the
+        * output list is guaranteed to be shorter than or equal to the
+        * input list, we overwrite the bottom of the input list.  This
+        * eliminates the need to memmove big chunks of the list over and
+        * over again.
+        */
+       if ( (num_msgs > 0) && (mode == MSGS_SEARCH) && (search_string) ) {
+
+               /* Call search module via hook mechanism.
+                * NULL means use any search function available.
+                * otherwise replace with a char * to name of search routine
+                */
+               CtdlModuleDoSearch(&num_search_msgs, &search_msgs, search_string, "fulltext");
+
+               if (num_search_msgs > 0) {
        
+                       int orig_num_msgs;
+
+                       orig_num_msgs = num_msgs;
+                       num_msgs = 0;
+                       for (i=0; i<orig_num_msgs; ++i) {
+                               for (j=0; j<num_search_msgs; ++j) {
+                                       if (msglist[i] == search_msgs[j]) {
+                                               msglist[num_msgs++] = msglist[i];
+                                       }
+                               }
+                       }
+               }
+               else {
+                       num_msgs = 0;   /* No messages qualify */
+               }
+               if (search_msgs != NULL) free(search_msgs);
+
+               /* Now that we've purged messages which don't contain the search
+                * string, treat a MSGS_SEARCH just like a MSGS_ALL from this
+                * point on.
+                */
+               mode = MSGS_ALL;
+       }
+
        /*
         * Now iterate through the message list, according to the
         * criteria supplied by the caller.
@@ -494,8 +646,14 @@ int CtdlForEachMessage(int mode, long ref,
        if (num_msgs > 0)
                for (a = 0; a < num_msgs; ++a) {
                        thismsg = msglist[a];
-                       is_seen = is_msg_in_mset(vbuf.v_seen, thismsg);
-                       if (is_seen) lastold = thismsg;
+                       if (mode == MSGS_ALL) {
+                               is_seen = 0;
+                       }
+                       else {
+                               is_seen = is_msg_in_sequence_set(
+                                                       vbuf.v_seen, thismsg);
+                               if (is_seen) lastold = thismsg;
+                       }
                        if ((thismsg > 0L)
                            && (
 
@@ -518,7 +676,8 @@ int CtdlForEachMessage(int mode, long ref,
                                ++num_processed;
                        }
                }
-       free(msglist);          /* Clean up */
+       cdb_free(cdbfr);        /* Clean up */
+       if (need_to_free_re) regfree(&re);
        return num_processed;
 }
 
@@ -526,25 +685,28 @@ int CtdlForEachMessage(int mode, long ref,
 
 /*
  * cmd_msgs()  -  get list of message #'s in this room
- *                implements the MSGS server command using CtdlForEachMessage()
+ *             implements the MSGS server command using CtdlForEachMessage()
  */
 void cmd_msgs(char *cmdbuf)
 {
        int mode = 0;
-       char which[SIZ];
-       char buf[SIZ];
-       char tfield[SIZ];
-       char tvalue[SIZ];
+       char which[16];
+       char buf[256];
+       char tfield[256];
+       char tvalue[256];
        int cm_ref = 0;
        int i;
        int with_template = 0;
        struct CtdlMessage *template = NULL;
+       int with_headers = 0;
+       char search_string[1024];
 
-       extract(which, cmdbuf, 0);
+       extract_token(which, cmdbuf, 0, '|', sizeof which);
        cm_ref = extract_int(cmdbuf, 1);
+       extract_token(search_string, cmdbuf, 1, '|', sizeof search_string);
        with_template = extract_int(cmdbuf, 2);
+       with_headers = extract_int(cmdbuf, 3);
 
-       mode = MSGS_ALL;
        strcat(which, "   ");
        if (!strncasecmp(which, "OLD", 3))
                mode = MSGS_OLD;
@@ -556,12 +718,22 @@ void cmd_msgs(char *cmdbuf)
                mode = MSGS_LAST;
        else if (!strncasecmp(which, "GT", 2))
                mode = MSGS_GT;
+       else if (!strncasecmp(which, "SEARCH", 6))
+               mode = MSGS_SEARCH;
+       else
+               mode = MSGS_ALL;
 
        if ((!(CC->logged_in)) && (!(CC->internal_pgm))) {
                cprintf("%d not logged in\n", ERROR + NOT_LOGGED_IN);
                return;
        }
 
+       if ( (mode == MSGS_SEARCH) && (!config.c_enable_fulltext) ) {
+               cprintf("%d Full text index is not enabled on this server.\n",
+                       ERROR + CMD_NOT_SUPPORTED);
+               return;
+       }
+
        if (with_template) {
                unbuffer_output();
                cprintf("%d Send template then receive message list\n",
@@ -569,9 +741,12 @@ void cmd_msgs(char *cmdbuf)
                template = (struct CtdlMessage *)
                        malloc(sizeof(struct CtdlMessage));
                memset(template, 0, sizeof(struct CtdlMessage));
-               while(client_gets(buf), strcmp(buf,"000")) {
-                       extract(tfield, buf, 0);
-                       extract(tvalue, buf, 1);
+               template->cm_magic = CTDLMESSAGE_MAGIC;
+               template->cm_anon_type = MES_NORMAL;
+
+               while(client_getln(buf, sizeof buf) >= 0 && strcmp(buf,"000")) {
+                       extract_token(tfield, buf, 0, '|', sizeof tfield);
+                       extract_token(tvalue, buf, 1, '|', sizeof tvalue);
                        for (i='A'; i<='Z'; ++i) if (msgkeys[i]!=NULL) {
                                if (!strcasecmp(tfield, msgkeys[i])) {
                                        template->cm_fields[i] =
@@ -582,11 +757,17 @@ void cmd_msgs(char *cmdbuf)
                buffer_output();
        }
        else {
-               cprintf("%d Message list...\n", LISTING_FOLLOWS);
+               cprintf("%d  \n", LISTING_FOLLOWS);
        }
 
-       CtdlForEachMessage(mode, cm_ref,
-               NULL, template, simple_listing, NULL);
+       CtdlForEachMessage(mode,
+                       ( (mode == MSGS_SEARCH) ? 0 : cm_ref ),
+                       ( (mode == MSGS_SEARCH) ? search_string : NULL ),
+                       NULL,
+                       template,
+                       (with_headers ? headers_listing : simple_listing),
+                       NULL
+       );
        if (template != NULL) CtdlFreeMessage(template);
        cprintf("000\n");
 }
@@ -624,20 +805,19 @@ void do_help_subst(char *buffer)
        help_subst(buffer, "^variantname", CITADEL);
        snprintf(buf2, sizeof buf2, "%d", config.c_maxsessions);
        help_subst(buffer, "^maxsessions", buf2);
-       help_subst(buffer, "^bbsdir", BBSDIR);
+       help_subst(buffer, "^bbsdir", ctdl_message_dir);
 }
 
 
 
 /*
  * memfmout()  -  Citadel text formatter and paginator.
- *             Although the original purpose of this routine was to format
- *             text to the reader's screen width, all we're really using it
- *             for here is to format text out to 80 columns before sending it
- *             to the client.  The client software may reformat it again.
+ *          Although the original purpose of this routine was to format
+ *          text to the reader's screen width, all we're really using it
+ *          for here is to format text out to 80 columns before sending it
+ *          to the client.  The client software may reformat it again.
  */
 void memfmout(
-       int width,              /* screen width to use */
        char *mptr,             /* where are we going to get our text from? */
        char subst,             /* nonzero if we should do substitutions */
        char *nl)               /* string to terminate lines with */
@@ -648,6 +828,7 @@ void memfmout(
        cit_uint8_t ch;
        char aaa[140];
        char buffer[SIZ];
+       static int width = 80;
 
        strcpy(aaa, "");
        old = 255;
@@ -675,15 +856,13 @@ void memfmout(
                old = real;
                real = ch;
 
-               if (((ch == 13) || (ch == 10)) && (old != 13) && (old != 10))
+               if (((ch == 13) || (ch == 10)) && (old != 13) && (old != 10)) {
                        ch = 32;
+               }
                if (((old == 13) || (old == 10)) && (isspace(real))) {
                        cprintf("%s", nl);
                        c = 1;
                }
-               if (ch > 126)
-                       continue;
-
                if (ch > 32) {
                        if (((strlen(aaa) + c) > (width - 5)) && (strlen(aaa) > (width - 5))) {
                                cprintf("%s%s", nl, aaa);
@@ -721,32 +900,53 @@ void memfmout(
  * Callback function for mime parser that simply lists the part
  */
 void list_this_part(char *name, char *filename, char *partnum, char *disp,
-                   void *content, char *cbtype, size_t length, char *encoding,
+                   void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
                    void *cbuserdata)
 {
-
-       cprintf("part=%s|%s|%s|%s|%s|%ld\n",
-               name, filename, partnum, disp, cbtype, (long)length);
+       struct ma_info *ma;
+       
+       ma = (struct ma_info *)cbuserdata;
+       if (ma->is_ma == 0) {
+               cprintf("part=%s|%s|%s|%s|%s|%ld\n",
+                       name, filename, partnum, disp, cbtype, (long)length);
+       }
 }
 
 /* 
  * Callback function for multipart prefix
  */
 void list_this_pref(char *name, char *filename, char *partnum, char *disp,
-                   void *content, char *cbtype, size_t length, char *encoding,
+                   void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
                    void *cbuserdata)
 {
-       cprintf("pref=%s|%s\n", partnum, cbtype);
+       struct ma_info *ma;
+       
+       ma = (struct ma_info *)cbuserdata;
+       if (!strcasecmp(cbtype, "multipart/alternative")) {
+               ++ma->is_ma;
+       }
+
+       if (ma->is_ma == 0) {
+               cprintf("pref=%s|%s\n", partnum, cbtype);
+       }
 }
 
 /* 
  * Callback function for multipart sufffix
  */
 void list_this_suff(char *name, char *filename, char *partnum, char *disp,
-                   void *content, char *cbtype, size_t length, char *encoding,
+                   void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
                    void *cbuserdata)
 {
-       cprintf("suff=%s|%s\n", partnum, cbtype);
+       struct ma_info *ma;
+       
+       ma = (struct ma_info *)cbuserdata;
+       if (ma->is_ma == 0) {
+               cprintf("suff=%s|%s\n", partnum, cbtype);
+       }
+       if (!strcasecmp(cbtype, "multipart/alternative")) {
+               --ma->is_ma;
+       }
 }
 
 
@@ -754,8 +954,8 @@ void list_this_suff(char *name, char *filename, char *partnum, char *disp,
  * Callback function for mime parser that opens a section for downloading
  */
 void mime_download(char *name, char *filename, char *partnum, char *disp,
-                  void *content, char *cbtype, size_t length, char *encoding,
-                  void *cbuserdata)
+                  void *content, char *cbtype, char *cbcharset, size_t length,
+                  char *encoding, void *cbuserdata)
 {
 
        /* Silently go away if there's already a download open... */
@@ -763,7 +963,7 @@ void mime_download(char *name, char *filename, char *partnum, char *disp,
                return;
 
        /* ...or if this is not the desired section */
-       if (strcasecmp(desired_section, partnum))
+       if (strcasecmp(CC->download_desired_section, partnum))
                return;
 
        CC->download_fp = tmpfile();
@@ -779,6 +979,27 @@ void mime_download(char *name, char *filename, char *partnum, char *disp,
 
 
 
+/*
+ * Callback function for mime parser that outputs a section all at once
+ */
+void mime_spew_section(char *name, char *filename, char *partnum, char *disp,
+                  void *content, char *cbtype, char *cbcharset, size_t length,
+                  char *encoding, void *cbuserdata)
+{
+       int *found_it = (int *)cbuserdata;
+
+       /* ...or if this is not the desired section */
+       if (strcasecmp(CC->download_desired_section, partnum))
+               return;
+
+       *found_it = 1;
+
+       cprintf("%d %d\n", BINARY_FOLLOWS, (int)length);
+       client_write(content, length);
+}
+
+
+
 /*
  * Load a message from disk into memory.
  * This is used by CtdlOutputMsg() and other fetch functions.
@@ -791,17 +1012,18 @@ struct CtdlMessage *CtdlFetchMessage(long msgnum, int with_body)
        struct cdbdata *dmsgtext;
        struct CtdlMessage *ret = NULL;
        char *mptr;
+       char *upper_bound;
        cit_uint8_t ch;
        cit_uint8_t field_header;
-       size_t field_length;
 
-       lprintf(CTDL_DEBUG, "CtdlFetchMessage(%ld, %d)\n", msgnum, with_body);
+       CtdlLogPrintf(CTDL_DEBUG, "CtdlFetchMessage(%ld, %d)\n", msgnum, with_body);
 
        dmsgtext = cdb_fetch(CDB_MSGMAIN, &msgnum, sizeof(long));
        if (dmsgtext == NULL) {
                return NULL;
        }
        mptr = dmsgtext->ptr;
+       upper_bound = mptr + dmsgtext->len;
 
        /* Parse the three bytes that begin EVERY message on disk.
         * The first is always 0xFF, the on-disk magic number.
@@ -810,9 +1032,7 @@ struct CtdlMessage *CtdlFetchMessage(long msgnum, int with_body)
         */
        ch = *mptr++;
        if (ch != 255) {
-               lprintf(CTDL_ERR,
-                       "Message %ld appears to be corrupted.\n",
-                       msgnum);
+               CtdlLogPrintf(CTDL_ERR, "Message %ld appears to be corrupted.\n", msgnum);
                cdb_free(dmsgtext);
                return NULL;
        }
@@ -829,16 +1049,15 @@ struct CtdlMessage *CtdlFetchMessage(long msgnum, int with_body)
         * have just processed the 'M' (message text) field.
         */
        do {
-               field_length = strlen(mptr);
-               if (field_length == 0)
+               if (mptr >= upper_bound) {
                        break;
+               }
                field_header = *mptr++;
-               ret->cm_fields[field_header] = malloc(field_length + 1);
-               strcpy(ret->cm_fields[field_header], mptr);
+               ret->cm_fields[field_header] = strdup(mptr);
 
                while (*mptr++ != 0);   /* advance to next field */
 
-       } while ((field_length > 0) && (field_header != 'M'));
+       } while ((mptr < upper_bound) && (field_header != 'M'));
 
        cdb_free(dmsgtext);
 
@@ -855,7 +1074,7 @@ struct CtdlMessage *CtdlFetchMessage(long msgnum, int with_body)
                }
        }
        if (ret->cm_fields['M'] == NULL) {
-               ret->cm_fields['M'] = strdup("<no text>\n");
+               ret->cm_fields['M'] = strdup("\r\n\r\n (no text)\r\n");
        }
 
        /* Perform "before read" hooks (aborting if any return nonzero) */
@@ -876,7 +1095,7 @@ int is_valid_message(struct CtdlMessage *msg) {
        if (msg == NULL)
                return 0;
        if ((msg->cm_magic) != CTDLMESSAGE_MAGIC) {
-               lprintf(CTDL_WARNING, "is_valid_message() -- self-check failed\n");
+               CtdlLogPrintf(CTDL_WARNING, "is_valid_message() -- self-check failed\n");
                return 0;
        }
        return 1;
@@ -890,7 +1109,11 @@ void CtdlFreeMessage(struct CtdlMessage *msg)
 {
        int i;
 
-       if (is_valid_message(msg) == 0) return;
+       if (is_valid_message(msg) == 0) 
+       {
+               if (msg != NULL) free (msg);
+               return;
+       }
 
        for (i = 0; i < 256; ++i)
                if (msg->cm_fields[i] != NULL) {
@@ -913,97 +1136,140 @@ void CtdlFreeMessage(struct CtdlMessage *msg)
  *
  */
 void fixed_output_pre(char *name, char *filename, char *partnum, char *disp,
-               void *content, char *cbtype, size_t length, char *encoding,
+               void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
                void *cbuserdata)
 {
-               lprintf(CTDL_DEBUG, "fixed_output_pre() type=<%s>\n", cbtype);  
-               if (!strcasecmp(cbtype, "multipart/alternative")) {
-                       ++ma->is_ma;
-                       ma->did_print = 0;
-                       return;
-               }
+       struct ma_info *ma;
+       
+       ma = (struct ma_info *)cbuserdata;
+       CtdlLogPrintf(CTDL_DEBUG, "fixed_output_pre() type=<%s>\n", cbtype);    
+       if (!strcasecmp(cbtype, "multipart/alternative")) {
+               ++ma->is_ma;
+               ma->did_print = 0;
+       }
+       if (!strcasecmp(cbtype, "message/rfc822")) {
+               ++ma->freeze;
+       }
 }
 
 /*
  * Post callback function for multipart/alternative
  */
 void fixed_output_post(char *name, char *filename, char *partnum, char *disp,
-               void *content, char *cbtype, size_t length, char *encoding,
-               void *cbuserdata)
+               void *content, char *cbtype, char *cbcharset, size_t length,
+               char *encoding, void *cbuserdata)
 {
-               lprintf(CTDL_DEBUG, "fixed_output_post() type=<%s>\n", cbtype); 
-               if (!strcasecmp(cbtype, "multipart/alternative")) {
-                       --ma->is_ma;
-                       ma->did_print = 0;
-                       return;
-               }
+       struct ma_info *ma;
+       
+       ma = (struct ma_info *)cbuserdata;
+       CtdlLogPrintf(CTDL_DEBUG, "fixed_output_post() type=<%s>\n", cbtype);   
+       if (!strcasecmp(cbtype, "multipart/alternative")) {
+               --ma->is_ma;
+               ma->did_print = 0;
+       }
+       if (!strcasecmp(cbtype, "message/rfc822")) {
+               --ma->freeze;
+       }
 }
 
 /*
  * Inline callback function for mime parser that wants to display text
  */
 void fixed_output(char *name, char *filename, char *partnum, char *disp,
-               void *content, char *cbtype, size_t length, char *encoding,
-               void *cbuserdata)
-       {
-               char *ptr;
-               char *wptr;
-               size_t wlen;
-
-               lprintf(CTDL_DEBUG, "fixed_output() type=<%s>\n", cbtype);      
-
-               /*
-                * If we're in the middle of a multipart/alternative scope and
-                * we've already printed another section, skip this one.
-                */     
-               if ( (ma->is_ma == 1) && (ma->did_print == 1) ) {
-                       lprintf(CTDL_DEBUG, "Skipping part %s (%s)\n", partnum, cbtype);
-                       return;
-               }
-               ma->did_print = 1;
-       
-               if ( (!strcasecmp(cbtype, "text/plain")) 
-                  || (strlen(cbtype)==0) ) {
-                       wptr = content;
-                       if (length > 0) {
-                               client_write(wptr, length);
-                               if (wptr[length-1] != '\n') {
-                                       cprintf("\n");
-                               }
-                       }
-               }
-               else if (!strcasecmp(cbtype, "text/html")) {
-                       ptr = html_to_ascii(content, 80, 0);
-                       wlen = strlen(ptr);
-                       client_write(ptr, wlen);
-                       if (ptr[wlen-1] != '\n') {
+               void *content, char *cbtype, char *cbcharset, size_t length,
+               char *encoding, void *cbuserdata)
+{
+       char *ptr;
+       char *wptr;
+       size_t wlen;
+       struct ma_info *ma;
+
+       ma = (struct ma_info *)cbuserdata;
+
+       CtdlLogPrintf(CTDL_DEBUG,
+               "fixed_output() part %s: %s (%s) (%ld bytes)\n",
+               partnum, filename, cbtype, (long)length);
+
+       /*
+        * If we're in the middle of a multipart/alternative scope and
+        * we've already printed another section, skip this one.
+        */     
+       if ( (ma->is_ma) && (ma->did_print) ) {
+               CtdlLogPrintf(CTDL_DEBUG, "Skipping part %s (%s)\n", partnum, cbtype);
+               return;
+       }
+       ma->did_print = 1;
+
+       if ( (!strcasecmp(cbtype, "text/plain")) 
+          || (IsEmptyStr(cbtype)) ) {
+               wptr = content;
+               if (length > 0) {
+                       client_write(wptr, length);
+                       if (wptr[length-1] != '\n') {
                                cprintf("\n");
                        }
-                       free(ptr);
                }
-               else if (strncasecmp(cbtype, "multipart/", 10)) {
-                       cprintf("Part %s: %s (%s) (%ld bytes)\r\n",
-                               partnum, filename, cbtype, (long)length);
+               return;
+       }
+
+       if (!strcasecmp(cbtype, "text/html")) {
+               ptr = html_to_ascii(content, length, 80, 0);
+               wlen = strlen(ptr);
+               client_write(ptr, wlen);
+               if (ptr[wlen-1] != '\n') {
+                       cprintf("\n");
+               }
+               free(ptr);
+               return;
+       }
+
+       if (ma->use_fo_hooks) {
+               if (PerformFixedOutputHooks(cbtype, content, length)) {
+               /* above function returns nonzero if it handled the part */
+                       return;
                }
        }
 
+       if (strncasecmp(cbtype, "multipart/", 10)) {
+               cprintf("Part %s: %s (%s) (%ld bytes)\r\n",
+                       partnum, filename, cbtype, (long)length);
+               return;
+       }
+}
+
 /*
  * The client is elegant and sophisticated and wants to be choosy about
  * MIME content types, so figure out which multipart/alternative part
  * we're going to send.
+ *
+ * We use a system of weights.  When we find a part that matches one of the
+ * MIME types we've declared as preferential, we can store it in ma->chosen_part
+ * and then set ma->chosen_pref to that MIME type's position in our preference
+ * list.  If we then hit another match, we only replace the first match if
+ * the preference value is lower.
  */
 void choose_preferred(char *name, char *filename, char *partnum, char *disp,
-               void *content, char *cbtype, size_t length, char *encoding,
-               void *cbuserdata)
+               void *content, char *cbtype, char *cbcharset, size_t length,
+               char *encoding, void *cbuserdata)
 {
-       char buf[SIZ];
+       char buf[1024];
        int i;
+       struct ma_info *ma;
+       
+       ma = (struct ma_info *)cbuserdata;
 
-       if (ma->is_ma > 0) {
-               for (i=0; i<num_tokens(CC->preferred_formats, '|'); ++i) {
-                       extract(buf, CC->preferred_formats, i);
-                       if (!strcasecmp(buf, cbtype)) {
-                               strcpy(ma->chosen_part, partnum);
+       // NOTE: REMOVING THIS CONDITIONAL FIXES BUG 220
+       //       http://bugzilla.citadel.org/show_bug.cgi?id=220
+       // I don't know if there are any side effects!  Please TEST TEST TEST
+       //if (ma->is_ma > 0) {
+
+       for (i=0; i<num_tokens(CC->preferred_formats, '|'); ++i) {
+               extract_token(buf, CC->preferred_formats, i, '|', sizeof buf);
+               if ( (!strcasecmp(buf, cbtype)) && (!ma->freeze) ) {
+                       if (i < ma->chosen_pref) {
+                               CtdlLogPrintf(CTDL_DEBUG, "Setting chosen part: <%s>\n", partnum);
+                               safestrncpy(ma->chosen_part, partnum, sizeof ma->chosen_part);
+                               ma->chosen_pref = i;
                        }
                }
        }
@@ -1013,13 +1279,16 @@ void choose_preferred(char *name, char *filename, char *partnum, char *disp,
  * Now that we've chosen our preferred part, output it.
  */
 void output_preferred(char *name, char *filename, char *partnum, char *disp,
-               void *content, char *cbtype, size_t length, char *encoding,
-               void *cbuserdata)
+               void *content, char *cbtype, char *cbcharset, size_t length,
+               char *encoding, void *cbuserdata)
 {
        int i;
-       char buf[SIZ];
+       char buf[128];
        int add_newline = 0;
        char *text_content;
+       struct ma_info *ma;
+       
+       ma = (struct ma_info *)cbuserdata;
 
        /* This is not the MIME part you're looking for... */
        if (strcasecmp(partnum, ma->chosen_part)) return;
@@ -1028,7 +1297,7 @@ void output_preferred(char *name, char *filename, char *partnum, char *disp,
         * list, we can simply output it verbatim.
         */
        for (i=0; i<num_tokens(CC->preferred_formats, '|'); ++i) {
-               extract(buf, CC->preferred_formats, i);
+               extract_token(buf, CC->preferred_formats, i, '|', sizeof buf);
                if (!strcasecmp(buf, cbtype)) {
                        /* Yeah!  Go!  W00t!! */
 
@@ -1036,16 +1305,19 @@ void output_preferred(char *name, char *filename, char *partnum, char *disp,
                        if (text_content[length-1] != '\n') {
                                ++add_newline;
                        }
-
-                       cprintf("Content-type: %s\n", cbtype);
-                       cprintf("Content-length: %d\n",
+                       cprintf("Content-type: %s", cbtype);
+                       if (!IsEmptyStr(cbcharset)) {
+                               cprintf("; charset=%s", cbcharset);
+                       }
+                       cprintf("\nContent-length: %d\n",
                                (int)(length + add_newline) );
-                       if (strlen(encoding) > 0) {
+                       if (!IsEmptyStr(encoding)) {
                                cprintf("Content-transfer-encoding: %s\n", encoding);
                        }
                        else {
                                cprintf("Content-transfer-encoding: 7bit\n");
                        }
+                       cprintf("X-Citadel-MSG4-Partnum: %s\n", partnum);
                        cprintf("\n");
                        client_write(content, length);
                        if (add_newline) cprintf("\n");
@@ -1055,26 +1327,62 @@ void output_preferred(char *name, char *filename, char *partnum, char *disp,
 
        /* No translations required or possible: output as text/plain */
        cprintf("Content-type: text/plain\n\n");
-       fixed_output(name, filename, partnum, disp, content, cbtype,
+       fixed_output(name, filename, partnum, disp, content, cbtype, cbcharset,
                        length, encoding, cbuserdata);
 }
 
 
+struct encapmsg {
+       char desired_section[64];
+       char *msg;
+       size_t msglen;
+};
+
+
+/*
+ * Callback function for
+ */
+void extract_encapsulated_message(char *name, char *filename, char *partnum, char *disp,
+                  void *content, char *cbtype, char *cbcharset, size_t length,
+                  char *encoding, void *cbuserdata)
+{
+       struct encapmsg *encap;
+
+       encap = (struct encapmsg *)cbuserdata;
+
+       /* Only proceed if this is the desired section... */
+       if (!strcasecmp(encap->desired_section, partnum)) {
+               encap->msglen = length;
+               encap->msg = malloc(length + 2);
+               memcpy(encap->msg, content, length);
+               return;
+       }
+
+}
+
+
+
+
 /*
  * Get a message off disk.  (returns om_* values found in msgbase.h)
  * 
  */
 int CtdlOutputMsg(long msg_num,                /* message number (local) to fetch */
-               int mode,               /* how would you like that message? */
-               int headers_only,       /* eschew the message body? */
-               int do_proto,           /* do Citadel protocol responses? */
-               int crlf                /* Use CRLF newlines instead of LF? */
+                 int mode,             /* how would you like that message? */
+                 int headers_only,     /* eschew the message body? */
+                 int do_proto,         /* do Citadel protocol responses? */
+                 int crlf,             /* Use CRLF newlines instead of LF? */
+                 char *section,        /* NULL or a message/rfc822 section */
+                 int flags             /* should the bessage be exported clean? */
 ) {
        struct CtdlMessage *TheMessage = NULL;
        int retcode = om_no_such_msg;
+       struct encapmsg encap;
 
-       lprintf(CTDL_DEBUG, "CtdlOutputMsg() msgnum=%ld, mode=%d\n", 
-               msg_num, mode);
+       CtdlLogPrintf(CTDL_DEBUG, "CtdlOutputMsg() msgnum=%ld, mode=%d, section=%s\n", 
+               msg_num, mode,
+               (section ? section : "<>")
+       );
 
        if ((!(CC->logged_in)) && (!(CC->internal_pgm))) {
                if (do_proto) cprintf("%d Not logged in.\n",
@@ -1085,11 +1393,11 @@ int CtdlOutputMsg(long msg_num,         /* message number (local) to fetch */
        /* FIXME: check message id against msglist for this room */
 
        /*
-        * Fetch the message from disk.  If we're in sooper-fast headers
+        * Fetch the message from disk.  If we're in any sort of headers
         * only mode, request that we don't even bother loading the body
         * into memory.
         */
-       if (headers_only == HEADERS_FAST) {
+       if ( (headers_only == HEADERS_FAST) || (headers_only == HEADERS_ONLY) ) {
                TheMessage = CtdlFetchMessage(msg_num, 0);
        }
        else {
@@ -1101,69 +1409,209 @@ int CtdlOutputMsg(long msg_num,                /* message number (local) to fetch */
                        ERROR + MESSAGE_NOT_FOUND, msg_num);
                return(om_no_such_msg);
        }
-       
-       TRACE;
-       retcode = CtdlOutputPreLoadedMsg(
-                       TheMessage, msg_num, mode,
-                       headers_only, do_proto, crlf);
 
-       TRACE;
+       /* Here is the weird form of this command, to process only an
+        * encapsulated message/rfc822 section.
+        */
+       if (section) if (!IsEmptyStr(section)) if (strcmp(section, "0")) {
+               memset(&encap, 0, sizeof encap);
+               safestrncpy(encap.desired_section, section, sizeof encap.desired_section);
+               mime_parser(TheMessage->cm_fields['M'],
+                       NULL,
+                       *extract_encapsulated_message,
+                       NULL, NULL, (void *)&encap, 0
+               );
+               CtdlFreeMessage(TheMessage);
+               TheMessage = NULL;
+
+               if (encap.msg) {
+                       encap.msg[encap.msglen] = 0;
+                       TheMessage = convert_internet_message(encap.msg);
+                       encap.msg = NULL;       /* no free() here, TheMessage owns it now */
+
+                       /* Now we let it fall through to the bottom of this
+                        * function, because TheMessage now contains the
+                        * encapsulated message instead of the top-level
+                        * message.  Isn't that neat?
+                        */
+
+               }
+               else {
+                       if (do_proto) cprintf("%d msg %ld has no part %s\n",
+                               ERROR + MESSAGE_NOT_FOUND, msg_num, section);
+                       retcode = om_no_such_msg;
+               }
+
+       }
+
+       /* Ok, output the message now */
+       retcode = CtdlOutputPreLoadedMsg(TheMessage, mode, headers_only, do_proto, crlf, flags);
        CtdlFreeMessage(TheMessage);
 
        return(retcode);
 }
 
+char *qp_encode_email_addrs(char *source)
+{
+       char user[256], node[256], name[256];
+       const char headerStr[] = "=?UTF-8?Q?";
+       char *Encoded;
+       char *EncodedName;
+       char *nPtr;
+       int need_to_encode = 0;
+       long SourceLen;
+       long EncodedMaxLen;
+       long nColons = 0;
+       long *AddrPtr;
+       long *AddrUtf8;
+       long nAddrPtrMax = 50;
+       long nmax;
+       int InQuotes = 0;
+       int i, n;
+
+       
+       if (source == NULL)
+               return source;
+
+       AddrPtr = malloc (sizeof (long) * nAddrPtrMax);
+       AddrUtf8 = malloc (sizeof (long) * nAddrPtrMax);
+       memset(AddrUtf8, 0, sizeof (long) * nAddrPtrMax);
+       *AddrPtr = 0;
+       i = 0;
+       while (!IsEmptyStr (&source[i])) {
+               if (nColons > nAddrPtrMax){
+                       long *ptr;
+
+                       ptr = (long *) malloc(sizeof (long) * nAddrPtrMax * 2);
+                       memcpy (ptr, AddrPtr, sizeof (long) * nAddrPtrMax);
+                       free (AddrPtr), AddrPtr = ptr;
+                       ptr = (long *) malloc(sizeof (long) * nAddrPtrMax * 2);
+                       memset(ptr + sizeof (long) * nAddrPtrMax, 0, sizeof (long) * nAddrPtrMax - 1);
+                       memcpy (ptr, AddrUtf8, sizeof (long) * nAddrPtrMax);
+                       free (AddrUtf8), AddrUtf8 = ptr;
+                       nAddrPtrMax *= 2;                               
+               }
+               if (((unsigned char) source[i] < 32) || 
+                   ((unsigned char) source[i] > 126)) {
+                       need_to_encode = 1;
+                       AddrUtf8[nColons] = 1;
+               }
+               if (source[i] == '"')
+                       InQuotes = !InQuotes;
+               if (!InQuotes && source[i] == ',') {
+                       nColons++;
+                       AddrPtr[nColons] = i;
+               }
+               i++;
+       }
+       if (need_to_encode == 0) {
+               free(AddrPtr);
+               free(AddrUtf8);
+               return source;
+       }
+
+       SourceLen = i;
+       EncodedMaxLen = nColons * (sizeof(headerStr) + 3) + SourceLen * 3;
+       Encoded = (char*) malloc (EncodedMaxLen);
+
+       for (i = 1; i <= nColons; i++)
+               source[AddrPtr[i]++] = '\0';
+
+       nPtr = Encoded;
+       *nPtr = '\0';
+       for (i = 0; i <= nColons && nPtr != NULL; i++) {
+               nmax = EncodedMaxLen - (nPtr - Encoded);
+               if (AddrUtf8[i]) {
+                       process_rfc822_addr(&source[AddrPtr[i]], 
+                                           user,
+                                           node,
+                                           name);
+                       /* TODO: libIDN here ! */
+                       if (IsEmptyStr(name)) {
+                               n = snprintf(nPtr, nmax, 
+                                            (i==0)?"%s@%s" : ",%s@%s",
+                                            user, node);
+                       }
+                       else {
+                               EncodedName = rfc2047encode(name, strlen(name));                        
+                               n = snprintf(nPtr, nmax, 
+                                            (i==0)?"%s <%s@%s>" : ",%s <%s@%s>",
+                                            EncodedName, user, node);
+                               free(EncodedName);
+                       }
+               }
+               else { 
+                       n = snprintf(nPtr, nmax, 
+                                    (i==0)?"%s" : ",%s",
+                                    &source[AddrPtr[i]]);
+               }
+               if (n > 0 )
+                       nPtr += n;
+               else { 
+                       char *ptr, *nnPtr;
+                       ptr = (char*) malloc(EncodedMaxLen * 2);
+                       memcpy(ptr, Encoded, EncodedMaxLen);
+                       nnPtr = ptr + (nPtr - Encoded), nPtr = nnPtr;
+                       free(Encoded), Encoded = ptr;
+                       EncodedMaxLen *= 2;
+                       i--; /* do it once more with properly lengthened buffer */
+               }
+       }
+       for (i = 1; i <= nColons; i++)
+               source[--AddrPtr[i]] = ',';
+       free(AddrUtf8);
+       free(AddrPtr);
+       return Encoded;
+}
 
 /*
  * Get a message off disk.  (returns om_* values found in msgbase.h)
- * 
  */
 int CtdlOutputPreLoadedMsg(
                struct CtdlMessage *TheMessage,
-               long msg_num,
                int mode,               /* how would you like that message? */
                int headers_only,       /* eschew the message body? */
                int do_proto,           /* do Citadel protocol responses? */
-               int crlf                /* Use CRLF newlines instead of LF? */
+               int crlf,               /* Use CRLF newlines instead of LF? */
+               int flags               /* should the bessage be exported clean? */
 ) {
-       int i, k;
+       int i, j, k;
        char buf[SIZ];
        cit_uint8_t ch;
-       char allkeys[SIZ];
-       char display_name[SIZ];
-       char *mptr;
+       char allkeys[30];
+       char display_name[256];
+       char *mptr, *mpptr;
        char *nl;       /* newline string */
        int suppress_f = 0;
        int subject_found = 0;
+       struct ma_info ma;
 
-       /* buffers needed for RFC822 translation */
-       char suser[SIZ];
-       char luser[SIZ];
-       char fuser[SIZ];
-       char snode[SIZ];
-       char lnode[SIZ];
-       char mid[SIZ];
-       char datestamp[SIZ];
-       /*                                       */
-
-       lprintf(CTDL_DEBUG, "CtdlOutputPreLoadedMsg(TheMessage=%s, %ld, %d, %d, %d, %d\n",
+       /* Buffers needed for RFC822 translation.  These are all filled
+        * using functions that are bounds-checked, and therefore we can
+        * make them substantially smaller than SIZ.
+        */
+       char suser[100];
+       char luser[100];
+       char fuser[100];
+       char snode[100];
+       char lnode[100];
+       char mid[100];
+       char datestamp[100];
+
+       CtdlLogPrintf(CTDL_DEBUG, "CtdlOutputPreLoadedMsg(TheMessage=%s, %d, %d, %d, %d\n",
                ((TheMessage == NULL) ? "NULL" : "not null"),
-               msg_num,
                mode, headers_only, do_proto, crlf);
 
-       TRACE;
-       snprintf(mid, sizeof mid, "%ld", msg_num);
+       strcpy(mid, "unknown");
        nl = (crlf ? "\r\n" : "\n");
 
-       TRACE;
        if (!is_valid_message(TheMessage)) {
-               lprintf(CTDL_ERR,
+               CtdlLogPrintf(CTDL_ERR,
                        "ERROR: invalid preloaded message for output\n");
                return(om_no_such_msg);
        }
 
        /* Are we downloading a MIME component? */
-       TRACE;
        if (mode == MT_DOWNLOAD) {
                if (TheMessage->cm_format_type != FMT_RFC822) {
                        if (do_proto)
@@ -1176,9 +1624,7 @@ int CtdlOutputPreLoadedMsg(
                } else {
                        /* Parse the message text component */
                        mptr = TheMessage->cm_fields['M'];
-                       mime_parser(mptr, NULL,
-                               *mime_download, NULL, NULL,
-                               NULL, 0);
+                       mime_parser(mptr, NULL, *mime_download, NULL, NULL, NULL, 0);
                        /* If there's no file open by this time, the requested
                         * section wasn't found, so print an error
                         */
@@ -1186,16 +1632,40 @@ int CtdlOutputPreLoadedMsg(
                                if (do_proto) cprintf(
                                        "%d Section %s not found.\n",
                                        ERROR + FILE_NOT_FOUND,
-                                       desired_section);
+                                       CC->download_desired_section);
+                       }
+               }
+               return((CC->download_fp != NULL) ? om_ok : om_mime_error);
+       }
+
+       /* MT_SPEW_SECTION is like MT_DOWNLOAD except it outputs the whole MIME part
+        * in a single server operation instead of opening a download file.
+        */
+       if (mode == MT_SPEW_SECTION) {
+               if (TheMessage->cm_format_type != FMT_RFC822) {
+                       if (do_proto)
+                               cprintf("%d This is not a MIME message.\n",
+                               ERROR + ILLEGAL_VALUE);
+               } else {
+                       /* Parse the message text component */
+                       int found_it = 0;
+
+                       mptr = TheMessage->cm_fields['M'];
+                       mime_parser(mptr, NULL, *mime_spew_section, NULL, NULL, (void *)&found_it, 0);
+                       /* If section wasn't found, print an error
+                        */
+                       if (!found_it) {
+                               if (do_proto) cprintf(
+                                       "%d Section %s not found.\n",
+                                       ERROR + FILE_NOT_FOUND,
+                                       CC->download_desired_section);
                        }
                }
-               TRACE;
                return((CC->download_fp != NULL) ? om_ok : om_mime_error);
        }
-       TRACE;
 
        /* now for the user-mode message reading loops */
-       if (do_proto) cprintf("%d Message %ld:\n", LISTING_FOLLOWS, msg_num);
+       if (do_proto) cprintf("%d msg:\n", LISTING_FOLLOWS);
 
        /* Does the caller want to skip the headers? */
        if (headers_only == HEADERS_NONE) goto START_TEXT;
@@ -1207,7 +1677,7 @@ int CtdlOutputPreLoadedMsg(
 
        /* nhdr=yes means that we're only displaying headers, no body */
        if ( (TheMessage->cm_anon_type == MES_ANONONLY)
-           && (mode == MT_CITADEL)
+          && (mode == MT_CITADEL)
           && (do_proto)
           ) {
                cprintf("nhdr=yes\n");
@@ -1215,20 +1685,19 @@ int CtdlOutputPreLoadedMsg(
 
        /* begin header processing loop for Citadel message format */
 
-       TRACE;
        if ((mode == MT_CITADEL) || (mode == MT_MIME)) {
 
-               strcpy(display_name, "<unknown>");
+               safestrncpy(display_name, "<unknown>", sizeof display_name);
                if (TheMessage->cm_fields['A']) {
                        strcpy(buf, TheMessage->cm_fields['A']);
                        if (TheMessage->cm_anon_type == MES_ANONONLY) {
-                               strcpy(display_name, "****");
+                               safestrncpy(display_name, "****", sizeof display_name);
                        }
                        else if (TheMessage->cm_anon_type == MES_ANONOPT) {
-                               strcpy(display_name, "anonymous");
+                               safestrncpy(display_name, "anonymous", sizeof display_name);
                        }
                        else {
-                               strcpy(display_name, buf);
+                               safestrncpy(display_name, buf, sizeof display_name);
                        }
                        if ((is_room_aide())
                            && ((TheMessage->cm_anon_type == MES_ANONONLY)
@@ -1245,13 +1714,13 @@ int CtdlOutputPreLoadedMsg(
                 */
                suppress_f = 0;
                if (TheMessage->cm_fields['N'] != NULL)
-                  if (strlen(TheMessage->cm_fields['N']) > 0)
+                  if (!IsEmptyStr(TheMessage->cm_fields['N']))
                      if (haschar(TheMessage->cm_fields['N'], '.') == 0) {
                        suppress_f = 1;
                }
                
                /* Now spew the header fields in the order we like them. */
-               strcpy(allkeys, FORDER);
+               safestrncpy(allkeys, FORDER, sizeof allkeys);
                for (i=0; i<strlen(allkeys); ++i) {
                        k = (int) allkeys[i];
                        if (k != 'M') {
@@ -1279,25 +1748,34 @@ int CtdlOutputPreLoadedMsg(
        }
 
        /* begin header processing loop for RFC822 transfer format */
-       TRACE;
 
        strcpy(suser, "");
        strcpy(luser, "");
        strcpy(fuser, "");
        strcpy(snode, NODENAME);
        strcpy(lnode, HUMANNODE);
-       TRACE;
        if (mode == MT_RFC822) {
                for (i = 0; i < 256; ++i) {
-       TRACE;
                        if (TheMessage->cm_fields[i]) {
-                               mptr = TheMessage->cm_fields[i];
-       TRACE;
-
+                               mptr = mpptr = TheMessage->cm_fields[i];
+                               
                                if (i == 'A') {
                                        safestrncpy(luser, mptr, sizeof luser);
                                        safestrncpy(suser, mptr, sizeof suser);
                                }
+                               else if (i == 'Y') {
+                                       if (flags & QP_EADDR != 0) 
+                                               mptr = qp_encode_email_addrs(mptr);
+                                       cprintf("CC: %s%s", mptr, nl);
+                               }
+                               else if (i == 'P') {
+                                       cprintf("Return-Path: %s%s", mptr, nl);
+                               }
+                               else if (i == 'V') {
+                                       if (flags & QP_EADDR != 0) 
+                                               mptr = qp_encode_email_addrs(mptr);
+                                       cprintf("Envelope-To: %s%s", mptr, nl);
+                               }
                                else if (i == 'U') {
                                        cprintf("Subject: %s%s", mptr, nl);
                                        subject_found = 1;
@@ -1308,62 +1786,83 @@ int CtdlOutputPreLoadedMsg(
                                        safestrncpy(lnode, mptr, sizeof lnode);
                                else if (i == 'F')
                                        safestrncpy(fuser, mptr, sizeof fuser);
-                               else if (i == 'O')
+                               /* else if (i == 'O')
                                        cprintf("X-Citadel-Room: %s%s",
-                                               mptr, nl);
+                                               mptr, nl); */
                                else if (i == 'N')
                                        safestrncpy(snode, mptr, sizeof snode);
                                else if (i == 'R')
-                                       cprintf("To: %s%s", mptr, nl);
+                               {
+                                       if (haschar(mptr, '@') == 0)
+                                       {
+                                               cprintf("To: %s@%s%s", mptr, config.c_fqdn, nl);
+                                       }
+                                       else
+                                       {
+                                               if (flags & QP_EADDR != 0) 
+                                                       mptr = qp_encode_email_addrs(mptr);
+                                               cprintf("To: %s%s", mptr, nl);
+                                       }
+                               }
                                else if (i == 'T') {
                                        datestring(datestamp, sizeof datestamp,
                                                atol(mptr), DATESTRING_RFC822);
                                        cprintf("Date: %s%s", datestamp, nl);
                                }
+                               else if (i == 'W') {
+                                       cprintf("References: ");
+                                       k = num_tokens(mptr, '|');
+                                       for (j=0; j<k; ++j) {
+                                               extract_token(buf, mptr, j, '|', sizeof buf);
+                                               cprintf("<%s>", buf);
+                                               if (j == (k-1)) {
+                                                       cprintf("%s", nl);
+                                               }
+                                               else {
+                                                       cprintf(" ");
+                                               }
+                                       }
+                               }
+                               if (mptr != mpptr)
+                                       free (mptr);
                        }
                }
                if (subject_found == 0) {
-       TRACE;
                        cprintf("Subject: (no subject)%s", nl);
                }
        }
-       TRACE;
 
-       for (i=0; i<strlen(suser); ++i) {
+       for (i=0; !IsEmptyStr(&suser[i]); ++i) {
                suser[i] = tolower(suser[i]);
                if (!isalnum(suser[i])) suser[i]='_';
        }
-       TRACE;
 
        if (mode == MT_RFC822) {
                if (!strcasecmp(snode, NODENAME)) {
                        safestrncpy(snode, FQDN, sizeof snode);
                }
-       TRACE;
 
                /* Construct a fun message id */
-               cprintf("Message-ID: <%s", mid);
+               cprintf("Message-ID: <%s", mid);/// todo: this possibly breaks threadding mails.
                if (strchr(mid, '@')==NULL) {
                        cprintf("@%s", snode);
                }
                cprintf(">%s", nl);
-       TRACE;
 
                if (!is_room_aide() && (TheMessage->cm_anon_type == MES_ANONONLY)) {
-                       cprintf("From: x@x.org (----)%s", nl);
+                       cprintf("From: \"----\" <x@x.org>%s", nl);
                }
                else if (!is_room_aide() && (TheMessage->cm_anon_type == MES_ANONOPT)) {
-                       cprintf("From: x@x.org (anonymous)%s", nl);
+                       cprintf("From: \"anonymous\" <x@x.org>%s", nl);
                }
-               else if (strlen(fuser) > 0) {
-                       cprintf("From: %s (%s)%s", fuser, luser, nl);
+               else if (!IsEmptyStr(fuser)) {
+                       cprintf("From: \"%s\" <%s>%s", luser, fuser, nl);
                }
                else {
-                       cprintf("From: %s@%s (%s)%s", suser, snode, luser, nl);
+                       cprintf("From: \"%s\" <%s@%s>%s", luser, suser, snode, nl);
                }
 
                cprintf("Organization: %s%s", lnode, nl);
-       TRACE;
 
                /* Blank line signifying RFC822 end-of-headers */
                if (TheMessage->cm_format_type != FMT_RFC822) {
@@ -1373,61 +1872,61 @@ int CtdlOutputPreLoadedMsg(
 
        /* end header processing loop ... at this point, we're in the text */
 START_TEXT:
-       TRACE;
        if (headers_only == HEADERS_FAST) goto DONE;
        mptr = TheMessage->cm_fields['M'];
 
        /* Tell the client about the MIME parts in this message */
        if (TheMessage->cm_format_type == FMT_RFC822) {
                if ( (mode == MT_CITADEL) || (mode == MT_MIME) ) {
+                       memset(&ma, 0, sizeof(struct ma_info));
                        mime_parser(mptr, NULL,
-                               *list_this_part,
-                               *list_this_pref,
-                               *list_this_suff,
-                               NULL, 0);
+                               (do_proto ? *list_this_part : NULL),
+                               (do_proto ? *list_this_pref : NULL),
+                               (do_proto ? *list_this_suff : NULL),
+                               (void *)&ma, 0);
                }
                else if (mode == MT_RFC822) {   /* unparsed RFC822 dump */
-                       /* FIXME ... we have to put some code in here to avoid
-                        * printing duplicate header information when both
-                        * Citadel and RFC822 headers exist.  Preference should
-                        * probably be given to the RFC822 headers.
-                        */
-                       int done_rfc822_hdrs = 0;
-                       while (ch=*(mptr++), ch!=0) {
+                       char *start_of_text = NULL;
+                       start_of_text = strstr(mptr, "\n\r\n");
+                       if (start_of_text == NULL) start_of_text = strstr(mptr, "\n\n");
+                       if (start_of_text == NULL) start_of_text = mptr;
+                       ++start_of_text;
+                       start_of_text = strstr(start_of_text, "\n");
+                       ++start_of_text;
+
+                       char outbuf[1024];
+                       int outlen = 0;
+                       int nllen = strlen(nl);
+                       while (ch=*mptr, ch!=0) {
                                if (ch==13) {
                                        /* do nothing */
                                }
-                               else if (ch==10) {
-                                       if (!done_rfc822_hdrs) {
-                                               if (headers_only != HEADERS_NONE) {
-                                                       cprintf("%s", nl);
-                                               }
-                                       }
-                                       else {
-                                               if (headers_only != HEADERS_ONLY) {
-                                                       cprintf("%s", nl);
-                                               }
-                                       }
-                                       if ((*(mptr) == 13) || (*(mptr) == 10)) {
-                                               done_rfc822_hdrs = 1;
-                                       }
-                               }
                                else {
-                                       if (done_rfc822_hdrs) {
-                                               if (headers_only != HEADERS_NONE) {
-                                                       cprintf("%c", ch);
+                                       if (
+                                               ((headers_only == HEADERS_NONE) && (mptr >= start_of_text))
+                                          ||   ((headers_only == HEADERS_ONLY) && (mptr < start_of_text))
+                                          ||   ((headers_only != HEADERS_NONE) && (headers_only != HEADERS_ONLY))
+                                       ) {
+                                               if (ch == 10) {
+                                                       sprintf(&outbuf[outlen], "%s", nl);
+                                                       outlen += nllen;
                                                }
-                                       }
-                                       else {
-                                               if (headers_only != HEADERS_ONLY) {
-                                                       cprintf("%c", ch);
+                                               else {
+                                                       outbuf[outlen++] = ch;
                                                }
                                        }
-                                       if ((*mptr == 13) || (*mptr == 10)) {
-                                               done_rfc822_hdrs = 1;
-                                       }
                                }
+                               ++mptr;
+                               if (outlen > 1000) {
+                                       client_write(outbuf, outlen);
+                                       outlen = 0;
+                               }
+                       }
+                       if (outlen > 0) {
+                               client_write(outbuf, outlen);
+                               outlen = 0;
                        }
+
                        goto DONE;
                }
        }
@@ -1440,30 +1939,33 @@ START_TEXT:
        if ( (mode == MT_CITADEL) || (mode == MT_MIME) ) {
                if (do_proto) cprintf("text\n");
        }
-       TRACE;
 
        /* If the format type on disk is 1 (fixed-format), then we want
         * everything to be output completely literally ... regardless of
         * what message transfer format is in use.
         */
-       TRACE;
        if (TheMessage->cm_format_type == FMT_FIXED) {
+               int buflen;
                if (mode == MT_MIME) {
                        cprintf("Content-type: text/plain\n\n");
                }
-               strcpy(buf, "");
+               *buf = '\0';
+               buflen = 0;
                while (ch = *mptr++, ch > 0) {
                        if (ch == 13)
                                ch = 10;
-                       if ((ch == 10) || (strlen(buf) > 250)) {
+                       if ((ch == 10) || (buflen > 250)) {
+                               buf[buflen] = '\0';
                                cprintf("%s%s", buf, nl);
-                               strcpy(buf, "");
+                               *buf = '\0';
+                               buflen = 0;
                        } else {
-                               buf[strlen(buf) + 1] = 0;
-                               buf[strlen(buf)] = ch;
+                               buf[buflen] = ch;
+                               buflen++;
                        }
                }
-               if (strlen(buf) > 0)
+               buf[buflen] = '\0';
+               if (!IsEmptyStr(buf))
                        cprintf("%s%s", buf, nl);
        }
 
@@ -1474,12 +1976,11 @@ START_TEXT:
         * for new paragraphs is correct and the client will reformat the
         * message to the reader's screen width.
         */
-       TRACE;
        if (TheMessage->cm_format_type == FMT_CITADEL) {
                if (mode == MT_MIME) {
                        cprintf("Content-type: text/x-citadel-variformat\n\n");
                }
-               memfmout(80, mptr, 0, nl);
+               memfmout(mptr, 0, nl);
        }
 
        /* If the message on disk is format 4 (MIME), we've gotta hand it
@@ -1487,29 +1988,30 @@ START_TEXT:
         * this message is format 1 (fixed format), so the callback function
         * we use will display those parts as-is.
         */
-       TRACE;
        if (TheMessage->cm_format_type == FMT_RFC822) {
-               CtdlAllocUserData(SYM_MA_INFO, sizeof(struct ma_info));
-               memset(ma, 0, sizeof(struct ma_info));
+               memset(&ma, 0, sizeof(struct ma_info));
 
                if (mode == MT_MIME) {
-                       strcpy(ma->chosen_part, "1");
+                       ma.use_fo_hooks = 0;
+                       strcpy(ma.chosen_part, "1");
+                       ma.chosen_pref = 9999;
                        mime_parser(mptr, NULL,
                                *choose_preferred, *fixed_output_pre,
-                               *fixed_output_post, NULL, 0);
+                               *fixed_output_post, (void *)&ma, 0);
                        mime_parser(mptr, NULL,
-                               *output_preferred, NULL, NULL, NULL, 0);
+                               *output_preferred, NULL, NULL, (void *)&ma, CC->msg4_dont_decode);
                }
                else {
+                       ma.use_fo_hooks = 1;
                        mime_parser(mptr, NULL,
                                *fixed_output, *fixed_output_pre,
-                               *fixed_output_post, NULL, 0);
+                               *fixed_output_post, (void *)&ma, 0);
                }
+
        }
 
 DONE:  /* now we're done */
        if (do_proto) cprintf("000\n");
-       TRACE;
        return(om_ok);
 }
 
@@ -1526,7 +2028,7 @@ void cmd_msg0(char *cmdbuf)
        msgid = extract_long(cmdbuf, 0);
        headers_only = extract_int(cmdbuf, 1);
 
-       CtdlOutputMsg(msgid, MT_CITADEL, headers_only, 1, 0);
+       CtdlOutputMsg(msgid, MT_CITADEL, headers_only, 1, 0, NULL, 0);
        return;
 }
 
@@ -1542,7 +2044,7 @@ void cmd_msg2(char *cmdbuf)
        msgid = extract_long(cmdbuf, 0);
        headers_only = extract_int(cmdbuf, 1);
 
-       CtdlOutputMsg(msgid, MT_RFC822, headers_only, 1, 1);
+       CtdlOutputMsg(msgid, MT_RFC822, headers_only, 1, 1, NULL, 0);
 }
 
 
@@ -1553,7 +2055,7 @@ void cmd_msg2(char *cmdbuf)
 void cmd_msg3(char *cmdbuf)
 {
        long msgnum;
-       struct CtdlMessage *msg;
+       struct CtdlMessage *msg = NULL;
        struct ser_ret smr;
 
        if (CC->internal_pgm == 0) {
@@ -1592,9 +2094,11 @@ void cmd_msg3(char *cmdbuf)
 void cmd_msg4(char *cmdbuf)
 {
        long msgid;
+       char section[64];
 
        msgid = extract_long(cmdbuf, 0);
-       CtdlOutputMsg(msgid, MT_MIME, 0, 1, 0);
+       extract_token(section, cmdbuf, 1, '|', sizeof section);
+       CtdlOutputMsg(msgid, MT_MIME, 0, 1, 0, (section[0] ? section : NULL) , 0);
 }
 
 
@@ -1604,9 +2108,14 @@ void cmd_msg4(char *cmdbuf)
  */
 void cmd_msgp(char *cmdbuf)
 {
-       safestrncpy(CC->preferred_formats, cmdbuf,
-                       sizeof(CC->preferred_formats));
-       cprintf("%d ok\n", CIT_OK);
+       if (!strcasecmp(cmdbuf, "dont_decode")) {
+               CC->msg4_dont_decode = 1;
+               cprintf("%d MSG4 will not pre-decode messages.\n", CIT_OK);
+       }
+       else {
+               safestrncpy(CC->preferred_formats, cmdbuf, sizeof(CC->preferred_formats));
+               cprintf("%d Preferred MIME formats have been set.\n", CIT_OK);
+       }
 }
 
 
@@ -1616,139 +2125,209 @@ void cmd_msgp(char *cmdbuf)
 void cmd_opna(char *cmdbuf)
 {
        long msgid;
-
-       CtdlAllocUserData(SYM_DESIRED_SECTION, SIZ);
+       char desired_section[128];
 
        msgid = extract_long(cmdbuf, 0);
-       extract(desired_section, cmdbuf, 1);
+       extract_token(desired_section, cmdbuf, 1, '|', sizeof desired_section);
+       safestrncpy(CC->download_desired_section, desired_section,
+               sizeof CC->download_desired_section);
+       CtdlOutputMsg(msgid, MT_DOWNLOAD, 0, 1, 1, NULL, 0);
+}                      
+
+
+/*
+ * Open a component of a MIME message and transmit it all at once
+ */
+void cmd_dlat(char *cmdbuf)
+{
+       long msgid;
+       char desired_section[128];
 
-       CtdlOutputMsg(msgid, MT_DOWNLOAD, 0, 1, 1);
+       msgid = extract_long(cmdbuf, 0);
+       extract_token(desired_section, cmdbuf, 1, '|', sizeof desired_section);
+       safestrncpy(CC->download_desired_section, desired_section,
+               sizeof CC->download_desired_section);
+       CtdlOutputMsg(msgid, MT_SPEW_SECTION, 0, 1, 1, NULL, 0);
 }                      
 
 
 /*
- * Save a message pointer into a specified room
+ * Save one or more message pointers into a specified room
  * (Returns 0 for success, nonzero for failure)
  * roomname may be NULL to use the current room
+ *
+ * Note that the 'supplied_msg' field may be set to NULL, in which case
+ * the message will be fetched from disk, by number, if we need to perform
+ * replication checks.  This adds an additional database read, so if the
+ * caller already has the message in memory then it should be supplied.  (Obviously
+ * this mode of operation only works if we're saving a single message.)
  */
-int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int flags) {
-       int i;
+int CtdlSaveMsgPointersInRoom(char *roomname, long newmsgidlist[], int num_newmsgs,
+                               int do_repl_check, struct CtdlMessage *supplied_msg)
+{
+       int i, j, unique;
        char hold_rm[ROOMNAMELEN];
-        struct cdbdata *cdbfr;
-        int num_msgs;
-        long *msglist;
-        long highest_msg = 0L;
+       struct cdbdata *cdbfr;
+       int num_msgs;
+       long *msglist;
+       long highest_msg = 0L;
+
+       long msgid = 0;
        struct CtdlMessage *msg = NULL;
 
-       lprintf(CTDL_DEBUG, "CtdlSaveMsgPointerInRoom(%s, %ld, %d)\n",
-               roomname, msgid, flags);
+       long *msgs_to_be_merged = NULL;
+       int num_msgs_to_be_merged = 0;
 
-       strcpy(hold_rm, CC->room.QRname);
-
-       /* We may need to check to see if this message is real */
-       if (  (flags & SM_VERIFY_GOODNESS)
-          || (flags & SM_DO_REPL_CHECK)
-          ) {
-               msg = CtdlFetchMessage(msgid, 1);
-               if (msg == NULL) return(ERROR + ILLEGAL_VALUE);
-       }
+       CtdlLogPrintf(CTDL_DEBUG,
+               "CtdlSaveMsgPointersInRoom(room=%s, num_msgs=%d, repl=%d)\n",
+               roomname, num_newmsgs, do_repl_check);
 
-       /* Perform replication checks if necessary */
-       if ( (flags & SM_DO_REPL_CHECK) && (msg != NULL) ) {
+       strcpy(hold_rm, CC->room.QRname);
 
-               if (getroom(&CC->room,
-                  ((roomname != NULL) ? roomname : CC->room.QRname) )
-                  != 0) {
-                       lprintf(CTDL_ERR, "No such room <%s>\n", roomname);
-                       if (msg != NULL) CtdlFreeMessage(msg);
-                       return(ERROR + ROOM_NOT_FOUND);
-               }
-
-               if (ReplicationChecks(msg) != 0) {
-                       getroom(&CC->room, hold_rm);
-                       if (msg != NULL) CtdlFreeMessage(msg);
-                       lprintf(CTDL_DEBUG,
-                               "Did replication, and newer exists\n");
-                       return(0);
-               }
-       }
+       /* Sanity checks */
+       if (newmsgidlist == NULL) return(ERROR + INTERNAL_ERROR);
+       if (num_newmsgs < 1) return(ERROR + INTERNAL_ERROR);
+       if (num_newmsgs > 1) supplied_msg = NULL;
 
        /* Now the regular stuff */
        if (lgetroom(&CC->room,
           ((roomname != NULL) ? roomname : CC->room.QRname) )
           != 0) {
-               lprintf(CTDL_ERR, "No such room <%s>\n", roomname);
-               if (msg != NULL) CtdlFreeMessage(msg);
+               CtdlLogPrintf(CTDL_ERR, "No such room <%s>\n", roomname);
                return(ERROR + ROOM_NOT_FOUND);
        }
 
-        cdbfr = cdb_fetch(CDB_MSGLISTS, &CC->room.QRnumber, sizeof(long));
-        if (cdbfr == NULL) {
-                msglist = NULL;
-                num_msgs = 0;
-        } else {
-                msglist = malloc(cdbfr->len);
-                if (msglist == NULL)
-                        lprintf(CTDL_ALERT, "ERROR malloc msglist!\n");
-                num_msgs = cdbfr->len / sizeof(long);
-                memcpy(msglist, cdbfr->ptr, cdbfr->len);
-                cdb_free(cdbfr);
-        }
+
+       msgs_to_be_merged = malloc(sizeof(long) * num_newmsgs);
+       num_msgs_to_be_merged = 0;
+
+
+       cdbfr = cdb_fetch(CDB_MSGLISTS, &CC->room.QRnumber, sizeof(long));
+       if (cdbfr == NULL) {
+               msglist = NULL;
+               num_msgs = 0;
+       } else {
+               msglist = (long *) cdbfr->ptr;
+               cdbfr->ptr = NULL;      /* CtdlSaveMsgPointerInRoom() now owns this memory */
+               num_msgs = cdbfr->len / sizeof(long);
+               cdb_free(cdbfr);
+       }
 
 
-       /* Make sure the message doesn't already exist in this room.  It
-        * is absolutely taboo to have more than one reference to the same
-        * message in a room.
+       /* Create a list of msgid's which were supplied by the caller, but do
+        * not already exist in the target room.  It is absolutely taboo to
+        * have more than one reference to the same message in a room.
         */
-        if (num_msgs > 0) for (i=0; i<num_msgs; ++i) {
-               if (msglist[i] == msgid) {
-                       lputroom(&CC->room);    /* unlock the room */
-                       getroom(&CC->room, hold_rm);
-                       if (msg != NULL) CtdlFreeMessage(msg);
-                       free(msglist);
-                       return(ERROR + ALREADY_EXISTS);
+       for (i=0; i<num_newmsgs; ++i) {
+               unique = 1;
+               if (num_msgs > 0) for (j=0; j<num_msgs; ++j) {
+                       if (msglist[j] == newmsgidlist[i]) {
+                               unique = 0;
+                       }
+               }
+               if (unique) {
+                       msgs_to_be_merged[num_msgs_to_be_merged++] = newmsgidlist[i];
                }
        }
 
-        /* Now add the new message */
-        ++num_msgs;
-        msglist = realloc(msglist, (num_msgs * sizeof(long)));
+       CtdlLogPrintf(9, "%d unique messages to be merged\n", num_msgs_to_be_merged);
 
-        if (msglist == NULL) {
-                lprintf(CTDL_ALERT, "ERROR: can't realloc message list!\n");
-        }
-        msglist[num_msgs - 1] = msgid;
+       /*
+        * Now merge the new messages
+        */
+       msglist = realloc(msglist, (sizeof(long) * (num_msgs + num_msgs_to_be_merged)) );
+       if (msglist == NULL) {
+               CtdlLogPrintf(CTDL_ALERT, "ERROR: can't realloc message list!\n");
+       }
+       memcpy(&msglist[num_msgs], msgs_to_be_merged, (sizeof(long) * num_msgs_to_be_merged) );
+       num_msgs += num_msgs_to_be_merged;
 
-        /* Sort the message list, so all the msgid's are in order */
-        num_msgs = sort_msglist(msglist, num_msgs);
+       /* Sort the message list, so all the msgid's are in order */
+       num_msgs = sort_msglist(msglist, num_msgs);
 
-        /* Determine the highest message number */
-        highest_msg = msglist[num_msgs - 1];
+       /* Determine the highest message number */
+       highest_msg = msglist[num_msgs - 1];
 
-        /* Write it back to disk. */
-        cdb_store(CDB_MSGLISTS, &CC->room.QRnumber, (int)sizeof(long),
-                  msglist, (int)(num_msgs * sizeof(long)));
+       /* Write it back to disk. */
+       cdb_store(CDB_MSGLISTS, &CC->room.QRnumber, (int)sizeof(long),
+                 msglist, (int)(num_msgs * sizeof(long)));
 
-        /* Free up the memory we used. */
-        free(msglist);
+       /* Free up the memory we used. */
+       free(msglist);
 
        /* Update the highest-message pointer and unlock the room. */
        CC->room.QRhighest = highest_msg;
        lputroom(&CC->room);
+
+       /* Perform replication checks if necessary */
+       if ( (DoesThisRoomNeedEuidIndexing(&CC->room)) && (do_repl_check) ) {
+               CtdlLogPrintf(CTDL_DEBUG, "CtdlSaveMsgPointerInRoom() doing repl checks\n");
+
+               for (i=0; i<num_msgs_to_be_merged; ++i) {
+                       msgid = msgs_to_be_merged[i];
+       
+                       if (supplied_msg != NULL) {
+                               msg = supplied_msg;
+                       }
+                       else {
+                               msg = CtdlFetchMessage(msgid, 0);
+                       }
+       
+                       if (msg != NULL) {
+                               ReplicationChecks(msg);
+               
+                               /* If the message has an Exclusive ID, index that... */
+                               if (msg->cm_fields['E'] != NULL) {
+                                       index_message_by_euid(msg->cm_fields['E'], &CC->room, msgid);
+                               }
+
+                               /* Free up the memory we may have allocated */
+                               if (msg != supplied_msg) {
+                                       CtdlFreeMessage(msg);
+                               }
+                       }
+       
+               }
+       }
+
+       else {
+               CtdlLogPrintf(CTDL_DEBUG, "CtdlSaveMsgPointerInRoom() skips repl checks\n");
+       }
+
+       /* Submit this room for processing by hooks */
+       PerformRoomHooks(&CC->room);
+
+       /* Go back to the room we were in before we wandered here... */
        getroom(&CC->room, hold_rm);
 
-       /* Bump the reference count for this message. */
-       if ((flags & SM_DONT_BUMP_REF)==0) {
-               AdjRefCount(msgid, +1);
+       /* Bump the reference count for all messages which were merged */
+       for (i=0; i<num_msgs_to_be_merged; ++i) {
+               AdjRefCount(msgs_to_be_merged[i], +1);
+       }
+
+       /* Free up memory... */
+       if (msgs_to_be_merged != NULL) {
+               free(msgs_to_be_merged);
        }
 
        /* Return success. */
-       if (msg != NULL) CtdlFreeMessage(msg);
-        return (0);
+       return (0);
+}
+
+
+/*
+ * This is the same as CtdlSaveMsgPointersInRoom() but it only accepts
+ * a single message.
+ */
+int CtdlSaveMsgPointerInRoom(char *roomname, long msgid,
+                       int do_repl_check, struct CtdlMessage *supplied_msg)
+{
+       return CtdlSaveMsgPointersInRoom(roomname, &msgid, 1, do_repl_check, supplied_msg);
 }
 
 
 
+
 /*
  * Message base operation to save a new message to the message store
  * (returns new message number)
@@ -1760,14 +2339,14 @@ int CtdlSaveMsgPointerInRoom(char *roomname, long msgid, int flags) {
 long send_message(struct CtdlMessage *msg) {
        long newmsgid;
        long retval;
-       char msgidbuf[SIZ];
-        struct ser_ret smr;
+       char msgidbuf[256];
+       struct ser_ret smr;
        int is_bigmsg = 0;
        char *holdM = NULL;
 
        /* Get a new message number */
        newmsgid = get_new_message_number();
-       snprintf(msgidbuf, sizeof msgidbuf, "%ld@%s", newmsgid, config.c_fqdn);
+       snprintf(msgidbuf, sizeof msgidbuf, "%010ld@%s", newmsgid, config.c_fqdn);
 
        /* Generate an ID if we don't have one already */
        if (msg->cm_fields['I']==NULL) {
@@ -1784,22 +2363,22 @@ long send_message(struct CtdlMessage *msg) {
        }
 
        /* Serialize our data structure for storage in the database */  
-        serialize_message(&smr, msg);
+       serialize_message(&smr, msg);
 
        if (is_bigmsg) {
                msg->cm_fields['M'] = holdM;
        }
 
-        if (smr.len == 0) {
-                cprintf("%d Unable to serialize message\n",
-                        ERROR + INTERNAL_ERROR);
-                return (-1L);
-        }
+       if (smr.len == 0) {
+               cprintf("%d Unable to serialize message\n",
+                       ERROR + INTERNAL_ERROR);
+               return (-1L);
+       }
 
        /* Write our little bundle of joy into the message base */
        if (cdb_store(CDB_MSGMAIN, &newmsgid, (int)sizeof(long),
                      smr.ser, smr.len) < 0) {
-               lprintf(CTDL_ERR, "Can't store message\n");
+               CtdlLogPrintf(CTDL_ERR, "Can't store message\n");
                retval = 0L;
        } else {
                if (is_bigmsg) {
@@ -1814,7 +2393,7 @@ long send_message(struct CtdlMessage *msg) {
        }
 
        /* Free the memory we used for the serialized message */
-        free(smr.ser);
+       free(smr.ser);
 
        /* Return the *local* message ID to the caller
         * (even if we're storing an incoming network message)
@@ -1834,21 +2413,31 @@ long send_message(struct CtdlMessage *msg) {
 void serialize_message(struct ser_ret *ret,            /* return values */
                        struct CtdlMessage *msg)        /* unserialized msg */
 {
-       size_t wlen;
+       size_t wlen, fieldlen;
        int i;
        static char *forder = FORDER;
 
-       if (is_valid_message(msg) == 0) return;         /* self check */
+       /*
+        * Check for valid message format
+        */
+       if (is_valid_message(msg) == 0) {
+               CtdlLogPrintf(CTDL_ERR, "serialize_message() aborting due to invalid message\n");
+               ret->len = 0;
+               ret->ser = NULL;
+               return;
+       }
 
        ret->len = 3;
        for (i=0; i<26; ++i) if (msg->cm_fields[(int)forder[i]] != NULL)
                ret->len = ret->len +
                        strlen(msg->cm_fields[(int)forder[i]]) + 2;
 
-       lprintf(CTDL_DEBUG, "serialize_message() calling malloc(%ld)\n", (long)ret->len);
        ret->ser = malloc(ret->len);
        if (ret->ser == NULL) {
+               CtdlLogPrintf(CTDL_ERR, "serialize_message() malloc(%ld) failed: %s\n",
+                       (long)ret->len, strerror(errno));
                ret->len = 0;
+               ret->ser = NULL;
                return;
        }
 
@@ -1858,96 +2447,94 @@ void serialize_message(struct ser_ret *ret,             /* return values */
        wlen = 3;
 
        for (i=0; i<26; ++i) if (msg->cm_fields[(int)forder[i]] != NULL) {
+               fieldlen = strlen(msg->cm_fields[(int)forder[i]]);
                ret->ser[wlen++] = (char)forder[i];
-               strcpy((char *)&ret->ser[wlen], msg->cm_fields[(int)forder[i]]);
-               wlen = wlen + strlen(msg->cm_fields[(int)forder[i]]) + 1;
+               safestrncpy((char *)&ret->ser[wlen], msg->cm_fields[(int)forder[i]], fieldlen+1);
+               wlen = wlen + fieldlen + 1;
        }
-       if (ret->len != wlen) lprintf(CTDL_ERR, "ERROR: len=%ld wlen=%ld\n",
+       if (ret->len != wlen) CtdlLogPrintf(CTDL_ERR, "ERROR: len=%ld wlen=%ld\n",
                (long)ret->len, (long)wlen);
 
        return;
 }
 
 
-
 /*
- * Back end for the ReplicationChecks() function
+ * Serialize a struct CtdlMessage into the format used on disk and network.
+ * 
+ * This function loads up a "struct ser_ret" (defined in server.h) which
+ * contains the length of the serialized message and a pointer to the
+ * serialized message in memory.  THE LATTER MUST BE FREED BY THE CALLER.
  */
-void check_repl(long msgnum, void *userdata) {
-       struct CtdlMessage *msg;
-       time_t timestamp = (-1L);
-
-       lprintf(CTDL_DEBUG, "check_repl() found message %ld\n", msgnum);
-       msg = CtdlFetchMessage(msgnum, 1);
-       if (msg == NULL) return;
-       if (msg->cm_fields['T'] != NULL) {
-               timestamp = atol(msg->cm_fields['T']);
-       }
-       CtdlFreeMessage(msg);
+void dump_message(struct CtdlMessage *msg,     /* unserialized msg */
+                 long Siz)                     /* how many chars ? */
+{
+       size_t wlen;
+       int i;
+       static char *forder = FORDER;
+       char *buf;
 
-       if (timestamp > msg_repl->highest) {
-               msg_repl->highest = timestamp;  /* newer! */
-               lprintf(CTDL_DEBUG, "newer!\n");
+       /*
+        * Check for valid message format
+        */
+       if (is_valid_message(msg) == 0) {
+               CtdlLogPrintf(CTDL_ERR, "dump_message() aborting due to invalid message\n");
                return;
        }
-       lprintf(CTDL_DEBUG, "older!\n");
 
-       /* Existing isn't newer?  Then delete the old one(s). */
-       CtdlDeleteMessages(CC->room.QRname, msgnum, "");
+       buf = (char*) malloc (Siz + 1);
+
+       wlen = 3;
+       
+       for (i=0; i<26; ++i) if (msg->cm_fields[(int)forder[i]] != NULL) {
+                       snprintf (buf, Siz, " msg[%c] = %s ...\n", (char) forder[i], 
+                                  msg->cm_fields[(int)forder[i]]);
+                       client_write (buf, strlen(buf));
+               }
+
+       return;
 }
 
 
+
 /*
- * Check to see if any messages already exist which carry the same Extended ID
- * as this one.  
- *
- * If any are found:
- * -> With older timestamps: delete them and return 0.  Message will be saved.
- * -> With newer timestamps: return 1.  Message save will be aborted.
+ * Check to see if any messages already exist in the current room which
+ * carry the same Exclusive ID as this one.  If any are found, delete them.
  */
-int ReplicationChecks(struct CtdlMessage *msg) {
-       struct CtdlMessage *template;
-       int abort_this = 0;
+void ReplicationChecks(struct CtdlMessage *msg) {
+       long old_msgnum = (-1L);
 
-       lprintf(CTDL_DEBUG, "ReplicationChecks() started\n");
-       /* No extended id?  Don't do anything. */
-       if (msg->cm_fields['E'] == NULL) return 0;
-       if (strlen(msg->cm_fields['E']) == 0) return 0;
-       lprintf(CTDL_DEBUG, "Extended ID: <%s>\n", msg->cm_fields['E']);
+       if (DoesThisRoomNeedEuidIndexing(&CC->room) == 0) return;
 
-       CtdlAllocUserData(SYM_REPL, sizeof(struct repl));
-       strcpy(msg_repl->extended_id, msg->cm_fields['E']);
-       msg_repl->highest = atol(msg->cm_fields['T']);
+       CtdlLogPrintf(CTDL_DEBUG, "Performing replication checks in <%s>\n",
+               CC->room.QRname);
 
-       template = (struct CtdlMessage *) malloc(sizeof(struct CtdlMessage));
-       memset(template, 0, sizeof(struct CtdlMessage));
-       template->cm_fields['E'] = strdup(msg->cm_fields['E']);
-
-       CtdlForEachMessage(MSGS_ALL, 0L, NULL, template, check_repl, NULL);
-
-       /* If a newer message exists with the same Extended ID, abort
-        * this save.
-        */
-       if (msg_repl->highest > atol(msg->cm_fields['T']) ) {
-               abort_this = 1;
-               }
+       /* No exclusive id?  Don't do anything. */
+       if (msg == NULL) return;
+       if (msg->cm_fields['E'] == NULL) return;
+       if (IsEmptyStr(msg->cm_fields['E'])) return;
+       /*CtdlLogPrintf(CTDL_DEBUG, "Exclusive ID: <%s> for room <%s>\n",
+               msg->cm_fields['E'], CC->room.QRname);*/
 
-       CtdlFreeMessage(template);
-       lprintf(CTDL_DEBUG, "ReplicationChecks() returning %d\n", abort_this);
-       return(abort_this);
+       old_msgnum = locate_message_by_euid(msg->cm_fields['E'], &CC->room);
+       if (old_msgnum > 0L) {
+               CtdlLogPrintf(CTDL_DEBUG, "ReplicationChecks() replacing message %ld\n", old_msgnum);
+               CtdlDeleteMessages(CC->room.QRname, &old_msgnum, 1, "");
+       }
 }
 
 
 
-
 /*
  * Save a message to disk and submit it into the delivery system.
  */
 long CtdlSubmitMsg(struct CtdlMessage *msg,    /* message to save */
-               struct recptypes *recps,        /* recipients (if mail) */
-               char *force                     /* force a particular room? */
+                  struct recptypes *recps,     /* recipients (if mail) */
+                  char *force,                 /* force a particular room? */
+                  int flags                    /* should the bessage be exported clean? */
 ) {
-       char aaa[SIZ];
+       char submit_filename[128];
+       char generated_timestamp[32];
        char hold_rm[ROOMNAMELEN];
        char actual_rm[ROOMNAMELEN];
        char force_room[ROOMNAMELEN];
@@ -1961,29 +2548,33 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,     /* message to save */
        FILE *network_fp = NULL;
        static int seqnum = 1;
        struct CtdlMessage *imsg = NULL;
-       char *instr;
+       char *instr = NULL;
+       size_t instr_alloc = 0;
        struct ser_ret smr;
        char *hold_R, *hold_D;
+       char *collected_addresses = NULL;
+       struct addresses_to_be_filed *aptr = NULL;
+       char *saved_rfc822_version = NULL;
+       int qualified_for_journaling = 0;
+       struct CitContext *CCC = CC;            /* CachedCitContext - performance boost */
 
-       lprintf(CTDL_DEBUG, "CtdlSubmitMsg() called\n");
+       CtdlLogPrintf(CTDL_DEBUG, "CtdlSubmitMsg() called\n");
        if (is_valid_message(msg) == 0) return(-1);     /* self check */
 
        /* If this message has no timestamp, we take the liberty of
         * giving it one, right now.
         */
        if (msg->cm_fields['T'] == NULL) {
-               lprintf(CTDL_DEBUG, "Generating timestamp\n");
-               snprintf(aaa, sizeof aaa, "%ld", (long)time(NULL));
-               msg->cm_fields['T'] = strdup(aaa);
+               snprintf(generated_timestamp, sizeof generated_timestamp, "%ld", (long)time(NULL));
+               msg->cm_fields['T'] = strdup(generated_timestamp);
        }
 
        /* If this message has no path, we generate one.
         */
        if (msg->cm_fields['P'] == NULL) {
-               lprintf(CTDL_DEBUG, "Generating path\n");
                if (msg->cm_fields['A'] != NULL) {
                        msg->cm_fields['P'] = strdup(msg->cm_fields['A']);
-                       for (a=0; a<strlen(msg->cm_fields['P']); ++a) {
+                       for (a=0; !IsEmptyStr(&msg->cm_fields['P'][a]); ++a) {
                                if (isspace(msg->cm_fields['P'][a])) {
                                        msg->cm_fields['P'][a] = ' ';
                                }
@@ -2002,10 +2593,9 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,      /* message to save */
        }
 
        /* Learn about what's inside, because it's what's inside that counts */
-       lprintf(CTDL_DEBUG, "Learning what's inside\n");
        if (msg->cm_fields['M'] == NULL) {
-               lprintf(CTDL_ERR, "ERROR: attempt to save message with NULL body\n");
-               return(-1);
+               CtdlLogPrintf(CTDL_ERR, "ERROR: attempt to save message with NULL body\n");
+               return(-2);
        }
 
        switch (msg->cm_format_type) {
@@ -2017,119 +2607,148 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,   /* message to save */
                break;
        case 4:
                strcpy(content_type, "text/plain");
-               /* advance past header fields */
-               mptr = msg->cm_fields['M'];
-               a = strlen(mptr);
-               while ((--a) > 0) {
-                       if (!strncasecmp(mptr, "Content-type: ", 14)) {
-                               safestrncpy(content_type, mptr,
-                                           sizeof(content_type));
-                               strcpy(content_type, &content_type[14]);
-                               for (a = 0; a < strlen(content_type); ++a)
-                                       if ((content_type[a] == ';')
-                                           || (content_type[a] == ' ')
-                                           || (content_type[a] == 13)
-                                           || (content_type[a] == 10))
-                                               content_type[a] = 0;
-                               break;
+               mptr = bmstrcasestr(msg->cm_fields['M'], "Content-type:");
+               if (mptr != NULL) {
+                       char *aptr;
+                       safestrncpy(content_type, &mptr[13], sizeof content_type);
+                       striplt(content_type);
+                       aptr = content_type;
+                       while (!IsEmptyStr(aptr)) {
+                               if ((*aptr == ';')
+                                   || (*aptr == ' ')
+                                   || (*aptr == 13)
+                                   || (*aptr == 10)) {
+                                       *aptr = 0;
+                               }
+                               else aptr++;
                        }
-                       ++mptr;
                }
        }
 
        /* Goto the correct room */
-       lprintf(CTDL_DEBUG, "Selected room %s\n", (recps) ? CC->room.QRname : SENTITEMS);
-       strcpy(hold_rm, CC->room.QRname);
-       strcpy(actual_rm, CC->room.QRname);
+       CtdlLogPrintf(CTDL_DEBUG, "Selected room %s\n", (recps) ? CCC->room.QRname : SENTITEMS);
+       strcpy(hold_rm, CCC->room.QRname);
+       strcpy(actual_rm, CCC->room.QRname);
        if (recps != NULL) {
                strcpy(actual_rm, SENTITEMS);
        }
 
        /* If the user is a twit, move to the twit room for posting */
-       lprintf(CTDL_DEBUG, "Handling twit stuff: %s\n",
-                       (CC->user.axlevel == 2) ? config.c_twitroom : "OK");
        if (TWITDETECT) {
-               if (CC->user.axlevel == 2) {
+               if (CCC->user.axlevel == 2) {
                        strcpy(hold_rm, actual_rm);
                        strcpy(actual_rm, config.c_twitroom);
+                       CtdlLogPrintf(CTDL_DEBUG, "Diverting to twit room\n");
                }
        }
 
        /* ...or if this message is destined for Aide> then go there. */
-       if (strlen(force_room) > 0) {
+       if (!IsEmptyStr(force_room)) {
                strcpy(actual_rm, force_room);
        }
 
-       lprintf(CTDL_DEBUG, "Final selection: %s\n", actual_rm);
-       if (strcasecmp(actual_rm, CC->room.QRname)) {
-               getroom(&CC->room, actual_rm);
+       CtdlLogPrintf(CTDL_DEBUG, "Final selection: %s\n", actual_rm);
+       if (strcasecmp(actual_rm, CCC->room.QRname)) {
+               /* getroom(&CCC->room, actual_rm); */
+               usergoto(actual_rm, 0, 1, NULL, NULL);
        }
 
        /*
         * If this message has no O (room) field, generate one.
         */
        if (msg->cm_fields['O'] == NULL) {
-               msg->cm_fields['O'] = strdup(CC->room.QRname);
+               msg->cm_fields['O'] = strdup(CCC->room.QRname);
        }
 
        /* Perform "before save" hooks (aborting if any return nonzero) */
-       lprintf(CTDL_DEBUG, "Performing before-save hooks\n");
-       if (PerformMessageHooks(msg, EVT_BEFORESAVE) > 0) return(-1);
+       CtdlLogPrintf(CTDL_DEBUG, "Performing before-save hooks\n");
+       if (PerformMessageHooks(msg, EVT_BEFORESAVE) > 0) return(-3);
 
-       /* If this message has an Extended ID, perform replication checks */
-       lprintf(CTDL_DEBUG, "Performing replication checks\n");
-       if (ReplicationChecks(msg) > 0) return(-1);
+       /*
+        * If this message has an Exclusive ID, and the room is replication
+        * checking enabled, then do replication checks.
+        */
+       if (DoesThisRoomNeedEuidIndexing(&CCC->room)) {
+               ReplicationChecks(msg);
+       }
 
        /* Save it to disk */
-       lprintf(CTDL_DEBUG, "Saving to disk\n");
+       CtdlLogPrintf(CTDL_DEBUG, "Saving to disk\n");
        newmsgid = send_message(msg);
-       if (newmsgid <= 0L) return(-1);
+       if (newmsgid <= 0L) return(-5);
 
        /* Write a supplemental message info record.  This doesn't have to
         * be a critical section because nobody else knows about this message
         * yet.
         */
-       lprintf(CTDL_DEBUG, "Creating MetaData record\n");
+       CtdlLogPrintf(CTDL_DEBUG, "Creating MetaData record\n");
        memset(&smi, 0, sizeof(struct MetaData));
        smi.meta_msgnum = newmsgid;
        smi.meta_refcount = 0;
-       safestrncpy(smi.meta_content_type, content_type, 64);
+       safestrncpy(smi.meta_content_type, content_type,
+                       sizeof smi.meta_content_type);
+
+       /*
+        * Measure how big this message will be when rendered as RFC822.
+        * We do this for two reasons:
+        * 1. We need the RFC822 length for the new metadata record, so the
+        *    POP and IMAP services don't have to calculate message lengths
+        *    while the user is waiting (multiplied by potentially hundreds
+        *    or thousands of messages).
+        * 2. If journaling is enabled, we will need an RFC822 version of the
+        *    message to attach to the journalized copy.
+        */
+       if (CCC->redirect_buffer != NULL) {
+               CtdlLogPrintf(CTDL_ALERT, "CCC->redirect_buffer is not NULL during message submission!\n");
+               abort();
+       }
+       CCC->redirect_buffer = malloc(SIZ);
+       CCC->redirect_len = 0;
+       CCC->redirect_alloc = SIZ;
+       CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, QP_EADDR);
+       smi.meta_rfc822_length = CCC->redirect_len;
+       saved_rfc822_version = CCC->redirect_buffer;
+       CCC->redirect_buffer = NULL;
+       CCC->redirect_len = 0;
+       CCC->redirect_alloc = 0;
+
        PutMetaData(&smi);
 
        /* Now figure out where to store the pointers */
-       lprintf(CTDL_DEBUG, "Storing pointers\n");
+       CtdlLogPrintf(CTDL_DEBUG, "Storing pointers\n");
 
        /* If this is being done by the networker delivering a private
         * message, we want to BYPASS saving the sender's copy (because there
         * is no local sender; it would otherwise go to the Trashcan).
         */
-       if ((!CC->internal_pgm) || (recps == NULL)) {
-               if (CtdlSaveMsgPointerInRoom(actual_rm, newmsgid, 0) != 0) {
-                       lprintf(CTDL_ERR, "ERROR saving message pointer!\n");
-                       CtdlSaveMsgPointerInRoom(config.c_aideroom, newmsgid, 0);
+       if ((!CCC->internal_pgm) || (recps == NULL)) {
+               if (CtdlSaveMsgPointerInRoom(actual_rm, newmsgid, 1, msg) != 0) {
+                       CtdlLogPrintf(CTDL_ERR, "ERROR saving message pointer!\n");
+                       CtdlSaveMsgPointerInRoom(config.c_aideroom, newmsgid, 0, msg);
                }
        }
 
        /* For internet mail, drop a copy in the outbound queue room */
        if (recps != NULL)
         if (recps->num_internet > 0) {
-               CtdlSaveMsgPointerInRoom(SMTP_SPOOLOUT_ROOM, newmsgid, 0);
+               CtdlSaveMsgPointerInRoom(SMTP_SPOOLOUT_ROOM, newmsgid, 0, msg);
        }
 
        /* If other rooms are specified, drop them there too. */
        if (recps != NULL)
         if (recps->num_room > 0)
          for (i=0; i<num_tokens(recps->recp_room, '|'); ++i) {
-               extract(recipient, recps->recp_room, i);
-               lprintf(CTDL_DEBUG, "Delivering to local room <%s>\n", recipient);
-               CtdlSaveMsgPointerInRoom(recipient, newmsgid, 0);
+               extract_token(recipient, recps->recp_room, i,
+                                       '|', sizeof recipient);
+               CtdlLogPrintf(CTDL_DEBUG, "Delivering to room <%s>\n", recipient);
+               CtdlSaveMsgPointerInRoom(recipient, newmsgid, 0, msg);
        }
 
        /* Bump this user's messages posted counter. */
-       lprintf(CTDL_DEBUG, "Updating user\n");
-       lgetuser(&CC->user, CC->curr_user);
-       CC->user.posted = CC->user.posted + 1;
-       lputuser(&CC->user);
+       CtdlLogPrintf(CTDL_DEBUG, "Updating user\n");
+       lgetuser(&CCC->user, CCC->curr_user);
+       CCC->user.posted = CCC->user.posted + 1;
+       lputuser(&CCC->user);
 
        /* If this is private, local mail, make a copy in the
         * recipient's mailbox and bump the reference count.
@@ -2137,22 +2756,51 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,     /* message to save */
        if (recps != NULL)
         if (recps->num_local > 0)
          for (i=0; i<num_tokens(recps->recp_local, '|'); ++i) {
-               extract(recipient, recps->recp_local, i);
-               lprintf(CTDL_DEBUG, "Delivering private local mail to <%s>\n",
+               extract_token(recipient, recps->recp_local, i,
+                                       '|', sizeof recipient);
+               CtdlLogPrintf(CTDL_DEBUG, "Delivering private local mail to <%s>\n",
                        recipient);
                if (getuser(&userbuf, recipient) == 0) {
+                       // Add a flag so the Funambol module knows its mail
+                       msg->cm_fields['W'] = strdup(recipient);
                        MailboxName(actual_rm, sizeof actual_rm, &userbuf, MAILROOM);
-                       CtdlSaveMsgPointerInRoom(actual_rm, newmsgid, 0);
+                       CtdlSaveMsgPointerInRoom(actual_rm, newmsgid, 0, msg);
                        BumpNewMailCounter(userbuf.usernum);
+                       if (!IsEmptyStr(config.c_funambol_host) || !IsEmptyStr(config.c_pager_program)) {
+                       /* Generate a instruction message for the Funambol notification
+                        * server, in the same style as the SMTP queue
+                        */
+                          instr_alloc = 1024;
+                          instr = malloc(instr_alloc);
+                          snprintf(instr, instr_alloc,
+                       "Content-type: %s\n\nmsgid|%ld\nsubmitted|%ld\n"
+                       "bounceto|%s@%s\n",
+                       SPOOLMIME, newmsgid, (long)time(NULL),
+                       msg->cm_fields['A'], msg->cm_fields['N']
+                       );
+
+                          imsg = malloc(sizeof(struct CtdlMessage));
+                          memset(imsg, 0, sizeof(struct CtdlMessage));
+                          imsg->cm_magic = CTDLMESSAGE_MAGIC;
+                          imsg->cm_anon_type = MES_NORMAL;
+                          imsg->cm_format_type = FMT_RFC822;
+                          imsg->cm_fields['A'] = strdup("Citadel");
+                          imsg->cm_fields['J'] = strdup("do not journal");
+                          imsg->cm_fields['M'] = instr;        /* imsg owns this memory now */
+                          imsg->cm_fields['W'] = strdup(recipient);
+                          CtdlSubmitMsg(imsg, NULL, FNBL_QUEUE_ROOM, 0);
+                          CtdlFreeMessage(imsg);
+                       }
                }
                else {
-                       lprintf(CTDL_DEBUG, "No user <%s>\n", recipient);
-                       CtdlSaveMsgPointerInRoom(config.c_aideroom, newmsgid, 0);
+                       CtdlLogPrintf(CTDL_DEBUG, "No user <%s>\n", recipient);
+                       CtdlSaveMsgPointerInRoom(config.c_aideroom,
+                               newmsgid, 0, msg);
                }
        }
 
        /* Perform "after save" hooks */
-       lprintf(CTDL_DEBUG, "Performing after-save hooks\n");
+       CtdlLogPrintf(CTDL_DEBUG, "Performing after-save hooks\n");
        PerformMessageHooks(msg, EVT_AFTERSAVE);
 
        /* For IGnet mail, we have to save a new copy into the spooler for
@@ -2166,21 +2814,23 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,     /* message to save */
        if (recps != NULL)
         if (recps->num_ignet > 0)
          for (i=0; i<num_tokens(recps->recp_ignet, '|'); ++i) {
-               extract(recipient, recps->recp_ignet, i);
+               extract_token(recipient, recps->recp_ignet, i,
+                               '|', sizeof recipient);
 
                hold_R = msg->cm_fields['R'];
                hold_D = msg->cm_fields['D'];
                msg->cm_fields['R'] = malloc(SIZ);
-               msg->cm_fields['D'] = malloc(SIZ);
-               extract_token(msg->cm_fields['R'], recipient, 0, '@');
-               extract_token(msg->cm_fields['D'], recipient, 1, '@');
+               msg->cm_fields['D'] = malloc(128);
+               extract_token(msg->cm_fields['R'], recipient, 0, '@', SIZ);
+               extract_token(msg->cm_fields['D'], recipient, 1, '@', 128);
                
                serialize_message(&smr, msg);
                if (smr.len > 0) {
-                       snprintf(aaa, sizeof aaa,
-                               "./network/spoolin/netmail.%04lx.%04x.%04x",
-                               (long) getpid(), CC->cs_pid, ++seqnum);
-                       network_fp = fopen(aaa, "wb+");
+                       snprintf(submit_filename, sizeof submit_filename,
+                                        "%s/netmail.%04lx.%04x.%04x",
+                                        ctdl_netin_dir,
+                                        (long) getpid(), CCC->cs_pid, ++seqnum);
+                       network_fp = fopen(submit_filename, "wb+");
                        if (network_fp != NULL) {
                                fwrite(smr.ser, smr.len, 1, network_fp);
                                fclose(network_fp);
@@ -2195,9 +2845,9 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
        }
 
        /* Go back to the room we started from */
-       lprintf(CTDL_DEBUG, "Returning to original room %s\n", hold_rm);
-       if (strcasecmp(hold_rm, CC->room.QRname))
-               getroom(&CC->room, hold_rm);
+       CtdlLogPrintf(CTDL_DEBUG, "Returning to original room %s\n", hold_rm);
+       if (strcasecmp(hold_rm, CCC->room.QRname))
+               usergoto(hold_rm, 0, 1, NULL, NULL);
 
        /* For internet mail, generate delivery instructions.
         * Yes, this is recursive.  Deal with it.  Infinite recursion does
@@ -2206,9 +2856,10 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,      /* message to save */
         */
        if (recps != NULL)
         if (recps->num_internet > 0) {
-               lprintf(CTDL_DEBUG, "Generating delivery instructions\n");
-               instr = malloc(SIZ * 2);
-               snprintf(instr, SIZ * 2,
+               CtdlLogPrintf(CTDL_DEBUG, "Generating delivery instructions\n");
+               instr_alloc = 1024;
+               instr = malloc(instr_alloc);
+               snprintf(instr, instr_alloc,
                        "Content-type: %s\n\nmsgid|%ld\nsubmitted|%ld\n"
                        "bounceto|%s@%s\n",
                        SPOOLMIME, newmsgid, (long)time(NULL),
@@ -2217,31 +2868,90 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,     /* message to save */
 
                for (i=0; i<num_tokens(recps->recp_internet, '|'); ++i) {
                        size_t tmp = strlen(instr);
-                       extract(recipient, recps->recp_internet, i);
-                       snprintf(&instr[tmp], SIZ * 2 - tmp,
-                                "remote|%s|0||\n", recipient);
+                       extract_token(recipient, recps->recp_internet, i, '|', sizeof recipient);
+                       if ((tmp + strlen(recipient) + 32) > instr_alloc) {
+                               instr_alloc = instr_alloc * 2;
+                               instr = realloc(instr, instr_alloc);
+                       }
+                       snprintf(&instr[tmp], instr_alloc - tmp, "remote|%s|0||\n", recipient);
                }
 
-               imsg = malloc(sizeof(struct CtdlMessage));
+               imsg = malloc(sizeof(struct CtdlMessage));
                memset(imsg, 0, sizeof(struct CtdlMessage));
                imsg->cm_magic = CTDLMESSAGE_MAGIC;
                imsg->cm_anon_type = MES_NORMAL;
                imsg->cm_format_type = FMT_RFC822;
                imsg->cm_fields['A'] = strdup("Citadel");
-               imsg->cm_fields['M'] = instr;
-               CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM);
+               imsg->cm_fields['J'] = strdup("do not journal");
+               imsg->cm_fields['M'] = instr;   /* imsg owns this memory now */
+               CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM, QP_EADDR);
                CtdlFreeMessage(imsg);
        }
 
+       /*
+        * Any addresses to harvest for someone's address book?
+        */
+       if ( (CCC->logged_in) && (recps != NULL) ) {
+               collected_addresses = harvest_collected_addresses(msg);
+       }
+
+       if (collected_addresses != NULL) {
+               begin_critical_section(S_ATBF);
+               aptr = (struct addresses_to_be_filed *)
+                       malloc(sizeof(struct addresses_to_be_filed));
+               aptr->next = atbf;
+               MailboxName(actual_rm, sizeof actual_rm,
+                       &CCC->user, USERCONTACTSROOM);
+               aptr->roomname = strdup(actual_rm);
+               aptr->collected_addresses = collected_addresses;
+               atbf = aptr;
+               end_critical_section(S_ATBF);
+       }
+
+       /*
+        * Determine whether this message qualifies for journaling.
+        */
+       if (msg->cm_fields['J'] != NULL) {
+               qualified_for_journaling = 0;
+       }
+       else {
+               if (recps == NULL) {
+                       qualified_for_journaling = config.c_journal_pubmsgs;
+               }
+               else if (recps->num_local + recps->num_ignet + recps->num_internet > 0) {
+                       qualified_for_journaling = config.c_journal_email;
+               }
+               else {
+                       qualified_for_journaling = config.c_journal_pubmsgs;
+               }
+       }
+
+       /*
+        * Do we have to perform journaling?  If so, hand off the saved
+        * RFC822 version will be handed off to the journaler for background
+        * submit.  Otherwise, we have to free the memory ourselves.
+        */
+       if (saved_rfc822_version != NULL) {
+               if (qualified_for_journaling) {
+                       JournalBackgroundSubmit(msg, saved_rfc822_version, recps);
+               }
+               else {
+                       free(saved_rfc822_version);
+               }
+       }
+
+       /* Done. */
        return(newmsgid);
 }
 
 
 
+
+
 /*
  * Convenience function for generating small administrative messages.
  */
-void quickie_message(char *from, char *to, char *room, char *text, 
+void quickie_message(char *from, char *fromaddr, char *to, char *room, char *text, 
                        int format_type, char *subject)
 {
        struct CtdlMessage *msg;
@@ -2252,21 +2962,35 @@ void quickie_message(char *from, char *to, char *room, char *text,
        msg->cm_magic = CTDLMESSAGE_MAGIC;
        msg->cm_anon_type = MES_NORMAL;
        msg->cm_format_type = format_type;
-       msg->cm_fields['A'] = strdup(from);
+
+       if (from != NULL) {
+               msg->cm_fields['A'] = strdup(from);
+       }
+       else if (fromaddr != NULL) {
+               msg->cm_fields['A'] = strdup(fromaddr);
+               if (strchr(msg->cm_fields['A'], '@')) {
+                       *strchr(msg->cm_fields['A'], '@') = 0;
+               }
+       }
+       else {
+               msg->cm_fields['A'] = strdup("Citadel");
+       }
+
+       if (fromaddr != NULL) msg->cm_fields['F'] = strdup(fromaddr);
        if (room != NULL) msg->cm_fields['O'] = strdup(room);
        msg->cm_fields['N'] = strdup(NODENAME);
        if (to != NULL) {
                msg->cm_fields['R'] = strdup(to);
-               recp = validate_recipients(to);
+               recp = validate_recipients(to, NULL, 0);
        }
        if (subject != NULL) {
                msg->cm_fields['U'] = strdup(subject);
        }
        msg->cm_fields['M'] = strdup(text);
 
-       CtdlSubmitMsg(msg, recp, room);
+       CtdlSubmitMsg(msg, recp, room, 0);
        CtdlFreeMessage(msg);
-       if (recp != NULL) free(recp);
+       if (recp != NULL) free_recipients(recp);
 }
 
 
@@ -2278,9 +3002,10 @@ char *CtdlReadMessageBody(char *terminator,      /* token signalling EOT */
                        size_t maxlen,          /* maximum message length */
                        char *exist,            /* if non-null, append to it;
                                                   exist is ALWAYS freed  */
-                       int crlf                /* CRLF newlines instead of LF */
+                       int crlf,               /* CRLF newlines instead of LF */
+                       int sock                /* socket handle or 0 for this session's client socket */
                        ) {
-       char buf[SIZ];
+       char buf[1024];
        int linelen;
        size_t message_len = 0;
        size_t buffer_len = 0;
@@ -2288,6 +3013,7 @@ char *CtdlReadMessageBody(char *terminator,       /* token signalling EOT */
        char *m;
        int flushing = 0;
        int finished = 0;
+       int dotdot = 0;
 
        if (exist == NULL) {
                m = malloc(4096);
@@ -2305,6 +3031,11 @@ char *CtdlReadMessageBody(char *terminator,      /* token signalling EOT */
                }
        }
 
+       /* Do we need to change leading ".." to "." for SMTP escaping? */
+       if (!strcmp(terminator, ".")) {
+               dotdot = 1;
+       }
+
        /* flush the input if we have nowhere to store it */
        if (m == NULL) {
                flushing = 1;
@@ -2312,7 +3043,12 @@ char *CtdlReadMessageBody(char *terminator,      /* token signalling EOT */
 
        /* read in the lines of message text one by one */
        do {
-               if (client_gets(buf) < 1) finished = 1;
+               if (sock > 0) {
+                       if (sock_getln(sock, buf, (sizeof buf - 3)) < 0) finished = 1;
+               }
+               else {
+                       if (client_getln(buf, (sizeof buf - 3)) < 1) finished = 1;
+               }
                if (!strcmp(buf, terminator)) finished = 1;
                if (crlf) {
                        strcat(buf, "\r\n");
@@ -2321,6 +3057,13 @@ char *CtdlReadMessageBody(char *terminator,      /* token signalling EOT */
                        strcat(buf, "\n");
                }
 
+               /* Unescape SMTP-style input of two dots at the beginning of the line */
+               if (dotdot) {
+                       if (!strncmp(buf, "..", 2)) {
+                               strcpy(buf, &buf[1]);
+                       }
+               }
+
                if ( (!flushing) && (!finished) ) {
                        /* Measure the line */
                        linelen = strlen(buf);
@@ -2333,7 +3076,7 @@ char *CtdlReadMessageBody(char *terminator,       /* token signalling EOT */
                                } else {
                                        buffer_len = (buffer_len * 2);
                                        m = ptr;
-                                       lprintf(CTDL_DEBUG, "buffer_len is now %ld\n", (long)buffer_len);
+                                       CtdlLogPrintf(CTDL_DEBUG, "buffer_len is now %ld\n", (long)buffer_len);
                                }
                        }
        
@@ -2367,15 +3110,19 @@ char *CtdlReadMessageBody(char *terminator,     /* token signalling EOT */
 struct CtdlMessage *CtdlMakeMessage(
        struct ctdluser *author,        /* author's user structure */
        char *recipient,                /* NULL if it's not mail */
+       char *recp_cc,                  /* NULL if it's not mail */
        char *room,                     /* room where it's going */
        int type,                       /* see MES_ types in header file */
        int format_type,                /* variformat, plain text, MIME... */
        char *fake_name,                /* who we're masquerading as */
+       char *my_email,                 /* which of my email addresses to use (empty is ok) */
        char *subject,                  /* Subject (optional) */
-       char *preformatted_text         /* ...or NULL to read text from client */
+       char *supplied_euid,            /* ...or NULL if this is irrelevant */
+       char *preformatted_text,        /* ...or NULL to read text from client */
+       char *references                /* Thread references */
 ) {
-       char dest_node[SIZ];
-       char buf[SIZ];
+       char dest_node[256];
+       char buf[1024];
        struct CtdlMessage *msg;
 
        msg = malloc(sizeof(struct CtdlMessage));
@@ -2388,9 +3135,19 @@ struct CtdlMessage *CtdlMakeMessage(
        strcpy(dest_node, "");
 
        striplt(recipient);
+       striplt(recp_cc);
+
+       /* Path or Return-Path */
+       if (my_email == NULL) my_email = "";
 
-       snprintf(buf, sizeof buf, "cit%ld", author->usernum);   /* Path */
-       msg->cm_fields['P'] = strdup(buf);
+       if (!IsEmptyStr(my_email)) {
+               msg->cm_fields['P'] = strdup(my_email);
+       }
+       else {
+               snprintf(buf, sizeof buf, "%s", author->fullname);
+               msg->cm_fields['P'] = strdup(buf);
+       }
+       convert_spaces_to_underscores(msg->cm_fields['P']);
 
        snprintf(buf, sizeof buf, "%ld", (long)time(NULL));     /* timestamp */
        msg->cm_fields['T'] = strdup(buf);
@@ -2413,18 +3170,49 @@ struct CtdlMessage *CtdlMakeMessage(
        if (recipient[0] != 0) {
                msg->cm_fields['R'] = strdup(recipient);
        }
+       if (recp_cc[0] != 0) {
+               msg->cm_fields['Y'] = strdup(recp_cc);
+       }
        if (dest_node[0] != 0) {
                msg->cm_fields['D'] = strdup(dest_node);
        }
 
-       if ( (author == &CC->user) && (strlen(CC->cs_inet_email) > 0) ) {
+       if (!IsEmptyStr(my_email)) {
+               msg->cm_fields['F'] = strdup(my_email);
+       }
+       else if ( (author == &CC->user) && (!IsEmptyStr(CC->cs_inet_email)) ) {
                msg->cm_fields['F'] = strdup(CC->cs_inet_email);
        }
 
        if (subject != NULL) {
+               long length;
                striplt(subject);
-               if (strlen(subject) > 0) {
-                       msg->cm_fields['U'] = strdup(subject);
+               length = strlen(subject);
+               if (length > 0) {
+                       long i;
+                       long IsAscii;
+                       IsAscii = -1;
+                       i = 0;
+                       while ((subject[i] != '\0') &&
+                              (IsAscii = isascii(subject[i]) != 0 ))
+                               i++;
+                       if (IsAscii != 0)
+                               msg->cm_fields['U'] = strdup(subject);
+                       else /* ok, we've got utf8 in the string. */
+                       {
+                               msg->cm_fields['U'] = rfc2047encode(subject, length);
+                       }
+
+               }
+       }
+
+       if (supplied_euid != NULL) {
+               msg->cm_fields['E'] = strdup(supplied_euid);
+       }
+
+       if (references != NULL) {
+               if (!IsEmptyStr(references)) {
+                       msg->cm_fields['W'] = strdup(references);
                }
        }
 
@@ -2432,8 +3220,7 @@ struct CtdlMessage *CtdlMakeMessage(
                msg->cm_fields['M'] = preformatted_text;
        }
        else {
-               msg->cm_fields['M'] = CtdlReadMessageBody("000",
-                                       config.c_maxmsglen, NULL, 0);
+               msg->cm_fields['M'] = CtdlReadMessageBody("000", config.c_maxmsglen, NULL, 0, 0);
        }
 
        return(msg);
@@ -2445,12 +3232,65 @@ struct CtdlMessage *CtdlMakeMessage(
  * room.  Returns a *CITADEL ERROR CODE* and puts a message in errmsgbuf, or
  * returns 0 on success.
  */
-int CtdlDoIHavePermissionToPostInThisRoom(char *errmsgbuf, size_t n) {
+int CtdlDoIHavePermissionToPostInThisRoom(char *errmsgbuf, 
+                                         size_t n, 
+                                         const char* RemoteIdentifier,
+                                         int PostPublic) {
+       int ra;
 
-       if (!(CC->logged_in)) {
+       if (!(CC->logged_in) && 
+           (PostPublic == POST_LOGGED_IN)) {
                snprintf(errmsgbuf, n, "Not logged in.");
                return (ERROR + NOT_LOGGED_IN);
        }
+       else if (PostPublic == CHECK_EXISTANCE) {
+               return (0); // We're Evaling whether a recipient exists
+       }
+       else if (!(CC->logged_in)) {
+               
+               if ((CC->room.QRflags & QR_READONLY)) {
+                       snprintf(errmsgbuf, n, "Not logged in.");
+                       return (ERROR + NOT_LOGGED_IN);
+               }
+               if (CC->room.QRflags2 & QR2_MODERATED) {
+                       snprintf(errmsgbuf, n, "Not logged in Moderation feature not yet implemented!");
+                       return (ERROR + NOT_LOGGED_IN);
+               }
+               if ((PostPublic!=POST_LMTP) &&(CC->room.QRflags2 & QR2_SMTP_PUBLIC) == 0) {
+                       SpoolControl *sc;
+                       char filename[SIZ];
+                       int found;
+
+                       if (RemoteIdentifier == NULL)
+                       {
+                               snprintf(errmsgbuf, n, "Need sender to permit access.");
+                               return (ERROR + USERNAME_REQUIRED);
+                       }
+
+                       assoc_file_name(filename, sizeof filename, &CC->room, ctdl_netcfg_dir);
+                       begin_critical_section(S_NETCONFIGS);
+                       if (!read_spoolcontrol_file(&sc, filename))
+                       {
+                               end_critical_section(S_NETCONFIGS);
+                               snprintf(errmsgbuf, n,
+                                       "This mailing list only accepts posts from subscribers.");
+                               return (ERROR + NO_SUCH_USER);
+                       }
+                       end_critical_section(S_NETCONFIGS);
+                       found = is_recipient (sc, RemoteIdentifier);
+                       free_spoolcontrol_struct(&sc);
+                       if (found) {
+                               return (0);
+                       }
+                       else {
+                               snprintf(errmsgbuf, n,
+                                       "This mailing list only accepts posts from subscribers.");
+                               return (ERROR + NO_SUCH_USER);
+                       }
+               }
+               return (0);
+
+       }
 
        if ((CC->user.axlevel < 2)
            && ((CC->room.QRflags & QR_MAILBOX) == 0)) {
@@ -2459,15 +3299,9 @@ int CtdlDoIHavePermissionToPostInThisRoom(char *errmsgbuf, size_t n) {
                return (ERROR + HIGHER_ACCESS_REQUIRED);
        }
 
-       if ((CC->user.axlevel < 4)
-          && (CC->room.QRflags & QR_NETWORK)) {
-               snprintf(errmsgbuf, n, "Need net privileges to enter here.");
-               return (ERROR + HIGHER_ACCESS_REQUIRED);
-       }
-
-       if ((CC->user.axlevel < 6)
-          && (CC->room.QRflags & QR_READONLY)) {
-               snprintf(errmsgbuf, n, "Sorry, this is a read-only room.");
+       CtdlRoomAccess(&CC->room, &CC->user, &ra, NULL);
+       if (!(ra & UA_POSTALLOWED)) {
+               snprintf(errmsgbuf, n, "Higher access is required to post in this room.");
                return (ERROR + HIGHER_ACCESS_REQUIRED);
        }
 
@@ -2499,60 +3333,105 @@ int CtdlCheckInternetMailPermission(struct ctdluser *who) {
 }
 
 
-
 /*
  * Validate recipients, count delivery types and errors, and handle aliasing
  * FIXME check for dupes!!!!!
+ *
+ * Returns 0 if all addresses are ok, ret->num_error = -1 if no addresses 
+ * were specified, or the number of addresses found invalid.
+ *
+ * Caller needs to free the result using free_recipients()
  */
-struct recptypes *validate_recipients(char *recipients) {
+struct recptypes *validate_recipients(char *supplied_recipients, 
+                                     const char *RemoteIdentifier, 
+                                     int Flags) {
        struct recptypes *ret;
-       char this_recp[SIZ];
-       char this_recp_cooked[SIZ];
+       char *recipients = NULL;
+       char this_recp[256];
+       char this_recp_cooked[256];
        char append[SIZ];
-       int num_recps;
+       int num_recps = 0;
        int i, j;
        int mailtype;
        int invalid;
        struct ctdluser tempUS;
        struct ctdlroom tempQR;
+       struct ctdlroom tempQR2;
+       int err = 0;
+       char errmsg[SIZ];
+       int in_quotes = 0;
 
        /* Initialize */
        ret = (struct recptypes *) malloc(sizeof(struct recptypes));
        if (ret == NULL) return(NULL);
-       memset(ret, 0, sizeof(struct recptypes));
 
-       ret->num_local = 0;
-       ret->num_internet = 0;
-       ret->num_ignet = 0;
-       ret->num_error = 0;
-       ret->num_room = 0;
+       /* Set all strings to null and numeric values to zero */
+       memset(ret, 0, sizeof(struct recptypes));
 
-       if (recipients == NULL) {
-               num_recps = 0;
-       }
-       else if (strlen(recipients) == 0) {
-               num_recps = 0;
+       if (supplied_recipients == NULL) {
+               recipients = strdup("");
        }
        else {
-               /* Change all valid separator characters to commas */
-               for (i=0; i<strlen(recipients); ++i) {
-                       if ((recipients[i] == ';') || (recipients[i] == '|')) {
-                               recipients[i] = ',';
-                       }
-               }
+               recipients = strdup(supplied_recipients);
+       }
+
+       /* Allocate some memory.  Yes, this allocates 500% more memory than we will
+        * actually need, but it's healthier for the heap than doing lots of tiny
+        * realloc() calls instead.
+        */
 
-               /* Count 'em up */
-               num_recps = num_tokens(recipients, ',');
+       ret->errormsg = malloc(strlen(recipients) + 1024);
+       ret->recp_local = malloc(strlen(recipients) + 1024);
+       ret->recp_internet = malloc(strlen(recipients) + 1024);
+       ret->recp_ignet = malloc(strlen(recipients) + 1024);
+       ret->recp_room = malloc(strlen(recipients) + 1024);
+       ret->display_recp = malloc(strlen(recipients) + 1024);
+
+       ret->errormsg[0] = 0;
+       ret->recp_local[0] = 0;
+       ret->recp_internet[0] = 0;
+       ret->recp_ignet[0] = 0;
+       ret->recp_room[0] = 0;
+       ret->display_recp[0] = 0;
+
+       ret->recptypes_magic = RECPTYPES_MAGIC;
+
+       /* Change all valid separator characters to commas */
+       for (i=0; !IsEmptyStr(&recipients[i]); ++i) {
+               if ((recipients[i] == ';') || (recipients[i] == '|')) {
+                       recipients[i] = ',';
+               }
        }
 
-       if (num_recps > 0) for (i=0; i<num_recps; ++i) {
-               extract_token(this_recp, recipients, i, ',');
+       /* Now start extracting recipients... */
+
+       while (!IsEmptyStr(recipients)) {
+
+               for (i=0; i<=strlen(recipients); ++i) {
+                       if (recipients[i] == '\"') in_quotes = 1 - in_quotes;
+                       if ( ( (recipients[i] == ',') && (!in_quotes) ) || (recipients[i] == 0) ) {
+                               safestrncpy(this_recp, recipients, i+1);
+                               this_recp[i] = 0;
+                               if (recipients[i] == ',') {
+                                       strcpy(recipients, &recipients[i+1]);
+                               }
+                               else {
+                                       strcpy(recipients, "");
+                               }
+                               break;
+                       }
+               }
+
                striplt(this_recp);
-               lprintf(CTDL_DEBUG, "Evaluating recipient #%d <%s>\n", i, this_recp);
+               if (IsEmptyStr(this_recp))
+                       break;
+               CtdlLogPrintf(CTDL_DEBUG, "Evaluating recipient #%d: %s\n", num_recps, this_recp);
+               ++num_recps;
                mailtype = alias(this_recp);
                mailtype = alias(this_recp);
                mailtype = alias(this_recp);
-               for (j=0; j<=strlen(this_recp); ++j) {
+               j = 0;
+               for (j=0; !IsEmptyStr(&this_recp[j]); ++j) {
                        if (this_recp[j]=='_') {
                                this_recp_cooked[j] = ' ';
                        }
@@ -2560,21 +3439,53 @@ struct recptypes *validate_recipients(char *recipients) {
                                this_recp_cooked[j] = this_recp[j];
                        }
                }
+               this_recp_cooked[j] = '\0';
                invalid = 0;
+               errmsg[0] = 0;
                switch(mailtype) {
                        case MES_LOCAL:
                                if (!strcasecmp(this_recp, "sysop")) {
                                        ++ret->num_room;
                                        strcpy(this_recp, config.c_aideroom);
-                                       if (strlen(ret->recp_room) > 0) {
+                                       if (!IsEmptyStr(ret->recp_room)) {
                                                strcat(ret->recp_room, "|");
                                        }
                                        strcat(ret->recp_room, this_recp);
                                }
+                               else if ( (!strncasecmp(this_recp, "room_", 5))
+                                     && (!getroom(&tempQR, &this_recp_cooked[5])) ) {
+
+                                       /* Save room so we can restore it later */
+                                       tempQR2 = CC->room;
+                                       CC->room = tempQR;
+                                       
+                                       /* Check permissions to send mail to this room */
+                                       err = CtdlDoIHavePermissionToPostInThisRoom(errmsg, 
+                                                                                   sizeof errmsg, 
+                                                                                   RemoteIdentifier,
+                                                                                   Flags
+                                       );
+                                       if (err)
+                                       {
+                                               ++ret->num_error;
+                                               invalid = 1;
+                                       } 
+                                       else {
+                                               ++ret->num_room;
+                                               if (!IsEmptyStr(ret->recp_room)) {
+                                                       strcat(ret->recp_room, "|");
+                                               }
+                                               strcat(ret->recp_room, &this_recp_cooked[5]);
+                                       }
+                                       
+                                       /* Restore room in case something needs it */
+                                       CC->room = tempQR2;
+
+                               }
                                else if (getuser(&tempUS, this_recp) == 0) {
                                        ++ret->num_local;
                                        strcpy(this_recp, tempUS.fullname);
-                                       if (strlen(ret->recp_local) > 0) {
+                                       if (!IsEmptyStr(ret->recp_local)) {
                                                strcat(ret->recp_local, "|");
                                        }
                                        strcat(ret->recp_local, this_recp);
@@ -2582,19 +3493,11 @@ struct recptypes *validate_recipients(char *recipients) {
                                else if (getuser(&tempUS, this_recp_cooked) == 0) {
                                        ++ret->num_local;
                                        strcpy(this_recp, tempUS.fullname);
-                                       if (strlen(ret->recp_local) > 0) {
+                                       if (!IsEmptyStr(ret->recp_local)) {
                                                strcat(ret->recp_local, "|");
                                        }
                                        strcat(ret->recp_local, this_recp);
                                }
-                               else if ( (!strncasecmp(this_recp, "room_", 5))
-                                     && (!getroom(&tempQR, &this_recp_cooked[5])) ) {
-                                       ++ret->num_room;
-                                       if (strlen(ret->recp_room) > 0) {
-                                               strcat(ret->recp_room, "|");
-                                       }
-                                       strcat(ret->recp_room, &this_recp_cooked[5]);
-                               }
                                else {
                                        ++ret->num_error;
                                        invalid = 1;
@@ -2607,13 +3510,13 @@ struct recptypes *validate_recipients(char *recipients) {
                                 * because if the address were valid, we would have
                                 * already translated it to a local address by now.
                                 */
-                               if (IsDirectory(this_recp)) {
+                               if (IsDirectory(this_recp, 0)) {
                                        ++ret->num_error;
                                        invalid = 1;
                                }
                                else {
                                        ++ret->num_internet;
-                                       if (strlen(ret->recp_internet) > 0) {
+                                       if (!IsEmptyStr(ret->recp_internet)) {
                                                strcat(ret->recp_internet, "|");
                                        }
                                        strcat(ret->recp_internet, this_recp);
@@ -2621,7 +3524,7 @@ struct recptypes *validate_recipients(char *recipients) {
                                break;
                        case MES_IGNET:
                                ++ret->num_ignet;
-                               if (strlen(ret->recp_ignet) > 0) {
+                               if (!IsEmptyStr(ret->recp_ignet)) {
                                        strcat(ret->recp_ignet, "|");
                                }
                                strcat(ret->recp_ignet, this_recp);
@@ -2632,26 +3535,25 @@ struct recptypes *validate_recipients(char *recipients) {
                                break;
                }
                if (invalid) {
-                       if (strlen(ret->errormsg) == 0) {
-                               snprintf(append, sizeof append,
-                                        "Invalid recipient: %s",
-                                        this_recp);
+                       if (IsEmptyStr(errmsg)) {
+                               snprintf(append, sizeof append, "Invalid recipient: %s", this_recp);
                        }
                        else {
-                               snprintf(append, sizeof append,
-                                        ", %s", this_recp);
+                               snprintf(append, sizeof append, "%s", errmsg);
                        }
-                       if ( (strlen(ret->errormsg) + strlen(append)) < SIZ) {
+                       if ( (strlen(ret->errormsg) + strlen(append) + 3) < SIZ) {
+                               if (!IsEmptyStr(ret->errormsg)) {
+                                       strcat(ret->errormsg, "; ");
+                               }
                                strcat(ret->errormsg, append);
                        }
                }
                else {
-                       if (strlen(ret->display_recp) == 0) {
+                       if (IsEmptyStr(ret->display_recp)) {
                                strcpy(append, this_recp);
                        }
                        else {
-                               snprintf(append, sizeof append, ", %s",
-                                        this_recp);
+                               snprintf(append, sizeof append, ", %s", this_recp);
                        }
                        if ( (strlen(ret->display_recp)+strlen(append)) < SIZ) {
                                strcat(ret->display_recp, append);
@@ -2661,21 +3563,46 @@ struct recptypes *validate_recipients(char *recipients) {
 
        if ((ret->num_local + ret->num_internet + ret->num_ignet +
           ret->num_room + ret->num_error) == 0) {
-               ++ret->num_error;
+               ret->num_error = (-1);
                strcpy(ret->errormsg, "No recipients specified.");
        }
 
-       lprintf(CTDL_DEBUG, "validate_recipients()\n");
-       lprintf(CTDL_DEBUG, " local: %d <%s>\n", ret->num_local, ret->recp_local);
-       lprintf(CTDL_DEBUG, "  room: %d <%s>\n", ret->num_room, ret->recp_room);
-       lprintf(CTDL_DEBUG, "  inet: %d <%s>\n", ret->num_internet, ret->recp_internet);
-       lprintf(CTDL_DEBUG, " ignet: %d <%s>\n", ret->num_ignet, ret->recp_ignet);
-       lprintf(CTDL_DEBUG, " error: %d <%s>\n", ret->num_error, ret->errormsg);
+       CtdlLogPrintf(CTDL_DEBUG, "validate_recipients()\n");
+       CtdlLogPrintf(CTDL_DEBUG, " local: %d <%s>\n", ret->num_local, ret->recp_local);
+       CtdlLogPrintf(CTDL_DEBUG, "  room: %d <%s>\n", ret->num_room, ret->recp_room);
+       CtdlLogPrintf(CTDL_DEBUG, "  inet: %d <%s>\n", ret->num_internet, ret->recp_internet);
+       CtdlLogPrintf(CTDL_DEBUG, " ignet: %d <%s>\n", ret->num_ignet, ret->recp_ignet);
+       CtdlLogPrintf(CTDL_DEBUG, " error: %d <%s>\n", ret->num_error, ret->errormsg);
 
+       free(recipients);
        return(ret);
 }
 
 
+/*
+ * Destructor for struct recptypes
+ */
+void free_recipients(struct recptypes *valid) {
+
+       if (valid == NULL) {
+               return;
+       }
+
+       if (valid->recptypes_magic != RECPTYPES_MAGIC) {
+               CtdlLogPrintf(CTDL_EMERG, "Attempt to call free_recipients() on some other data type!\n");
+               abort();
+       }
+
+       if (valid->errormsg != NULL)            free(valid->errormsg);
+       if (valid->recp_local != NULL)          free(valid->recp_local);
+       if (valid->recp_internet != NULL)       free(valid->recp_internet);
+       if (valid->recp_ignet != NULL)          free(valid->recp_ignet);
+       if (valid->recp_room != NULL)           free(valid->recp_room);
+       if (valid->display_recp != NULL)        free(valid->display_recp);
+       free(valid);
+}
+
+
 
 /*
  * message entry  -  mode 0 (normal)
@@ -2684,92 +3611,191 @@ void cmd_ent0(char *entargs)
 {
        int post = 0;
        char recp[SIZ];
-       char masquerade_as[SIZ];
+       char cc[SIZ];
+       char bcc[SIZ];
+       char supplied_euid[128];
        int anon_flag = 0;
        int format_type = 0;
-       char newusername[SIZ];
+       char newusername[256];
+       char newuseremail[256];
        struct CtdlMessage *msg;
        int anonymous = 0;
        char errmsg[SIZ];
        int err = 0;
        struct recptypes *valid = NULL;
+       struct recptypes *valid_to = NULL;
+       struct recptypes *valid_cc = NULL;
+       struct recptypes *valid_bcc = NULL;
        char subject[SIZ];
+       int subject_required = 0;
+       int do_confirm = 0;
+       long msgnum;
+       int i, j;
+       char buf[256];
+       int newuseremail_ok = 0;
+       char references[SIZ];
+       char *ptr;
 
        unbuffer_output();
 
        post = extract_int(entargs, 0);
-       extract(recp, entargs, 1);
+       extract_token(recp, entargs, 1, '|', sizeof recp);
        anon_flag = extract_int(entargs, 2);
        format_type = extract_int(entargs, 3);
-       extract(subject, entargs, 4);
+       extract_token(subject, entargs, 4, '|', sizeof subject);
+       extract_token(newusername, entargs, 5, '|', sizeof newusername);
+       do_confirm = extract_int(entargs, 6);
+       extract_token(cc, entargs, 7, '|', sizeof cc);
+       extract_token(bcc, entargs, 8, '|', sizeof bcc);
+       switch(CC->room.QRdefaultview) {
+               case VIEW_NOTES:
+               case VIEW_WIKI:
+                       extract_token(supplied_euid, entargs, 9, '|', sizeof supplied_euid);
+                       break;
+               default:
+                       supplied_euid[0] = 0;
+                       break;
+       }
+       extract_token(newuseremail, entargs, 10, '|', sizeof newuseremail);
+       extract_token(references, entargs, 11, '|', sizeof references);
+       for (ptr=references; *ptr != 0; ++ptr) {
+               if (*ptr == '!') *ptr = '|';
+       }
 
        /* first check to make sure the request is valid. */
 
-       err = CtdlDoIHavePermissionToPostInThisRoom(errmsg, sizeof errmsg);
-       if (err) {
+       err = CtdlDoIHavePermissionToPostInThisRoom(errmsg, sizeof errmsg, NULL, POST_LOGGED_IN);
+       if (err)
+       {
                cprintf("%d %s\n", err, errmsg);
                return;
        }
 
        /* Check some other permission type things. */
 
-       if (post == 2) {
-               if (CC->user.axlevel < 6) {
-                       cprintf("%d You don't have permission to masquerade.\n",
-                               ERROR + HIGHER_ACCESS_REQUIRED);
-                       return;
+       if (IsEmptyStr(newusername)) {
+               strcpy(newusername, CC->user.fullname);
+       }
+       if (  (CC->user.axlevel < 6)
+          && (strcasecmp(newusername, CC->user.fullname))
+          && (strcasecmp(newusername, CC->cs_inet_fn))
+       ) {     
+               cprintf("%d You don't have permission to author messages as '%s'.\n",
+                       ERROR + HIGHER_ACCESS_REQUIRED,
+                       newusername
+               );
+               return;
+       }
+
+
+       if (IsEmptyStr(newuseremail)) {
+               newuseremail_ok = 1;
+       }
+
+       if (!IsEmptyStr(newuseremail)) {
+               if (!strcasecmp(newuseremail, CC->cs_inet_email)) {
+                       newuseremail_ok = 1;
                }
-               extract(newusername, entargs, 5);
-               memset(CC->fake_postname, 0, sizeof(CC->fake_postname) );
-               safestrncpy(CC->fake_postname, newusername,
-                       sizeof(CC->fake_postname) );
-               cprintf("%d ok\n", CIT_OK);
+               else if (!IsEmptyStr(CC->cs_inet_other_emails)) {
+                       j = num_tokens(CC->cs_inet_other_emails, '|');
+                       for (i=0; i<j; ++i) {
+                               extract_token(buf, CC->cs_inet_other_emails, i, '|', sizeof buf);
+                               if (!strcasecmp(newuseremail, buf)) {
+                                       newuseremail_ok = 1;
+                               }
+                       }
+               }
+       }
+
+       if (!newuseremail_ok) {
+               cprintf("%d You don't have permission to author messages as '%s'.\n",
+                       ERROR + HIGHER_ACCESS_REQUIRED,
+                       newuseremail
+               );
                return;
        }
+
        CC->cs_flags |= CS_POSTING;
 
-       /* In the Mail> room we have to behave a little differently --
+       /* In mailbox rooms we have to behave a little differently --
         * make sure the user has specified at least one recipient.  Then
-        * validate the recipient(s).
+        * validate the recipient(s).  We do this for the Mail> room, as
+        * well as any room which has the "Mailbox" view set.
         */
-       if ( (CC->room.QRflags & QR_MAILBOX)
-          && (!strcasecmp(&CC->room.QRname[11], MAILROOM)) ) {
 
+       if (  ( (CC->room.QRflags & QR_MAILBOX) && (!strcasecmp(&CC->room.QRname[11], MAILROOM)) )
+          || ( (CC->room.QRflags & QR_MAILBOX) && (CC->curr_view == VIEW_MAILBOX) )
+       ) {
                if (CC->user.axlevel < 2) {
                        strcpy(recp, "sysop");
+                       strcpy(cc, "");
+                       strcpy(bcc, "");
+               }
+
+               valid_to = validate_recipients(recp, NULL, 0);
+               if (valid_to->num_error > 0) {
+                       cprintf("%d %s\n", ERROR + NO_SUCH_USER, valid_to->errormsg);
+                       free_recipients(valid_to);
+                       return;
                }
 
-               valid = validate_recipients(recp);
-               if (valid->num_error > 0) {
-                       cprintf("%d %s\n",
-                               ERROR + NO_SUCH_USER, valid->errormsg);
-                       free(valid);
+               valid_cc = validate_recipients(cc, NULL, 0);
+               if (valid_cc->num_error > 0) {
+                       cprintf("%d %s\n", ERROR + NO_SUCH_USER, valid_cc->errormsg);
+                       free_recipients(valid_to);
+                       free_recipients(valid_cc);
                        return;
                }
-               if (valid->num_internet > 0) {
+
+               valid_bcc = validate_recipients(bcc, NULL, 0);
+               if (valid_bcc->num_error > 0) {
+                       cprintf("%d %s\n", ERROR + NO_SUCH_USER, valid_bcc->errormsg);
+                       free_recipients(valid_to);
+                       free_recipients(valid_cc);
+                       free_recipients(valid_bcc);
+                       return;
+               }
+
+               /* Recipient required, but none were specified */
+               if ( (valid_to->num_error < 0) && (valid_cc->num_error < 0) && (valid_bcc->num_error < 0) ) {
+                       free_recipients(valid_to);
+                       free_recipients(valid_cc);
+                       free_recipients(valid_bcc);
+                       cprintf("%d At least one recipient is required.\n", ERROR + NO_SUCH_USER);
+                       return;
+               }
+
+               if (valid_to->num_internet + valid_cc->num_internet + valid_bcc->num_internet > 0) {
                        if (CtdlCheckInternetMailPermission(&CC->user)==0) {
                                cprintf("%d You do not have permission "
                                        "to send Internet mail.\n",
                                        ERROR + HIGHER_ACCESS_REQUIRED);
-                               free(valid);
+                               free_recipients(valid_to);
+                               free_recipients(valid_cc);
+                               free_recipients(valid_bcc);
                                return;
                        }
                }
 
-               if ( ( (valid->num_internet + valid->num_ignet) > 0)
+               if ( ( (valid_to->num_internet + valid_to->num_ignet + valid_cc->num_internet + valid_cc->num_ignet + valid_bcc->num_internet + valid_bcc->num_ignet) > 0)
                   && (CC->user.axlevel < 4) ) {
                        cprintf("%d Higher access required for network mail.\n",
                                ERROR + HIGHER_ACCESS_REQUIRED);
-                       free(valid);
+                       free_recipients(valid_to);
+                       free_recipients(valid_cc);
+                       free_recipients(valid_bcc);
                        return;
                }
        
-               if ((RESTRICT_INTERNET == 1) && (valid->num_internet > 0)
+               if ((RESTRICT_INTERNET == 1)
+                   && (valid_to->num_internet + valid_cc->num_internet + valid_bcc->num_internet > 0)
                    && ((CC->user.flags & US_INTERNET) == 0)
                    && (!CC->internal_pgm)) {
                        cprintf("%d You don't have access to Internet mail.\n",
                                ERROR + HIGHER_ACCESS_REQUIRED);
-                       free(valid);
+                       free_recipients(valid_to);
+                       free_recipients(valid_cc);
+                       free_recipients(valid_bcc);
                        return;
                }
 
@@ -2790,39 +3816,95 @@ void cmd_ent0(char *entargs)
                recp[0] = 0;
        }
 
+       /* Recommend to the client that the use of a message subject is
+        * strongly recommended in this room, if either the SUBJECTREQ flag
+        * is set, or if there is one or more Internet email recipients.
+        */
+       if (CC->room.QRflags2 & QR2_SUBJECTREQ) subject_required = 1;
+       if (valid_to) if (valid_to->num_internet > 0) subject_required = 1;
+       if (valid_cc) if (valid_cc->num_internet > 0) subject_required = 1;
+       if (valid_bcc) if (valid_bcc->num_internet > 0) subject_required = 1;
+
        /* If we're only checking the validity of the request, return
         * success without creating the message.
         */
        if (post == 0) {
-               cprintf("%d %s\n", CIT_OK,
-                       ((valid != NULL) ? valid->display_recp : "") );
-               free(valid);
+               cprintf("%d %s|%d\n", CIT_OK,
+                       ((valid_to != NULL) ? valid_to->display_recp : ""), 
+                       subject_required);
+               free_recipients(valid_to);
+               free_recipients(valid_cc);
+               free_recipients(valid_bcc);
                return;
        }
 
-       /* Handle author masquerading */
-       if (CC->fake_postname[0]) {
-               strcpy(masquerade_as, CC->fake_postname);
+       /* We don't need these anymore because we'll do it differently below */
+       free_recipients(valid_to);
+       free_recipients(valid_cc);
+       free_recipients(valid_bcc);
+
+       /* Read in the message from the client. */
+       if (do_confirm) {
+               cprintf("%d send message\n", START_CHAT_MODE);
+       } else {
+               cprintf("%d send message\n", SEND_LISTING);
        }
-       else if (CC->fake_username[0]) {
-               strcpy(masquerade_as, CC->fake_username);
+
+       msg = CtdlMakeMessage(&CC->user, recp, cc,
+               CC->room.QRname, anonymous, format_type,
+               newusername, newuseremail, subject,
+               ((!IsEmptyStr(supplied_euid)) ? supplied_euid : NULL),
+               NULL, references);
+
+       /* Put together one big recipients struct containing to/cc/bcc all in
+        * one.  This is for the envelope.
+        */
+       char *all_recps = malloc(SIZ * 3);
+       strcpy(all_recps, recp);
+       if (!IsEmptyStr(cc)) {
+               if (!IsEmptyStr(all_recps)) {
+                       strcat(all_recps, ",");
+               }
+               strcat(all_recps, cc);
+       }
+       if (!IsEmptyStr(bcc)) {
+               if (!IsEmptyStr(all_recps)) {
+                       strcat(all_recps, ",");
+               }
+               strcat(all_recps, bcc);
+       }
+       if (!IsEmptyStr(all_recps)) {
+               valid = validate_recipients(all_recps, NULL, 0);
        }
        else {
-               strcpy(masquerade_as, "");
+               valid = NULL;
        }
-
-       /* Read in the message from the client. */
-       cprintf("%d send message\n", SEND_LISTING);
-       msg = CtdlMakeMessage(&CC->user, recp,
-               CC->room.QRname, anonymous, format_type,
-               masquerade_as, subject, NULL);
+       free(all_recps);
 
        if (msg != NULL) {
-               CtdlSubmitMsg(msg, valid, "");
+               msgnum = CtdlSubmitMsg(msg, valid, "", QP_EADDR);
+
+               if (do_confirm) {
+                       cprintf("%ld\n", msgnum);
+                       if (msgnum >= 0L) {
+                               cprintf("Message accepted.\n");
+                       }
+                       else {
+                               cprintf("Internal error.\n");
+                       }
+                       if (msg->cm_fields['E'] != NULL) {
+                               cprintf("%s\n", msg->cm_fields['E']);
+                       } else {
+                               cprintf("\n");
+                       }
+                       cprintf("000\n");
+               }
+
                CtdlFreeMessage(msg);
        }
-       CC->fake_postname[0] = '\0';
-       free(valid);
+       if (valid != NULL) {
+               free_recipients(valid);
+       }
        return;
 }
 
@@ -2833,36 +3915,44 @@ void cmd_ent0(char *entargs)
  * (returns the actual number of messages deleted)
  */
 int CtdlDeleteMessages(char *room_name,                /* which room */
-                      long dmsgnum,            /* or "0" for any */
-                      char *content_type       /* or "" for any */
+                       long *dmsgnums,         /* array of msg numbers to be deleted */
+                       int num_dmsgnums,       /* number of msgs to be deleted, or 0 for "any" */
+                       char *content_type      /* or "" for any.  regular expressions expected. */
 )
 {
-
        struct ctdlroom qrbuf;
        struct cdbdata *cdbfr;
        long *msglist = NULL;
        long *dellist = NULL;
        int num_msgs = 0;
-       int i;
+       int i, j;
        int num_deleted = 0;
        int delete_this;
        struct MetaData smi;
+       regex_t re;
+       regmatch_t pm;
+       int need_to_free_re = 0;
 
-       lprintf(CTDL_DEBUG, "CtdlDeleteMessages(%s, %ld, %s)\n",
-               room_name, dmsgnum, content_type);
+       if (content_type) if (!IsEmptyStr(content_type)) {
+               regcomp(&re, content_type, 0);
+               need_to_free_re = 1;
+       }
+       CtdlLogPrintf(CTDL_DEBUG, "CtdlDeleteMessages(%s, %d msgs, %s)\n",
+               room_name, num_dmsgnums, content_type);
 
        /* get room record, obtaining a lock... */
        if (lgetroom(&qrbuf, room_name) != 0) {
-               lprintf(CTDL_ERR, "CtdlDeleteMessages(): Room <%s> not found\n",
+               CtdlLogPrintf(CTDL_ERR, "CtdlDeleteMessages(): Room <%s> not found\n",
                        room_name);
+               if (need_to_free_re) regfree(&re);
                return (0);     /* room not found */
        }
        cdbfr = cdb_fetch(CDB_MSGLISTS, &qrbuf.QRnumber, sizeof(long));
 
        if (cdbfr != NULL) {
-               msglist = malloc(cdbfr->len);
                dellist = malloc(cdbfr->len);
-               memcpy(msglist, cdbfr->ptr, cdbfr->len);
+               msglist = (long *) cdbfr->ptr;
+               cdbfr->ptr = NULL;      /* CtdlDeleteMessages() now owns this memory */
                num_msgs = cdbfr->len / sizeof(long);
                cdb_free(cdbfr);
        }
@@ -2872,15 +3962,25 @@ int CtdlDeleteMessages(char *room_name,         /* which room */
 
                        /* Set/clear a bit for each criterion */
 
-                       if ((dmsgnum == 0L) || (msglist[i] == dmsgnum)) {
+                       /* 0 messages in the list or a null list means that we are
+                        * interested in deleting any messages which meet the other criteria.
+                        */
+                       if ((num_dmsgnums == 0) || (dmsgnums == NULL)) {
                                delete_this |= 0x01;
                        }
-                       if (strlen(content_type) == 0) {
+                       else {
+                               for (j=0; j<num_dmsgnums; ++j) {
+                                       if (msglist[i] == dmsgnums[j]) {
+                                               delete_this |= 0x01;
+                                       }
+                               }
+                       }
+
+                       if (IsEmptyStr(content_type)) {
                                delete_this |= 0x02;
                        } else {
                                GetMetaData(&smi, msglist[i]);
-                               if (!strcasecmp(smi.meta_content_type,
-                                               content_type)) {
+                               if (regexec(&re, smi.meta_content_type, 1, &pm, 0) == 0) {
                                        delete_this |= 0x02;
                                }
                        }
@@ -2916,7 +4016,8 @@ int CtdlDeleteMessages(char *room_name,           /* which room */
        /* Now free the memory we used, and go away. */
        if (msglist != NULL) free(msglist);
        if (dellist != NULL) free(dellist);
-       lprintf(CTDL_DEBUG, "%d message(s) deleted.\n", num_deleted);
+       CtdlLogPrintf(CTDL_DEBUG, "%d message(s) deleted.\n", num_deleted);
+       if (need_to_free_re) regfree(&re);
        return (num_deleted);
 }
 
@@ -2927,52 +4028,68 @@ int CtdlDeleteMessages(char *room_name,         /* which room */
  * the current room (returns 1 for yes, 0 for no)
  */
 int CtdlDoIHavePermissionToDeleteMessagesFromThisRoom(void) {
-       getuser(&CC->user, CC->curr_user);
-       if ((CC->user.axlevel < 6)
-           && (CC->user.usernum != CC->room.QRroomaide)
-           && ((CC->room.QRflags & QR_MAILBOX) == 0)
-           && (!(CC->internal_pgm))) {
-               return(0);
-       }
-       return(1);
+       int ra;
+       CtdlRoomAccess(&CC->room, &CC->user, &ra, NULL);
+       if (ra & UA_DELETEALLOWED) return(1);
+       return(0);
 }
 
 
 
+
 /*
  * Delete message from current room
  */
-void cmd_dele(char *delstr)
+void cmd_dele(char *args)
 {
-       long delnum;
        int num_deleted;
+       int i;
+       char msgset[SIZ];
+       char msgtok[32];
+       long *msgs;
+       int num_msgs = 0;
+
+       extract_token(msgset, args, 0, '|', sizeof msgset);
+       num_msgs = num_tokens(msgset, ',');
+       if (num_msgs < 1) {
+               cprintf("%d Nothing to do.\n", CIT_OK);
+               return;
+       }
 
        if (CtdlDoIHavePermissionToDeleteMessagesFromThisRoom() == 0) {
                cprintf("%d Higher access required.\n",
                        ERROR + HIGHER_ACCESS_REQUIRED);
                return;
        }
-       delnum = extract_long(delstr, 0);
 
-       num_deleted = CtdlDeleteMessages(CC->room.QRname, delnum, "");
+       /*
+        * Build our message set to be moved/copied
+        */
+       msgs = malloc(num_msgs * sizeof(long));
+       for (i=0; i<num_msgs; ++i) {
+               extract_token(msgtok, msgset, i, ',', sizeof msgtok);
+               msgs[i] = atol(msgtok);
+       }
+
+       num_deleted = CtdlDeleteMessages(CC->room.QRname, msgs, num_msgs, "");
+       free(msgs);
 
        if (num_deleted) {
                cprintf("%d %d message%s deleted.\n", CIT_OK,
                        num_deleted, ((num_deleted != 1) ? "s" : ""));
        } else {
-               cprintf("%d Message %ld not found.\n", ERROR + MESSAGE_NOT_FOUND, delnum);
+               cprintf("%d Message not found.\n", ERROR + MESSAGE_NOT_FOUND);
        }
 }
 
 
 /*
- * Back end API function for moves and deletes
+ * Back end API function for moves and deletes (multiple messages)
  */
-int CtdlCopyMsgToRoom(long msgnum, char *dest) {
+int CtdlCopyMsgsToRoom(long *msgnums, int num_msgs, char *dest) {
        int err;
 
-       err = CtdlSaveMsgPointerInRoom(dest, msgnum,
-               (SM_VERIFY_GOODNESS | SM_DO_REPL_CHECK) );
+       err = CtdlSaveMsgPointersInRoom(dest, msgnums, num_msgs, 1, NULL);
        if (err != 0) return(err);
 
        return(0);
@@ -2980,21 +4097,34 @@ int CtdlCopyMsgToRoom(long msgnum, char *dest) {
 
 
 
+
 /*
  * move or copy a message to another room
  */
 void cmd_move(char *args)
 {
-       long num;
-       char targ[SIZ];
+       char msgset[SIZ];
+       char msgtok[32];
+       long *msgs;
+       int num_msgs = 0;
+
+       char targ[ROOMNAMELEN];
        struct ctdlroom qtemp;
        int err;
        int is_copy = 0;
        int ra;
        int permit = 0;
+       int i;
 
-       num = extract_long(args, 0);
-       extract(targ, args, 1);
+       extract_token(msgset, args, 0, '|', sizeof msgset);
+       num_msgs = num_tokens(msgset, ',');
+       if (num_msgs < 1) {
+               cprintf("%d Nothing to do.\n", CIT_OK);
+               return;
+       }
+
+       extract_token(targ, args, 1, '|', sizeof targ);
+       convert_room_name_macros(targ, sizeof targ);
        targ[ROOMNAMELEN - 1] = 0;
        is_copy = extract_int(args, 2);
 
@@ -3005,7 +4135,7 @@ void cmd_move(char *args)
        }
 
        getuser(&CC->user, CC->curr_user);
-       ra = CtdlRoomAccess(&qtemp, &CC->user);
+       CtdlRoomAccess(&qtemp, &CC->user, &ra, NULL);
 
        /* Check for permission to perform this operation.
         * Remember: "CC->room" is source, "qtemp" is target.
@@ -3027,6 +4157,13 @@ void cmd_move(char *args)
           && (!(CC->room.QRflags & QR_MAILBOX))
           && (qtemp.QRflags & QR_MAILBOX)) permit = 1;
 
+       /* Permit message removal from collaborative delete rooms */
+       if (CC->room.QRflags2 & QR2_COLLABDEL) permit = 1;
+
+       /* Users allowed to post into the target room may move into it too. */
+       if ((CC->room.QRflags & QR_MAILBOX) && 
+           (qtemp.QRflags & UA_POSTALLOWED))  permit = 1;
+
        /* User must have access to target room */
        if (!(ra & UA_KNOWN))  permit = 0;
 
@@ -3036,10 +4173,23 @@ void cmd_move(char *args)
                return;
        }
 
-       err = CtdlCopyMsgToRoom(num, targ);
+       /*
+        * Build our message set to be moved/copied
+        */
+       msgs = malloc(num_msgs * sizeof(long));
+       for (i=0; i<num_msgs; ++i) {
+               extract_token(msgtok, msgset, i, ',', sizeof msgtok);
+               msgs[i] = atol(msgtok);
+       }
+
+       /*
+        * Do the copy
+        */
+       err = CtdlCopyMsgsToRoom(msgs, num_msgs, targ);
        if (err != 0) {
-               cprintf("%d Cannot store message in %s: error %d\n",
+               cprintf("%d Cannot store message(s) in %s: error %d\n",
                        err, targ, err);
+               free(msgs);
                return;
        }
 
@@ -3047,10 +4197,11 @@ void cmd_move(char *args)
         * if this is a 'move' rather than a 'copy' operation.
         */
        if (is_copy == 0) {
-               CtdlDeleteMessages(CC->room.QRname, num, "");
+               CtdlDeleteMessages(CC->room.QRname, msgs, num_msgs, "");
        }
+       free(msgs);
 
-       cprintf("%d Message %s.\n", CIT_OK, (is_copy ? "copied" : "moved") );
+       cprintf("%d Message(s) %s.\n", CIT_OK, (is_copy ? "copied" : "moved") );
 }
 
 
@@ -3093,9 +4244,6 @@ void PutMetaData(struct MetaData *smibuf)
        /* Use the negative of the message number for the metadata db index */
        TheIndex = (0L - smibuf->meta_msgnum);
 
-       lprintf(CTDL_DEBUG, "PutMetaData(%ld) - ref count is %d\n",
-               smibuf->meta_msgnum, smibuf->meta_refcount);
-
        cdb_store(CDB_MSGMAIN,
                  &TheIndex, (int)sizeof(long),
                  smibuf, (int)sizeof(struct MetaData));
@@ -3103,10 +4251,112 @@ void PutMetaData(struct MetaData *smibuf)
 }
 
 /*
- * AdjRefCount  -  change the reference count for a message;
- *                 delete the message if it reaches zero
+ * AdjRefCount  -  submit an adjustment to the reference count for a message.
+ *                 (These are just queued -- we actually process them later.)
  */
 void AdjRefCount(long msgnum, int incr)
+{
+       struct arcq new_arcq;
+
+       begin_critical_section(S_SUPPMSGMAIN);
+       if (arcfp == NULL) {
+               arcfp = fopen(file_arcq, "ab+");
+       }
+       end_critical_section(S_SUPPMSGMAIN);
+
+       /* msgnum < 0 means that we're trying to close the file */
+       if (msgnum < 0) {
+               CtdlLogPrintf(CTDL_DEBUG, "Closing the AdjRefCount queue file\n");
+               begin_critical_section(S_SUPPMSGMAIN);
+               if (arcfp != NULL) {
+                       fclose(arcfp);
+                       arcfp = NULL;
+               }
+               end_critical_section(S_SUPPMSGMAIN);
+               return;
+       }
+
+       /*
+        * If we can't open the queue, perform the operation synchronously.
+        */
+       if (arcfp == NULL) {
+               TDAP_AdjRefCount(msgnum, incr);
+               return;
+       }
+
+       new_arcq.arcq_msgnum = msgnum;
+       new_arcq.arcq_delta = incr;
+       fwrite(&new_arcq, sizeof(struct arcq), 1, arcfp);
+       fflush(arcfp);
+
+       return;
+}
+
+
+/*
+ * TDAP_ProcessAdjRefCountQueue()
+ *
+ * Process the queue of message count adjustments that was created by calls
+ * to AdjRefCount() ... by reading the queue and calling TDAP_AdjRefCount()
+ * for each one.  This should be an "off hours" operation.
+ */
+int TDAP_ProcessAdjRefCountQueue(void)
+{
+       char file_arcq_temp[PATH_MAX];
+       int r;
+       FILE *fp;
+       struct arcq arcq_rec;
+       int num_records_processed = 0;
+
+       snprintf(file_arcq_temp, sizeof file_arcq_temp, "%s2", file_arcq);
+
+       begin_critical_section(S_SUPPMSGMAIN);
+       if (arcfp != NULL) {
+               fclose(arcfp);
+               arcfp = NULL;
+       }
+
+       r = link(file_arcq, file_arcq_temp);
+       if (r != 0) {
+               CtdlLogPrintf(CTDL_CRIT, "%s: %s\n", file_arcq_temp, strerror(errno));
+               end_critical_section(S_SUPPMSGMAIN);
+               return(num_records_processed);
+       }
+
+       unlink(file_arcq);
+       end_critical_section(S_SUPPMSGMAIN);
+
+       fp = fopen(file_arcq_temp, "rb");
+       if (fp == NULL) {
+               CtdlLogPrintf(CTDL_CRIT, "%s: %s\n", file_arcq_temp, strerror(errno));
+               return(num_records_processed);
+       }
+
+       while (fread(&arcq_rec, sizeof(struct arcq), 1, fp) == 1) {
+               TDAP_AdjRefCount(arcq_rec.arcq_msgnum, arcq_rec.arcq_delta);
+               ++num_records_processed;
+       }
+
+       fclose(fp);
+       r = unlink(file_arcq_temp);
+       if (r != 0) {
+               CtdlLogPrintf(CTDL_CRIT, "%s: %s\n", file_arcq_temp, strerror(errno));
+       }
+
+       return(num_records_processed);
+}
+
+
+
+/*
+ * TDAP_AdjRefCount  -  adjust the reference count for a message.
+ *                      This one does it "for real" because it's called by
+ *                      the autopurger function that processes the queue
+ *                      created by AdjRefCount().   If a message's reference
+ *                      count becomes zero, we also delete the message from
+ *                      disk and de-index it.
+ */
+void TDAP_AdjRefCount(long msgnum, int incr)
 {
 
        struct MetaData smi;
@@ -3118,27 +4368,31 @@ void AdjRefCount(long msgnum, int incr)
         */
        begin_critical_section(S_SUPPMSGMAIN);
        GetMetaData(&smi, msgnum);
-       lprintf(CTDL_DEBUG, "Ref count for message <%ld> before write is <%d>\n",
-               msgnum, smi.meta_refcount);
        smi.meta_refcount += incr;
        PutMetaData(&smi);
        end_critical_section(S_SUPPMSGMAIN);
-       lprintf(CTDL_DEBUG, "Ref count for message <%ld> after write is <%d>\n",
-               msgnum, smi.meta_refcount);
+       CtdlLogPrintf(CTDL_DEBUG, "msg %ld ref count delta %d, is now %d\n",
+               msgnum, incr, smi.meta_refcount);
 
        /* If the reference count is now zero, delete the message
         * (and its supplementary record as well).
         */
        if (smi.meta_refcount == 0) {
-               lprintf(CTDL_DEBUG, "Deleting message <%ld>\n", msgnum);
+               CtdlLogPrintf(CTDL_DEBUG, "Deleting message <%ld>\n", msgnum);
+               
+               /* Call delete hooks with NULL room to show it has gone altogether */
+               PerformDeleteHooks(NULL, msgnum);
+
+               /* Remove from message base */
                delnum = msgnum;
                cdb_delete(CDB_MSGMAIN, &delnum, (int)sizeof(long));
                cdb_delete(CDB_BIGMSGS, &delnum, (int)sizeof(long));
 
-               /* We have to delete the metadata record too! */
+               /* Remove metadata record */
                delnum = (0L - msgnum);
                cdb_delete(CDB_MSGMAIN, &delnum, (int)sizeof(long));
        }
+
 }
 
 /*
@@ -3166,23 +4420,23 @@ void CtdlWriteObject(char *req_room,            /* Room to stuff it in */
        char *encoded_message = NULL;
        off_t raw_length = 0;
 
-       if (is_mailbox != NULL)
+       if (is_mailbox != NULL) {
                MailboxName(roomname, sizeof roomname, is_mailbox, req_room);
-       else
+       }
+       else {
                safestrncpy(roomname, req_room, sizeof(roomname));
-       lprintf(CTDL_DEBUG, "CtdlWriteObject() to <%s> (flags=%d)\n", roomname, flags);
-
+       }
 
        fp = fopen(tempfilename, "rb");
        if (fp == NULL) {
-               lprintf(CTDL_CRIT, "Cannot open %s: %s\n",
+               CtdlLogPrintf(CTDL_CRIT, "Cannot open %s: %s\n",
                        tempfilename, strerror(errno));
                return;
        }
        fseek(fp, 0L, SEEK_END);
        raw_length = ftell(fp);
        rewind(fp);
-       lprintf(CTDL_DEBUG, "Raw length is %ld\n", (long)raw_length);
+       CtdlLogPrintf(CTDL_DEBUG, "Raw length is %ld\n", (long)raw_length);
 
        raw_message = malloc((size_t)raw_length + 2);
        fread(raw_message, (size_t)raw_length, 1, fp);
@@ -3213,7 +4467,8 @@ void CtdlWriteObject(char *req_room,              /* Room to stuff it in */
                CtdlEncodeBase64(
                        &encoded_message[strlen(encoded_message)],
                        raw_message,
-                       (int)raw_length
+                       (int)raw_length,
+                       0
                );
        }
        else {
@@ -3227,7 +4482,7 @@ void CtdlWriteObject(char *req_room,              /* Room to stuff it in */
 
        free(raw_message);
 
-       lprintf(CTDL_DEBUG, "Allocating\n");
+       CtdlLogPrintf(CTDL_DEBUG, "Allocating\n");
        msg = malloc(sizeof(struct CtdlMessage));
        memset(msg, 0, sizeof(struct CtdlMessage));
        msg->cm_magic = CTDLMESSAGE_MAGIC;
@@ -3251,11 +4506,12 @@ void CtdlWriteObject(char *req_room,            /* Room to stuff it in */
         * other objects of this type that are currently in the room.
         */
        if (is_unique) {
-               lprintf(CTDL_DEBUG, "Deleted %d other msgs of this type\n",
-                       CtdlDeleteMessages(roomname, 0L, content_type));
+               CtdlLogPrintf(CTDL_DEBUG, "Deleted %d other msgs of this type\n",
+                       CtdlDeleteMessages(roomname, NULL, 0, content_type)
+               );
        }
        /* Now write the data */
-       CtdlSubmitMsg(msg, NULL, roomname);
+       CtdlSubmitMsg(msg, NULL, roomname, 0);
        CtdlFreeMessage(msg);
 }
 
@@ -3286,7 +4542,7 @@ char *CtdlGetSysConfig(char *sysconfname) {
        /* We want the last (and probably only) config in this room */
        begin_critical_section(S_CONFIG);
        config_msgnum = (-1L);
-       CtdlForEachMessage(MSGS_LAST, 1, sysconfname, NULL,
+       CtdlForEachMessage(MSGS_LAST, 1, NULL, sysconfname, NULL,
                CtdlGetSysConfigBackend, NULL);
        msgnum = config_msgnum;
        end_critical_section(S_CONFIG);
@@ -3295,10 +4551,10 @@ char *CtdlGetSysConfig(char *sysconfname) {
                conf = NULL;
        }
        else {
-               msg = CtdlFetchMessage(msgnum, 1);
-               if (msg != NULL) {
-                       conf = strdup(msg->cm_fields['M']);
-                       CtdlFreeMessage(msg);
+               msg = CtdlFetchMessage(msgnum, 1);
+               if (msg != NULL) {
+                       conf = strdup(msg->cm_fields['M']);
+                       CtdlFreeMessage(msg);
                }
                else {
                        conf = NULL;
@@ -3308,9 +4564,9 @@ char *CtdlGetSysConfig(char *sysconfname) {
        getroom(&CC->room, hold_rm);
 
        if (conf != NULL) do {
-               extract_token(buf, conf, 0, '\n');
+               extract_token(buf, conf, 0, '\n', sizeof buf);
                strcpy(conf, &conf[strlen(buf)+1]);
-       } while ( (strlen(conf)>0) && (strlen(buf)>0) );
+       } while ( (!IsEmptyStr(conf)) && (!IsEmptyStr(buf)) );
 
        return(conf);
 }
@@ -3319,7 +4575,7 @@ void CtdlPutSysConfig(char *sysconfname, char *sysconfdata) {
        char temp[PATH_MAX];
        FILE *fp;
 
-       strcpy(temp, tmpnam(NULL));
+       CtdlMakeTempFileName(temp, sizeof temp);
 
        fp = fopen(temp, "w");
        if (fp == NULL) return;
@@ -3335,27 +4591,28 @@ void CtdlPutSysConfig(char *sysconfname, char *sysconfdata) {
 /*
  * Determine whether a given Internet address belongs to the current user
  */
-int CtdlIsMe(char *addr) {
+int CtdlIsMe(char *addr, int addr_buf_len)
+{
        struct recptypes *recp;
        int i;
 
-       recp = validate_recipients(addr);
+       recp = validate_recipients(addr, NULL, 0);
        if (recp == NULL) return(0);
 
        if (recp->num_local == 0) {
-               free(recp);
+               free_recipients(recp);
                return(0);
        }
 
        for (i=0; i<recp->num_local; ++i) {
-               extract(addr, recp->recp_local, i);
+               extract_token(addr, recp->recp_local, i, '|', addr_buf_len);
                if (!strcasecmp(addr, CC->user.fullname)) {
-                       free(recp);
+                       free_recipients(recp);
                        return(1);
                }
        }
 
-       free(recp);
+       free_recipients(recp);
        return(0);
 }
 
@@ -3364,12 +4621,12 @@ int CtdlIsMe(char *addr) {
  * Citadel protocol command to do the same
  */
 void cmd_isme(char *argbuf) {
-       char addr[SIZ];
+       char addr[256];
 
        if (CtdlAccessCheck(ac_logged_in)) return;
-       extract(addr, argbuf, 0);
+       extract_token(addr, argbuf, 0, '|', sizeof addr);
 
-       if (CtdlIsMe(addr)) {
+       if (CtdlIsMe(addr, sizeof addr)) {
                cprintf("%d %s\n", CIT_OK, addr);
        }
        else {