* New parameter to CtdlSaveMsgPointersInRoom() to suppress the reference count adjust...
[citadel.git] / citadel / msgbase.c
index da1498f2b715d7e3051679f99bcfac29c9fc1aae..98510111a8ff51f0d365914409b94594c236ad8a 100644 (file)
@@ -3,6 +3,21 @@
  *
  * Implements the message store.
  *
+ * Copyright (c) 1987-2010 by the citadel.org team
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
 #include "sysdep.h"
@@ -45,7 +60,6 @@
 #include "file_ops.h"
 #include "config.h"
 #include "control.h"
-#include "html.h"
 #include "genstamp.h"
 #include "internet_addressing.h"
 #include "euidindex.h"
@@ -55,6 +69,8 @@
 #include "serv_network.h"
 #include "threads.h"
 
+#include "ctdl_module.h"
+
 long config_msgnum;
 struct addresses_to_be_filed *atbf = NULL;
 
@@ -90,7 +106,8 @@ char *msgkeys[] = {
        "hnod",
        "msgn",
        "jrnl",
-       NULL, NULL,
+       NULL,
+       "list",
        "text",
        "node",
        "room",
@@ -101,7 +118,7 @@ char *msgkeys[] = {
        "time",
        "subj",
        NULL,
-       NULL,
+       "wefw",
        NULL,
        "cccc",
        NULL
@@ -183,7 +200,7 @@ int alias(char *name)
        }
 
        if (strcasecmp(original_name, name)) {
-               lprintf(CTDL_INFO, "%s is being forwarded to %s\n", original_name, name);
+               CtdlLogPrintf(CTDL_INFO, "%s is being forwarded to %s\n", original_name, name);
        }
 
        /* Change "user @ xxx" to "user" if xxx is an alias for this host */
@@ -191,7 +208,7 @@ int alias(char *name)
                if (name[a] == '@') {
                        if (CtdlHostAlias(&name[a+1]) == hostalias_localhost) {
                                name[a] = 0;
-                               lprintf(CTDL_INFO, "Changed to <%s>\n", name);
+                               CtdlLogPrintf(CTDL_INFO, "Changed to <%s>\n", name);
                        }
                }
        }
@@ -279,6 +296,25 @@ void headers_listing(long msgnum, void *userdata)
        CtdlFreeMessage(msg);
 }
 
+/*
+ * Back end for the MSGS command: output EUID header.
+ */
+void headers_euid(long msgnum, void *userdata)
+{
+       struct CtdlMessage *msg;
+
+       msg = CtdlFetchMessage(msgnum, 0);
+       if (msg == NULL) {
+               cprintf("%ld||\n", msgnum);
+               return;
+       }
+
+       cprintf("%ld|%s|\n", msgnum, (msg->cm_fields['E'] ? msg->cm_fields['E'] : ""));
+       CtdlFreeMessage(msg);
+}
+
+
+
 
 
 /* Determine if a given message matches the fields in a message template.
@@ -298,6 +334,8 @@ int CtdlMsgCmp(struct CtdlMessage *msg, struct CtdlMessage *template) {
        for (i='A'; i<='Z'; ++i) {
                if (template->cm_fields[i] != NULL) {
                        if (msg->cm_fields[i] == NULL) {
+                               /* Considered equal if temmplate is empty string */
+                               if (IsEmptyStr(template->cm_fields[i])) continue;
                                return 1;
                        }
                        if (strcasecmp(msg->cm_fields[i],
@@ -335,40 +373,47 @@ void CtdlSetSeen(long *target_msgnums, int num_target_msgnums,
                int target_setting, int which_set,
                struct ctdluser *which_user, struct ctdlroom *which_room) {
        struct cdbdata *cdbfr;
-       int i, j, k;
+       int i, k;
        int is_seen = 0;
        int was_seen = 0;
        long lo = (-1L);
        long hi = (-1L);
-       long t = (-1L);
-       int trimming = 0;
        struct visit vbuf;
        long *msglist;
        int num_msgs = 0;
-       char vset[SIZ];
+       StrBuf *vset;
+       StrBuf *setstr;
+       StrBuf *lostr;
+       StrBuf *histr;
+       const char *pvset;
        char *is_set;   /* actually an array of booleans */
-       int num_sets;
-       int s;
-       char setstr[SIZ], lostr[SIZ], histr[SIZ];
-       size_t tmp;
 
        /* Don't bother doing *anything* if we were passed a list of zero messages */
        if (num_target_msgnums < 1) {
                return;
        }
 
-       lprintf(CTDL_DEBUG, "CtdlSetSeen(%d msgs starting with %ld, %d, %d)\n",
+       /* If no room was specified, we go with the current room. */
+       if (!which_room) {
+               which_room = &CC->room;
+       }
+
+       /* If no user was specified, we go with the current user. */
+       if (!which_user) {
+               which_user = &CC->user;
+       }
+
+       CtdlLogPrintf(CTDL_DEBUG, "CtdlSetSeen(%d msgs starting with %ld, %s, %d) in <%s>\n",
                num_target_msgnums, target_msgnums[0],
-               target_setting, which_set);
+               (target_setting ? "SET" : "CLEAR"),
+               which_set,
+               which_room->QRname);
 
        /* Learn about the user and room in question */
-       CtdlGetRelationship(&vbuf,
-               ((which_user != NULL) ? which_user : &CC->user),
-               ((which_room != NULL) ? which_room : &CC->room)
-       );
+       CtdlGetRelationship(&vbuf, which_user, which_room);
 
        /* Load the message list */
-       cdbfr = cdb_fetch(CDB_MSGLISTS, &CC->room.QRnumber, sizeof(long));
+       cdbfr = cdb_fetch(CDB_MSGLISTS, &which_room->QRnumber, sizeof(long));
        if (cdbfr != NULL) {
                msglist = (long *) cdbfr->ptr;
                cdbfr->ptr = NULL;      /* CtdlSetSeen() now owns this memory */
@@ -383,33 +428,52 @@ void CtdlSetSeen(long *target_msgnums, int num_target_msgnums,
 
        /* Decide which message set we're manipulating */
        switch(which_set) {
-               case ctdlsetseen_seen:
-                       safestrncpy(vset, vbuf.v_seen, sizeof vset);
-                       break;
-               case ctdlsetseen_answered:
-                       safestrncpy(vset, vbuf.v_answered, sizeof vset);
-                       break;
+       case ctdlsetseen_seen:
+               vset = NewStrBufPlain(vbuf.v_seen, -1);
+               break;
+       case ctdlsetseen_answered:
+               vset = NewStrBufPlain(vbuf.v_answered, -1);
+               break;
+       default:
+               vset = NewStrBuf();
        }
 
-       /* lprintf(CTDL_DEBUG, "before optimize: %s\n", vset); */
+
+#if 0  /* This is a special diagnostic section.  Do not allow it to run during normal operation. */
+       CtdlLogPrintf(CTDL_DEBUG, "There are %d messages in the room.\n", num_msgs);
+       for (i=0; i<num_msgs; ++i) {
+               if ((i > 0) && (msglist[i] <= msglist[i-1])) abort();
+       }
+       CtdlLogPrintf(CTDL_DEBUG, "We are twiddling %d of them.\n", num_target_msgnums);
+       for (k=0; k<num_target_msgnums; ++k) {
+               if ((k > 0) && (target_msgnums[k] <= target_msgnums[k-1])) abort();
+       }
+#endif
+
+       CtdlLogPrintf(CTDL_DEBUG, "before update: %s\n", ChrPtr(vset));
 
        /* Translate the existing sequence set into an array of booleans */
-       num_sets = num_tokens(vset, ',');
-       for (s=0; s<num_sets; ++s) {
-               extract_token(setstr, vset, s, ',', sizeof setstr);
-
-               extract_token(lostr, setstr, 0, ':', sizeof lostr);
-               if (num_tokens(setstr, ':') >= 2) {
-                       extract_token(histr, setstr, 1, ':', sizeof histr);
-                       if (!strcmp(histr, "*")) {
-                               snprintf(histr, sizeof histr, "%ld", LONG_MAX);
-                       }
+       setstr = NewStrBuf();
+       lostr = NewStrBuf();
+       histr = NewStrBuf();
+       pvset = NULL;
+       while (StrBufExtract_NextToken(setstr, vset, &pvset, ',') >= 0) {
+
+               StrBufExtract_token(lostr, setstr, 0, ':');
+               if (StrBufNum_tokens(setstr, ':') >= 2) {
+                       StrBufExtract_token(histr, setstr, 1, ':');
+               }
+               else {
+                       FlushStrBuf(histr);
+                       StrBufAppendBuf(histr, lostr, 0);
+               }
+               lo = StrTol(lostr);
+               if (!strcmp(ChrPtr(histr), "*")) {
+                       hi = LONG_MAX;
                }
                else {
-                       strcpy(histr, lostr);
+                       hi = StrTol(histr);
                }
-               lo = atol(lostr);
-               hi = atol(histr);
 
                for (i = 0; i < num_msgs; ++i) {
                        if ((msglist[i] >= lo) && (msglist[i] <= hi)) {
@@ -417,85 +481,114 @@ void CtdlSetSeen(long *target_msgnums, int num_target_msgnums,
                        }
                }
        }
+       FreeStrBuf(&setstr);
+       FreeStrBuf(&lostr);
+       FreeStrBuf(&histr);
+
 
        /* Now translate the array of booleans back into a sequence set */
-       strcpy(vset, "");
-       lo = (-1L);
-       hi = (-1L);
+       FlushStrBuf(vset);
+       was_seen = 0;
+       lo = (-1);
+       hi = (-1);
 
        for (i=0; i<num_msgs; ++i) {
+               is_seen = is_set[i];
 
-               is_seen = is_set[i];    /* Default to existing setting */
-
+               /* Apply changes */
                for (k=0; k<num_target_msgnums; ++k) {
                        if (msglist[i] == target_msgnums[k]) {
                                is_seen = target_setting;
                        }
                }
 
-               if (is_seen) {
-                       if (lo < 0L) lo = msglist[i];
-                       hi = msglist[i];
+               if ((was_seen == 0) && (is_seen == 1)) {
+                       lo = msglist[i];
                }
+               else if ((was_seen == 1) && (is_seen == 0)) {
+                       hi = msglist[i-1];
 
-               if (  ((is_seen == 0) && (was_seen == 1))
-                  || ((is_seen == 1) && (i == num_msgs-1)) ) {
-
-                       /* begin trim-o-matic code */
-                       j=9;
-                       trimming = 0;
-                       while ( (strlen(vset) + 20) > sizeof vset) {
-                               remove_token(vset, 0, ',');
-                               trimming = 1;
-                               if (j--) break; /* loop no more than 9 times */
+                       if (StrLength(vset) > 0) {
+                               StrBufAppendBufPlain(vset, HKEY(","), 0);
+                       }
+                       if (lo == hi) {
+                               StrBufAppendPrintf(vset, "%ld", hi);
                        }
-                       if ( (trimming) && (which_set == ctdlsetseen_seen) ) {
-                               t = atol(vset);
-                               if (t<2) t=2;
-                               --t;
-                               snprintf(lostr, sizeof lostr,
-                                       "1:%ld,%s", t, vset);
-                               safestrncpy(vset, lostr, sizeof vset);
+                       else {
+                               StrBufAppendPrintf(vset, "%ld:%ld", lo, hi);
                        }
-                       /* end trim-o-matic code */
+               }
 
-                       tmp = strlen(vset);
-                       if (tmp > 0) {
-                               strcat(vset, ",");
-                               ++tmp;
+               if ((is_seen) && (i == num_msgs - 1)) {
+                       if (StrLength(vset) > 0) {
+                               StrBufAppendBufPlain(vset, HKEY(","), 0);
                        }
-                       if (lo == hi) {
-                               snprintf(&vset[tmp], (sizeof vset) - tmp,
-                                        "%ld", lo);
+                       if ((i==0) || (was_seen == 0)) {
+                               StrBufAppendPrintf(vset, "%ld", msglist[i]);
                        }
                        else {
-                               snprintf(&vset[tmp], (sizeof vset) - tmp,
-                                        "%ld:%ld", lo, hi);
+                               StrBufAppendPrintf(vset, "%ld:%ld", lo, msglist[i]);
                        }
-                       lo = (-1L);
-                       hi = (-1L);
                }
+
                was_seen = is_seen;
        }
 
+       /*
+        * We will have to stuff this string back into a 4096 byte buffer, so if it's
+        * larger than that now, truncate it by removing tokens from the beginning.
+        * The limit of 100 iterations is there to prevent an infinite loop in case
+        * something unexpected happens.
+        */
+       int number_of_truncations = 0;
+       while ( (StrLength(vset) > SIZ) && (number_of_truncations < 100) ) {
+               StrBufRemove_token(vset, 0, ',');
+               ++number_of_truncations;
+       }
+
+       /*
+        * If we're truncating the sequence set of messages marked with the 'seen' flag,
+        * we want the earliest messages (the truncated ones) to be marked, not unmarked.
+        * Otherwise messages at the beginning will suddenly appear to be 'unseen'.
+        */
+       if ( (which_set == ctdlsetseen_seen) && (number_of_truncations > 0) ) {
+               StrBuf *first_tok;
+               first_tok = NewStrBuf();
+               StrBufExtract_token(first_tok, vset, 0, ',');
+               StrBufRemove_token(vset, 0, ',');
+
+               if (StrBufNum_tokens(first_tok, ':') > 1) {
+                       StrBufRemove_token(first_tok, 0, ':');
+               }
+               
+               StrBuf *new_set;
+               new_set = NewStrBuf();
+               StrBufAppendBufPlain(new_set, HKEY("1:"), 0);
+               StrBufAppendBuf(new_set, first_tok, 0);
+               StrBufAppendBufPlain(new_set, HKEY(":"), 0);
+               StrBufAppendBuf(new_set, vset, 0);
+
+               FreeStrBuf(&vset);
+               FreeStrBuf(&first_tok);
+               vset = new_set;
+       }
+
+       CtdlLogPrintf(CTDL_DEBUG, " after update: %s\n", ChrPtr(vset));
+
        /* Decide which message set we're manipulating */
        switch (which_set) {
                case ctdlsetseen_seen:
-                       safestrncpy(vbuf.v_seen, vset, sizeof vbuf.v_seen);
+                       safestrncpy(vbuf.v_seen, ChrPtr(vset), sizeof vbuf.v_seen);
                        break;
                case ctdlsetseen_answered:
-                       safestrncpy(vbuf.v_answered, vset,
-                                               sizeof vbuf.v_answered);
+                       safestrncpy(vbuf.v_answered, ChrPtr(vset), sizeof vbuf.v_answered);
                        break;
        }
-       free(is_set);
 
-       /* lprintf(CTDL_DEBUG, " after optimize: %s\n", vset); */
+       free(is_set);
        free(msglist);
-       CtdlSetRelationship(&vbuf,
-               ((which_user != NULL) ? which_user : &CC->user),
-               ((which_room != NULL) ? which_room : &CC->room)
-       );
+       CtdlSetRelationship(&vbuf, which_user, which_room);
+       FreeStrBuf(&vset);
 }
 
 
@@ -506,7 +599,7 @@ void CtdlSetSeen(long *target_msgnums, int num_target_msgnums,
 int CtdlForEachMessage(int mode, long ref, char *search_string,
                        char *content_type,
                        struct CtdlMessage *compare,
-                       void (*CallBack) (long, void *),
+                        ForEachMsgCallback CallBack,
                        void *userdata)
 {
 
@@ -528,13 +621,13 @@ int CtdlForEachMessage(int mode, long ref, char *search_string,
        int need_to_free_re = 0;
        regmatch_t pm;
 
-       if (content_type) if (!IsEmptyStr(content_type)) {
+       if ((content_type) && (!IsEmptyStr(content_type))) {
                regcomp(&re, content_type, 0);
                need_to_free_re = 1;
        }
 
        /* Learn about the user and room in question */
-       getuser(&CC->user, CC->curr_user);
+       CtdlGetUser(&CC->user, CC->curr_user);
        CtdlGetRelationship(&vbuf, &CC->user, &CC->room);
 
        /* Load the message list */
@@ -556,7 +649,7 @@ int CtdlForEachMessage(int mode, long ref, char *search_string,
                /* If the caller is looking for a specific MIME type, filter
                 * out all messages which are not of the type requested.
                 */
-               if (content_type != NULL) if (!IsEmptyStr(content_type)) {
+               if ((content_type != NULL) && (!IsEmptyStr(content_type))) {
 
                        /* This call to GetMetaData() sits inside this loop
                         * so that we only do the extra database read per msg
@@ -664,6 +757,7 @@ int CtdlForEachMessage(int mode, long ref, char *search_string,
                                       || ((mode == MSGS_LAST) && (a >= (num_msgs - ref)))
                                   || ((mode == MSGS_FIRST) && (a < ref))
                                || ((mode == MSGS_GT) && (thismsg > ref))
+                               || ((mode == MSGS_LT) && (thismsg < ref))
                                || ((mode == MSGS_EQ) && (thismsg == ref))
                            )
                            ) {
@@ -699,14 +793,26 @@ void cmd_msgs(char *cmdbuf)
        int i;
        int with_template = 0;
        struct CtdlMessage *template = NULL;
-       int with_headers = 0;
        char search_string[1024];
+       ForEachMsgCallback CallBack;
 
        extract_token(which, cmdbuf, 0, '|', sizeof which);
        cm_ref = extract_int(cmdbuf, 1);
        extract_token(search_string, cmdbuf, 1, '|', sizeof search_string);
        with_template = extract_int(cmdbuf, 2);
-       with_headers = extract_int(cmdbuf, 3);
+       switch (extract_int(cmdbuf, 3))
+       {
+       default:
+       case MSG_HDRS_BRIEF:
+               CallBack = simple_listing;
+               break;
+       case MSG_HDRS_ALL:
+               CallBack = headers_listing;
+               break;
+       case MSG_HDRS_EUID:
+               CallBack = headers_euid;
+               break;
+       }
 
        strcat(which, "   ");
        if (!strncasecmp(which, "OLD", 3))
@@ -719,6 +825,8 @@ void cmd_msgs(char *cmdbuf)
                mode = MSGS_LAST;
        else if (!strncasecmp(which, "GT", 2))
                mode = MSGS_GT;
+       else if (!strncasecmp(which, "LT", 2))
+               mode = MSGS_LT;
        else if (!strncasecmp(which, "SEARCH", 6))
                mode = MSGS_SEARCH;
        else
@@ -745,7 +853,7 @@ void cmd_msgs(char *cmdbuf)
                template->cm_magic = CTDLMESSAGE_MAGIC;
                template->cm_anon_type = MES_NORMAL;
 
-               while(client_getln(buf, sizeof buf), strcmp(buf,"000")) {
+               while(client_getln(buf, sizeof buf) >= 0 && strcmp(buf,"000")) {
                        extract_token(tfield, buf, 0, '|', sizeof tfield);
                        extract_token(tvalue, buf, 1, '|', sizeof tvalue);
                        for (i='A'; i<='Z'; ++i) if (msgkeys[i]!=NULL) {
@@ -762,13 +870,12 @@ void cmd_msgs(char *cmdbuf)
        }
 
        CtdlForEachMessage(mode,
-                       ( (mode == MSGS_SEARCH) ? 0 : cm_ref ),
-                       ( (mode == MSGS_SEARCH) ? search_string : NULL ),
-                       NULL,
-                       template,
-                       (with_headers ? headers_listing : simple_listing),
-                       NULL
-       );
+                          ( (mode == MSGS_SEARCH) ? 0 : cm_ref ),
+                          ( (mode == MSGS_SEARCH) ? search_string : NULL ),
+                          NULL,
+                          template,
+                          CallBack,
+                          NULL);
        if (template != NULL) CtdlFreeMessage(template);
        cprintf("000\n");
 }
@@ -820,79 +927,56 @@ void do_help_subst(char *buffer)
  */
 void memfmout(
        char *mptr,             /* where are we going to get our text from? */
-       char subst,             /* nonzero if we should do substitutions */
-       char *nl)               /* string to terminate lines with */
-{
-       int a, b, c;
-       int real = 0;
-       int old = 0;
-       cit_uint8_t ch;
-       char aaa[140];
-       char buffer[SIZ];
-       static int width = 80;
-
-       strcpy(aaa, "");
-       old = 255;
-       strcpy(buffer, "");
-       c = 1;                  /* c is the current pos */
-
-       do {
-               if (subst) {
-                       while (ch = *mptr, ((ch != 0) && (strlen(buffer) < 126))) {
-                               ch = *mptr++;
-                               buffer[strlen(buffer) + 1] = 0;
-                               buffer[strlen(buffer)] = ch;
-                       }
-
-                       if (buffer[0] == '^')
-                               do_help_subst(buffer);
-
-                       buffer[strlen(buffer) + 1] = 0;
-                       a = buffer[0];
-                       strcpy(buffer, &buffer[1]);
-               } else {
-                       ch = *mptr++;
-               }
-
-               old = real;
-               real = ch;
-
-               if (((ch == 13) || (ch == 10)) && (old != 13) && (old != 10)) {
-                       ch = 32;
+       const char *nl          /* string to terminate lines with */
+) {
+       int column = 0;
+       char ch = 0;
+       char outbuf[1024];
+       int len = 0;
+       int nllen = 0;
+
+       if (!mptr) return;
+       nllen = strlen(nl);
+       while (ch=*(mptr++), ch > 0) {
+
+               if (ch == '\n') {
+                       client_write(outbuf, len);
+                       len = 0;
+                       client_write(nl, nllen);
+                       column = 0;
                }
-               if (((old == 13) || (old == 10)) && (isspace(real))) {
-                       cprintf("%s", nl);
-                       c = 1;
+               else if (ch == '\r') {
+                       /* Ignore carriage returns.  Newlines are always LF or CRLF but never CR. */
                }
-               if (ch > 32) {
-                       if (((strlen(aaa) + c) > (width - 5)) && (strlen(aaa) > (width - 5))) {
-                               cprintf("%s%s", nl, aaa);
-                               c = strlen(aaa);
-                               aaa[0] = 0;
+               else if (isspace(ch)) {
+                       if (column > 72) {              /* Beyond 72 columns, break on the next space */
+                               client_write(outbuf, len);
+                               len = 0;
+                               client_write(nl, nllen);
+                               column = 0;
                        }
-                       b = strlen(aaa);
-                       aaa[b] = ch;
-                       aaa[b + 1] = 0;
-               }
-               if (ch == 32) {
-                       if ((strlen(aaa) + c) > (width - 5)) {
-                               cprintf("%s", nl);
-                               c = 1;
+                       else {
+                               outbuf[len++] = ch;
+                               ++column;
                        }
-                       cprintf("%s ", aaa);
-                       ++c;
-                       c = c + strlen(aaa);
-                       strcpy(aaa, "");
                }
-               if ((ch == 13) || (ch == 10)) {
-                       cprintf("%s%s", aaa, nl);
-                       c = 1;
-                       strcpy(aaa, "");
+               else {
+                       outbuf[len++] = ch;
+                       ++column;
+                       if (column > 1000) {            /* Beyond 1000 columns, break anywhere */
+                               client_write(outbuf, len);
+                               len = 0;
+                               client_write(nl, nllen);
+                               column = 0;
+                       }
                }
-
-       } while (ch > 0);
-
-       cprintf("%s%s", aaa, nl);
+       }
+       if (len) {
+               client_write(outbuf, len);
+               len = 0;
+               client_write(nl, nllen);
+               column = 0;
+       }
 }
 
 
@@ -902,14 +986,21 @@ void memfmout(
  */
 void list_this_part(char *name, char *filename, char *partnum, char *disp,
                    void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
-                   void *cbuserdata)
+                   char *cbid, void *cbuserdata)
 {
        struct ma_info *ma;
        
        ma = (struct ma_info *)cbuserdata;
        if (ma->is_ma == 0) {
-               cprintf("part=%s|%s|%s|%s|%s|%ld\n",
-                       name, filename, partnum, disp, cbtype, (long)length);
+               cprintf("part=%s|%s|%s|%s|%s|%ld|%s|%s\n",
+                       name, 
+                       filename, 
+                       partnum, 
+                       disp, 
+                       cbtype, 
+                       (long)length, 
+                       cbid, 
+                       cbcharset);
        }
 }
 
@@ -918,7 +1009,7 @@ void list_this_part(char *name, char *filename, char *partnum, char *disp,
  */
 void list_this_pref(char *name, char *filename, char *partnum, char *disp,
                    void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
-                   void *cbuserdata)
+                   char *cbid, void *cbuserdata)
 {
        struct ma_info *ma;
        
@@ -937,7 +1028,7 @@ void list_this_pref(char *name, char *filename, char *partnum, char *disp,
  */
 void list_this_suff(char *name, char *filename, char *partnum, char *disp,
                    void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
-                   void *cbuserdata)
+                   char *cbid, void *cbuserdata)
 {
        struct ma_info *ma;
        
@@ -956,47 +1047,56 @@ void list_this_suff(char *name, char *filename, char *partnum, char *disp,
  */
 void mime_download(char *name, char *filename, char *partnum, char *disp,
                   void *content, char *cbtype, char *cbcharset, size_t length,
-                  char *encoding, void *cbuserdata)
+                  char *encoding, char *cbid, void *cbuserdata)
 {
+       int rv = 0;
 
-       /* Silently go away if there's already a download open... */
+       /* Silently go away if there's already a download open. */
        if (CC->download_fp != NULL)
                return;
 
-       /* ...or if this is not the desired section */
-       if (strcasecmp(CC->download_desired_section, partnum))
-               return;
-
-       CC->download_fp = tmpfile();
-       if (CC->download_fp == NULL)
-               return;
-
-       fwrite(content, length, 1, CC->download_fp);
-       fflush(CC->download_fp);
-       rewind(CC->download_fp);
-
-       OpenCmdResult(filename, cbtype);
+       if (
+               (!IsEmptyStr(partnum) && (!strcasecmp(CC->download_desired_section, partnum)))
+       ||      (!IsEmptyStr(cbid) && (!strcasecmp(CC->download_desired_section, cbid)))
+       ) {
+               CC->download_fp = tmpfile();
+               if (CC->download_fp == NULL)
+                       return;
+       
+               rv = fwrite(content, length, 1, CC->download_fp);
+               fflush(CC->download_fp);
+               rewind(CC->download_fp);
+       
+               OpenCmdResult(filename, cbtype);
+       }
 }
 
 
 
 /*
- * Callback function for mime parser that outputs a section all at once
+ * Callback function for mime parser that outputs a section all at once.
+ * We can specify the desired section by part number *or* content-id.
  */
 void mime_spew_section(char *name, char *filename, char *partnum, char *disp,
                   void *content, char *cbtype, char *cbcharset, size_t length,
-                  char *encoding, void *cbuserdata)
+                  char *encoding, char *cbid, void *cbuserdata)
 {
        int *found_it = (int *)cbuserdata;
 
-       /* ...or if this is not the desired section */
-       if (strcasecmp(CC->download_desired_section, partnum))
-               return;
-
-       *found_it = 1;
-
-       cprintf("%d %d\n", BINARY_FOLLOWS, (int)length);
-       client_write(content, length);
+       if (
+               (!IsEmptyStr(partnum) && (!strcasecmp(CC->download_desired_section, partnum)))
+       ||      (!IsEmptyStr(cbid) && (!strcasecmp(CC->download_desired_section, cbid)))
+       ) {
+               *found_it = 1;
+               cprintf("%d %d|-1|%s|%s|%s\n",
+                       BINARY_FOLLOWS,
+                       (int)length,
+                       filename,
+                       cbtype,
+                       cbcharset
+               );
+               client_write(content, length);
+       }
 }
 
 
@@ -1017,7 +1117,7 @@ struct CtdlMessage *CtdlFetchMessage(long msgnum, int with_body)
        cit_uint8_t ch;
        cit_uint8_t field_header;
 
-       lprintf(CTDL_DEBUG, "CtdlFetchMessage(%ld, %d)\n", msgnum, with_body);
+       CtdlLogPrintf(CTDL_DEBUG, "CtdlFetchMessage(%ld, %d)\n", msgnum, with_body);
 
        dmsgtext = cdb_fetch(CDB_MSGMAIN, &msgnum, sizeof(long));
        if (dmsgtext == NULL) {
@@ -1033,7 +1133,7 @@ struct CtdlMessage *CtdlFetchMessage(long msgnum, int with_body)
         */
        ch = *mptr++;
        if (ch != 255) {
-               lprintf(CTDL_ERR, "Message %ld appears to be corrupted.\n", msgnum);
+               CtdlLogPrintf(CTDL_ERR, "Message %ld appears to be corrupted.\n", msgnum);
                cdb_free(dmsgtext);
                return NULL;
        }
@@ -1096,7 +1196,7 @@ int is_valid_message(struct CtdlMessage *msg) {
        if (msg == NULL)
                return 0;
        if ((msg->cm_magic) != CTDLMESSAGE_MAGIC) {
-               lprintf(CTDL_WARNING, "is_valid_message() -- self-check failed\n");
+               CtdlLogPrintf(CTDL_WARNING, "is_valid_message() -- self-check failed\n");
                return 0;
        }
        return 1;
@@ -1138,12 +1238,12 @@ void CtdlFreeMessage(struct CtdlMessage *msg)
  */
 void fixed_output_pre(char *name, char *filename, char *partnum, char *disp,
                void *content, char *cbtype, char *cbcharset, size_t length, char *encoding,
-               void *cbuserdata)
+               char *cbid, void *cbuserdata)
 {
        struct ma_info *ma;
        
        ma = (struct ma_info *)cbuserdata;
-       lprintf(CTDL_DEBUG, "fixed_output_pre() type=<%s>\n", cbtype);  
+       CtdlLogPrintf(CTDL_DEBUG, "fixed_output_pre() type=<%s>\n", cbtype);    
        if (!strcasecmp(cbtype, "multipart/alternative")) {
                ++ma->is_ma;
                ma->did_print = 0;
@@ -1158,12 +1258,12 @@ void fixed_output_pre(char *name, char *filename, char *partnum, char *disp,
  */
 void fixed_output_post(char *name, char *filename, char *partnum, char *disp,
                void *content, char *cbtype, char *cbcharset, size_t length,
-               char *encoding, void *cbuserdata)
+               char *encoding, char *cbid, void *cbuserdata)
 {
        struct ma_info *ma;
        
        ma = (struct ma_info *)cbuserdata;
-       lprintf(CTDL_DEBUG, "fixed_output_post() type=<%s>\n", cbtype); 
+       CtdlLogPrintf(CTDL_DEBUG, "fixed_output_post() type=<%s>\n", cbtype);   
        if (!strcasecmp(cbtype, "multipart/alternative")) {
                --ma->is_ma;
                ma->did_print = 0;
@@ -1178,7 +1278,7 @@ void fixed_output_post(char *name, char *filename, char *partnum, char *disp,
  */
 void fixed_output(char *name, char *filename, char *partnum, char *disp,
                void *content, char *cbtype, char *cbcharset, size_t length,
-               char *encoding, void *cbuserdata)
+               char *encoding, char *cbid, void *cbuserdata)
 {
        char *ptr;
        char *wptr;
@@ -1187,7 +1287,7 @@ void fixed_output(char *name, char *filename, char *partnum, char *disp,
 
        ma = (struct ma_info *)cbuserdata;
 
-       lprintf(CTDL_DEBUG,
+       CtdlLogPrintf(CTDL_DEBUG,
                "fixed_output() part %s: %s (%s) (%ld bytes)\n",
                partnum, filename, cbtype, (long)length);
 
@@ -1196,7 +1296,7 @@ void fixed_output(char *name, char *filename, char *partnum, char *disp,
         * we've already printed another section, skip this one.
         */     
        if ( (ma->is_ma) && (ma->did_print) ) {
-               lprintf(CTDL_DEBUG, "Skipping part %s (%s)\n", partnum, cbtype);
+               CtdlLogPrintf(CTDL_DEBUG, "Skipping part %s (%s)\n", partnum, cbtype);
                return;
        }
        ma->did_print = 1;
@@ -1251,7 +1351,7 @@ void fixed_output(char *name, char *filename, char *partnum, char *disp,
  */
 void choose_preferred(char *name, char *filename, char *partnum, char *disp,
                void *content, char *cbtype, char *cbcharset, size_t length,
-               char *encoding, void *cbuserdata)
+               char *encoding, char *cbid, void *cbuserdata)
 {
        char buf[1024];
        int i;
@@ -1268,7 +1368,7 @@ void choose_preferred(char *name, char *filename, char *partnum, char *disp,
                extract_token(buf, CC->preferred_formats, i, '|', sizeof buf);
                if ( (!strcasecmp(buf, cbtype)) && (!ma->freeze) ) {
                        if (i < ma->chosen_pref) {
-                               lprintf(CTDL_DEBUG, "Setting chosen part: <%s>\n", partnum);
+                               CtdlLogPrintf(CTDL_DEBUG, "Setting chosen part: <%s>\n", partnum);
                                safestrncpy(ma->chosen_part, partnum, sizeof ma->chosen_part);
                                ma->chosen_pref = i;
                        }
@@ -1281,7 +1381,7 @@ void choose_preferred(char *name, char *filename, char *partnum, char *disp,
  */
 void output_preferred(char *name, char *filename, char *partnum, char *disp,
                void *content, char *cbtype, char *cbcharset, size_t length,
-               char *encoding, void *cbuserdata)
+               char *encoding, char *cbid, void *cbuserdata)
 {
        int i;
        char buf[128];
@@ -1329,7 +1429,7 @@ void output_preferred(char *name, char *filename, char *partnum, char *disp,
        /* No translations required or possible: output as text/plain */
        cprintf("Content-type: text/plain\n\n");
        fixed_output(name, filename, partnum, disp, content, cbtype, cbcharset,
-                       length, encoding, cbuserdata);
+                       length, encoding, cbid, cbuserdata);
 }
 
 
@@ -1345,7 +1445,7 @@ struct encapmsg {
  */
 void extract_encapsulated_message(char *name, char *filename, char *partnum, char *disp,
                   void *content, char *cbtype, char *cbcharset, size_t length,
-                  char *encoding, void *cbuserdata)
+                  char *encoding, char *cbid, void *cbuserdata)
 {
        struct encapmsg *encap;
 
@@ -1364,40 +1464,58 @@ void extract_encapsulated_message(char *name, char *filename, char *partnum, cha
 
 
 
+
+
+int CtdlDoIHavePermissionToReadMessagesInThisRoom(void) {
+       if ((!(CC->logged_in)) && (!(CC->internal_pgm))) {
+               return(om_not_logged_in);
+       }
+       return(om_ok);
+}
+
+
 /*
  * Get a message off disk.  (returns om_* values found in msgbase.h)
  * 
  */
 int CtdlOutputMsg(long msg_num,                /* message number (local) to fetch */
-               int mode,               /* how would you like that message? */
-               int headers_only,       /* eschew the message body? */
-               int do_proto,           /* do Citadel protocol responses? */
-               int crlf,               /* Use CRLF newlines instead of LF? */
-               char *section           /* NULL or a message/rfc822 section */
+                 int mode,             /* how would you like that message? */
+                 int headers_only,     /* eschew the message body? */
+                 int do_proto,         /* do Citadel protocol responses? */
+                 int crlf,             /* Use CRLF newlines instead of LF? */
+                 char *section,        /* NULL or a message/rfc822 section */
+                 int flags             /* various flags; see msgbase.h */
 ) {
        struct CtdlMessage *TheMessage = NULL;
        int retcode = om_no_such_msg;
        struct encapmsg encap;
+       int r;
 
-       lprintf(CTDL_DEBUG, "CtdlOutputMsg() msgnum=%ld, mode=%d, section=%s\n", 
+       CtdlLogPrintf(CTDL_DEBUG, "CtdlOutputMsg(msgnum=%ld, mode=%d, section=%s)\n", 
                msg_num, mode,
                (section ? section : "<>")
        );
 
-       if ((!(CC->logged_in)) && (!(CC->internal_pgm))) {
-               if (do_proto) cprintf("%d Not logged in.\n",
-                       ERROR + NOT_LOGGED_IN);
-               return(om_not_logged_in);
+       r = CtdlDoIHavePermissionToReadMessagesInThisRoom();
+       if (r != om_ok) {
+               if (do_proto) {
+                       if (r == om_not_logged_in) {
+                               cprintf("%d Not logged in.\n", ERROR + NOT_LOGGED_IN);
+                       }
+                       else {
+                               cprintf("%d An unknown error has occurred.\n", ERROR);
+                       }
+               }
+               return(r);
        }
 
        /* FIXME: check message id against msglist for this room */
 
        /*
-        * Fetch the message from disk.  If we're in any sort of headers
-        * only mode, request that we don't even bother loading the body
-        * into memory.
+        * Fetch the message from disk.  If we're in HEADERS_FAST mode,
+        * request that we don't even bother loading the body into memory.
         */
-       if ( (headers_only == HEADERS_FAST) || (headers_only == HEADERS_ONLY) ) {
+       if (headers_only == HEADERS_FAST) {
                TheMessage = CtdlFetchMessage(msg_num, 0);
        }
        else {
@@ -1445,13 +1563,154 @@ int CtdlOutputMsg(long msg_num,                /* message number (local) to fetch */
        }
 
        /* Ok, output the message now */
-       retcode = CtdlOutputPreLoadedMsg(TheMessage, mode, headers_only, do_proto, crlf);
+       retcode = CtdlOutputPreLoadedMsg(TheMessage, mode, headers_only, do_proto, crlf, flags);
        CtdlFreeMessage(TheMessage);
 
        return(retcode);
 }
 
 
+char *qp_encode_email_addrs(char *source)
+{
+       char user[256], node[256], name[256];
+       const char headerStr[] = "=?UTF-8?Q?";
+       char *Encoded;
+       char *EncodedName;
+       char *nPtr;
+       int need_to_encode = 0;
+       long SourceLen;
+       long EncodedMaxLen;
+       long nColons = 0;
+       long *AddrPtr;
+       long *AddrUtf8;
+       long nAddrPtrMax = 50;
+       long nmax;
+       int InQuotes = 0;
+       int i, n;
+
+       if (source == NULL) return source;
+       if (IsEmptyStr(source)) return source;
+
+       AddrPtr = malloc (sizeof (long) * nAddrPtrMax);
+       AddrUtf8 = malloc (sizeof (long) * nAddrPtrMax);
+       memset(AddrUtf8, 0, sizeof (long) * nAddrPtrMax);
+       *AddrPtr = 0;
+       i = 0;
+       while (!IsEmptyStr (&source[i])) {
+               if (nColons >= nAddrPtrMax){
+                       long *ptr;
+
+                       ptr = (long *) malloc(sizeof (long) * nAddrPtrMax * 2);
+                       memcpy (ptr, AddrPtr, sizeof (long) * nAddrPtrMax);
+                       free (AddrPtr), AddrPtr = ptr;
+
+                       ptr = (long *) malloc(sizeof (long) * nAddrPtrMax * 2);
+                       memset(&ptr[nAddrPtrMax], 0, 
+                              sizeof (long) * nAddrPtrMax);
+
+                       memcpy (ptr, AddrUtf8, sizeof (long) * nAddrPtrMax);
+                       free (AddrUtf8), AddrUtf8 = ptr;
+                       nAddrPtrMax *= 2;                               
+               }
+               if (((unsigned char) source[i] < 32) || 
+                   ((unsigned char) source[i] > 126)) {
+                       need_to_encode = 1;
+                       AddrUtf8[nColons] = 1;
+               }
+               if (source[i] == '"')
+                       InQuotes = !InQuotes;
+               if (!InQuotes && source[i] == ',') {
+                       AddrPtr[nColons] = i;
+                       nColons++;
+               }
+               i++;
+       }
+       if (need_to_encode == 0) {
+               free(AddrPtr);
+               free(AddrUtf8);
+               return source;
+       }
+
+       SourceLen = i;
+       EncodedMaxLen = nColons * (sizeof(headerStr) + 3) + SourceLen * 3;
+       Encoded = (char*) malloc (EncodedMaxLen);
+
+       for (i = 0; i < nColons; i++)
+               source[AddrPtr[i]++] = '\0';
+
+       nPtr = Encoded;
+       *nPtr = '\0';
+       for (i = 0; i < nColons && nPtr != NULL; i++) {
+               nmax = EncodedMaxLen - (nPtr - Encoded);
+               if (AddrUtf8[i]) {
+                       process_rfc822_addr(&source[AddrPtr[i]], 
+                                           user,
+                                           node,
+                                           name);
+                       /* TODO: libIDN here ! */
+                       if (IsEmptyStr(name)) {
+                               n = snprintf(nPtr, nmax, 
+                                            (i==0)?"%s@%s" : ",%s@%s",
+                                            user, node);
+                       }
+                       else {
+                               EncodedName = rfc2047encode(name, strlen(name));                        
+                               n = snprintf(nPtr, nmax, 
+                                            (i==0)?"%s <%s@%s>" : ",%s <%s@%s>",
+                                            EncodedName, user, node);
+                               free(EncodedName);
+                       }
+               }
+               else { 
+                       n = snprintf(nPtr, nmax, 
+                                    (i==0)?"%s" : ",%s",
+                                    &source[AddrPtr[i]]);
+               }
+               if (n > 0 )
+                       nPtr += n;
+               else { 
+                       char *ptr, *nnPtr;
+                       ptr = (char*) malloc(EncodedMaxLen * 2);
+                       memcpy(ptr, Encoded, EncodedMaxLen);
+                       nnPtr = ptr + (nPtr - Encoded), nPtr = nnPtr;
+                       free(Encoded), Encoded = ptr;
+                       EncodedMaxLen *= 2;
+                       i--; /* do it once more with properly lengthened buffer */
+               }
+       }
+       for (i = 0; i < nColons; i++)
+               source[--AddrPtr[i]] = ',';
+       free(AddrUtf8);
+       free(AddrPtr);
+       return Encoded;
+}
+
+
+/* If the last item in a list of recipients was truncated to a partial address,
+ * remove it completely in order to avoid choking libSieve
+ */
+void sanitize_truncated_recipient(char *str)
+{
+       if (!str) return;
+       if (num_tokens(str, ',') < 2) return;
+
+       int len = strlen(str);
+       if (len < 900) return;
+       if (len > 998) str[998] = 0;
+
+       char *cptr = strrchr(str, ',');
+       if (!cptr) return;
+
+       char *lptr = strchr(cptr, '<');
+       char *rptr = strchr(cptr, '>');
+
+       if ( (lptr) && (rptr) && (rptr > lptr) ) return;
+
+       *cptr = 0;
+}
+
+
+
 /*
  * Get a message off disk.  (returns om_* values found in msgbase.h)
  */
@@ -1460,15 +1719,16 @@ int CtdlOutputPreLoadedMsg(
                int mode,               /* how would you like that message? */
                int headers_only,       /* eschew the message body? */
                int do_proto,           /* do Citadel protocol responses? */
-               int crlf                /* Use CRLF newlines instead of LF? */
+               int crlf,               /* Use CRLF newlines instead of LF? */
+               int flags               /* should the bessage be exported clean? */
 ) {
-       int i, k;
+       int i, j, k;
        char buf[SIZ];
-       cit_uint8_t ch;
+       cit_uint8_t ch, prev_ch;
        char allkeys[30];
        char display_name[256];
-       char *mptr;
-       char *nl;       /* newline string */
+       char *mptr, *mpptr;
+       const char *nl; /* newline string */
        int suppress_f = 0;
        int subject_found = 0;
        struct ma_info ma;
@@ -1481,11 +1741,10 @@ int CtdlOutputPreLoadedMsg(
        char luser[100];
        char fuser[100];
        char snode[100];
-       char lnode[100];
        char mid[100];
        char datestamp[100];
 
-       lprintf(CTDL_DEBUG, "CtdlOutputPreLoadedMsg(TheMessage=%s, %d, %d, %d, %d\n",
+       CtdlLogPrintf(CTDL_DEBUG, "CtdlOutputPreLoadedMsg(TheMessage=%s, %d, %d, %d, %d\n",
                ((TheMessage == NULL) ? "NULL" : "not null"),
                mode, headers_only, do_proto, crlf);
 
@@ -1493,11 +1752,19 @@ int CtdlOutputPreLoadedMsg(
        nl = (crlf ? "\r\n" : "\n");
 
        if (!is_valid_message(TheMessage)) {
-               lprintf(CTDL_ERR,
+               CtdlLogPrintf(CTDL_ERR,
                        "ERROR: invalid preloaded message for output\n");
+               cit_backtrace ();
                return(om_no_such_msg);
        }
 
+       /* Suppress envelope recipients if required to avoid disclosing BCC addresses.
+        * Pad it with spaces in order to avoid changing the RFC822 length of the message.
+        */
+       if ( (flags & SUPPRESS_ENV_TO) && (TheMessage->cm_fields['V'] != NULL) ) {
+               memset(TheMessage->cm_fields['V'], ' ', strlen(TheMessage->cm_fields['V']));
+       }
+               
        /* Are we downloading a MIME component? */
        if (mode == MT_DOWNLOAD) {
                if (TheMessage->cm_format_type != FMT_RFC822) {
@@ -1564,7 +1831,7 @@ int CtdlOutputPreLoadedMsg(
 
        /* nhdr=yes means that we're only displaying headers, no body */
        if ( (TheMessage->cm_anon_type == MES_ANONONLY)
-          && (mode == MT_CITADEL)
+          && ((mode == MT_CITADEL) || (mode == MT_MIME))
           && (do_proto)
           ) {
                cprintf("nhdr=yes\n");
@@ -1605,7 +1872,7 @@ int CtdlOutputPreLoadedMsg(
                      if (haschar(TheMessage->cm_fields['N'], '.') == 0) {
                        suppress_f = 1;
                }
-               
+
                /* Now spew the header fields in the order we like them. */
                safestrncpy(allkeys, FORDER, sizeof allkeys);
                for (i=0; i<strlen(allkeys); ++i) {
@@ -1613,6 +1880,9 @@ int CtdlOutputPreLoadedMsg(
                        if (k != 'M') {
                                if ( (TheMessage->cm_fields[k] != NULL)
                                   && (msgkeys[k] != NULL) ) {
+                                       if ((k == 'V') || (k == 'R') || (k == 'Y')) {
+                                               sanitize_truncated_recipient(TheMessage->cm_fields[k]);
+                                       }
                                        if (k == 'A') {
                                                if (do_proto) cprintf("%s=%s\n",
                                                        msgkeys[k],
@@ -1640,23 +1910,31 @@ int CtdlOutputPreLoadedMsg(
        strcpy(luser, "");
        strcpy(fuser, "");
        strcpy(snode, NODENAME);
-       strcpy(lnode, HUMANNODE);
        if (mode == MT_RFC822) {
                for (i = 0; i < 256; ++i) {
                        if (TheMessage->cm_fields[i]) {
-                               mptr = TheMessage->cm_fields[i];
-
+                               mptr = mpptr = TheMessage->cm_fields[i];
+                               
                                if (i == 'A') {
                                        safestrncpy(luser, mptr, sizeof luser);
                                        safestrncpy(suser, mptr, sizeof suser);
                                }
                                else if (i == 'Y') {
+                                       if ((flags & QP_EADDR) != 0) {
+                                               mptr = qp_encode_email_addrs(mptr);
+                                       }
+                                       sanitize_truncated_recipient(mptr);
                                        cprintf("CC: %s%s", mptr, nl);
                                }
                                else if (i == 'P') {
                                        cprintf("Return-Path: %s%s", mptr, nl);
                                }
+                               else if (i == 'L') {
+                                       cprintf("List-ID: %s%s", mptr, nl);
+                               }
                                else if (i == 'V') {
+                                       if ((flags & QP_EADDR) != 0) 
+                                               mptr = qp_encode_email_addrs(mptr);
                                        cprintf("Envelope-To: %s%s", mptr, nl);
                                }
                                else if (i == 'U') {
@@ -1665,8 +1943,6 @@ int CtdlOutputPreLoadedMsg(
                                }
                                else if (i == 'I')
                                        safestrncpy(mid, mptr, sizeof mid);
-                               else if (i == 'H')
-                                       safestrncpy(lnode, mptr, sizeof lnode);
                                else if (i == 'F')
                                        safestrncpy(fuser, mptr, sizeof fuser);
                                /* else if (i == 'O')
@@ -1678,11 +1954,18 @@ int CtdlOutputPreLoadedMsg(
                                {
                                        if (haschar(mptr, '@') == 0)
                                        {
-                                               cprintf("To: %s@%s%s", mptr, config.c_fqdn, nl);
+                                               sanitize_truncated_recipient(mptr);
+                                               cprintf("To: %s@%s", mptr, config.c_fqdn);
+                                               cprintf("%s", nl);
                                        }
                                        else
                                        {
-                                               cprintf("To: %s%s", mptr, nl);
+                                               if ((flags & QP_EADDR) != 0) {
+                                                       mptr = qp_encode_email_addrs(mptr);
+                                               }
+                                               sanitize_truncated_recipient(mptr);
+                                               cprintf("To: %s", mptr);
+                                               cprintf("%s", nl);
                                        }
                                }
                                else if (i == 'T') {
@@ -1690,6 +1973,22 @@ int CtdlOutputPreLoadedMsg(
                                                atol(mptr), DATESTRING_RFC822);
                                        cprintf("Date: %s%s", datestamp, nl);
                                }
+                               else if (i == 'W') {
+                                       cprintf("References: ");
+                                       k = num_tokens(mptr, '|');
+                                       for (j=0; j<k; ++j) {
+                                               extract_token(buf, mptr, j, '|', sizeof buf);
+                                               cprintf("<%s>", buf);
+                                               if (j == (k-1)) {
+                                                       cprintf("%s", nl);
+                                               }
+                                               else {
+                                                       cprintf(" ");
+                                               }
+                                       }
+                               }
+                               if (mptr != mpptr)
+                                       free (mptr);
                        }
                }
                if (subject_found == 0) {
@@ -1727,8 +2026,6 @@ int CtdlOutputPreLoadedMsg(
                        cprintf("From: \"%s\" <%s@%s>%s", luser, suser, snode, nl);
                }
 
-               cprintf("Organization: %s%s", lnode, nl);
-
                /* Blank line signifying RFC822 end-of-headers */
                if (TheMessage->cm_format_type != FMT_RFC822) {
                        cprintf("%s", nl);
@@ -1751,36 +2048,48 @@ START_TEXT:
                                (void *)&ma, 0);
                }
                else if (mode == MT_RFC822) {   /* unparsed RFC822 dump */
-                       char *start_of_text = NULL;
-                       start_of_text = strstr(mptr, "\n\r\n");
-                       if (start_of_text == NULL) start_of_text = strstr(mptr, "\n\n");
-                       if (start_of_text == NULL) start_of_text = mptr;
-                       ++start_of_text;
-                       start_of_text = strstr(start_of_text, "\n");
-                       ++start_of_text;
+                       int eoh = 0;
 
                        char outbuf[1024];
                        int outlen = 0;
                        int nllen = strlen(nl);
-                       while (ch=*mptr, ch!=0) {
-                               if (ch==13) {
+                       prev_ch = 0;
+                       while (*mptr != '\0') {
+                               if (*mptr == '\r') {
                                        /* do nothing */
                                }
                                else {
+                                       if ((!eoh) &&
+                                           (*mptr == '\n'))
+                                       {
+                                               eoh = (*(mptr+1) == '\r') && (*(mptr+2) == '\n');
+                                               if (!eoh)
+                                                       eoh = *(mptr+1) == '\n';
+                                       }
+
                                        if (
-                                               ((headers_only == HEADERS_NONE) && (mptr >= start_of_text))
-                                          ||   ((headers_only == HEADERS_ONLY) && (mptr < start_of_text))
+                                               ((headers_only == HEADERS_NONE) && (eoh))
+                                          ||   ((headers_only == HEADERS_ONLY) && (!eoh))
                                           ||   ((headers_only != HEADERS_NONE) && (headers_only != HEADERS_ONLY))
                                        ) {
-                                               if (ch == 10) {
-                                                       sprintf(&outbuf[outlen], "%s", nl);
+                                               if (*mptr == '\n') {
+                                                       memcpy(&outbuf[outlen], nl, nllen);
                                                        outlen += nllen;
+                                                       outbuf[outlen] = '\0';
                                                }
                                                else {
-                                                       outbuf[outlen++] = ch;
+                                                       outbuf[outlen++] = *mptr;
                                                }
                                        }
                                }
+                               if (flags & ESC_DOT)
+                               {
+                                       if ((prev_ch == '\n') && (*mptr == '.') && ((*(mptr+1) == '\r') || (*(mptr+1) == '\n')))
+                                       {
+                                               outbuf[outlen++] = '.';
+                                       }
+                               }
+                               prev_ch = *mptr;
                                ++mptr;
                                if (outlen > 1000) {
                                        client_write(outbuf, outlen);
@@ -1811,19 +2120,49 @@ START_TEXT:
         */
        if (TheMessage->cm_format_type == FMT_FIXED) {
                int buflen;
+               int xlline = 0;
+               int nllen = strlen (nl);
                if (mode == MT_MIME) {
                        cprintf("Content-type: text/plain\n\n");
                }
                *buf = '\0';
                buflen = 0;
                while (ch = *mptr++, ch > 0) {
-                       if (ch == 13)
-                               ch = 10;
-                       if ((ch == 10) || (buflen > 250)) {
+                       if (ch == '\n')
+                               ch = '\r';
+
+                       if ((buflen > 250) && (!xlline)){
+                               int tbuflen;
+                               tbuflen = buflen;
+
+                               while ((buflen > 0) && 
+                                      (!isspace(buf[buflen])))
+                                       buflen --;
+                               if (buflen == 0) {
+                                       xlline = 1;
+                               }
+                               else {
+                                       mptr -= tbuflen - buflen;
+                                       buf[buflen] = '\0';
+                                       ch = '\r';
+                               }
+                       }
+                       /* if we reach the outer bounds of our buffer, 
+                          abort without respect what whe purge. */
+                       if (xlline && 
+                           ((isspace(ch)) || 
+                            (buflen > SIZ - nllen - 2)))
+                               ch = '\r';
+
+                       if (ch == '\r') {
+                               memcpy (&buf[buflen], nl, nllen);
+                               buflen += nllen;
                                buf[buflen] = '\0';
-                               cprintf("%s%s", buf, nl);
+
+                               client_write(buf, buflen);
                                *buf = '\0';
                                buflen = 0;
+                               xlline = 0;
                        } else {
                                buf[buflen] = ch;
                                buflen++;
@@ -1845,7 +2184,7 @@ START_TEXT:
                if (mode == MT_MIME) {
                        cprintf("Content-type: text/x-citadel-variformat\n\n");
                }
-               memfmout(mptr, 0, nl);
+               memfmout(mptr, nl);
        }
 
        /* If the message on disk is format 4 (MIME), we've gotta hand it
@@ -1893,7 +2232,7 @@ void cmd_msg0(char *cmdbuf)
        msgid = extract_long(cmdbuf, 0);
        headers_only = extract_int(cmdbuf, 1);
 
-       CtdlOutputMsg(msgid, MT_CITADEL, headers_only, 1, 0, NULL);
+       CtdlOutputMsg(msgid, MT_CITADEL, headers_only, 1, 0, NULL, 0);
        return;
 }
 
@@ -1909,7 +2248,7 @@ void cmd_msg2(char *cmdbuf)
        msgid = extract_long(cmdbuf, 0);
        headers_only = extract_int(cmdbuf, 1);
 
-       CtdlOutputMsg(msgid, MT_RFC822, headers_only, 1, 1, NULL);
+       CtdlOutputMsg(msgid, MT_RFC822, headers_only, 1, 1, NULL, 0);
 }
 
 
@@ -1963,7 +2302,7 @@ void cmd_msg4(char *cmdbuf)
 
        msgid = extract_long(cmdbuf, 0);
        extract_token(section, cmdbuf, 1, '|', sizeof section);
-       CtdlOutputMsg(msgid, MT_MIME, 0, 1, 0, (section[0] ? section : NULL) );
+       CtdlOutputMsg(msgid, MT_MIME, 0, 1, 0, (section[0] ? section : NULL) , 0);
 }
 
 
@@ -1996,7 +2335,7 @@ void cmd_opna(char *cmdbuf)
        extract_token(desired_section, cmdbuf, 1, '|', sizeof desired_section);
        safestrncpy(CC->download_desired_section, desired_section,
                sizeof CC->download_desired_section);
-       CtdlOutputMsg(msgid, MT_DOWNLOAD, 0, 1, 1, NULL);
+       CtdlOutputMsg(msgid, MT_DOWNLOAD, 0, 1, 1, NULL, 0);
 }                      
 
 
@@ -2012,7 +2351,7 @@ void cmd_dlat(char *cmdbuf)
        extract_token(desired_section, cmdbuf, 1, '|', sizeof desired_section);
        safestrncpy(CC->download_desired_section, desired_section,
                sizeof CC->download_desired_section);
-       CtdlOutputMsg(msgid, MT_SPEW_SECTION, 0, 1, 1, NULL);
+       CtdlOutputMsg(msgid, MT_SPEW_SECTION, 0, 1, 1, NULL, 0);
 }                      
 
 
@@ -2028,8 +2367,8 @@ void cmd_dlat(char *cmdbuf)
  * this mode of operation only works if we're saving a single message.)
  */
 int CtdlSaveMsgPointersInRoom(char *roomname, long newmsgidlist[], int num_newmsgs,
-                               int do_repl_check, struct CtdlMessage *supplied_msg)
-{
+                       int do_repl_check, struct CtdlMessage *supplied_msg, int suppress_refcount_adj
+{
        int i, j, unique;
        char hold_rm[ROOMNAMELEN];
        struct cdbdata *cdbfr;
@@ -2043,9 +2382,10 @@ int CtdlSaveMsgPointersInRoom(char *roomname, long newmsgidlist[], int num_newms
        long *msgs_to_be_merged = NULL;
        int num_msgs_to_be_merged = 0;
 
-       lprintf(CTDL_DEBUG,
-               "CtdlSaveMsgPointersInRoom(room=%s, num_msgs=%d, repl=%d)\n",
-               roomname, num_newmsgs, do_repl_check);
+       CtdlLogPrintf(CTDL_DEBUG,
+               "CtdlSaveMsgPointersInRoom(room=%s, num_msgs=%d, repl=%d, suppress_rca=%d)\n",
+               roomname, num_newmsgs, do_repl_check, suppress_refcount_adj
+       );
 
        strcpy(hold_rm, CC->room.QRname);
 
@@ -2055,10 +2395,10 @@ int CtdlSaveMsgPointersInRoom(char *roomname, long newmsgidlist[], int num_newms
        if (num_newmsgs > 1) supplied_msg = NULL;
 
        /* Now the regular stuff */
-       if (lgetroom(&CC->room,
+       if (CtdlGetRoomLock(&CC->room,
           ((roomname != NULL) ? roomname : CC->room.QRname) )
           != 0) {
-               lprintf(CTDL_ERR, "No such room <%s>\n", roomname);
+               CtdlLogPrintf(CTDL_ERR, "No such room <%s>\n", roomname);
                return(ERROR + ROOM_NOT_FOUND);
        }
 
@@ -2095,14 +2435,14 @@ int CtdlSaveMsgPointersInRoom(char *roomname, long newmsgidlist[], int num_newms
                }
        }
 
-       lprintf(9, "%d unique messages to be merged\n", num_msgs_to_be_merged);
+       CtdlLogPrintf(9, "%d unique messages to be merged\n", num_msgs_to_be_merged);
 
        /*
         * Now merge the new messages
         */
        msglist = realloc(msglist, (sizeof(long) * (num_msgs + num_msgs_to_be_merged)) );
        if (msglist == NULL) {
-               lprintf(CTDL_ALERT, "ERROR: can't realloc message list!\n");
+               CtdlLogPrintf(CTDL_ALERT, "ERROR: can't realloc message list!\n");
        }
        memcpy(&msglist[num_msgs], msgs_to_be_merged, (sizeof(long) * num_msgs_to_be_merged) );
        num_msgs += num_msgs_to_be_merged;
@@ -2122,11 +2462,11 @@ int CtdlSaveMsgPointersInRoom(char *roomname, long newmsgidlist[], int num_newms
 
        /* Update the highest-message pointer and unlock the room. */
        CC->room.QRhighest = highest_msg;
-       lputroom(&CC->room);
+       CtdlPutRoomLock(&CC->room);
 
        /* Perform replication checks if necessary */
        if ( (DoesThisRoomNeedEuidIndexing(&CC->room)) && (do_repl_check) ) {
-               lprintf(CTDL_DEBUG, "CtdlSaveMsgPointerInRoom() doing repl checks\n");
+               CtdlLogPrintf(CTDL_DEBUG, "CtdlSaveMsgPointerInRoom() doing repl checks\n");
 
                for (i=0; i<num_msgs_to_be_merged; ++i) {
                        msgid = msgs_to_be_merged[i];
@@ -2156,18 +2496,20 @@ int CtdlSaveMsgPointersInRoom(char *roomname, long newmsgidlist[], int num_newms
        }
 
        else {
-               lprintf(CTDL_DEBUG, "CtdlSaveMsgPointerInRoom() skips repl checks\n");
+               CtdlLogPrintf(CTDL_DEBUG, "CtdlSaveMsgPointerInRoom() skips repl checks\n");
        }
 
        /* Submit this room for processing by hooks */
        PerformRoomHooks(&CC->room);
 
        /* Go back to the room we were in before we wandered here... */
-       getroom(&CC->room, hold_rm);
+       CtdlGetRoom(&CC->room, hold_rm);
 
        /* Bump the reference count for all messages which were merged */
-       for (i=0; i<num_msgs_to_be_merged; ++i) {
-               AdjRefCount(msgs_to_be_merged[i], +1);
+       if (!suppress_refcount_adj) {
+               for (i=0; i<num_msgs_to_be_merged; ++i) {
+                       AdjRefCount(msgs_to_be_merged[i], +1);
+               }
        }
 
        /* Free up memory... */
@@ -2187,7 +2529,7 @@ int CtdlSaveMsgPointersInRoom(char *roomname, long newmsgidlist[], int num_newms
 int CtdlSaveMsgPointerInRoom(char *roomname, long msgid,
                        int do_repl_check, struct CtdlMessage *supplied_msg)
 {
-       return CtdlSaveMsgPointersInRoom(roomname, &msgid, 1, do_repl_check, supplied_msg);
+       return CtdlSaveMsgPointersInRoom(roomname, &msgid, 1, do_repl_check, supplied_msg, 0);
 }
 
 
@@ -2243,7 +2585,7 @@ long send_message(struct CtdlMessage *msg) {
        /* Write our little bundle of joy into the message base */
        if (cdb_store(CDB_MSGMAIN, &newmsgid, (int)sizeof(long),
                      smr.ser, smr.len) < 0) {
-               lprintf(CTDL_ERR, "Can't store message\n");
+               CtdlLogPrintf(CTDL_ERR, "Can't store message\n");
                retval = 0L;
        } else {
                if (is_bigmsg) {
@@ -2286,7 +2628,7 @@ void serialize_message(struct ser_ret *ret,               /* return values */
         * Check for valid message format
         */
        if (is_valid_message(msg) == 0) {
-               lprintf(CTDL_ERR, "serialize_message() aborting due to invalid message\n");
+               CtdlLogPrintf(CTDL_ERR, "serialize_message() aborting due to invalid message\n");
                ret->len = 0;
                ret->ser = NULL;
                return;
@@ -2299,7 +2641,7 @@ void serialize_message(struct ser_ret *ret,               /* return values */
 
        ret->ser = malloc(ret->len);
        if (ret->ser == NULL) {
-               lprintf(CTDL_ERR, "serialize_message() malloc(%ld) failed: %s\n",
+               CtdlLogPrintf(CTDL_ERR, "serialize_message() malloc(%ld) failed: %s\n",
                        (long)ret->len, strerror(errno));
                ret->len = 0;
                ret->ser = NULL;
@@ -2317,7 +2659,7 @@ void serialize_message(struct ser_ret *ret,               /* return values */
                safestrncpy((char *)&ret->ser[wlen], msg->cm_fields[(int)forder[i]], fieldlen+1);
                wlen = wlen + fieldlen + 1;
        }
-       if (ret->len != wlen) lprintf(CTDL_ERR, "ERROR: len=%ld wlen=%ld\n",
+       if (ret->len != wlen) CtdlLogPrintf(CTDL_ERR, "ERROR: len=%ld wlen=%ld\n",
                (long)ret->len, (long)wlen);
 
        return;
@@ -2343,7 +2685,7 @@ void dump_message(struct CtdlMessage *msg,        /* unserialized msg */
         * Check for valid message format
         */
        if (is_valid_message(msg) == 0) {
-               lprintf(CTDL_ERR, "dump_message() aborting due to invalid message\n");
+               CtdlLogPrintf(CTDL_ERR, "dump_message() aborting due to invalid message\n");
                return;
        }
 
@@ -2371,19 +2713,19 @@ void ReplicationChecks(struct CtdlMessage *msg) {
 
        if (DoesThisRoomNeedEuidIndexing(&CC->room) == 0) return;
 
-       lprintf(CTDL_DEBUG, "Performing replication checks in <%s>\n",
+       CtdlLogPrintf(CTDL_DEBUG, "Performing replication checks in <%s>\n",
                CC->room.QRname);
 
        /* No exclusive id?  Don't do anything. */
        if (msg == NULL) return;
        if (msg->cm_fields['E'] == NULL) return;
        if (IsEmptyStr(msg->cm_fields['E'])) return;
-       /*lprintf(CTDL_DEBUG, "Exclusive ID: <%s> for room <%s>\n",
+       /*CtdlLogPrintf(CTDL_DEBUG, "Exclusive ID: <%s> for room <%s>\n",
                msg->cm_fields['E'], CC->room.QRname);*/
 
-       old_msgnum = locate_message_by_euid(msg->cm_fields['E'], &CC->room);
+       old_msgnum = CtdlLocateMessageByEuid(msg->cm_fields['E'], &CC->room);
        if (old_msgnum > 0L) {
-               lprintf(CTDL_DEBUG, "ReplicationChecks() replacing message %ld\n", old_msgnum);
+               CtdlLogPrintf(CTDL_DEBUG, "ReplicationChecks() replacing message %ld\n", old_msgnum);
                CtdlDeleteMessages(CC->room.QRname, &old_msgnum, 1, "");
        }
 }
@@ -2394,8 +2736,9 @@ void ReplicationChecks(struct CtdlMessage *msg) {
  * Save a message to disk and submit it into the delivery system.
  */
 long CtdlSubmitMsg(struct CtdlMessage *msg,    /* message to save */
-               struct recptypes *recps,        /* recipients (if mail) */
-               char *force                     /* force a particular room? */
+                  struct recptypes *recps,     /* recipients (if mail) */
+                  char *force,                 /* force a particular room? */
+                  int flags                    /* should the message be exported clean? */
 ) {
        char submit_filename[128];
        char generated_timestamp[32];
@@ -2405,7 +2748,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
        char content_type[SIZ];                 /* We have to learn this */
        char recipient[SIZ];
        long newmsgid;
-       char *mptr = NULL;
+       const char *mptr = NULL;
        struct ctdluser userbuf;
        int a, i;
        struct MetaData smi;
@@ -2420,9 +2763,12 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,      /* message to save */
        struct addresses_to_be_filed *aptr = NULL;
        char *saved_rfc822_version = NULL;
        int qualified_for_journaling = 0;
-       struct CitContext *CCC = CC;            /* CachedCitContext - performance boost */
+       CitContext *CCC = CC;           /* CachedCitContext - performance boost */
+       char bounce_to[1024] = "";
+       size_t tmp = 0;
+       int rv = 0;
 
-       lprintf(CTDL_DEBUG, "CtdlSubmitMsg() called\n");
+       CtdlLogPrintf(CTDL_DEBUG, "CtdlSubmitMsg() called\n");
        if (is_valid_message(msg) == 0) return(-1);     /* self check */
 
        /* If this message has no timestamp, we take the liberty of
@@ -2458,7 +2804,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
 
        /* Learn about what's inside, because it's what's inside that counts */
        if (msg->cm_fields['M'] == NULL) {
-               lprintf(CTDL_ERR, "ERROR: attempt to save message with NULL body\n");
+               CtdlLogPrintf(CTDL_ERR, "ERROR: attempt to save message with NULL body\n");
                return(-2);
        }
 
@@ -2490,7 +2836,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
        }
 
        /* Goto the correct room */
-       lprintf(CTDL_DEBUG, "Selected room %s\n", (recps) ? CCC->room.QRname : SENTITEMS);
+       CtdlLogPrintf(CTDL_DEBUG, "Selected room %s\n", (recps) ? CCC->room.QRname : SENTITEMS);
        strcpy(hold_rm, CCC->room.QRname);
        strcpy(actual_rm, CCC->room.QRname);
        if (recps != NULL) {
@@ -2499,10 +2845,10 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,     /* message to save */
 
        /* If the user is a twit, move to the twit room for posting */
        if (TWITDETECT) {
-               if (CCC->user.axlevel == 2) {
+               if (CCC->user.axlevel == AxProbU) {
                        strcpy(hold_rm, actual_rm);
                        strcpy(actual_rm, config.c_twitroom);
-                       lprintf(CTDL_DEBUG, "Diverting to twit room\n");
+                       CtdlLogPrintf(CTDL_DEBUG, "Diverting to twit room\n");
                }
        }
 
@@ -2511,10 +2857,10 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,     /* message to save */
                strcpy(actual_rm, force_room);
        }
 
-       lprintf(CTDL_DEBUG, "Final selection: %s\n", actual_rm);
+       CtdlLogPrintf(CTDL_DEBUG, "Final selection: %s\n", actual_rm);
        if (strcasecmp(actual_rm, CCC->room.QRname)) {
-               /* getroom(&CCC->room, actual_rm); */
-               usergoto(actual_rm, 0, 1, NULL, NULL);
+               /* CtdlGetRoom(&CCC->room, actual_rm); */
+               CtdlUserGoto(actual_rm, 0, 1, NULL, NULL);
        }
 
        /*
@@ -2525,7 +2871,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
        }
 
        /* Perform "before save" hooks (aborting if any return nonzero) */
-       lprintf(CTDL_DEBUG, "Performing before-save hooks\n");
+       CtdlLogPrintf(CTDL_DEBUG, "Performing before-save hooks\n");
        if (PerformMessageHooks(msg, EVT_BEFORESAVE) > 0) return(-3);
 
        /*
@@ -2537,7 +2883,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
        }
 
        /* Save it to disk */
-       lprintf(CTDL_DEBUG, "Saving to disk\n");
+       CtdlLogPrintf(CTDL_DEBUG, "Saving to disk\n");
        newmsgid = send_message(msg);
        if (newmsgid <= 0L) return(-5);
 
@@ -2545,7 +2891,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
         * be a critical section because nobody else knows about this message
         * yet.
         */
-       lprintf(CTDL_DEBUG, "Creating MetaData record\n");
+       CtdlLogPrintf(CTDL_DEBUG, "Creating MetaData record\n");
        memset(&smi, 0, sizeof(struct MetaData));
        smi.meta_msgnum = newmsgid;
        smi.meta_refcount = 0;
@@ -2563,13 +2909,13 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,     /* message to save */
         *    message to attach to the journalized copy.
         */
        if (CCC->redirect_buffer != NULL) {
-               lprintf(CTDL_ALERT, "CCC->redirect_buffer is not NULL during message submission!\n");
+               CtdlLogPrintf(CTDL_ALERT, "CCC->redirect_buffer is not NULL during message submission!\n");
                abort();
        }
        CCC->redirect_buffer = malloc(SIZ);
        CCC->redirect_len = 0;
        CCC->redirect_alloc = SIZ;
-       CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1);
+       CtdlOutputPreLoadedMsg(msg, MT_RFC822, HEADERS_ALL, 0, 1, QP_EADDR);
        smi.meta_rfc822_length = CCC->redirect_len;
        saved_rfc822_version = CCC->redirect_buffer;
        CCC->redirect_buffer = NULL;
@@ -2579,7 +2925,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
        PutMetaData(&smi);
 
        /* Now figure out where to store the pointers */
-       lprintf(CTDL_DEBUG, "Storing pointers\n");
+       CtdlLogPrintf(CTDL_DEBUG, "Storing pointers\n");
 
        /* If this is being done by the networker delivering a private
         * message, we want to BYPASS saving the sender's copy (because there
@@ -2587,49 +2933,57 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,     /* message to save */
         */
        if ((!CCC->internal_pgm) || (recps == NULL)) {
                if (CtdlSaveMsgPointerInRoom(actual_rm, newmsgid, 1, msg) != 0) {
-                       lprintf(CTDL_ERR, "ERROR saving message pointer!\n");
+                       CtdlLogPrintf(CTDL_ERR, "ERROR saving message pointer!\n");
                        CtdlSaveMsgPointerInRoom(config.c_aideroom, newmsgid, 0, msg);
                }
        }
 
        /* For internet mail, drop a copy in the outbound queue room */
-       if (recps != NULL)
-        if (recps->num_internet > 0) {
+       if ((recps != NULL) && (recps->num_internet > 0)) {
                CtdlSaveMsgPointerInRoom(SMTP_SPOOLOUT_ROOM, newmsgid, 0, msg);
        }
 
        /* If other rooms are specified, drop them there too. */
-       if (recps != NULL)
-        if (recps->num_room > 0)
+       if ((recps != NULL) && (recps->num_room > 0))
          for (i=0; i<num_tokens(recps->recp_room, '|'); ++i) {
                extract_token(recipient, recps->recp_room, i,
                                        '|', sizeof recipient);
-               lprintf(CTDL_DEBUG, "Delivering to room <%s>\n", recipient);
+               CtdlLogPrintf(CTDL_DEBUG, "Delivering to room <%s>\n", recipient);
                CtdlSaveMsgPointerInRoom(recipient, newmsgid, 0, msg);
        }
 
        /* Bump this user's messages posted counter. */
-       lprintf(CTDL_DEBUG, "Updating user\n");
-       lgetuser(&CCC->user, CCC->curr_user);
+       CtdlLogPrintf(CTDL_DEBUG, "Updating user\n");
+       CtdlGetUserLock(&CCC->user, CCC->curr_user);
        CCC->user.posted = CCC->user.posted + 1;
-       lputuser(&CCC->user);
+       CtdlPutUserLock(&CCC->user);
+
+       /* Decide where bounces need to be delivered */
+       if ((recps != NULL) && (recps->bounce_to != NULL)) {
+               safestrncpy(bounce_to, recps->bounce_to, sizeof bounce_to);
+       }
+       else if (CCC->logged_in) {
+               snprintf(bounce_to, sizeof bounce_to, "%s@%s", CCC->user.fullname, config.c_nodename);
+       }
+       else {
+               snprintf(bounce_to, sizeof bounce_to, "%s@%s", msg->cm_fields['A'], msg->cm_fields['N']);
+       }
 
        /* If this is private, local mail, make a copy in the
         * recipient's mailbox and bump the reference count.
         */
-       if (recps != NULL)
-        if (recps->num_local > 0)
+       if ((recps != NULL) && (recps->num_local > 0))
          for (i=0; i<num_tokens(recps->recp_local, '|'); ++i) {
                extract_token(recipient, recps->recp_local, i,
                                        '|', sizeof recipient);
-               lprintf(CTDL_DEBUG, "Delivering private local mail to <%s>\n",
+               CtdlLogPrintf(CTDL_DEBUG, "Delivering private local mail to <%s>\n",
                        recipient);
-               if (getuser(&userbuf, recipient) == 0) {
+               if (CtdlGetUser(&userbuf, recipient) == 0) {
                        // Add a flag so the Funambol module knows its mail
                        msg->cm_fields['W'] = strdup(recipient);
-                       MailboxName(actual_rm, sizeof actual_rm, &userbuf, MAILROOM);
+                       CtdlMailboxName(actual_rm, sizeof actual_rm, &userbuf, MAILROOM);
                        CtdlSaveMsgPointerInRoom(actual_rm, newmsgid, 0, msg);
-                       BumpNewMailCounter(userbuf.usernum);
+                       CtdlBumpNewMailCounter(userbuf.usernum);
                        if (!IsEmptyStr(config.c_funambol_host) || !IsEmptyStr(config.c_pager_program)) {
                        /* Generate a instruction message for the Funambol notification
                         * server, in the same style as the SMTP queue
@@ -2638,9 +2992,9 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
                           instr = malloc(instr_alloc);
                           snprintf(instr, instr_alloc,
                        "Content-type: %s\n\nmsgid|%ld\nsubmitted|%ld\n"
-                       "bounceto|%s@%s\n",
+                       "bounceto|%s\n",
                        SPOOLMIME, newmsgid, (long)time(NULL),
-                       msg->cm_fields['A'], msg->cm_fields['N']
+                       bounce_to
                        );
 
                           imsg = malloc(sizeof(struct CtdlMessage));
@@ -2652,19 +3006,19 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,     /* message to save */
                           imsg->cm_fields['J'] = strdup("do not journal");
                           imsg->cm_fields['M'] = instr;        /* imsg owns this memory now */
                           imsg->cm_fields['W'] = strdup(recipient);
-                          CtdlSubmitMsg(imsg, NULL, FNBL_QUEUE_ROOM);
+                          CtdlSubmitMsg(imsg, NULL, FNBL_QUEUE_ROOM, 0);
                           CtdlFreeMessage(imsg);
                        }
                }
                else {
-                       lprintf(CTDL_DEBUG, "No user <%s>\n", recipient);
+                       CtdlLogPrintf(CTDL_DEBUG, "No user <%s>\n", recipient);
                        CtdlSaveMsgPointerInRoom(config.c_aideroom,
                                newmsgid, 0, msg);
                }
        }
 
        /* Perform "after save" hooks */
-       lprintf(CTDL_DEBUG, "Performing after-save hooks\n");
+       CtdlLogPrintf(CTDL_DEBUG, "Performing after-save hooks\n");
        PerformMessageHooks(msg, EVT_AFTERSAVE);
 
        /* For IGnet mail, we have to save a new copy into the spooler for
@@ -2675,8 +3029,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
         * node.  We'll revisit this again in a year or so when everyone has
         * a network spool receiver that can handle the new style messages.
         */
-       if (recps != NULL)
-        if (recps->num_ignet > 0)
+       if ((recps != NULL) && (recps->num_ignet > 0))
          for (i=0; i<num_tokens(recps->recp_ignet, '|'); ++i) {
                extract_token(recipient, recps->recp_ignet, i,
                                '|', sizeof recipient);
@@ -2696,7 +3049,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
                                         (long) getpid(), CCC->cs_pid, ++seqnum);
                        network_fp = fopen(submit_filename, "wb+");
                        if (network_fp != NULL) {
-                               fwrite(smr.ser, smr.len, 1, network_fp);
+                               rv = fwrite(smr.ser, smr.len, 1, network_fp);
                                fclose(network_fp);
                        }
                        free(smr.ser);
@@ -2709,29 +3062,33 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,     /* message to save */
        }
 
        /* Go back to the room we started from */
-       lprintf(CTDL_DEBUG, "Returning to original room %s\n", hold_rm);
+       CtdlLogPrintf(CTDL_DEBUG, "Returning to original room %s\n", hold_rm);
        if (strcasecmp(hold_rm, CCC->room.QRname))
-               usergoto(hold_rm, 0, 1, NULL, NULL);
+               CtdlUserGoto(hold_rm, 0, 1, NULL, NULL);
 
        /* For internet mail, generate delivery instructions.
         * Yes, this is recursive.  Deal with it.  Infinite recursion does
         * not happen because the delivery instructions message does not
         * contain a recipient.
         */
-       if (recps != NULL)
-        if (recps->num_internet > 0) {
-               lprintf(CTDL_DEBUG, "Generating delivery instructions\n");
+       if ((recps != NULL) && (recps->num_internet > 0)) {
+               CtdlLogPrintf(CTDL_DEBUG, "Generating delivery instructions\n");
                instr_alloc = 1024;
                instr = malloc(instr_alloc);
                snprintf(instr, instr_alloc,
                        "Content-type: %s\n\nmsgid|%ld\nsubmitted|%ld\n"
-                       "bounceto|%s@%s\n",
+                       "bounceto|%s\n",
                        SPOOLMIME, newmsgid, (long)time(NULL),
-                       msg->cm_fields['A'], msg->cm_fields['N']
+                       bounce_to
                );
 
+               if (recps->envelope_from != NULL) {
+                       tmp = strlen(instr);
+                       snprintf(&instr[tmp], instr_alloc-tmp, "envelope_from|%s\n", recps->envelope_from);
+               }
+
                for (i=0; i<num_tokens(recps->recp_internet, '|'); ++i) {
-                       size_t tmp = strlen(instr);
+                       tmp = strlen(instr);
                        extract_token(recipient, recps->recp_internet, i, '|', sizeof recipient);
                        if ((tmp + strlen(recipient) + 32) > instr_alloc) {
                                instr_alloc = instr_alloc * 2;
@@ -2748,7 +3105,7 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,       /* message to save */
                imsg->cm_fields['A'] = strdup("Citadel");
                imsg->cm_fields['J'] = strdup("do not journal");
                imsg->cm_fields['M'] = instr;   /* imsg owns this memory now */
-               CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM);
+               CtdlSubmitMsg(imsg, NULL, SMTP_SPOOLOUT_ROOM, QP_EADDR);
                CtdlFreeMessage(imsg);
        }
 
@@ -2760,14 +3117,14 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,     /* message to save */
        }
 
        if (collected_addresses != NULL) {
-               begin_critical_section(S_ATBF);
                aptr = (struct addresses_to_be_filed *)
                        malloc(sizeof(struct addresses_to_be_filed));
-               aptr->next = atbf;
-               MailboxName(actual_rm, sizeof actual_rm,
+               CtdlMailboxName(actual_rm, sizeof actual_rm,
                        &CCC->user, USERCONTACTSROOM);
                aptr->roomname = strdup(actual_rm);
                aptr->collected_addresses = collected_addresses;
+               begin_critical_section(S_ATBF);
+               aptr->next = atbf;
                atbf = aptr;
                end_critical_section(S_ATBF);
        }
@@ -2810,13 +3167,17 @@ long CtdlSubmitMsg(struct CtdlMessage *msg,     /* message to save */
 
 
 
+void aide_message (char *text, char *subject)
+{
+       quickie_message("Citadel",NULL,NULL,AIDEROOM,text,FMT_CITADEL,subject);
+}
 
 
 /*
  * Convenience function for generating small administrative messages.
  */
-void quickie_message(char *from, char *fromaddr, char *to, char *room, char *text, 
-                       int format_type, char *subject)
+void quickie_message(const char *from, const char *fromaddr, char *to, char *room, const char *text, 
+                       int format_type, const char *subject)
 {
        struct CtdlMessage *msg;
        struct recptypes *recp = NULL;
@@ -2852,7 +3213,7 @@ void quickie_message(char *from, char *fromaddr, char *to, char *room, char *tex
        }
        msg->cm_fields['M'] = strdup(text);
 
-       CtdlSubmitMsg(msg, recp, room);
+       CtdlSubmitMsg(msg, recp, room, 0);
        CtdlFreeMessage(msg);
        if (recp != NULL) free_recipients(recp);
 }
@@ -2862,105 +3223,102 @@ void quickie_message(char *from, char *fromaddr, char *to, char *room, char *tex
 /*
  * Back end function used by CtdlMakeMessage() and similar functions
  */
-char *CtdlReadMessageBody(char *terminator,    /* token signalling EOT */
-                       size_t maxlen,          /* maximum message length */
-                       char *exist,            /* if non-null, append to it;
-                                                  exist is ALWAYS freed  */
-                       int crlf,               /* CRLF newlines instead of LF */
-                       int sock                /* socket handle or 0 for this session's client socket */
-                       ) {
-       char buf[1024];
-       int linelen;
-       size_t message_len = 0;
-       size_t buffer_len = 0;
-       char *ptr;
-       char *m;
+StrBuf *CtdlReadMessageBodyBuf(char *terminator,       /* token signalling EOT */
+                              long tlen,
+                              size_t maxlen,           /* maximum message length */
+                              char *exist,             /* if non-null, append to it;
+                                                          exist is ALWAYS freed  */
+                              int crlf,                /* CRLF newlines instead of LF */
+                              int *sock                /* socket handle or 0 for this session's client socket */
+                       ) 
+{
+       StrBuf *Message;
+       StrBuf *LineBuf;
        int flushing = 0;
        int finished = 0;
        int dotdot = 0;
 
+       LineBuf = NewStrBufPlain(NULL, SIZ);
        if (exist == NULL) {
-               m = malloc(4096);
-               m[0] = 0;
-               buffer_len = 4096;
-               message_len = 0;
+               Message = NewStrBufPlain(NULL, 4 * SIZ);
        }
        else {
-               message_len = strlen(exist);
-               buffer_len = message_len + 4096;
-               m = realloc(exist, buffer_len);
-               if (m == NULL) {
-                       free(exist);
-                       return m;
-               }
+               Message = NewStrBufPlain(exist, -1);
+               free(exist);
        }
 
        /* Do we need to change leading ".." to "." for SMTP escaping? */
-       if (!strcmp(terminator, ".")) {
+       if ((tlen == 1) && (*terminator == '.')) {
                dotdot = 1;
        }
 
-       /* flush the input if we have nowhere to store it */
-       if (m == NULL) {
-               flushing = 1;
-       }
-
        /* read in the lines of message text one by one */
        do {
-               if (sock > 0) {
-                       if (sock_getln(sock, buf, (sizeof buf - 3)) < 0) finished = 1;
-               }
-               else {
-                       if (client_getln(buf, (sizeof buf - 3)) < 1) finished = 1;
-               }
-               if (!strcmp(buf, terminator)) finished = 1;
-               if (crlf) {
-                       strcat(buf, "\r\n");
+               if (sock != NULL) {
+                       if ((CtdlSockGetLine(sock, LineBuf) < 0) ||
+                           (*sock == -1))
+                               finished = 1;
                }
                else {
-                       strcat(buf, "\n");
-               }
-
-               /* Unescape SMTP-style input of two dots at the beginning of the line */
-               if (dotdot) {
-                       if (!strncmp(buf, "..", 2)) {
-                               strcpy(buf, &buf[1]);
-                       }
+                       if (CtdlClientGetLine(LineBuf) < 0) finished = 1;
                }
+               if ((StrLength(LineBuf) == tlen) && 
+                   (!strcmp(ChrPtr(LineBuf), terminator)))
+                       finished = 1;
 
                if ( (!flushing) && (!finished) ) {
-                       /* Measure the line */
-                       linelen = strlen(buf);
-       
-                       /* augment the buffer if we have to */
-                       if ((message_len + linelen) >= buffer_len) {
-                               ptr = realloc(m, (buffer_len * 2) );
-                               if (ptr == NULL) {      /* flush if can't allocate */
-                                       flushing = 1;
-                               } else {
-                                       buffer_len = (buffer_len * 2);
-                                       m = ptr;
-                                       lprintf(CTDL_DEBUG, "buffer_len is now %ld\n", (long)buffer_len);
-                               }
+                       if (crlf) {
+                               StrBufAppendBufPlain(LineBuf, HKEY("\r\n"), 0);
                        }
-       
-                       /* Add the new line to the buffer.  NOTE: this loop must avoid
-                       * using functions like strcat() and strlen() because they
-                       * traverse the entire buffer upon every call, and doing that
-                       * for a multi-megabyte message slows it down beyond usability.
-                       */
-                       strcpy(&m[message_len], buf);
-                       message_len += linelen;
+                       else {
+                               StrBufAppendBufPlain(LineBuf, HKEY("\n"), 0);
+                       }
+                       
+                       /* Unescape SMTP-style input of two dots at the beginning of the line */
+                       if ((dotdot) &&
+                           (StrLength(LineBuf) == 2) && 
+                           (!strcmp(ChrPtr(LineBuf), "..")))
+                       {
+                               StrBufCutLeft(LineBuf, 1);
+                       }
+                       
+                       StrBufAppendBuf(Message, LineBuf, 0);
                }
 
                /* if we've hit the max msg length, flush the rest */
-               if (message_len >= maxlen) flushing = 1;
+               if (StrLength(Message) >= maxlen) flushing = 1;
 
        } while (!finished);
-       return(m);
+       FreeStrBuf(&LineBuf);
+       return Message;
 }
 
 
+/*
+ * Back end function used by CtdlMakeMessage() and similar functions
+ */
+char *CtdlReadMessageBody(char *terminator,    /* token signalling EOT */
+                         long tlen,
+                         size_t maxlen,                /* maximum message length */
+                         char *exist,          /* if non-null, append to it;
+                                                  exist is ALWAYS freed  */
+                         int crlf,             /* CRLF newlines instead of LF */
+                         int *sock             /* socket handle or 0 for this session's client socket */
+       ) 
+{
+       StrBuf *Message;
+
+       Message = CtdlReadMessageBodyBuf(terminator,
+                                        tlen,
+                                        maxlen,
+                                        exist,
+                                        crlf,
+                                        sock);
+       if (Message == NULL)
+               return NULL;
+       else
+               return SmashStrBuf(&Message);
+}
 
 
 /*
@@ -2982,12 +3340,12 @@ struct CtdlMessage *CtdlMakeMessage(
        char *my_email,                 /* which of my email addresses to use (empty is ok) */
        char *subject,                  /* Subject (optional) */
        char *supplied_euid,            /* ...or NULL if this is irrelevant */
-       char *preformatted_text         /* ...or NULL to read text from client */
+       char *preformatted_text,        /* ...or NULL to read text from client */
+       char *references                /* Thread references */
 ) {
        char dest_node[256];
        char buf[1024];
        struct CtdlMessage *msg;
-       int i;
 
        msg = malloc(sizeof(struct CtdlMessage));
        memset(msg, 0, sizeof(struct CtdlMessage));
@@ -2998,8 +3356,8 @@ struct CtdlMessage *CtdlMakeMessage(
        /* Don't confuse the poor folks if it's not routed mail. */
        strcpy(dest_node, "");
 
-       striplt(recipient);
-       striplt(recp_cc);
+       if (recipient != NULL) striplt(recipient);
+       if (recp_cc != NULL) striplt(recp_cc);
 
        /* Path or Return-Path */
        if (my_email == NULL) my_email = "";
@@ -3011,19 +3369,17 @@ struct CtdlMessage *CtdlMakeMessage(
                snprintf(buf, sizeof buf, "%s", author->fullname);
                msg->cm_fields['P'] = strdup(buf);
        }
-       for (i=0; (msg->cm_fields['P'][i]!=0); ++i) {
-               if (isspace(msg->cm_fields['P'][i])) {
-                       msg->cm_fields['P'][i] = '_';
-               }
-       }
+       convert_spaces_to_underscores(msg->cm_fields['P']);
 
        snprintf(buf, sizeof buf, "%ld", (long)time(NULL));     /* timestamp */
        msg->cm_fields['T'] = strdup(buf);
 
-       if (fake_name[0])                                       /* author */
+       if ((fake_name != NULL) && (fake_name[0])) {            /* author */
                msg->cm_fields['A'] = strdup(fake_name);
-       else
+       }
+       else {
                msg->cm_fields['A'] = strdup(author->fullname);
+       }
 
        if (CC->room.QRflags & QR_MAILBOX) {            /* room */
                msg->cm_fields['O'] = strdup(&CC->room.QRname[11]);
@@ -3035,10 +3391,10 @@ struct CtdlMessage *CtdlMakeMessage(
        msg->cm_fields['N'] = strdup(NODENAME);         /* nodename */
        msg->cm_fields['H'] = strdup(HUMANNODE);                /* hnodename */
 
-       if (recipient[0] != 0) {
+       if ((recipient != NULL) && (recipient[0] != 0)) {
                msg->cm_fields['R'] = strdup(recipient);
        }
-       if (recp_cc[0] != 0) {
+       if ((recp_cc != NULL) && (recp_cc[0] != 0)) {
                msg->cm_fields['Y'] = strdup(recp_cc);
        }
        if (dest_node[0] != 0) {
@@ -3078,11 +3434,17 @@ struct CtdlMessage *CtdlMakeMessage(
                msg->cm_fields['E'] = strdup(supplied_euid);
        }
 
+       if (references != NULL) {
+               if (!IsEmptyStr(references)) {
+                       msg->cm_fields['W'] = strdup(references);
+               }
+       }
+
        if (preformatted_text != NULL) {
                msg->cm_fields['M'] = preformatted_text;
        }
        else {
-               msg->cm_fields['M'] = CtdlReadMessageBody("000", config.c_maxmsglen, NULL, 0, 0);
+               msg->cm_fields['M'] = CtdlReadMessageBody(HKEY("000"), config.c_maxmsglen, NULL, 0, 0);
        }
 
        return(msg);
@@ -3154,7 +3516,7 @@ int CtdlDoIHavePermissionToPostInThisRoom(char *errmsgbuf,
 
        }
 
-       if ((CC->user.axlevel < 2)
+       if ((CC->user.axlevel < AxProbU)
            && ((CC->room.QRflags & QR_MAILBOX) == 0)) {
                snprintf(errmsgbuf, n, "Need to be validated to enter "
                                "(except in %s> to sysop)", MAILROOM);
@@ -3179,7 +3541,7 @@ int CtdlDoIHavePermissionToPostInThisRoom(char *errmsgbuf,
 int CtdlCheckInternetMailPermission(struct ctdluser *who) {
 
        /* Do not allow twits to send Internet mail */
-       if (who->axlevel <= 2) return(0);
+       if (who->axlevel <= AxProbU) return(0);
 
        /* Globally enabled? */
        if (config.c_restrict == 0) return(1);
@@ -3188,7 +3550,7 @@ int CtdlCheckInternetMailPermission(struct ctdluser *who) {
        if (who->flags & US_INTERNET) return(2);
 
        /* Aide level access? */
-       if (who->axlevel >= 6) return(3);
+       if (who->axlevel >= AxAideU) return(3);
 
        /* No mail for you! */
        return(0);
@@ -3204,7 +3566,7 @@ int CtdlCheckInternetMailPermission(struct ctdluser *who) {
  *
  * Caller needs to free the result using free_recipients()
  */
-struct recptypes *validate_recipients(char *supplied_recipients, 
+struct recptypes *validate_recipients(const char *supplied_recipients, 
                                      const char *RemoteIdentifier, 
                                      int Flags) {
        struct recptypes *ret;
@@ -3287,7 +3649,7 @@ struct recptypes *validate_recipients(char *supplied_recipients,
                striplt(this_recp);
                if (IsEmptyStr(this_recp))
                        break;
-               lprintf(CTDL_DEBUG, "Evaluating recipient #%d: %s\n", num_recps, this_recp);
+               CtdlLogPrintf(CTDL_DEBUG, "Evaluating recipient #%d: %s\n", num_recps, this_recp);
                ++num_recps;
                mailtype = alias(this_recp);
                mailtype = alias(this_recp);
@@ -3314,24 +3676,8 @@ struct recptypes *validate_recipients(char *supplied_recipients,
                                        }
                                        strcat(ret->recp_room, this_recp);
                                }
-                               else if (getuser(&tempUS, this_recp) == 0) {
-                                       ++ret->num_local;
-                                       strcpy(this_recp, tempUS.fullname);
-                                       if (!IsEmptyStr(ret->recp_local)) {
-                                               strcat(ret->recp_local, "|");
-                                       }
-                                       strcat(ret->recp_local, this_recp);
-                               }
-                               else if (getuser(&tempUS, this_recp_cooked) == 0) {
-                                       ++ret->num_local;
-                                       strcpy(this_recp, tempUS.fullname);
-                                       if (!IsEmptyStr(ret->recp_local)) {
-                                               strcat(ret->recp_local, "|");
-                                       }
-                                       strcat(ret->recp_local, this_recp);
-                               }
                                else if ( (!strncasecmp(this_recp, "room_", 5))
-                                     && (!getroom(&tempQR, &this_recp_cooked[5])) ) {
+                                     && (!CtdlGetRoom(&tempQR, &this_recp_cooked[5])) ) {
 
                                        /* Save room so we can restore it later */
                                        tempQR2 = CC->room;
@@ -3360,6 +3706,22 @@ struct recptypes *validate_recipients(char *supplied_recipients,
                                        CC->room = tempQR2;
 
                                }
+                               else if (CtdlGetUser(&tempUS, this_recp) == 0) {
+                                       ++ret->num_local;
+                                       strcpy(this_recp, tempUS.fullname);
+                                       if (!IsEmptyStr(ret->recp_local)) {
+                                               strcat(ret->recp_local, "|");
+                                       }
+                                       strcat(ret->recp_local, this_recp);
+                               }
+                               else if (CtdlGetUser(&tempUS, this_recp_cooked) == 0) {
+                                       ++ret->num_local;
+                                       strcpy(this_recp, tempUS.fullname);
+                                       if (!IsEmptyStr(ret->recp_local)) {
+                                               strcat(ret->recp_local, "|");
+                                       }
+                                       strcat(ret->recp_local, this_recp);
+                               }
                                else {
                                        ++ret->num_error;
                                        invalid = 1;
@@ -3429,12 +3791,12 @@ struct recptypes *validate_recipients(char *supplied_recipients,
                strcpy(ret->errormsg, "No recipients specified.");
        }
 
-       lprintf(CTDL_DEBUG, "validate_recipients()\n");
-       lprintf(CTDL_DEBUG, " local: %d <%s>\n", ret->num_local, ret->recp_local);
-       lprintf(CTDL_DEBUG, "  room: %d <%s>\n", ret->num_room, ret->recp_room);
-       lprintf(CTDL_DEBUG, "  inet: %d <%s>\n", ret->num_internet, ret->recp_internet);
-       lprintf(CTDL_DEBUG, " ignet: %d <%s>\n", ret->num_ignet, ret->recp_ignet);
-       lprintf(CTDL_DEBUG, " error: %d <%s>\n", ret->num_error, ret->errormsg);
+       CtdlLogPrintf(CTDL_DEBUG, "validate_recipients()\n");
+       CtdlLogPrintf(CTDL_DEBUG, " local: %d <%s>\n", ret->num_local, ret->recp_local);
+       CtdlLogPrintf(CTDL_DEBUG, "  room: %d <%s>\n", ret->num_room, ret->recp_room);
+       CtdlLogPrintf(CTDL_DEBUG, "  inet: %d <%s>\n", ret->num_internet, ret->recp_internet);
+       CtdlLogPrintf(CTDL_DEBUG, " ignet: %d <%s>\n", ret->num_ignet, ret->recp_ignet);
+       CtdlLogPrintf(CTDL_DEBUG, " error: %d <%s>\n", ret->num_error, ret->errormsg);
 
        free(recipients);
        return(ret);
@@ -3451,7 +3813,7 @@ void free_recipients(struct recptypes *valid) {
        }
 
        if (valid->recptypes_magic != RECPTYPES_MAGIC) {
-               lprintf(CTDL_EMERG, "Attempt to call free_recipients() on some other data type!\n");
+               CtdlLogPrintf(CTDL_EMERG, "Attempt to call free_recipients() on some other data type!\n");
                abort();
        }
 
@@ -3461,6 +3823,8 @@ void free_recipients(struct recptypes *valid) {
        if (valid->recp_ignet != NULL)          free(valid->recp_ignet);
        if (valid->recp_room != NULL)           free(valid->recp_room);
        if (valid->display_recp != NULL)        free(valid->display_recp);
+       if (valid->bounce_to != NULL)           free(valid->bounce_to);
+       if (valid->envelope_from != NULL)       free(valid->envelope_from);
        free(valid);
 }
 
@@ -3495,6 +3859,8 @@ void cmd_ent0(char *entargs)
        int i, j;
        char buf[256];
        int newuseremail_ok = 0;
+       char references[SIZ];
+       char *ptr;
 
        unbuffer_output();
 
@@ -3517,6 +3883,10 @@ void cmd_ent0(char *entargs)
                        break;
        }
        extract_token(newuseremail, entargs, 10, '|', sizeof newuseremail);
+       extract_token(references, entargs, 11, '|', sizeof references);
+       for (ptr=references; *ptr != 0; ++ptr) {
+               if (*ptr == '!') *ptr = '|';
+       }
 
        /* first check to make sure the request is valid. */
 
@@ -3532,7 +3902,7 @@ void cmd_ent0(char *entargs)
        if (IsEmptyStr(newusername)) {
                strcpy(newusername, CC->user.fullname);
        }
-       if (  (CC->user.axlevel < 6)
+       if (  (CC->user.axlevel < AxAideU)
           && (strcasecmp(newusername, CC->user.fullname))
           && (strcasecmp(newusername, CC->cs_inet_fn))
        ) {     
@@ -3582,7 +3952,7 @@ void cmd_ent0(char *entargs)
        if (  ( (CC->room.QRflags & QR_MAILBOX) && (!strcasecmp(&CC->room.QRname[11], MAILROOM)) )
           || ( (CC->room.QRflags & QR_MAILBOX) && (CC->curr_view == VIEW_MAILBOX) )
        ) {
-               if (CC->user.axlevel < 2) {
+               if (CC->user.axlevel < AxProbU) {
                        strcpy(recp, "sysop");
                        strcpy(cc, "");
                        strcpy(bcc, "");
@@ -3634,7 +4004,7 @@ void cmd_ent0(char *entargs)
                }
 
                if ( ( (valid_to->num_internet + valid_to->num_ignet + valid_cc->num_internet + valid_cc->num_ignet + valid_bcc->num_internet + valid_bcc->num_ignet) > 0)
-                  && (CC->user.axlevel < 4) ) {
+                  && (CC->user.axlevel < AxNetU) ) {
                        cprintf("%d Higher access required for network mail.\n",
                                ERROR + HIGHER_ACCESS_REQUIRED);
                        free_recipients(valid_to);
@@ -3677,9 +4047,9 @@ void cmd_ent0(char *entargs)
         * is set, or if there is one or more Internet email recipients.
         */
        if (CC->room.QRflags2 & QR2_SUBJECTREQ) subject_required = 1;
-       if (valid_to) if (valid_to->num_internet > 0) subject_required = 1;
-       if (valid_cc) if (valid_cc->num_internet > 0) subject_required = 1;
-       if (valid_bcc) if (valid_bcc->num_internet > 0) subject_required = 1;
+       if ((valid_to)  && (valid_to->num_internet > 0))        subject_required = 1;
+       if ((valid_cc)  && (valid_cc->num_internet > 0))        subject_required = 1;
+       if ((valid_bcc) && (valid_bcc->num_internet > 0))       subject_required = 1;
 
        /* If we're only checking the validity of the request, return
         * success without creating the message.
@@ -3710,7 +4080,7 @@ void cmd_ent0(char *entargs)
                CC->room.QRname, anonymous, format_type,
                newusername, newuseremail, subject,
                ((!IsEmptyStr(supplied_euid)) ? supplied_euid : NULL),
-               NULL);
+               NULL, references);
 
        /* Put together one big recipients struct containing to/cc/bcc all in
         * one.  This is for the envelope.
@@ -3738,7 +4108,7 @@ void cmd_ent0(char *entargs)
        free(all_recps);
 
        if (msg != NULL) {
-               msgnum = CtdlSubmitMsg(msg, valid, "");
+               msgnum = CtdlSubmitMsg(msg, valid, "", QP_EADDR);
 
                if (do_confirm) {
                        cprintf("%ld\n", msgnum);
@@ -3793,12 +4163,12 @@ int CtdlDeleteMessages(char *room_name,         /* which room */
                regcomp(&re, content_type, 0);
                need_to_free_re = 1;
        }
-       lprintf(CTDL_DEBUG, "CtdlDeleteMessages(%s, %d msgs, %s)\n",
+       CtdlLogPrintf(CTDL_DEBUG, "CtdlDeleteMessages(%s, %d msgs, %s)\n",
                room_name, num_dmsgnums, content_type);
 
        /* get room record, obtaining a lock... */
-       if (lgetroom(&qrbuf, room_name) != 0) {
-               lprintf(CTDL_ERR, "CtdlDeleteMessages(): Room <%s> not found\n",
+       if (CtdlGetRoomLock(&qrbuf, room_name) != 0) {
+               CtdlLogPrintf(CTDL_ERR, "CtdlDeleteMessages(): Room <%s> not found\n",
                        room_name);
                if (need_to_free_re) regfree(&re);
                return (0);     /* room not found */
@@ -3852,9 +4222,12 @@ int CtdlDeleteMessages(char *room_name,          /* which room */
                cdb_store(CDB_MSGLISTS, &qrbuf.QRnumber, (int)sizeof(long),
                          msglist, (int)(num_msgs * sizeof(long)));
 
-               qrbuf.QRhighest = msglist[num_msgs - 1];
+               if (num_msgs > 0)
+                       qrbuf.QRhighest = msglist[num_msgs - 1];
+               else
+                       qrbuf.QRhighest = 0;
        }
-       lputroom(&qrbuf);
+       CtdlPutRoomLock(&qrbuf);
 
        /* Go through the messages we pulled out of the index, and decrement
         * their reference counts by 1.  If this is the only room the message
@@ -3872,7 +4245,7 @@ int CtdlDeleteMessages(char *room_name,           /* which room */
        /* Now free the memory we used, and go away. */
        if (msglist != NULL) free(msglist);
        if (dellist != NULL) free(dellist);
-       lprintf(CTDL_DEBUG, "%d message(s) deleted.\n", num_deleted);
+       CtdlLogPrintf(CTDL_DEBUG, "%d message(s) deleted.\n", num_deleted);
        if (need_to_free_re) regfree(&re);
        return (num_deleted);
 }
@@ -3939,19 +4312,6 @@ void cmd_dele(char *args)
 }
 
 
-/*
- * Back end API function for moves and deletes (multiple messages)
- */
-int CtdlCopyMsgsToRoom(long *msgnums, int num_msgs, char *dest) {
-       int err;
-
-       err = CtdlSaveMsgPointersInRoom(dest, msgnums, num_msgs, 1, NULL);
-       if (err != 0) return(err);
-
-       return(0);
-}
-
-
 
 
 /*
@@ -3984,13 +4344,17 @@ void cmd_move(char *args)
        targ[ROOMNAMELEN - 1] = 0;
        is_copy = extract_int(args, 2);
 
-       if (getroom(&qtemp, targ) != 0) {
-               cprintf("%d '%s' does not exist.\n",
-                       ERROR + ROOM_NOT_FOUND, targ);
+       if (CtdlGetRoom(&qtemp, targ) != 0) {
+               cprintf("%d '%s' does not exist.\n", ERROR + ROOM_NOT_FOUND, targ);
+               return;
+       }
+
+       if (!strcasecmp(qtemp.QRname, CC->room.QRname)) {
+               cprintf("%d Source and target rooms are the same.\n", ERROR + ALREADY_EXISTS);
                return;
        }
 
-       getuser(&CC->user, CC->curr_user);
+       CtdlGetUser(&CC->user, CC->curr_user);
        CtdlRoomAccess(&qtemp, &CC->user, &ra, NULL);
 
        /* Check for permission to perform this operation.
@@ -3999,7 +4363,7 @@ void cmd_move(char *args)
        permit = 0;
 
        /* Aides can move/copy */
-       if (CC->user.axlevel >= 6) permit = 1;
+       if (CC->user.axlevel >= AxAideU) permit = 1;
 
        /* Room aides can move/copy */
        if (CC->user.usernum == CC->room.QRroomaide) permit = 1;
@@ -4041,7 +4405,7 @@ void cmd_move(char *args)
        /*
         * Do the copy
         */
-       err = CtdlCopyMsgsToRoom(msgs, num_msgs, targ);
+       err = CtdlSaveMsgPointersInRoom(targ, msgs, num_msgs, 1, NULL, 0);
        if (err != 0) {
                cprintf("%d Cannot store message(s) in %s: error %d\n",
                        err, targ, err);
@@ -4113,6 +4477,11 @@ void PutMetaData(struct MetaData *smibuf)
 void AdjRefCount(long msgnum, int incr)
 {
        struct arcq new_arcq;
+       int rv = 0;
+
+       CtdlLogPrintf(CTDL_DEBUG, "AdjRefCount() msg %ld ref count delta %+d\n",
+               msgnum, incr
+       );
 
        begin_critical_section(S_SUPPMSGMAIN);
        if (arcfp == NULL) {
@@ -4122,7 +4491,7 @@ void AdjRefCount(long msgnum, int incr)
 
        /* msgnum < 0 means that we're trying to close the file */
        if (msgnum < 0) {
-               lprintf(CTDL_DEBUG, "Closing the AdjRefCount queue file\n");
+               CtdlLogPrintf(CTDL_DEBUG, "Closing the AdjRefCount queue file\n");
                begin_critical_section(S_SUPPMSGMAIN);
                if (arcfp != NULL) {
                        fclose(arcfp);
@@ -4142,7 +4511,7 @@ void AdjRefCount(long msgnum, int incr)
 
        new_arcq.arcq_msgnum = msgnum;
        new_arcq.arcq_delta = incr;
-       fwrite(&new_arcq, sizeof(struct arcq), 1, arcfp);
+       rv = fwrite(&new_arcq, sizeof(struct arcq), 1, arcfp);
        fflush(arcfp);
 
        return;
@@ -4164,7 +4533,7 @@ int TDAP_ProcessAdjRefCountQueue(void)
        struct arcq arcq_rec;
        int num_records_processed = 0;
 
-       snprintf(file_arcq_temp, sizeof file_arcq_temp, "%s2", file_arcq);
+       snprintf(file_arcq_temp, sizeof file_arcq_temp, "%s.%04x", file_arcq, rand());
 
        begin_critical_section(S_SUPPMSGMAIN);
        if (arcfp != NULL) {
@@ -4174,7 +4543,7 @@ int TDAP_ProcessAdjRefCountQueue(void)
 
        r = link(file_arcq, file_arcq_temp);
        if (r != 0) {
-               lprintf(CTDL_CRIT, "%s: %s\n", file_arcq_temp, strerror(errno));
+               CtdlLogPrintf(CTDL_CRIT, "%s: %s\n", file_arcq_temp, strerror(errno));
                end_critical_section(S_SUPPMSGMAIN);
                return(num_records_processed);
        }
@@ -4184,7 +4553,7 @@ int TDAP_ProcessAdjRefCountQueue(void)
 
        fp = fopen(file_arcq_temp, "rb");
        if (fp == NULL) {
-               lprintf(CTDL_CRIT, "%s: %s\n", file_arcq_temp, strerror(errno));
+               CtdlLogPrintf(CTDL_CRIT, "%s: %s\n", file_arcq_temp, strerror(errno));
                return(num_records_processed);
        }
 
@@ -4196,7 +4565,7 @@ int TDAP_ProcessAdjRefCountQueue(void)
        fclose(fp);
        r = unlink(file_arcq_temp);
        if (r != 0) {
-               lprintf(CTDL_CRIT, "%s: %s\n", file_arcq_temp, strerror(errno));
+               CtdlLogPrintf(CTDL_CRIT, "%s: %s\n", file_arcq_temp, strerror(errno));
        }
 
        return(num_records_processed);
@@ -4227,14 +4596,15 @@ void TDAP_AdjRefCount(long msgnum, int incr)
        smi.meta_refcount += incr;
        PutMetaData(&smi);
        end_critical_section(S_SUPPMSGMAIN);
-       lprintf(CTDL_DEBUG, "msg %ld ref count delta %d, is now %d\n",
-               msgnum, incr, smi.meta_refcount);
+       CtdlLogPrintf(CTDL_DEBUG, "TDAP_AdjRefCount() msg %ld ref count delta %+d, is now %d\n",
+               msgnum, incr, smi.meta_refcount
+       );
 
        /* If the reference count is now zero, delete the message
         * (and its supplementary record as well).
         */
        if (smi.meta_refcount == 0) {
-               lprintf(CTDL_DEBUG, "Deleting message <%ld>\n", msgnum);
+               CtdlLogPrintf(CTDL_DEBUG, "Deleting message <%ld>\n", msgnum);
                
                /* Call delete hooks with NULL room to show it has gone altogether */
                PerformDeleteHooks(NULL, msgnum);
@@ -4257,50 +4627,33 @@ void TDAP_AdjRefCount(long msgnum, int incr)
  * Note: this could be much more efficient.  Right now we use two temporary
  * files, and still pull the message into memory as with all others.
  */
-void CtdlWriteObject(char *req_room,           /* Room to stuff it in */
-                       char *content_type,     /* MIME type of this object */
-                       char *tempfilename,     /* Where to fetch it from */
+void CtdlWriteObject(char *req_room,                   /* Room to stuff it in */
+                       char *content_type,             /* MIME type of this object */
+                       char *raw_message,              /* Data to be written */
+                       off_t raw_length,               /* Size of raw_message */
                        struct ctdluser *is_mailbox,    /* Mailbox room? */
-                       int is_binary,          /* Is encoding necessary? */
-                       int is_unique,          /* Del others of this type? */
-                       unsigned int flags      /* Internal save flags */
+                       int is_binary,                  /* Is encoding necessary? */
+                       int is_unique,                  /* Del others of this type? */
+                       unsigned int flags              /* Internal save flags */
                        )
 {
 
-       FILE *fp;
        struct ctdlroom qrbuf;
        char roomname[ROOMNAMELEN];
        struct CtdlMessage *msg;
-
-       char *raw_message = NULL;
        char *encoded_message = NULL;
-       off_t raw_length = 0;
 
        if (is_mailbox != NULL) {
-               MailboxName(roomname, sizeof roomname, is_mailbox, req_room);
+               CtdlMailboxName(roomname, sizeof roomname, is_mailbox, req_room);
        }
        else {
                safestrncpy(roomname, req_room, sizeof(roomname));
        }
 
-       fp = fopen(tempfilename, "rb");
-       if (fp == NULL) {
-               lprintf(CTDL_CRIT, "Cannot open %s: %s\n",
-                       tempfilename, strerror(errno));
-               return;
-       }
-       fseek(fp, 0L, SEEK_END);
-       raw_length = ftell(fp);
-       rewind(fp);
-       lprintf(CTDL_DEBUG, "Raw length is %ld\n", (long)raw_length);
-
-       raw_message = malloc((size_t)raw_length + 2);
-       fread(raw_message, (size_t)raw_length, 1, fp);
-       fclose(fp);
+       CtdlLogPrintf(CTDL_DEBUG, "Raw length is %ld\n", (long)raw_length);
 
        if (is_binary) {
-               encoded_message = malloc((size_t)
-                       (((raw_length * 134) / 100) + 4096 ) );
+               encoded_message = malloc((size_t) (((raw_length * 134) / 100) + 4096 ) );
        }
        else {
                encoded_message = malloc((size_t)(raw_length + 4096));
@@ -4328,7 +4681,6 @@ void CtdlWriteObject(char *req_room,              /* Room to stuff it in */
                );
        }
        else {
-               raw_message[raw_length] = 0;
                memcpy(
                        &encoded_message[strlen(encoded_message)],
                        raw_message,
@@ -4336,9 +4688,7 @@ void CtdlWriteObject(char *req_room,              /* Room to stuff it in */
                );
        }
 
-       free(raw_message);
-
-       lprintf(CTDL_DEBUG, "Allocating\n");
+       CtdlLogPrintf(CTDL_DEBUG, "Allocating\n");
        msg = malloc(sizeof(struct CtdlMessage));
        memset(msg, 0, sizeof(struct CtdlMessage));
        msg->cm_magic = CTDLMESSAGE_MAGIC;
@@ -4353,8 +4703,8 @@ void CtdlWriteObject(char *req_room,              /* Room to stuff it in */
        msg->cm_fields['M'] = encoded_message;
 
        /* Create the requested room if we have to. */
-       if (getroom(&qrbuf, roomname) != 0) {
-               create_room(roomname, 
+       if (CtdlGetRoom(&qrbuf, roomname) != 0) {
+               CtdlCreateRoom(roomname, 
                        ( (is_mailbox != NULL) ? 5 : 3 ),
                        "", 0, 1, 0, VIEW_BBS);
        }
@@ -4362,12 +4712,12 @@ void CtdlWriteObject(char *req_room,            /* Room to stuff it in */
         * other objects of this type that are currently in the room.
         */
        if (is_unique) {
-               lprintf(CTDL_DEBUG, "Deleted %d other msgs of this type\n",
+               CtdlLogPrintf(CTDL_DEBUG, "Deleted %d other msgs of this type\n",
                        CtdlDeleteMessages(roomname, NULL, 0, content_type)
                );
        }
        /* Now write the data */
-       CtdlSubmitMsg(msg, NULL, roomname);
+       CtdlSubmitMsg(msg, NULL, roomname, 0);
        CtdlFreeMessage(msg);
 }
 
@@ -4389,8 +4739,8 @@ char *CtdlGetSysConfig(char *sysconfname) {
        char buf[SIZ];
        
        strcpy(hold_rm, CC->room.QRname);
-       if (getroom(&CC->room, SYSCONFIGROOM) != 0) {
-               getroom(&CC->room, hold_rm);
+       if (CtdlGetRoom(&CC->room, SYSCONFIGROOM) != 0) {
+               CtdlGetRoom(&CC->room, hold_rm);
                return NULL;
        }
 
@@ -4417,7 +4767,7 @@ char *CtdlGetSysConfig(char *sysconfname) {
                }
        }
 
-       getroom(&CC->room, hold_rm);
+       CtdlGetRoom(&CC->room, hold_rm);
 
        if (conf != NULL) do {
                extract_token(buf, conf, 0, '\n', sizeof buf);
@@ -4427,20 +4777,9 @@ char *CtdlGetSysConfig(char *sysconfname) {
        return(conf);
 }
 
-void CtdlPutSysConfig(char *sysconfname, char *sysconfdata) {
-       char temp[PATH_MAX];
-       FILE *fp;
 
-       CtdlMakeTempFileName(temp, sizeof temp);
-
-       fp = fopen(temp, "w");
-       if (fp == NULL) return;
-       fprintf(fp, "%s", sysconfdata);
-       fclose(fp);
-
-       /* this handy API function does all the work for us */
-       CtdlWriteObject(SYSCONFIGROOM, sysconfname, temp, NULL, 0, 1, 0);
-       unlink(temp);
+void CtdlPutSysConfig(char *sysconfname, char *sysconfdata) {
+       CtdlWriteObject(SYSCONFIGROOM, sysconfname, sysconfdata, (strlen(sysconfdata)+1), NULL, 0, 1, 0);
 }
 
 
@@ -4490,3 +4829,29 @@ void cmd_isme(char *argbuf) {
        }
 
 }
+
+
+/*****************************************************************************/
+/*                      MODULE INITIALIZATION STUFF                          */
+/*****************************************************************************/
+
+CTDL_MODULE_INIT(msgbase)
+{
+       if (!threading) {
+               CtdlRegisterProtoHook(cmd_msgs, "MSGS", "Output a list of messages in the current room");
+               CtdlRegisterProtoHook(cmd_msg0, "MSG0", "Output a message in plain text format");
+               CtdlRegisterProtoHook(cmd_msg2, "MSG2", "Output a message in RFC822 format");
+               CtdlRegisterProtoHook(cmd_msg3, "MSG3", "Output a message in raw format (deprecated)");
+               CtdlRegisterProtoHook(cmd_msg4, "MSG4", "Output a message in the client's preferred format");
+               CtdlRegisterProtoHook(cmd_msgp, "MSGP", "Select preferred format for MSG4 output");
+               CtdlRegisterProtoHook(cmd_opna, "OPNA", "Open an attachment for download");
+               CtdlRegisterProtoHook(cmd_dlat, "DLAT", "Download an attachment");
+               CtdlRegisterProtoHook(cmd_ent0, "ENT0", "Enter a message");
+               CtdlRegisterProtoHook(cmd_dele, "DELE", "Delete a message");
+               CtdlRegisterProtoHook(cmd_move, "MOVE", "Move or copy a message to another room");
+               CtdlRegisterProtoHook(cmd_isme, "ISME", "Determine whether an email address belongs to a user");
+       }
+
+        /* return our Subversion id for the Log */
+       return "$Id$";
+}