use an Enum for the cm_fields vector instead of nameless chars
[citadel.git] / citadel / internet_addressing.c
index 6f28a69a336ff79376637ab89caa7fefebc08e3d..b4663fc36e3a2d1d654d5a69aefa93d5edca8e33 100644 (file)
@@ -1,6 +1,4 @@
 /*
- * $Id$
- *
  * This file contains functions which handle the mapping of Internet addresses
  * to users on the Citadel system.
  */
 #include "room_ops.h"
 #include "parsedate.h"
 #include "database.h"
+#include "ctdl_module.h"
+#ifdef HAVE_ICONV
+#include <iconv.h>
+
+#if 0
+/* This is the non-define version in case of s.b. needing to debug */
+inline void FindNextEnd (char *bptr, char *end)
+{
+       /* Find the next ?Q? */
+       end = strchr(bptr + 2, '?');
+       if (end == NULL) return NULL;
+       if (((*(end + 1) == 'B') || (*(end + 1) == 'Q')) && 
+           (*(end + 2) == '?')) {
+               /* skip on to the end of the cluster, the next ?= */
+               end = strstr(end + 3, "?=");
+       }
+       else
+               /* sort of half valid encoding, try to find an end. */
+               end = strstr(bptr, "?=");
+}
+#endif
+
+#define FindNextEnd(bptr, end) { \
+       end = strchr(bptr + 2, '?'); \
+       if (end != NULL) { \
+               if (((*(end + 1) == 'B') || (*(end + 1) == 'Q')) && (*(end + 2) == '?')) { \
+                       end = strstr(end + 3, "?="); \
+               } else end = strstr(bptr, "?="); \
+       } \
+}
+
+/*
+ * Handle subjects with RFC2047 encoding such as:
+ * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
+ */
+void utf8ify_rfc822_string(char *buf) {
+       char *start, *end, *next, *nextend, *ptr;
+       char newbuf[1024];
+       char charset[128];
+       char encoding[16];
+       char istr[1024];
+       iconv_t ic = (iconv_t)(-1) ;
+       char *ibuf;                     /**< Buffer of characters to be converted */
+       char *obuf;                     /**< Buffer for converted characters */
+       size_t ibuflen;                 /**< Length of input buffer */
+       size_t obuflen;                 /**< Length of output buffer */
+       char *isav;                     /**< Saved pointer to input buffer */
+       char *osav;                     /**< Saved pointer to output buffer */
+       int passes = 0;
+       int i, len, delta;
+       int illegal_non_rfc2047_encoding = 0;
+
+       /* Sometimes, badly formed messages contain strings which were simply
+        *  written out directly in some foreign character set instead of
+        *  using RFC2047 encoding.  This is illegal but we will attempt to
+        *  handle it anyway by converting from a user-specified default
+        *  charset to UTF-8 if we see any nonprintable characters.
+        */
+       len = strlen(buf);
+       for (i=0; i<len; ++i) {
+               if ((buf[i] < 32) || (buf[i] > 126)) {
+                       illegal_non_rfc2047_encoding = 1;
+                       i = len; ///< take a shortcut, it won't be more than one.
+               }
+       }
+       if (illegal_non_rfc2047_encoding) {
+               const char *default_header_charset = "iso-8859-1";
+               if ( (strcasecmp(default_header_charset, "UTF-8")) && (strcasecmp(default_header_charset, "us-ascii")) ) {
+                       ctdl_iconv_open("UTF-8", default_header_charset, &ic);
+                       if (ic != (iconv_t)(-1) ) {
+                               ibuf = malloc(1024);
+                               isav = ibuf;
+                               safestrncpy(ibuf, buf, 1024);
+                               ibuflen = strlen(ibuf);
+                               obuflen = 1024;
+                               obuf = (char *) malloc(obuflen);
+                               osav = obuf;
+                               iconv(ic, &ibuf, &ibuflen, &obuf, &obuflen);
+                               osav[1024-obuflen] = 0;
+                               strcpy(buf, osav);
+                               free(osav);
+                               iconv_close(ic);
+                               free(isav);
+                       }
+               }
+       }
+
+       /* pre evaluate the first pair */
+       nextend = end = NULL;
+       len = strlen(buf);
+       start = strstr(buf, "=?");
+       if (start != NULL) 
+               FindNextEnd (start, end);
+
+       while ((start != NULL) && (end != NULL))
+       {
+               next = strstr(end, "=?");
+               if (next != NULL)
+                       FindNextEnd(next, nextend);
+               if (nextend == NULL)
+                       next = NULL;
+
+               /* did we find two partitions */
+               if ((next != NULL) && 
+                   ((next - end) > 2))
+               {
+                       ptr = end + 2;
+                       while ((ptr < next) && 
+                              (isspace(*ptr) ||
+                               (*ptr == '\r') ||
+                               (*ptr == '\n') || 
+                               (*ptr == '\t')))
+                               ptr ++;
+                       /* did we find a gab just filled with blanks? */
+                       if (ptr == next)
+                       {
+                               memmove (end + 2,
+                                        next,
+                                        len - (next - start));
+
+                               /* now terminate the gab at the end */
+                               delta = (next - end) - 2;
+                               len -= delta;
+                               buf[len] = '\0';
+
+                               /* move next to its new location. */
+                               next -= delta;
+                               nextend -= delta;
+                       }
+               }
+               /* our next-pair is our new first pair now. */
+               start = next;
+               end = nextend;
+       }
+
+       /* Now we handle foreign character sets properly encoded
+        * in RFC2047 format.
+        */
+       start = strstr(buf, "=?");
+       FindNextEnd((start != NULL)? start : buf, end);
+       while (start != NULL && end != NULL && end > start)
+       {
+               extract_token(charset, start, 1, '?', sizeof charset);
+               extract_token(encoding, start, 2, '?', sizeof encoding);
+               extract_token(istr, start, 3, '?', sizeof istr);
+
+               ibuf = malloc(1024);
+               isav = ibuf;
+               if (!strcasecmp(encoding, "B")) {       /**< base64 */
+                       ibuflen = CtdlDecodeBase64(ibuf, istr, strlen(istr));
+               }
+               else if (!strcasecmp(encoding, "Q")) {  /**< quoted-printable */
+                       size_t len;
+                       long pos;
+                       
+                       len = strlen(istr);
+                       pos = 0;
+                       while (pos < len)
+                       {
+                               if (istr[pos] == '_') istr[pos] = ' ';
+                               pos++;
+                       }
+
+                       ibuflen = CtdlDecodeQuotedPrintable(ibuf, istr, len);
+               }
+               else {
+                       strcpy(ibuf, istr);             /**< unknown encoding */
+                       ibuflen = strlen(istr);
+               }
+
+               ctdl_iconv_open("UTF-8", charset, &ic);
+               if (ic != (iconv_t)(-1) ) {
+                       obuflen = 1024;
+                       obuf = (char *) malloc(obuflen);
+                       osav = obuf;
+                       iconv(ic, &ibuf, &ibuflen, &obuf, &obuflen);
+                       osav[1024-obuflen] = 0;
+
+                       end = start;
+                       end++;
+                       strcpy(start, "");
+                       remove_token(end, 0, '?');
+                       remove_token(end, 0, '?');
+                       remove_token(end, 0, '?');
+                       remove_token(end, 0, '?');
+                       strcpy(end, &end[1]);
+
+                       snprintf(newbuf, sizeof newbuf, "%s%s%s", buf, osav, end);
+                       strcpy(buf, newbuf);
+                       free(osav);
+                       iconv_close(ic);
+               }
+               else {
+                       end = start;
+                       end++;
+                       strcpy(start, "");
+                       remove_token(end, 0, '?');
+                       remove_token(end, 0, '?');
+                       remove_token(end, 0, '?');
+                       remove_token(end, 0, '?');
+                       strcpy(end, &end[1]);
+
+                       snprintf(newbuf, sizeof newbuf, "%s(unreadable)%s", buf, end);
+                       strcpy(buf, newbuf);
+               }
+
+               free(isav);
+
+               /*
+                * Since spammers will go to all sorts of absurd lengths to get their
+                * messages through, there are LOTS of corrupt headers out there.
+                * So, prevent a really badly formed RFC2047 header from throwing
+                * this function into an infinite loop.
+                */
+               ++passes;
+               if (passes > 20) return;
+
+               start = strstr(buf, "=?");
+               FindNextEnd((start != NULL)? start : buf, end);
+       }
 
+}
+#else
+inline void utf8ify_rfc822_string(char *a){};
 
-#ifndef HAVE_SNPRINTF
-#include "snprintf.h"
 #endif
 
 
+
 struct trynamebuf {
        char buffer1[SIZ];
        char buffer2[SIZ];
@@ -67,6 +287,7 @@ int CtdlHostAlias(char *fqdn) {
        int i;
        char buf[256];
        char host[256], type[256];
+       int found = 0;
 
        if (fqdn == NULL) return(hostalias_nomatch);
        if (IsEmptyStr(fqdn)) return(hostalias_nomatch);
@@ -81,22 +302,23 @@ int CtdlHostAlias(char *fqdn) {
                extract_token(host, buf, 0, '|', sizeof host);
                extract_token(type, buf, 1, '|', sizeof type);
 
-               if ( (!strcasecmp(type, "localhost"))
-                  && (!strcasecmp(fqdn, host)))
-                       return(hostalias_localhost);
-
-               if ( (!strcasecmp(type, "gatewaydomain"))
-                  && (!strcasecmp(&fqdn[strlen(fqdn)-strlen(host)], host)))
-                       return(hostalias_gatewaydomain);
+               found = 0;
 
-               if ( (!strcasecmp(type, "directory"))
-                  && (!strcasecmp(&fqdn[strlen(fqdn)-strlen(host)], host)))
-                       return(hostalias_directory);
+               /* Process these in a specific order, in case there are multiple matches.
+                * We want directory to override masq, for example.
+                */
 
-               if ( (!strcasecmp(type, "masqdomain"))
-                  && (!strcasecmp(&fqdn[strlen(fqdn)-strlen(host)], host)))
-                       return(hostalias_masq);
+               if ( (!strcasecmp(type, "masqdomain")) && (!strcasecmp(fqdn, host))) {
+                       found = hostalias_masq;
+               }
+               if ( (!strcasecmp(type, "localhost")) && (!strcasecmp(fqdn, host))) {
+                       found = hostalias_localhost;
+               }
+               if ( (!strcasecmp(type, "directory")) && (!strcasecmp(fqdn, host))) {
+                       found = hostalias_directory;
+               }
 
+               if (found) return(found);
        }
 
        return(hostalias_nomatch);
@@ -136,22 +358,54 @@ int fuzzy_match(struct ctdluser *us, char *matchstring) {
 /*
  * Unfold a multi-line field into a single line, removing multi-whitespaces
  */
-void unfold_rfc822_field(char *field) {
-       int i;
+void unfold_rfc822_field(char **field, char **FieldEnd) 
+{
        int quote = 0;
+       char *pField = *field;
+       char *sField;
+       char *pFieldEnd = *FieldEnd;
+
+       while (isspace(*pField))
+               pField++;
+       /* remove leading/trailing whitespace */
+       ;
 
-       striplt(field);         /* remove leading/trailing whitespace */
+       while (isspace(*pFieldEnd))
+               pFieldEnd --;
 
+       *FieldEnd = pFieldEnd;
        /* convert non-space whitespace to spaces, and remove double blanks */
-       for (i=0; i<strlen(field); ++i) {
-               if (field[i]=='\"') quote = 1 - quote;
-               if (!quote) {
-                       if (isspace(field[i])) field[i] = ' ';
-                       while (isspace(field[i]) && isspace(field[i+1])) {
-                               strcpy(&field[i+1], &field[i+2]);
+       for (sField = *field = pField; 
+            sField < pFieldEnd; 
+            pField++, sField++)
+       {
+               if ((*sField=='\r') || (*sField=='\n')) {
+                   sField++;
+                   if  (*sField == '\n')
+                       sField++;
+                   *pField = *sField;
+               }
+               else {
+                       if (*sField=='\"') quote = 1 - quote;
+                       if (!quote) {
+                               if (isspace(*sField))
+                               {
+                                       *pField = ' ';
+                                       pField++;
+                                       sField++;
+                                       
+                                       while ((sField < pFieldEnd) && 
+                                              isspace(*sField))
+                                               sField++;
+                                       *pField = *sField;
+                               }
+                               else *pField = *sField;
                        }
+                       else *pField = *sField;
                }
        }
+       *pField = '\0';
+       *FieldEnd = pField - 1;
 }
 
 
@@ -292,11 +546,12 @@ void process_rfc822_addr(const char *rfc822, char *user, char *node, char *name)
  * structure, implying that the source field should be removed from the
  * message text.
  */
-int convert_field(struct CtdlMessage *msg, int beg, int end) {
-       char *rfc822;
-       char *key, *value;
+int convert_field(struct CtdlMessage *msg, const char *beg, const char *end) {
+       char *key, *value, *valueend;
+       long len;
+       const char *pos;
        int i;
-       int colonpos = (-1);
+       const char *colonpos = NULL;
        int processed = 0;
        char buf[SIZ];
        char user[1024];
@@ -304,19 +559,25 @@ int convert_field(struct CtdlMessage *msg, int beg, int end) {
        char name[1024];
        char addr[1024];
        time_t parsed_date;
+       long valuelen;
 
-       rfc822 = msg->cm_fields['M'];   /* M field contains rfc822 text */
-       for (i = end; i >= beg; --i) {
-               if (rfc822[i] == ':') colonpos = i;
+       for (pos = end; pos >= beg; pos--) {
+               if (*pos == ':') colonpos = pos;
        }
 
-       if (colonpos < 0) return(0);    /* no colon? not a valid header line */
+       if (colonpos == NULL) return(0);        /* no colon? not a valid header line */
 
-       key = malloc((end - beg) + 2);
-       safestrncpy(key, &rfc822[beg], (end-beg)+1);
-       key[colonpos - beg] = 0;
+       len = end - beg;
+       key = malloc(len + 2);
+       memcpy(key, beg, len + 1);
+       key[len] = '\0';
+       valueend = key + len;
+       * ( key + (colonpos - beg) ) = '\0';
        value = &key[(colonpos - beg) + 1];
-       unfold_rfc822_field(value);
+/*     printf("Header: [%s]\nValue: [%s]\n", key, value); */
+       unfold_rfc822_field(&value, &valueend);
+       valuelen = valueend - value + 1;
+/*     printf("UnfoldedValue: [%s]\n", value); */
 
        /*
         * Here's the big rfc822-to-citadel loop.
@@ -329,85 +590,98 @@ int convert_field(struct CtdlMessage *msg, int beg, int end) {
                parsed_date = parsedate(value);
                if (parsed_date < 0L) parsed_date = time(NULL);
                snprintf(buf, sizeof buf, "%ld", (long)parsed_date );
-               if (msg->cm_fields['T'] == NULL)
-                       msg->cm_fields['T'] = strdup(buf);
+               if (msg->cm_fields[eTimestamp] == NULL)
+                       msg->cm_fields[eTimestamp] = strdup(buf);
                processed = 1;
        }
 
        else if (!strcasecmp(key, "From")) {
                process_rfc822_addr(value, user, node, name);
-               CtdlLogPrintf(CTDL_DEBUG, "Converted to <%s@%s> (%s)\n", user, node, name);
+               syslog(LOG_DEBUG, "Converted to <%s@%s> (%s)\n", user, node, name);
                snprintf(addr, sizeof addr, "%s@%s", user, node);
-               if (msg->cm_fields['A'] == NULL)
-                       msg->cm_fields['A'] = strdup(name);
-               processed = 1;
-               if (msg->cm_fields['F'] == NULL)
-                       msg->cm_fields['F'] = strdup(addr);
+               if (msg->cm_fields[eAuthor] == NULL)
+                       msg->cm_fields[eAuthor] = strdup(name);
+               if (msg->cm_fields[erFc822Addr] == NULL)
+                       msg->cm_fields[erFc822Addr] = strdup(addr);
                processed = 1;
        }
 
        else if (!strcasecmp(key, "Subject")) {
-               if (msg->cm_fields['U'] == NULL)
-                       msg->cm_fields['U'] = strdup(value);
+               if (msg->cm_fields[eMsgSubject] == NULL)
+                       msg->cm_fields[eMsgSubject] = strndup(value, valuelen);
+               processed = 1;
+       }
+
+       else if (!strcasecmp(key, "List-ID")) {
+               if (msg->cm_fields[eListID] == NULL)
+                       msg->cm_fields[eListID] = strndup(value, valuelen);
                processed = 1;
        }
 
        else if (!strcasecmp(key, "To")) {
-               if (msg->cm_fields['R'] == NULL)
-                       msg->cm_fields['R'] = strdup(value);
+               if (msg->cm_fields[eRecipient] == NULL)
+                       msg->cm_fields[eRecipient] = strndup(value, valuelen);
                processed = 1;
        }
 
        else if (!strcasecmp(key, "CC")) {
-               if (msg->cm_fields['Y'] == NULL)
-                       msg->cm_fields['Y'] = strdup(value);
+               if (msg->cm_fields[eCarbonCopY] == NULL)
+                       msg->cm_fields[eCarbonCopY] = strndup(value, valuelen);
                processed = 1;
        }
 
        else if (!strcasecmp(key, "Message-ID")) {
-               if (msg->cm_fields['I'] != NULL) {
-                       CtdlLogPrintf(CTDL_WARNING, "duplicate message id\n");
+               if (msg->cm_fields[emessageId] != NULL) {
+                       syslog(LOG_WARNING, "duplicate message id\n");
                }
 
-               if (msg->cm_fields['I'] == NULL) {
-                       msg->cm_fields['I'] = strdup(value);
+               if (msg->cm_fields[emessageId] == NULL) {
+                       msg->cm_fields[emessageId] = strndup(value, valuelen);
 
                        /* Strip angle brackets */
-                       while (haschar(msg->cm_fields['I'], '<') > 0) {
-                               strcpy(&msg->cm_fields['I'][0],
-                                       &msg->cm_fields['I'][1]);
+                       while (haschar(msg->cm_fields[emessageId], '<') > 0) {
+                               strcpy(&msg->cm_fields[emessageId][0],
+                                       &msg->cm_fields[emessageId][1]);
                        }
-                       for (i = 0; i<strlen(msg->cm_fields['I']); ++i)
-                               if (msg->cm_fields['I'][i] == '>')
-                                       msg->cm_fields['I'][i] = 0;
+                       for (i = 0; i<strlen(msg->cm_fields[emessageId]); ++i)
+                               if (msg->cm_fields[emessageId][i] == '>')
+                                       msg->cm_fields[emessageId][i] = 0;
                }
 
                processed = 1;
        }
 
        else if (!strcasecmp(key, "Return-Path")) {
-               if (msg->cm_fields['P'] == NULL)
-                       msg->cm_fields['P'] = strdup(value);
+               if (msg->cm_fields[eMessagePath] == NULL)
+                       msg->cm_fields[eMessagePath] = strndup(value, valuelen);
                processed = 1;
        }
 
        else if (!strcasecmp(key, "Envelope-To")) {
-               if (msg->cm_fields['V'] == NULL)
-                       msg->cm_fields['V'] = strdup(value);
+               if (msg->cm_fields[eenVelopeTo] == NULL)
+                       msg->cm_fields[eenVelopeTo] = strndup(value, valuelen);
                processed = 1;
        }
 
        else if (!strcasecmp(key, "References")) {
-               if (msg->cm_fields['W'] != NULL) {
-                       free(msg->cm_fields['W']);
+               if (msg->cm_fields[eWeferences] != NULL) {
+                       free(msg->cm_fields[eWeferences]);
                }
-               msg->cm_fields['W'] = strdup(value);
+               msg->cm_fields[eWeferences] = strndup(value, valuelen);
+               processed = 1;
+       }
+
+       else if (!strcasecmp(key, "Reply-To")) {
+               if (msg->cm_fields[eReplyTo] != NULL) {
+                       free(msg->cm_fields[eReplyTo]);
+               }
+               msg->cm_fields[eReplyTo] = strndup(value, valuelen);
                processed = 1;
        }
 
        else if (!strcasecmp(key, "In-reply-to")) {
-               if (msg->cm_fields['W'] == NULL) {              /* References: supersedes In-reply-to: */
-                       msg->cm_fields['W'] = strdup(value);
+               if (msg->cm_fields[eWeferences] == NULL) {              /* References: supersedes In-reply-to: */
+                       msg->cm_fields[eWeferences] = strndup(value, valuelen);
                }
                processed = 1;
        }
@@ -463,12 +737,21 @@ void convert_references_to_wefewences(char *str) {
  * again.
  */
 struct CtdlMessage *convert_internet_message(char *rfc822) {
+       StrBuf *RFCBuf = NewStrBufPlain(rfc822, -1);
+       free (rfc822);
+       return convert_internet_message_buf(&RFCBuf);
+}
 
+
+
+struct CtdlMessage *convert_internet_message_buf(StrBuf **rfc822)
+{
        struct CtdlMessage *msg;
-       int pos, beg, end, msglen;
-       int done;
+       const char *pos, *beg, *end, *totalend;
+       int done, alldone = 0;
        char buf[SIZ];
        int converted;
+       StrBuf *OtherHeaders;
 
        msg = malloc(sizeof(struct CtdlMessage));
        if (msg == NULL) return msg;
@@ -477,36 +760,39 @@ struct CtdlMessage *convert_internet_message(char *rfc822) {
        msg->cm_magic = CTDLMESSAGE_MAGIC;      /* self check */
        msg->cm_anon_type = 0;                  /* never anonymous */
        msg->cm_format_type = FMT_RFC822;       /* internet message */
-       msg->cm_fields['M'] = rfc822;
 
-       pos = 0;
+       pos = ChrPtr(*rfc822);
+       totalend = pos + StrLength(*rfc822);
        done = 0;
+       OtherHeaders = NewStrBufPlain(NULL, StrLength(*rfc822));
 
-       while (!done) {
+       while (!alldone) {
 
                /* Locate beginning and end of field, keeping in mind that
                 * some fields might be multiline
                 */
-               beg = pos;
-               end = (-1);
+               end = beg = pos;
 
-               msglen = strlen(rfc822);        
-               while ( (end < 0) && (done == 0) ) {
+               while ((end < totalend) && 
+                      (end == beg) && 
+                      (done == 0) ) 
+               {
 
-                       if ((rfc822[pos]=='\n')
-                          && (!isspace(rfc822[pos+1]))) {
+                       if ( (*pos=='\n') && ((*(pos+1))!=0x20) && ((*(pos+1))!=0x09) )
+                       {
                                end = pos;
                        }
 
                        /* done with headers? */
-                       if (   (rfc822[pos]=='\n')
-                          && ( (rfc822[pos+1]=='\n')
-                             ||(rfc822[pos+1]=='\r')) ) {
-                               end = pos;
-                               done = 1;
+                       if ((*pos=='\n') &&
+                           ( (*(pos+1)=='\n') ||
+                             (*(pos+1)=='\r')) ) 
+                       {
+                               alldone = 1;
                        }
 
-                       if (pos >= (msglen-1) ) {
+                       if (pos >= (totalend - 1) )
+                       {
                                end = pos;
                                done = 1;
                        }
@@ -519,28 +805,34 @@ struct CtdlMessage *convert_internet_message(char *rfc822) {
                converted = convert_field(msg, beg, end);
 
                /* Strip the field out of the RFC822 header if we used it */
-               if (converted) {
-                       strcpy(&rfc822[beg], &rfc822[pos]);
-                       pos = beg;
+               if (!converted) {
+                       StrBufAppendBufPlain(OtherHeaders, beg, end - beg, 0);
+                       StrBufAppendBufPlain(OtherHeaders, HKEY("\n"), 0);
                }
 
                /* If we've hit the end of the message, bail out */
-               if (pos > strlen(rfc822)) done = 1;
+               if (pos >= totalend)
+                       alldone = 1;
        }
+       StrBufAppendBufPlain(OtherHeaders, HKEY("\n"), 0);
+       if (pos < totalend)
+               StrBufAppendBufPlain(OtherHeaders, pos, totalend - pos, 0);
+       FreeStrBuf(rfc822);
+       msg->cm_fields[eMesageText] = SmashStrBuf(&OtherHeaders);
 
        /* Follow-up sanity checks... */
 
        /* If there's no timestamp on this message, set it to now. */
-       if (msg->cm_fields['T'] == NULL) {
+       if (msg->cm_fields[eTimestamp] == NULL) {
                snprintf(buf, sizeof buf, "%ld", (long)time(NULL));
-               msg->cm_fields['T'] = strdup(buf);
+               msg->cm_fields[eTimestamp] = strdup(buf);
        }
 
        /* If a W (references, or rather, Wefewences) field is present, we
         * have to convert it from RFC822 format to Citadel format.
         */
-       if (msg->cm_fields['W'] != NULL) {
-               convert_references_to_wefewences(msg->cm_fields['W']);
+       if (msg->cm_fields[eWeferences] != NULL) {
+               convert_references_to_wefewences(msg->cm_fields[eWeferences]);
        }
 
        return msg;
@@ -555,11 +847,11 @@ struct CtdlMessage *convert_internet_message(char *rfc822) {
  * The caller is responsible for freeing the returned buffer.  If the requested
  * field is not present, or anything else goes wrong, it returns NULL.
  */
-char *rfc822_fetch_field(char *rfc822, char *fieldname) {
+char *rfc822_fetch_field(const char *rfc822, const char *fieldname) {
        char *fieldbuf = NULL;
-       char *end_of_headers;
-       char *field_start;
-       char *ptr;
+       const char *end_of_headers;
+       const char *field_start;
+       const char *ptr;
        char *cont;
        char fieldhdr[SIZ];
 
@@ -570,13 +862,13 @@ char *rfc822_fetch_field(char *rfc822, char *fieldname) {
        snprintf(fieldhdr, sizeof fieldhdr, "%s:", fieldname);
 
        /* Locate the end of the headers, so we don't run past that point */
-       end_of_headers = bmstrcasestr(rfc822, "\n\r\n");
+       end_of_headers = cbmstrcasestr(rfc822, "\n\r\n");
        if (end_of_headers == NULL) {
-               end_of_headers = bmstrcasestr(rfc822, "\n\n");
+               end_of_headers = cbmstrcasestr(rfc822, "\n\n");
        }
        if (end_of_headers == NULL) return (NULL);
 
-       field_start = bmstrcasestr(rfc822, fieldhdr);
+       field_start = cbmstrcasestr(rfc822, fieldhdr);
        if (field_start == NULL) return(NULL);
        if (field_start > end_of_headers) return(NULL);
 
@@ -584,11 +876,11 @@ char *rfc822_fetch_field(char *rfc822, char *fieldname) {
        strcpy(fieldbuf, "");
 
        ptr = field_start;
-       ptr = memreadline(ptr, fieldbuf, SIZ-strlen(fieldbuf) );
+       ptr = cmemreadline(ptr, fieldbuf, SIZ-strlen(fieldbuf) );
        while ( (isspace(ptr[0])) && (ptr < end_of_headers) ) {
                strcat(fieldbuf, " ");
                cont = &fieldbuf[strlen(fieldbuf)];
-               ptr = memreadline(ptr, cont, SIZ-strlen(fieldbuf) );
+               ptr = cmemreadline(ptr, cont, SIZ-strlen(fieldbuf) );
                striplt(cont);
        }
 
@@ -619,7 +911,7 @@ void directory_key(char *key, char *addr) {
        }
        key[keylen++] = 0;
 
-       CtdlLogPrintf(CTDL_DEBUG, "Directory key is <%s>\n", key);
+       syslog(LOG_DEBUG, "Directory key is <%s>\n", key);
 }
 
 
@@ -659,17 +951,15 @@ void CtdlDirectoryInit(void) {
 /*
  * Add an Internet e-mail address to the directory for a user
  */
-void CtdlDirectoryAddUser(char *internet_addr, char *citadel_addr) {
+int CtdlDirectoryAddUser(char *internet_addr, char *citadel_addr) {
        char key[SIZ];
 
-       CtdlLogPrintf(CTDL_DEBUG, "Dir: %s --> %s\n",
-               internet_addr, citadel_addr);
-       if (IsDirectory(internet_addr, 0) == 0) return;
-
+       if (IsDirectory(internet_addr, 0) == 0) 
+               return 0;
+       syslog(LOG_DEBUG, "Create directory entry: %s --> %s\n", internet_addr, citadel_addr);
        directory_key(key, internet_addr);
-
-       cdb_store(CDB_DIRECTORY, key, strlen(key),
-               citadel_addr, strlen(citadel_addr)+1 );
+       cdb_store(CDB_DIRECTORY, key, strlen(key), citadel_addr, strlen(citadel_addr)+1 );
+       return 1;
 }
 
 
@@ -679,11 +969,12 @@ void CtdlDirectoryAddUser(char *internet_addr, char *citadel_addr) {
  * (NOTE: we don't actually use or need the citadel_addr variable; it's merely
  * here because the callback API expects to be able to send it.)
  */
-void CtdlDirectoryDelUser(char *internet_addr, char *citadel_addr) {
+int CtdlDirectoryDelUser(char *internet_addr, char *citadel_addr) {
        char key[SIZ];
 
+       syslog(LOG_DEBUG, "Delete directory entry: %s --> %s\n", internet_addr, citadel_addr);
        directory_key(key, internet_addr);
-       cdb_delete(CDB_DIRECTORY, key, strlen(key) );
+       return cdb_delete(CDB_DIRECTORY, key, strlen(key) ) == 0;
 }
 
 
@@ -727,20 +1018,20 @@ char *harvest_collected_addresses(struct CtdlMessage *msg) {
        char user[256], node[256], name[256];
        int is_harvestable;
        int i, j, h;
-       int field = 0;
+       eMsgField field = 0;
 
        if (msg == NULL) return(NULL);
 
        is_harvestable = 1;
        strcpy(addr, "");       
-       if (msg->cm_fields['A'] != NULL) {
-               strcat(addr, msg->cm_fields['A']);
+       if (msg->cm_fields[eAuthor] != NULL) {
+               strcat(addr, msg->cm_fields[eAuthor]);
        }
-       if (msg->cm_fields['F'] != NULL) {
+       if (msg->cm_fields[erFc822Addr] != NULL) {
                strcat(addr, " <");
-               strcat(addr, msg->cm_fields['F']);
+               strcat(addr, msg->cm_fields[erFc822Addr]);
                strcat(addr, ">");
-               if (IsDirectory(msg->cm_fields['F'], 0)) {
+               if (IsDirectory(msg->cm_fields[erFc822Addr], 0)) {
                        is_harvestable = 0;
                }
        }
@@ -756,12 +1047,14 @@ char *harvest_collected_addresses(struct CtdlMessage *msg) {
 
        /* Scan both the R (To) and Y (CC) fields */
        for (i = 0; i < 2; ++i) {
-               if (i == 0) field = 'R' ;
-               if (i == 1) field = 'Y' ;
+               if (i == 0) field = eRecipient;
+               if (i == 1) field = eCarbonCopY;
 
                if (msg->cm_fields[field] != NULL) {
                        for (j=0; j<num_tokens(msg->cm_fields[field], ','); ++j) {
                                extract_token(addr, msg->cm_fields[field], j, ',', sizeof addr);
+                               if (strstr(addr, "=?") != NULL)
+                                       utf8ify_rfc822_string(addr);
                                process_rfc822_addr(addr, user, node, name);
                                h = CtdlHostAlias(node);
                                if ( (h != hostalias_localhost) && (h != hostalias_directory) ) {