]> code.citadel.org Git - citadel.git/blobdiff - libcitadel/lib/stringbuf.c
* move utf8 handling stuff into strbuf, so we can be more exact about our buffer...
[citadel.git] / libcitadel / lib / stringbuf.c
index 6d429a6898f84e0eb8ada8eadacdf5f41d40340a..4bec25f3dde47740c3e4d931ee054b3c6c4331a3 100644 (file)
@@ -1,3 +1,4 @@
+#include "../sysdep.h"
 #include <ctype.h>
 #include <errno.h>
 #include <string.h>
@@ -6,10 +7,25 @@
 #include <stdio.h>
 #include <sys/select.h>
 #include <fcntl.h>
+#include <sys/types.h>
 #define SHOW_ME_VAPPEND_PRINTF
 #include <stdarg.h>
 #include "libcitadel.h"
 
+#ifdef HAVE_ICONV
+#include <iconv.h>
+#endif
+
+#if HAVE_BACKTRACE
+#include <execinfo.h>
+#endif
+
+#ifdef HAVE_ZLIB
+#include <zlib.h>
+int ZEXPORT compress_gzip(Bytef * dest, size_t * destLen,
+                          const Bytef * source, uLong sourceLen, int level);
+#endif
+int BaseStrBufSize = 64;
 
 /**
  * Private Structure for the Stringbuffer
 struct StrBuf {
        char *buf;         /**< the pointer to the dynamic buffer */
        long BufSize;      /**< how many spcae do we optain */
-       long BufUsed;      /**< Number of Chars used excluding the trailing \0 */
+       long BufUsed;      /**< StNumber of Chars used excluding the trailing \0 */
        int ConstBuf;      /**< are we just a wrapper arround a static buffer and musn't we be changed? */
+#ifdef SIZE_DEBUG
+       long nIncreases;
+       char bt [SIZ];
+       char bt_lastinc [SIZ];
+#endif
 };
 
+#ifdef SIZE_DEBUG
+#if HAVE_BACKTRACE
+static void StrBufBacktrace(StrBuf *Buf, int which)
+{
+       int n;
+       char *pstart, *pch;
+       void *stack_frames[50];
+       size_t size, i;
+       char **strings;
+
+       if (which)
+               pstart = pch = Buf->bt;
+       else
+               pstart = pch = Buf->bt_lastinc;
+       size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
+       strings = backtrace_symbols(stack_frames, size);
+       for (i = 0; i < size; i++) {
+               if (strings != NULL)
+                       n = snprintf(pch, SIZ - (pch - pstart), "%s\\n", strings[i]);
+               else
+                       n = snprintf(pch, SIZ - (pch - pstart), "%p\\n", stack_frames[i]);
+               pch += n;
+       }
+       free(strings);
+
+
+}
+#endif
+#endif
 
 /** 
  * \Brief Cast operator to Plain String 
+ * Note: if the buffer is altered by StrBuf operations, this pointer may become 
+ *  invalid. So don't lean on it after altering the buffer!
+ *  Since this operation is considered cheap, rather call it often than risking
+ *  your pointer to become invalid!
  * \param Str the string we want to get the c-string representation for
  * \returns the Pointer to the Content. Don't mess with it!
  */
@@ -59,7 +113,7 @@ static int IncreaseBuf(StrBuf *Buf, int KeepOriginal, int DestSize)
                return -1;
                
        if (DestSize > 0)
-               while (NewSize < DestSize)
+               while (NewSize <= DestSize)
                        NewSize *= 2;
 
        NewBuf= (char*) malloc(NewSize);
@@ -74,10 +128,52 @@ static int IncreaseBuf(StrBuf *Buf, int KeepOriginal, int DestSize)
        }
        free (Buf->buf);
        Buf->buf = NewBuf;
-       Buf->BufSize *= 2;
+       Buf->BufSize = NewSize;
+#ifdef SIZE_DEBUG
+       Buf->nIncreases++;
+#if HAVE_BACKTRACE
+       StrBufBacktrace(Buf, 1);
+#endif
+#endif
        return Buf->BufSize;
 }
 
+/**
+ * \brief shrink an _EMPTY_ buffer if its Buffer superseeds threshhold to NewSize. Buffercontent is thoroughly ignored and flushed.
+ * \param Buf Buffer to shrink (has to be empty)
+ * \param ThreshHold if the buffer is bigger then this, its readjusted
+ * \param NewSize if we Shrink it, how big are we going to be afterwards?
+ */
+void ReAdjustEmptyBuf(StrBuf *Buf, long ThreshHold, long NewSize)
+{
+       if (Buf->BufUsed > ThreshHold) {
+               free(Buf->buf);
+               Buf->buf = (char*) malloc(NewSize);
+               Buf->BufUsed = 0;
+               Buf->BufSize = NewSize;
+       }
+}
+
+/**
+ * \brief shrink long term buffers to their real size so they don't waste memory
+ * \param Buf buffer to shrink
+ * \param Force if not set, will just executed if the buffer is much to big; set for lifetime strings
+ * \returns physical size of the buffer
+ */
+long StrBufShrinkToFit(StrBuf *Buf, int Force)
+{
+       if (Force || 
+           (Buf->BufUsed + (Buf->BufUsed / 3) > Buf->BufSize))
+       {
+               char *TmpBuf = (char*) malloc(Buf->BufUsed + 1);
+               memcpy (TmpBuf, Buf->buf, Buf->BufUsed + 1);
+               Buf->BufSize = Buf->BufUsed + 1;
+               free(Buf->buf);
+               Buf->buf = TmpBuf;
+       }
+       return Buf->BufUsed;
+}
+
 /**
  * Allocate a new buffer with default buffer size
  * \returns the new stringbuffer
@@ -87,11 +183,19 @@ StrBuf* NewStrBuf(void)
        StrBuf *NewBuf;
 
        NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
-       NewBuf->buf = (char*) malloc(SIZ);
+       NewBuf->buf = (char*) malloc(BaseStrBufSize);
        NewBuf->buf[0] = '\0';
-       NewBuf->BufSize = SIZ;
+       NewBuf->BufSize = BaseStrBufSize;
        NewBuf->BufUsed = 0;
        NewBuf->ConstBuf = 0;
+#ifdef SIZE_DEBUG
+       NewBuf->nIncreases = 0;
+       NewBuf->bt[0] = '\0';
+       NewBuf->bt_lastinc[0] = '\0';
+#if HAVE_BACKTRACE
+       StrBufBacktrace(NewBuf, 0);
+#endif
+#endif
        return NewBuf;
 }
 
@@ -113,6 +217,14 @@ StrBuf* NewStrBufDup(const StrBuf *CopyMe)
        NewBuf->BufUsed = CopyMe->BufUsed;
        NewBuf->BufSize = CopyMe->BufSize;
        NewBuf->ConstBuf = 0;
+#ifdef SIZE_DEBUG
+       NewBuf->nIncreases = 0;
+       NewBuf->bt[0] = '\0';
+       NewBuf->bt_lastinc[0] = '\0';
+#if HAVE_BACKTRACE
+       StrBufBacktrace(NewBuf, 0);
+#endif
+#endif
        return NewBuf;
 }
 
@@ -127,7 +239,7 @@ StrBuf* NewStrBufDup(const StrBuf *CopyMe)
 StrBuf* NewStrBufPlain(const char* ptr, int nChars)
 {
        StrBuf *NewBuf;
-       size_t Siz = SIZ;
+       size_t Siz = BaseStrBufSize;
        size_t CopySize;
 
        NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
@@ -151,6 +263,14 @@ StrBuf* NewStrBufPlain(const char* ptr, int nChars)
                NewBuf->BufUsed = 0;
        }
        NewBuf->ConstBuf = 0;
+#ifdef SIZE_DEBUG
+       NewBuf->nIncreases = 0;
+       NewBuf->bt[0] = '\0';
+       NewBuf->bt_lastinc[0] = '\0';
+#if HAVE_BACKTRACE
+       StrBufBacktrace(NewBuf, 0);
+#endif
+#endif
        return NewBuf;
 }
 
@@ -197,6 +317,11 @@ StrBuf* _NewConstStrBuf(const char* StringConstant, size_t SizeOfStrConstant)
        NewBuf->BufSize = SizeOfStrConstant;
        NewBuf->BufUsed = SizeOfStrConstant;
        NewBuf->ConstBuf = 1;
+#ifdef SIZE_DEBUG
+       NewBuf->nIncreases = 0;
+       NewBuf->bt[0] = '\0';
+       NewBuf->bt_lastinc[0] = '\0';
+#endif
        return NewBuf;
 }
 
@@ -207,6 +332,8 @@ StrBuf* _NewConstStrBuf(const char* StringConstant, size_t SizeOfStrConstant)
  */
 int FlushStrBuf(StrBuf *buf)
 {
+       if (buf == NULL)
+               return -1;
        if (buf->ConstBuf)
                return -1;       
        buf->buf[0] ='\0';
@@ -214,6 +341,26 @@ int FlushStrBuf(StrBuf *buf)
        return 0;
 }
 
+/**
+ * \brief wipe the content of a Buf thoroughly (overwrite it -> expensive); keep its struct
+ * \param buf Buffer to wipe
+ */
+int FLUSHStrBuf(StrBuf *buf)
+{
+       if (buf == NULL)
+               return -1;
+       if (buf->ConstBuf)
+               return -1;
+       if (buf->BufUsed > 0) {
+               memset(buf->buf, 0, buf->BufUsed);
+               buf->BufUsed = 0;
+       }
+       return 0;
+}
+
+#ifdef SIZE_DEBUG
+int hFreeDbglog = -1;
+#endif
 /**
  * \brief Release a Buffer
  * Its a double pointer, so it can NULL your pointer
@@ -224,6 +371,35 @@ void FreeStrBuf (StrBuf **FreeMe)
 {
        if (*FreeMe == NULL)
                return;
+#ifdef SIZE_DEBUG
+       if (hFreeDbglog == -1){
+               pid_t pid = getpid();
+               char path [SIZ];
+               snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
+               hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
+       }
+       if ((*FreeMe)->nIncreases > 0)
+       {
+               char buf[SIZ * 3];
+               long n;
+               n = snprintf(buf, SIZ * 3, "+|%ld|%ld|%ld|%s|%s|\n",
+                            (*FreeMe)->nIncreases,
+                            (*FreeMe)->BufUsed,
+                            (*FreeMe)->BufSize,
+                            (*FreeMe)->bt,
+                            (*FreeMe)->bt_lastinc);
+               n = write(hFreeDbglog, buf, n);
+       }
+       else
+       {
+               char buf[128];
+               long n;
+               n = snprintf(buf, 128, "_|0|%ld%ld|\n",
+                            (*FreeMe)->BufUsed,
+                            (*FreeMe)->BufSize);
+               n = write(hFreeDbglog, buf, n);
+       }
+#endif
        if (!(*FreeMe)->ConstBuf) 
                free((*FreeMe)->buf);
        free(*FreeMe);
@@ -240,6 +416,35 @@ void HFreeStrBuf (void *VFreeMe)
        StrBuf *FreeMe = (StrBuf*)VFreeMe;
        if (FreeMe == NULL)
                return;
+#ifdef SIZE_DEBUG
+       if (hFreeDbglog == -1){
+               pid_t pid = getpid();
+               char path [SIZ];
+               snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
+               hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
+       }
+       if (FreeMe->nIncreases > 0)
+       {
+               char buf[SIZ * 3];
+               long n;
+               n = snprintf(buf, SIZ * 3, "+|%ld|%ld|%ld|%s|%s|\n",
+                            FreeMe->nIncreases,
+                            FreeMe->BufUsed,
+                            FreeMe->BufSize,
+                            FreeMe->bt,
+                            FreeMe->bt_lastinc);
+               write(hFreeDbglog, buf, n);
+       }
+       else
+       {
+               char buf[128];
+               long n;
+               n = snprintf(buf, 128, "_|%ld|%ld%ld|\n",
+                            FreeMe->nIncreases,
+                            FreeMe->BufUsed,
+                            FreeMe->BufSize);
+       }
+#endif
        if (!FreeMe->ConstBuf) 
                free(FreeMe->buf);
        free(FreeMe);
@@ -250,6 +455,8 @@ void HFreeStrBuf (void *VFreeMe)
  */
 long StrTol(const StrBuf *Buf)
 {
+       if (Buf == NULL)
+               return 0;
        if(Buf->BufUsed > 0)
                return atol(Buf->buf);
        else
@@ -261,12 +468,28 @@ long StrTol(const StrBuf *Buf)
  */
 int StrToi(const StrBuf *Buf)
 {
-       if(Buf->BufUsed > 0)
+       if (Buf == NULL)
+               return 0;
+       if (Buf->BufUsed > 0)
                return atoi(Buf->buf);
        else
                return 0;
 }
 
+/**
+ * \brief Checks to see if the string is a pure number 
+ */
+int StrBufIsNumber(const StrBuf *Buf) {
+  char * pEnd;
+  if (Buf == NULL) {
+       return 0;
+  }
+  strtoll(Buf->buf, &pEnd, 10);
+  if (pEnd == NULL && ((Buf->buf)-pEnd) != 0) {
+    return 1;
+  }
+  return 0;
+} 
 /**
  * \brief modifies a Single char of the Buf
  * You can point to it via char* or a zero-based integer
@@ -292,12 +515,12 @@ long StrBufPeek(StrBuf *Buf, const char* ptr, long nThChar, char PeekValue)
  * \param AppendBuf Buffer to copy at the end of our buffer
  * \param Offset Should we start copying from an offset?
  */
-void StrBufAppendBuf(StrBuf *Buf, const StrBuf *AppendBuf, size_t Offset)
+void StrBufAppendBuf(StrBuf *Buf, const StrBuf *AppendBuf, unsigned long Offset)
 {
-       if ((AppendBuf == NULL) || (Buf == NULL))
+       if ((AppendBuf == NULL) || (Buf == NULL) || (AppendBuf->buf == NULL))
                return;
 
-       if (Buf->BufSize - Offset < AppendBuf->BufUsed + Buf->BufUsed)
+       if (Buf->BufSize - Offset < AppendBuf->BufUsed + Buf->BufUsed + 1)
                IncreaseBuf(Buf, 
                            (Buf->BufUsed > 0), 
                            AppendBuf->BufUsed + Buf->BufUsed);
@@ -317,9 +540,10 @@ void StrBufAppendBuf(StrBuf *Buf, const StrBuf *AppendBuf, size_t Offset)
  * \param AppendSize number of bytes to copy; set to -1 if we should count it in advance
  * \param Offset Should we start copying from an offset?
  */
-void StrBufAppendBufPlain(StrBuf *Buf, const char *AppendBuf, long AppendSize, size_t Offset)
+void StrBufAppendBufPlain(StrBuf *Buf, const char *AppendBuf, long AppendSize, unsigned long Offset)
 {
        long aps;
+       long BufSizeRequired;
 
        if ((AppendBuf == NULL) || (Buf == NULL))
                return;
@@ -329,8 +553,9 @@ void StrBufAppendBufPlain(StrBuf *Buf, const char *AppendBuf, long AppendSize, s
        else
                aps = AppendSize - Offset;
 
-       if (Buf->BufSize < Buf->BufUsed + aps)
-               IncreaseBuf(Buf, (Buf->BufUsed > 0), Buf->BufUsed + aps);
+       BufSizeRequired = Buf->BufUsed + aps + 1;
+       if (Buf->BufSize <= BufSizeRequired)
+               IncreaseBuf(Buf, (Buf->BufUsed > 0), BufSizeRequired);
 
        memcpy(Buf->buf + Buf->BufUsed, 
               AppendBuf + Offset, 
@@ -408,7 +633,8 @@ void StrBufUrlescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
  * \param Source       source buffer; set to NULL if you just have a C-String
  * \param PlainIn       Plain-C string to append; set to NULL if unused
  * \param nbsp         If nonzero, spaces are converted to non-breaking spaces.
- * \param nolinebreaks if set, linebreaks are removed from the string.
+ * \param nolinebreaks if set to 1, linebreaks are removed from the string.
+ *                      if set to 2, linebreaks are replaced by &ltbr/&gt
  */
 long StrEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
 {
@@ -434,12 +660,12 @@ long StrEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int
                return -1;
 
        bptr = Target->buf + Target->BufUsed;
-       eptr = Target->buf + Target->BufSize - 6; /* our biggest unit to put in...  */
+       eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
 
        while (aptr < eiptr){
                if(bptr >= eptr) {
                        IncreaseBuf(Target, 1, -1);
-                       eptr = Target->buf + Target->BufSize - 6; 
+                       eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
                        bptr = Target->buf + Target->BufUsed;
                }
                if (*aptr == '<') {
@@ -457,7 +683,7 @@ long StrEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int
                        bptr += 5;
                        Target->BufUsed += 5;
                }
-               else if (*aptr == '\"') {
+               else if (*aptr == '"') {
                        memcpy(bptr, "&quot;", 6);
                        bptr += 6;
                        Target->BufUsed += 6;
@@ -487,10 +713,17 @@ long StrEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int
                        bptr += 6;
                        Target->BufUsed += 6;
                }
-               else if ((*aptr == '\n') && (nolinebreaks)) {
+               else if ((*aptr == '\n') && (nolinebreaks == 1)) {
                        *bptr='\0';     /* nothing */
                }
-               else if ((*aptr == '\r') && (nolinebreaks)) {
+               else if ((*aptr == '\n') && (nolinebreaks == 2)) {
+                       memcpy(bptr, "&lt;br/&gt;", 11);
+                       bptr += 11;
+                       Target->BufUsed += 11;
+               }
+
+
+               else if ((*aptr == '\r') && (nolinebreaks != 0)) {
                        *bptr='\0';     /* nothing */
                }
                else{
@@ -536,13 +769,13 @@ void StrMsgEscAppend(StrBuf *Target, StrBuf *Source, const char *PlainIn)
        if (len == 0) 
                return;
 
-       eptr = Target->buf + Target->BufSize - 6
+       eptr = Target->buf + Target->BufSize - 8
        tptr = Target->buf + Target->BufUsed;
        
        while (aptr < eiptr){
                if(tptr >= eptr) {
                        IncreaseBuf(Target, 1, -1);
-                       eptr = Target->buf + Target->BufSize - 6
+                       eptr = Target->buf + Target->BufSize - 8
                        tptr = Target->buf + Target->BufUsed;
                }
               
@@ -570,6 +803,70 @@ void StrMsgEscAppend(StrBuf *Target, StrBuf *Source, const char *PlainIn)
        *tptr = '\0';
 }
 
+/*
+ * \brief Append a string, escaping characters which have meaning in JavaScript strings .  
+ *
+ * \param Target       target buffer
+ * \param Source       source buffer; set to NULL if you just have a C-String
+ * \param PlainIn       Plain-C string to append; set to NULL if unused
+ */
+long StrECMAEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
+{
+       const char *aptr, *eiptr;
+       char *bptr, *eptr;
+       long len;
+
+       if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
+               return -1;
+
+       if (PlainIn != NULL) {
+               aptr = PlainIn;
+               len = strlen(PlainIn);
+               eiptr = aptr + len;
+       }
+       else {
+               aptr = Source->buf;
+               eiptr = aptr + Source->BufUsed;
+               len = Source->BufUsed;
+       }
+
+       if (len == 0) 
+               return -1;
+
+       bptr = Target->buf + Target->BufUsed;
+       eptr = Target->buf + Target->BufSize - 3; /* our biggest unit to put in...  */
+
+       while (aptr < eiptr){
+               if(bptr >= eptr) {
+                       IncreaseBuf(Target, 1, -1);
+                       eptr = Target->buf + Target->BufSize - 3; 
+                       bptr = Target->buf + Target->BufUsed;
+               }
+               else if (*aptr == '"') {
+                       *bptr = '\\';
+                       bptr ++;
+                       *bptr = '"';
+                       bptr ++;
+                       Target->BufUsed += 2;
+               } else if (*aptr == '\\') {
+                       *bptr = '\\';
+                       bptr ++;
+                       *bptr = '\\';
+                       bptr ++;
+                       Target->BufUsed += 2;
+               }
+               else{
+                       *bptr = *aptr;
+                       bptr++;
+                       Target->BufUsed ++;
+               }
+               aptr ++;
+       }
+       *bptr = '\0';
+       if ((bptr == eptr - 1 ) && !IsEmptyStr(aptr) )
+               return -1;
+       return Target->BufUsed;
+}
 
 /**
  * \brief extracts a substring from Source into dest
@@ -579,7 +876,7 @@ void StrMsgEscAppend(StrBuf *Target, StrBuf *Source, const char *PlainIn)
  * \param nChars number of chars to copy
  * \returns the number of chars copied; may be different from nChars due to the size of Source
  */
-int StrBufSub(StrBuf *dest, const StrBuf *Source, size_t Offset, size_t nChars)
+int StrBufSub(StrBuf *dest, const StrBuf *Source, unsigned long Offset, size_t nChars)
 {
        size_t NCharsRemain;
        if (Offset > Source->BufUsed)
@@ -589,7 +886,7 @@ int StrBufSub(StrBuf *dest, const StrBuf *Source, size_t Offset, size_t nChars)
        }
        if (Offset + nChars < Source->BufUsed)
        {
-               if (nChars > dest->BufSize)
+               if (nChars >= dest->BufSize)
                        IncreaseBuf(dest, 0, nChars + 1);
                memcpy(dest->buf, Source->buf + Offset, nChars);
                dest->BufUsed = nChars;
@@ -597,7 +894,7 @@ int StrBufSub(StrBuf *dest, const StrBuf *Source, size_t Offset, size_t nChars)
                return nChars;
        }
        NCharsRemain = Source->BufUsed - Offset;
-       if (NCharsRemain > dest->BufSize)
+       if (NCharsRemain  >= dest->BufSize)
                IncreaseBuf(dest, 0, NCharsRemain + 1);
        memcpy(dest->buf, Source->buf + Offset, NCharsRemain);
        dest->BufUsed = NCharsRemain;
@@ -629,6 +926,7 @@ void StrBufVAppendPrintf(StrBuf *Buf, const char *format, va_list ap)
                newused = Offset + nWritten;
                if (newused >= Buf->BufSize) {
                        IncreaseBuf(Buf, 1, newused);
+                       newused = Buf->BufSize + 1;
                }
                else {
                        Buf->BufUsed = Offset + nWritten;
@@ -661,6 +959,7 @@ void StrBufAppendPrintf(StrBuf *Buf, const char *format, ...)
                newused = Buf->BufUsed + nWritten;
                if (newused >= Buf->BufSize) {
                        IncreaseBuf(Buf, 1, newused);
+                       newused = Buf->BufSize + 1;
                }
                else {
                        Buf->BufUsed += nWritten;
@@ -685,9 +984,12 @@ void StrBufPrintf(StrBuf *Buf, const char *format, ...)
                va_start(arg_ptr, format);
                nWritten = vsnprintf(Buf->buf, Buf->BufSize, format, arg_ptr);
                va_end(arg_ptr);
-               Buf->BufUsed = nWritten ;
-               if (nWritten >= Buf->BufSize)
+               if (nWritten >= Buf->BufSize) {
                        IncreaseBuf(Buf, 0, 0);
+                       nWritten = Buf->BufSize + 1;
+                       continue;
+               }
+               Buf->BufUsed = nWritten ;
        }
 }
 
@@ -698,11 +1000,76 @@ void StrBufPrintf(StrBuf *Buf, const char *format, ...)
  * \param tok    Tokenizer char to count
  * \returns numbers of tokenizer chars found
  */
-inline int StrBufNum_tokens(const StrBuf *source, char tok)
+int StrBufNum_tokens(const StrBuf *source, char tok)
 {
+       if (source == NULL)
+               return 0;
        return num_tokens(source->buf, tok);
 }
 
+/*
+ * remove_token() - a tokenizer that kills, maims, and destroys
+ */
+/**
+ * \brief a string tokenizer
+ * \param Source StringBuffer to read into
+ * \param parmnum n'th parameter to remove
+ * \param separator tokenizer param
+ * \returns -1 if not found, else length of token.
+ */
+int StrBufRemove_token(StrBuf *Source, int parmnum, char separator)
+{
+       int ReducedBy;
+       char *d, *s;            /* dest, source */
+       int count = 0;
+
+       /* Find desired parameter */
+       d = Source->buf;
+       while (count < parmnum) {
+               /* End of string, bail! */
+               if (!*d) {
+                       d = NULL;
+                       break;
+               }
+               if (*d == separator) {
+                       count++;
+               }
+               d++;
+       }
+       if (!d) return 0;               /* Parameter not found */
+
+       /* Find next parameter */
+       s = d;
+       while (*s && *s != separator) {
+               s++;
+       }
+       if (*s == separator)
+               s++;
+       ReducedBy = d - s;
+
+       /* Hack and slash */
+       if (*s) {
+               memmove(d, s, Source->BufUsed - (s - Source->buf) + 1);
+               Source->BufUsed -= (ReducedBy + 1);
+       }
+       else if (d == Source->buf) {
+               *d = 0;
+               Source->BufUsed = 0;
+       }
+       else {
+               *--d = 0;
+               Source->BufUsed -= (ReducedBy + 1);
+       }
+       /*
+       while (*s) {
+               *d++ = *s++;
+       }
+       *d = 0;
+       */
+       return ReducedBy;
+}
+
+
 /**
  * \brief a string tokenizer
  * \param dest Destination StringBuffer
@@ -716,20 +1083,22 @@ int StrBufExtract_token(StrBuf *dest, const StrBuf *Source, int parmnum, char se
        const char *s, *e;              //* source * /
        int len = 0;                    //* running total length of extracted string * /
        int current_token = 0;          //* token currently being processed * /
+        
+       if (dest != NULL) {
+               dest->buf[0] = '\0';
+               dest->BufUsed = 0;
+       }
+       else
+               return(-1);
 
        if ((Source == NULL) || (Source->BufUsed ==0)) {
                return(-1);
        }
        s = Source->buf;
        e = s + Source->BufUsed;
-       if (dest == NULL) {
-               return(-1);
-       }
 
        //cit_backtrace();
        //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
-       dest->buf[0] = '\0';
-       dest->BufUsed = 0;
 
        while ((s<e) && !IsEmptyStr(s)) {
                if (*s == separator) {
@@ -761,6 +1130,9 @@ int StrBufExtract_token(StrBuf *dest, const StrBuf *Source, int parmnum, char se
 }
 
 
+
+
+
 /**
  * \brief a string tokenizer to fetch an integer
  * \param dest Destination StringBuffer
@@ -777,6 +1149,7 @@ int StrBufExtract_int(const StrBuf* Source, int parmnum, char separator)
        buf[0] = '\0';
        tmp.BufSize = 64;
        tmp.BufUsed = 0;
+       tmp.ConstBuf = 1;
        if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
                return(atoi(buf));
        else
@@ -799,6 +1172,7 @@ long StrBufExtract_long(const StrBuf* Source, int parmnum, char separator)
        buf[0] = '\0';
        tmp.BufSize = 64;
        tmp.BufUsed = 0;
+       tmp.ConstBuf = 1;
        if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
                return(atoi(buf));
        else
@@ -823,6 +1197,7 @@ unsigned long StrBufExtract_unsigned_long(const StrBuf* Source, int parmnum, cha
        buf[0] = '\0';
        tmp.BufSize = 64;
        tmp.BufUsed = 0;
+       tmp.ConstBuf = 1;
        if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0) {
                pnum = &buf[0];
                if (*pnum == '-')
@@ -836,67 +1211,268 @@ unsigned long StrBufExtract_unsigned_long(const StrBuf* Source, int parmnum, cha
 
 
 /**
- * \brief Read a line from socket
- * flushes and closes the FD on error
- * \param buf the buffer to get the input to
- * \param fd pointer to the filedescriptor to read
- * \param append Append to an existing string or replace?
- * \param Error strerror() on error 
- * \returns numbers of chars read
+ * \brief a string tokenizer
+ * \param dest Destination StringBuffer
+ * \param Source StringBuffer to read into
+ * \param pStart pointer to the end of the last token. Feed with NULL.
+ * \param separator tokenizer param
+ * \returns -1 if not found, else length of token.
  */
-int StrBufTCP_read_line(StrBuf *buf, int *fd, int append, const char **Error)
+int StrBufExtract_NextToken(StrBuf *dest, const StrBuf *Source, const char **pStart, char separator)
 {
-       int len, rlen, slen;
+       const char *s, *EndBuffer;      //* source * /
+       int len = 0;                    //* running total length of extracted string * /
+       int current_token = 0;          //* token currently being processed * /
+        
+       if (dest != NULL) {
+               dest->buf[0] = '\0';
+               dest->BufUsed = 0;
+       }
+       else
+               return(-1);
 
-       if (!append)
-               FlushStrBuf(buf);
+       if ((Source == NULL) || 
+           (Source->BufUsed ==0)) {
+               return(-1);
+       }
+       if (*pStart == NULL)
+               *pStart = Source->buf;
 
-       slen = len = buf->BufUsed;
-       while (1) {
-               rlen = read(*fd, &buf->buf[len], 1);
-               if (rlen < 1) {
-                       *Error = strerror(errno);
-                       
-                       close(*fd);
-                       *fd = -1;
-                       
-                       return -1;
+       EndBuffer = Source->buf + Source->BufUsed;
+
+       if ((*pStart < Source->buf) || 
+           (*pStart >  EndBuffer)) {
+               return (-1);
+       }
+
+
+       s = *pStart;
+
+       //cit_backtrace();
+       //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
+
+       while ((s<EndBuffer) && !IsEmptyStr(s)) {
+               if (*s == separator) {
+                       ++current_token;
                }
-               if (buf->buf[len] == '\n')
+               if (len >= dest->BufSize)
+                       if (!IncreaseBuf(dest, 1, -1)) {
+                               *pStart = EndBuffer + 1;
+                               break;
+                       }
+               if ( (current_token == 0) && 
+                    (*s != separator)) {
+                       dest->buf[len] = *s;
+                       ++len;
+               }
+               else if (current_token > 0) {
+                       *pStart = s;
                        break;
-               if (buf->buf[len] != '\r')
-                       len ++;
-               if (!(len < buf->BufSize)) {
-                       buf->BufUsed = len;
-                       buf->buf[len+1] = '\0';
-                       IncreaseBuf(buf, 1, -1);
                }
+               ++s;
        }
-       buf->BufUsed = len;
-       buf->buf[len] = '\0';
-       return len - slen;
+       *pStart = s;
+       (*pStart) ++;
+
+       dest->buf[len] = '\0';
+       dest->BufUsed = len;
+               //lprintf (CTDL_DEBUG,"test <!: %s\n", dest);
+       //lprintf (CTDL_DEBUG,"test <: %d; %s\n", len, dest);
+       return(len);
 }
 
+
 /**
- * \brief Read a line from socket
- * flushes and closes the FD on error
- * \param buf the buffer to get the input to
- * \param fd pointer to the filedescriptor to read
- * \param append Append to an existing string or replace?
- * \param Error strerror() on error 
- * \returns numbers of chars read
+ * \brief a string tokenizer
+ * \param dest Destination StringBuffer
+ * \param Source StringBuffer to read into
+ * \param pStart pointer to the end of the last token. Feed with NULL.
+ * \param separator tokenizer param
+ * \returns -1 if not found, else length of token.
  */
-int StrBufTCP_read_buffered_line(StrBuf *Line, 
-                                StrBuf *buf, 
-                                int *fd, 
-                                int timeout, 
-                                int selectresolution, 
-                                const char **Error)
+int StrBufSkip_NTokenS(const StrBuf *Source, const char **pStart, char separator, int nTokens)
 {
-       int len, rlen;
-       int nSuccessLess = 0;
-       fd_set rfds;
-       char *pch = NULL;
+       const char *s, *EndBuffer;      //* source * /
+       int len = 0;                    //* running total length of extracted string * /
+       int current_token = 0;          //* token currently being processed * /
+
+       if ((Source == NULL) || 
+           (Source->BufUsed ==0)) {
+               return(-1);
+       }
+       if (nTokens == 0)
+               return Source->BufUsed;
+
+       if (*pStart == NULL)
+               *pStart = Source->buf;
+
+       EndBuffer = Source->buf + Source->BufUsed;
+
+       if ((*pStart < Source->buf) || 
+           (*pStart >  EndBuffer)) {
+               return (-1);
+       }
+
+
+       s = *pStart;
+
+       //cit_backtrace();
+       //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
+
+       while ((s<EndBuffer) && !IsEmptyStr(s)) {
+               if (*s == separator) {
+                       ++current_token;
+               }
+               if (current_token >= nTokens) {
+                       break;
+               }
+               ++s;
+       }
+       *pStart = s;
+       (*pStart) ++;
+
+       return(len);
+}
+
+/**
+ * \brief a string tokenizer to fetch an integer
+ * \param dest Destination StringBuffer
+ * \param parmnum n'th parameter to extract
+ * \param separator tokenizer param
+ * \returns 0 if not found, else integer representation of the token
+ */
+int StrBufExtractNext_int(const StrBuf* Source, const char **pStart, char separator)
+{
+       StrBuf tmp;
+       char buf[64];
+       
+       tmp.buf = buf;
+       buf[0] = '\0';
+       tmp.BufSize = 64;
+       tmp.BufUsed = 0;
+       tmp.ConstBuf = 1;
+       if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
+               return(atoi(buf));
+       else
+               return 0;
+}
+
+/**
+ * \brief a string tokenizer to fetch a long integer
+ * \param dest Destination StringBuffer
+ * \param parmnum n'th parameter to extract
+ * \param separator tokenizer param
+ * \returns 0 if not found, else long integer representation of the token
+ */
+long StrBufExtractNext_long(const StrBuf* Source, const char **pStart, char separator)
+{
+       StrBuf tmp;
+       char buf[64];
+       
+       tmp.buf = buf;
+       buf[0] = '\0';
+       tmp.BufSize = 64;
+       tmp.BufUsed = 0;
+       tmp.ConstBuf = 1;
+       if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
+               return(atoi(buf));
+       else
+               return 0;
+}
+
+
+/**
+ * \brief a string tokenizer to fetch an unsigned long
+ * \param dest Destination StringBuffer
+ * \param parmnum n'th parameter to extract
+ * \param separator tokenizer param
+ * \returns 0 if not found, else unsigned long representation of the token
+ */
+unsigned long StrBufExtractNext_unsigned_long(const StrBuf* Source, const char **pStart, char separator)
+{
+       StrBuf tmp;
+       char buf[64];
+       char *pnum;
+       
+       tmp.buf = buf;
+       buf[0] = '\0';
+       tmp.BufSize = 64;
+       tmp.BufUsed = 0;
+       tmp.ConstBuf = 1;
+       if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0) {
+               pnum = &buf[0];
+               if (*pnum == '-')
+                       pnum ++;
+               return (unsigned long) atol(pnum);
+       }
+       else 
+               return 0;
+}
+
+
+
+/**
+ * \brief Read a line from socket
+ * flushes and closes the FD on error
+ * \param buf the buffer to get the input to
+ * \param fd pointer to the filedescriptor to read
+ * \param append Append to an existing string or replace?
+ * \param Error strerror() on error 
+ * \returns numbers of chars read
+ */
+int StrBufTCP_read_line(StrBuf *buf, int *fd, int append, const char **Error)
+{
+       int len, rlen, slen;
+
+       if (!append)
+               FlushStrBuf(buf);
+
+       slen = len = buf->BufUsed;
+       while (1) {
+               rlen = read(*fd, &buf->buf[len], 1);
+               if (rlen < 1) {
+                       *Error = strerror(errno);
+                       
+                       close(*fd);
+                       *fd = -1;
+                       
+                       return -1;
+               }
+               if (buf->buf[len] == '\n')
+                       break;
+               if (buf->buf[len] != '\r')
+                       len ++;
+               if (len >= buf->BufSize) {
+                       buf->BufUsed = len;
+                       buf->buf[len+1] = '\0';
+                       IncreaseBuf(buf, 1, -1);
+               }
+       }
+       buf->BufUsed = len;
+       buf->buf[len] = '\0';
+       return len - slen;
+}
+
+/**
+ * \brief Read a line from socket
+ * flushes and closes the FD on error
+ * \param buf the buffer to get the input to
+ * \param fd pointer to the filedescriptor to read
+ * \param append Append to an existing string or replace?
+ * \param Error strerror() on error 
+ * \returns numbers of chars read
+ */
+int StrBufTCP_read_buffered_line(StrBuf *Line, 
+                                StrBuf *buf, 
+                                int *fd, 
+                                int timeout, 
+                                int selectresolution, 
+                                const char **Error)
+{
+       int len, rlen;
+       int nSuccessLess = 0;
+       fd_set rfds;
+       char *pch = NULL;
         int fdflags;
        struct timeval tv;
 
@@ -912,7 +1488,7 @@ int StrBufTCP_read_buffered_line(StrBuf *Line,
                        return len - rlen;
                }
        }
-
+       
        if (buf->BufSize - buf->BufUsed < 10)
                IncreaseBuf(buf, 1, -1);
 
@@ -968,6 +1544,129 @@ int StrBufTCP_read_buffered_line(StrBuf *Line,
 
 }
 
+/**
+ * \brief Read a line from socket
+ * flushes and closes the FD on error
+ * \param buf the buffer to get the input to
+ * \param Pos pointer to the current read position, should be NULL initialized!
+ * \param fd pointer to the filedescriptor to read
+ * \param append Append to an existing string or replace?
+ * \param Error strerror() on error 
+ * \returns numbers of chars read
+ */
+int StrBufTCP_read_buffered_line_fast(StrBuf *Line, 
+                                     StrBuf *IOBuf, 
+                                     const char **Pos,
+                                     int *fd, 
+                                     int timeout, 
+                                     int selectresolution, 
+                                     const char **Error)
+{
+       const char *pche = NULL;
+       const char *pos = NULL;
+       int len, rlen;
+       int nSuccessLess = 0;
+       fd_set rfds;
+       const char *pch = NULL;
+        int fdflags;
+       struct timeval tv;
+       
+       pos = *Pos;
+       if ((IOBuf->BufUsed > 0) && 
+           (pos != NULL) && 
+           (pos < IOBuf->buf + IOBuf->BufUsed)) 
+       {
+               pche = IOBuf->buf + IOBuf->BufUsed;
+               pch = pos;
+               while ((pch < pche) && (*pch != '\n'))
+                       pch ++;
+               if ((pch >= pche) || (*pch == '\0'))
+                       pch = NULL;
+               if ((pch != NULL) && 
+                   (pch <= pche)) 
+               {
+                       rlen = 0;
+                       len = pch - pos;
+                       if (len > 0 && (*(pch - 1) == '\r') )
+                               rlen ++;
+                       StrBufSub(Line, IOBuf, (pos - IOBuf->buf), len - rlen);
+                       *Pos = pch + 1;
+                       return len - rlen;
+               }
+       }
+       
+       if (pos != NULL) {
+               if (pos > pche)
+                       FlushStrBuf(IOBuf);
+               else 
+                       StrBufCutLeft(IOBuf, (pos - IOBuf->buf));
+               *Pos = NULL;
+       }
+       
+       if (IOBuf->BufSize - IOBuf->BufUsed < 10) {
+               IncreaseBuf(IOBuf, 1, -1);
+       }
+
+       fdflags = fcntl(*fd, F_GETFL);
+       if ((fdflags & O_NONBLOCK) == O_NONBLOCK)
+               return -1;
+
+       pch = NULL;
+       while ((nSuccessLess < timeout) && (pch == NULL)) {
+               tv.tv_sec = selectresolution;
+               tv.tv_usec = 0;
+               
+               FD_ZERO(&rfds);
+               FD_SET(*fd, &rfds);
+               if (select(*fd + 1, NULL, &rfds, NULL, &tv) == -1) {
+                       *Error = strerror(errno);
+                       close (*fd);
+                       *fd = -1;
+                       return -1;
+               }               
+               if (FD_ISSET(*fd, &rfds) != 0) {
+                       rlen = read(*fd, 
+                                   &IOBuf->buf[IOBuf->BufUsed], 
+                                   IOBuf->BufSize - IOBuf->BufUsed - 1);
+                       if (rlen < 1) {
+                               *Error = strerror(errno);
+                               close(*fd);
+                               *fd = -1;
+                               return -1;
+                       }
+                       else if (rlen > 0) {
+                               nSuccessLess = 0;
+                               IOBuf->BufUsed += rlen;
+                               IOBuf->buf[IOBuf->BufUsed] = '\0';
+                               if (IOBuf->BufUsed + 10 > IOBuf->BufSize) {
+                                       IncreaseBuf(IOBuf, 1, -1);
+                               }
+
+                               pche = IOBuf->buf + IOBuf->BufUsed;
+                               pch = IOBuf->buf;
+                               while ((pch < pche) && (*pch != '\n'))
+                                       pch ++;
+                               if ((pch >= pche) || (*pch == '\0'))
+                                       pch = NULL;
+                               continue;
+                       }
+               }
+               nSuccessLess ++;
+       }
+       if (pch != NULL) {
+               pos = IOBuf->buf;
+               rlen = 0;
+               len = pch - pos;
+               if (len > 0 && (*(pch - 1) == '\r') )
+                       rlen ++;
+               StrBufSub(Line, IOBuf, 0, len - rlen);
+               *Pos = pos + len + 1;
+               return len - rlen;
+       }
+       return -1;
+
+}
+
 /**
  * \brief Input binary data from socket
  * flushes and closes the FD on error
@@ -990,7 +1689,7 @@ int StrBufReadBLOB(StrBuf *Buf, int *fd, int append, long nBytes, const char **E
                return -1;
        if (!append)
                FlushStrBuf(Buf);
-       if (Buf->BufUsed + nBytes > Buf->BufSize)
+       if (Buf->BufUsed + nBytes >= Buf->BufSize)
                IncreaseBuf(Buf, 1, Buf->BufUsed + nBytes);
 
        ptr = Buf->buf + Buf->BufUsed;
@@ -1025,6 +1724,142 @@ int StrBufReadBLOB(StrBuf *Buf, int *fd, int append, long nBytes, const char **E
        return nRead;
 }
 
+/**
+ * \brief Input binary data from socket
+ * flushes and closes the FD on error
+ * \param buf the buffer to get the input to
+ * \param fd pointer to the filedescriptor to read
+ * \param append Append to an existing string or replace?
+ * \param nBytes the maximal number of bytes to read
+ * \param Error strerror() on error 
+ * \returns numbers of chars read
+ */
+int StrBufReadBLOBBuffered(StrBuf *Blob, 
+                          StrBuf *IOBuf, 
+                          const char **Pos,
+                          int *fd, 
+                          int append, 
+                          long nBytes, 
+                          int check, 
+                          const char **Error)
+{
+       const char *pche;
+       const char *pos;
+       int nSelects = 0;
+       int SelRes;
+        fd_set wset;
+        int fdflags;
+       int len = 0;
+       int rlen, slen;
+       int nRead = 0;
+       char *ptr;
+       const char *pch;
+
+       if ((Blob == NULL) || (*fd == -1) || (IOBuf == NULL) || (Pos == NULL))
+               return -1;
+       if (!append)
+               FlushStrBuf(Blob);
+       if (Blob->BufUsed + nBytes >= Blob->BufSize) 
+               IncreaseBuf(Blob, append, Blob->BufUsed + nBytes);
+       
+       pos = *Pos;
+
+       if (pos > 0)
+               len = pos - IOBuf->buf;
+       rlen = IOBuf->BufUsed - len;
+
+
+       if ((IOBuf->BufUsed > 0) && 
+           (pos != NULL) && 
+           (pos < IOBuf->buf + IOBuf->BufUsed)) 
+       {
+               pche = IOBuf->buf + IOBuf->BufUsed;
+               pch = pos;
+
+               if (rlen < nBytes) {
+                       memcpy(Blob->buf + Blob->BufUsed, pos, rlen);
+                       Blob->BufUsed += rlen;
+                       Blob->buf[Blob->BufUsed] = '\0';
+                       nRead = rlen;
+                       *Pos = NULL; 
+               }
+               if (rlen >= nBytes) {
+                       memcpy(Blob->buf + Blob->BufUsed, pos, nBytes);
+                       Blob->BufUsed += nBytes;
+                       Blob->buf[Blob->BufUsed] = '\0';
+                       if (rlen == nBytes) {
+                               *Pos = NULL; 
+                               FlushStrBuf(IOBuf);
+                       }
+                       else 
+                               *Pos += nBytes;
+                       return nBytes;
+               }
+       }
+
+       FlushStrBuf(IOBuf);
+       if (IOBuf->BufSize < nBytes - nRead)
+               IncreaseBuf(IOBuf, 0, nBytes - nRead);
+       ptr = IOBuf->buf;
+
+       slen = len = Blob->BufUsed;
+
+       fdflags = fcntl(*fd, F_GETFL);
+
+       SelRes = 1;
+       nBytes -= nRead;
+       nRead = 0;
+       while (nRead < nBytes) {
+               if ((fdflags & O_NONBLOCK) == O_NONBLOCK) {
+                        FD_ZERO(&wset);
+                        FD_SET(*fd, &wset);
+                       SelRes = select(*fd + 1, NULL, &wset, NULL, NULL);
+               }
+               if (SelRes == -1) {
+                       *Error = strerror(errno);
+                       return -1;
+               }
+               else if (SelRes) {
+                       nSelects = 0;
+                       rlen = read(*fd, 
+                                   ptr,
+                                   nBytes - nRead);
+                       if (rlen == -1) {
+                               close(*fd);
+                               *fd = -1;
+                               *Error = strerror(errno);
+                               return rlen;
+                       }
+               }
+               else {
+                       nSelects ++;
+                       if ((check == NNN_TERM) && 
+                           (nRead > 5) &&
+                           (strncmp(IOBuf->buf + IOBuf->BufUsed - 5, "\n000\n", 5) == 0)) 
+                       {
+                               StrBufPlain(Blob, HKEY("\n000\n"));
+                               StrBufCutRight(Blob, 5);
+                               return Blob->BufUsed;
+                       }
+                       if (nSelects > 10) {
+                               FlushStrBuf(IOBuf);
+                               return -1;
+                       }
+               }
+               if (rlen > 0) {
+                       nRead += rlen;
+                       ptr += rlen;
+                       IOBuf->BufUsed += rlen;
+               }
+       }
+       if (nRead > nBytes) {
+               *Pos = IOBuf->buf + nBytes;
+       }
+       Blob->buf[Blob->BufUsed] = '\0';
+       StrBufAppendBufPlain(Blob, IOBuf->buf, nBytes, 0);
+       return nRead;
+}
+
 /**
  * \brief Cut nChars from the start of the string
  * \param Buf Buffer to modify
@@ -1056,6 +1891,73 @@ void StrBufCutRight(StrBuf *Buf, int nChars)
        Buf->buf[Buf->BufUsed] = '\0';
 }
 
+/**
+ * \brief Cut the string after n Chars
+ * \param Buf Buffer to modify
+ * \param AfternChars after how many chars should we trunkate the string?
+ * \param At if non-null and points inside of our string, cut it there.
+ */
+void StrBufCutAt(StrBuf *Buf, int AfternChars, const char *At)
+{
+       if (At != NULL){
+               AfternChars = At - Buf->buf;
+       }
+
+       if ((AfternChars < 0) || (AfternChars >= Buf->BufUsed))
+               return;
+       Buf->BufUsed = AfternChars;
+       Buf->buf[Buf->BufUsed] = '\0';
+}
+
+
+/*
+ * Strip leading and trailing spaces from a string; with premeasured and adjusted length.
+ * buf - the string to modify
+ * len - length of the string. 
+ */
+void StrBufTrim(StrBuf *Buf)
+{
+       int delta = 0;
+       if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
+
+       while ((Buf->BufUsed > delta) && (isspace(Buf->buf[delta]))){
+               delta ++;
+       }
+       if (delta > 0) StrBufCutLeft(Buf, delta);
+
+       if (Buf->BufUsed == 0) return;
+       while (isspace(Buf->buf[Buf->BufUsed - 1])){
+               Buf->BufUsed --;
+       }
+       Buf->buf[Buf->BufUsed] = '\0';
+}
+
+
+void StrBufUpCase(StrBuf *Buf) 
+{
+       char *pch, *pche;
+
+       pch = Buf->buf;
+       pche = pch + Buf->BufUsed;
+       while (pch < pche) {
+               *pch = toupper(*pch);
+               pch ++;
+       }
+}
+
+
+void StrBufLowerCase(StrBuf *Buf) 
+{
+       char *pch, *pche;
+
+       pch = Buf->buf;
+       pche = pch + Buf->BufUsed;
+       while (pch < pche) {
+               *pch = tolower(*pch);
+               pch ++;
+       }
+}
+
 
 /**
  * \brief unhide special chars hidden to the HTML escaper
@@ -1208,19 +2110,27 @@ int CompressBuffer(StrBuf *Buf)
 #ifdef HAVE_ZLIB
        char *compressed_data = NULL;
        size_t compressed_len, bufsize;
+       int i = 0;
        
        bufsize = compressed_len = ((Buf->BufUsed * 101) / 100) + 100;
        compressed_data = malloc(compressed_len);
        
+       /* Flush some space after the used payload so valgrind shuts up... */
+        while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
+               Buf->buf[Buf->BufUsed + i++] = '\0';
        if (compress_gzip((Bytef *) compressed_data,
                          &compressed_len,
                          (Bytef *) Buf->buf,
                          (uLongf) Buf->BufUsed, Z_BEST_SPEED) == Z_OK) {
-               if (!ConstBuf)
+               if (!Buf->ConstBuf)
                        free(Buf->buf);
                Buf->buf = compressed_data;
                Buf->BufUsed = compressed_len;
                Buf->BufSize = bufsize;
+               /* Flush some space after the used payload so valgrind shuts up... */
+               i = 0;
+               while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
+                       Buf->buf[Buf->BufUsed + i++] = '\0';
                return 1;
        } else {
                free(compressed_data);
@@ -1249,6 +2159,51 @@ int StrBufDecodeBase64(StrBuf *Buf)
        return siz;
 }
 
+/**
+ * \brief decode a buffer from base 64 encoding; destroys original
+ * \param Buf Buffor to transform
+ */
+int StrBufDecodeHex(StrBuf *Buf)
+{
+       unsigned int ch;
+       char *pch, *pche, *pchi;
+
+       if (Buf == NULL) return -1;
+
+       pch = pchi = Buf->buf;
+       pche = pch + Buf->BufUsed;
+
+       while (pchi < pche){
+               ch = decode_hex(pchi);
+               *pch = ch;
+               pch ++;
+               pchi += 2;
+       }
+
+       *pch = '\0';
+       Buf->BufUsed = pch - Buf->buf;
+       return Buf->BufUsed;
+}
+
+/**
+ * \brief replace all chars >0x20 && < 0x7F with Mute
+ * \param Mute char to put over invalid chars
+ * \param Buf Buffor to transform
+ */
+int StrBufSanitizeAscii(StrBuf *Buf, const char Mute)
+{
+       unsigned char *pch;
+
+       if (Buf == NULL) return -1;
+       pch = (unsigned char *)Buf->buf;
+       while (pch < (unsigned char *)Buf->buf + Buf->BufUsed) {
+               if ((*pch < 0x20) || (*pch > 0x7F))
+                       *pch = Mute;
+               pch ++;
+       }
+       return Buf->BufUsed;
+}
+
 
 /**
  * \brief  remove escaped strings from i.e. the url string (like %20 for blanks)
@@ -1338,12 +2293,12 @@ int StrBufRFC2047encode(StrBuf **target, const StrBuf *source)
        }
        if (*target == NULL)
                *target = NewStrBufPlain(NULL, sizeof(headerStr) + source->BufUsed * 2);
-       else if (sizeof(headerStr) + source->BufUsed > (*target)->BufSize)
+       else if (sizeof(headerStr) + source->BufUsed >= (*target)->BufSize)
                IncreaseBuf(*target, sizeof(headerStr) + source->BufUsed, 0);
        memcpy ((*target)->buf, headerStr, sizeof(headerStr) - 1);
        (*target)->BufUsed = sizeof(headerStr) - 1;
        for (i=0; (i < source->BufUsed); ++i) {
-               if ((*target)->BufUsed + 4 > (*target)->BufSize)
+               if ((*target)->BufUsed + 4 >= (*target)->BufSize)
                        IncreaseBuf(*target, 1, 0);
                ch = (unsigned char) source->buf[i];
                if ((ch < 32) || (ch > 126) || (ch == 61)) {
@@ -1356,7 +2311,7 @@ int StrBufRFC2047encode(StrBuf **target, const StrBuf *source)
                }
        }
        
-       if ((*target)->BufUsed + 4 > (*target)->BufSize)
+       if ((*target)->BufUsed + 4 >= (*target)->BufSize)
                IncreaseBuf(*target, 1, 0);
 
        (*target)->buf[(*target)->BufUsed++] = '?';
@@ -1381,3 +2336,444 @@ void StrBufReplaceChars(StrBuf *buf, char search, char replace)
                        buf->buf[i] = replace;
 
 }
+
+
+
+/*
+ * Wrapper around iconv_open()
+ * Our version adds aliases for non-standard Microsoft charsets
+ * such as 'MS950', aliasing them to names like 'CP950'
+ *
+ * tocode      Target encoding
+ * fromcode    Source encoding
+ */
+void  ctdl_iconv_open(const char *tocode, const char *fromcode, void *pic)
+{
+#ifdef HAVE_ICONV
+       iconv_t ic = (iconv_t)(-1) ;
+       ic = iconv_open(tocode, fromcode);
+       if (ic == (iconv_t)(-1) ) {
+               char alias_fromcode[64];
+               if ( (strlen(fromcode) == 5) && (!strncasecmp(fromcode, "MS", 2)) ) {
+                       safestrncpy(alias_fromcode, fromcode, sizeof alias_fromcode);
+                       alias_fromcode[0] = 'C';
+                       alias_fromcode[1] = 'P';
+                       ic = iconv_open(tocode, alias_fromcode);
+               }
+       }
+       *(iconv_t *)pic = ic;
+#endif
+}
+
+
+
+static inline char *FindNextEnd (const StrBuf *Buf, char *bptr)
+{
+       char * end;
+       /* Find the next ?Q? */
+       if (Buf->BufUsed - (bptr - Buf->buf)  < 6)
+               return NULL;
+
+       end = strchr(bptr + 2, '?');
+
+       if (end == NULL)
+               return NULL;
+
+       if ((Buf->BufUsed - (end - Buf->buf) > 3) &&
+           ((*(end + 1) == 'B') || (*(end + 1) == 'Q')) && 
+           (*(end + 2) == '?')) {
+               /* skip on to the end of the cluster, the next ?= */
+               end = strstr(end + 3, "?=");
+       }
+       else
+               /* sort of half valid encoding, try to find an end. */
+               end = strstr(bptr, "?=");
+       return end;
+}
+
+
+void StrBufConvert(StrBuf *ConvertBuf, StrBuf *TmpBuf, void *pic)
+{
+#ifdef HAVE_ICONV
+       int BufSize;
+       iconv_t ic;
+       char *ibuf;                     /**< Buffer of characters to be converted */
+       char *obuf;                     /**< Buffer for converted characters */
+       size_t ibuflen;                 /**< Length of input buffer */
+       size_t obuflen;                 /**< Length of output buffer */
+
+
+       if (ConvertBuf->BufUsed >= TmpBuf->BufSize)
+               IncreaseBuf(TmpBuf, 0, ConvertBuf->BufUsed);
+
+       ic = *(iconv_t*)pic;
+       ibuf = ConvertBuf->buf;
+       ibuflen = ConvertBuf->BufUsed;
+       obuf = TmpBuf->buf;
+       obuflen = TmpBuf->BufSize;
+       
+       iconv(ic, &ibuf, &ibuflen, &obuf, &obuflen);
+
+       /* little card game: wheres the red lady? */
+       ibuf = ConvertBuf->buf;
+       BufSize = ConvertBuf->BufSize;
+
+       ConvertBuf->buf = TmpBuf->buf;
+       ConvertBuf->BufSize = TmpBuf->BufSize;
+       ConvertBuf->BufUsed = TmpBuf->BufSize - obuflen;
+       ConvertBuf->buf[ConvertBuf->BufUsed] = '\0';
+       
+       TmpBuf->buf = ibuf;
+       TmpBuf->BufSize = BufSize;
+       TmpBuf->BufUsed = 0;
+       TmpBuf->buf[0] = '\0';
+#endif
+}
+
+
+
+
+inline static void DecodeSegment(StrBuf *Target, 
+                                const StrBuf *DecodeMe, 
+                                char *SegmentStart, 
+                                char *SegmentEnd, 
+                                StrBuf *ConvertBuf,
+                                StrBuf *ConvertBuf2, 
+                                StrBuf *FoundCharset)
+{
+       StrBuf StaticBuf;
+       char charset[128];
+       char encoding[16];
+#ifdef HAVE_ICONV
+       iconv_t ic = (iconv_t)(-1);
+#endif
+       /* Now we handle foreign character sets properly encoded
+        * in RFC2047 format.
+        */
+       StaticBuf.buf = SegmentStart;
+       StaticBuf.BufUsed = SegmentEnd - SegmentStart;
+       StaticBuf.BufSize = DecodeMe->BufSize - (SegmentStart - DecodeMe->buf);
+       extract_token(charset, SegmentStart, 1, '?', sizeof charset);
+       if (FoundCharset != NULL) {
+               FlushStrBuf(FoundCharset);
+               StrBufAppendBufPlain(FoundCharset, charset, -1, 0);
+       }
+       extract_token(encoding, SegmentStart, 2, '?', sizeof encoding);
+       StrBufExtract_token(ConvertBuf, &StaticBuf, 3, '?');
+       
+       *encoding = toupper(*encoding);
+       if (*encoding == 'B') { /**< base64 */
+               ConvertBuf2->BufUsed = CtdlDecodeBase64(ConvertBuf2->buf, 
+                                                       ConvertBuf->buf, 
+                                                       ConvertBuf->BufUsed);
+       }
+       else if (*encoding == 'Q') {    /**< quoted-printable */
+               long pos;
+               
+               pos = 0;
+               while (pos < ConvertBuf->BufUsed)
+               {
+                       if (ConvertBuf->buf[pos] == '_') 
+                               ConvertBuf->buf[pos] = ' ';
+                       pos++;
+               }
+               
+               ConvertBuf2->BufUsed = CtdlDecodeQuotedPrintable(
+                       ConvertBuf2->buf, 
+                       ConvertBuf->buf,
+                       ConvertBuf->BufUsed);
+       }
+       else {
+               StrBufAppendBuf(ConvertBuf2, ConvertBuf, 0);
+       }
+#ifdef HAVE_ICONV
+       ctdl_iconv_open("UTF-8", charset, &ic);
+       if (ic != (iconv_t)(-1) ) {             
+#endif
+               StrBufConvert(ConvertBuf2, ConvertBuf, &ic);
+               StrBufAppendBuf(Target, ConvertBuf2, 0);
+#ifdef HAVE_ICONV
+               iconv_close(ic);
+       }
+       else {
+               StrBufAppendBufPlain(Target, HKEY("(unreadable)"), 0);
+       }
+#endif
+}
+/*
+ * Handle subjects with RFC2047 encoding such as:
+ * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
+ */
+void StrBuf_RFC822_to_Utf8(StrBuf *Target, const StrBuf *DecodeMe, const StrBuf* DefaultCharset, StrBuf *FoundCharset)
+{
+       StrBuf *ConvertBuf, *ConvertBuf2;
+       char *start, *end, *next, *nextend, *ptr = NULL;
+#ifdef HAVE_ICONV
+       iconv_t ic = (iconv_t)(-1) ;
+#endif
+       const char *eptr;
+       int passes = 0;
+       int i, len, delta;
+       int illegal_non_rfc2047_encoding = 0;
+
+       /* Sometimes, badly formed messages contain strings which were simply
+        *  written out directly in some foreign character set instead of
+        *  using RFC2047 encoding.  This is illegal but we will attempt to
+        *  handle it anyway by converting from a user-specified default
+        *  charset to UTF-8 if we see any nonprintable characters.
+        */
+       
+       len = StrLength(DecodeMe);
+       for (i=0; i<DecodeMe->BufUsed; ++i) {
+               if ((DecodeMe->buf[i] < 32) || (DecodeMe->buf[i] > 126)) {
+                       illegal_non_rfc2047_encoding = 1;
+                       break;
+               }
+       }
+
+       ConvertBuf = NewStrBufPlain(NULL, StrLength(DecodeMe));
+       if ((illegal_non_rfc2047_encoding) &&
+           (strcasecmp(ChrPtr(DefaultCharset), "UTF-8")) && 
+           (strcasecmp(ChrPtr(DefaultCharset), "us-ascii")) )
+       {
+#ifdef HAVE_ICONV
+               ctdl_iconv_open("UTF-8", ChrPtr(DefaultCharset), &ic);
+               if (ic != (iconv_t)(-1) ) {
+                       StrBufConvert((StrBuf*)DecodeMe, ConvertBuf, &ic);///TODO: don't void const?
+                       iconv_close(ic);
+               }
+#endif
+       }
+
+       /* pre evaluate the first pair */
+       nextend = end = NULL;
+       len = StrLength(DecodeMe);
+       start = strstr(DecodeMe->buf, "=?");
+       eptr = DecodeMe->buf + DecodeMe->BufUsed;
+       if (start != NULL) 
+               end = FindNextEnd (DecodeMe, start);
+       else {
+               StrBufAppendBuf(Target, DecodeMe, 0);
+               FreeStrBuf(&ConvertBuf);
+               return;
+       }
+
+       ConvertBuf2 = NewStrBufPlain(NULL, StrLength(DecodeMe));
+
+       if (start != DecodeMe->buf)
+               StrBufAppendBufPlain(Target, DecodeMe->buf, start - DecodeMe->buf, 0);
+       /*
+        * Since spammers will go to all sorts of absurd lengths to get their
+        * messages through, there are LOTS of corrupt headers out there.
+        * So, prevent a really badly formed RFC2047 header from throwing
+        * this function into an infinite loop.
+        */
+       while ((start != NULL) && 
+              (end != NULL) && 
+              (start < eptr) && 
+              (end < eptr) && 
+              (passes < 20))
+       {
+               passes++;
+               DecodeSegment(Target, 
+                             DecodeMe, 
+                             start, 
+                             end, 
+                             ConvertBuf,
+                             ConvertBuf2,
+                             FoundCharset);
+               
+               next = strstr(end, "=?");
+               nextend = NULL;
+               if ((next != NULL) && 
+                   (next < eptr))
+                       nextend = FindNextEnd(DecodeMe, next);
+               if (nextend == NULL)
+                       next = NULL;
+
+               /* did we find two partitions */
+               if ((next != NULL) && 
+                   ((next - end) > 2))
+               {
+                       ptr = end + 2;
+                       while ((ptr < next) && 
+                              (isspace(*ptr) ||
+                               (*ptr == '\r') ||
+                               (*ptr == '\n') || 
+                               (*ptr == '\t')))
+                               ptr ++;
+                       /* did we find a gab just filled with blanks? */
+                       if (ptr == next)
+                       {
+                               memmove (end + 2,
+                                        next,
+                                        len - (next - start));
+                               
+                               /* now terminate the gab at the end */
+                               delta = (next - end) - 2; ////TODO: const! 
+                               ((StrBuf*)DecodeMe)->BufUsed -= delta;
+                               ((StrBuf*)DecodeMe)->buf[DecodeMe->BufUsed] = '\0';
+
+                               /* move next to its new location. */
+                               next -= delta;
+                               nextend -= delta;
+                       }
+               }
+               /* our next-pair is our new first pair now. */
+               ptr = end + 2;
+               start = next;
+               end = nextend;
+       }
+       end = ptr;
+       nextend = DecodeMe->buf + DecodeMe->BufUsed;
+       if ((end != NULL) && (end < nextend)) {
+               ptr = end;
+               while ( (ptr < nextend) &&
+                       (isspace(*ptr) ||
+                        (*ptr == '\r') ||
+                        (*ptr == '\n') || 
+                        (*ptr == '\t')))
+                       ptr ++;
+               if (ptr < nextend)
+                       StrBufAppendBufPlain(Target, end, nextend - end, 0);
+       }
+       FreeStrBuf(&ConvertBuf);
+       FreeStrBuf(&ConvertBuf2);
+}
+
+/**
+ * \brief evaluate the length of an utf8 special character sequence
+ * \param Char the character to examine
+ * \returns width of utf8 chars in bytes
+ */
+static inline int Ctdl_GetUtf8SequenceLength(char *CharS, char *CharE)
+{
+       int n = 1;
+        char test = (1<<7);
+       
+       while ((n < 8) && ((test & *CharS) != 0)) {
+               test = test << 1;
+               n ++;
+       }
+       if ((n > 6) || ((CharE - CharS) > n))
+               n = 1;
+       return n;
+}
+
+/**
+ * \brief detect whether this char starts an utf-8 encoded char
+ * \param Char character to inspect
+ * \returns yes or no
+ */
+static inline int Ctdl_IsUtf8SequenceStart(char Char)
+{
+/** 11??.???? indicates an UTF8 Sequence. */
+       return ((Char & 0xC0) != 0);
+}
+
+/**
+ * \brief measure the number of glyphs in an UTF8 string...
+ * \param str string to measure
+ * \returns the length of str
+ */
+long StrBuf_Utf8StrLen(StrBuf *Buf)
+{
+       int n = 0;
+       int m = 0;
+       char *aptr, *eptr;
+
+       if ((Buf == NULL) || (Buf->BufUsed == 0))
+               return 0;
+       aptr = Buf->buf;
+       eptr = Buf->buf + Buf->BufUsed;
+       while ((aptr < eptr) && (*aptr != '\0')) {
+               if (Ctdl_IsUtf8SequenceStart(*aptr)){
+                       m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
+                       while ((aptr < eptr) && (m-- > 0) && (*aptr++ != '\0'))
+                               n ++;
+               }
+               else {
+                       n++;
+                       aptr++;
+               }
+                       
+       }
+       return n;
+}
+
+/**
+ * \brief cuts a string after maxlen glyphs
+ * \param str string to cut to maxlen glyphs
+ * \param maxlen how long may the string become?
+ * \returns pointer to maxlen or the end of the string
+ */
+long StrBuf_Utf8StrCut(StrBuf *Buf, int maxlen)
+{
+       char *aptr, *eptr;
+       int n = 0, m = 0;
+
+       aptr = Buf->buf;
+       eptr = Buf->buf + Buf->BufUsed;
+       while ((aptr < eptr) && (*aptr != '\0')) {
+               if (Ctdl_IsUtf8SequenceStart(*aptr)){
+                       m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
+                       while ((m-- > 0) && (*aptr++ != '\0'))
+                               n ++;
+               }
+               else {
+                       n++;
+                       aptr++;
+               }
+               if (n > maxlen) {
+                       *aptr = '\0';
+                       Buf->BufUsed = aptr - Buf->buf;
+                       return Buf->BufUsed;
+               }                       
+       }
+       return Buf->BufUsed;
+
+}
+
+
+
+int StrBufSipLine(StrBuf *LineBuf, StrBuf *Buf, const char **Ptr)
+{
+       const char *aptr, *ptr, *eptr;
+       char *optr, *xptr;
+
+       if (Buf == NULL)
+               return 0;
+
+       if (*Ptr==NULL)
+               ptr = aptr = Buf->buf;
+       else
+               ptr = aptr = *Ptr;
+
+       optr = LineBuf->buf;
+       eptr = Buf->buf + Buf->BufUsed;
+       xptr = LineBuf->buf + LineBuf->BufSize - 1;
+
+       while ((*ptr != '\n') &&
+              (*ptr != '\r') &&
+              (ptr < eptr))
+       {
+               *optr = *ptr;
+               optr++; ptr++;
+               if (optr == xptr) {
+                       LineBuf->BufUsed = optr - LineBuf->buf;
+                       IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
+                       optr = LineBuf->buf + LineBuf->BufUsed;
+                       xptr = LineBuf->buf + LineBuf->BufSize - 1;
+               }
+       }
+       LineBuf->BufUsed = optr - LineBuf->buf;
+       *optr = '\0';       
+       if (*ptr == '\r')
+               ptr ++;
+       if (*ptr == '\n')
+               ptr ++;
+
+       *Ptr = ptr;
+
+       return Buf->BufUsed - (ptr - Buf->buf);
+}