]> code.citadel.org Git - citadel.git/blobdiff - libcitadel/lib/stringbuf.c
* move utf8 handling stuff into strbuf, so we can be more exact about our buffer...
[citadel.git] / libcitadel / lib / stringbuf.c
index 03459bd789fe42fbb8c2ff8b470ed72a5555f81f..4bec25f3dde47740c3e4d931ee054b3c6c4331a3 100644 (file)
@@ -7,6 +7,7 @@
 #include <stdio.h>
 #include <sys/select.h>
 #include <fcntl.h>
+#include <sys/types.h>
 #define SHOW_ME_VAPPEND_PRINTF
 #include <stdarg.h>
 #include "libcitadel.h"
 #include <iconv.h>
 #endif
 
-#ifdef HAVE_ZLIB
-#include <zlib.h>
+#if HAVE_BACKTRACE
+#include <execinfo.h>
 #endif
 
-
 #ifdef HAVE_ZLIB
 #include <zlib.h>
 int ZEXPORT compress_gzip(Bytef * dest, size_t * destLen,
                           const Bytef * source, uLong sourceLen, int level);
 #endif
+int BaseStrBufSize = 64;
 
 /**
  * Private Structure for the Stringbuffer
@@ -34,8 +35,42 @@ struct StrBuf {
        long BufSize;      /**< how many spcae do we optain */
        long BufUsed;      /**< StNumber of Chars used excluding the trailing \0 */
        int ConstBuf;      /**< are we just a wrapper arround a static buffer and musn't we be changed? */
+#ifdef SIZE_DEBUG
+       long nIncreases;
+       char bt [SIZ];
+       char bt_lastinc [SIZ];
+#endif
 };
 
+#ifdef SIZE_DEBUG
+#if HAVE_BACKTRACE
+static void StrBufBacktrace(StrBuf *Buf, int which)
+{
+       int n;
+       char *pstart, *pch;
+       void *stack_frames[50];
+       size_t size, i;
+       char **strings;
+
+       if (which)
+               pstart = pch = Buf->bt;
+       else
+               pstart = pch = Buf->bt_lastinc;
+       size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
+       strings = backtrace_symbols(stack_frames, size);
+       for (i = 0; i < size; i++) {
+               if (strings != NULL)
+                       n = snprintf(pch, SIZ - (pch - pstart), "%s\\n", strings[i]);
+               else
+                       n = snprintf(pch, SIZ - (pch - pstart), "%p\\n", stack_frames[i]);
+               pch += n;
+       }
+       free(strings);
+
+
+}
+#endif
+#endif
 
 /** 
  * \Brief Cast operator to Plain String 
@@ -93,10 +128,52 @@ static int IncreaseBuf(StrBuf *Buf, int KeepOriginal, int DestSize)
        }
        free (Buf->buf);
        Buf->buf = NewBuf;
-       Buf->BufSize *= 2;
+       Buf->BufSize = NewSize;
+#ifdef SIZE_DEBUG
+       Buf->nIncreases++;
+#if HAVE_BACKTRACE
+       StrBufBacktrace(Buf, 1);
+#endif
+#endif
        return Buf->BufSize;
 }
 
+/**
+ * \brief shrink an _EMPTY_ buffer if its Buffer superseeds threshhold to NewSize. Buffercontent is thoroughly ignored and flushed.
+ * \param Buf Buffer to shrink (has to be empty)
+ * \param ThreshHold if the buffer is bigger then this, its readjusted
+ * \param NewSize if we Shrink it, how big are we going to be afterwards?
+ */
+void ReAdjustEmptyBuf(StrBuf *Buf, long ThreshHold, long NewSize)
+{
+       if (Buf->BufUsed > ThreshHold) {
+               free(Buf->buf);
+               Buf->buf = (char*) malloc(NewSize);
+               Buf->BufUsed = 0;
+               Buf->BufSize = NewSize;
+       }
+}
+
+/**
+ * \brief shrink long term buffers to their real size so they don't waste memory
+ * \param Buf buffer to shrink
+ * \param Force if not set, will just executed if the buffer is much to big; set for lifetime strings
+ * \returns physical size of the buffer
+ */
+long StrBufShrinkToFit(StrBuf *Buf, int Force)
+{
+       if (Force || 
+           (Buf->BufUsed + (Buf->BufUsed / 3) > Buf->BufSize))
+       {
+               char *TmpBuf = (char*) malloc(Buf->BufUsed + 1);
+               memcpy (TmpBuf, Buf->buf, Buf->BufUsed + 1);
+               Buf->BufSize = Buf->BufUsed + 1;
+               free(Buf->buf);
+               Buf->buf = TmpBuf;
+       }
+       return Buf->BufUsed;
+}
+
 /**
  * Allocate a new buffer with default buffer size
  * \returns the new stringbuffer
@@ -106,11 +183,19 @@ StrBuf* NewStrBuf(void)
        StrBuf *NewBuf;
 
        NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
-       NewBuf->buf = (char*) malloc(SIZ);
+       NewBuf->buf = (char*) malloc(BaseStrBufSize);
        NewBuf->buf[0] = '\0';
-       NewBuf->BufSize = SIZ;
+       NewBuf->BufSize = BaseStrBufSize;
        NewBuf->BufUsed = 0;
        NewBuf->ConstBuf = 0;
+#ifdef SIZE_DEBUG
+       NewBuf->nIncreases = 0;
+       NewBuf->bt[0] = '\0';
+       NewBuf->bt_lastinc[0] = '\0';
+#if HAVE_BACKTRACE
+       StrBufBacktrace(NewBuf, 0);
+#endif
+#endif
        return NewBuf;
 }
 
@@ -132,6 +217,14 @@ StrBuf* NewStrBufDup(const StrBuf *CopyMe)
        NewBuf->BufUsed = CopyMe->BufUsed;
        NewBuf->BufSize = CopyMe->BufSize;
        NewBuf->ConstBuf = 0;
+#ifdef SIZE_DEBUG
+       NewBuf->nIncreases = 0;
+       NewBuf->bt[0] = '\0';
+       NewBuf->bt_lastinc[0] = '\0';
+#if HAVE_BACKTRACE
+       StrBufBacktrace(NewBuf, 0);
+#endif
+#endif
        return NewBuf;
 }
 
@@ -146,7 +239,7 @@ StrBuf* NewStrBufDup(const StrBuf *CopyMe)
 StrBuf* NewStrBufPlain(const char* ptr, int nChars)
 {
        StrBuf *NewBuf;
-       size_t Siz = SIZ;
+       size_t Siz = BaseStrBufSize;
        size_t CopySize;
 
        NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
@@ -170,6 +263,14 @@ StrBuf* NewStrBufPlain(const char* ptr, int nChars)
                NewBuf->BufUsed = 0;
        }
        NewBuf->ConstBuf = 0;
+#ifdef SIZE_DEBUG
+       NewBuf->nIncreases = 0;
+       NewBuf->bt[0] = '\0';
+       NewBuf->bt_lastinc[0] = '\0';
+#if HAVE_BACKTRACE
+       StrBufBacktrace(NewBuf, 0);
+#endif
+#endif
        return NewBuf;
 }
 
@@ -216,6 +317,11 @@ StrBuf* _NewConstStrBuf(const char* StringConstant, size_t SizeOfStrConstant)
        NewBuf->BufSize = SizeOfStrConstant;
        NewBuf->BufUsed = SizeOfStrConstant;
        NewBuf->ConstBuf = 1;
+#ifdef SIZE_DEBUG
+       NewBuf->nIncreases = 0;
+       NewBuf->bt[0] = '\0';
+       NewBuf->bt_lastinc[0] = '\0';
+#endif
        return NewBuf;
 }
 
@@ -235,6 +341,26 @@ int FlushStrBuf(StrBuf *buf)
        return 0;
 }
 
+/**
+ * \brief wipe the content of a Buf thoroughly (overwrite it -> expensive); keep its struct
+ * \param buf Buffer to wipe
+ */
+int FLUSHStrBuf(StrBuf *buf)
+{
+       if (buf == NULL)
+               return -1;
+       if (buf->ConstBuf)
+               return -1;
+       if (buf->BufUsed > 0) {
+               memset(buf->buf, 0, buf->BufUsed);
+               buf->BufUsed = 0;
+       }
+       return 0;
+}
+
+#ifdef SIZE_DEBUG
+int hFreeDbglog = -1;
+#endif
 /**
  * \brief Release a Buffer
  * Its a double pointer, so it can NULL your pointer
@@ -245,6 +371,35 @@ void FreeStrBuf (StrBuf **FreeMe)
 {
        if (*FreeMe == NULL)
                return;
+#ifdef SIZE_DEBUG
+       if (hFreeDbglog == -1){
+               pid_t pid = getpid();
+               char path [SIZ];
+               snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
+               hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
+       }
+       if ((*FreeMe)->nIncreases > 0)
+       {
+               char buf[SIZ * 3];
+               long n;
+               n = snprintf(buf, SIZ * 3, "+|%ld|%ld|%ld|%s|%s|\n",
+                            (*FreeMe)->nIncreases,
+                            (*FreeMe)->BufUsed,
+                            (*FreeMe)->BufSize,
+                            (*FreeMe)->bt,
+                            (*FreeMe)->bt_lastinc);
+               n = write(hFreeDbglog, buf, n);
+       }
+       else
+       {
+               char buf[128];
+               long n;
+               n = snprintf(buf, 128, "_|0|%ld%ld|\n",
+                            (*FreeMe)->BufUsed,
+                            (*FreeMe)->BufSize);
+               n = write(hFreeDbglog, buf, n);
+       }
+#endif
        if (!(*FreeMe)->ConstBuf) 
                free((*FreeMe)->buf);
        free(*FreeMe);
@@ -261,6 +416,35 @@ void HFreeStrBuf (void *VFreeMe)
        StrBuf *FreeMe = (StrBuf*)VFreeMe;
        if (FreeMe == NULL)
                return;
+#ifdef SIZE_DEBUG
+       if (hFreeDbglog == -1){
+               pid_t pid = getpid();
+               char path [SIZ];
+               snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
+               hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
+       }
+       if (FreeMe->nIncreases > 0)
+       {
+               char buf[SIZ * 3];
+               long n;
+               n = snprintf(buf, SIZ * 3, "+|%ld|%ld|%ld|%s|%s|\n",
+                            FreeMe->nIncreases,
+                            FreeMe->BufUsed,
+                            FreeMe->BufSize,
+                            FreeMe->bt,
+                            FreeMe->bt_lastinc);
+               write(hFreeDbglog, buf, n);
+       }
+       else
+       {
+               char buf[128];
+               long n;
+               n = snprintf(buf, 128, "_|%ld|%ld%ld|\n",
+                            FreeMe->nIncreases,
+                            FreeMe->BufUsed,
+                            FreeMe->BufSize);
+       }
+#endif
        if (!FreeMe->ConstBuf) 
                free(FreeMe->buf);
        free(FreeMe);
@@ -291,14 +475,15 @@ int StrToi(const StrBuf *Buf)
        else
                return 0;
 }
+
 /**
  * \brief Checks to see if the string is a pure number 
  */
 int StrBufIsNumber(const StrBuf *Buf) {
+  char * pEnd;
   if (Buf == NULL) {
        return 0;
   }
-  char * pEnd;
   strtoll(Buf->buf, &pEnd, 10);
   if (pEnd == NULL && ((Buf->buf)-pEnd) != 0) {
     return 1;
@@ -658,13 +843,17 @@ long StrECMAEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
                        bptr = Target->buf + Target->BufUsed;
                }
                else if (*aptr == '"') {
-                       memcpy(bptr, "\\\"", 2);
-                       bptr += 2;
+                       *bptr = '\\';
+                       bptr ++;
+                       *bptr = '"';
+                       bptr ++;
                        Target->BufUsed += 2;
                } else if (*aptr == '\\') {
-                 memcpy(bptr, "\\\\", 2);
-                 bptr += 2;
-                 Target->BufUsed += 2;
+                       *bptr = '\\';
+                       bptr ++;
+                       *bptr = '\\';
+                       bptr ++;
+                       Target->BufUsed += 2;
                }
                else{
                        *bptr = *aptr;
@@ -674,7 +863,7 @@ long StrECMAEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
                aptr ++;
        }
        *bptr = '\0';
-       if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
+       if ((bptr == eptr - 1 ) && !IsEmptyStr(aptr) )
                return -1;
        return Target->BufUsed;
 }
@@ -737,6 +926,7 @@ void StrBufVAppendPrintf(StrBuf *Buf, const char *format, va_list ap)
                newused = Offset + nWritten;
                if (newused >= Buf->BufSize) {
                        IncreaseBuf(Buf, 1, newused);
+                       newused = Buf->BufSize + 1;
                }
                else {
                        Buf->BufUsed = Offset + nWritten;
@@ -769,6 +959,7 @@ void StrBufAppendPrintf(StrBuf *Buf, const char *format, ...)
                newused = Buf->BufUsed + nWritten;
                if (newused >= Buf->BufSize) {
                        IncreaseBuf(Buf, 1, newused);
+                       newused = Buf->BufSize + 1;
                }
                else {
                        Buf->BufUsed += nWritten;
@@ -793,9 +984,12 @@ void StrBufPrintf(StrBuf *Buf, const char *format, ...)
                va_start(arg_ptr, format);
                nWritten = vsnprintf(Buf->buf, Buf->BufSize, format, arg_ptr);
                va_end(arg_ptr);
-               Buf->BufUsed = nWritten ;
-               if (nWritten >= Buf->BufSize)
+               if (nWritten >= Buf->BufSize) {
                        IncreaseBuf(Buf, 0, 0);
+                       nWritten = Buf->BufSize + 1;
+                       continue;
+               }
+               Buf->BufUsed = nWritten ;
        }
 }
 
@@ -1361,47 +1555,63 @@ int StrBufTCP_read_buffered_line(StrBuf *Line,
  * \returns numbers of chars read
  */
 int StrBufTCP_read_buffered_line_fast(StrBuf *Line, 
-                                     StrBuf *buf, 
+                                     StrBuf *IOBuf, 
                                      const char **Pos,
                                      int *fd, 
                                      int timeout, 
                                      int selectresolution, 
                                      const char **Error)
 {
+       const char *pche = NULL;
+       const char *pos = NULL;
        int len, rlen;
        int nSuccessLess = 0;
        fd_set rfds;
        const char *pch = NULL;
         int fdflags;
        struct timeval tv;
-
-       if ((buf->BufUsed > 0) && (Pos != NULL)) {
-               if (*Pos == NULL)
-                       *Pos = buf->buf;
-               pch = strchr(*Pos, '\n');
-               if (pch != NULL) {
+       
+       pos = *Pos;
+       if ((IOBuf->BufUsed > 0) && 
+           (pos != NULL) && 
+           (pos < IOBuf->buf + IOBuf->BufUsed)) 
+       {
+               pche = IOBuf->buf + IOBuf->BufUsed;
+               pch = pos;
+               while ((pch < pche) && (*pch != '\n'))
+                       pch ++;
+               if ((pch >= pche) || (*pch == '\0'))
+                       pch = NULL;
+               if ((pch != NULL) && 
+                   (pch <= pche)) 
+               {
                        rlen = 0;
-                       len = pch - *Pos;
+                       len = pch - pos;
                        if (len > 0 && (*(pch - 1) == '\r') )
                                rlen ++;
-                       StrBufSub(Line, buf, (*Pos - buf->buf), len - rlen);
+                       StrBufSub(Line, IOBuf, (pos - IOBuf->buf), len - rlen);
                        *Pos = pch + 1;
                        return len - rlen;
                }
        }
        
-       if (*Pos != NULL) {
-               StrBufCutLeft(buf, (*Pos - buf->buf));
+       if (pos != NULL) {
+               if (pos > pche)
+                       FlushStrBuf(IOBuf);
+               else 
+                       StrBufCutLeft(IOBuf, (pos - IOBuf->buf));
                *Pos = NULL;
        }
        
-       if (buf->BufSize - buf->BufUsed < 10)
-               IncreaseBuf(buf, 1, -1);
+       if (IOBuf->BufSize - IOBuf->BufUsed < 10) {
+               IncreaseBuf(IOBuf, 1, -1);
+       }
 
        fdflags = fcntl(*fd, F_GETFL);
        if ((fdflags & O_NONBLOCK) == O_NONBLOCK)
                return -1;
 
+       pch = NULL;
        while ((nSuccessLess < timeout) && (pch == NULL)) {
                tv.tv_sec = selectresolution;
                tv.tv_usec = 0;
@@ -1414,10 +1624,10 @@ int StrBufTCP_read_buffered_line_fast(StrBuf *Line,
                        *fd = -1;
                        return -1;
                }               
-               if (FD_ISSET(*fd, &rfds)) {
+               if (FD_ISSET(*fd, &rfds) != 0) {
                        rlen = read(*fd, 
-                                   &buf->buf[buf->BufUsed], 
-                                   buf->BufSize - buf->BufUsed - 1);
+                                   &IOBuf->buf[IOBuf->BufUsed], 
+                                   IOBuf->BufSize - IOBuf->BufUsed - 1);
                        if (rlen < 1) {
                                *Error = strerror(errno);
                                close(*fd);
@@ -1426,25 +1636,31 @@ int StrBufTCP_read_buffered_line_fast(StrBuf *Line,
                        }
                        else if (rlen > 0) {
                                nSuccessLess = 0;
-                               buf->BufUsed += rlen;
-                               buf->buf[buf->BufUsed] = '\0';
-                               if (buf->BufUsed + 10 > buf->BufSize) {
-                                       IncreaseBuf(buf, 1, -1);
+                               IOBuf->BufUsed += rlen;
+                               IOBuf->buf[IOBuf->BufUsed] = '\0';
+                               if (IOBuf->BufUsed + 10 > IOBuf->BufSize) {
+                                       IncreaseBuf(IOBuf, 1, -1);
                                }
-                               pch = strchr(buf->buf, '\n');
+
+                               pche = IOBuf->buf + IOBuf->BufUsed;
+                               pch = IOBuf->buf;
+                               while ((pch < pche) && (*pch != '\n'))
+                                       pch ++;
+                               if ((pch >= pche) || (*pch == '\0'))
+                                       pch = NULL;
                                continue;
                        }
                }
                nSuccessLess ++;
        }
        if (pch != NULL) {
-               *Pos = buf->buf;
+               pos = IOBuf->buf;
                rlen = 0;
-               len = pch - *Pos;
+               len = pch - pos;
                if (len > 0 && (*(pch - 1) == '\r') )
                        rlen ++;
-               StrBufSub(Line, buf, 0, len - rlen);
-               *Pos = *Pos + len + 1;
+               StrBufSub(Line, IOBuf, 0, len - rlen);
+               *Pos = pos + len + 1;
                return len - rlen;
        }
        return -1;
@@ -1518,61 +1734,81 @@ int StrBufReadBLOB(StrBuf *Buf, int *fd, int append, long nBytes, const char **E
  * \param Error strerror() on error 
  * \returns numbers of chars read
  */
-int StrBufReadBLOBBuffered(StrBuf *Buf
+int StrBufReadBLOBBuffered(StrBuf *Blob
                           StrBuf *IOBuf, 
-                          const char **BufPos,
+                          const char **Pos,
                           int *fd, 
                           int append, 
                           long nBytes, 
                           int check, 
                           const char **Error)
 {
+       const char *pche;
+       const char *pos;
        int nSelects = 0;
        int SelRes;
         fd_set wset;
         int fdflags;
-       int len, rlen, slen;
+       int len = 0;
+       int rlen, slen;
        int nRead = 0;
        char *ptr;
+       const char *pch;
 
-       if ((Buf == NULL) || (*fd == -1) || (IOBuf == NULL))
+       if ((Blob == NULL) || (*fd == -1) || (IOBuf == NULL) || (Pos == NULL))
                return -1;
        if (!append)
-               FlushStrBuf(Buf);
-       if (Buf->BufUsed + nBytes >= Buf->BufSize)
-               IncreaseBuf(Buf, 1, Buf->BufUsed + nBytes);
-
+               FlushStrBuf(Blob);
+       if (Blob->BufUsed + nBytes >= Blob->BufSize) 
+               IncreaseBuf(Blob, append, Blob->BufUsed + nBytes);
+       
+       pos = *Pos;
 
-       len = *BufPos - IOBuf->buf;
+       if (pos > 0)
+               len = pos - IOBuf->buf;
        rlen = IOBuf->BufUsed - len;
 
+
        if ((IOBuf->BufUsed > 0) && 
-           ((IOBuf->BufUsed - len > 0))) {
-               if (rlen <= nBytes) {
-                       memcpy(Buf->buf + Buf->BufUsed, *BufPos, rlen);
-                       Buf->BufUsed += rlen;
-                       Buf->buf[Buf->BufUsed] = '\0';
-                       *BufPos = NULL; FlushStrBuf(IOBuf);
+           (pos != NULL) && 
+           (pos < IOBuf->buf + IOBuf->BufUsed)) 
+       {
+               pche = IOBuf->buf + IOBuf->BufUsed;
+               pch = pos;
+
+               if (rlen < nBytes) {
+                       memcpy(Blob->buf + Blob->BufUsed, pos, rlen);
+                       Blob->BufUsed += rlen;
+                       Blob->buf[Blob->BufUsed] = '\0';
                        nRead = rlen;
-                       if (nRead == nBytes)
-                               return nBytes;
-               }
-               else {
-                       memcpy(Buf->buf + Buf->BufUsed, *BufPos, nBytes);
-                       Buf->BufUsed += nBytes;
-                       Buf->buf[Buf->BufUsed] = '\0';
-                       *BufPos += nBytes;
+                       *Pos = NULL; 
+               }
+               if (rlen >= nBytes) {
+                       memcpy(Blob->buf + Blob->BufUsed, pos, nBytes);
+                       Blob->BufUsed += nBytes;
+                       Blob->buf[Blob->BufUsed] = '\0';
+                       if (rlen == nBytes) {
+                               *Pos = NULL; 
+                               FlushStrBuf(IOBuf);
+                       }
+                       else 
+                               *Pos += nBytes;
                        return nBytes;
                }
        }
 
-       ptr = Buf->buf + Buf->BufUsed;
+       FlushStrBuf(IOBuf);
+       if (IOBuf->BufSize < nBytes - nRead)
+               IncreaseBuf(IOBuf, 0, nBytes - nRead);
+       ptr = IOBuf->buf;
 
-       slen = len = Buf->BufUsed;
+       slen = len = Blob->BufUsed;
 
        fdflags = fcntl(*fd, F_GETFL);
 
        SelRes = 1;
+       nBytes -= nRead;
+       nRead = 0;
        while (nRead < nBytes) {
                if ((fdflags & O_NONBLOCK) == O_NONBLOCK) {
                         FD_ZERO(&wset);
@@ -1585,9 +1821,10 @@ int StrBufReadBLOBBuffered(StrBuf *Buf,
                }
                else if (SelRes) {
                        nSelects = 0;
-                       if ((rlen = read(*fd, 
-                                        ptr,
-                                        nBytes - nRead)) == -1) {
+                       rlen = read(*fd, 
+                                   ptr,
+                                   nBytes - nRead);
+                       if (rlen == -1) {
                                close(*fd);
                                *fd = -1;
                                *Error = strerror(errno);
@@ -1598,22 +1835,28 @@ int StrBufReadBLOBBuffered(StrBuf *Buf,
                        nSelects ++;
                        if ((check == NNN_TERM) && 
                            (nRead > 5) &&
-                           (strncmp(Buf->buf + Buf->BufUsed - 5, "\n000\n", 5) == 0)) 
+                           (strncmp(IOBuf->buf + IOBuf->BufUsed - 5, "\n000\n", 5) == 0)) 
                        {
-                               StrBufPlain(IOBuf, HKEY("\n000\n"));
-                               StrBufCutRight(Buf, 5);
-                               return Buf->BufUsed;
+                               StrBufPlain(Blob, HKEY("\n000\n"));
+                               StrBufCutRight(Blob, 5);
+                               return Blob->BufUsed;
                        }
                        if (nSelects > 10) {
-                               FlushStrBuf(Buf);
+                               FlushStrBuf(IOBuf);
                                return -1;
                        }
                }
-               nRead += rlen;
-               ptr += rlen;
-               Buf->BufUsed += rlen;
+               if (rlen > 0) {
+                       nRead += rlen;
+                       ptr += rlen;
+                       IOBuf->BufUsed += rlen;
+               }
        }
-       Buf->buf[Buf->BufUsed] = '\0';
+       if (nRead > nBytes) {
+               *Pos = IOBuf->buf + nBytes;
+       }
+       Blob->buf[Blob->BufUsed] = '\0';
+       StrBufAppendBufPlain(Blob, IOBuf->buf, nBytes, 0);
        return nRead;
 }
 
@@ -1916,6 +2159,32 @@ int StrBufDecodeBase64(StrBuf *Buf)
        return siz;
 }
 
+/**
+ * \brief decode a buffer from base 64 encoding; destroys original
+ * \param Buf Buffor to transform
+ */
+int StrBufDecodeHex(StrBuf *Buf)
+{
+       unsigned int ch;
+       char *pch, *pche, *pchi;
+
+       if (Buf == NULL) return -1;
+
+       pch = pchi = Buf->buf;
+       pche = pch + Buf->BufUsed;
+
+       while (pchi < pche){
+               ch = decode_hex(pchi);
+               *pch = ch;
+               pch ++;
+               pchi += 2;
+       }
+
+       *pch = '\0';
+       Buf->BufUsed = pch - Buf->buf;
+       return Buf->BufUsed;
+}
+
 /**
  * \brief replace all chars >0x20 && < 0x7F with Mute
  * \param Mute char to put over invalid chars
@@ -2372,23 +2641,97 @@ void StrBuf_RFC822_to_Utf8(StrBuf *Target, const StrBuf *DecodeMe, const StrBuf*
        FreeStrBuf(&ConvertBuf2);
 }
 
+/**
+ * \brief evaluate the length of an utf8 special character sequence
+ * \param Char the character to examine
+ * \returns width of utf8 chars in bytes
+ */
+static inline int Ctdl_GetUtf8SequenceLength(char *CharS, char *CharE)
+{
+       int n = 1;
+        char test = (1<<7);
+       
+       while ((n < 8) && ((test & *CharS) != 0)) {
+               test = test << 1;
+               n ++;
+       }
+       if ((n > 6) || ((CharE - CharS) > n))
+               n = 1;
+       return n;
+}
 
+/**
+ * \brief detect whether this char starts an utf-8 encoded char
+ * \param Char character to inspect
+ * \returns yes or no
+ */
+static inline int Ctdl_IsUtf8SequenceStart(char Char)
+{
+/** 11??.???? indicates an UTF8 Sequence. */
+       return ((Char & 0xC0) != 0);
+}
 
+/**
+ * \brief measure the number of glyphs in an UTF8 string...
+ * \param str string to measure
+ * \returns the length of str
+ */
 long StrBuf_Utf8StrLen(StrBuf *Buf)
 {
-       return Ctdl_Utf8StrLen(Buf->buf);
+       int n = 0;
+       int m = 0;
+       char *aptr, *eptr;
+
+       if ((Buf == NULL) || (Buf->BufUsed == 0))
+               return 0;
+       aptr = Buf->buf;
+       eptr = Buf->buf + Buf->BufUsed;
+       while ((aptr < eptr) && (*aptr != '\0')) {
+               if (Ctdl_IsUtf8SequenceStart(*aptr)){
+                       m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
+                       while ((aptr < eptr) && (m-- > 0) && (*aptr++ != '\0'))
+                               n ++;
+               }
+               else {
+                       n++;
+                       aptr++;
+               }
+                       
+       }
+       return n;
 }
 
+/**
+ * \brief cuts a string after maxlen glyphs
+ * \param str string to cut to maxlen glyphs
+ * \param maxlen how long may the string become?
+ * \returns pointer to maxlen or the end of the string
+ */
 long StrBuf_Utf8StrCut(StrBuf *Buf, int maxlen)
 {
-       char *CutAt;
+       char *aptr, *eptr;
+       int n = 0, m = 0;
 
-       CutAt = Ctdl_Utf8StrCut(Buf->buf, maxlen);
-       if (CutAt != NULL) {
-               Buf->BufUsed = CutAt - Buf->buf;
-               Buf->buf[Buf->BufUsed] = '\0';
+       aptr = Buf->buf;
+       eptr = Buf->buf + Buf->BufUsed;
+       while ((aptr < eptr) && (*aptr != '\0')) {
+               if (Ctdl_IsUtf8SequenceStart(*aptr)){
+                       m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
+                       while ((m-- > 0) && (*aptr++ != '\0'))
+                               n ++;
+               }
+               else {
+                       n++;
+                       aptr++;
+               }
+               if (n > maxlen) {
+                       *aptr = '\0';
+                       Buf->BufUsed = aptr - Buf->buf;
+                       return Buf->BufUsed;
+               }                       
        }
-       return Buf->BufUsed;    
+       return Buf->BufUsed;
+
 }