libcitadel/lib/stringbuf.c

   1 #include "../sysdep.h"
   2 #include <ctype.h>
   3 #include <errno.h>
   4 #include <string.h>
   5 #include <unistd.h>
   6 #include <string.h>
   7 #include <stdio.h>
   8 #include <sys/select.h>
   9 #include <fcntl.h>
  10 #include <sys/types.h>
  11 #define SHOW_ME_VAPPEND_PRINTF
  12 #include <stdarg.h>
  13 #include "libcitadel.h"
  14
  15 #ifdef HAVE_ICONV
  16 #include <iconv.h>
  17 #endif
  18
  19 #ifdef HAVE_BACKTRACE
  20 #include <execinfo.h>
  21 #endif
  22
  23 #ifdef HAVE_ZLIB
  24 #include <zlib.h>
  25 int ZEXPORT compress_gzip(Bytef * dest, size_t * destLen,
  26                           const Bytef * source, uLong sourceLen, int level);
  27 #endif
  28 int BaseStrBufSize = 64;
  29
  30 const char *StrBufNOTNULL = ((char*) NULL) - 1;
  31
  32 const char HexList[256][3] = {
  33 "00","01","02","03","04","05","06","07","08","09","0A","0B","0C","0D","0E","0F",
  34 "10","11","12","13","14","15","16","17","18","19","1A","1B","1C","1D","1E","1F",
  35 "20","21","22","23","24","25","26","27","28","29","2A","2B","2C","2D","2E","2F",
  36 "30","31","32","33","34","35","36","37","38","39","3A","3B","3C","3D","3E","3F",
  37 "40","41","42","43","44","45","46","47","48","49","4A","4B","4C","4D","4E","4F",
  38 "50","51","52","53","54","55","56","57","58","59","5A","5B","5C","5D","5E","5F",
  39 "60","61","62","63","64","65","66","67","68","69","6A","6B","6C","6D","6E","6F",
  40 "70","71","72","73","74","75","76","77","78","79","7A","7B","7C","7D","7E","7F",
  41 "80","81","82","83","84","85","86","87","88","89","8A","8B","8C","8D","8E","8F",
  42 "90","91","92","93","94","95","96","97","98","99","9A","9B","9C","9D","9E","9F",
  43 "A0","A1","A2","A3","A4","A5","A6","A7","A8","A9","AA","AB","AC","AD","AE","AF",
  44 "B0","B1","B2","B3","B4","B5","B6","B7","B8","B9","BA","BB","BC","BD","BE","BF",
  45 "C0","C1","C2","C3","C4","C5","C6","C7","C8","C9","CA","CB","CC","CD","CE","CF",
  46 "D0","D1","D2","D3","D4","D5","D6","D7","D8","D9","DA","DB","DC","DD","DE","DF",
  47 "E0","E1","E2","E3","E4","E5","E6","E7","E8","E9","EA","EB","EC","ED","EE","EF",
  48 "F0","F1","F2","F3","F4","F5","F6","F7","F8","F9","FA","FB","FC","FD","FE","FF"};
  49
  50 /**
  51  * @defgroup StrBuf Stringbuffer, A class for manipulating strings with dynamic buffers
  52  * StrBuf is a versatile class, aiding the handling of dynamic strings
  53  *  * reduce de/reallocations
  54  *  * reduce the need to remeasure it
  55  *  * reduce scanning over the string (in @ref StrBuf_NextTokenizer "Tokenizers")
  56  *  * allow asyncroneous IO for line and Blob based operations
  57  *  * reduce the use of memove in those
  58  *  * Quick filling in several operations with append functions
  59  */
  60
  61 /**
  62  * @defgroup StrBuf_DeConstructors Create/Destroy StrBufs
  63  * @ingroup StrBuf
  64  */
  65
  66 /**
  67  * @defgroup StrBuf_Cast Cast operators to interact with char* based code
  68  * @ingroup StrBuf
  69  * use these operators to interfere with code demanding char*;
  70  * if you need to own the content, smash me. Avoid, since we loose the length information.
  71  */
  72
  73 /**
  74  * @defgroup StrBuf_Filler Create/Replace/Append Content into a StrBuf
  75  * @ingroup StrBuf
  76  * operations to get your Strings into a StrBuf, manipulating them, or appending
  77  */
  78 /**
  79  * @defgroup StrBuf_NextTokenizer Fast tokenizer to pull tokens in sequence
  80  * @ingroup StrBuf
  81  * Quick tokenizer; demands of the user to pull its tokens in sequence
  82  */
  83
  84 /**
  85  * @defgroup StrBuf_Tokenizer tokenizer Functions; Slow ones.
  86  * @ingroup StrBuf
  87  * versatile tokenizer; random access to tokens, but slower; Prefer the @ref StrBuf_NextTokenizer "Next Tokenizer"
  88  */
  89
  90 /**
  91  * @defgroup StrBuf_BufferedIO Buffered IO with Asynchroneous reads and no unneeded memmoves (the fast ones)
  92  * @ingroup StrBuf
  93  * File IO to fill StrBufs; Works with work-buffer shared across several calls;
  94  * External Cursor to maintain the current read position inside of the buffer
  95  * the non-fast ones will use memove to keep the start of the buffer the read buffer (which is slower)
  96  */
  97
  98 /**
  99  * @defgroup StrBuf_IO FileIO; Prefer @ref StrBuf_BufferedIO
 100  * @ingroup StrBuf
 101  * Slow I/O; avoid.
 102  */
 103
 104 /**
 105  * @defgroup StrBuf_DeEnCoder functions to translate the contents of a buffer
 106  * @ingroup StrBuf
 107  * these functions translate the content of a buffer into another representation;
 108  * some are combined Fillers and encoders
 109  */
 110
 111 /**
 112  * Private Structure for the Stringbuffer
 113  */
 114 struct StrBuf {
 115         char *buf;         /**< the pointer to the dynamic buffer */
 116         long BufSize;      /**< how many spcae do we optain */
 117         long BufUsed;      /**< StNumber of Chars used excluding the trailing \\0 */
 118         int ConstBuf;      /**< are we just a wrapper arround a static buffer and musn't we be changed? */
 119 #ifdef SIZE_DEBUG
 120         long nIncreases;   /**< for profiling; cound how many times we needed more */
 121         char bt [SIZ];     /**< Stacktrace of last increase */
 122         char bt_lastinc [SIZ]; /**< How much did we increase last time? */
 123 #endif
 124 };
 125
 126
 127 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE);
 128 static inline int Ctdl_IsUtf8SequenceStart(const char Char);
 129
 130 #ifdef SIZE_DEBUG
 131 #ifdef HAVE_BACKTRACE
 132 static void StrBufBacktrace(StrBuf *Buf, int which)
 133 {
 134         int n;
 135         char *pstart, *pch;
 136         void *stack_frames[50];
 137         size_t size, i;
 138         char **strings;
 139
 140         if (which)
 141                 pstart = pch = Buf->bt;
 142         else
 143                 pstart = pch = Buf->bt_lastinc;
 144         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
 145         strings = backtrace_symbols(stack_frames, size);
 146         for (i = 0; i < size; i++) {
 147                 if (strings != NULL)
 148                         n = snprintf(pch, SIZ - (pch - pstart), "%s\\n", strings[i]);
 149                 else
 150                         n = snprintf(pch, SIZ - (pch - pstart), "%p\\n", stack_frames[i]);
 151                 pch += n;
 152         }
 153         free(strings);
 154
 155
 156 }
 157 #endif
 158 #endif
 159
 160 /**
 161  * @ingroup StrBuf_Cast
 162  * @brief Cast operator to Plain String
 163  * @note if the buffer is altered by StrBuf operations, this pointer may become
 164  *  invalid. So don't lean on it after altering the buffer!
 165  *  Since this operation is considered cheap, rather call it often than risking
 166  *  your pointer to become invalid!
 167  * @param Str the string we want to get the c-string representation for
 168  * @returns the Pointer to the Content. Don't mess with it!
 169  */
 170 inline const char *ChrPtr(const StrBuf *Str)
 171 {
 172         if (Str == NULL)
 173                 return "";
 174         return Str->buf;
 175 }
 176
 177 /**
 178  * @ingroup StrBuf_Cast
 179  * @brief since we know strlen()'s result, provide it here.
 180  * @param Str the string to return the length to
 181  * @returns contentlength of the buffer
 182  */
 183 inline int StrLength(const StrBuf *Str)
 184 {
 185         return (Str != NULL) ? Str->BufUsed : 0;
 186 }
 187
 188 /**
 189  * @ingroup StrBuf_DeConstructors
 190  * @brief local utility function to resize the buffer
 191  * @param Buf the buffer whichs storage we should increase
 192  * @param KeepOriginal should we copy the original buffer or just start over with a new one
 193  * @param DestSize what should fit in after?
 194  */
 195 static int IncreaseBuf(StrBuf *Buf, int KeepOriginal, int DestSize)
 196 {
 197         char *NewBuf;
 198         size_t NewSize = Buf->BufSize * 2;
 199
 200         if (Buf->ConstBuf)
 201                 return -1;
 202
 203         if (DestSize > 0)
 204                 while (NewSize <= DestSize)
 205                         NewSize *= 2;
 206
 207         NewBuf= (char*) malloc(NewSize);
 208         if (NewBuf == NULL)
 209                 return -1;
 210
 211         if (KeepOriginal && (Buf->BufUsed > 0))
 212         {
 213                 memcpy(NewBuf, Buf->buf, Buf->BufUsed);
 214         }
 215         else
 216         {
 217                 NewBuf[0] = '\0';
 218                 Buf->BufUsed = 0;
 219         }
 220         free (Buf->buf);
 221         Buf->buf = NewBuf;
 222         Buf->BufSize = NewSize;
 223 #ifdef SIZE_DEBUG
 224         Buf->nIncreases++;
 225 #ifdef HAVE_BACKTRACE
 226         StrBufBacktrace(Buf, 1);
 227 #endif
 228 #endif
 229         return Buf->BufSize;
 230 }
 231
 232 /**
 233  * @ingroup StrBuf_DeConstructors
 234  * @brief shrink an _EMPTY_ buffer if its Buffer superseeds threshhold to NewSize. Buffercontent is thoroughly ignored and flushed.
 235  * @param Buf Buffer to shrink (has to be empty)
 236  * @param ThreshHold if the buffer is bigger then this, its readjusted
 237  * @param NewSize if we Shrink it, how big are we going to be afterwards?
 238  */
 239 void ReAdjustEmptyBuf(StrBuf *Buf, long ThreshHold, long NewSize)
 240 {
 241         if (Buf->BufUsed > ThreshHold) {
 242                 free(Buf->buf);
 243                 Buf->buf = (char*) malloc(NewSize);
 244                 Buf->BufUsed = 0;
 245                 Buf->BufSize = NewSize;
 246         }
 247 }
 248
 249 /**
 250  * @ingroup StrBuf_DeConstructors
 251  * @brief shrink long term buffers to their real size so they don't waste memory
 252  * @param Buf buffer to shrink
 253  * @param Force if not set, will just executed if the buffer is much to big; set for lifetime strings
 254  * @returns physical size of the buffer
 255  */
 256 long StrBufShrinkToFit(StrBuf *Buf, int Force)
 257 {
 258         if (Force ||
 259             (Buf->BufUsed + (Buf->BufUsed / 3) > Buf->BufSize))
 260         {
 261                 char *TmpBuf = (char*) malloc(Buf->BufUsed + 1);
 262                 memcpy (TmpBuf, Buf->buf, Buf->BufUsed + 1);
 263                 Buf->BufSize = Buf->BufUsed + 1;
 264                 free(Buf->buf);
 265                 Buf->buf = TmpBuf;
 266         }
 267         return Buf->BufUsed;
 268 }
 269
 270 /**
 271  * @ingroup StrBuf_DeConstructors
 272  * @brief Allocate a new buffer with default buffer size
 273  * @returns the new stringbuffer
 274  */
 275 StrBuf* NewStrBuf(void)
 276 {
 277         StrBuf *NewBuf;
 278
 279         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 280         NewBuf->buf = (char*) malloc(BaseStrBufSize);
 281         NewBuf->buf[0] = '\0';
 282         NewBuf->BufSize = BaseStrBufSize;
 283         NewBuf->BufUsed = 0;
 284         NewBuf->ConstBuf = 0;
 285 #ifdef SIZE_DEBUG
 286         NewBuf->nIncreases = 0;
 287         NewBuf->bt[0] = '\0';
 288         NewBuf->bt_lastinc[0] = '\0';
 289 #ifdef HAVE_BACKTRACE
 290         StrBufBacktrace(NewBuf, 0);
 291 #endif
 292 #endif
 293         return NewBuf;
 294 }
 295
 296 /**
 297  * @ingroup StrBuf_DeConstructors
 298  * @brief Copy Constructor; returns a duplicate of CopyMe
 299  * @param CopyMe Buffer to faxmilate
 300  * @returns the new stringbuffer
 301  */
 302 StrBuf* NewStrBufDup(const StrBuf *CopyMe)
 303 {
 304         StrBuf *NewBuf;
 305
 306         if (CopyMe == NULL)
 307                 return NewStrBuf();
 308
 309         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 310         NewBuf->buf = (char*) malloc(CopyMe->BufSize);
 311         memcpy(NewBuf->buf, CopyMe->buf, CopyMe->BufUsed + 1);
 312         NewBuf->BufUsed = CopyMe->BufUsed;
 313         NewBuf->BufSize = CopyMe->BufSize;
 314         NewBuf->ConstBuf = 0;
 315 #ifdef SIZE_DEBUG
 316         NewBuf->nIncreases = 0;
 317         NewBuf->bt[0] = '\0';
 318         NewBuf->bt_lastinc[0] = '\0';
 319 #ifdef HAVE_BACKTRACE
 320         StrBufBacktrace(NewBuf, 0);
 321 #endif
 322 #endif
 323         return NewBuf;
 324 }
 325
 326 /**
 327  * @ingroup StrBuf_DeConstructors
 328  * @brief create a new Buffer using an existing c-string
 329  * this function should also be used if you want to pre-suggest
 330  * the buffer size to allocate in conjunction with ptr == NULL
 331  * @param ptr the c-string to copy; may be NULL to create a blank instance
 332  * @param nChars How many chars should we copy; -1 if we should measure the length ourselves
 333  * @returns the new stringbuffer
 334  */
 335 StrBuf* NewStrBufPlain(const char* ptr, int nChars)
 336 {
 337         StrBuf *NewBuf;
 338         size_t Siz = BaseStrBufSize;
 339         size_t CopySize;
 340
 341         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 342         if (nChars < 0)
 343                 CopySize = strlen((ptr != NULL)?ptr:"");
 344         else
 345                 CopySize = nChars;
 346
 347         while (Siz <= CopySize)
 348                 Siz *= 2;
 349
 350         NewBuf->buf = (char*) malloc(Siz);
 351         NewBuf->BufSize = Siz;
 352         if (ptr != NULL) {
 353                 memcpy(NewBuf->buf, ptr, CopySize);
 354                 NewBuf->buf[CopySize] = '\0';
 355                 NewBuf->BufUsed = CopySize;
 356         }
 357         else {
 358                 NewBuf->buf[0] = '\0';
 359                 NewBuf->BufUsed = 0;
 360         }
 361         NewBuf->ConstBuf = 0;
 362 #ifdef SIZE_DEBUG
 363         NewBuf->nIncreases = 0;
 364         NewBuf->bt[0] = '\0';
 365         NewBuf->bt_lastinc[0] = '\0';
 366 #ifdef HAVE_BACKTRACE
 367         StrBufBacktrace(NewBuf, 0);
 368 #endif
 369 #endif
 370         return NewBuf;
 371 }
 372
 373 /**
 374  * @ingroup StrBuf_DeConstructors
 375  * @brief Set an existing buffer from a c-string
 376  * @param Buf buffer to load
 377  * @param ptr c-string to put into
 378  * @param nChars set to -1 if we should work 0-terminated
 379  * @returns the new length of the string
 380  */
 381 int StrBufPlain(StrBuf *Buf, const char* ptr, int nChars)
 382 {
 383         size_t Siz = Buf->BufSize;
 384         size_t CopySize;
 385
 386         if (nChars < 0)
 387                 CopySize = strlen(ptr);
 388         else
 389                 CopySize = nChars;
 390
 391         while (Siz <= CopySize)
 392                 Siz *= 2;
 393
 394         if (Siz != Buf->BufSize)
 395                 IncreaseBuf(Buf, 0, Siz);
 396         memcpy(Buf->buf, ptr, CopySize);
 397         Buf->buf[CopySize] = '\0';
 398         Buf->BufUsed = CopySize;
 399         Buf->ConstBuf = 0;
 400         return CopySize;
 401 }
 402
 403
 404 /**
 405  * @ingroup StrBuf_DeConstructors
 406  * @brief use strbuf as wrapper for a string constant for easy handling
 407  * @param StringConstant a string to wrap
 408  * @param SizeOfStrConstant should be sizeof(StringConstant)-1
 409  */
 410 StrBuf* _NewConstStrBuf(const char* StringConstant, size_t SizeOfStrConstant)
 411 {
 412         StrBuf *NewBuf;
 413
 414         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 415         NewBuf->buf = (char*) StringConstant;
 416         NewBuf->BufSize = SizeOfStrConstant;
 417         NewBuf->BufUsed = SizeOfStrConstant;
 418         NewBuf->ConstBuf = 1;
 419 #ifdef SIZE_DEBUG
 420         NewBuf->nIncreases = 0;
 421         NewBuf->bt[0] = '\0';
 422         NewBuf->bt_lastinc[0] = '\0';
 423 #endif
 424         return NewBuf;
 425 }
 426
 427
 428 /**
 429  * @ingroup StrBuf_DeConstructors
 430  * @brief flush the content of a Buf; keep its struct
 431  * @param buf Buffer to flush
 432  */
 433 int FlushStrBuf(StrBuf *buf)
 434 {
 435         if (buf == NULL)
 436                 return -1;
 437         if (buf->ConstBuf)
 438                 return -1;
 439         buf->buf[0] ='\0';
 440         buf->BufUsed = 0;
 441         return 0;
 442 }
 443
 444 /**
 445  * @ingroup StrBuf_DeConstructors
 446  * @brief wipe the content of a Buf thoroughly (overwrite it -> expensive); keep its struct
 447  * @param buf Buffer to wipe
 448  */
 449 int FLUSHStrBuf(StrBuf *buf)
 450 {
 451         if (buf == NULL)
 452                 return -1;
 453         if (buf->ConstBuf)
 454                 return -1;
 455         if (buf->BufUsed > 0) {
 456                 memset(buf->buf, 0, buf->BufUsed);
 457                 buf->BufUsed = 0;
 458         }
 459         return 0;
 460 }
 461
 462 #ifdef SIZE_DEBUG
 463 int hFreeDbglog = -1;
 464 #endif
 465 /**
 466  * @ingroup StrBuf_DeConstructors
 467  * @brief Release a Buffer
 468  * Its a double pointer, so it can NULL your pointer
 469  * so fancy SIG11 appear instead of random results
 470  * @param FreeMe Pointer Pointer to the buffer to free
 471  */
 472 void FreeStrBuf (StrBuf **FreeMe)
 473 {
 474         if (*FreeMe == NULL)
 475                 return;
 476 #ifdef SIZE_DEBUG
 477         if (hFreeDbglog == -1){
 478                 pid_t pid = getpid();
 479                 char path [SIZ];
 480                 snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
 481                 hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
 482         }
 483         if ((*FreeMe)->nIncreases > 0)
 484         {
 485                 char buf[SIZ * 3];
 486                 long n;
 487                 n = snprintf(buf, SIZ * 3, "+|%ld|%ld|%ld|%s|%s|\n",
 488                              (*FreeMe)->nIncreases,
 489                              (*FreeMe)->BufUsed,
 490                              (*FreeMe)->BufSize,
 491                              (*FreeMe)->bt,
 492                              (*FreeMe)->bt_lastinc);
 493                 n = write(hFreeDbglog, buf, n);
 494         }
 495         else
 496         {
 497                 char buf[128];
 498                 long n;
 499                 n = snprintf(buf, 128, "_|0|%ld%ld|\n",
 500                              (*FreeMe)->BufUsed,
 501                              (*FreeMe)->BufSize);
 502                 n = write(hFreeDbglog, buf, n);
 503         }
 504 #endif
 505         if (!(*FreeMe)->ConstBuf)
 506                 free((*FreeMe)->buf);
 507         free(*FreeMe);
 508         *FreeMe = NULL;
 509 }
 510
 511 /**
 512  * @ingroup StrBuf_DeConstructors
 513  * @brief flatten a Buffer to the Char * we return
 514  * Its a double pointer, so it can NULL your pointer
 515  * so fancy SIG11 appear instead of random results
 516  * The Callee then owns the buffer and is responsible for freeing it.
 517  * @param SmashMe Pointer Pointer to the buffer to release Buf from and free
 518  * @returns the pointer of the buffer; Callee owns the memory thereafter.
 519  */
 520 char *SmashStrBuf (StrBuf **SmashMe)
 521 {
 522         char *Ret;
 523
 524         if (*SmashMe == NULL)
 525                 return NULL;
 526 #ifdef SIZE_DEBUG
 527         if (hFreeDbglog == -1){
 528                 pid_t pid = getpid();
 529                 char path [SIZ];
 530                 snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
 531                 hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
 532         }
 533         if ((*SmashMe)->nIncreases > 0)
 534         {
 535                 char buf[SIZ * 3];
 536                 long n;
 537                 n = snprintf(buf, SIZ * 3, "S+|%ld|%ld|%ld|%s|%s|\n",
 538                              (*SmashMe)->nIncreases,
 539                              (*SmashMe)->BufUsed,
 540                              (*SmashMe)->BufSize,
 541                              (*SmashMe)->bt,
 542                              (*SmashMe)->bt_lastinc);
 543                 n = write(hFreeDbglog, buf, n);
 544         }
 545         else
 546         {
 547                 char buf[128];
 548                 long n;
 549                 n = snprintf(buf, 128, "S_|0|%ld%ld|\n",
 550                              (*SmashMe)->BufUsed,
 551                              (*SmashMe)->BufSize);
 552                 n = write(hFreeDbglog, buf, n);
 553         }
 554 #endif
 555         Ret = (*SmashMe)->buf;
 556         free(*SmashMe);
 557         *SmashMe = NULL;
 558         return Ret;
 559 }
 560
 561 /**
 562  * @ingroup StrBuf_DeConstructors
 563  * @brief Release the buffer
 564  * If you want put your StrBuf into a Hash, use this as Destructor.
 565  * @param VFreeMe untyped pointer to a StrBuf. be shure to do the right thing [TM]
 566  */
 567 void HFreeStrBuf (void *VFreeMe)
 568 {
 569         StrBuf *FreeMe = (StrBuf*)VFreeMe;
 570         if (FreeMe == NULL)
 571                 return;
 572 #ifdef SIZE_DEBUG
 573         if (hFreeDbglog == -1){
 574                 pid_t pid = getpid();
 575                 char path [SIZ];
 576                 snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
 577                 hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
 578         }
 579         if (FreeMe->nIncreases > 0)
 580         {
 581                 char buf[SIZ * 3];
 582                 long n;
 583                 n = snprintf(buf, SIZ * 3, "+|%ld|%ld|%ld|%s|%s|\n",
 584                              FreeMe->nIncreases,
 585                              FreeMe->BufUsed,
 586                              FreeMe->BufSize,
 587                              FreeMe->bt,
 588                              FreeMe->bt_lastinc);
 589                 write(hFreeDbglog, buf, n);
 590         }
 591         else
 592         {
 593                 char buf[128];
 594                 long n;
 595                 n = snprintf(buf, 128, "_|%ld|%ld%ld|\n",
 596                              FreeMe->nIncreases,
 597                              FreeMe->BufUsed,
 598                              FreeMe->BufSize);
 599         }
 600 #endif
 601         if (!FreeMe->ConstBuf)
 602                 free(FreeMe->buf);
 603         free(FreeMe);
 604 }
 605
 606 /**
 607  * @ingroup StrBuf
 608  * @brief Wrapper around atol
 609  */
 610 long StrTol(const StrBuf *Buf)
 611 {
 612         if (Buf == NULL)
 613                 return 0;
 614         if(Buf->BufUsed > 0)
 615                 return atol(Buf->buf);
 616         else
 617                 return 0;
 618 }
 619
 620 /**
 621  * @ingroup StrBuf
 622  * @brief Wrapper around atoi
 623  */
 624 int StrToi(const StrBuf *Buf)
 625 {
 626         if (Buf == NULL)
 627                 return 0;
 628         if (Buf->BufUsed > 0)
 629                 return atoi(Buf->buf);
 630         else
 631                 return 0;
 632 }
 633
 634 /**
 635  * @ingroup StrBuf
 636  * @brief Checks to see if the string is a pure number
 637  */
 638 int StrBufIsNumber(const StrBuf *Buf) {
 639   char * pEnd;
 640   if (Buf == NULL) {
 641         return 0;
 642   }
 643   strtoll(Buf->buf, &pEnd, 10);
 644   if (pEnd == NULL && ((Buf->buf)-pEnd) != 0) {
 645     return 1;
 646   }
 647   return 0;
 648 }
 649 /**
 650  * @ingroup StrBuf
 651  * @brief modifies a Single char of the Buf
 652  * You can point to it via char* or a zero-based integer
 653  * @param Buf The buffer to manipulate
 654  * @param ptr char* to zero; use NULL if unused
 655  * @param nThChar zero based pointer into the string; use -1 if unused
 656  * @param PeekValue The Character to place into the position
 657  */
 658 long StrBufPeek(StrBuf *Buf, const char* ptr, long nThChar, char PeekValue)
 659 {
 660         if (Buf == NULL)
 661                 return -1;
 662         if (ptr != NULL)
 663                 nThChar = ptr - Buf->buf;
 664         if ((nThChar < 0) || (nThChar > Buf->BufUsed))
 665                 return -1;
 666         Buf->buf[nThChar] = PeekValue;
 667         return nThChar;
 668 }
 669
 670 /**
 671  * @ingroup StrBuf
 672  * @brief Append a StringBuffer to the buffer
 673  * @param Buf Buffer to modify
 674  * @param AppendBuf Buffer to copy at the end of our buffer
 675  * @param Offset Should we start copying from an offset?
 676  */
 677 void StrBufAppendBuf(StrBuf *Buf, const StrBuf *AppendBuf, unsigned long Offset)
 678 {
 679         if ((AppendBuf == NULL) || (Buf == NULL) || (AppendBuf->buf == NULL))
 680                 return;
 681
 682         if (Buf->BufSize - Offset < AppendBuf->BufUsed + Buf->BufUsed + 1)
 683                 IncreaseBuf(Buf,
 684                             (Buf->BufUsed > 0),
 685                             AppendBuf->BufUsed + Buf->BufUsed);
 686
 687         memcpy(Buf->buf + Buf->BufUsed,
 688                AppendBuf->buf + Offset,
 689                AppendBuf->BufUsed - Offset);
 690         Buf->BufUsed += AppendBuf->BufUsed - Offset;
 691         Buf->buf[Buf->BufUsed] = '\0';
 692 }
 693
 694
 695 /**
 696  * @ingroup StrBuf
 697  * @brief Append a C-String to the buffer
 698  * @param Buf Buffer to modify
 699  * @param AppendBuf Buffer to copy at the end of our buffer
 700  * @param AppendSize number of bytes to copy; set to -1 if we should count it in advance
 701  * @param Offset Should we start copying from an offset?
 702  */
 703 void StrBufAppendBufPlain(StrBuf *Buf, const char *AppendBuf, long AppendSize, unsigned long Offset)
 704 {
 705         long aps;
 706         long BufSizeRequired;
 707
 708         if ((AppendBuf == NULL) || (Buf == NULL))
 709                 return;
 710
 711         if (AppendSize < 0 )
 712                 aps = strlen(AppendBuf + Offset);
 713         else
 714                 aps = AppendSize - Offset;
 715
 716         BufSizeRequired = Buf->BufUsed + aps + 1;
 717         if (Buf->BufSize <= BufSizeRequired)
 718                 IncreaseBuf(Buf, (Buf->BufUsed > 0), BufSizeRequired);
 719
 720         memcpy(Buf->buf + Buf->BufUsed,
 721                AppendBuf + Offset,
 722                aps);
 723         Buf->BufUsed += aps;
 724         Buf->buf[Buf->BufUsed] = '\0';
 725 }
 726
 727 /**
 728  * @ingroup StrBuf
 729  * @brief Callback for cURL to append the webserver reply to a buffer
 730  * @param ptr pre-defined by the cURL API; see man 3 curl for mre info
 731  * @param size pre-defined by the cURL API; see man 3 curl for mre info
 732  * @param nmemb pre-defined by the cURL API; see man 3 curl for mre info
 733  * @param stream pre-defined by the cURL API; see man 3 curl for mre info
 734  */
 735 size_t CurlFillStrBuf_callback(void *ptr, size_t size, size_t nmemb, void *stream)
 736 {
 737
 738         StrBuf *Target;
 739
 740         Target = stream;
 741         if (ptr == NULL)
 742                 return 0;
 743
 744         StrBufAppendBufPlain(Target, ptr, size * nmemb, 0);
 745         return size * nmemb;
 746 }
 747
 748
 749 /**
 750  * @ingroup StrBuf_DeEnCoder
 751  * @brief Escape a string for feeding out as a URL while appending it to a Buffer
 752  * @param OutBuf the output buffer
 753  * @param In Buffer to encode
 754  * @param PlainIn way in from plain old c strings
 755  */
 756 void StrBufUrlescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
 757 {
 758         const char *pch, *pche;
 759         char *pt, *pte;
 760         int len;
 761
 762         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
 763                 return;
 764         if (PlainIn != NULL) {
 765                 len = strlen(PlainIn);
 766                 pch = PlainIn;
 767                 pche = pch + len;
 768         }
 769         else {
 770                 pch = In->buf;
 771                 pche = pch + In->BufUsed;
 772                 len = In->BufUsed;
 773         }
 774
 775         if (len == 0)
 776                 return;
 777
 778         pt = OutBuf->buf + OutBuf->BufUsed;
 779         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
 780
 781         while (pch < pche) {
 782                 if (pt >= pte) {
 783                         IncreaseBuf(OutBuf, 1, -1);
 784                         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
 785                         pt = OutBuf->buf + OutBuf->BufUsed;
 786                 }
 787
 788                 if((*pch >= 'a' && *pch <= 'z') ||
 789                    (*pch >= '@' && *pch <= 'Z') || /* @ A-Z */
 790                    (*pch >= '0' && *pch <= ':') || /* 0-9 : */
 791                    (*pch == '!') || (*pch == '_') ||
 792                    (*pch == ',') || (*pch == '.') ||
 793                    (*pch == ','))
 794                 {
 795                         *(pt++) = *(pch++);
 796                         OutBuf->BufUsed++;
 797                 }
 798                 else {
 799                         *pt = '%';
 800                         *(pt + 1) = HexList[(unsigned char)*pch][0];
 801                         *(pt + 2) = HexList[(unsigned char)*pch][1];
 802                         pt += 3;
 803                         OutBuf->BufUsed += 3;
 804                         pch ++;
 805                 }
 806         }
 807         *pt = '\0';
 808 }
 809
 810 /**
 811  * @ingroup StrBuf_DeEnCoder
 812  * @brief Append a string, escaping characters which have meaning in HTML.
 813  *
 814  * @param Target        target buffer
 815  * @param Source        source buffer; set to NULL if you just have a C-String
 816  * @param PlainIn       Plain-C string to append; set to NULL if unused
 817  * @param nbsp          If nonzero, spaces are converted to non-breaking spaces.
 818  * @param nolinebreaks  if set to 1, linebreaks are removed from the string.
 819  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
 820  */
 821 long StrEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
 822 {
 823         const char *aptr, *eiptr;
 824         char *bptr, *eptr;
 825         long len;
 826
 827         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
 828                 return -1;
 829
 830         if (PlainIn != NULL) {
 831                 aptr = PlainIn;
 832                 len = strlen(PlainIn);
 833                 eiptr = aptr + len;
 834         }
 835         else {
 836                 aptr = Source->buf;
 837                 eiptr = aptr + Source->BufUsed;
 838                 len = Source->BufUsed;
 839         }
 840
 841         if (len == 0)
 842                 return -1;
 843
 844         bptr = Target->buf + Target->BufUsed;
 845         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
 846
 847         while (aptr < eiptr){
 848                 if(bptr >= eptr) {
 849                         IncreaseBuf(Target, 1, -1);
 850                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
 851                         bptr = Target->buf + Target->BufUsed;
 852                 }
 853                 if (*aptr == '<') {
 854                         memcpy(bptr, "&lt;", 4);
 855                         bptr += 4;
 856                         Target->BufUsed += 4;
 857                 }
 858                 else if (*aptr == '>') {
 859                         memcpy(bptr, "&gt;", 4);
 860                         bptr += 4;
 861                         Target->BufUsed += 4;
 862                 }
 863                 else if (*aptr == '&') {
 864                         memcpy(bptr, "&amp;", 5);
 865                         bptr += 5;
 866                         Target->BufUsed += 5;
 867                 }
 868                 else if (*aptr == '"') {
 869                         memcpy(bptr, "&quot;", 6);
 870                         bptr += 6;
 871                         Target->BufUsed += 6;
 872                 }
 873                 else if (*aptr == '\'') {
 874                         memcpy(bptr, "&#39;", 5);
 875                         bptr += 5;
 876                         Target->BufUsed += 5;
 877                 }
 878                 else if (*aptr == LB) {
 879                         *bptr = '<';
 880                         bptr ++;
 881                         Target->BufUsed ++;
 882                 }
 883                 else if (*aptr == RB) {
 884                         *bptr = '>';
 885                         bptr ++;
 886                         Target->BufUsed ++;
 887                 }
 888                 else if (*aptr == QU) {
 889                         *bptr ='"';
 890                         bptr ++;
 891                         Target->BufUsed ++;
 892                 }
 893                 else if ((*aptr == 32) && (nbsp == 1)) {
 894                         memcpy(bptr, "&nbsp;", 6);
 895                         bptr += 6;
 896                         Target->BufUsed += 6;
 897                 }
 898                 else if ((*aptr == '\n') && (nolinebreaks == 1)) {
 899                         *bptr='\0';     /* nothing */
 900                 }
 901                 else if ((*aptr == '\n') && (nolinebreaks == 2)) {
 902                         memcpy(bptr, "&lt;br/&gt;", 11);
 903                         bptr += 11;
 904                         Target->BufUsed += 11;
 905                 }
 906
 907
 908                 else if ((*aptr == '\r') && (nolinebreaks != 0)) {
 909                         *bptr='\0';     /* nothing */
 910                 }
 911                 else{
 912                         *bptr = *aptr;
 913                         bptr++;
 914                         Target->BufUsed ++;
 915                 }
 916                 aptr ++;
 917         }
 918         *bptr = '\0';
 919         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
 920                 return -1;
 921         return Target->BufUsed;
 922 }
 923
 924 /**
 925  * @ingroup StrBuf_DeEnCoder
 926  * @brief Append a string, escaping characters which have meaning in HTML.
 927  * Converts linebreaks into blanks; escapes single quotes
 928  * @param Target        target buffer
 929  * @param Source        source buffer; set to NULL if you just have a C-String
 930  * @param PlainIn       Plain-C string to append; set to NULL if unused
 931  */
 932 void StrMsgEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
 933 {
 934         const char *aptr, *eiptr;
 935         char *tptr, *eptr;
 936         long len;
 937
 938         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
 939                 return ;
 940
 941         if (PlainIn != NULL) {
 942                 aptr = PlainIn;
 943                 len = strlen(PlainIn);
 944                 eiptr = aptr + len;
 945         }
 946         else {
 947                 aptr = Source->buf;
 948                 eiptr = aptr + Source->BufUsed;
 949                 len = Source->BufUsed;
 950         }
 951
 952         if (len == 0)
 953                 return;
 954
 955         eptr = Target->buf + Target->BufSize - 8;
 956         tptr = Target->buf + Target->BufUsed;
 957
 958         while (aptr < eiptr){
 959                 if(tptr >= eptr) {
 960                         IncreaseBuf(Target, 1, -1);
 961                         eptr = Target->buf + Target->BufSize - 8;
 962                         tptr = Target->buf + Target->BufUsed;
 963                 }
 964
 965                 if (*aptr == '\n') {
 966                         *tptr = ' ';
 967                         Target->BufUsed++;
 968                 }
 969                 else if (*aptr == '\r') {
 970                         *tptr = ' ';
 971                         Target->BufUsed++;
 972                 }
 973                 else if (*aptr == '\'') {
 974                         *(tptr++) = '&';
 975                         *(tptr++) = '#';
 976                         *(tptr++) = '3';
 977                         *(tptr++) = '9';
 978                         *tptr = ';';
 979                         Target->BufUsed += 5;
 980                 } else {
 981                         *tptr = *aptr;
 982                         Target->BufUsed++;
 983                 }
 984                 tptr++; aptr++;
 985         }
 986         *tptr = '\0';
 987 }
 988
 989
 990
 991 /**
 992  * @ingroup StrBuf_DeEnCoder
 993  * @brief Append a string, escaping characters which have meaning in ICAL.
 994  * [\n,]
 995  * @param Target        target buffer
 996  * @param Source        source buffer; set to NULL if you just have a C-String
 997  * @param PlainIn       Plain-C string to append; set to NULL if unused
 998  */
 999 void StrIcalEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
1000 {
1001         const char *aptr, *eiptr;
1002         char *tptr, *eptr;
1003         long len;
1004
1005         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
1006                 return ;
1007
1008         if (PlainIn != NULL) {
1009                 aptr = PlainIn;
1010                 len = strlen(PlainIn);
1011                 eiptr = aptr + len;
1012         }
1013         else {
1014                 aptr = Source->buf;
1015                 eiptr = aptr + Source->BufUsed;
1016                 len = Source->BufUsed;
1017         }
1018
1019         if (len == 0)
1020                 return;
1021
1022         eptr = Target->buf + Target->BufSize - 8;
1023         tptr = Target->buf + Target->BufUsed;
1024
1025         while (aptr < eiptr){
1026                 if(tptr + 3 >= eptr) {
1027                         IncreaseBuf(Target, 1, -1);
1028                         eptr = Target->buf + Target->BufSize - 8;
1029                         tptr = Target->buf + Target->BufUsed;
1030                 }
1031
1032                 if (*aptr == '\n') {
1033                         *tptr = '\\';
1034                         Target->BufUsed++;
1035                         tptr++;
1036                         *tptr = 'n';
1037                         Target->BufUsed++;
1038                 }
1039                 else if (*aptr == '\r') {
1040                         *tptr = '\\';
1041                         Target->BufUsed++;
1042                         tptr++;
1043                         *tptr = 'r';
1044                         Target->BufUsed++;
1045                 }
1046                 else if (*aptr == ',') {
1047                         *tptr = '\\';
1048                         Target->BufUsed++;
1049                         tptr++;
1050                         *tptr = ',';
1051                         Target->BufUsed++;
1052                 } else {
1053                         *tptr = *aptr;
1054                         Target->BufUsed++;
1055                 }
1056                 tptr++; aptr++;
1057         }
1058         *tptr = '\0';
1059 }
1060
1061 /**
1062  * @ingroup StrBuf_DeEnCoder
1063  * @brief Append a string, escaping characters which have meaning in JavaScript strings .
1064  *
1065  * @param Target        target buffer
1066  * @param Source        source buffer; set to NULL if you just have a C-String
1067  * @param PlainIn       Plain-C string to append; set to NULL if unused
1068  * @returns size of result or -1
1069  */
1070 long StrECMAEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
1071 {
1072         const char *aptr, *eiptr;
1073         char *bptr, *eptr;
1074         long len;
1075
1076         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
1077                 return -1;
1078
1079         if (PlainIn != NULL) {
1080                 aptr = PlainIn;
1081                 len = strlen(PlainIn);
1082                 eiptr = aptr + len;
1083         }
1084         else {
1085                 aptr = Source->buf;
1086                 eiptr = aptr + Source->BufUsed;
1087                 len = Source->BufUsed;
1088         }
1089
1090         if (len == 0)
1091                 return -1;
1092
1093         bptr = Target->buf + Target->BufUsed;
1094         eptr = Target->buf + Target->BufSize - 3; /* our biggest unit to put in...  */
1095
1096         while (aptr < eiptr){
1097                 if(bptr >= eptr) {
1098                         IncreaseBuf(Target, 1, -1);
1099                         eptr = Target->buf + Target->BufSize - 3;
1100                         bptr = Target->buf + Target->BufUsed;
1101                 }
1102                 if (*aptr == '"') {
1103                         *bptr = '\\';
1104                         bptr ++;
1105                         *bptr = '"';
1106                         bptr ++;
1107                         Target->BufUsed += 2;
1108                 } else if (*aptr == '\\') {
1109                         *bptr = '\\';
1110                         bptr ++;
1111                         *bptr = '\\';
1112                         bptr ++;
1113                         Target->BufUsed += 2;
1114                 }
1115                 else{
1116                         *bptr = *aptr;
1117                         bptr++;
1118                         Target->BufUsed ++;
1119                 }
1120                 aptr ++;
1121         }
1122         *bptr = '\0';
1123         if ((bptr == eptr - 1 ) && !IsEmptyStr(aptr) )
1124                 return -1;
1125         return Target->BufUsed;
1126 }
1127
1128 /**
1129  * @ingroup StrBuf_DeEnCoder
1130  * @brief Append a string, escaping characters which have meaning in HTML + json.
1131  *
1132  * @param Target        target buffer
1133  * @param Source        source buffer; set to NULL if you just have a C-String
1134  * @param PlainIn       Plain-C string to append; set to NULL if unused
1135  * @param nbsp          If nonzero, spaces are converted to non-breaking spaces.
1136  * @param nolinebreaks  if set to 1, linebreaks are removed from the string.
1137  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
1138  */
1139 long StrHtmlEcmaEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
1140 {
1141         const char *aptr, *eiptr;
1142         char *bptr, *eptr;
1143         long len;
1144         int IsUtf8Sequence = 0;
1145
1146         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
1147                 return -1;
1148
1149         if (PlainIn != NULL) {
1150                 aptr = PlainIn;
1151                 len = strlen(PlainIn);
1152                 eiptr = aptr + len;
1153         }
1154         else {
1155                 aptr = Source->buf;
1156                 eiptr = aptr + Source->BufUsed;
1157                 len = Source->BufUsed;
1158         }
1159
1160         if (len == 0)
1161                 return -1;
1162
1163         bptr = Target->buf + Target->BufUsed;
1164         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
1165
1166         while (aptr < eiptr){
1167                 if(bptr >= eptr) {
1168                         IncreaseBuf(Target, 1, -1);
1169                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
1170                         bptr = Target->buf + Target->BufUsed;
1171                 }
1172                 if (*aptr == '<') {
1173                         memcpy(bptr, "&lt;", 4);
1174                         bptr += 4;
1175                         Target->BufUsed += 4;
1176                 }
1177                 else if (*aptr == '>') {
1178                         memcpy(bptr, "&gt;", 4);
1179                         bptr += 4;
1180                         Target->BufUsed += 4;
1181                 }
1182                 else if (*aptr == '&') {
1183                         memcpy(bptr, "&amp;", 5);
1184                         bptr += 5;
1185                         Target->BufUsed += 5;
1186                 }
1187                 else if (*aptr == LB) {
1188                         *bptr = '<';
1189                         bptr ++;
1190                         Target->BufUsed ++;
1191                 }
1192                 else if (*aptr == RB) {
1193                         *bptr = '>';
1194                         bptr ++;
1195                         Target->BufUsed ++;
1196                 }
1197                 else if ((*aptr == 32) && (nbsp == 1)) {
1198                         memcpy(bptr, "&nbsp;", 6);
1199                         bptr += 6;
1200                         Target->BufUsed += 6;
1201                 }
1202                 else if ((*aptr == '\n') && (nolinebreaks == 1)) {
1203                         *bptr='\0';     /* nothing */
1204                 }
1205                 else if ((*aptr == '\n') && (nolinebreaks == 2)) {
1206                         memcpy(bptr, "&lt;br/&gt;", 11);
1207                         bptr += 11;
1208                         Target->BufUsed += 11;
1209                 }
1210
1211                 else if ((*aptr == '\r') && (nolinebreaks != 0)) {
1212                         *bptr='\0';     /* nothing */
1213                 }
1214
1215                 else if ((*aptr == '"') || (*aptr == QU)) {
1216                         *bptr = '\\';
1217                         bptr ++;
1218                         *bptr = '"';
1219                         bptr ++;
1220                         Target->BufUsed += 2;
1221                 } else if (*aptr == '\\') {
1222                         *bptr = '\\';
1223                         bptr ++;
1224                         *bptr = '\\';
1225                         bptr ++;
1226                         Target->BufUsed += 2;
1227                 }
1228                 else {
1229                         if (((unsigned char)*aptr) >= 0x20)
1230                         {
1231                                 IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(aptr, eiptr);
1232
1233                                 *bptr = *aptr;
1234                                 Target->BufUsed ++;
1235                                 while (IsUtf8Sequence > 1){
1236                                         if(bptr + IsUtf8Sequence >= eptr) {
1237                                                 IncreaseBuf(Target, 1, -1);
1238                                                 eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
1239                                                 bptr = Target->buf + Target->BufUsed - 1;
1240                                         }
1241                                         bptr++; aptr++;
1242                                         IsUtf8Sequence --;
1243                                         *bptr = *aptr;
1244                                         Target->BufUsed ++;
1245                                 }
1246                                 bptr++;
1247                         }
1248
1249                 }
1250                 aptr ++;
1251         }
1252         *bptr = '\0';
1253         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
1254                 return -1;
1255         return Target->BufUsed;
1256 }
1257
1258
1259 /**
1260  * @ingroup StrBuf
1261  * @brief extracts a substring from Source into dest
1262  * @param dest buffer to place substring into
1263  * @param Source string to copy substring from
1264  * @param Offset chars to skip from start
1265  * @param nChars number of chars to copy
1266  * @returns the number of chars copied; may be different from nChars due to the size of Source
1267  */
1268 int StrBufSub(StrBuf *dest, const StrBuf *Source, unsigned long Offset, size_t nChars)
1269 {
1270         size_t NCharsRemain;
1271         if (Offset > Source->BufUsed)
1272         {
1273                 FlushStrBuf(dest);
1274                 return 0;
1275         }
1276         if (Offset + nChars < Source->BufUsed)
1277         {
1278                 if (nChars >= dest->BufSize)
1279                         IncreaseBuf(dest, 0, nChars + 1);
1280                 memcpy(dest->buf, Source->buf + Offset, nChars);
1281                 dest->BufUsed = nChars;
1282                 dest->buf[dest->BufUsed] = '\0';
1283                 return nChars;
1284         }
1285         NCharsRemain = Source->BufUsed - Offset;
1286         if (NCharsRemain  >= dest->BufSize)
1287                 IncreaseBuf(dest, 0, NCharsRemain + 1);
1288         memcpy(dest->buf, Source->buf + Offset, NCharsRemain);
1289         dest->BufUsed = NCharsRemain;
1290         dest->buf[dest->BufUsed] = '\0';
1291         return NCharsRemain;
1292 }
1293
1294 /**
1295  * @ingroup StrBuf
1296  * @brief sprintf like function appending the formated string to the buffer
1297  * vsnprintf version to wrap into own calls
1298  * @param Buf Buffer to extend by format and Params
1299  * @param format printf alike format to add
1300  * @param ap va_list containing the items for format
1301  */
1302 void StrBufVAppendPrintf(StrBuf *Buf, const char *format, va_list ap)
1303 {
1304         va_list apl;
1305         size_t BufSize;
1306         size_t nWritten;
1307         size_t Offset;
1308         size_t newused;
1309
1310         if ((Buf == NULL)  || (format == NULL))
1311                 return;
1312
1313         BufSize = Buf->BufSize;
1314         nWritten = Buf->BufSize + 1;
1315         Offset = Buf->BufUsed;
1316         newused = Offset + nWritten;
1317
1318         while (newused >= BufSize) {
1319                 va_copy(apl, ap);
1320                 nWritten = vsnprintf(Buf->buf + Offset,
1321                                      Buf->BufSize - Offset,
1322                                      format, apl);
1323                 va_end(apl);
1324                 newused = Offset + nWritten;
1325                 if (newused >= Buf->BufSize) {
1326                         IncreaseBuf(Buf, 1, newused);
1327                         newused = Buf->BufSize + 1;
1328                 }
1329                 else {
1330                         Buf->BufUsed = Offset + nWritten;
1331                         BufSize = Buf->BufSize;
1332                 }
1333
1334         }
1335 }
1336
1337 /**
1338  * @ingroup StrBuf
1339  * @brief sprintf like function appending the formated string to the buffer
1340  * @param Buf Buffer to extend by format and Params
1341  * @param format printf alike format to add
1342  */
1343 void StrBufAppendPrintf(StrBuf *Buf, const char *format, ...)
1344 {
1345         size_t BufSize;
1346         size_t nWritten;
1347         size_t Offset;
1348         size_t newused;
1349         va_list arg_ptr;
1350
1351         if ((Buf == NULL)  || (format == NULL))
1352                 return;
1353
1354         BufSize = Buf->BufSize;
1355         nWritten = Buf->BufSize + 1;
1356         Offset = Buf->BufUsed;
1357         newused = Offset + nWritten;
1358
1359         while (newused >= BufSize) {
1360                 va_start(arg_ptr, format);
1361                 nWritten = vsnprintf(Buf->buf + Buf->BufUsed,
1362                                      Buf->BufSize - Buf->BufUsed,
1363                                      format, arg_ptr);
1364                 va_end(arg_ptr);
1365                 newused = Buf->BufUsed + nWritten;
1366                 if (newused >= Buf->BufSize) {
1367                         IncreaseBuf(Buf, 1, newused);
1368                         newused = Buf->BufSize + 1;
1369                 }
1370                 else {
1371                         Buf->BufUsed += nWritten;
1372                         BufSize = Buf->BufSize;
1373                 }
1374
1375         }
1376 }
1377
1378 /**
1379  * @ingroup StrBuf
1380  * @brief sprintf like function putting the formated string into the buffer
1381  * @param Buf Buffer to extend by format and Parameters
1382  * @param format printf alike format to add
1383  */
1384 void StrBufPrintf(StrBuf *Buf, const char *format, ...)
1385 {
1386         size_t nWritten;
1387         va_list arg_ptr;
1388
1389         if ((Buf == NULL)  || (format == NULL))
1390                 return;
1391
1392         nWritten = Buf->BufSize + 1;
1393         while (nWritten >= Buf->BufSize) {
1394                 va_start(arg_ptr, format);
1395                 nWritten = vsnprintf(Buf->buf, Buf->BufSize, format, arg_ptr);
1396                 va_end(arg_ptr);
1397                 if (nWritten >= Buf->BufSize) {
1398                         IncreaseBuf(Buf, 0, 0);
1399                         nWritten = Buf->BufSize + 1;
1400                         continue;
1401                 }
1402                 Buf->BufUsed = nWritten ;
1403         }
1404 }
1405
1406
1407 /**
1408  * @ingroup StrBuf_Tokenizer
1409  * @brief Counts the numbmer of tokens in a buffer
1410  * @param source String to count tokens in
1411  * @param tok    Tokenizer char to count
1412  * @returns numbers of tokenizer chars found
1413  */
1414 int StrBufNum_tokens(const StrBuf *source, char tok)
1415 {
1416         if (source == NULL)
1417                 return 0;
1418         return num_tokens(source->buf, tok);
1419 }
1420
1421 /*
1422  * remove_token() - a tokenizer that kills, maims, and destroys
1423  */
1424 /**
1425  * @ingroup StrBuf_Tokenizer
1426  * @brief a string tokenizer
1427  * @param Source StringBuffer to read into
1428  * @param parmnum n'th Parameter to remove
1429  * @param separator tokenizer character
1430  * @returns -1 if not found, else length of token.
1431  */
1432 int StrBufRemove_token(StrBuf *Source, int parmnum, char separator)
1433 {
1434         int ReducedBy;
1435         char *d, *s, *end;              /* dest, source */
1436         int count = 0;
1437
1438         /* Find desired @parameter */
1439         end = Source->buf + Source->BufUsed;
1440         d = Source->buf;
1441         while ((d <= end) &&
1442                (count < parmnum))
1443         {
1444                 /* End of string, bail! */
1445                 if (!*d) {
1446                         d = NULL;
1447                         break;
1448                 }
1449                 if (*d == separator) {
1450                         count++;
1451                 }
1452                 d++;
1453         }
1454         if ((d == NULL) || (d >= end))
1455                 return 0;               /* @Parameter not found */
1456
1457         /* Find next @parameter */
1458         s = d;
1459         while ((s <= end) &&
1460                (*s && *s != separator))
1461         {
1462                 s++;
1463         }
1464         if (*s == separator)
1465                 s++;
1466         ReducedBy = d - s;
1467
1468         /* Hack and slash */
1469         if (s >= end) {
1470                 return 0;
1471         }
1472         else if (*s) {
1473                 memmove(d, s, Source->BufUsed - (s - Source->buf));
1474                 Source->BufUsed += ReducedBy;
1475                 Source->buf[Source->BufUsed] = '\0';
1476         }
1477         else if (d == Source->buf) {
1478                 *d = 0;
1479                 Source->BufUsed = 0;
1480         }
1481         else {
1482                 *--d = '\0';
1483                 Source->BufUsed += ReducedBy;
1484         }
1485         /*
1486         while (*s) {
1487                 *d++ = *s++;
1488         }
1489         *d = 0;
1490         */
1491         return ReducedBy;
1492 }
1493
1494
1495 /**
1496  * @ingroup StrBuf_Tokenizer
1497  * @brief a string tokenizer
1498  * @param dest Destination StringBuffer
1499  * @param Source StringBuffer to read into
1500  * @param parmnum n'th Parameter to extract
1501  * @param separator tokenizer character
1502  * @returns -1 if not found, else length of token.
1503  */
1504 int StrBufExtract_token(StrBuf *dest, const StrBuf *Source, int parmnum, char separator)
1505 {
1506         const char *s, *e;              //* source * /
1507         int len = 0;                    //* running total length of extracted string * /
1508         int current_token = 0;          //* token currently being processed * /
1509
1510         if (dest != NULL) {
1511                 dest->buf[0] = '\0';
1512                 dest->BufUsed = 0;
1513         }
1514         else
1515                 return(-1);
1516
1517         if ((Source == NULL) || (Source->BufUsed ==0)) {
1518                 return(-1);
1519         }
1520         s = Source->buf;
1521         e = s + Source->BufUsed;
1522
1523         //cit_backtrace();
1524         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1525
1526         while ((s<e) && !IsEmptyStr(s)) {
1527                 if (*s == separator) {
1528                         ++current_token;
1529                 }
1530                 if (len >= dest->BufSize) {
1531                         dest->BufUsed = len;
1532                         if (IncreaseBuf(dest, 1, -1) < 0) {
1533                                 dest->BufUsed --;
1534                                 break;
1535                         }
1536                 }
1537                 if ( (current_token == parmnum) &&
1538                      (*s != separator)) {
1539                         dest->buf[len] = *s;
1540                         ++len;
1541                 }
1542                 else if (current_token > parmnum) {
1543                         break;
1544                 }
1545                 ++s;
1546         }
1547
1548         dest->buf[len] = '\0';
1549         dest->BufUsed = len;
1550
1551         if (current_token < parmnum) {
1552                 //lprintf (CTDL_DEBUG,"test <!: %s\n", dest);
1553                 return(-1);
1554         }
1555         //lprintf (CTDL_DEBUG,"test <: %d; %s\n", len, dest);
1556         return(len);
1557 }
1558
1559
1560
1561
1562
1563 /**
1564  * @ingroup StrBuf_Tokenizer
1565  * @brief a string tokenizer to fetch an integer
1566  * @param Source String containing tokens
1567  * @param parmnum n'th Parameter to extract
1568  * @param separator tokenizer character
1569  * @returns 0 if not found, else integer representation of the token
1570  */
1571 int StrBufExtract_int(const StrBuf* Source, int parmnum, char separator)
1572 {
1573         StrBuf tmp;
1574         char buf[64];
1575
1576         tmp.buf = buf;
1577         buf[0] = '\0';
1578         tmp.BufSize = 64;
1579         tmp.BufUsed = 0;
1580         tmp.ConstBuf = 1;
1581         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1582                 return(atoi(buf));
1583         else
1584                 return 0;
1585 }
1586
1587 /**
1588  * @ingroup StrBuf_Tokenizer
1589  * @brief a string tokenizer to fetch a long integer
1590  * @param Source String containing tokens
1591  * @param parmnum n'th Parameter to extract
1592  * @param separator tokenizer character
1593  * @returns 0 if not found, else long integer representation of the token
1594  */
1595 long StrBufExtract_long(const StrBuf* Source, int parmnum, char separator)
1596 {
1597         StrBuf tmp;
1598         char buf[64];
1599
1600         tmp.buf = buf;
1601         buf[0] = '\0';
1602         tmp.BufSize = 64;
1603         tmp.BufUsed = 0;
1604         tmp.ConstBuf = 1;
1605         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1606                 return(atoi(buf));
1607         else
1608                 return 0;
1609 }
1610
1611
1612 /**
1613  * @ingroup StrBuf_Tokenizer
1614  * @brief a string tokenizer to fetch an unsigned long
1615  * @param Source String containing tokens
1616  * @param parmnum n'th Parameter to extract
1617  * @param separator tokenizer character
1618  * @returns 0 if not found, else unsigned long representation of the token
1619  */
1620 unsigned long StrBufExtract_unsigned_long(const StrBuf* Source, int parmnum, char separator)
1621 {
1622         StrBuf tmp;
1623         char buf[64];
1624         char *pnum;
1625
1626         tmp.buf = buf;
1627         buf[0] = '\0';
1628         tmp.BufSize = 64;
1629         tmp.BufUsed = 0;
1630         tmp.ConstBuf = 1;
1631         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0) {
1632                 pnum = &buf[0];
1633                 if (*pnum == '-')
1634                         pnum ++;
1635                 return (unsigned long) atol(pnum);
1636         }
1637         else
1638                 return 0;
1639 }
1640
1641
1642
1643 /**
1644  * @ingroup StrBuf_NextTokenizer
1645  * @brief a string tokenizer; Bounds checker
1646  *  function to make shure whether StrBufExtract_NextToken and friends have reached the end of the string.
1647  * @param Source our tokenbuffer
1648  * @param pStart the token iterator pointer to inspect
1649  * @returns whether the revolving pointer is inside of the search range
1650  */
1651 int StrBufHaveNextToken(const StrBuf *Source, const char **pStart)
1652 {
1653         if ((Source == NULL) ||
1654             (*pStart == StrBufNOTNULL) ||
1655             (Source->BufUsed == 0))
1656         {
1657                 return 0;
1658         }
1659         if (*pStart == NULL)
1660         {
1661                 return 1;
1662         }
1663         else if (*pStart > Source->buf + Source->BufUsed)
1664         {
1665                 return 0;
1666         }
1667         else if (*pStart <= Source->buf)
1668         {
1669                 return 0;
1670         }
1671
1672         return 1;
1673 }
1674
1675 /**
1676  * @ingroup StrBuf_NextTokenizer
1677  * @brief a string tokenizer
1678  * @param dest Destination StringBuffer
1679  * @param Source StringBuffer to read into
1680  * @param pStart pointer to the end of the last token. Feed with NULL on start.
1681  * @param separator tokenizer
1682  * @returns -1 if not found, else length of token.
1683  */
1684 int StrBufExtract_NextToken(StrBuf *dest, const StrBuf *Source, const char **pStart, char separator)
1685 {
1686         const char *s;          /* source */
1687         const char *EndBuffer;  /* end stop of source buffer */
1688         int current_token = 0;  /* token currently being processed */
1689         int len = 0;            /* running total length of extracted string */
1690
1691         if ((Source          == NULL) ||
1692             (Source->BufUsed == 0)      )
1693         {
1694                 *pStart = StrBufNOTNULL;
1695                 return -1;
1696         }
1697
1698         EndBuffer = Source->buf + Source->BufUsed;
1699
1700         if (dest != NULL)
1701         {
1702                 dest->buf[0] = '\0';
1703                 dest->BufUsed = 0;
1704         }
1705         else
1706         {
1707                 *pStart = EndBuffer + 1;
1708                 return -1;
1709         }
1710
1711         if (*pStart == NULL)
1712         {
1713                 *pStart = Source->buf; /* we're starting to examine this buffer. */
1714         }
1715         else if ((*pStart < Source->buf) ||
1716                  (*pStart > EndBuffer  )   )
1717         {
1718                 return -1; /* no more tokens to find. */
1719         }
1720
1721         s = *pStart;
1722         /* start to find the next token */
1723         while ((s <= EndBuffer)      &&
1724                (current_token == 0) )
1725         {
1726                 if (*s == separator)
1727                 {
1728                         /* we found the next token */
1729                         ++current_token;
1730                 }
1731
1732                 if (len >= dest->BufSize)
1733                 {
1734                         /* our Dest-buffer isn't big enough, increase it. */
1735                         dest->BufUsed = len;
1736
1737                         if (IncreaseBuf(dest, 1, -1) < 0) {
1738                                 /* WHUT? no more mem? bail out. */
1739                                 s = EndBuffer;
1740                                 dest->BufUsed --;
1741                                 break;
1742                         }
1743                 }
1744
1745                 if ( (current_token == 0 ) &&   /* are we in our target token? */
1746                      (!IsEmptyStr(s)     ) &&
1747                      (separator     != *s)    ) /* don't copy the token itself */
1748                 {
1749                         dest->buf[len] = *s;    /* Copy the payload */
1750                         ++len;                  /* remember the bigger size. */
1751                 }
1752
1753                 ++s;
1754         }
1755
1756         /* did we reach the end? */
1757         if ((s > EndBuffer)) {
1758                 EndBuffer = StrBufNOTNULL;
1759                 *pStart = EndBuffer;
1760         }
1761         else {
1762                 *pStart = s;  /* remember the position for the next run */
1763         }
1764
1765         /* sanitize our extracted token */
1766         dest->buf[len] = '\0';
1767         dest->BufUsed  = len;
1768
1769         return (len);
1770 }
1771
1772
1773 /**
1774  * @ingroup StrBuf_NextTokenizer
1775  * @brief a string tokenizer
1776  * @param Source StringBuffer to read from
1777  * @param pStart pointer to the end of the last token. Feed with NULL.
1778  * @param separator tokenizer character
1779  * @param nTokens number of tokens to fastforward over
1780  * @returns -1 if not found, else length of token.
1781  */
1782 int StrBufSkip_NTokenS(const StrBuf *Source, const char **pStart, char separator, int nTokens)
1783 {
1784         const char *s, *EndBuffer;      //* source * /
1785         int len = 0;                    //* running total length of extracted string * /
1786         int current_token = 0;          //* token currently being processed * /
1787
1788         if ((Source == NULL) ||
1789             (Source->BufUsed ==0)) {
1790                 return(-1);
1791         }
1792         if (nTokens == 0)
1793                 return Source->BufUsed;
1794
1795         if (*pStart == NULL)
1796                 *pStart = Source->buf;
1797
1798         EndBuffer = Source->buf + Source->BufUsed;
1799
1800         if ((*pStart < Source->buf) ||
1801             (*pStart >  EndBuffer)) {
1802                 return (-1);
1803         }
1804
1805
1806         s = *pStart;
1807
1808         //cit_backtrace();
1809         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1810
1811         while ((s<EndBuffer) && !IsEmptyStr(s)) {
1812                 if (*s == separator) {
1813                         ++current_token;
1814                 }
1815                 if (current_token >= nTokens) {
1816                         break;
1817                 }
1818                 ++s;
1819         }
1820         *pStart = s;
1821         (*pStart) ++;
1822
1823         return(len);
1824 }
1825
1826 /**
1827  * @ingroup StrBuf_NextTokenizer
1828  * @brief a string tokenizer to fetch an integer
1829  * @param Source StringBuffer to read from
1830  * @param pStart Cursor on the tokenstring
1831  * @param separator tokenizer character
1832  * @returns 0 if not found, else integer representation of the token
1833  */
1834 int StrBufExtractNext_int(const StrBuf* Source, const char **pStart, char separator)
1835 {
1836         StrBuf tmp;
1837         char buf[64];
1838
1839         tmp.buf = buf;
1840         buf[0] = '\0';
1841         tmp.BufSize = 64;
1842         tmp.BufUsed = 0;
1843         tmp.ConstBuf = 1;
1844         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1845                 return(atoi(buf));
1846         else
1847                 return 0;
1848 }
1849
1850 /**
1851  * @ingroup StrBuf_NextTokenizer
1852  * @brief a string tokenizer to fetch a long integer
1853  * @param Source StringBuffer to read from
1854  * @param pStart Cursor on the tokenstring
1855  * @param separator tokenizer character
1856  * @returns 0 if not found, else long integer representation of the token
1857  */
1858 long StrBufExtractNext_long(const StrBuf* Source, const char **pStart, char separator)
1859 {
1860         StrBuf tmp;
1861         char buf[64];
1862
1863         tmp.buf = buf;
1864         buf[0] = '\0';
1865         tmp.BufSize = 64;
1866         tmp.BufUsed = 0;
1867         tmp.ConstBuf = 1;
1868         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1869                 return(atoi(buf));
1870         else
1871                 return 0;
1872 }
1873
1874
1875 /**
1876  * @ingroup StrBuf_NextTokenizer
1877  * @brief a string tokenizer to fetch an unsigned long
1878  * @param Source StringBuffer to read from
1879  * @param pStart Cursor on the tokenstring
1880  * @param separator tokenizer character
1881  * @returns 0 if not found, else unsigned long representation of the token
1882  */
1883 unsigned long StrBufExtractNext_unsigned_long(const StrBuf* Source, const char **pStart, char separator)
1884 {
1885         StrBuf tmp;
1886         char buf[64];
1887         char *pnum;
1888
1889         tmp.buf = buf;
1890         buf[0] = '\0';
1891         tmp.BufSize = 64;
1892         tmp.BufUsed = 0;
1893         tmp.ConstBuf = 1;
1894         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0) {
1895                 pnum = &buf[0];
1896                 if (*pnum == '-')
1897                         pnum ++;
1898                 return (unsigned long) atol(pnum);
1899         }
1900         else
1901                 return 0;
1902 }
1903
1904
1905
1906 /**
1907  * @ingroup StrBuf_IO
1908  * @brief Read a line from socket
1909  * flushes and closes the FD on error
1910  * @param buf the buffer to get the input to
1911  * @param fd pointer to the filedescriptor to read
1912  * @param append Append to an existing string or replace?
1913  * @param Error strerror() on error
1914  * @returns numbers of chars read
1915  */
1916 int StrBufTCP_read_line(StrBuf *buf, int *fd, int append, const char **Error)
1917 {
1918         int len, rlen, slen;
1919
1920         if (!append)
1921                 FlushStrBuf(buf);
1922
1923         slen = len = buf->BufUsed;
1924         while (1) {
1925                 rlen = read(*fd, &buf->buf[len], 1);
1926                 if (rlen < 1) {
1927                         *Error = strerror(errno);
1928
1929                         close(*fd);
1930                         *fd = -1;
1931
1932                         return -1;
1933                 }
1934                 if (buf->buf[len] == '\n')
1935                         break;
1936                 if (buf->buf[len] != '\r')
1937                         len ++;
1938                 if (len + 2 >= buf->BufSize) {
1939                         buf->BufUsed = len;
1940                         buf->buf[len+1] = '\0';
1941                         IncreaseBuf(buf, 1, -1);
1942                 }
1943         }
1944         buf->BufUsed = len;
1945         buf->buf[len] = '\0';
1946         return len - slen;
1947 }
1948
1949 /**
1950  * @ingroup StrBuf_BufferedIO
1951  * @brief Read a line from socket
1952  * flushes and closes the FD on error
1953  * @param Line the line to read from the fd / I/O Buffer
1954  * @param buf the buffer to get the input to
1955  * @param fd pointer to the filedescriptor to read
1956  * @param timeout number of successless selects until we bail out
1957  * @param selectresolution how long to wait on each select
1958  * @param Error strerror() on error
1959  * @returns numbers of chars read
1960  */
1961 int StrBufTCP_read_buffered_line(StrBuf *Line,
1962                                  StrBuf *buf,
1963                                  int *fd,
1964                                  int timeout,
1965                                  int selectresolution,
1966                                  const char **Error)
1967 {
1968         int len, rlen;
1969         int nSuccessLess = 0;
1970         fd_set rfds;
1971         char *pch = NULL;
1972         int fdflags;
1973         int IsNonBlock;
1974         struct timeval tv;
1975
1976         if (buf->BufUsed > 0) {
1977                 pch = strchr(buf->buf, '\n');
1978                 if (pch != NULL) {
1979                         rlen = 0;
1980                         len = pch - buf->buf;
1981                         if (len > 0 && (*(pch - 1) == '\r') )
1982                                 rlen ++;
1983                         StrBufSub(Line, buf, 0, len - rlen);
1984                         StrBufCutLeft(buf, len + 1);
1985                         return len - rlen;
1986                 }
1987         }
1988
1989         if (buf->BufSize - buf->BufUsed < 10)
1990                 IncreaseBuf(buf, 1, -1);
1991
1992         fdflags = fcntl(*fd, F_GETFL);
1993         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
1994
1995         while ((nSuccessLess < timeout) && (pch == NULL)) {
1996                 if (IsNonBlock){
1997                         tv.tv_sec = selectresolution;
1998                         tv.tv_usec = 0;
1999
2000                         FD_ZERO(&rfds);
2001                         FD_SET(*fd, &rfds);
2002                         if (select(*fd + 1, NULL, &rfds, NULL, &tv) == -1) {
2003                                 *Error = strerror(errno);
2004                                 close (*fd);
2005                                 *fd = -1;
2006                                 return -1;
2007                         }
2008                 }
2009                 if (IsNonBlock && !  FD_ISSET(*fd, &rfds)) {
2010                         nSuccessLess ++;
2011                         continue;
2012                 }
2013                 rlen = read(*fd,
2014                             &buf->buf[buf->BufUsed],
2015                             buf->BufSize - buf->BufUsed - 1);
2016                 if (rlen < 1) {
2017                         *Error = strerror(errno);
2018                         close(*fd);
2019                         *fd = -1;
2020                         return -1;
2021                 }
2022                 else if (rlen > 0) {
2023                         nSuccessLess = 0;
2024                         buf->BufUsed += rlen;
2025                         buf->buf[buf->BufUsed] = '\0';
2026                         if (buf->BufUsed + 10 > buf->BufSize) {
2027                                 IncreaseBuf(buf, 1, -1);
2028                         }
2029                         pch = strchr(buf->buf, '\n');
2030                         continue;
2031                 }
2032
2033         }
2034         if (pch != NULL) {
2035                 rlen = 0;
2036                 len = pch - buf->buf;
2037                 if (len > 0 && (*(pch - 1) == '\r') )
2038                         rlen ++;
2039                 StrBufSub(Line, buf, 0, len - rlen);
2040                 StrBufCutLeft(buf, len + 1);
2041                 return len - rlen;
2042         }
2043         return -1;
2044
2045 }
2046
2047 static const char *ErrRBLF_PreConditionFailed="StrBufTCP_read_buffered_line_fast: Wrong arguments or invalid Filedescriptor";
2048 static const char *ErrRBLF_SelectFailed="StrBufTCP_read_buffered_line_fast: Select failed without reason";
2049 static const char *ErrRBLF_NotEnoughSentFromServer="StrBufTCP_read_buffered_line_fast: No complete line was sent from peer";
2050 /**
2051  * @ingroup StrBuf_BufferedIO
2052  * @brief Read a line from socket
2053  * flushes and closes the FD on error
2054  * @param Line Line to read from the fd / I/O Buffer
2055  * @param IOBuf the buffer to get the input to
2056  * @param Pos pointer to the current read position, should be NULL initialized!
2057  * @param fd pointer to the filedescriptor to read
2058  * @param timeout number of successless selects until we bail out
2059  * @param selectresolution how long to wait on each select
2060  * @param Error strerror() on error
2061  * @returns numbers of chars read
2062  */
2063 int StrBufTCP_read_buffered_line_fast(StrBuf *Line,
2064                                       StrBuf *IOBuf,
2065                                       const char **Pos,
2066                                       int *fd,
2067                                       int timeout,
2068                                       int selectresolution,
2069                                       const char **Error)
2070 {
2071         const char *pche = NULL;
2072         const char *pos = NULL;
2073         int len, rlen;
2074         int nSuccessLess = 0;
2075         fd_set rfds;
2076         const char *pch = NULL;
2077         int fdflags;
2078         int IsNonBlock;
2079         struct timeval tv;
2080
2081         if ((Line == NULL) ||
2082             (IOBuf == NULL) ||
2083             (*fd == -1))
2084         {
2085                 *Error = ErrRBLF_PreConditionFailed;
2086                 return -1;
2087         }
2088
2089         pos = *Pos;
2090         if ((IOBuf->BufUsed > 0) &&
2091             (pos != NULL) &&
2092             (pos < IOBuf->buf + IOBuf->BufUsed))
2093         {
2094                 pche = IOBuf->buf + IOBuf->BufUsed;
2095                 pch = pos;
2096                 while ((pch < pche) && (*pch != '\n'))
2097                         pch ++;
2098                 if ((pch >= pche) || (*pch == '\0'))
2099                         pch = NULL;
2100                 if ((pch != NULL) &&
2101                     (pch <= pche))
2102                 {
2103                         rlen = 0;
2104                         len = pch - pos;
2105                         if (len > 0 && (*(pch - 1) == '\r') )
2106                                 rlen ++;
2107                         StrBufSub(Line, IOBuf, (pos - IOBuf->buf), len - rlen);
2108                         *Pos = pch + 1;
2109                         return len - rlen;
2110                 }
2111         }
2112
2113         if (pos != NULL) {
2114                 if (pos > pche)
2115                         FlushStrBuf(IOBuf);
2116                 else
2117                         StrBufCutLeft(IOBuf, (pos - IOBuf->buf));
2118                 *Pos = NULL;
2119         }
2120
2121         if (IOBuf->BufSize - IOBuf->BufUsed < 10) {
2122                 IncreaseBuf(IOBuf, 1, -1);
2123         }
2124
2125         fdflags = fcntl(*fd, F_GETFL);
2126         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
2127
2128         pch = NULL;
2129         while ((nSuccessLess < timeout) &&
2130                (pch == NULL) &&
2131                (*fd != -1)) {
2132                 if (IsNonBlock)
2133                 {
2134                         tv.tv_sec = 1;
2135                         tv.tv_usec = 0;
2136
2137                         FD_ZERO(&rfds);
2138                         FD_SET(*fd, &rfds);
2139                         if (select((*fd) + 1, &rfds, NULL, NULL, &tv) == -1) {
2140                                 *Error = strerror(errno);
2141                                 close (*fd);
2142                                 *fd = -1;
2143                                 if (*Error == NULL)
2144                                         *Error = ErrRBLF_SelectFailed;
2145                                 return -1;
2146                         }
2147                         if (! FD_ISSET(*fd, &rfds) != 0) {
2148                                 nSuccessLess ++;
2149                                 continue;
2150                         }
2151                 }
2152                 rlen = read(*fd,
2153                             &IOBuf->buf[IOBuf->BufUsed],
2154                             IOBuf->BufSize - IOBuf->BufUsed - 1);
2155                 if (rlen < 1) {
2156                         *Error = strerror(errno);
2157                         close(*fd);
2158                         *fd = -1;
2159                         return -1;
2160                 }
2161                 else if (rlen > 0) {
2162                         nSuccessLess = 0;
2163                         IOBuf->BufUsed += rlen;
2164                         IOBuf->buf[IOBuf->BufUsed] = '\0';
2165                         if (IOBuf->BufUsed + 10 > IOBuf->BufSize) {
2166                                 IncreaseBuf(IOBuf, 1, -1);
2167                                 *Pos = NULL;
2168                         }
2169
2170                         pche = IOBuf->buf + IOBuf->BufUsed;
2171                         pch = IOBuf->buf;
2172                         while ((pch < pche) && (*pch != '\n'))
2173                                 pch ++;
2174                         if ((pch >= pche) || (*pch == '\0'))
2175                                 pch = NULL;
2176                         continue;
2177                 }
2178         }
2179         if (pch != NULL) {
2180                 pos = IOBuf->buf;
2181                 rlen = 0;
2182                 len = pch - pos;
2183                 if (len > 0 && (*(pch - 1) == '\r') )
2184                         rlen ++;
2185                 StrBufSub(Line, IOBuf, 0, len - rlen);
2186                 *Pos = pos + len + 1;
2187                 return len - rlen;
2188         }
2189         *Error = ErrRBLF_NotEnoughSentFromServer;
2190         return -1;
2191
2192 }
2193
2194 static const char *ErrRBLF_BLOBPreConditionFailed="StrBufReadBLOB: Wrong arguments or invalid Filedescriptor";
2195 /**
2196  * @ingroup StrBuf_IO
2197  * @brief Input binary data from socket
2198  * flushes and closes the FD on error
2199  * @param Buf the buffer to get the input to
2200  * @param fd pointer to the filedescriptor to read
2201  * @param append Append to an existing string or replace?
2202  * @param nBytes the maximal number of bytes to read
2203  * @param Error strerror() on error
2204  * @returns numbers of chars read
2205  */
2206 int StrBufReadBLOB(StrBuf *Buf, int *fd, int append, long nBytes, const char **Error)
2207 {
2208         int fdflags;
2209         int len, rlen, slen;
2210         int nSuccessLess;
2211         int nRead = 0;
2212         char *ptr;
2213         int IsNonBlock;
2214         struct timeval tv;
2215         fd_set rfds;
2216
2217         if ((Buf == NULL) || (*fd == -1))
2218         {
2219                 *Error = ErrRBLF_BLOBPreConditionFailed;
2220                 return -1;
2221         }
2222         if (!append)
2223                 FlushStrBuf(Buf);
2224         if (Buf->BufUsed + nBytes >= Buf->BufSize)
2225                 IncreaseBuf(Buf, 1, Buf->BufUsed + nBytes);
2226
2227         ptr = Buf->buf + Buf->BufUsed;
2228
2229         slen = len = Buf->BufUsed;
2230
2231         fdflags = fcntl(*fd, F_GETFL);
2232         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
2233         nSuccessLess = 0;
2234         while ((nRead < nBytes) &&
2235                (*fd != -1))
2236         {
2237                 if (IsNonBlock)
2238                 {
2239                         tv.tv_sec = 1;
2240                         tv.tv_usec = 0;
2241
2242                         FD_ZERO(&rfds);
2243                         FD_SET(*fd, &rfds);
2244                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
2245                                 *Error = strerror(errno);
2246                                 close (*fd);
2247                                 *fd = -1;
2248                                 if (*Error == NULL)
2249                                         *Error = ErrRBLF_SelectFailed;
2250                                 return -1;
2251                         }
2252                         if (! FD_ISSET(*fd, &rfds) != 0) {
2253                                 nSuccessLess ++;
2254                                 continue;
2255                         }
2256                 }
2257
2258                 if ((rlen = read(*fd,
2259                                  ptr,
2260                                  nBytes - nRead)) == -1) {
2261                         close(*fd);
2262                         *fd = -1;
2263                         *Error = strerror(errno);
2264                         return rlen;
2265                 }
2266                 nRead += rlen;
2267                 ptr += rlen;
2268                 Buf->BufUsed += rlen;
2269         }
2270         Buf->buf[Buf->BufUsed] = '\0';
2271         return nRead;
2272 }
2273
2274 const char *ErrRBB_BLOBFPreConditionFailed = "StrBufReadBLOBBuffered: to many selects; aborting.";
2275 const char *ErrRBB_too_many_selects = "StrBufReadBLOBBuffered: to many selects; aborting.";
2276 /**
2277  * @ingroup StrBuf_BufferedIO
2278  * @brief Input binary data from socket
2279  * flushes and closes the FD on error
2280  * @param Blob put binary thing here
2281  * @param IOBuf the buffer to get the input to
2282  * @param Pos offset inside of IOBuf
2283  * @param fd pointer to the filedescriptor to read
2284  * @param append Append to an existing string or replace?
2285  * @param nBytes the maximal number of bytes to read
2286  * @param check whether we should search for '000\n' terminators in case of timeouts
2287  * @param Error strerror() on error
2288  * @returns numbers of chars read
2289  */
2290 int StrBufReadBLOBBuffered(StrBuf *Blob,
2291                            StrBuf *IOBuf,
2292                            const char **Pos,
2293                            int *fd,
2294                            int append,
2295                            long nBytes,
2296                            int check,
2297                            const char **Error)
2298 {
2299         const char *pche;
2300         const char *pos;
2301         int nSelects = 0;
2302         int SelRes;
2303         int fdflags;
2304         int len = 0;
2305         int rlen, slen;
2306         int nRead = 0;
2307         int nAlreadyRead = 0;
2308         int IsNonBlock;
2309         char *ptr;
2310         fd_set rfds;
2311         const char *pch;
2312         struct timeval tv;
2313         int nSuccessLess;
2314
2315         if ((Blob == NULL) || (*fd == -1) || (IOBuf == NULL) || (Pos == NULL))
2316         {
2317                 *Error = ErrRBB_BLOBFPreConditionFailed;
2318                 return -1;
2319         }
2320
2321         if (!append)
2322                 FlushStrBuf(Blob);
2323         if (Blob->BufUsed + nBytes >= Blob->BufSize)
2324                 IncreaseBuf(Blob, append, Blob->BufUsed + nBytes);
2325
2326         pos = *Pos;
2327
2328         if (pos > 0)
2329                 len = pos - IOBuf->buf;
2330         rlen = IOBuf->BufUsed - len;
2331
2332
2333         if ((IOBuf->BufUsed > 0) &&
2334             (pos != NULL) &&
2335             (pos < IOBuf->buf + IOBuf->BufUsed))
2336         {
2337                 pche = IOBuf->buf + IOBuf->BufUsed;
2338                 pch = pos;
2339
2340                 if (rlen < nBytes) {
2341                         memcpy(Blob->buf + Blob->BufUsed, pos, rlen);
2342                         Blob->BufUsed += rlen;
2343                         Blob->buf[Blob->BufUsed] = '\0';
2344                         nAlreadyRead = nRead = rlen;
2345                         *Pos = NULL;
2346                 }
2347                 if (rlen >= nBytes) {
2348                         memcpy(Blob->buf + Blob->BufUsed, pos, nBytes);
2349                         Blob->BufUsed += nBytes;
2350                         Blob->buf[Blob->BufUsed] = '\0';
2351                         if (rlen == nBytes) {
2352                                 *Pos = NULL;
2353                                 FlushStrBuf(IOBuf);
2354                         }
2355                         else
2356                                 *Pos += nBytes;
2357                         return nBytes;
2358                 }
2359         }
2360
2361         FlushStrBuf(IOBuf);
2362         if (IOBuf->BufSize < nBytes - nRead)
2363                 IncreaseBuf(IOBuf, 0, nBytes - nRead);
2364         ptr = IOBuf->buf;
2365
2366         slen = len = Blob->BufUsed;
2367
2368         fdflags = fcntl(*fd, F_GETFL);
2369         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
2370
2371         SelRes = 1;
2372         nBytes -= nRead;
2373         nRead = 0;
2374         while ((nRead < nBytes) &&
2375                (*fd != -1)) {
2376                 if (IsNonBlock)
2377                 {
2378                         tv.tv_sec = 1;
2379                         tv.tv_usec = 0;
2380
2381                         FD_ZERO(&rfds);
2382                         FD_SET(*fd, &rfds);
2383                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
2384                                 *Error = strerror(errno);
2385                                 close (*fd);
2386                                 *fd = -1;
2387                                 if (*Error == NULL)
2388                                         *Error = ErrRBLF_SelectFailed;
2389                                 return -1;
2390                         }
2391                         if (! FD_ISSET(*fd, &rfds) != 0) {
2392                                 nSuccessLess ++;
2393                                 continue;
2394                         }
2395                 }
2396                 nSuccessLess = 0;
2397                 rlen = read(*fd,
2398                             ptr,
2399                             nBytes - nRead);
2400                 if (rlen == -1) {
2401                         close(*fd);
2402                         *fd = -1;
2403                         *Error = strerror(errno);
2404                         return rlen;
2405                 }
2406                 else if (rlen == 0){
2407                         nSuccessLess ++;
2408                         if ((check == NNN_TERM) &&
2409                             (nRead > 5) &&
2410                             (strncmp(IOBuf->buf + IOBuf->BufUsed - 5, "\n000\n", 5) == 0))
2411                         {
2412                                 StrBufPlain(Blob, HKEY("\n000\n"));
2413                                 StrBufCutRight(Blob, 5);
2414                                 return Blob->BufUsed;
2415                         }
2416                         if (nSelects > 10) {
2417                                 FlushStrBuf(IOBuf);
2418                                 *Error = ErrRBB_too_many_selects;
2419                                 return -1;
2420                         }
2421                 }
2422                 else if (rlen > 0) {
2423                         nRead += rlen;
2424                         ptr += rlen;
2425                         IOBuf->BufUsed += rlen;
2426                 }
2427         }
2428         if (nRead > nBytes) {
2429                 *Pos = IOBuf->buf + nBytes;
2430         }
2431         Blob->buf[Blob->BufUsed] = '\0';
2432         StrBufAppendBufPlain(Blob, IOBuf->buf, nBytes, 0);
2433         if (*Pos == NULL) {
2434                 FlushStrBuf(IOBuf);
2435         }
2436         return nRead + nAlreadyRead;
2437 }
2438
2439 /**
2440  * @ingroup StrBuf
2441  * @brief Cut nChars from the start of the string
2442  * @param Buf Buffer to modify
2443  * @param nChars how many chars should be skipped?
2444  */
2445 void StrBufCutLeft(StrBuf *Buf, int nChars)
2446 {
2447         if (nChars >= Buf->BufUsed) {
2448                 FlushStrBuf(Buf);
2449                 return;
2450         }
2451         memmove(Buf->buf, Buf->buf + nChars, Buf->BufUsed - nChars);
2452         Buf->BufUsed -= nChars;
2453         Buf->buf[Buf->BufUsed] = '\0';
2454 }
2455
2456 /**
2457  * @ingroup StrBuf
2458  * @brief Cut the trailing n Chars from the string
2459  * @param Buf Buffer to modify
2460  * @param nChars how many chars should be trunkated?
2461  */
2462 void StrBufCutRight(StrBuf *Buf, int nChars)
2463 {
2464         if (nChars >= Buf->BufUsed) {
2465                 FlushStrBuf(Buf);
2466                 return;
2467         }
2468         Buf->BufUsed -= nChars;
2469         Buf->buf[Buf->BufUsed] = '\0';
2470 }
2471
2472 /**
2473  * @ingroup StrBuf
2474  * @brief Cut the string after n Chars
2475  * @param Buf Buffer to modify
2476  * @param AfternChars after how many chars should we trunkate the string?
2477  * @param At if non-null and points inside of our string, cut it there.
2478  */
2479 void StrBufCutAt(StrBuf *Buf, int AfternChars, const char *At)
2480 {
2481         if (At != NULL){
2482                 AfternChars = At - Buf->buf;
2483         }
2484
2485         if ((AfternChars < 0) || (AfternChars >= Buf->BufUsed))
2486                 return;
2487         Buf->BufUsed = AfternChars;
2488         Buf->buf[Buf->BufUsed] = '\0';
2489 }
2490
2491
2492 /**
2493  * @ingroup StrBuf
2494  * @brief Strip leading and trailing spaces from a string; with premeasured and adjusted length.
2495  * @param Buf the string to modify
2496  */
2497 void StrBufTrim(StrBuf *Buf)
2498 {
2499         int delta = 0;
2500         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
2501
2502         while ((Buf->BufUsed > delta) && (isspace(Buf->buf[delta]))){
2503                 delta ++;
2504         }
2505         if (delta > 0) StrBufCutLeft(Buf, delta);
2506
2507         if (Buf->BufUsed == 0) return;
2508         while (isspace(Buf->buf[Buf->BufUsed - 1])){
2509                 Buf->BufUsed --;
2510         }
2511         Buf->buf[Buf->BufUsed] = '\0';
2512 }
2513
2514 /**
2515  * @ingroup StrBuf
2516  * @brief uppercase the contents of a buffer
2517  * @param Buf the buffer to translate
2518  */
2519 void StrBufUpCase(StrBuf *Buf)
2520 {
2521         char *pch, *pche;
2522
2523         pch = Buf->buf;
2524         pche = pch + Buf->BufUsed;
2525         while (pch < pche) {
2526                 *pch = toupper(*pch);
2527                 pch ++;
2528         }
2529 }
2530
2531
2532 /**
2533  * @ingroup StrBuf
2534  * @brief lowercase the contents of a buffer
2535  * @param Buf the buffer to translate
2536  */
2537 void StrBufLowerCase(StrBuf *Buf)
2538 {
2539         char *pch, *pche;
2540
2541         pch = Buf->buf;
2542         pche = pch + Buf->BufUsed;
2543         while (pch < pche) {
2544                 *pch = tolower(*pch);
2545                 pch ++;
2546         }
2547 }
2548
2549 /**
2550  * @ingroup StrBuf
2551  * @brief removes double slashes from pathnames
2552  * @param Dir directory string to filter
2553  * @param RemoveTrailingSlash allows / disallows trailing slashes
2554  */
2555 void StrBufStripSlashes(StrBuf *Dir, int RemoveTrailingSlash)
2556 {
2557         char *a, *b;
2558
2559         a = b = Dir->buf;
2560
2561         while (!IsEmptyStr(a)) {
2562                 if (*a == '/') {
2563                         while (*a == '/')
2564                                 a++;
2565                         *b = '/';
2566                         b++;
2567                 }
2568                 else {
2569                         *b = *a;
2570                         b++; a++;
2571                 }
2572         }
2573         if ((RemoveTrailingSlash) && (*(b - 1) != '/')){
2574                 *b = '/';
2575                 b++;
2576         }
2577         *b = '\0';
2578         Dir->BufUsed = b - Dir->buf;
2579 }
2580
2581 /**
2582  * @ingroup StrBuf_DeEnCoder
2583  * @brief unhide special chars hidden to the HTML escaper
2584  * @param target buffer to put the unescaped string in
2585  * @param source buffer to unescape
2586  */
2587 void StrBufEUid_unescapize(StrBuf *target, const StrBuf *source)
2588 {
2589         int a, b, len;
2590         char hex[3];
2591
2592         if (target != NULL)
2593                 FlushStrBuf(target);
2594
2595         if (source == NULL ||target == NULL)
2596         {
2597                 return;
2598         }
2599
2600         len = source->BufUsed;
2601         for (a = 0; a < len; ++a) {
2602                 if (target->BufUsed >= target->BufSize)
2603                         IncreaseBuf(target, 1, -1);
2604
2605                 if (source->buf[a] == '=') {
2606                         hex[0] = source->buf[a + 1];
2607                         hex[1] = source->buf[a + 2];
2608                         hex[2] = 0;
2609                         b = 0;
2610                         sscanf(hex, "%02x", &b);
2611                         target->buf[target->BufUsed] = b;
2612                         target->buf[++target->BufUsed] = 0;
2613                         a += 2;
2614                 }
2615                 else {
2616                         target->buf[target->BufUsed] = source->buf[a];
2617                         target->buf[++target->BufUsed] = 0;
2618                 }
2619         }
2620 }
2621
2622
2623 /**
2624  * @ingroup StrBuf_DeEnCoder
2625  * @brief hide special chars from the HTML escapers and friends
2626  * @param target buffer to put the escaped string in
2627  * @param source buffer to escape
2628  */
2629 void StrBufEUid_escapize(StrBuf *target, const StrBuf *source)
2630 {
2631         int i, len;
2632
2633         if (target != NULL)
2634                 FlushStrBuf(target);
2635
2636         if (source == NULL ||target == NULL)
2637         {
2638                 return;
2639         }
2640
2641         len = source->BufUsed;
2642         for (i=0; i<len; ++i) {
2643                 if (target->BufUsed + 4 >= target->BufSize)
2644                         IncreaseBuf(target, 1, -1);
2645                 if ( (isalnum(source->buf[i])) ||
2646                      (source->buf[i]=='-') ||
2647                      (source->buf[i]=='_') ) {
2648                         target->buf[target->BufUsed++] = source->buf[i];
2649                 }
2650                 else {
2651                         sprintf(&target->buf[target->BufUsed],
2652                                 "=%02X",
2653                                 (0xFF &source->buf[i]));
2654                         target->BufUsed += 3;
2655                 }
2656         }
2657         target->buf[target->BufUsed + 1] = '\0';
2658 }
2659
2660 #ifdef HAVE_ZLIB
2661 #define DEF_MEM_LEVEL 8 /*< memlevel??? */
2662 #define OS_CODE 0x03    /*< unix */
2663
2664 /**
2665  * @ingroup StrBuf_DeEnCoder
2666  * @brief uses the same calling syntax as compress2(), but it
2667  *   creates a stream compatible with HTTP "Content-encoding: gzip"
2668  * @param dest compressed buffer
2669  * @param destLen length of the compresed data
2670  * @param source source to encode
2671  * @param sourceLen length of source to encode
2672  * @param level compression level
2673  */
2674 int ZEXPORT compress_gzip(Bytef * dest,
2675                           size_t * destLen,
2676                           const Bytef * source,
2677                           uLong sourceLen,
2678                           int level)
2679 {
2680         const int gz_magic[2] = { 0x1f, 0x8b }; /* gzip magic header */
2681
2682         /* write gzip header */
2683         snprintf((char *) dest, *destLen,
2684                  "%c%c%c%c%c%c%c%c%c%c",
2685                  gz_magic[0], gz_magic[1], Z_DEFLATED,
2686                  0 /*flags */ , 0, 0, 0, 0 /*time */ , 0 /* xflags */ ,
2687                  OS_CODE);
2688
2689         /* normal deflate */
2690         z_stream stream;
2691         int err;
2692         stream.next_in = (Bytef *) source;
2693         stream.avail_in = (uInt) sourceLen;
2694         stream.next_out = dest + 10L;   // after header
2695         stream.avail_out = (uInt) * destLen;
2696         if ((uLong) stream.avail_out != *destLen)
2697                 return Z_BUF_ERROR;
2698
2699         stream.zalloc = (alloc_func) 0;
2700         stream.zfree = (free_func) 0;
2701         stream.opaque = (voidpf) 0;
2702
2703         err = deflateInit2(&stream, level, Z_DEFLATED, -MAX_WBITS,
2704                            DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
2705         if (err != Z_OK)
2706                 return err;
2707
2708         err = deflate(&stream, Z_FINISH);
2709         if (err != Z_STREAM_END) {
2710                 deflateEnd(&stream);
2711                 return err == Z_OK ? Z_BUF_ERROR : err;
2712         }
2713         *destLen = stream.total_out + 10L;
2714
2715         /* write CRC and Length */
2716         uLong crc = crc32(0L, source, sourceLen);
2717         int n;
2718         for (n = 0; n < 4; ++n, ++*destLen) {
2719                 dest[*destLen] = (int) (crc & 0xff);
2720                 crc >>= 8;
2721         }
2722         uLong len = stream.total_in;
2723         for (n = 0; n < 4; ++n, ++*destLen) {
2724                 dest[*destLen] = (int) (len & 0xff);
2725                 len >>= 8;
2726         }
2727         err = deflateEnd(&stream);
2728         return err;
2729 }
2730 #endif
2731
2732
2733 /**
2734  * @ingroup StrBuf_DeEnCoder
2735  * @brief compress the buffer with gzip
2736  * Attention! If you feed this a Const String, you must maintain the uncompressed buffer yourself!
2737  * @param Buf buffer whose content is to be gzipped
2738  */
2739 int CompressBuffer(StrBuf *Buf)
2740 {
2741 #ifdef HAVE_ZLIB
2742         char *compressed_data = NULL;
2743         size_t compressed_len, bufsize;
2744         int i = 0;
2745
2746         bufsize = compressed_len = Buf->BufUsed +  (Buf->BufUsed / 100) + 100;
2747         compressed_data = malloc(compressed_len);
2748
2749         if (compressed_data == NULL)
2750                 return -1;
2751         /* Flush some space after the used payload so valgrind shuts up... */
2752         while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
2753                 Buf->buf[Buf->BufUsed + i++] = '\0';
2754         if (compress_gzip((Bytef *) compressed_data,
2755                           &compressed_len,
2756                           (Bytef *) Buf->buf,
2757                           (uLongf) Buf->BufUsed, Z_BEST_SPEED) == Z_OK) {
2758                 if (!Buf->ConstBuf)
2759                         free(Buf->buf);
2760                 Buf->buf = compressed_data;
2761                 Buf->BufUsed = compressed_len;
2762                 Buf->BufSize = bufsize;
2763                 /* Flush some space after the used payload so valgrind shuts up... */
2764                 i = 0;
2765                 while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
2766                         Buf->buf[Buf->BufUsed + i++] = '\0';
2767                 return 1;
2768         } else {
2769                 free(compressed_data);
2770         }
2771 #endif  /* HAVE_ZLIB */
2772         return 0;
2773 }
2774
2775 /**
2776  * @ingroup StrBuf_DeEnCoder
2777  * @brief decode a buffer from base 64 encoding; destroys original
2778  * @param Buf Buffor to transform
2779  */
2780 int StrBufDecodeBase64(StrBuf *Buf)
2781 {
2782         char *xferbuf;
2783         size_t siz;
2784         if (Buf == NULL) return -1;
2785
2786         xferbuf = (char*) malloc(Buf->BufSize);
2787         siz = CtdlDecodeBase64(xferbuf,
2788                                Buf->buf,
2789                                Buf->BufUsed);
2790         free(Buf->buf);
2791         Buf->buf = xferbuf;
2792         Buf->BufUsed = siz;
2793         return siz;
2794 }
2795
2796 /**
2797  * @ingroup StrBuf_DeEnCoder
2798  * @brief decode a buffer from base 64 encoding; destroys original
2799  * @param Buf Buffor to transform
2800  */
2801 int StrBufDecodeHex(StrBuf *Buf)
2802 {
2803         unsigned int ch;
2804         char *pch, *pche, *pchi;
2805
2806         if (Buf == NULL) return -1;
2807
2808         pch = pchi = Buf->buf;
2809         pche = pch + Buf->BufUsed;
2810
2811         while (pchi < pche){
2812                 ch = decode_hex(pchi);
2813                 *pch = ch;
2814                 pch ++;
2815                 pchi += 2;
2816         }
2817
2818         *pch = '\0';
2819         Buf->BufUsed = pch - Buf->buf;
2820         return Buf->BufUsed;
2821 }
2822
2823 /**
2824  * @ingroup StrBuf_DeEnCoder
2825  * @brief replace all chars >0x20 && < 0x7F with Mute
2826  * @param Mute char to put over invalid chars
2827  * @param Buf Buffor to transform
2828  */
2829 int StrBufSanitizeAscii(StrBuf *Buf, const char Mute)
2830 {
2831         unsigned char *pch;
2832
2833         if (Buf == NULL) return -1;
2834         pch = (unsigned char *)Buf->buf;
2835         while (pch < (unsigned char *)Buf->buf + Buf->BufUsed) {
2836                 if ((*pch < 0x20) || (*pch > 0x7F))
2837                         *pch = Mute;
2838                 pch ++;
2839         }
2840         return Buf->BufUsed;
2841 }
2842
2843
2844 /**
2845  * @ingroup StrBuf_DeEnCoder
2846  * @brief remove escaped strings from i.e. the url string (like %20 for blanks)
2847  * @param Buf Buffer to translate
2848  * @param StripBlanks Reduce several blanks to one?
2849  */
2850 long StrBufUnescape(StrBuf *Buf, int StripBlanks)
2851 {
2852         int a, b;
2853         char hex[3];
2854         long len;
2855
2856         while ((Buf->BufUsed > 0) && (isspace(Buf->buf[Buf->BufUsed - 1]))){
2857                 Buf->buf[Buf->BufUsed - 1] = '\0';
2858                 Buf->BufUsed --;
2859         }
2860
2861         a = 0;
2862         while (a < Buf->BufUsed) {
2863                 if (Buf->buf[a] == '+')
2864                         Buf->buf[a] = ' ';
2865                 else if (Buf->buf[a] == '%') {
2866                         /* don't let % chars through, rather truncate the input. */
2867                         if (a + 2 > Buf->BufUsed) {
2868                                 Buf->buf[a] = '\0';
2869                                 Buf->BufUsed = a;
2870                         }
2871                         else {
2872                                 hex[0] = Buf->buf[a + 1];
2873                                 hex[1] = Buf->buf[a + 2];
2874                                 hex[2] = 0;
2875                                 b = 0;
2876                                 sscanf(hex, "%02x", &b);
2877                                 Buf->buf[a] = (char) b;
2878                                 len = Buf->BufUsed - a - 2;
2879                                 if (len > 0)
2880                                         memmove(&Buf->buf[a + 1], &Buf->buf[a + 3], len);
2881
2882                                 Buf->BufUsed -=2;
2883                         }
2884                 }
2885                 a++;
2886         }
2887         return a;
2888 }
2889
2890
2891 /**
2892  * @ingroup StrBuf_DeEnCoder
2893  * @brief       RFC2047-encode a header field if necessary.
2894  *              If no non-ASCII characters are found, the string
2895  *              will be copied verbatim without encoding.
2896  *
2897  * @param       target          Target buffer.
2898  * @param       source          Source string to be encoded.
2899  * @returns     encoded length; -1 if non success.
2900  */
2901 int StrBufRFC2047encode(StrBuf **target, const StrBuf *source)
2902 {
2903         const char headerStr[] = "=?UTF-8?Q?";
2904         int need_to_encode = 0;
2905         int i = 0;
2906         unsigned char ch;
2907
2908         if ((source == NULL) ||
2909             (target == NULL))
2910             return -1;
2911
2912         while ((i < source->BufUsed) &&
2913                (!IsEmptyStr (&source->buf[i])) &&
2914                (need_to_encode == 0)) {
2915                 if (((unsigned char) source->buf[i] < 32) ||
2916                     ((unsigned char) source->buf[i] > 126)) {
2917                         need_to_encode = 1;
2918                 }
2919                 i++;
2920         }
2921
2922         if (!need_to_encode) {
2923                 if (*target == NULL) {
2924                         *target = NewStrBufPlain(source->buf, source->BufUsed);
2925                 }
2926                 else {
2927                         FlushStrBuf(*target);
2928                         StrBufAppendBuf(*target, source, 0);
2929                 }
2930                 return (*target)->BufUsed;
2931         }
2932         if (*target == NULL)
2933                 *target = NewStrBufPlain(NULL, sizeof(headerStr) + source->BufUsed * 2);
2934         else if (sizeof(headerStr) + source->BufUsed >= (*target)->BufSize)
2935                 IncreaseBuf(*target, sizeof(headerStr) + source->BufUsed, 0);
2936         memcpy ((*target)->buf, headerStr, sizeof(headerStr) - 1);
2937         (*target)->BufUsed = sizeof(headerStr) - 1;
2938         for (i=0; (i < source->BufUsed); ++i) {
2939                 if ((*target)->BufUsed + 4 >= (*target)->BufSize)
2940                         IncreaseBuf(*target, 1, 0);
2941                 ch = (unsigned char) source->buf[i];
2942                 if ((ch < 32) || (ch > 126) || (ch == 61)) {
2943                         sprintf(&(*target)->buf[(*target)->BufUsed], "=%02X", ch);
2944                         (*target)->BufUsed += 3;
2945                 }
2946                 else {
2947                         (*target)->buf[(*target)->BufUsed] = ch;
2948                         (*target)->BufUsed++;
2949                 }
2950         }
2951
2952         if ((*target)->BufUsed + 4 >= (*target)->BufSize)
2953                 IncreaseBuf(*target, 1, 0);
2954
2955         (*target)->buf[(*target)->BufUsed++] = '?';
2956         (*target)->buf[(*target)->BufUsed++] = '=';
2957         (*target)->buf[(*target)->BufUsed] = '\0';
2958         return (*target)->BufUsed;;
2959 }
2960
2961 /**
2962  * @ingroup StrBuf
2963  * @brief replaces all occurances of 'search' by 'replace'
2964  * @param buf Buffer to modify
2965  * @param search character to search
2966  * @param replace character to replace search by
2967  */
2968 void StrBufReplaceChars(StrBuf *buf, char search, char replace)
2969 {
2970         long i;
2971         if (buf == NULL)
2972                 return;
2973         for (i=0; i<buf->BufUsed; i++)
2974                 if (buf->buf[i] == search)
2975                         buf->buf[i] = replace;
2976
2977 }
2978
2979
2980
2981 /**
2982  * @ingroup StrBuf_DeEnCoder
2983  * @brief Wrapper around iconv_open()
2984  * Our version adds aliases for non-standard Microsoft charsets
2985  * such as 'MS950', aliasing them to names like 'CP950'
2986  *
2987  * @param tocode        Target encoding
2988  * @param fromcode      Source encoding
2989  * @param pic           anonimized pointer to iconv struct
2990  */
2991 void  ctdl_iconv_open(const char *tocode, const char *fromcode, void *pic)
2992 {
2993 #ifdef HAVE_ICONV
2994         iconv_t ic = (iconv_t)(-1) ;
2995         ic = iconv_open(tocode, fromcode);
2996         if (ic == (iconv_t)(-1) ) {
2997                 char alias_fromcode[64];
2998                 if ( (strlen(fromcode) == 5) && (!strncasecmp(fromcode, "MS", 2)) ) {
2999                         safestrncpy(alias_fromcode, fromcode, sizeof alias_fromcode);
3000                         alias_fromcode[0] = 'C';
3001                         alias_fromcode[1] = 'P';
3002                         ic = iconv_open(tocode, alias_fromcode);
3003                 }
3004         }
3005         *(iconv_t *)pic = ic;
3006 #endif
3007 }
3008
3009
3010 /**
3011  * @ingroup StrBuf_DeEnCoder
3012  * @brief find one chunk of a RFC822 encoded string
3013  * @param Buffer where to search
3014  * @param bptr where to start searching
3015  * @returns found position, NULL if none.
3016  */
3017 static inline char *FindNextEnd (const StrBuf *Buf, char *bptr)
3018 {
3019         char * end;
3020         /* Find the next ?Q? */
3021         if (Buf->BufUsed - (bptr - Buf->buf)  < 6)
3022                 return NULL;
3023
3024         end = strchr(bptr + 2, '?');
3025
3026         if (end == NULL)
3027                 return NULL;
3028
3029         if ((Buf->BufUsed - (end - Buf->buf) > 3) &&
3030             ((*(end + 1) == 'B') || (*(end + 1) == 'Q')) &&
3031             (*(end + 2) == '?')) {
3032                 /* skip on to the end of the cluster, the next ?= */
3033                 end = strstr(end + 3, "?=");
3034         }
3035         else
3036                 /* sort of half valid encoding, try to find an end. */
3037                 end = strstr(bptr, "?=");
3038         return end;
3039 }
3040
3041 /**
3042  * @ingroup StrBuf
3043  * @brief swaps the contents of two StrBufs
3044  * this is to be used to have cheap switched between a work-buffer and a target buffer
3045  * @param A First one
3046  * @param B second one
3047  */
3048 static inline void SwapBuffers(StrBuf *A, StrBuf *B)
3049 {
3050         StrBuf C;
3051
3052         memcpy(&C, A, sizeof(*A));
3053         memcpy(A, B, sizeof(*B));
3054         memcpy(B, &C, sizeof(C));
3055
3056 }
3057
3058
3059 /**
3060  * @ingroup StrBuf_DeEnCoder
3061  * @brief convert one buffer according to the preselected iconv pointer PIC
3062  * @param ConvertBuf buffer we need to translate
3063  * @param TmpBuf To share a workbuffer over several iterations. prepare to have it filled with useless stuff afterwards.
3064  * @param pic Pointer to the iconv-session Object
3065  */
3066 void StrBufConvert(StrBuf *ConvertBuf, StrBuf *TmpBuf, void *pic)
3067 {
3068 #ifdef HAVE_ICONV
3069         long trycount = 0;
3070         size_t siz;
3071         iconv_t ic;
3072         char *ibuf;                     /**< Buffer of characters to be converted */
3073         char *obuf;                     /**< Buffer for converted characters */
3074         size_t ibuflen;                 /**< Length of input buffer */
3075         size_t obuflen;                 /**< Length of output buffer */
3076
3077
3078         /* since we're converting to utf-8, one glyph may take up to 6 bytes */
3079         if (ConvertBuf->BufUsed * 6 >= TmpBuf->BufSize)
3080                 IncreaseBuf(TmpBuf, 0, ConvertBuf->BufUsed * 6);
3081 TRYAGAIN:
3082         ic = *(iconv_t*)pic;
3083         ibuf = ConvertBuf->buf;
3084         ibuflen = ConvertBuf->BufUsed;
3085         obuf = TmpBuf->buf;
3086         obuflen = TmpBuf->BufSize;
3087
3088         siz = iconv(ic, &ibuf, &ibuflen, &obuf, &obuflen);
3089
3090         if (siz < 0) {
3091                 if (errno == E2BIG) {
3092                         trycount ++;
3093                         IncreaseBuf(TmpBuf, 0, 0);
3094                         if (trycount < 5)
3095                                 goto TRYAGAIN;
3096
3097                 }
3098                 else if (errno == EILSEQ){
3099                         /* hm, invalid utf8 sequence... what to do now? */
3100                         /* An invalid multibyte sequence has been encountered in the input */
3101                 }
3102                 else if (errno == EINVAL) {
3103                         /* An incomplete multibyte sequence has been encountered in the input. */
3104                 }
3105
3106                 FlushStrBuf(TmpBuf);
3107         }
3108         else {
3109                 TmpBuf->BufUsed = TmpBuf->BufSize - obuflen;
3110                 TmpBuf->buf[TmpBuf->BufUsed] = '\0';
3111
3112                 /* little card game: wheres the red lady? */
3113                 SwapBuffers(ConvertBuf, TmpBuf);
3114                 FlushStrBuf(TmpBuf);
3115         }
3116 #endif
3117 }
3118
3119
3120 /**
3121  * @ingroup StrBuf_DeEnCoder
3122  * @brief catches one RFC822 encoded segment, and decodes it.
3123  * @param Target buffer to fill with result
3124  * @param DecodeMe buffer with stuff to process
3125  * @param SegmentStart points to our current segment in DecodeMe
3126  * @param SegmentEnd Points to the end of our current segment in DecodeMe
3127  * @param ConvertBuf Workbuffer shared between several iterations. Random content; needs to be valid
3128  * @param ConvertBuf2 Workbuffer shared between several iterations. Random content; needs to be valid
3129  * @param FoundCharset Characterset to default decoding to; if we find another we will overwrite it.
3130  */
3131 inline static void DecodeSegment(StrBuf *Target,
3132                                  const StrBuf *DecodeMe,
3133                                  char *SegmentStart,
3134                                  char *SegmentEnd,
3135                                  StrBuf *ConvertBuf,
3136                                  StrBuf *ConvertBuf2,
3137                                  StrBuf *FoundCharset)
3138 {
3139         StrBuf StaticBuf;
3140         char charset[128];
3141         char encoding[16];
3142 #ifdef HAVE_ICONV
3143         iconv_t ic = (iconv_t)(-1);
3144 #else
3145         void *ic = NULL;
3146 #endif
3147         /* Now we handle foreign character sets properly encoded
3148          * in RFC2047 format.
3149          */
3150         StaticBuf.buf = SegmentStart;
3151         StaticBuf.BufUsed = SegmentEnd - SegmentStart;
3152         StaticBuf.BufSize = DecodeMe->BufSize - (SegmentStart - DecodeMe->buf);
3153         extract_token(charset, SegmentStart, 1, '?', sizeof charset);
3154         if (FoundCharset != NULL) {
3155                 FlushStrBuf(FoundCharset);
3156                 StrBufAppendBufPlain(FoundCharset, charset, -1, 0);
3157         }
3158         extract_token(encoding, SegmentStart, 2, '?', sizeof encoding);
3159         StrBufExtract_token(ConvertBuf, &StaticBuf, 3, '?');
3160
3161         *encoding = toupper(*encoding);
3162         if (*encoding == 'B') { /**< base64 */
3163                 ConvertBuf2->BufUsed = CtdlDecodeBase64(ConvertBuf2->buf,
3164                                                         ConvertBuf->buf,
3165                                                         ConvertBuf->BufUsed);
3166         }
3167         else if (*encoding == 'Q') {    /**< quoted-printable */
3168                 long pos;
3169
3170                 pos = 0;
3171                 while (pos < ConvertBuf->BufUsed)
3172                 {
3173                         if (ConvertBuf->buf[pos] == '_')
3174                                 ConvertBuf->buf[pos] = ' ';
3175                         pos++;
3176                 }
3177
3178                 ConvertBuf2->BufUsed = CtdlDecodeQuotedPrintable(
3179                         ConvertBuf2->buf,
3180                         ConvertBuf->buf,
3181                         ConvertBuf->BufUsed);
3182         }
3183         else {
3184                 StrBufAppendBuf(ConvertBuf2, ConvertBuf, 0);
3185         }
3186 #ifdef HAVE_ICONV
3187         ctdl_iconv_open("UTF-8", charset, &ic);
3188         if (ic != (iconv_t)(-1) ) {
3189 #endif
3190                 StrBufConvert(ConvertBuf2, ConvertBuf, &ic);
3191                 StrBufAppendBuf(Target, ConvertBuf2, 0);
3192 #ifdef HAVE_ICONV
3193                 iconv_close(ic);
3194         }
3195         else {
3196                 StrBufAppendBufPlain(Target, HKEY("(unreadable)"), 0);
3197         }
3198 #endif
3199 }
3200
3201 /**
3202  * @ingroup StrBuf_DeEnCoder
3203  * @brief Handle subjects with RFC2047 encoding such as:
3204  * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
3205  * @param Target where to put the decoded string to
3206  * @param DecodeMe buffer with encoded string
3207  * @param DefaultCharset if we don't find one, which should we use?
3208  * @param FoundCharset overrides DefaultCharset if non-empty; If we find a charset inside of the string,
3209  *        put it here for later use where no string might be known.
3210  */
3211 void StrBuf_RFC822_to_Utf8(StrBuf *Target, const StrBuf *DecodeMe, const StrBuf* DefaultCharset, StrBuf *FoundCharset)
3212 {
3213         StrBuf *DecodedInvalidBuf = NULL;
3214         StrBuf *ConvertBuf, *ConvertBuf2;
3215         const StrBuf *DecodeMee = DecodeMe;
3216         char *start, *end, *next, *nextend, *ptr = NULL;
3217 #ifdef HAVE_ICONV
3218         iconv_t ic = (iconv_t)(-1) ;
3219 #endif
3220         const char *eptr;
3221         int passes = 0;
3222         int i, len, delta;
3223         int illegal_non_rfc2047_encoding = 0;
3224
3225         /* Sometimes, badly formed messages contain strings which were simply
3226          *  written out directly in some foreign character set instead of
3227          *  using RFC2047 encoding.  This is illegal but we will attempt to
3228          *  handle it anyway by converting from a user-specified default
3229          *  charset to UTF-8 if we see any nonprintable characters.
3230          */
3231
3232         len = StrLength(DecodeMe);
3233         for (i=0; i<DecodeMe->BufUsed; ++i) {
3234                 if ((DecodeMe->buf[i] < 32) || (DecodeMe->buf[i] > 126)) {
3235                         illegal_non_rfc2047_encoding = 1;
3236                         break;
3237                 }
3238         }
3239
3240         ConvertBuf = NewStrBufPlain(NULL, StrLength(DecodeMe));
3241         if ((illegal_non_rfc2047_encoding) &&
3242             (strcasecmp(ChrPtr(DefaultCharset), "UTF-8")) &&
3243             (strcasecmp(ChrPtr(DefaultCharset), "us-ascii")) )
3244         {
3245 #ifdef HAVE_ICONV
3246                 ctdl_iconv_open("UTF-8", ChrPtr(DefaultCharset), &ic);
3247                 if (ic != (iconv_t)(-1) ) {
3248                         DecodedInvalidBuf = NewStrBufDup(DecodeMe);
3249                         StrBufConvert(DecodedInvalidBuf, ConvertBuf, &ic);///TODO: don't void const?
3250                         DecodeMee = DecodedInvalidBuf;
3251                         iconv_close(ic);
3252                 }
3253 #endif
3254         }
3255
3256         /* pre evaluate the first pair */
3257         nextend = end = NULL;
3258         len = StrLength(DecodeMee);
3259         start = strstr(DecodeMee->buf, "=?");
3260         eptr = DecodeMee->buf + DecodeMee->BufUsed;
3261         if (start != NULL)
3262                 end = FindNextEnd (DecodeMee, start);
3263         else {
3264                 StrBufAppendBuf(Target, DecodeMee, 0);
3265                 FreeStrBuf(&ConvertBuf);
3266                 FreeStrBuf(&DecodedInvalidBuf);
3267                 return;
3268         }
3269
3270         ConvertBuf2 = NewStrBufPlain(NULL, StrLength(DecodeMee));
3271
3272         if (start != DecodeMee->buf) {
3273                 long nFront;
3274
3275                 nFront = start - DecodeMee->buf;
3276                 StrBufAppendBufPlain(Target, DecodeMee->buf, nFront, 0);
3277                 len -= nFront;
3278         }
3279         /*
3280          * Since spammers will go to all sorts of absurd lengths to get their
3281          * messages through, there are LOTS of corrupt headers out there.
3282          * So, prevent a really badly formed RFC2047 header from throwing
3283          * this function into an infinite loop.
3284          */
3285         while ((start != NULL) &&
3286                (end != NULL) &&
3287                (start < eptr) &&
3288                (end < eptr) &&
3289                (passes < 20))
3290         {
3291                 passes++;
3292                 DecodeSegment(Target,
3293                               DecodeMee,
3294                               start,
3295                               end,
3296                               ConvertBuf,
3297                               ConvertBuf2,
3298                               FoundCharset);
3299
3300                 next = strstr(end, "=?");
3301                 nextend = NULL;
3302                 if ((next != NULL) &&
3303                     (next < eptr))
3304                         nextend = FindNextEnd(DecodeMee, next);
3305                 if (nextend == NULL)
3306                         next = NULL;
3307
3308                 /* did we find two partitions */
3309                 if ((next != NULL) &&
3310                     ((next - end) > 2))
3311                 {
3312                         ptr = end + 2;
3313                         while ((ptr < next) &&
3314                                (isspace(*ptr) ||
3315                                 (*ptr == '\r') ||
3316                                 (*ptr == '\n') ||
3317                                 (*ptr == '\t')))
3318                                 ptr ++;
3319                         /* did we find a gab just filled with blanks? */
3320                         if (ptr == next)
3321                         {
3322                                 long gap = next - start;
3323                                 memmove (end + 2,
3324                                          next,
3325                                          len - (gap));
3326                                 len -= gap;
3327                                 /* now terminate the gab at the end */
3328                                 delta = (next - end) - 2; ////TODO: const!
3329                                 ((StrBuf*)DecodeMee)->BufUsed -= delta;
3330                                 ((StrBuf*)DecodeMee)->buf[DecodeMee->BufUsed] = '\0';
3331
3332                                 /* move next to its new location. */
3333                                 next -= delta;
3334                                 nextend -= delta;
3335                         }
3336                 }
3337                 /* our next-pair is our new first pair now. */
3338                 ptr = end + 2;
3339                 start = next;
3340                 end = nextend;
3341         }
3342         end = ptr;
3343         nextend = DecodeMee->buf + DecodeMee->BufUsed;
3344         if ((end != NULL) && (end < nextend)) {
3345                 ptr = end;
3346                 while ( (ptr < nextend) &&
3347                         (isspace(*ptr) ||
3348                          (*ptr == '\r') ||
3349                          (*ptr == '\n') ||
3350                          (*ptr == '\t')))
3351                         ptr ++;
3352                 if (ptr < nextend)
3353                         StrBufAppendBufPlain(Target, end, nextend - end, 0);
3354         }
3355         FreeStrBuf(&ConvertBuf);
3356         FreeStrBuf(&ConvertBuf2);
3357         FreeStrBuf(&DecodedInvalidBuf);
3358 }
3359
3360 /**
3361  * @ingroup StrBuf
3362  * @brief evaluate the length of an utf8 special character sequence
3363  * @param Char the character to examine
3364  * @returns width of utf8 chars in bytes
3365  */
3366 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE)
3367 {
3368         int n = 1;
3369         char test = (1<<7);
3370
3371         while ((n < 8) && ((test & *CharS) != 0)) {
3372                 test = test << 1;
3373                 n ++;
3374         }
3375         if ((n > 6) || ((CharE - CharS) < n))
3376                 n = 1;
3377         return n;
3378 }
3379
3380 /**
3381  * @ingroup StrBuf
3382  * @brief detect whether this char starts an utf-8 encoded char
3383  * @param Char character to inspect
3384  * @returns yes or no
3385  */
3386 static inline int Ctdl_IsUtf8SequenceStart(const char Char)
3387 {
3388 /** 11??.???? indicates an UTF8 Sequence. */
3389         return ((Char & 0xC0) != 0);
3390 }
3391
3392 /**
3393  * @ingroup StrBuf
3394  * @brief measure the number of glyphs in an UTF8 string...
3395  * @param Buf string to measure
3396  * @returns the number of glyphs in Buf
3397  */
3398 long StrBuf_Utf8StrLen(StrBuf *Buf)
3399 {
3400         int n = 0;
3401         int m = 0;
3402         char *aptr, *eptr;
3403
3404         if ((Buf == NULL) || (Buf->BufUsed == 0))
3405                 return 0;
3406         aptr = Buf->buf;
3407         eptr = Buf->buf + Buf->BufUsed;
3408         while ((aptr < eptr) && (*aptr != '\0')) {
3409                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
3410                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
3411                         while ((aptr < eptr) && (*aptr++ != '\0')&& (m-- > 0) );
3412                         n ++;
3413                 }
3414                 else {
3415                         n++;
3416                         aptr++;
3417                 }
3418         }
3419         return n;
3420 }
3421
3422 /**
3423  * @ingroup StrBuf
3424  * @brief cuts a string after maxlen glyphs
3425  * @param Buf string to cut to maxlen glyphs
3426  * @param maxlen how long may the string become?
3427  * @returns current length of the string
3428  */
3429 long StrBuf_Utf8StrCut(StrBuf *Buf, int maxlen)
3430 {
3431         char *aptr, *eptr;
3432         int n = 0, m = 0;
3433
3434         aptr = Buf->buf;
3435         eptr = Buf->buf + Buf->BufUsed;
3436         while ((aptr < eptr) && (*aptr != '\0')) {
3437                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
3438                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
3439                         while ((*aptr++ != '\0') && (m-- > 0));
3440                         n ++;
3441                 }
3442                 else {
3443                         n++;
3444                         aptr++;
3445                 }
3446                 if (n > maxlen) {
3447                         *aptr = '\0';
3448                         Buf->BufUsed = aptr - Buf->buf;
3449                         return Buf->BufUsed;
3450                 }
3451         }
3452         return Buf->BufUsed;
3453
3454 }
3455
3456
3457 /**
3458  * @ingroup StrBuf
3459  * @brief extract a "next line" from Buf; Ptr to persist across several iterations
3460  * @param LineBuf your line will be copied here.
3461  * @param Buf BLOB with lines of text...
3462  * @param Ptr moved arround to keep the next-line across several iterations
3463  *        has to be &NULL on start; will be &NotNULL on end of buffer
3464  * @returns size of copied buffer
3465  */
3466 int StrBufSipLine(StrBuf *LineBuf, StrBuf *Buf, const char **Ptr)
3467 {
3468         const char *aptr, *ptr, *eptr;
3469         char *optr, *xptr;
3470
3471         if ((Buf == NULL) || (*Ptr == StrBufNOTNULL)) {
3472                 *Ptr = StrBufNOTNULL;
3473                 return 0;
3474         }
3475
3476         FlushStrBuf(LineBuf);
3477         if (*Ptr==NULL)
3478                 ptr = aptr = Buf->buf;
3479         else
3480                 ptr = aptr = *Ptr;
3481
3482         optr = LineBuf->buf;
3483         eptr = Buf->buf + Buf->BufUsed;
3484         xptr = LineBuf->buf + LineBuf->BufSize - 1;
3485
3486         while ((ptr <= eptr) &&
3487                (*ptr != '\n') &&
3488                (*ptr != '\r') )
3489         {
3490                 *optr = *ptr;
3491                 optr++; ptr++;
3492                 if (optr == xptr) {
3493                         LineBuf->BufUsed = optr - LineBuf->buf;
3494                         IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
3495                         optr = LineBuf->buf + LineBuf->BufUsed;
3496                         xptr = LineBuf->buf + LineBuf->BufSize - 1;
3497                 }
3498         }
3499
3500         if ((ptr >= eptr) && (optr > LineBuf->buf))
3501                 optr --;
3502         LineBuf->BufUsed = optr - LineBuf->buf;
3503         *optr = '\0';
3504         if ((ptr <= eptr) && (*ptr == '\r'))
3505                 ptr ++;
3506         if ((ptr <= eptr) && (*ptr == '\n'))
3507                 ptr ++;
3508
3509         if (ptr < eptr) {
3510                 *Ptr = ptr;
3511         }
3512         else {
3513                 *Ptr = StrBufNOTNULL;
3514         }
3515
3516         return Buf->BufUsed - (ptr - Buf->buf);
3517 }
3518