libcitadel/lib/stringbuf.c

   1 #include "sysdep.h"
   2 #include <ctype.h>
   3 #include <errno.h>
   4 #include <string.h>
   5 #include <unistd.h>
   6 #include <string.h>
   7 #include <stdio.h>
   8 #include <sys/select.h>
   9 #include <fcntl.h>
  10 #include <sys/types.h>
  11 #define SHOW_ME_VAPPEND_PRINTF
  12 #include <stdarg.h>
  13 #include "libcitadel.h"
  14
  15 #ifdef HAVE_ICONV
  16 #include <iconv.h>
  17 #endif
  18
  19 #ifdef HAVE_BACKTRACE
  20 #include <execinfo.h>
  21 #endif
  22
  23 #ifdef HAVE_ZLIB
  24 #include <zlib.h>
  25 int ZEXPORT compress_gzip(Bytef * dest, size_t * destLen,
  26                           const Bytef * source, uLong sourceLen, int level);
  27 #endif
  28 int BaseStrBufSize = 64;
  29
  30 const char *StrBufNOTNULL = ((char*) NULL) - 1;
  31
  32 const char HexList[256][3] = {
  33         "00","01","02","03","04","05","06","07","08","09","0A","0B","0C","0D","0E","0F",
  34         "10","11","12","13","14","15","16","17","18","19","1A","1B","1C","1D","1E","1F",
  35         "20","21","22","23","24","25","26","27","28","29","2A","2B","2C","2D","2E","2F",
  36         "30","31","32","33","34","35","36","37","38","39","3A","3B","3C","3D","3E","3F",
  37         "40","41","42","43","44","45","46","47","48","49","4A","4B","4C","4D","4E","4F",
  38         "50","51","52","53","54","55","56","57","58","59","5A","5B","5C","5D","5E","5F",
  39         "60","61","62","63","64","65","66","67","68","69","6A","6B","6C","6D","6E","6F",
  40         "70","71","72","73","74","75","76","77","78","79","7A","7B","7C","7D","7E","7F",
  41         "80","81","82","83","84","85","86","87","88","89","8A","8B","8C","8D","8E","8F",
  42         "90","91","92","93","94","95","96","97","98","99","9A","9B","9C","9D","9E","9F",
  43         "A0","A1","A2","A3","A4","A5","A6","A7","A8","A9","AA","AB","AC","AD","AE","AF",
  44         "B0","B1","B2","B3","B4","B5","B6","B7","B8","B9","BA","BB","BC","BD","BE","BF",
  45         "C0","C1","C2","C3","C4","C5","C6","C7","C8","C9","CA","CB","CC","CD","CE","CF",
  46         "D0","D1","D2","D3","D4","D5","D6","D7","D8","D9","DA","DB","DC","DD","DE","DF",
  47         "E0","E1","E2","E3","E4","E5","E6","E7","E8","E9","EA","EB","EC","ED","EE","EF",
  48         "F0","F1","F2","F3","F4","F5","F6","F7","F8","F9","FA","FB","FC","FD","FE","FF"};
  49
  50 /**
  51  * @defgroup StrBuf Stringbuffer, A class for manipulating strings with dynamic buffers
  52  * StrBuf is a versatile class, aiding the handling of dynamic strings
  53  *  * reduce de/reallocations
  54  *  * reduce the need to remeasure it
  55  *  * reduce scanning over the string (in @ref StrBuf_NextTokenizer "Tokenizers")
  56  *  * allow asyncroneous IO for line and Blob based operations
  57  *  * reduce the use of memove in those
  58  *  * Quick filling in several operations with append functions
  59  */
  60
  61 /**
  62  * @defgroup StrBuf_DeConstructors Create/Destroy StrBufs
  63  * @ingroup StrBuf
  64  */
  65
  66 /**
  67  * @defgroup StrBuf_Cast Cast operators to interact with char* based code
  68  * @ingroup StrBuf
  69  * use these operators to interfere with code demanding char*;
  70  * if you need to own the content, smash me. Avoid, since we loose the length information.
  71  */
  72
  73 /**
  74  * @defgroup StrBuf_Filler Create/Replace/Append Content into a StrBuf
  75  * @ingroup StrBuf
  76  * operations to get your Strings into a StrBuf, manipulating them, or appending
  77  */
  78 /**
  79  * @defgroup StrBuf_NextTokenizer Fast tokenizer to pull tokens in sequence
  80  * @ingroup StrBuf
  81  * Quick tokenizer; demands of the user to pull its tokens in sequence
  82  */
  83
  84 /**
  85  * @defgroup StrBuf_Tokenizer tokenizer Functions; Slow ones.
  86  * @ingroup StrBuf
  87  * versatile tokenizer; random access to tokens, but slower; Prefer the @ref StrBuf_NextTokenizer "Next Tokenizer"
  88  */
  89
  90 /**
  91  * @defgroup StrBuf_BufferedIO Buffered IO with Asynchroneous reads and no unneeded memmoves (the fast ones)
  92  * @ingroup StrBuf
  93  * File IO to fill StrBufs; Works with work-buffer shared across several calls;
  94  * External Cursor to maintain the current read position inside of the buffer
  95  * the non-fast ones will use memove to keep the start of the buffer the read buffer (which is slower)
  96  */
  97
  98 /**
  99  * @defgroup StrBuf_IO FileIO; Prefer @ref StrBuf_BufferedIO
 100  * @ingroup StrBuf
 101  * Slow I/O; avoid.
 102  */
 103
 104 /**
 105  * @defgroup StrBuf_DeEnCoder functions to translate the contents of a buffer
 106  * @ingroup StrBuf
 107  * these functions translate the content of a buffer into another representation;
 108  * some are combined Fillers and encoders
 109  */
 110
 111 /**
 112  * Private Structure for the Stringbuffer
 113  */
 114 struct StrBuf {
 115         char *buf;         /**< the pointer to the dynamic buffer */
 116         long BufSize;      /**< how many spcae do we optain */
 117         long BufUsed;      /**< StNumber of Chars used excluding the trailing \\0 */
 118         int ConstBuf;      /**< are we just a wrapper arround a static buffer and musn't we be changed? */
 119 #ifdef SIZE_DEBUG
 120         long nIncreases;   /**< for profiling; cound how many times we needed more */
 121         char bt [SIZ];     /**< Stacktrace of last increase */
 122         char bt_lastinc [SIZ]; /**< How much did we increase last time? */
 123 #endif
 124 };
 125
 126
 127 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE);
 128 static inline int Ctdl_IsUtf8SequenceStart(const char Char);
 129
 130 #ifdef SIZE_DEBUG
 131 #ifdef HAVE_BACKTRACE
 132 static void StrBufBacktrace(StrBuf *Buf, int which)
 133 {
 134         int n;
 135         char *pstart, *pch;
 136         void *stack_frames[50];
 137         size_t size, i;
 138         char **strings;
 139
 140         if (which)
 141                 pstart = pch = Buf->bt;
 142         else
 143                 pstart = pch = Buf->bt_lastinc;
 144         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
 145         strings = backtrace_symbols(stack_frames, size);
 146         for (i = 0; i < size; i++) {
 147                 if (strings != NULL)
 148                         n = snprintf(pch, SIZ - (pch - pstart), "%s\\n", strings[i]);
 149                 else
 150                         n = snprintf(pch, SIZ - (pch - pstart), "%p\\n", stack_frames[i]);
 151                 pch += n;
 152         }
 153         free(strings);
 154
 155
 156 }
 157 #endif
 158
 159 void dbg_FreeStrBuf(StrBuf *FreeMe, char *FromWhere)
 160 {
 161         if (hFreeDbglog == -1){
 162                 pid_t pid = getpid();
 163                 char path [SIZ];
 164                 snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
 165                 hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
 166         }
 167         if ((*FreeMe)->nIncreases > 0)
 168         {
 169                 char buf[SIZ * 3];
 170                 long n;
 171                 n = snprintf(buf, SIZ * 3, "%c+|%ld|%ld|%ld|%s|%s|\n",
 172                              FromWhere,
 173                              (*FreeMe)->nIncreases,
 174                              (*FreeMe)->BufUsed,
 175                              (*FreeMe)->BufSize,
 176                              (*FreeMe)->bt,
 177                              (*FreeMe)->bt_lastinc);
 178                 n = write(hFreeDbglog, buf, n);
 179         }
 180         else
 181         {
 182                 char buf[128];
 183                 long n;
 184                 n = snprintf(buf, 128, "%c_|0|%ld%ld|\n",
 185                              FromWhere,
 186                              (*FreeMe)->BufUsed,
 187                              (*FreeMe)->BufSize);
 188                 n = write(hFreeDbglog, buf, n);
 189         }
 190 }
 191
 192 void dbg_IncreaseBuf(StrBuf *IncMe)
 193 {
 194         Buf->nIncreases++;
 195 #ifdef HAVE_BACKTRACE
 196         StrBufBacktrace(Buf, 1);
 197 #endif
 198 }
 199
 200 void dbg_Init(StrBuf *Buf)
 201 {
 202         Buf->nIncreases = 0;
 203         Buf->bt[0] = '\0';
 204         Buf->bt_lastinc[0] = '\0';
 205 #ifdef HAVE_BACKTRACE
 206         StrBufBacktrace(Buf, 0);
 207 #endif
 208 }
 209
 210 #else
 211 /* void it... */
 212 #define dbg_FreeStrBuf(a, b)
 213 #define dbg_IncreaseBuf(a)
 214 #define dbg_Init(a)
 215
 216 #endif
 217
 218 /**
 219  * @ingroup StrBuf_Cast
 220  * @brief Cast operator to Plain String
 221  * @note if the buffer is altered by StrBuf operations, this pointer may become
 222  *  invalid. So don't lean on it after altering the buffer!
 223  *  Since this operation is considered cheap, rather call it often than risking
 224  *  your pointer to become invalid!
 225  * @param Str the string we want to get the c-string representation for
 226  * @returns the Pointer to the Content. Don't mess with it!
 227  */
 228 inline const char *ChrPtr(const StrBuf *Str)
 229 {
 230         if (Str == NULL)
 231                 return "";
 232         return Str->buf;
 233 }
 234
 235 /**
 236  * @ingroup StrBuf_Cast
 237  * @brief since we know strlen()'s result, provide it here.
 238  * @param Str the string to return the length to
 239  * @returns contentlength of the buffer
 240  */
 241 inline int StrLength(const StrBuf *Str)
 242 {
 243         return (Str != NULL) ? Str->BufUsed : 0;
 244 }
 245
 246 /**
 247  * @ingroup StrBuf_DeConstructors
 248  * @brief local utility function to resize the buffer
 249  * @param Buf the buffer whichs storage we should increase
 250  * @param KeepOriginal should we copy the original buffer or just start over with a new one
 251  * @param DestSize what should fit in after?
 252  */
 253 static int IncreaseBuf(StrBuf *Buf, int KeepOriginal, int DestSize)
 254 {
 255         char *NewBuf;
 256         size_t NewSize = Buf->BufSize * 2;
 257
 258         if (Buf->ConstBuf)
 259                 return -1;
 260
 261         if (DestSize > 0)
 262                 while (NewSize <= DestSize)
 263                         NewSize *= 2;
 264
 265         NewBuf= (char*) malloc(NewSize);
 266         if (NewBuf == NULL)
 267                 return -1;
 268
 269         if (KeepOriginal && (Buf->BufUsed > 0))
 270         {
 271                 memcpy(NewBuf, Buf->buf, Buf->BufUsed);
 272         }
 273         else
 274         {
 275                 NewBuf[0] = '\0';
 276                 Buf->BufUsed = 0;
 277         }
 278         free (Buf->buf);
 279         Buf->buf = NewBuf;
 280         Buf->BufSize = NewSize;
 281
 282         dbg_IncreaseBuf(Buf);
 283
 284         return Buf->BufSize;
 285 }
 286
 287 /**
 288  * @ingroup StrBuf_DeConstructors
 289  * @brief shrink an _EMPTY_ buffer if its Buffer superseeds threshhold to NewSize. Buffercontent is thoroughly ignored and flushed.
 290  * @param Buf Buffer to shrink (has to be empty)
 291  * @param ThreshHold if the buffer is bigger then this, its readjusted
 292  * @param NewSize if we Shrink it, how big are we going to be afterwards?
 293  */
 294 void ReAdjustEmptyBuf(StrBuf *Buf, long ThreshHold, long NewSize)
 295 {
 296         if ((Buf != NULL) && (Buf->BufUsed > ThreshHold)) {
 297                 free(Buf->buf);
 298                 Buf->buf = (char*) malloc(NewSize);
 299                 Buf->BufUsed = 0;
 300                 Buf->BufSize = NewSize;
 301         }
 302 }
 303
 304 /**
 305  * @ingroup StrBuf_DeConstructors
 306  * @brief shrink long term buffers to their real size so they don't waste memory
 307  * @param Buf buffer to shrink
 308  * @param Force if not set, will just executed if the buffer is much to big; set for lifetime strings
 309  * @returns physical size of the buffer
 310  */
 311 long StrBufShrinkToFit(StrBuf *Buf, int Force)
 312 {
 313         if (Buf == NULL)
 314                 return -1;
 315         if (Force ||
 316             (Buf->BufUsed + (Buf->BufUsed / 3) > Buf->BufSize))
 317         {
 318                 char *TmpBuf = (char*) malloc(Buf->BufUsed + 1);
 319                 memcpy (TmpBuf, Buf->buf, Buf->BufUsed + 1);
 320                 Buf->BufSize = Buf->BufUsed + 1;
 321                 free(Buf->buf);
 322                 Buf->buf = TmpBuf;
 323         }
 324         return Buf->BufUsed;
 325 }
 326
 327 /**
 328  * @ingroup StrBuf_DeConstructors
 329  * @brief Allocate a new buffer with default buffer size
 330  * @returns the new stringbuffer
 331  */
 332 StrBuf* NewStrBuf(void)
 333 {
 334         StrBuf *NewBuf;
 335
 336         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 337         NewBuf->buf = (char*) malloc(BaseStrBufSize);
 338         NewBuf->buf[0] = '\0';
 339         NewBuf->BufSize = BaseStrBufSize;
 340         NewBuf->BufUsed = 0;
 341         NewBuf->ConstBuf = 0;
 342
 343         dbg_Init (NewBuf);
 344
 345         return NewBuf;
 346 }
 347
 348 /**
 349  * @ingroup StrBuf_DeConstructors
 350  * @brief Copy Constructor; returns a duplicate of CopyMe
 351  * @param CopyMe Buffer to faxmilate
 352  * @returns the new stringbuffer
 353  */
 354 StrBuf* NewStrBufDup(const StrBuf *CopyMe)
 355 {
 356         StrBuf *NewBuf;
 357
 358         if (CopyMe == NULL)
 359                 return NewStrBuf();
 360
 361         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 362         NewBuf->buf = (char*) malloc(CopyMe->BufSize);
 363         memcpy(NewBuf->buf, CopyMe->buf, CopyMe->BufUsed + 1);
 364         NewBuf->BufUsed = CopyMe->BufUsed;
 365         NewBuf->BufSize = CopyMe->BufSize;
 366         NewBuf->ConstBuf = 0;
 367
 368         dbg_Init(NewBuf);
 369
 370         return NewBuf;
 371 }
 372
 373 /**
 374  * @ingroup StrBuf_DeConstructors
 375  * @brief create a new Buffer using an existing c-string
 376  * this function should also be used if you want to pre-suggest
 377  * the buffer size to allocate in conjunction with ptr == NULL
 378  * @param ptr the c-string to copy; may be NULL to create a blank instance
 379  * @param nChars How many chars should we copy; -1 if we should measure the length ourselves
 380  * @returns the new stringbuffer
 381  */
 382 StrBuf* NewStrBufPlain(const char* ptr, int nChars)
 383 {
 384         StrBuf *NewBuf;
 385         size_t Siz = BaseStrBufSize;
 386         size_t CopySize;
 387
 388         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 389         if (nChars < 0)
 390                 CopySize = strlen((ptr != NULL)?ptr:"");
 391         else
 392                 CopySize = nChars;
 393
 394         while (Siz <= CopySize)
 395                 Siz *= 2;
 396
 397         NewBuf->buf = (char*) malloc(Siz);
 398         NewBuf->BufSize = Siz;
 399         if (ptr != NULL) {
 400                 memcpy(NewBuf->buf, ptr, CopySize);
 401                 NewBuf->buf[CopySize] = '\0';
 402                 NewBuf->BufUsed = CopySize;
 403         }
 404         else {
 405                 NewBuf->buf[0] = '\0';
 406                 NewBuf->BufUsed = 0;
 407         }
 408         NewBuf->ConstBuf = 0;
 409
 410         dbg_Init(NewBuf)
 411
 412                 return NewBuf;
 413 }
 414
 415 /**
 416  * @ingroup StrBuf_DeConstructors
 417  * @brief Set an existing buffer from a c-string
 418  * @param Buf buffer to load
 419  * @param ptr c-string to put into
 420  * @param nChars set to -1 if we should work 0-terminated
 421  * @returns the new length of the string
 422  */
 423 int StrBufPlain(StrBuf *Buf, const char* ptr, int nChars)
 424 {
 425         size_t Siz = Buf->BufSize;
 426         size_t CopySize;
 427
 428         if (nChars < 0)
 429                 CopySize = strlen(ptr);
 430         else
 431                 CopySize = nChars;
 432
 433         while (Siz <= CopySize)
 434                 Siz *= 2;
 435
 436         if (Siz != Buf->BufSize)
 437                 IncreaseBuf(Buf, 0, Siz);
 438         memcpy(Buf->buf, ptr, CopySize);
 439         Buf->buf[CopySize] = '\0';
 440         Buf->BufUsed = CopySize;
 441         Buf->ConstBuf = 0;
 442         return CopySize;
 443 }
 444
 445
 446 /**
 447  * @ingroup StrBuf_DeConstructors
 448  * @brief use strbuf as wrapper for a string constant for easy handling
 449  * @param StringConstant a string to wrap
 450  * @param SizeOfStrConstant should be sizeof(StringConstant)-1
 451  */
 452 StrBuf* _NewConstStrBuf(const char* StringConstant, size_t SizeOfStrConstant)
 453 {
 454         StrBuf *NewBuf;
 455
 456         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 457         NewBuf->buf = (char*) StringConstant;
 458         NewBuf->BufSize = SizeOfStrConstant;
 459         NewBuf->BufUsed = SizeOfStrConstant;
 460         NewBuf->ConstBuf = 1;
 461
 462         dbg_Init(NewBuf);
 463
 464         return NewBuf;
 465 }
 466
 467
 468 /**
 469  * @ingroup StrBuf_DeConstructors
 470  * @brief flush the content of a Buf; keep its struct
 471  * @param buf Buffer to flush
 472  */
 473 int FlushStrBuf(StrBuf *buf)
 474 {
 475         if (buf == NULL)
 476                 return -1;
 477         if (buf->ConstBuf)
 478                 return -1;
 479         buf->buf[0] ='\0';
 480         buf->BufUsed = 0;
 481         return 0;
 482 }
 483
 484 /**
 485  * @ingroup StrBuf_DeConstructors
 486  * @brief wipe the content of a Buf thoroughly (overwrite it -> expensive); keep its struct
 487  * @param buf Buffer to wipe
 488  */
 489 int FLUSHStrBuf(StrBuf *buf)
 490 {
 491         if (buf == NULL)
 492                 return -1;
 493         if (buf->ConstBuf)
 494                 return -1;
 495         if (buf->BufUsed > 0) {
 496                 memset(buf->buf, 0, buf->BufUsed);
 497                 buf->BufUsed = 0;
 498         }
 499         return 0;
 500 }
 501
 502 #ifdef SIZE_DEBUG
 503 int hFreeDbglog = -1;
 504 #endif
 505 /**
 506  * @ingroup StrBuf_DeConstructors
 507  * @brief Release a Buffer
 508  * Its a double pointer, so it can NULL your pointer
 509  * so fancy SIG11 appear instead of random results
 510  * @param FreeMe Pointer Pointer to the buffer to free
 511  */
 512 void FreeStrBuf (StrBuf **FreeMe)
 513 {
 514         if (*FreeMe == NULL)
 515                 return;
 516
 517         dbg_FreeStrBuf(FreeMe, 'F');
 518
 519         if (!(*FreeMe)->ConstBuf)
 520                 free((*FreeMe)->buf);
 521         free(*FreeMe);
 522         *FreeMe = NULL;
 523 }
 524
 525 /**
 526  * @ingroup StrBuf_DeConstructors
 527  * @brief flatten a Buffer to the Char * we return
 528  * Its a double pointer, so it can NULL your pointer
 529  * so fancy SIG11 appear instead of random results
 530  * The Callee then owns the buffer and is responsible for freeing it.
 531  * @param SmashMe Pointer Pointer to the buffer to release Buf from and free
 532  * @returns the pointer of the buffer; Callee owns the memory thereafter.
 533  */
 534 char *SmashStrBuf (StrBuf **SmashMe)
 535 {
 536         char *Ret;
 537
 538         if ((SmashMe == NULL) || (*SmashMe == NULL))
 539                 return NULL;
 540
 541         dbg_FreeStrBuf(SmashMe, 'S');
 542
 543         Ret = (*SmashMe)->buf;
 544         free(*SmashMe);
 545         *SmashMe = NULL;
 546         return Ret;
 547 }
 548
 549 /**
 550  * @ingroup StrBuf_DeConstructors
 551  * @brief Release the buffer
 552  * If you want put your StrBuf into a Hash, use this as Destructor.
 553  * @param VFreeMe untyped pointer to a StrBuf. be shure to do the right thing [TM]
 554  */
 555 void HFreeStrBuf (void *VFreeMe)
 556 {
 557         StrBuf *FreeMe = (StrBuf*)VFreeMe;
 558         if (FreeMe == NULL)
 559                 return;
 560
 561         dbg_FreeStrBuf(SmashMe, 'H');
 562
 563         if (!FreeMe->ConstBuf)
 564                 free(FreeMe->buf);
 565         free(FreeMe);
 566 }
 567
 568
 569 /*******************************************************************************
 570  *                      Simple string transformations                          *
 571  *******************************************************************************/
 572
 573 /**
 574  * @ingroup StrBuf
 575  * @brief Wrapper around atol
 576  */
 577 long StrTol(const StrBuf *Buf)
 578 {
 579         if (Buf == NULL)
 580                 return 0;
 581         if(Buf->BufUsed > 0)
 582                 return atol(Buf->buf);
 583         else
 584                 return 0;
 585 }
 586
 587 /**
 588  * @ingroup StrBuf
 589  * @brief Wrapper around atoi
 590  */
 591 int StrToi(const StrBuf *Buf)
 592 {
 593         if (Buf == NULL)
 594                 return 0;
 595         if (Buf->BufUsed > 0)
 596                 return atoi(Buf->buf);
 597         else
 598                 return 0;
 599 }
 600
 601 /**
 602  * @ingroup StrBuf
 603  * @brief Checks to see if the string is a pure number
 604  * @param Buf The buffer to inspect
 605  * @returns 1 if its a pure number, 0, if not.
 606  */
 607 int StrBufIsNumber(const StrBuf *Buf) {
 608         char * pEnd;
 609         if ((Buf == NULL) || (Buf->BufUsed == 0)) {
 610                 return 0;
 611         }
 612         strtoll(Buf->buf, &pEnd, 10);
 613         if (pEnd == Buf->buf)
 614                 return 0;
 615         if ((pEnd != NULL) && (pEnd == Buf->buf + Buf->BufUsed))
 616                 return 1;
 617         if (Buf->buf == pEnd)
 618                 return 0;
 619         return 0;
 620 }
 621 /**
 622  * @ingroup StrBuf_Filler
 623  * @brief modifies a Single char of the Buf
 624  * You can point to it via char* or a zero-based integer
 625  * @param Buf The buffer to manipulate
 626  * @param ptr char* to zero; use NULL if unused
 627  * @param nThChar zero based pointer into the string; use -1 if unused
 628  * @param PeekValue The Character to place into the position
 629  */
 630 long StrBufPeek(StrBuf *Buf, const char* ptr, long nThChar, char PeekValue)
 631 {
 632         if (Buf == NULL)
 633                 return -1;
 634         if (ptr != NULL)
 635                 nThChar = ptr - Buf->buf;
 636         if ((nThChar < 0) || (nThChar > Buf->BufUsed))
 637                 return -1;
 638         Buf->buf[nThChar] = PeekValue;
 639         return nThChar;
 640 }
 641
 642 /**
 643  * @ingroup StrBuf_Filler
 644  * @brief Append a StringBuffer to the buffer
 645  * @param Buf Buffer to modify
 646  * @param AppendBuf Buffer to copy at the end of our buffer
 647  * @param Offset Should we start copying from an offset?
 648  */
 649 void StrBufAppendBuf(StrBuf *Buf, const StrBuf *AppendBuf, unsigned long Offset)
 650 {
 651         if ((AppendBuf == NULL) || (Buf == NULL) || (AppendBuf->buf == NULL))
 652                 return;
 653
 654         if (Buf->BufSize - Offset < AppendBuf->BufUsed + Buf->BufUsed + 1)
 655                 IncreaseBuf(Buf,
 656                             (Buf->BufUsed > 0),
 657                             AppendBuf->BufUsed + Buf->BufUsed);
 658
 659         memcpy(Buf->buf + Buf->BufUsed,
 660                AppendBuf->buf + Offset,
 661                AppendBuf->BufUsed - Offset);
 662         Buf->BufUsed += AppendBuf->BufUsed - Offset;
 663         Buf->buf[Buf->BufUsed] = '\0';
 664 }
 665
 666
 667 /**
 668  * @ingroup StrBuf_Filler
 669  * @brief Append a C-String to the buffer
 670  * @param Buf Buffer to modify
 671  * @param AppendBuf Buffer to copy at the end of our buffer
 672  * @param AppendSize number of bytes to copy; set to -1 if we should count it in advance
 673  * @param Offset Should we start copying from an offset?
 674  */
 675 void StrBufAppendBufPlain(StrBuf *Buf, const char *AppendBuf, long AppendSize, unsigned long Offset)
 676 {
 677         long aps;
 678         long BufSizeRequired;
 679
 680         if ((AppendBuf == NULL) || (Buf == NULL))
 681                 return;
 682
 683         if (AppendSize < 0 )
 684                 aps = strlen(AppendBuf + Offset);
 685         else
 686                 aps = AppendSize - Offset;
 687
 688         BufSizeRequired = Buf->BufUsed + aps + 1;
 689         if (Buf->BufSize <= BufSizeRequired)
 690                 IncreaseBuf(Buf, (Buf->BufUsed > 0), BufSizeRequired);
 691
 692         memcpy(Buf->buf + Buf->BufUsed,
 693                AppendBuf + Offset,
 694                aps);
 695         Buf->BufUsed += aps;
 696         Buf->buf[Buf->BufUsed] = '\0';
 697 }
 698
 699 /**
 700  * @ingroup StrBuf_Filler
 701  * @brief sprintf like function appending the formated string to the buffer
 702  * vsnprintf version to wrap into own calls
 703  * @param Buf Buffer to extend by format and Params
 704  * @param format printf alike format to add
 705  * @param ap va_list containing the items for format
 706  */
 707 void StrBufVAppendPrintf(StrBuf *Buf, const char *format, va_list ap)
 708 {
 709         va_list apl;
 710         size_t BufSize;
 711         size_t nWritten;
 712         size_t Offset;
 713         size_t newused;
 714
 715         if ((Buf == NULL)  || (format == NULL))
 716                 return;
 717
 718         BufSize = Buf->BufSize;
 719         nWritten = Buf->BufSize + 1;
 720         Offset = Buf->BufUsed;
 721         newused = Offset + nWritten;
 722
 723         while (newused >= BufSize) {
 724                 va_copy(apl, ap);
 725                 nWritten = vsnprintf(Buf->buf + Offset,
 726                                      Buf->BufSize - Offset,
 727                                      format, apl);
 728                 va_end(apl);
 729                 newused = Offset + nWritten;
 730                 if (newused >= Buf->BufSize) {
 731                         IncreaseBuf(Buf, 1, newused);
 732                         newused = Buf->BufSize + 1;
 733                 }
 734                 else {
 735                         Buf->BufUsed = Offset + nWritten;
 736                         BufSize = Buf->BufSize;
 737                 }
 738
 739         }
 740 }
 741
 742 /**
 743  * @ingroup StrBuf_Filler
 744  * @brief sprintf like function appending the formated string to the buffer
 745  * @param Buf Buffer to extend by format and Params
 746  * @param format printf alike format to add
 747  */
 748 void StrBufAppendPrintf(StrBuf *Buf, const char *format, ...)
 749 {
 750         size_t BufSize;
 751         size_t nWritten;
 752         size_t Offset;
 753         size_t newused;
 754         va_list arg_ptr;
 755
 756         if ((Buf == NULL)  || (format == NULL))
 757                 return;
 758
 759         BufSize = Buf->BufSize;
 760         nWritten = Buf->BufSize + 1;
 761         Offset = Buf->BufUsed;
 762         newused = Offset + nWritten;
 763
 764         while (newused >= BufSize) {
 765                 va_start(arg_ptr, format);
 766                 nWritten = vsnprintf(Buf->buf + Buf->BufUsed,
 767                                      Buf->BufSize - Buf->BufUsed,
 768                                      format, arg_ptr);
 769                 va_end(arg_ptr);
 770                 newused = Buf->BufUsed + nWritten;
 771                 if (newused >= Buf->BufSize) {
 772                         IncreaseBuf(Buf, 1, newused);
 773                         newused = Buf->BufSize + 1;
 774                 }
 775                 else {
 776                         Buf->BufUsed += nWritten;
 777                         BufSize = Buf->BufSize;
 778                 }
 779
 780         }
 781 }
 782
 783 /**
 784  * @ingroup StrBuf_Filler
 785  * @brief sprintf like function putting the formated string into the buffer
 786  * @param Buf Buffer to extend by format and Parameters
 787  * @param format printf alike format to add
 788  */
 789 void StrBufPrintf(StrBuf *Buf, const char *format, ...)
 790 {
 791         size_t nWritten;
 792         va_list arg_ptr;
 793
 794         if ((Buf == NULL)  || (format == NULL))
 795                 return;
 796
 797         nWritten = Buf->BufSize + 1;
 798         while (nWritten >= Buf->BufSize) {
 799                 va_start(arg_ptr, format);
 800                 nWritten = vsnprintf(Buf->buf, Buf->BufSize, format, arg_ptr);
 801                 va_end(arg_ptr);
 802                 if (nWritten >= Buf->BufSize) {
 803                         IncreaseBuf(Buf, 0, 0);
 804                         nWritten = Buf->BufSize + 1;
 805                         continue;
 806                 }
 807                 Buf->BufUsed = nWritten ;
 808         }
 809 }
 810
 811 /**
 812  * @ingroup StrBuf_Filler
 813  * @brief Callback for cURL to append the webserver reply to a buffer
 814  * @param ptr pre-defined by the cURL API; see man 3 curl for mre info
 815  * @param size pre-defined by the cURL API; see man 3 curl for mre info
 816  * @param nmemb pre-defined by the cURL API; see man 3 curl for mre info
 817  * @param stream pre-defined by the cURL API; see man 3 curl for mre info
 818  */
 819 size_t CurlFillStrBuf_callback(void *ptr, size_t size, size_t nmemb, void *stream)
 820 {
 821
 822         StrBuf *Target;
 823
 824         Target = stream;
 825         if (ptr == NULL)
 826                 return 0;
 827
 828         StrBufAppendBufPlain(Target, ptr, size * nmemb, 0);
 829         return size * nmemb;
 830 }
 831
 832
 833 /**
 834  * @ingroup StrBuf
 835  * @brief extracts a substring from Source into dest
 836  * @param dest buffer to place substring into
 837  * @param Source string to copy substring from
 838  * @param Offset chars to skip from start
 839  * @param nChars number of chars to copy
 840  * @returns the number of chars copied; may be different from nChars due to the size of Source
 841  */
 842 int StrBufSub(StrBuf *dest, const StrBuf *Source, unsigned long Offset, size_t nChars)
 843 {
 844         size_t NCharsRemain;
 845         if (Offset > Source->BufUsed)
 846         {
 847                 FlushStrBuf(dest);
 848                 return 0;
 849         }
 850         if (Offset + nChars < Source->BufUsed)
 851         {
 852                 if (nChars >= dest->BufSize)
 853                         IncreaseBuf(dest, 0, nChars + 1);
 854                 memcpy(dest->buf, Source->buf + Offset, nChars);
 855                 dest->BufUsed = nChars;
 856                 dest->buf[dest->BufUsed] = '\0';
 857                 return nChars;
 858         }
 859         NCharsRemain = Source->BufUsed - Offset;
 860         if (NCharsRemain  >= dest->BufSize)
 861                 IncreaseBuf(dest, 0, NCharsRemain + 1);
 862         memcpy(dest->buf, Source->buf + Offset, NCharsRemain);
 863         dest->BufUsed = NCharsRemain;
 864         dest->buf[dest->BufUsed] = '\0';
 865         return NCharsRemain;
 866 }
 867
 868 /**
 869  * @ingroup StrBuf
 870  * @brief Cut nChars from the start of the string
 871  * @param Buf Buffer to modify
 872  * @param nChars how many chars should be skipped?
 873  */
 874 void StrBufCutLeft(StrBuf *Buf, int nChars)
 875 {
 876         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
 877         if (nChars >= Buf->BufUsed) {
 878                 FlushStrBuf(Buf);
 879                 return;
 880         }
 881         memmove(Buf->buf, Buf->buf + nChars, Buf->BufUsed - nChars);
 882         Buf->BufUsed -= nChars;
 883         Buf->buf[Buf->BufUsed] = '\0';
 884 }
 885
 886 /**
 887  * @ingroup StrBuf
 888  * @brief Cut the trailing n Chars from the string
 889  * @param Buf Buffer to modify
 890  * @param nChars how many chars should be trunkated?
 891  */
 892 void StrBufCutRight(StrBuf *Buf, int nChars)
 893 {
 894         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
 895         if (nChars >= Buf->BufUsed) {
 896                 FlushStrBuf(Buf);
 897                 return;
 898         }
 899         Buf->BufUsed -= nChars;
 900         Buf->buf[Buf->BufUsed] = '\0';
 901 }
 902
 903 /**
 904  * @ingroup StrBuf
 905  * @brief Cut the string after n Chars
 906  * @param Buf Buffer to modify
 907  * @param AfternChars after how many chars should we trunkate the string?
 908  * @param At if non-null and points inside of our string, cut it there.
 909  */
 910 void StrBufCutAt(StrBuf *Buf, int AfternChars, const char *At)
 911 {
 912         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
 913         if (At != NULL){
 914                 AfternChars = At - Buf->buf;
 915         }
 916
 917         if ((AfternChars < 0) || (AfternChars >= Buf->BufUsed))
 918                 return;
 919         Buf->BufUsed = AfternChars;
 920         Buf->buf[Buf->BufUsed] = '\0';
 921 }
 922
 923
 924 /**
 925  * @ingroup StrBuf
 926  * @brief Strip leading and trailing spaces from a string; with premeasured and adjusted length.
 927  * @param Buf the string to modify
 928  */
 929 void StrBufTrim(StrBuf *Buf)
 930 {
 931         int delta = 0;
 932         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
 933
 934         while ((Buf->BufUsed > 0) &&
 935                isspace(Buf->buf[Buf->BufUsed - 1]))
 936         {
 937                 Buf->BufUsed --;
 938         }
 939         Buf->buf[Buf->BufUsed] = '\0';
 940
 941         if (Buf->BufUsed == 0) return;
 942
 943         while ((Buf->BufUsed > delta) && (isspace(Buf->buf[delta]))){
 944                 delta ++;
 945         }
 946         if (delta > 0) StrBufCutLeft(Buf, delta);
 947
 948 }
 949
 950 /**
 951  * @ingroup StrBuf_Filler
 952  * @brief uppercase the contents of a buffer
 953  * @param Buf the buffer to translate
 954  */
 955 void StrBufUpCase(StrBuf *Buf)
 956 {
 957         char *pch, *pche;
 958
 959         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
 960
 961         pch = Buf->buf;
 962         pche = pch + Buf->BufUsed;
 963         while (pch < pche) {
 964                 *pch = toupper(*pch);
 965                 pch ++;
 966         }
 967 }
 968
 969
 970 /**
 971  * @ingroup StrBuf_Filler
 972  * @brief lowercase the contents of a buffer
 973  * @param Buf the buffer to translate
 974  */
 975 void StrBufLowerCase(StrBuf *Buf)
 976 {
 977         char *pch, *pche;
 978
 979         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
 980
 981         pch = Buf->buf;
 982         pche = pch + Buf->BufUsed;
 983         while (pch < pche) {
 984                 *pch = tolower(*pch);
 985                 pch ++;
 986         }
 987 }
 988
 989
 990 /*******************************************************************************
 991  *           a tokenizer that kills, maims, and destroys                       *
 992  *******************************************************************************/
 993
 994 /**
 995  * @ingroup StrBuf_Tokenizer
 996  * @brief Replace a token at a given place with a given length by another token with given length
 997  * @param Buf String where to work on
 998  * @param where where inside of the Buf is the search-token
 999  * @param HowLong How long is the token to be replaced
1000  * @param Repl Token to insert at 'where'
1001  * @param ReplLen Length of repl
1002  * @returns -1 if fail else length of resulting Buf
1003  */
1004 int StrBufReplaceToken(StrBuf *Buf, long where, long HowLong,
1005                        const char *Repl, long ReplLen)
1006 {
1007
1008         if ((Buf == NULL) ||
1009             (where > Buf->BufUsed) ||
1010             (where + HowLong > Buf->BufUsed))
1011                 return -1;
1012
1013         if (where + ReplLen - HowLong > Buf->BufSize)
1014                 if (IncreaseBuf(Buf, 1, Buf->BufUsed + ReplLen) < 0)
1015                         return -1;
1016
1017         memmove(Buf->buf + where + HowLong,
1018                 Buf->buf + where + ReplLen,
1019                 Buf->BufUsed - where - HowLong);
1020
1021         memcpy(Buf->buf + where,
1022                Repl, ReplLen);
1023
1024         Buf->BufUsed += ReplLen - HowLong;
1025
1026         return Buf->BufUsed;
1027 }
1028
1029 /**
1030  * @ingroup StrBuf_Tokenizer
1031  * @brief Counts the numbmer of tokens in a buffer
1032  * @param source String to count tokens in
1033  * @param tok    Tokenizer char to count
1034  * @returns numbers of tokenizer chars found
1035  */
1036 int StrBufNum_tokens(const StrBuf *source, char tok)
1037 {
1038         char *pch, *pche;
1039         long NTokens;
1040         if ((source == NULL) || (source->BufUsed == 0))
1041                 return 0;
1042         if ((source->BufUsed == 1) && (*source->buf == tok))
1043                 return 2;
1044         NTokens = 1;
1045         pch = source->buf;
1046         pche = pch + source->BufUsed;
1047         while (pch < pche)
1048         {
1049                 if (*pch == tok)
1050                         NTokens ++;
1051                 pch ++;
1052         }
1053         return NTokens;
1054 }
1055
1056 /**
1057  * @ingroup StrBuf_Tokenizer
1058  * @brief a string tokenizer
1059  * @param Source StringBuffer to read into
1060  * @param parmnum n'th Parameter to remove
1061  * @param separator tokenizer character
1062  * @returns -1 if not found, else length of token.
1063  */
1064 int StrBufRemove_token(StrBuf *Source, int parmnum, char separator)
1065 {
1066         int ReducedBy;
1067         char *d, *s, *end;              /* dest, source */
1068         int count = 0;
1069
1070         /* Find desired @parameter */
1071         end = Source->buf + Source->BufUsed;
1072         d = Source->buf;
1073         while ((d <= end) &&
1074                (count < parmnum))
1075         {
1076                 /* End of string, bail! */
1077                 if (!*d) {
1078                         d = NULL;
1079                         break;
1080                 }
1081                 if (*d == separator) {
1082                         count++;
1083                 }
1084                 d++;
1085         }
1086         if ((d == NULL) || (d >= end))
1087                 return 0;               /* @Parameter not found */
1088
1089         /* Find next @parameter */
1090         s = d;
1091         while ((s <= end) &&
1092                (*s && *s != separator))
1093         {
1094                 s++;
1095         }
1096         if (*s == separator)
1097                 s++;
1098         ReducedBy = d - s;
1099
1100         /* Hack and slash */
1101         if (s >= end) {
1102                 return 0;
1103         }
1104         else if (*s) {
1105                 memmove(d, s, Source->BufUsed - (s - Source->buf));
1106                 Source->BufUsed += ReducedBy;
1107                 Source->buf[Source->BufUsed] = '\0';
1108         }
1109         else if (d == Source->buf) {
1110                 *d = 0;
1111                 Source->BufUsed = 0;
1112         }
1113         else {
1114                 *--d = '\0';
1115                 Source->BufUsed += ReducedBy;
1116         }
1117         /*
1118         while (*s) {
1119                 *d++ = *s++;
1120         }
1121         *d = 0;
1122         */
1123         return ReducedBy;
1124 }
1125
1126
1127 /**
1128  * @ingroup StrBuf_Tokenizer
1129  * @brief a string tokenizer
1130  * @param dest Destination StringBuffer
1131  * @param Source StringBuffer to read into
1132  * @param parmnum n'th Parameter to extract
1133  * @param separator tokenizer character
1134  * @returns -1 if not found, else length of token.
1135  */
1136 int StrBufExtract_token(StrBuf *dest, const StrBuf *Source, int parmnum, char separator)
1137 {
1138         const char *s, *e;              //* source * /
1139         int len = 0;                    //* running total length of extracted string * /
1140         int current_token = 0;          //* token currently being processed * /
1141
1142         if (dest != NULL) {
1143                 dest->buf[0] = '\0';
1144                 dest->BufUsed = 0;
1145         }
1146         else
1147                 return(-1);
1148
1149         if ((Source == NULL) || (Source->BufUsed ==0)) {
1150                 return(-1);
1151         }
1152         s = Source->buf;
1153         e = s + Source->BufUsed;
1154
1155         //cit_backtrace();
1156         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1157
1158         while ((s<e) && !IsEmptyStr(s)) {
1159                 if (*s == separator) {
1160                         ++current_token;
1161                 }
1162                 if (len >= dest->BufSize) {
1163                         dest->BufUsed = len;
1164                         if (IncreaseBuf(dest, 1, -1) < 0) {
1165                                 dest->BufUsed --;
1166                                 break;
1167                         }
1168                 }
1169                 if ( (current_token == parmnum) &&
1170                      (*s != separator)) {
1171                         dest->buf[len] = *s;
1172                         ++len;
1173                 }
1174                 else if (current_token > parmnum) {
1175                         break;
1176                 }
1177                 ++s;
1178         }
1179
1180         dest->buf[len] = '\0';
1181         dest->BufUsed = len;
1182
1183         if (current_token < parmnum) {
1184                 //lprintf (CTDL_DEBUG,"test <!: %s\n", dest);
1185                 return(-1);
1186         }
1187         //lprintf (CTDL_DEBUG,"test <: %d; %s\n", len, dest);
1188         return(len);
1189 }
1190
1191
1192
1193
1194
1195 /**
1196  * @ingroup StrBuf_Tokenizer
1197  * @brief a string tokenizer to fetch an integer
1198  * @param Source String containing tokens
1199  * @param parmnum n'th Parameter to extract
1200  * @param separator tokenizer character
1201  * @returns 0 if not found, else integer representation of the token
1202  */
1203 int StrBufExtract_int(const StrBuf* Source, int parmnum, char separator)
1204 {
1205         StrBuf tmp;
1206         char buf[64];
1207
1208         tmp.buf = buf;
1209         buf[0] = '\0';
1210         tmp.BufSize = 64;
1211         tmp.BufUsed = 0;
1212         tmp.ConstBuf = 1;
1213         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1214                 return(atoi(buf));
1215         else
1216                 return 0;
1217 }
1218
1219 /**
1220  * @ingroup StrBuf_Tokenizer
1221  * @brief a string tokenizer to fetch a long integer
1222  * @param Source String containing tokens
1223  * @param parmnum n'th Parameter to extract
1224  * @param separator tokenizer character
1225  * @returns 0 if not found, else long integer representation of the token
1226  */
1227 long StrBufExtract_long(const StrBuf* Source, int parmnum, char separator)
1228 {
1229         StrBuf tmp;
1230         char buf[64];
1231
1232         tmp.buf = buf;
1233         buf[0] = '\0';
1234         tmp.BufSize = 64;
1235         tmp.BufUsed = 0;
1236         tmp.ConstBuf = 1;
1237         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1238                 return(atoi(buf));
1239         else
1240                 return 0;
1241 }
1242
1243
1244 /**
1245  * @ingroup StrBuf_Tokenizer
1246  * @brief a string tokenizer to fetch an unsigned long
1247  * @param Source String containing tokens
1248  * @param parmnum n'th Parameter to extract
1249  * @param separator tokenizer character
1250  * @returns 0 if not found, else unsigned long representation of the token
1251  */
1252 unsigned long StrBufExtract_unsigned_long(const StrBuf* Source, int parmnum, char separator)
1253 {
1254         StrBuf tmp;
1255         char buf[64];
1256         char *pnum;
1257
1258         tmp.buf = buf;
1259         buf[0] = '\0';
1260         tmp.BufSize = 64;
1261         tmp.BufUsed = 0;
1262         tmp.ConstBuf = 1;
1263         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0) {
1264                 pnum = &buf[0];
1265                 if (*pnum == '-')
1266                         pnum ++;
1267                 return (unsigned long) atol(pnum);
1268         }
1269         else
1270                 return 0;
1271 }
1272
1273
1274
1275 /**
1276  * @ingroup StrBuf_NextTokenizer
1277  * @brief a string tokenizer; Bounds checker
1278  *  function to make shure whether StrBufExtract_NextToken and friends have reached the end of the string.
1279  * @param Source our tokenbuffer
1280  * @param pStart the token iterator pointer to inspect
1281  * @returns whether the revolving pointer is inside of the search range
1282  */
1283 int StrBufHaveNextToken(const StrBuf *Source, const char **pStart)
1284 {
1285         if ((Source == NULL) ||
1286             (*pStart == StrBufNOTNULL) ||
1287             (Source->BufUsed == 0))
1288         {
1289                 return 0;
1290         }
1291         if (*pStart == NULL)
1292         {
1293                 return 1;
1294         }
1295         else if (*pStart > Source->buf + Source->BufUsed)
1296         {
1297                 return 0;
1298         }
1299         else if (*pStart <= Source->buf)
1300         {
1301                 return 0;
1302         }
1303
1304         return 1;
1305 }
1306
1307 /**
1308  * @ingroup StrBuf_NextTokenizer
1309  * @brief a string tokenizer
1310  * @param dest Destination StringBuffer
1311  * @param Source StringBuffer to read into
1312  * @param pStart pointer to the end of the last token. Feed with NULL on start.
1313  * @param separator tokenizer
1314  * @returns -1 if not found, else length of token.
1315  */
1316 int StrBufExtract_NextToken(StrBuf *dest, const StrBuf *Source, const char **pStart, char separator)
1317 {
1318         const char *s;          /* source */
1319         const char *EndBuffer;  /* end stop of source buffer */
1320         int current_token = 0;  /* token currently being processed */
1321         int len = 0;            /* running total length of extracted string */
1322
1323         if ((Source          == NULL) ||
1324             (Source->BufUsed == 0)      )
1325         {
1326                 *pStart = StrBufNOTNULL;
1327                 return -1;
1328         }
1329
1330         EndBuffer = Source->buf + Source->BufUsed;
1331
1332         if (dest != NULL)
1333         {
1334                 dest->buf[0] = '\0';
1335                 dest->BufUsed = 0;
1336         }
1337         else
1338         {
1339                 *pStart = EndBuffer + 1;
1340                 return -1;
1341         }
1342
1343         if (*pStart == NULL)
1344         {
1345                 *pStart = Source->buf; /* we're starting to examine this buffer. */
1346         }
1347         else if ((*pStart < Source->buf) ||
1348                  (*pStart > EndBuffer  )   )
1349         {
1350                 return -1; /* no more tokens to find. */
1351         }
1352
1353         s = *pStart;
1354         /* start to find the next token */
1355         while ((s <= EndBuffer)      &&
1356                (current_token == 0) )
1357         {
1358                 if (*s == separator)
1359                 {
1360                         /* we found the next token */
1361                         ++current_token;
1362                 }
1363
1364                 if (len >= dest->BufSize)
1365                 {
1366                         /* our Dest-buffer isn't big enough, increase it. */
1367                         dest->BufUsed = len;
1368
1369                         if (IncreaseBuf(dest, 1, -1) < 0) {
1370                                 /* WHUT? no more mem? bail out. */
1371                                 s = EndBuffer;
1372                                 dest->BufUsed --;
1373                                 break;
1374                         }
1375                 }
1376
1377                 if ( (current_token == 0 ) &&   /* are we in our target token? */
1378                      (!IsEmptyStr(s)     ) &&
1379                      (separator     != *s)    ) /* don't copy the token itself */
1380                 {
1381                         dest->buf[len] = *s;    /* Copy the payload */
1382                         ++len;                  /* remember the bigger size. */
1383                 }
1384
1385                 ++s;
1386         }
1387
1388         /* did we reach the end? */
1389         if ((s > EndBuffer)) {
1390                 EndBuffer = StrBufNOTNULL;
1391                 *pStart = EndBuffer;
1392         }
1393         else {
1394                 *pStart = s;  /* remember the position for the next run */
1395         }
1396
1397         /* sanitize our extracted token */
1398         dest->buf[len] = '\0';
1399         dest->BufUsed  = len;
1400
1401         return (len);
1402 }
1403
1404
1405 /**
1406  * @ingroup StrBuf_NextTokenizer
1407  * @brief a string tokenizer
1408  * @param Source StringBuffer to read from
1409  * @param pStart pointer to the end of the last token. Feed with NULL.
1410  * @param separator tokenizer character
1411  * @param nTokens number of tokens to fastforward over
1412  * @returns -1 if not found, else length of token.
1413  */
1414 int StrBufSkip_NTokenS(const StrBuf *Source, const char **pStart, char separator, int nTokens)
1415 {
1416         const char *s, *EndBuffer;      //* source * /
1417         int len = 0;                    //* running total length of extracted string * /
1418         int current_token = 0;          //* token currently being processed * /
1419
1420         if ((Source == NULL) ||
1421             (Source->BufUsed ==0)) {
1422                 return(-1);
1423         }
1424         if (nTokens == 0)
1425                 return Source->BufUsed;
1426
1427         if (*pStart == NULL)
1428                 *pStart = Source->buf;
1429
1430         EndBuffer = Source->buf + Source->BufUsed;
1431
1432         if ((*pStart < Source->buf) ||
1433             (*pStart >  EndBuffer)) {
1434                 return (-1);
1435         }
1436
1437
1438         s = *pStart;
1439
1440         //cit_backtrace();
1441         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1442
1443         while ((s<EndBuffer) && !IsEmptyStr(s)) {
1444                 if (*s == separator) {
1445                         ++current_token;
1446                 }
1447                 if (current_token >= nTokens) {
1448                         break;
1449                 }
1450                 ++s;
1451         }
1452         *pStart = s;
1453         (*pStart) ++;
1454
1455         return(len);
1456 }
1457
1458 /**
1459  * @ingroup StrBuf_NextTokenizer
1460  * @brief a string tokenizer to fetch an integer
1461  * @param Source StringBuffer to read from
1462  * @param pStart Cursor on the tokenstring
1463  * @param separator tokenizer character
1464  * @returns 0 if not found, else integer representation of the token
1465  */
1466 int StrBufExtractNext_int(const StrBuf* Source, const char **pStart, char separator)
1467 {
1468         StrBuf tmp;
1469         char buf[64];
1470
1471         tmp.buf = buf;
1472         buf[0] = '\0';
1473         tmp.BufSize = 64;
1474         tmp.BufUsed = 0;
1475         tmp.ConstBuf = 1;
1476         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1477                 return(atoi(buf));
1478         else
1479                 return 0;
1480 }
1481
1482 /**
1483  * @ingroup StrBuf_NextTokenizer
1484  * @brief a string tokenizer to fetch a long integer
1485  * @param Source StringBuffer to read from
1486  * @param pStart Cursor on the tokenstring
1487  * @param separator tokenizer character
1488  * @returns 0 if not found, else long integer representation of the token
1489  */
1490 long StrBufExtractNext_long(const StrBuf* Source, const char **pStart, char separator)
1491 {
1492         StrBuf tmp;
1493         char buf[64];
1494
1495         tmp.buf = buf;
1496         buf[0] = '\0';
1497         tmp.BufSize = 64;
1498         tmp.BufUsed = 0;
1499         tmp.ConstBuf = 1;
1500         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1501                 return(atoi(buf));
1502         else
1503                 return 0;
1504 }
1505
1506
1507 /**
1508  * @ingroup StrBuf_NextTokenizer
1509  * @brief a string tokenizer to fetch an unsigned long
1510  * @param Source StringBuffer to read from
1511  * @param pStart Cursor on the tokenstring
1512  * @param separator tokenizer character
1513  * @returns 0 if not found, else unsigned long representation of the token
1514  */
1515 unsigned long StrBufExtractNext_unsigned_long(const StrBuf* Source, const char **pStart, char separator)
1516 {
1517         StrBuf tmp;
1518         char buf[64];
1519         char *pnum;
1520
1521         tmp.buf = buf;
1522         buf[0] = '\0';
1523         tmp.BufSize = 64;
1524         tmp.BufUsed = 0;
1525         tmp.ConstBuf = 1;
1526         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0) {
1527                 pnum = &buf[0];
1528                 if (*pnum == '-')
1529                         pnum ++;
1530                 return (unsigned long) atol(pnum);
1531         }
1532         else
1533                 return 0;
1534 }
1535
1536
1537
1538
1539
1540 /*******************************************************************************
1541  *                             Escape Appending                                *
1542  *******************************************************************************/
1543
1544 /**
1545  * @ingroup StrBuf_DeEnCoder
1546  * @brief Escape a string for feeding out as a URL while appending it to a Buffer
1547  * @param OutBuf the output buffer
1548  * @param In Buffer to encode
1549  * @param PlainIn way in from plain old c strings
1550  */
1551 void StrBufUrlescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
1552 {
1553         const char *pch, *pche;
1554         char *pt, *pte;
1555         int len;
1556
1557         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1558                 return;
1559         if (PlainIn != NULL) {
1560                 len = strlen(PlainIn);
1561                 pch = PlainIn;
1562                 pche = pch + len;
1563         }
1564         else {
1565                 pch = In->buf;
1566                 pche = pch + In->BufUsed;
1567                 len = In->BufUsed;
1568         }
1569
1570         if (len == 0)
1571                 return;
1572
1573         pt = OutBuf->buf + OutBuf->BufUsed;
1574         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1575
1576         while (pch < pche) {
1577                 if (pt >= pte) {
1578                         IncreaseBuf(OutBuf, 1, -1);
1579                         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1580                         pt = OutBuf->buf + OutBuf->BufUsed;
1581                 }
1582
1583                 if((*pch >= 'a' && *pch <= 'z') ||
1584                    (*pch >= '@' && *pch <= 'Z') || /* @ A-Z */
1585                    (*pch >= '0' && *pch <= ':') || /* 0-9 : */
1586                    (*pch == '!') || (*pch == '_') ||
1587                    (*pch == ',') || (*pch == '.'))
1588                 {
1589                         *(pt++) = *(pch++);
1590                         OutBuf->BufUsed++;
1591                 }
1592                 else {
1593                         *pt = '%';
1594                         *(pt + 1) = HexList[(unsigned char)*pch][0];
1595                         *(pt + 2) = HexList[(unsigned char)*pch][1];
1596                         pt += 3;
1597                         OutBuf->BufUsed += 3;
1598                         pch ++;
1599                 }
1600         }
1601         *pt = '\0';
1602 }
1603
1604 /**
1605  * @ingroup StrBuf_DeEnCoder
1606  * @brief append a string in hex encoding to the buffer
1607  * @param OutBuf the output buffer
1608  * @param In Buffer to encode
1609  * @param PlainIn way in from plain old c strings
1610  */
1611 void StrBufHexescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
1612 {
1613         const char *pch, *pche;
1614         char *pt, *pte;
1615         int len;
1616
1617         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1618                 return;
1619         if (PlainIn != NULL) {
1620                 len = strlen(PlainIn);
1621                 pch = PlainIn;
1622                 pche = pch + len;
1623         }
1624         else {
1625                 pch = In->buf;
1626                 pche = pch + In->BufUsed;
1627                 len = In->BufUsed;
1628         }
1629
1630         if (len == 0)
1631                 return;
1632
1633         pt = OutBuf->buf + OutBuf->BufUsed;
1634         pte = OutBuf->buf + OutBuf->BufSize - 3; /**< we max append 3 chars at once plus the \0 */
1635
1636         while (pch < pche) {
1637                 if (pt >= pte) {
1638                         IncreaseBuf(OutBuf, 1, -1);
1639                         pte = OutBuf->buf + OutBuf->BufSize - 3; /**< we max append 3 chars at once plus the \0 */
1640                         pt = OutBuf->buf + OutBuf->BufUsed;
1641                 }
1642
1643                 *pt = HexList[(unsigned char)*pch][0];
1644                 pt ++;
1645                 *pt = HexList[(unsigned char)*pch][1];
1646                 pt ++; pch ++; OutBuf->BufUsed += 2;
1647         }
1648         *pt = '\0';
1649 }
1650
1651 /**
1652  * @ingroup StrBuf_DeEnCoder
1653  * @brief Append a string, escaping characters which have meaning in HTML.
1654  *
1655  * @param Target        target buffer
1656  * @param Source        source buffer; set to NULL if you just have a C-String
1657  * @param PlainIn       Plain-C string to append; set to NULL if unused
1658  * @param nbsp          If nonzero, spaces are converted to non-breaking spaces.
1659  * @param nolinebreaks  if set to 1, linebreaks are removed from the string.
1660  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
1661  */
1662 long StrEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
1663 {
1664         const char *aptr, *eiptr;
1665         char *bptr, *eptr;
1666         long len;
1667
1668         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
1669                 return -1;
1670
1671         if (PlainIn != NULL) {
1672                 aptr = PlainIn;
1673                 len = strlen(PlainIn);
1674                 eiptr = aptr + len;
1675         }
1676         else {
1677                 aptr = Source->buf;
1678                 eiptr = aptr + Source->BufUsed;
1679                 len = Source->BufUsed;
1680         }
1681
1682         if (len == 0)
1683                 return -1;
1684
1685         bptr = Target->buf + Target->BufUsed;
1686         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
1687
1688         while (aptr < eiptr){
1689                 if(bptr >= eptr) {
1690                         IncreaseBuf(Target, 1, -1);
1691                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
1692                         bptr = Target->buf + Target->BufUsed;
1693                 }
1694                 if (*aptr == '<') {
1695                         memcpy(bptr, "&lt;", 4);
1696                         bptr += 4;
1697                         Target->BufUsed += 4;
1698                 }
1699                 else if (*aptr == '>') {
1700                         memcpy(bptr, "&gt;", 4);
1701                         bptr += 4;
1702                         Target->BufUsed += 4;
1703                 }
1704                 else if (*aptr == '&') {
1705                         memcpy(bptr, "&amp;", 5);
1706                         bptr += 5;
1707                         Target->BufUsed += 5;
1708                 }
1709                 else if (*aptr == '"') {
1710                         memcpy(bptr, "&quot;", 6);
1711                         bptr += 6;
1712                         Target->BufUsed += 6;
1713                 }
1714                 else if (*aptr == '\'') {
1715                         memcpy(bptr, "&#39;", 5);
1716                         bptr += 5;
1717                         Target->BufUsed += 5;
1718                 }
1719                 else if (*aptr == LB) {
1720                         *bptr = '<';
1721                         bptr ++;
1722                         Target->BufUsed ++;
1723                 }
1724                 else if (*aptr == RB) {
1725                         *bptr = '>';
1726                         bptr ++;
1727                         Target->BufUsed ++;
1728                 }
1729                 else if (*aptr == QU) {
1730                         *bptr ='"';
1731                         bptr ++;
1732                         Target->BufUsed ++;
1733                 }
1734                 else if ((*aptr == 32) && (nbsp == 1)) {
1735                         memcpy(bptr, "&nbsp;", 6);
1736                         bptr += 6;
1737                         Target->BufUsed += 6;
1738                 }
1739                 else if ((*aptr == '\n') && (nolinebreaks == 1)) {
1740                         *bptr='\0';     /* nothing */
1741                 }
1742                 else if ((*aptr == '\n') && (nolinebreaks == 2)) {
1743                         memcpy(bptr, "&lt;br/&gt;", 11);
1744                         bptr += 11;
1745                         Target->BufUsed += 11;
1746                 }
1747
1748
1749                 else if ((*aptr == '\r') && (nolinebreaks != 0)) {
1750                         *bptr='\0';     /* nothing */
1751                 }
1752                 else{
1753                         *bptr = *aptr;
1754                         bptr++;
1755                         Target->BufUsed ++;
1756                 }
1757                 aptr ++;
1758         }
1759         *bptr = '\0';
1760         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
1761                 return -1;
1762         return Target->BufUsed;
1763 }
1764
1765 /**
1766  * @ingroup StrBuf_DeEnCoder
1767  * @brief Append a string, escaping characters which have meaning in HTML.
1768  * Converts linebreaks into blanks; escapes single quotes
1769  * @param Target        target buffer
1770  * @param Source        source buffer; set to NULL if you just have a C-String
1771  * @param PlainIn       Plain-C string to append; set to NULL if unused
1772  */
1773 void StrMsgEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
1774 {
1775         const char *aptr, *eiptr;
1776         char *tptr, *eptr;
1777         long len;
1778
1779         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
1780                 return ;
1781
1782         if (PlainIn != NULL) {
1783                 aptr = PlainIn;
1784                 len = strlen(PlainIn);
1785                 eiptr = aptr + len;
1786         }
1787         else {
1788                 aptr = Source->buf;
1789                 eiptr = aptr + Source->BufUsed;
1790                 len = Source->BufUsed;
1791         }
1792
1793         if (len == 0)
1794                 return;
1795
1796         eptr = Target->buf + Target->BufSize - 8;
1797         tptr = Target->buf + Target->BufUsed;
1798
1799         while (aptr < eiptr){
1800                 if(tptr >= eptr) {
1801                         IncreaseBuf(Target, 1, -1);
1802                         eptr = Target->buf + Target->BufSize - 8;
1803                         tptr = Target->buf + Target->BufUsed;
1804                 }
1805
1806                 if (*aptr == '\n') {
1807                         *tptr = ' ';
1808                         Target->BufUsed++;
1809                 }
1810                 else if (*aptr == '\r') {
1811                         *tptr = ' ';
1812                         Target->BufUsed++;
1813                 }
1814                 else if (*aptr == '\'') {
1815                         *(tptr++) = '&';
1816                         *(tptr++) = '#';
1817                         *(tptr++) = '3';
1818                         *(tptr++) = '9';
1819                         *tptr = ';';
1820                         Target->BufUsed += 5;
1821                 } else {
1822                         *tptr = *aptr;
1823                         Target->BufUsed++;
1824                 }
1825                 tptr++; aptr++;
1826         }
1827         *tptr = '\0';
1828 }
1829
1830
1831
1832 /**
1833  * @ingroup StrBuf_DeEnCoder
1834  * @brief Append a string, escaping characters which have meaning in ICAL.
1835  * [\n,]
1836  * @param Target        target buffer
1837  * @param Source        source buffer; set to NULL if you just have a C-String
1838  * @param PlainIn       Plain-C string to append; set to NULL if unused
1839  */
1840 void StrIcalEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
1841 {
1842         const char *aptr, *eiptr;
1843         char *tptr, *eptr;
1844         long len;
1845
1846         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
1847                 return ;
1848
1849         if (PlainIn != NULL) {
1850                 aptr = PlainIn;
1851                 len = strlen(PlainIn);
1852                 eiptr = aptr + len;
1853         }
1854         else {
1855                 aptr = Source->buf;
1856                 eiptr = aptr + Source->BufUsed;
1857                 len = Source->BufUsed;
1858         }
1859
1860         if (len == 0)
1861                 return;
1862
1863         eptr = Target->buf + Target->BufSize - 8;
1864         tptr = Target->buf + Target->BufUsed;
1865
1866         while (aptr < eiptr){
1867                 if(tptr + 3 >= eptr) {
1868                         IncreaseBuf(Target, 1, -1);
1869                         eptr = Target->buf + Target->BufSize - 8;
1870                         tptr = Target->buf + Target->BufUsed;
1871                 }
1872
1873                 if (*aptr == '\n') {
1874                         *tptr = '\\';
1875                         Target->BufUsed++;
1876                         tptr++;
1877                         *tptr = 'n';
1878                         Target->BufUsed++;
1879                 }
1880                 else if (*aptr == '\r') {
1881                         *tptr = '\\';
1882                         Target->BufUsed++;
1883                         tptr++;
1884                         *tptr = 'r';
1885                         Target->BufUsed++;
1886                 }
1887                 else if (*aptr == ',') {
1888                         *tptr = '\\';
1889                         Target->BufUsed++;
1890                         tptr++;
1891                         *tptr = ',';
1892                         Target->BufUsed++;
1893                 } else {
1894                         *tptr = *aptr;
1895                         Target->BufUsed++;
1896                 }
1897                 tptr++; aptr++;
1898         }
1899         *tptr = '\0';
1900 }
1901
1902 /**
1903  * @ingroup StrBuf_DeEnCoder
1904  * @brief Append a string, escaping characters which have meaning in JavaScript strings .
1905  *
1906  * @param Target        target buffer
1907  * @param Source        source buffer; set to NULL if you just have a C-String
1908  * @param PlainIn       Plain-C string to append; set to NULL if unused
1909  * @returns size of result or -1
1910  */
1911 long StrECMAEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
1912 {
1913         const char *aptr, *eiptr;
1914         char *bptr, *eptr;
1915         long len;
1916
1917         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
1918                 return -1;
1919
1920         if (PlainIn != NULL) {
1921                 aptr = PlainIn;
1922                 len = strlen(PlainIn);
1923                 eiptr = aptr + len;
1924         }
1925         else {
1926                 aptr = Source->buf;
1927                 eiptr = aptr + Source->BufUsed;
1928                 len = Source->BufUsed;
1929         }
1930
1931         if (len == 0)
1932                 return -1;
1933
1934         bptr = Target->buf + Target->BufUsed;
1935         eptr = Target->buf + Target->BufSize - 3; /* our biggest unit to put in...  */
1936
1937         while (aptr < eiptr){
1938                 if(bptr >= eptr) {
1939                         IncreaseBuf(Target, 1, -1);
1940                         eptr = Target->buf + Target->BufSize - 3;
1941                         bptr = Target->buf + Target->BufUsed;
1942                 }
1943                 if (*aptr == '"') {
1944                         *bptr = '\\';
1945                         bptr ++;
1946                         *bptr = '"';
1947                         bptr ++;
1948                         Target->BufUsed += 2;
1949                 } else if (*aptr == '\\') {
1950                         *bptr = '\\';
1951                         bptr ++;
1952                         *bptr = '\\';
1953                         bptr ++;
1954                         Target->BufUsed += 2;
1955                 }
1956                 else{
1957                         *bptr = *aptr;
1958                         bptr++;
1959                         Target->BufUsed ++;
1960                 }
1961                 aptr ++;
1962         }
1963         *bptr = '\0';
1964         if ((bptr == eptr - 1 ) && !IsEmptyStr(aptr) )
1965                 return -1;
1966         return Target->BufUsed;
1967 }
1968
1969 /**
1970  * @ingroup StrBuf_DeEnCoder
1971  * @brief Append a string, escaping characters which have meaning in HTML + json.
1972  *
1973  * @param Target        target buffer
1974  * @param Source        source buffer; set to NULL if you just have a C-String
1975  * @param PlainIn       Plain-C string to append; set to NULL if unused
1976  * @param nbsp          If nonzero, spaces are converted to non-breaking spaces.
1977  * @param nolinebreaks  if set to 1, linebreaks are removed from the string.
1978  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
1979  */
1980 long StrHtmlEcmaEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
1981 {
1982         const char *aptr, *eiptr;
1983         char *bptr, *eptr;
1984         long len;
1985         int IsUtf8Sequence = 0;
1986
1987         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
1988                 return -1;
1989
1990         if (PlainIn != NULL) {
1991                 aptr = PlainIn;
1992                 len = strlen(PlainIn);
1993                 eiptr = aptr + len;
1994         }
1995         else {
1996                 aptr = Source->buf;
1997                 eiptr = aptr + Source->BufUsed;
1998                 len = Source->BufUsed;
1999         }
2000
2001         if (len == 0)
2002                 return -1;
2003
2004         bptr = Target->buf + Target->BufUsed;
2005         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2006
2007         while (aptr < eiptr){
2008                 if(bptr >= eptr) {
2009                         IncreaseBuf(Target, 1, -1);
2010                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2011                         bptr = Target->buf + Target->BufUsed;
2012                 }
2013                 if (*aptr == '<') {
2014                         memcpy(bptr, "&lt;", 4);
2015                         bptr += 4;
2016                         Target->BufUsed += 4;
2017                 }
2018                 else if (*aptr == '>') {
2019                         memcpy(bptr, "&gt;", 4);
2020                         bptr += 4;
2021                         Target->BufUsed += 4;
2022                 }
2023                 else if (*aptr == '&') {
2024                         memcpy(bptr, "&amp;", 5);
2025                         bptr += 5;
2026                         Target->BufUsed += 5;
2027                 }
2028                 else if (*aptr == LB) {
2029                         *bptr = '<';
2030                         bptr ++;
2031                         Target->BufUsed ++;
2032                 }
2033                 else if (*aptr == RB) {
2034                         *bptr = '>';
2035                         bptr ++;
2036                         Target->BufUsed ++;
2037                 }
2038                 else if ((*aptr == 32) && (nbsp == 1)) {
2039                         memcpy(bptr, "&nbsp;", 6);
2040                         bptr += 6;
2041                         Target->BufUsed += 6;
2042                 }
2043                 else if ((*aptr == '\n') && (nolinebreaks == 1)) {
2044                         *bptr='\0';     /* nothing */
2045                 }
2046                 else if ((*aptr == '\n') && (nolinebreaks == 2)) {
2047                         memcpy(bptr, "&lt;br/&gt;", 11);
2048                         bptr += 11;
2049                         Target->BufUsed += 11;
2050                 }
2051
2052                 else if ((*aptr == '\r') && (nolinebreaks != 0)) {
2053                         *bptr='\0';     /* nothing */
2054                 }
2055
2056                 else if ((*aptr == '"') || (*aptr == QU)) {
2057                         *bptr = '\\';
2058                         bptr ++;
2059                         *bptr = '"';
2060                         bptr ++;
2061                         Target->BufUsed += 2;
2062                 } else if (*aptr == '\\') {
2063                         *bptr = '\\';
2064                         bptr ++;
2065                         *bptr = '\\';
2066                         bptr ++;
2067                         Target->BufUsed += 2;
2068                 }
2069                 else {
2070                         if (((unsigned char)*aptr) >= 0x20)
2071                         {
2072                                 IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(aptr, eiptr);
2073
2074                                 *bptr = *aptr;
2075                                 Target->BufUsed ++;
2076                                 while (IsUtf8Sequence > 1){
2077                                         if(bptr + IsUtf8Sequence >= eptr) {
2078                                                 IncreaseBuf(Target, 1, -1);
2079                                                 eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2080                                                 bptr = Target->buf + Target->BufUsed - 1;
2081                                         }
2082                                         bptr++; aptr++;
2083                                         IsUtf8Sequence --;
2084                                         *bptr = *aptr;
2085                                         Target->BufUsed ++;
2086                                 }
2087                                 bptr++;
2088                         }
2089
2090                 }
2091                 aptr ++;
2092         }
2093         *bptr = '\0';
2094         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
2095                 return -1;
2096         return Target->BufUsed;
2097 }
2098
2099 /**
2100  * @ingroup StrBuf_DeEnCoder
2101  * @brief unhide special chars hidden to the HTML escaper
2102  * @param target buffer to put the unescaped string in
2103  * @param source buffer to unescape
2104  */
2105 void StrBufEUid_unescapize(StrBuf *target, const StrBuf *source)
2106 {
2107         int a, b, len;
2108         char hex[3];
2109
2110         if (target != NULL)
2111                 FlushStrBuf(target);
2112
2113         if (source == NULL ||target == NULL)
2114         {
2115                 return;
2116         }
2117
2118         len = source->BufUsed;
2119         for (a = 0; a < len; ++a) {
2120                 if (target->BufUsed >= target->BufSize)
2121                         IncreaseBuf(target, 1, -1);
2122
2123                 if (source->buf[a] == '=') {
2124                         hex[0] = source->buf[a + 1];
2125                         hex[1] = source->buf[a + 2];
2126                         hex[2] = 0;
2127                         b = 0;
2128                         sscanf(hex, "%02x", &b);
2129                         target->buf[target->BufUsed] = b;
2130                         target->buf[++target->BufUsed] = 0;
2131                         a += 2;
2132                 }
2133                 else {
2134                         target->buf[target->BufUsed] = source->buf[a];
2135                         target->buf[++target->BufUsed] = 0;
2136                 }
2137         }
2138 }
2139
2140
2141 /**
2142  * @ingroup StrBuf_DeEnCoder
2143  * @brief hide special chars from the HTML escapers and friends
2144  * @param target buffer to put the escaped string in
2145  * @param source buffer to escape
2146  */
2147 void StrBufEUid_escapize(StrBuf *target, const StrBuf *source)
2148 {
2149         int i, len;
2150
2151         if (target != NULL)
2152                 FlushStrBuf(target);
2153
2154         if (source == NULL ||target == NULL)
2155         {
2156                 return;
2157         }
2158
2159         len = source->BufUsed;
2160         for (i=0; i<len; ++i) {
2161                 if (target->BufUsed + 4 >= target->BufSize)
2162                         IncreaseBuf(target, 1, -1);
2163                 if ( (isalnum(source->buf[i])) ||
2164                      (source->buf[i]=='-') ||
2165                      (source->buf[i]=='_') ) {
2166                         target->buf[target->BufUsed++] = source->buf[i];
2167                 }
2168                 else {
2169                         sprintf(&target->buf[target->BufUsed],
2170                                 "=%02X",
2171                                 (0xFF &source->buf[i]));
2172                         target->BufUsed += 3;
2173                 }
2174         }
2175         target->buf[target->BufUsed + 1] = '\0';
2176 }
2177
2178
2179 /*******************************************************************************
2180  *                      Quoted Printable de/encoding                           *
2181  *******************************************************************************/
2182
2183 /**
2184  * @ingroup StrBuf_DeEnCoder
2185  * @brief decode a buffer from base 64 encoding; destroys original
2186  * @param Buf Buffor to transform
2187  */
2188 int StrBufDecodeBase64(StrBuf *Buf)
2189 {
2190         char *xferbuf;
2191         size_t siz;
2192         if (Buf == NULL) return -1;
2193
2194         xferbuf = (char*) malloc(Buf->BufSize);
2195         siz = CtdlDecodeBase64(xferbuf,
2196                                Buf->buf,
2197                                Buf->BufUsed);
2198         free(Buf->buf);
2199         Buf->buf = xferbuf;
2200         Buf->BufUsed = siz;
2201         return siz;
2202 }
2203
2204 /**
2205  * @ingroup StrBuf_DeEnCoder
2206  * @brief decode a buffer from base 64 encoding; destroys original
2207  * @param Buf Buffor to transform
2208  */
2209 int StrBufDecodeHex(StrBuf *Buf)
2210 {
2211         unsigned int ch;
2212         char *pch, *pche, *pchi;
2213
2214         if (Buf == NULL) return -1;
2215
2216         pch = pchi = Buf->buf;
2217         pche = pch + Buf->BufUsed;
2218
2219         while (pchi < pche){
2220                 ch = decode_hex(pchi);
2221                 *pch = ch;
2222                 pch ++;
2223                 pchi += 2;
2224         }
2225
2226         *pch = '\0';
2227         Buf->BufUsed = pch - Buf->buf;
2228         return Buf->BufUsed;
2229 }
2230
2231 /**
2232  * @ingroup StrBuf_DeEnCoder
2233  * @brief replace all chars >0x20 && < 0x7F with Mute
2234  * @param Mute char to put over invalid chars
2235  * @param Buf Buffor to transform
2236  */
2237 int StrBufSanitizeAscii(StrBuf *Buf, const char Mute)
2238 {
2239         unsigned char *pch;
2240
2241         if (Buf == NULL) return -1;
2242         pch = (unsigned char *)Buf->buf;
2243         while (pch < (unsigned char *)Buf->buf + Buf->BufUsed) {
2244                 if ((*pch < 0x20) || (*pch > 0x7F))
2245                         *pch = Mute;
2246                 pch ++;
2247         }
2248         return Buf->BufUsed;
2249 }
2250
2251
2252 /**
2253  * @ingroup StrBuf_DeEnCoder
2254  * @brief remove escaped strings from i.e. the url string (like %20 for blanks)
2255  * @param Buf Buffer to translate
2256  * @param StripBlanks Reduce several blanks to one?
2257  */
2258 long StrBufUnescape(StrBuf *Buf, int StripBlanks)
2259 {
2260         int a, b;
2261         char hex[3];
2262         long len;
2263
2264         if (Buf == NULL)
2265                 return -1;
2266
2267         while ((Buf->BufUsed > 0) && (isspace(Buf->buf[Buf->BufUsed - 1]))){
2268                 Buf->buf[Buf->BufUsed - 1] = '\0';
2269                 Buf->BufUsed --;
2270         }
2271
2272         a = 0;
2273         while (a < Buf->BufUsed) {
2274                 if (Buf->buf[a] == '+')
2275                         Buf->buf[a] = ' ';
2276                 else if (Buf->buf[a] == '%') {
2277                         /* don't let % chars through, rather truncate the input. */
2278                         if (a + 2 > Buf->BufUsed) {
2279                                 Buf->buf[a] = '\0';
2280                                 Buf->BufUsed = a;
2281                         }
2282                         else {
2283                                 hex[0] = Buf->buf[a + 1];
2284                                 hex[1] = Buf->buf[a + 2];
2285                                 hex[2] = 0;
2286                                 b = 0;
2287                                 sscanf(hex, "%02x", &b);
2288                                 Buf->buf[a] = (char) b;
2289                                 len = Buf->BufUsed - a - 2;
2290                                 if (len > 0)
2291                                         memmove(&Buf->buf[a + 1], &Buf->buf[a + 3], len);
2292
2293                                 Buf->BufUsed -=2;
2294                         }
2295                 }
2296                 a++;
2297         }
2298         return a;
2299 }
2300
2301
2302 /**
2303  * @ingroup StrBuf_DeEnCoder
2304  * @brief       RFC2047-encode a header field if necessary.
2305  *              If no non-ASCII characters are found, the string
2306  *              will be copied verbatim without encoding.
2307  *
2308  * @param       target          Target buffer.
2309  * @param       source          Source string to be encoded.
2310  * @returns     encoded length; -1 if non success.
2311  */
2312 int StrBufRFC2047encode(StrBuf **target, const StrBuf *source)
2313 {
2314         const char headerStr[] = "=?UTF-8?Q?";
2315         int need_to_encode = 0;
2316         int i = 0;
2317         unsigned char ch;
2318
2319         if ((source == NULL) ||
2320             (target == NULL))
2321             return -1;
2322
2323         while ((i < source->BufUsed) &&
2324                (!IsEmptyStr (&source->buf[i])) &&
2325                (need_to_encode == 0)) {
2326                 if (((unsigned char) source->buf[i] < 32) ||
2327                     ((unsigned char) source->buf[i] > 126)) {
2328                         need_to_encode = 1;
2329                 }
2330                 i++;
2331         }
2332
2333         if (!need_to_encode) {
2334                 if (*target == NULL) {
2335                         *target = NewStrBufPlain(source->buf, source->BufUsed);
2336                 }
2337                 else {
2338                         FlushStrBuf(*target);
2339                         StrBufAppendBuf(*target, source, 0);
2340                 }
2341                 return (*target)->BufUsed;
2342         }
2343         if (*target == NULL)
2344                 *target = NewStrBufPlain(NULL, sizeof(headerStr) + source->BufUsed * 2);
2345         else if (sizeof(headerStr) + source->BufUsed >= (*target)->BufSize)
2346                 IncreaseBuf(*target, sizeof(headerStr) + source->BufUsed, 0);
2347         memcpy ((*target)->buf, headerStr, sizeof(headerStr) - 1);
2348         (*target)->BufUsed = sizeof(headerStr) - 1;
2349         for (i=0; (i < source->BufUsed); ++i) {
2350                 if ((*target)->BufUsed + 4 >= (*target)->BufSize)
2351                         IncreaseBuf(*target, 1, 0);
2352                 ch = (unsigned char) source->buf[i];
2353                 if ((ch < 32) || (ch > 126) || (ch == 61)) {
2354                         sprintf(&(*target)->buf[(*target)->BufUsed], "=%02X", ch);
2355                         (*target)->BufUsed += 3;
2356                 }
2357                 else {
2358                         (*target)->buf[(*target)->BufUsed] = ch;
2359                         (*target)->BufUsed++;
2360                 }
2361         }
2362
2363         if ((*target)->BufUsed + 4 >= (*target)->BufSize)
2364                 IncreaseBuf(*target, 1, 0);
2365
2366         (*target)->buf[(*target)->BufUsed++] = '?';
2367         (*target)->buf[(*target)->BufUsed++] = '=';
2368         (*target)->buf[(*target)->BufUsed] = '\0';
2369         return (*target)->BufUsed;;
2370 }
2371
2372 /**
2373  * @ingroup StrBuf
2374  * @brief replaces all occurances of 'search' by 'replace'
2375  * @param buf Buffer to modify
2376  * @param search character to search
2377  * @param replace character to replace search by
2378  */
2379 void StrBufReplaceChars(StrBuf *buf, char search, char replace)
2380 {
2381         long i;
2382         if (buf == NULL)
2383                 return;
2384         for (i=0; i<buf->BufUsed; i++)
2385                 if (buf->buf[i] == search)
2386                         buf->buf[i] = replace;
2387
2388 }
2389
2390 /**
2391  * @ingroup StrBuf
2392  * @brief removes all \r s from the string, or replaces them with \n if its not a combination of both.
2393  * @param buf Buffer to modify
2394  */
2395 void StrBufToUnixLF(StrBuf *buf)
2396 {
2397         char *pche, *pchS, *pchT;
2398         if (buf == NULL)
2399                 return;
2400
2401         pche = buf->buf + buf->BufUsed;
2402         pchS = pchT = buf->buf;
2403         while (pchS < pche)
2404         {
2405                 if (*pchS == '\r')
2406                 {
2407                         pchS ++;
2408                         if (*pchS != '\n') {
2409                                 *pchT = '\n';
2410                                 pchT++;
2411                         }
2412                 }
2413                 *pchT = *pchS;
2414                 pchT++; pchS++;
2415         }
2416         *pchT = '\0';
2417         buf->BufUsed = pchT - buf->buf;
2418 }
2419
2420
2421 /*******************************************************************************
2422  *                 Iconv Wrapper; RFC822 de/encoding                           *
2423  *******************************************************************************/
2424
2425 /**
2426  * @ingroup StrBuf_DeEnCoder
2427  * @brief Wrapper around iconv_open()
2428  * Our version adds aliases for non-standard Microsoft charsets
2429  * such as 'MS950', aliasing them to names like 'CP950'
2430  *
2431  * @param tocode        Target encoding
2432  * @param fromcode      Source encoding
2433  * @param pic           anonimized pointer to iconv struct
2434  */
2435 void  ctdl_iconv_open(const char *tocode, const char *fromcode, void *pic)
2436 {
2437 #ifdef HAVE_ICONV
2438         iconv_t ic = (iconv_t)(-1) ;
2439         ic = iconv_open(tocode, fromcode);
2440         if (ic == (iconv_t)(-1) ) {
2441                 char alias_fromcode[64];
2442                 if ( (strlen(fromcode) == 5) && (!strncasecmp(fromcode, "MS", 2)) ) {
2443                         safestrncpy(alias_fromcode, fromcode, sizeof alias_fromcode);
2444                         alias_fromcode[0] = 'C';
2445                         alias_fromcode[1] = 'P';
2446                         ic = iconv_open(tocode, alias_fromcode);
2447                 }
2448         }
2449         *(iconv_t *)pic = ic;
2450 #endif
2451 }
2452
2453
2454 /**
2455  * @ingroup StrBuf_DeEnCoder
2456  * @brief find one chunk of a RFC822 encoded string
2457  * @param Buffer where to search
2458  * @param bptr where to start searching
2459  * @returns found position, NULL if none.
2460  */
2461 static inline const char *FindNextEnd (const StrBuf *Buf, const char *bptr)
2462 {
2463         const char * end;
2464         /* Find the next ?Q? */
2465         if (Buf->BufUsed - (bptr - Buf->buf)  < 6)
2466                 return NULL;
2467
2468         end = strchr(bptr + 2, '?');
2469
2470         if (end == NULL)
2471                 return NULL;
2472
2473         if ((Buf->BufUsed - (end - Buf->buf) > 3) &&
2474             ((*(end + 1) == 'B') || (*(end + 1) == 'Q')) &&
2475             (*(end + 2) == '?')) {
2476                 /* skip on to the end of the cluster, the next ?= */
2477                 end = strstr(end + 3, "?=");
2478         }
2479         else
2480                 /* sort of half valid encoding, try to find an end. */
2481                 end = strstr(bptr, "?=");
2482         return end;
2483 }
2484
2485 /**
2486  * @ingroup StrBuf
2487  * @brief swaps the contents of two StrBufs
2488  * this is to be used to have cheap switched between a work-buffer and a target buffer
2489  * @param A First one
2490  * @param B second one
2491  */
2492 static inline void SwapBuffers(StrBuf *A, StrBuf *B)
2493 {
2494         StrBuf C;
2495
2496         memcpy(&C, A, sizeof(*A));
2497         memcpy(A, B, sizeof(*B));
2498         memcpy(B, &C, sizeof(C));
2499
2500 }
2501
2502
2503 /**
2504  * @ingroup StrBuf_DeEnCoder
2505  * @brief convert one buffer according to the preselected iconv pointer PIC
2506  * @param ConvertBuf buffer we need to translate
2507  * @param TmpBuf To share a workbuffer over several iterations. prepare to have it filled with useless stuff afterwards.
2508  * @param pic Pointer to the iconv-session Object
2509  */
2510 void StrBufConvert(StrBuf *ConvertBuf, StrBuf *TmpBuf, void *pic)
2511 {
2512 #ifdef HAVE_ICONV
2513         long trycount = 0;
2514         size_t siz;
2515         iconv_t ic;
2516         char *ibuf;                     /**< Buffer of characters to be converted */
2517         char *obuf;                     /**< Buffer for converted characters */
2518         size_t ibuflen;                 /**< Length of input buffer */
2519         size_t obuflen;                 /**< Length of output buffer */
2520
2521
2522         /* since we're converting to utf-8, one glyph may take up to 6 bytes */
2523         if (ConvertBuf->BufUsed * 6 >= TmpBuf->BufSize)
2524                 IncreaseBuf(TmpBuf, 0, ConvertBuf->BufUsed * 6);
2525 TRYAGAIN:
2526         ic = *(iconv_t*)pic;
2527         ibuf = ConvertBuf->buf;
2528         ibuflen = ConvertBuf->BufUsed;
2529         obuf = TmpBuf->buf;
2530         obuflen = TmpBuf->BufSize;
2531
2532         siz = iconv(ic, &ibuf, &ibuflen, &obuf, &obuflen);
2533
2534         if (siz < 0) {
2535                 if (errno == E2BIG) {
2536                         trycount ++;
2537                         IncreaseBuf(TmpBuf, 0, 0);
2538                         if (trycount < 5)
2539                                 goto TRYAGAIN;
2540
2541                 }
2542                 else if (errno == EILSEQ){
2543                         /* hm, invalid utf8 sequence... what to do now? */
2544                         /* An invalid multibyte sequence has been encountered in the input */
2545                 }
2546                 else if (errno == EINVAL) {
2547                         /* An incomplete multibyte sequence has been encountered in the input. */
2548                 }
2549
2550                 FlushStrBuf(TmpBuf);
2551         }
2552         else {
2553                 TmpBuf->BufUsed = TmpBuf->BufSize - obuflen;
2554                 TmpBuf->buf[TmpBuf->BufUsed] = '\0';
2555
2556                 /* little card game: wheres the red lady? */
2557                 SwapBuffers(ConvertBuf, TmpBuf);
2558                 FlushStrBuf(TmpBuf);
2559         }
2560 #endif
2561 }
2562
2563
2564 /**
2565  * @ingroup StrBuf_DeEnCoder
2566  * @brief catches one RFC822 encoded segment, and decodes it.
2567  * @param Target buffer to fill with result
2568  * @param DecodeMe buffer with stuff to process
2569  * @param SegmentStart points to our current segment in DecodeMe
2570  * @param SegmentEnd Points to the end of our current segment in DecodeMe
2571  * @param ConvertBuf Workbuffer shared between several iterations. Random content; needs to be valid
2572  * @param ConvertBuf2 Workbuffer shared between several iterations. Random content; needs to be valid
2573  * @param FoundCharset Characterset to default decoding to; if we find another we will overwrite it.
2574  */
2575 inline static void DecodeSegment(StrBuf *Target,
2576                                  const StrBuf *DecodeMe,
2577                                  const char *SegmentStart,
2578                                  const char *SegmentEnd,
2579                                  StrBuf *ConvertBuf,
2580                                  StrBuf *ConvertBuf2,
2581                                  StrBuf *FoundCharset)
2582 {
2583         StrBuf StaticBuf;
2584         char charset[128];
2585         char encoding[16];
2586 #ifdef HAVE_ICONV
2587         iconv_t ic = (iconv_t)(-1);
2588 #else
2589         void *ic = NULL;
2590 #endif
2591         /* Now we handle foreign character sets properly encoded
2592          * in RFC2047 format.
2593          */
2594         StaticBuf.buf = (char*) SegmentStart; /*< it will just be read there... */
2595         StaticBuf.BufUsed = SegmentEnd - SegmentStart;
2596         StaticBuf.BufSize = DecodeMe->BufSize - (SegmentStart - DecodeMe->buf);
2597         extract_token(charset, SegmentStart, 1, '?', sizeof charset);
2598         if (FoundCharset != NULL) {
2599                 FlushStrBuf(FoundCharset);
2600                 StrBufAppendBufPlain(FoundCharset, charset, -1, 0);
2601         }
2602         extract_token(encoding, SegmentStart, 2, '?', sizeof encoding);
2603         StrBufExtract_token(ConvertBuf, &StaticBuf, 3, '?');
2604
2605         *encoding = toupper(*encoding);
2606         if (*encoding == 'B') { /**< base64 */
2607                 ConvertBuf2->BufUsed = CtdlDecodeBase64(ConvertBuf2->buf,
2608                                                         ConvertBuf->buf,
2609                                                         ConvertBuf->BufUsed);
2610         }
2611         else if (*encoding == 'Q') {    /**< quoted-printable */
2612                 long pos;
2613
2614                 pos = 0;
2615                 while (pos < ConvertBuf->BufUsed)
2616                 {
2617                         if (ConvertBuf->buf[pos] == '_')
2618                                 ConvertBuf->buf[pos] = ' ';
2619                         pos++;
2620                 }
2621
2622                 ConvertBuf2->BufUsed = CtdlDecodeQuotedPrintable(
2623                         ConvertBuf2->buf,
2624                         ConvertBuf->buf,
2625                         ConvertBuf->BufUsed);
2626         }
2627         else {
2628                 StrBufAppendBuf(ConvertBuf2, ConvertBuf, 0);
2629         }
2630 #ifdef HAVE_ICONV
2631         ctdl_iconv_open("UTF-8", charset, &ic);
2632         if (ic != (iconv_t)(-1) ) {
2633 #endif
2634                 StrBufConvert(ConvertBuf2, ConvertBuf, &ic);
2635                 StrBufAppendBuf(Target, ConvertBuf2, 0);
2636 #ifdef HAVE_ICONV
2637                 iconv_close(ic);
2638         }
2639         else {
2640                 StrBufAppendBufPlain(Target, HKEY("(unreadable)"), 0);
2641         }
2642 #endif
2643 }
2644
2645 /**
2646  * @ingroup StrBuf_DeEnCoder
2647  * @brief Handle subjects with RFC2047 encoding such as:
2648  * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
2649  * @param Target where to put the decoded string to
2650  * @param DecodeMe buffer with encoded string
2651  * @param DefaultCharset if we don't find one, which should we use?
2652  * @param FoundCharset overrides DefaultCharset if non-empty; If we find a charset inside of the string,
2653  *        put it here for later use where no string might be known.
2654  */
2655 void StrBuf_RFC822_to_Utf8(StrBuf *Target, const StrBuf *DecodeMe, const StrBuf* DefaultCharset, StrBuf *FoundCharset)
2656 {
2657         StrBuf *DecodedInvalidBuf = NULL;
2658         StrBuf *ConvertBuf, *ConvertBuf2;
2659         const StrBuf *DecodeMee = DecodeMe;
2660         const char *start, *end, *next, *nextend, *ptr = NULL;
2661 #ifdef HAVE_ICONV
2662         iconv_t ic = (iconv_t)(-1) ;
2663 #endif
2664         const char *eptr;
2665         int passes = 0;
2666         int i, len;
2667         int illegal_non_rfc2047_encoding = 0;
2668
2669         /* Sometimes, badly formed messages contain strings which were simply
2670          *  written out directly in some foreign character set instead of
2671          *  using RFC2047 encoding.  This is illegal but we will attempt to
2672          *  handle it anyway by converting from a user-specified default
2673          *  charset to UTF-8 if we see any nonprintable characters.
2674          */
2675
2676         len = StrLength(DecodeMe);
2677         for (i=0; i<DecodeMe->BufUsed; ++i) {
2678                 if ((DecodeMe->buf[i] < 32) || (DecodeMe->buf[i] > 126)) {
2679                         illegal_non_rfc2047_encoding = 1;
2680                         break;
2681                 }
2682         }
2683
2684         ConvertBuf = NewStrBufPlain(NULL, StrLength(DecodeMe));
2685         if ((illegal_non_rfc2047_encoding) &&
2686             (strcasecmp(ChrPtr(DefaultCharset), "UTF-8")) &&
2687             (strcasecmp(ChrPtr(DefaultCharset), "us-ascii")) )
2688         {
2689 #ifdef HAVE_ICONV
2690                 ctdl_iconv_open("UTF-8", ChrPtr(DefaultCharset), &ic);
2691                 if (ic != (iconv_t)(-1) ) {
2692                         DecodedInvalidBuf = NewStrBufDup(DecodeMe);
2693                         StrBufConvert(DecodedInvalidBuf, ConvertBuf, &ic);///TODO: don't void const?
2694                         DecodeMee = DecodedInvalidBuf;
2695                         iconv_close(ic);
2696                 }
2697 #endif
2698         }
2699
2700         /* pre evaluate the first pair */
2701         nextend = end = NULL;
2702         len = StrLength(DecodeMee);
2703         start = strstr(DecodeMee->buf, "=?");
2704         eptr = DecodeMee->buf + DecodeMee->BufUsed;
2705         if (start != NULL)
2706                 end = FindNextEnd (DecodeMee, start);
2707         else {
2708                 StrBufAppendBuf(Target, DecodeMee, 0);
2709                 FreeStrBuf(&ConvertBuf);
2710                 FreeStrBuf(&DecodedInvalidBuf);
2711                 return;
2712         }
2713
2714         ConvertBuf2 = NewStrBufPlain(NULL, StrLength(DecodeMee));
2715
2716         if (start != DecodeMee->buf) {
2717                 long nFront;
2718
2719                 nFront = start - DecodeMee->buf;
2720                 StrBufAppendBufPlain(Target, DecodeMee->buf, nFront, 0);
2721                 len -= nFront;
2722         }
2723         /*
2724          * Since spammers will go to all sorts of absurd lengths to get their
2725          * messages through, there are LOTS of corrupt headers out there.
2726          * So, prevent a really badly formed RFC2047 header from throwing
2727          * this function into an infinite loop.
2728          */
2729         while ((start != NULL) &&
2730                (end != NULL) &&
2731                (start < eptr) &&
2732                (end < eptr) &&
2733                (passes < 20))
2734         {
2735                 passes++;
2736                 DecodeSegment(Target,
2737                               DecodeMee,
2738                               start,
2739                               end,
2740                               ConvertBuf,
2741                               ConvertBuf2,
2742                               FoundCharset);
2743
2744                 next = strstr(end, "=?");
2745                 nextend = NULL;
2746                 if ((next != NULL) &&
2747                     (next < eptr))
2748                         nextend = FindNextEnd(DecodeMee, next);
2749                 if (nextend == NULL)
2750                         next = NULL;
2751
2752                 /* did we find two partitions */
2753                 if ((next != NULL) &&
2754                     ((next - end) > 2))
2755                 {
2756                         ptr = end + 2;
2757                         while ((ptr < next) &&
2758                                (isspace(*ptr) ||
2759                                 (*ptr == '\r') ||
2760                                 (*ptr == '\n') ||
2761                                 (*ptr == '\t')))
2762                                 ptr ++;
2763                         /*
2764                          * did we find a gab just filled with blanks?
2765                          * if not, copy its stuff over.
2766                          */
2767                         if (ptr != next)
2768                         {
2769                                 StrBufAppendBufPlain(Target,
2770                                                      end + 2,
2771                                                      next - end - 2,
2772                                                      0);
2773                         }
2774                 }
2775                 /* our next-pair is our new first pair now. */
2776                 ptr = end + 2;
2777                 start = next;
2778                 end = nextend;
2779         }
2780         end = ptr;
2781         nextend = DecodeMee->buf + DecodeMee->BufUsed;
2782         if ((end != NULL) && (end < nextend)) {
2783                 ptr = end;
2784                 while ( (ptr < nextend) &&
2785                         (isspace(*ptr) ||
2786                          (*ptr == '\r') ||
2787                          (*ptr == '\n') ||
2788                          (*ptr == '\t')))
2789                         ptr ++;
2790                 if (ptr < nextend)
2791                         StrBufAppendBufPlain(Target, end, nextend - end, 0);
2792         }
2793         FreeStrBuf(&ConvertBuf);
2794         FreeStrBuf(&ConvertBuf2);
2795         FreeStrBuf(&DecodedInvalidBuf);
2796 }
2797
2798 /*******************************************************************************
2799  *                   Manipulating UTF-8 Strings                                *
2800  *******************************************************************************/
2801
2802 /**
2803  * @ingroup StrBuf
2804  * @brief evaluate the length of an utf8 special character sequence
2805  * @param Char the character to examine
2806  * @returns width of utf8 chars in bytes
2807  */
2808 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE)
2809 {
2810         int n = 1;
2811         char test = (1<<7);
2812
2813         while ((n < 8) && ((test & *CharS) != 0)) {
2814                 test = test << 1;
2815                 n ++;
2816         }
2817         if ((n > 6) || ((CharE - CharS) < n))
2818                 n = 1;
2819         return n;
2820 }
2821
2822 /**
2823  * @ingroup StrBuf
2824  * @brief detect whether this char starts an utf-8 encoded char
2825  * @param Char character to inspect
2826  * @returns yes or no
2827  */
2828 static inline int Ctdl_IsUtf8SequenceStart(const char Char)
2829 {
2830 /** 11??.???? indicates an UTF8 Sequence. */
2831         return ((Char & 0xC0) != 0);
2832 }
2833
2834 /**
2835  * @ingroup StrBuf
2836  * @brief measure the number of glyphs in an UTF8 string...
2837  * @param Buf string to measure
2838  * @returns the number of glyphs in Buf
2839  */
2840 long StrBuf_Utf8StrLen(StrBuf *Buf)
2841 {
2842         int n = 0;
2843         int m = 0;
2844         char *aptr, *eptr;
2845
2846         if ((Buf == NULL) || (Buf->BufUsed == 0))
2847                 return 0;
2848         aptr = Buf->buf;
2849         eptr = Buf->buf + Buf->BufUsed;
2850         while ((aptr < eptr) && (*aptr != '\0')) {
2851                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
2852                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
2853                         while ((aptr < eptr) && (*aptr++ != '\0')&& (m-- > 0) );
2854                         n ++;
2855                 }
2856                 else {
2857                         n++;
2858                         aptr++;
2859                 }
2860         }
2861         return n;
2862 }
2863
2864 /**
2865  * @ingroup StrBuf
2866  * @brief cuts a string after maxlen glyphs
2867  * @param Buf string to cut to maxlen glyphs
2868  * @param maxlen how long may the string become?
2869  * @returns current length of the string
2870  */
2871 long StrBuf_Utf8StrCut(StrBuf *Buf, int maxlen)
2872 {
2873         char *aptr, *eptr;
2874         int n = 0, m = 0;
2875
2876         aptr = Buf->buf;
2877         eptr = Buf->buf + Buf->BufUsed;
2878         while ((aptr < eptr) && (*aptr != '\0')) {
2879                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
2880                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
2881                         while ((*aptr++ != '\0') && (m-- > 0));
2882                         n ++;
2883                 }
2884                 else {
2885                         n++;
2886                         aptr++;
2887                 }
2888                 if (n > maxlen) {
2889                         *aptr = '\0';
2890                         Buf->BufUsed = aptr - Buf->buf;
2891                         return Buf->BufUsed;
2892                 }
2893         }
2894         return Buf->BufUsed;
2895
2896 }
2897
2898
2899
2900
2901
2902 /*******************************************************************************
2903  *                               wrapping ZLib                                 *
2904  *******************************************************************************/
2905
2906 #ifdef HAVE_ZLIB
2907 #define DEF_MEM_LEVEL 8 /*< memlevel??? */
2908 #define OS_CODE 0x03    /*< unix */
2909
2910 /**
2911  * @ingroup StrBuf_DeEnCoder
2912  * @brief uses the same calling syntax as compress2(), but it
2913  *   creates a stream compatible with HTTP "Content-encoding: gzip"
2914  * @param dest compressed buffer
2915  * @param destLen length of the compresed data
2916  * @param source source to encode
2917  * @param sourceLen length of source to encode
2918  * @param level compression level
2919  */
2920 int ZEXPORT compress_gzip(Bytef * dest,
2921                           size_t * destLen,
2922                           const Bytef * source,
2923                           uLong sourceLen,
2924                           int level)
2925 {
2926         const int gz_magic[2] = { 0x1f, 0x8b }; /* gzip magic header */
2927
2928         /* write gzip header */
2929         snprintf((char *) dest, *destLen,
2930                  "%c%c%c%c%c%c%c%c%c%c",
2931                  gz_magic[0], gz_magic[1], Z_DEFLATED,
2932                  0 /*flags */ , 0, 0, 0, 0 /*time */ , 0 /* xflags */ ,
2933                  OS_CODE);
2934
2935         /* normal deflate */
2936         z_stream stream;
2937         int err;
2938         stream.next_in = (Bytef *) source;
2939         stream.avail_in = (uInt) sourceLen;
2940         stream.next_out = dest + 10L;   // after header
2941         stream.avail_out = (uInt) * destLen;
2942         if ((uLong) stream.avail_out != *destLen)
2943                 return Z_BUF_ERROR;
2944
2945         stream.zalloc = (alloc_func) 0;
2946         stream.zfree = (free_func) 0;
2947         stream.opaque = (voidpf) 0;
2948
2949         err = deflateInit2(&stream, level, Z_DEFLATED, -MAX_WBITS,
2950                            DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
2951         if (err != Z_OK)
2952                 return err;
2953
2954         err = deflate(&stream, Z_FINISH);
2955         if (err != Z_STREAM_END) {
2956                 deflateEnd(&stream);
2957                 return err == Z_OK ? Z_BUF_ERROR : err;
2958         }
2959         *destLen = stream.total_out + 10L;
2960
2961         /* write CRC and Length */
2962         uLong crc = crc32(0L, source, sourceLen);
2963         int n;
2964         for (n = 0; n < 4; ++n, ++*destLen) {
2965                 dest[*destLen] = (int) (crc & 0xff);
2966                 crc >>= 8;
2967         }
2968         uLong len = stream.total_in;
2969         for (n = 0; n < 4; ++n, ++*destLen) {
2970                 dest[*destLen] = (int) (len & 0xff);
2971                 len >>= 8;
2972         }
2973         err = deflateEnd(&stream);
2974         return err;
2975 }
2976 #endif
2977
2978
2979 /**
2980  * @ingroup StrBuf_DeEnCoder
2981  * @brief compress the buffer with gzip
2982  * Attention! If you feed this a Const String, you must maintain the uncompressed buffer yourself!
2983  * @param Buf buffer whose content is to be gzipped
2984  */
2985 int CompressBuffer(StrBuf *Buf)
2986 {
2987 #ifdef HAVE_ZLIB
2988         char *compressed_data = NULL;
2989         size_t compressed_len, bufsize;
2990         int i = 0;
2991
2992         bufsize = compressed_len = Buf->BufUsed +  (Buf->BufUsed / 100) + 100;
2993         compressed_data = malloc(compressed_len);
2994
2995         if (compressed_data == NULL)
2996                 return -1;
2997         /* Flush some space after the used payload so valgrind shuts up... */
2998         while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
2999                 Buf->buf[Buf->BufUsed + i++] = '\0';
3000         if (compress_gzip((Bytef *) compressed_data,
3001                           &compressed_len,
3002                           (Bytef *) Buf->buf,
3003                           (uLongf) Buf->BufUsed, Z_BEST_SPEED) == Z_OK) {
3004                 if (!Buf->ConstBuf)
3005                         free(Buf->buf);
3006                 Buf->buf = compressed_data;
3007                 Buf->BufUsed = compressed_len;
3008                 Buf->BufSize = bufsize;
3009                 /* Flush some space after the used payload so valgrind shuts up... */
3010                 i = 0;
3011                 while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
3012                         Buf->buf[Buf->BufUsed + i++] = '\0';
3013                 return 1;
3014         } else {
3015                 free(compressed_data);
3016         }
3017 #endif  /* HAVE_ZLIB */
3018         return 0;
3019 }
3020
3021
3022
3023 /*******************************************************************************
3024  *           File I/O; Prefer buffered read since its faster!                  *
3025  *******************************************************************************/
3026
3027 /**
3028  * @ingroup StrBuf_IO
3029  * @brief Read a line from socket
3030  * flushes and closes the FD on error
3031  * @param buf the buffer to get the input to
3032  * @param fd pointer to the filedescriptor to read
3033  * @param append Append to an existing string or replace?
3034  * @param Error strerror() on error
3035  * @returns numbers of chars read
3036  */
3037 int StrBufTCP_read_line(StrBuf *buf, int *fd, int append, const char **Error)
3038 {
3039         int len, rlen, slen;
3040
3041         if (!append)
3042                 FlushStrBuf(buf);
3043
3044         slen = len = buf->BufUsed;
3045         while (1) {
3046                 rlen = read(*fd, &buf->buf[len], 1);
3047                 if (rlen < 1) {
3048                         *Error = strerror(errno);
3049
3050                         close(*fd);
3051                         *fd = -1;
3052
3053                         return -1;
3054                 }
3055                 if (buf->buf[len] == '\n')
3056                         break;
3057                 if (buf->buf[len] != '\r')
3058                         len ++;
3059                 if (len + 2 >= buf->BufSize) {
3060                         buf->BufUsed = len;
3061                         buf->buf[len+1] = '\0';
3062                         IncreaseBuf(buf, 1, -1);
3063                 }
3064         }
3065         buf->BufUsed = len;
3066         buf->buf[len] = '\0';
3067         return len - slen;
3068 }
3069
3070 /**
3071  * @ingroup StrBuf_BufferedIO
3072  * @brief Read a line from socket
3073  * flushes and closes the FD on error
3074  * @param Line the line to read from the fd / I/O Buffer
3075  * @param buf the buffer to get the input to
3076  * @param fd pointer to the filedescriptor to read
3077  * @param timeout number of successless selects until we bail out
3078  * @param selectresolution how long to wait on each select
3079  * @param Error strerror() on error
3080  * @returns numbers of chars read
3081  */
3082 int StrBufTCP_read_buffered_line(StrBuf *Line,
3083                                  StrBuf *buf,
3084                                  int *fd,
3085                                  int timeout,
3086                                  int selectresolution,
3087                                  const char **Error)
3088 {
3089         int len, rlen;
3090         int nSuccessLess = 0;
3091         fd_set rfds;
3092         char *pch = NULL;
3093         int fdflags;
3094         int IsNonBlock;
3095         struct timeval tv;
3096
3097         if (buf->BufUsed > 0) {
3098                 pch = strchr(buf->buf, '\n');
3099                 if (pch != NULL) {
3100                         rlen = 0;
3101                         len = pch - buf->buf;
3102                         if (len > 0 && (*(pch - 1) == '\r') )
3103                                 rlen ++;
3104                         StrBufSub(Line, buf, 0, len - rlen);
3105                         StrBufCutLeft(buf, len + 1);
3106                         return len - rlen;
3107                 }
3108         }
3109
3110         if (buf->BufSize - buf->BufUsed < 10)
3111                 IncreaseBuf(buf, 1, -1);
3112
3113         fdflags = fcntl(*fd, F_GETFL);
3114         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
3115
3116         while ((nSuccessLess < timeout) && (pch == NULL)) {
3117                 if (IsNonBlock){
3118                         tv.tv_sec = selectresolution;
3119                         tv.tv_usec = 0;
3120
3121                         FD_ZERO(&rfds);
3122                         FD_SET(*fd, &rfds);
3123                         if (select(*fd + 1, NULL, &rfds, NULL, &tv) == -1) {
3124                                 *Error = strerror(errno);
3125                                 close (*fd);
3126                                 *fd = -1;
3127                                 return -1;
3128                         }
3129                 }
3130                 if (IsNonBlock && !  FD_ISSET(*fd, &rfds)) {
3131                         nSuccessLess ++;
3132                         continue;
3133                 }
3134                 rlen = read(*fd,
3135                             &buf->buf[buf->BufUsed],
3136                             buf->BufSize - buf->BufUsed - 1);
3137                 if (rlen < 1) {
3138                         *Error = strerror(errno);
3139                         close(*fd);
3140                         *fd = -1;
3141                         return -1;
3142                 }
3143                 else if (rlen > 0) {
3144                         nSuccessLess = 0;
3145                         buf->BufUsed += rlen;
3146                         buf->buf[buf->BufUsed] = '\0';
3147                         if (buf->BufUsed + 10 > buf->BufSize) {
3148                                 IncreaseBuf(buf, 1, -1);
3149                         }
3150                         pch = strchr(buf->buf, '\n');
3151                         continue;
3152                 }
3153
3154         }
3155         if (pch != NULL) {
3156                 rlen = 0;
3157                 len = pch - buf->buf;
3158                 if (len > 0 && (*(pch - 1) == '\r') )
3159                         rlen ++;
3160                 StrBufSub(Line, buf, 0, len - rlen);
3161                 StrBufCutLeft(buf, len + 1);
3162                 return len - rlen;
3163         }
3164         return -1;
3165
3166 }
3167
3168 static const char *ErrRBLF_PreConditionFailed="StrBufTCP_read_buffered_line_fast: Wrong arguments or invalid Filedescriptor";
3169 static const char *ErrRBLF_SelectFailed="StrBufTCP_read_buffered_line_fast: Select failed without reason";
3170 static const char *ErrRBLF_NotEnoughSentFromServer="StrBufTCP_read_buffered_line_fast: No complete line was sent from peer";
3171 /**
3172  * @ingroup StrBuf_BufferedIO
3173  * @brief Read a line from socket
3174  * flushes and closes the FD on error
3175  * @param Line where to append our Line read from the fd / I/O Buffer;
3176  * @param IOBuf the buffer to get the input to; lifetime pair to FD
3177  * @param Pos pointer to the current read position, should be NULL initialized on opening the FD it belongs to.!
3178  * @param fd pointer to the filedescriptor to read
3179  * @param timeout number of successless selects until we bail out
3180  * @param selectresolution how long to wait on each select
3181  * @param Error strerror() on error
3182  * @returns numbers of chars read or -1 in case of error. "\n" will become 0
3183  */
3184 int StrBufTCP_read_buffered_line_fast(StrBuf *Line,
3185                                       StrBuf *IOBuf,
3186                                       const char **Pos,
3187                                       int *fd,
3188                                       int timeout,
3189                                       int selectresolution,
3190                                       const char **Error)
3191 {
3192         const char *pche = NULL;
3193         const char *pos = NULL;
3194         const char *pLF;
3195         int len, rlen, retlen;
3196         int nSuccessLess = 0;
3197         fd_set rfds;
3198         const char *pch = NULL;
3199         int fdflags;
3200         int IsNonBlock;
3201         struct timeval tv;
3202
3203         retlen = 0;
3204         if ((Line == NULL) ||
3205             (Pos == NULL) ||
3206             (IOBuf == NULL) ||
3207             (*fd == -1))
3208         {
3209                 if (Pos != NULL)
3210                         *Pos = NULL;
3211                 *Error = ErrRBLF_PreConditionFailed;
3212                 return -1;
3213         }
3214
3215         pos = *Pos;
3216         if ((IOBuf->BufUsed > 0) &&
3217             (pos != NULL) &&
3218             (pos < IOBuf->buf + IOBuf->BufUsed))
3219         {
3220                 char *pcht;
3221
3222                 pche = IOBuf->buf + IOBuf->BufUsed;
3223                 pch = pos;
3224                 pcht = Line->buf;
3225
3226                 while ((pch < pche) && (*pch != '\n'))
3227                 {
3228                         if (Line->BufUsed + 10 > Line->BufSize)
3229                         {
3230                                 long apos;
3231                                 apos = pcht - Line->buf;
3232                                 *pcht = '\0';
3233                                 IncreaseBuf(Line, 1, -1);
3234                                 pcht = Line->buf + apos;
3235                         }
3236                         *pcht++ = *pch++;
3237                         Line->BufUsed++;
3238                         retlen++;
3239                 }
3240
3241                 len = pch - pos;
3242                 if (len > 0 && (*(pch - 1) == '\r') )
3243                 {
3244                         retlen--;
3245                         len --;
3246                         pcht --;
3247                         Line->BufUsed --;
3248                 }
3249                 *pcht = '\0';
3250
3251                 if ((pch >= pche) || (*pch == '\0'))
3252                 {
3253                         FlushStrBuf(IOBuf);
3254                         *Pos = NULL;
3255                         pch = NULL;
3256                         pos = 0;
3257                 }
3258
3259                 if ((pch != NULL) &&
3260                     (pch <= pche))
3261                 {
3262                         if (pch + 1 >= pche) {
3263                                 *Pos = NULL;
3264                                 FlushStrBuf(IOBuf);
3265                         }
3266                         else
3267                                 *Pos = pch + 1;
3268
3269                         return retlen;
3270                 }
3271                 else
3272                         FlushStrBuf(IOBuf);
3273         }
3274
3275         /* If we come here, Pos is Unset since we read everything into Line, and now go for more. */
3276
3277         if (IOBuf->BufSize - IOBuf->BufUsed < 10)
3278                 IncreaseBuf(IOBuf, 1, -1);
3279
3280         fdflags = fcntl(*fd, F_GETFL);
3281         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
3282
3283         pLF = NULL;
3284         while ((nSuccessLess < timeout) &&
3285                (pLF == NULL) &&
3286                (*fd != -1)) {
3287                 if (IsNonBlock)
3288                 {
3289                         tv.tv_sec = 1;
3290                         tv.tv_usec = 0;
3291
3292                         FD_ZERO(&rfds);
3293                         FD_SET(*fd, &rfds);
3294                         if (select((*fd) + 1, &rfds, NULL, NULL, &tv) == -1) {
3295                                 *Error = strerror(errno);
3296                                 close (*fd);
3297                                 *fd = -1;
3298                                 if (*Error == NULL)
3299                                         *Error = ErrRBLF_SelectFailed;
3300                                 return -1;
3301                         }
3302                         if (! FD_ISSET(*fd, &rfds) != 0) {
3303                                 nSuccessLess ++;
3304                                 continue;
3305                         }
3306                 }
3307                 rlen = read(*fd,
3308                             &IOBuf->buf[IOBuf->BufUsed],
3309                             IOBuf->BufSize - IOBuf->BufUsed - 1);
3310                 if (rlen < 1) {
3311                         *Error = strerror(errno);
3312                         close(*fd);
3313                         *fd = -1;
3314                         return -1;
3315                 }
3316                 else if (rlen > 0) {
3317                         nSuccessLess = 0;
3318                         pLF = IOBuf->buf + IOBuf->BufUsed;
3319                         IOBuf->BufUsed += rlen;
3320                         IOBuf->buf[IOBuf->BufUsed] = '\0';
3321
3322                         pche = IOBuf->buf + IOBuf->BufUsed;
3323
3324                         while ((pLF < pche) && (*pLF != '\n'))
3325                                 pLF ++;
3326                         if ((pLF >= pche) || (*pLF == '\0'))
3327                                 pLF = NULL;
3328
3329                         if (IOBuf->BufUsed + 10 > IOBuf->BufSize)
3330                         {
3331                                 long apos;
3332
3333                                 if (pLF != NULL) apos = pLF - IOBuf->buf;
3334                                 IncreaseBuf(IOBuf, 1, -1);
3335                                 if (pLF != NULL) pLF = IOBuf->buf + apos;
3336                         }
3337
3338                         continue;
3339                 }
3340         }
3341         *Pos = NULL;
3342         if (pLF != NULL) {
3343                 pos = IOBuf->buf;
3344                 len = pLF - pos;
3345                 if (len > 0 && (*(pLF - 1) == '\r') )
3346                         len --;
3347                 StrBufAppendBufPlain(Line, ChrPtr(IOBuf), len, 0);
3348                 if (pLF + 1 >= IOBuf->buf + IOBuf->BufUsed)
3349                 {
3350                         FlushStrBuf(IOBuf);
3351                 }
3352                 else
3353                         *Pos = pLF + 1;
3354                 return retlen + len;
3355         }
3356         *Error = ErrRBLF_NotEnoughSentFromServer;
3357         return -1;
3358
3359 }
3360
3361 static const char *ErrRBLF_BLOBPreConditionFailed="StrBufReadBLOB: Wrong arguments or invalid Filedescriptor";
3362 /**
3363  * @ingroup StrBuf_IO
3364  * @brief Input binary data from socket
3365  * flushes and closes the FD on error
3366  * @param Buf the buffer to get the input to
3367  * @param fd pointer to the filedescriptor to read
3368  * @param append Append to an existing string or replace?
3369  * @param nBytes the maximal number of bytes to read
3370  * @param Error strerror() on error
3371  * @returns numbers of chars read
3372  */
3373 int StrBufReadBLOB(StrBuf *Buf, int *fd, int append, long nBytes, const char **Error)
3374 {
3375         int fdflags;
3376         int len, rlen, slen;
3377         int nSuccessLess;
3378         int nRead = 0;
3379         char *ptr;
3380         int IsNonBlock;
3381         struct timeval tv;
3382         fd_set rfds;
3383
3384         if ((Buf == NULL) || (*fd == -1))
3385         {
3386                 *Error = ErrRBLF_BLOBPreConditionFailed;
3387                 return -1;
3388         }
3389         if (!append)
3390                 FlushStrBuf(Buf);
3391         if (Buf->BufUsed + nBytes >= Buf->BufSize)
3392                 IncreaseBuf(Buf, 1, Buf->BufUsed + nBytes);
3393
3394         ptr = Buf->buf + Buf->BufUsed;
3395
3396         slen = len = Buf->BufUsed;
3397
3398         fdflags = fcntl(*fd, F_GETFL);
3399         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
3400         nSuccessLess = 0;
3401         while ((nRead < nBytes) &&
3402                (*fd != -1))
3403         {
3404                 if (IsNonBlock)
3405                 {
3406                         tv.tv_sec = 1;
3407                         tv.tv_usec = 0;
3408
3409                         FD_ZERO(&rfds);
3410                         FD_SET(*fd, &rfds);
3411                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
3412                                 *Error = strerror(errno);
3413                                 close (*fd);
3414                                 *fd = -1;
3415                                 if (*Error == NULL)
3416                                         *Error = ErrRBLF_SelectFailed;
3417                                 return -1;
3418                         }
3419                         if (! FD_ISSET(*fd, &rfds) != 0) {
3420                                 nSuccessLess ++;
3421                                 continue;
3422                         }
3423                 }
3424
3425                 if ((rlen = read(*fd,
3426                                  ptr,
3427                                  nBytes - nRead)) == -1) {
3428                         close(*fd);
3429                         *fd = -1;
3430                         *Error = strerror(errno);
3431                         return rlen;
3432                 }
3433                 nRead += rlen;
3434                 ptr += rlen;
3435                 Buf->BufUsed += rlen;
3436         }
3437         Buf->buf[Buf->BufUsed] = '\0';
3438         return nRead;
3439 }
3440
3441 const char *ErrRBB_BLOBFPreConditionFailed = "StrBufReadBLOBBuffered: to many selects; aborting.";
3442 const char *ErrRBB_too_many_selects = "StrBufReadBLOBBuffered: to many selects; aborting.";
3443 /**
3444  * @ingroup StrBuf_BufferedIO
3445  * @brief Input binary data from socket
3446  * flushes and closes the FD on error
3447  * @param Blob put binary thing here
3448  * @param IOBuf the buffer to get the input to
3449  * @param Pos offset inside of IOBuf
3450  * @param fd pointer to the filedescriptor to read
3451  * @param append Append to an existing string or replace?
3452  * @param nBytes the maximal number of bytes to read
3453  * @param check whether we should search for '000\n' terminators in case of timeouts
3454  * @param Error strerror() on error
3455  * @returns numbers of chars read
3456  */
3457 int StrBufReadBLOBBuffered(StrBuf *Blob,
3458                            StrBuf *IOBuf,
3459                            const char **Pos,
3460                            int *fd,
3461                            int append,
3462                            long nBytes,
3463                            int check,
3464                            const char **Error)
3465 {
3466         const char *pche;
3467         const char *pos;
3468         int nSelects = 0;
3469         int SelRes;
3470         int fdflags;
3471         int len = 0;
3472         int rlen, slen;
3473         int nRead = 0;
3474         int nAlreadyRead = 0;
3475         int IsNonBlock;
3476         char *ptr;
3477         fd_set rfds;
3478         const char *pch;
3479         struct timeval tv;
3480         int nSuccessLess;
3481
3482         if ((Blob == NULL) || (*fd == -1) || (IOBuf == NULL) || (Pos == NULL))
3483         {
3484                 if (*Pos != NULL)
3485                         *Pos = NULL;
3486                 *Error = ErrRBB_BLOBFPreConditionFailed;
3487                 return -1;
3488         }
3489
3490         if (!append)
3491                 FlushStrBuf(Blob);
3492         if (Blob->BufUsed + nBytes >= Blob->BufSize)
3493                 IncreaseBuf(Blob, append, Blob->BufUsed + nBytes);
3494
3495         pos = *Pos;
3496
3497         if (pos != NULL)
3498                 len = pos - IOBuf->buf;
3499         rlen = IOBuf->BufUsed - len;
3500
3501
3502         if ((IOBuf->BufUsed > 0) &&
3503             (pos != NULL) &&
3504             (pos < IOBuf->buf + IOBuf->BufUsed))
3505         {
3506                 pche = IOBuf->buf + IOBuf->BufUsed;
3507                 pch = pos;
3508
3509                 if (rlen < nBytes) {
3510                         memcpy(Blob->buf + Blob->BufUsed, pos, rlen);
3511                         Blob->BufUsed += rlen;
3512                         Blob->buf[Blob->BufUsed] = '\0';
3513                         nAlreadyRead = nRead = rlen;
3514                         *Pos = NULL;
3515                 }
3516                 if (rlen >= nBytes) {
3517                         memcpy(Blob->buf + Blob->BufUsed, pos, nBytes);
3518                         Blob->BufUsed += nBytes;
3519                         Blob->buf[Blob->BufUsed] = '\0';
3520                         if (rlen == nBytes) {
3521                                 *Pos = NULL;
3522                                 FlushStrBuf(IOBuf);
3523                         }
3524                         else
3525                                 *Pos += nBytes;
3526                         return nBytes;
3527                 }
3528         }
3529
3530         FlushStrBuf(IOBuf);
3531         *Pos = NULL;
3532         if (IOBuf->BufSize < nBytes - nRead)
3533                 IncreaseBuf(IOBuf, 0, nBytes - nRead);
3534         ptr = IOBuf->buf;
3535
3536         slen = len = Blob->BufUsed;
3537
3538         fdflags = fcntl(*fd, F_GETFL);
3539         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
3540
3541         SelRes = 1;
3542         nBytes -= nRead;
3543         nRead = 0;
3544         while ((nRead < nBytes) &&
3545                (*fd != -1)) {
3546                 if (IsNonBlock)
3547                 {
3548                         tv.tv_sec = 1;
3549                         tv.tv_usec = 0;
3550
3551                         FD_ZERO(&rfds);
3552                         FD_SET(*fd, &rfds);
3553                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
3554                                 *Error = strerror(errno);
3555                                 close (*fd);
3556                                 *fd = -1;
3557                                 if (*Error == NULL)
3558                                         *Error = ErrRBLF_SelectFailed;
3559                                 return -1;
3560                         }
3561                         if (! FD_ISSET(*fd, &rfds) != 0) {
3562                                 nSuccessLess ++;
3563                                 continue;
3564                         }
3565                 }
3566                 nSuccessLess = 0;
3567                 rlen = read(*fd,
3568                             ptr,
3569                             nBytes - nRead);
3570                 if (rlen == -1) {
3571                         close(*fd);
3572                         *fd = -1;
3573                         *Error = strerror(errno);
3574                         return rlen;
3575                 }
3576                 else if (rlen == 0){
3577                         nSuccessLess ++;
3578                         if ((check == NNN_TERM) &&
3579                             (nRead > 5) &&
3580                             (strncmp(IOBuf->buf + IOBuf->BufUsed - 5, "\n000\n", 5) == 0))
3581                         {
3582                                 StrBufPlain(Blob, HKEY("\n000\n"));
3583                                 StrBufCutRight(Blob, 5);
3584                                 return Blob->BufUsed;
3585                         }
3586                         if (nSelects > 10) {
3587                                 FlushStrBuf(IOBuf);
3588                                 *Error = ErrRBB_too_many_selects;
3589                                 return -1;
3590                         }
3591                 }
3592                 else if (rlen > 0) {
3593                         nRead += rlen;
3594                         ptr += rlen;
3595                         IOBuf->BufUsed += rlen;
3596                 }
3597         }
3598         if (nRead > nBytes) {
3599                 *Pos = IOBuf->buf + nBytes;
3600         }
3601         Blob->buf[Blob->BufUsed] = '\0';
3602         StrBufAppendBufPlain(Blob, IOBuf->buf, nBytes, 0);
3603         if (*Pos == NULL) {
3604                 FlushStrBuf(IOBuf);
3605         }
3606         return nRead + nAlreadyRead;
3607 }
3608
3609 /**
3610  * @ingroup StrBuf_IO
3611  * @brief extract a "next line" from Buf; Ptr to persist across several iterations
3612  * @param LineBuf your line will be copied here.
3613  * @param Buf BLOB with lines of text...
3614  * @param Ptr moved arround to keep the next-line across several iterations
3615  *        has to be &NULL on start; will be &NotNULL on end of buffer
3616  * @returns size of copied buffer
3617  */
3618 int StrBufSipLine(StrBuf *LineBuf, StrBuf *Buf, const char **Ptr)
3619 {
3620         const char *aptr, *ptr, *eptr;
3621         char *optr, *xptr;
3622
3623         if ((Buf == NULL) || (*Ptr == StrBufNOTNULL)) {
3624                 *Ptr = StrBufNOTNULL;
3625                 return 0;
3626         }
3627
3628         FlushStrBuf(LineBuf);
3629         if (*Ptr==NULL)
3630                 ptr = aptr = Buf->buf;
3631         else
3632                 ptr = aptr = *Ptr;
3633
3634         optr = LineBuf->buf;
3635         eptr = Buf->buf + Buf->BufUsed;
3636         xptr = LineBuf->buf + LineBuf->BufSize - 1;
3637
3638         while ((ptr <= eptr) &&
3639                (*ptr != '\n') &&
3640                (*ptr != '\r') )
3641         {
3642                 *optr = *ptr;
3643                 optr++; ptr++;
3644                 if (optr == xptr) {
3645                         LineBuf->BufUsed = optr - LineBuf->buf;
3646                         IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
3647                         optr = LineBuf->buf + LineBuf->BufUsed;
3648                         xptr = LineBuf->buf + LineBuf->BufSize - 1;
3649                 }
3650         }
3651
3652         if ((ptr >= eptr) && (optr > LineBuf->buf))
3653                 optr --;
3654         LineBuf->BufUsed = optr - LineBuf->buf;
3655         *optr = '\0';
3656         if ((ptr <= eptr) && (*ptr == '\r'))
3657                 ptr ++;
3658         if ((ptr <= eptr) && (*ptr == '\n'))
3659                 ptr ++;
3660
3661         if (ptr < eptr) {
3662                 *Ptr = ptr;
3663         }
3664         else {
3665                 *Ptr = StrBufNOTNULL;
3666         }
3667
3668         return Buf->BufUsed - (ptr - Buf->buf);
3669 }
3670
3671
3672 /**
3673  * @ingroup StrBuf_IO
3674  * @brief removes double slashes from pathnames
3675  * @param Dir directory string to filter
3676  * @param RemoveTrailingSlash allows / disallows trailing slashes
3677  */
3678 void StrBufStripSlashes(StrBuf *Dir, int RemoveTrailingSlash)
3679 {
3680         char *a, *b;
3681
3682         a = b = Dir->buf;
3683
3684         while (!IsEmptyStr(a)) {
3685                 if (*a == '/') {
3686                         while (*a == '/')
3687                                 a++;
3688                         *b = '/';
3689                         b++;
3690                 }
3691                 else {
3692                         *b = *a;
3693                         b++; a++;
3694                 }
3695         }
3696         if ((RemoveTrailingSlash) && (*(b - 1) != '/')){
3697                 *b = '/';
3698                 b++;
3699         }
3700         *b = '\0';
3701         Dir->BufUsed = b - Dir->buf;
3702 }
3703
3704