libcitadel/lib/stringbuf.c

   1 /*
   2  * Copyright (c) 1987-2013 by the citadel.org team
   3  *
   4  * This program is open source software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 3 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #define _GNU_SOURCE
  20 #include "sysdep.h"
  21 #include <ctype.h>
  22 #include <errno.h>
  23 #include <string.h>
  24 #include <unistd.h>
  25 #include <string.h>
  26 #include <stdio.h>
  27 #include <sys/select.h>
  28 #include <fcntl.h>
  29 #include <sys/types.h>
  30 #define SHOW_ME_VAPPEND_PRINTF
  31 #include <stdarg.h>
  32
  33 #include "libcitadel.h"
  34
  35 #include "b64/cencode.h"
  36 #include "b64/cdecode.h"
  37
  38 #ifdef HAVE_ICONV
  39 #include <iconv.h>
  40 #endif
  41
  42 #ifdef HAVE_BACKTRACE
  43 #include <execinfo.h>
  44 #endif
  45
  46 #ifdef UNDEF_MEMCPY
  47 #undef memcpy
  48 #endif
  49
  50 #ifdef HAVE_ZLIB
  51 #include <zlib.h>
  52 int ZEXPORT compress_gzip(Bytef * dest, size_t * destLen,
  53                           const Bytef * source, uLong sourceLen, int level);
  54 #endif
  55 int BaseStrBufSize = 64;
  56 int EnableSplice = 0;
  57 int ZLibCompressionRatio = -1; /* defaults to 6 */
  58 #ifdef HAVE_ZLIB
  59 #define DEF_MEM_LEVEL 8 /*< memlevel??? */
  60 #define OS_CODE 0x03    /*< unix */
  61 const int gz_magic[2] = { 0x1f, 0x8b }; /* gzip magic header */
  62 #endif
  63
  64 const char *StrBufNOTNULL = ((char*) NULL) - 1;
  65
  66 const char HexList[256][3] = {
  67         "00","01","02","03","04","05","06","07","08","09","0A","0B","0C","0D","0E","0F",
  68         "10","11","12","13","14","15","16","17","18","19","1A","1B","1C","1D","1E","1F",
  69         "20","21","22","23","24","25","26","27","28","29","2A","2B","2C","2D","2E","2F",
  70         "30","31","32","33","34","35","36","37","38","39","3A","3B","3C","3D","3E","3F",
  71         "40","41","42","43","44","45","46","47","48","49","4A","4B","4C","4D","4E","4F",
  72         "50","51","52","53","54","55","56","57","58","59","5A","5B","5C","5D","5E","5F",
  73         "60","61","62","63","64","65","66","67","68","69","6A","6B","6C","6D","6E","6F",
  74         "70","71","72","73","74","75","76","77","78","79","7A","7B","7C","7D","7E","7F",
  75         "80","81","82","83","84","85","86","87","88","89","8A","8B","8C","8D","8E","8F",
  76         "90","91","92","93","94","95","96","97","98","99","9A","9B","9C","9D","9E","9F",
  77         "A0","A1","A2","A3","A4","A5","A6","A7","A8","A9","AA","AB","AC","AD","AE","AF",
  78         "B0","B1","B2","B3","B4","B5","B6","B7","B8","B9","BA","BB","BC","BD","BE","BF",
  79         "C0","C1","C2","C3","C4","C5","C6","C7","C8","C9","CA","CB","CC","CD","CE","CF",
  80         "D0","D1","D2","D3","D4","D5","D6","D7","D8","D9","DA","DB","DC","DD","DE","DF",
  81         "E0","E1","E2","E3","E4","E5","E6","E7","E8","E9","EA","EB","EC","ED","EE","EF",
  82         "F0","F1","F2","F3","F4","F5","F6","F7","F8","F9","FA","FB","FC","FD","FE","FF"};
  83
  84 /**
  85  * @defgroup StrBuf Stringbuffer, A class for manipulating strings with dynamic buffers
  86  * StrBuf is a versatile class, aiding the handling of dynamic strings
  87  *  * reduce de/reallocations
  88  *  * reduce the need to remeasure it
  89  *  * reduce scanning over the string (in @ref StrBuf_NextTokenizer "Tokenizers")
  90  *  * allow asyncroneous IO for line and Blob based operations
  91  *  * reduce the use of memove in those
  92  *  * Quick filling in several operations with append functions
  93  */
  94
  95 /**
  96  * @defgroup StrBuf_DeConstructors Create/Destroy StrBufs
  97  * @ingroup StrBuf
  98  */
  99
 100 /**
 101  * @defgroup StrBuf_Cast Cast operators to interact with char* based code
 102  * @ingroup StrBuf
 103  * use these operators to interfere with code demanding char*;
 104  * if you need to own the content, smash me. Avoid, since we loose the length information.
 105  */
 106
 107 /**
 108  * @defgroup StrBuf_Filler Create/Replace/Append Content into a StrBuf
 109  * @ingroup StrBuf
 110  * operations to get your Strings into a StrBuf, manipulating them, or appending
 111  */
 112 /**
 113  * @defgroup StrBuf_NextTokenizer Fast tokenizer to pull tokens in sequence
 114  * @ingroup StrBuf
 115  * Quick tokenizer; demands of the user to pull its tokens in sequence
 116  */
 117
 118 /**
 119  * @defgroup StrBuf_Tokenizer tokenizer Functions; Slow ones.
 120  * @ingroup StrBuf
 121  * versatile tokenizer; random access to tokens, but slower; Prefer the @ref StrBuf_NextTokenizer "Next Tokenizer"
 122  */
 123
 124 /**
 125  * @defgroup StrBuf_BufferedIO Buffered IO with Asynchroneous reads and no unneeded memmoves (the fast ones)
 126  * @ingroup StrBuf
 127  * File IO to fill StrBufs; Works with work-buffer shared across several calls;
 128  * External Cursor to maintain the current read position inside of the buffer
 129  * the non-fast ones will use memove to keep the start of the buffer the read buffer (which is slower)
 130  */
 131
 132 /**
 133  * @defgroup StrBuf_IO FileIO; Prefer @ref StrBuf_BufferedIO
 134  * @ingroup StrBuf
 135  * Slow I/O; avoid.
 136  */
 137
 138 /**
 139  * @defgroup StrBuf_DeEnCoder functions to translate the contents of a buffer
 140  * @ingroup StrBuf
 141  * these functions translate the content of a buffer into another representation;
 142  * some are combined Fillers and encoders
 143  */
 144
 145 /**
 146  * Private Structure for the Stringbuffer
 147  */
 148 struct StrBuf {
 149         char *buf;         /**< the pointer to the dynamic buffer */
 150         long BufSize;      /**< how many spcae do we optain */
 151         long BufUsed;      /**< StNumber of Chars used excluding the trailing \\0 */
 152         int ConstBuf;      /**< are we just a wrapper arround a static buffer and musn't we be changed? */
 153 #ifdef SIZE_DEBUG
 154         long nIncreases;   /**< for profiling; cound how many times we needed more */
 155         char bt [SIZ];     /**< Stacktrace of last increase */
 156         char bt_lastinc [SIZ]; /**< How much did we increase last time? */
 157 #endif
 158 };
 159
 160
 161 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE);
 162 static inline int Ctdl_IsUtf8SequenceStart(const char Char);
 163
 164 #ifdef SIZE_DEBUG
 165 #ifdef HAVE_BACKTRACE
 166 static void StrBufBacktrace(StrBuf *Buf, int which)
 167 {
 168         int n;
 169         char *pstart, *pch;
 170         void *stack_frames[50];
 171         size_t size, i;
 172         char **strings;
 173
 174         if (which)
 175                 pstart = pch = Buf->bt;
 176         else
 177                 pstart = pch = Buf->bt_lastinc;
 178         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
 179         strings = backtrace_symbols(stack_frames, size);
 180         for (i = 0; i < size; i++) {
 181                 if (strings != NULL)
 182                         n = snprintf(pch, SIZ - (pch - pstart), "%s\\n", strings[i]);
 183                 else
 184                         n = snprintf(pch, SIZ - (pch - pstart), "%p\\n", stack_frames[i]);
 185                 pch += n;
 186         }
 187         free(strings);
 188
 189
 190 }
 191 #endif
 192
 193 void dbg_FreeStrBuf(StrBuf *FreeMe, char *FromWhere)
 194 {
 195         if (hFreeDbglog == -1){
 196                 pid_t pid = getpid();
 197                 char path [SIZ];
 198                 snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
 199                 hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
 200         }
 201         if ((*FreeMe)->nIncreases > 0)
 202         {
 203                 char buf[SIZ * 3];
 204                 long n;
 205                 n = snprintf(buf, SIZ * 3, "%c+|%ld|%ld|%ld|%s|%s|\n",
 206                              FromWhere,
 207                              (*FreeMe)->nIncreases,
 208                              (*FreeMe)->BufUsed,
 209                              (*FreeMe)->BufSize,
 210                              (*FreeMe)->bt,
 211                              (*FreeMe)->bt_lastinc);
 212                 n = write(hFreeDbglog, buf, n);
 213         }
 214         else
 215         {
 216                 char buf[128];
 217                 long n;
 218                 n = snprintf(buf, 128, "%c_|0|%ld%ld|\n",
 219                              FromWhere,
 220                              (*FreeMe)->BufUsed,
 221                              (*FreeMe)->BufSize);
 222                 n = write(hFreeDbglog, buf, n);
 223         }
 224 }
 225
 226 void dbg_IncreaseBuf(StrBuf *IncMe)
 227 {
 228         Buf->nIncreases++;
 229 #ifdef HAVE_BACKTRACE
 230         StrBufBacktrace(Buf, 1);
 231 #endif
 232 }
 233
 234 void dbg_Init(StrBuf *Buf)
 235 {
 236         Buf->nIncreases = 0;
 237         Buf->bt[0] = '\0';
 238         Buf->bt_lastinc[0] = '\0';
 239 #ifdef HAVE_BACKTRACE
 240         StrBufBacktrace(Buf, 0);
 241 #endif
 242 }
 243
 244 #else
 245 /* void it... */
 246 #define dbg_FreeStrBuf(a, b)
 247 #define dbg_IncreaseBuf(a)
 248 #define dbg_Init(a)
 249
 250 #endif
 251
 252 /**
 253  * @ingroup StrBuf
 254  * @brief swaps the contents of two StrBufs
 255  * this is to be used to have cheap switched between a work-buffer and a target buffer
 256  * @param A First one
 257  * @param B second one
 258  */
 259 static inline void SwapBuffers(StrBuf *A, StrBuf *B)
 260 {
 261         StrBuf C;
 262
 263         memcpy(&C, A, sizeof(*A));
 264         memcpy(A, B, sizeof(*B));
 265         memcpy(B, &C, sizeof(C));
 266
 267 }
 268
 269 /**
 270  * @ingroup StrBuf_Cast
 271  * @brief Cast operator to Plain String
 272  * @note if the buffer is altered by StrBuf operations, this pointer may become
 273  *  invalid. So don't lean on it after altering the buffer!
 274  *  Since this operation is considered cheap, rather call it often than risking
 275  *  your pointer to become invalid!
 276  * @param Str the string we want to get the c-string representation for
 277  * @returns the Pointer to the Content. Don't mess with it!
 278  */
 279 inline const char *ChrPtr(const StrBuf *Str)
 280 {
 281         if (Str == NULL)
 282                 return "";
 283         return Str->buf;
 284 }
 285
 286 /**
 287  * @ingroup StrBuf_Cast
 288  * @brief since we know strlen()'s result, provide it here.
 289  * @param Str the string to return the length to
 290  * @returns contentlength of the buffer
 291  */
 292 inline int StrLength(const StrBuf *Str)
 293 {
 294         return (Str != NULL) ? Str->BufUsed : 0;
 295 }
 296
 297 /**
 298  * @ingroup StrBuf_DeConstructors
 299  * @brief local utility function to resize the buffer
 300  * @param Buf the buffer whichs storage we should increase
 301  * @param KeepOriginal should we copy the original buffer or just start over with a new one
 302  * @param DestSize what should fit in after?
 303  */
 304 static int IncreaseBuf(StrBuf *Buf, int KeepOriginal, int DestSize)
 305 {
 306         char *NewBuf;
 307         size_t NewSize = Buf->BufSize * 2;
 308
 309         if (Buf->ConstBuf)
 310                 return -1;
 311
 312         if (DestSize > 0)
 313                 while ((NewSize <= DestSize) && (NewSize != 0))
 314                         NewSize *= 2;
 315
 316         if (NewSize == 0)
 317                 return -1;
 318
 319         NewBuf= (char*) malloc(NewSize);
 320         if (NewBuf == NULL)
 321                 return -1;
 322
 323         if (KeepOriginal && (Buf->BufUsed > 0))
 324         {
 325                 memcpy(NewBuf, Buf->buf, Buf->BufUsed);
 326         }
 327         else
 328         {
 329                 NewBuf[0] = '\0';
 330                 Buf->BufUsed = 0;
 331         }
 332         free (Buf->buf);
 333         Buf->buf = NewBuf;
 334         Buf->BufSize = NewSize;
 335
 336         dbg_IncreaseBuf(Buf);
 337
 338         return Buf->BufSize;
 339 }
 340
 341 /**
 342  * @ingroup StrBuf_DeConstructors
 343  * @brief shrink / increase an _EMPTY_ buffer to NewSize. Buffercontent is thoroughly ignored and flushed.
 344  * @param Buf Buffer to shrink (has to be empty)
 345  * @param ThreshHold if the buffer is bigger then this, its readjusted
 346  * @param NewSize if we Shrink it, how big are we going to be afterwards?
 347  */
 348 void ReAdjustEmptyBuf(StrBuf *Buf, long ThreshHold, long NewSize)
 349 {
 350         if ((Buf != NULL) &&
 351             (Buf->BufUsed == 0) &&
 352             (Buf->BufSize < ThreshHold)) {
 353                 free(Buf->buf);
 354                 Buf->buf = (char*) malloc(NewSize);
 355                 Buf->BufUsed = 0;
 356                 Buf->BufSize = NewSize;
 357         }
 358 }
 359
 360 /**
 361  * @ingroup StrBuf_DeConstructors
 362  * @brief shrink long term buffers to their real size so they don't waste memory
 363  * @param Buf buffer to shrink
 364  * @param Force if not set, will just executed if the buffer is much to big; set for lifetime strings
 365  * @returns physical size of the buffer
 366  */
 367 long StrBufShrinkToFit(StrBuf *Buf, int Force)
 368 {
 369         if (Buf == NULL)
 370                 return -1;
 371         if (Force ||
 372             (Buf->BufUsed + (Buf->BufUsed / 3) > Buf->BufSize))
 373         {
 374                 char *TmpBuf;
 375
 376                 TmpBuf = (char*) malloc(Buf->BufUsed + 1);
 377                 if (TmpBuf == NULL)
 378                         return -1;
 379
 380                 memcpy (TmpBuf, Buf->buf, Buf->BufUsed + 1);
 381                 Buf->BufSize = Buf->BufUsed + 1;
 382                 free(Buf->buf);
 383                 Buf->buf = TmpBuf;
 384         }
 385         return Buf->BufUsed;
 386 }
 387
 388 /**
 389  * @ingroup StrBuf_DeConstructors
 390  * @brief Allocate a new buffer with default buffer size
 391  * @returns the new stringbuffer
 392  */
 393 StrBuf* NewStrBuf(void)
 394 {
 395         StrBuf *NewBuf;
 396
 397         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 398         if (NewBuf == NULL)
 399                 return NULL;
 400
 401         NewBuf->buf = (char*) malloc(BaseStrBufSize);
 402         if (NewBuf->buf == NULL)
 403         {
 404                 free(NewBuf);
 405                 return NULL;
 406         }
 407         NewBuf->buf[0] = '\0';
 408         NewBuf->BufSize = BaseStrBufSize;
 409         NewBuf->BufUsed = 0;
 410         NewBuf->ConstBuf = 0;
 411
 412         dbg_Init (NewBuf);
 413
 414         return NewBuf;
 415 }
 416
 417 /**
 418  * @ingroup StrBuf_DeConstructors
 419  * @brief Copy Constructor; returns a duplicate of CopyMe
 420  * @param CopyMe Buffer to faxmilate
 421  * @returns the new stringbuffer
 422  */
 423 StrBuf* NewStrBufDup(const StrBuf *CopyMe)
 424 {
 425         StrBuf *NewBuf;
 426
 427         if (CopyMe == NULL)
 428                 return NewStrBuf();
 429
 430         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 431         if (NewBuf == NULL)
 432                 return NULL;
 433
 434         NewBuf->buf = (char*) malloc(CopyMe->BufSize);
 435         if (NewBuf->buf == NULL)
 436         {
 437                 free(NewBuf);
 438                 return NULL;
 439         }
 440
 441         memcpy(NewBuf->buf, CopyMe->buf, CopyMe->BufUsed + 1);
 442         NewBuf->BufUsed = CopyMe->BufUsed;
 443         NewBuf->BufSize = CopyMe->BufSize;
 444         NewBuf->ConstBuf = 0;
 445
 446         dbg_Init(NewBuf);
 447
 448         return NewBuf;
 449 }
 450
 451 /**
 452  * @ingroup StrBuf_DeConstructors
 453  * @brief Copy Constructor; CreateRelpaceMe will contain CopyFlushMe afterwards.
 454  * @param NoMe if non-NULL, we will use that buffer as value; KeepOriginal will abused as len.
 455  * @param CopyFlushMe Buffer to faxmilate if KeepOriginal, or to move into CreateRelpaceMe if !KeepOriginal.
 456  * @param CreateRelpaceMe If NULL, will be created, else Flushed and filled CopyFlushMe
 457  * @param KeepOriginal should CopyFlushMe remain intact? or may we Steal its buffer?
 458  * @returns the new stringbuffer
 459  */
 460 void NewStrBufDupAppendFlush(StrBuf **CreateRelpaceMe, StrBuf *CopyFlushMe, const char *NoMe, int KeepOriginal)
 461 {
 462         StrBuf *NewBuf;
 463
 464         if (CreateRelpaceMe == NULL)
 465                 return;
 466
 467         if (NoMe != NULL)
 468         {
 469                 if (*CreateRelpaceMe != NULL)
 470                         StrBufPlain(*CreateRelpaceMe, NoMe, KeepOriginal);
 471                 else
 472                         *CreateRelpaceMe = NewStrBufPlain(NoMe, KeepOriginal);
 473                 return;
 474         }
 475
 476         if (CopyFlushMe == NULL)
 477         {
 478                 if (*CreateRelpaceMe != NULL)
 479                         FlushStrBuf(*CreateRelpaceMe);
 480                 else
 481                         *CreateRelpaceMe = NewStrBuf();
 482                 return;
 483         }
 484
 485         /*
 486          * Randomly Chosen: bigger than 64 chars is cheaper to swap the buffers instead of copying.
 487          * else *CreateRelpaceMe may use more memory than needed in a longer term, CopyFlushMe might
 488          * be a big IO-Buffer...
 489          */
 490         if (KeepOriginal || (StrLength(CopyFlushMe) < 256))
 491         {
 492                 if (*CreateRelpaceMe == NULL)
 493                 {
 494                         *CreateRelpaceMe = NewBuf = NewStrBufPlain(NULL, CopyFlushMe->BufUsed);
 495                         dbg_Init(NewBuf);
 496                 }
 497                 else
 498                 {
 499                         NewBuf = *CreateRelpaceMe;
 500                         FlushStrBuf(NewBuf);
 501                 }
 502                 StrBufAppendBuf(NewBuf, CopyFlushMe, 0);
 503         }
 504         else
 505         {
 506                 if (*CreateRelpaceMe == NULL)
 507                 {
 508                         *CreateRelpaceMe = NewBuf = NewStrBufPlain(NULL, CopyFlushMe->BufUsed);
 509                         dbg_Init(NewBuf);
 510                 }
 511                 else
 512                         NewBuf = *CreateRelpaceMe;
 513                 SwapBuffers (NewBuf, CopyFlushMe);
 514         }
 515         if (!KeepOriginal)
 516                 FlushStrBuf(CopyFlushMe);
 517         return;
 518 }
 519
 520 /**
 521  * @ingroup StrBuf_DeConstructors
 522  * @brief create a new Buffer using an existing c-string
 523  * this function should also be used if you want to pre-suggest
 524  * the buffer size to allocate in conjunction with ptr == NULL
 525  * @param ptr the c-string to copy; may be NULL to create a blank instance
 526  * @param nChars How many chars should we copy; -1 if we should measure the length ourselves
 527  * @returns the new stringbuffer
 528  */
 529 StrBuf* NewStrBufPlain(const char* ptr, int nChars)
 530 {
 531         StrBuf *NewBuf;
 532         size_t Siz = BaseStrBufSize;
 533         size_t CopySize;
 534
 535         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 536         if (NewBuf == NULL)
 537                 return NULL;
 538
 539         if (nChars < 0)
 540                 CopySize = strlen((ptr != NULL)?ptr:"");
 541         else
 542                 CopySize = nChars;
 543
 544         while ((Siz <= CopySize) && (Siz != 0))
 545                 Siz *= 2;
 546
 547         if (Siz == 0)
 548         {
 549                 free(NewBuf);
 550                 return NULL;
 551         }
 552
 553         NewBuf->buf = (char*) malloc(Siz);
 554         if (NewBuf->buf == NULL)
 555         {
 556                 free(NewBuf);
 557                 return NULL;
 558         }
 559         NewBuf->BufSize = Siz;
 560         if (ptr != NULL) {
 561                 memcpy(NewBuf->buf, ptr, CopySize);
 562                 NewBuf->buf[CopySize] = '\0';
 563                 NewBuf->BufUsed = CopySize;
 564         }
 565         else {
 566                 NewBuf->buf[0] = '\0';
 567                 NewBuf->BufUsed = 0;
 568         }
 569         NewBuf->ConstBuf = 0;
 570
 571         dbg_Init(NewBuf);
 572
 573         return NewBuf;
 574 }
 575
 576 /**
 577  * @ingroup StrBuf_DeConstructors
 578  * @brief Set an existing buffer from a c-string
 579  * @param Buf buffer to load
 580  * @param ptr c-string to put into
 581  * @param nChars set to -1 if we should work 0-terminated
 582  * @returns the new length of the string
 583  */
 584 int StrBufPlain(StrBuf *Buf, const char* ptr, int nChars)
 585 {
 586         size_t Siz;
 587         size_t CopySize;
 588
 589         if (Buf == NULL)
 590                 return -1;
 591         if (ptr == NULL) {
 592                 FlushStrBuf(Buf);
 593                 return -1;
 594         }
 595
 596         Siz = Buf->BufSize;
 597
 598         if (nChars < 0)
 599                 CopySize = strlen(ptr);
 600         else
 601                 CopySize = nChars;
 602
 603         while ((Siz <= CopySize) && (Siz != 0))
 604                 Siz *= 2;
 605
 606         if (Siz == 0) {
 607                 FlushStrBuf(Buf);
 608                 return -1;
 609         }
 610
 611         if (Siz != Buf->BufSize)
 612                 IncreaseBuf(Buf, 0, Siz);
 613         memcpy(Buf->buf, ptr, CopySize);
 614         Buf->buf[CopySize] = '\0';
 615         Buf->BufUsed = CopySize;
 616         Buf->ConstBuf = 0;
 617         return CopySize;
 618 }
 619
 620
 621 /**
 622  * @ingroup StrBuf_DeConstructors
 623  * @brief use strbuf as wrapper for a string constant for easy handling
 624  * @param StringConstant a string to wrap
 625  * @param SizeOfStrConstant should be sizeof(StringConstant)-1
 626  */
 627 StrBuf* _NewConstStrBuf(const char* StringConstant, size_t SizeOfStrConstant)
 628 {
 629         StrBuf *NewBuf;
 630
 631         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 632         if (NewBuf == NULL)
 633                 return NULL;
 634         NewBuf->buf = (char*) StringConstant;
 635         NewBuf->BufSize = SizeOfStrConstant;
 636         NewBuf->BufUsed = SizeOfStrConstant;
 637         NewBuf->ConstBuf = 1;
 638
 639         dbg_Init(NewBuf);
 640
 641         return NewBuf;
 642 }
 643
 644
 645 /**
 646  * @ingroup StrBuf_DeConstructors
 647  * @brief flush the content of a Buf; keep its struct
 648  * @param buf Buffer to flush
 649  */
 650 int FlushStrBuf(StrBuf *buf)
 651 {
 652         if ((buf == NULL) || (buf->buf == NULL))
 653                 return -1;
 654         if (buf->ConstBuf)
 655                 return -1;
 656         buf->buf[0] ='\0';
 657         buf->BufUsed = 0;
 658         return 0;
 659 }
 660
 661 /**
 662  * @ingroup StrBuf_DeConstructors
 663  * @brief wipe the content of a Buf thoroughly (overwrite it -> expensive); keep its struct
 664  * @param buf Buffer to wipe
 665  */
 666 int FLUSHStrBuf(StrBuf *buf)
 667 {
 668         if (buf == NULL)
 669                 return -1;
 670         if (buf->ConstBuf)
 671                 return -1;
 672         if (buf->BufUsed > 0) {
 673                 memset(buf->buf, 0, buf->BufUsed);
 674                 buf->BufUsed = 0;
 675         }
 676         return 0;
 677 }
 678
 679 #ifdef SIZE_DEBUG
 680 int hFreeDbglog = -1;
 681 #endif
 682 /**
 683  * @ingroup StrBuf_DeConstructors
 684  * @brief Release a Buffer
 685  * Its a double pointer, so it can NULL your pointer
 686  * so fancy SIG11 appear instead of random results
 687  * @param FreeMe Pointer Pointer to the buffer to free
 688  */
 689 void FreeStrBuf (StrBuf **FreeMe)
 690 {
 691         if (*FreeMe == NULL)
 692                 return;
 693
 694         dbg_FreeStrBuf(FreeMe, 'F');
 695
 696         if (!(*FreeMe)->ConstBuf)
 697                 free((*FreeMe)->buf);
 698         free(*FreeMe);
 699         *FreeMe = NULL;
 700 }
 701
 702 /**
 703  * @ingroup StrBuf_DeConstructors
 704  * @brief flatten a Buffer to the Char * we return
 705  * Its a double pointer, so it can NULL your pointer
 706  * so fancy SIG11 appear instead of random results
 707  * The Callee then owns the buffer and is responsible for freeing it.
 708  * @param SmashMe Pointer Pointer to the buffer to release Buf from and free
 709  * @returns the pointer of the buffer; Callee owns the memory thereafter.
 710  */
 711 char *SmashStrBuf (StrBuf **SmashMe)
 712 {
 713         char *Ret;
 714
 715         if ((SmashMe == NULL) || (*SmashMe == NULL))
 716                 return NULL;
 717
 718         dbg_FreeStrBuf(SmashMe, 'S');
 719
 720         Ret = (*SmashMe)->buf;
 721         free(*SmashMe);
 722         *SmashMe = NULL;
 723         return Ret;
 724 }
 725
 726 /**
 727  * @ingroup StrBuf_DeConstructors
 728  * @brief Release the buffer
 729  * If you want put your StrBuf into a Hash, use this as Destructor.
 730  * @param VFreeMe untyped pointer to a StrBuf. be shure to do the right thing [TM]
 731  */
 732 void HFreeStrBuf (void *VFreeMe)
 733 {
 734         StrBuf *FreeMe = (StrBuf*)VFreeMe;
 735         if (FreeMe == NULL)
 736                 return;
 737
 738         dbg_FreeStrBuf(SmashMe, 'H');
 739
 740         if (!FreeMe->ConstBuf)
 741                 free(FreeMe->buf);
 742         free(FreeMe);
 743 }
 744
 745
 746 /*******************************************************************************
 747  *                      Simple string transformations                          *
 748  *******************************************************************************/
 749
 750 /**
 751  * @ingroup StrBuf
 752  * @brief Wrapper around atol
 753  */
 754 long StrTol(const StrBuf *Buf)
 755 {
 756         if (Buf == NULL)
 757                 return 0;
 758         if(Buf->BufUsed > 0)
 759                 return atol(Buf->buf);
 760         else
 761                 return 0;
 762 }
 763
 764 /**
 765  * @ingroup StrBuf
 766  * @brief Wrapper around atoi
 767  */
 768 int StrToi(const StrBuf *Buf)
 769 {
 770         if (Buf == NULL)
 771                 return 0;
 772         if (Buf->BufUsed > 0)
 773                 return atoi(Buf->buf);
 774         else
 775                 return 0;
 776 }
 777
 778 /**
 779  * @ingroup StrBuf
 780  * @brief Checks to see if the string is a pure number
 781  * @param Buf The buffer to inspect
 782  * @returns 1 if its a pure number, 0, if not.
 783  */
 784 int StrBufIsNumber(const StrBuf *Buf) {
 785         char * pEnd;
 786         if ((Buf == NULL) || (Buf->BufUsed == 0)) {
 787                 return 0;
 788         }
 789         strtoll(Buf->buf, &pEnd, 10);
 790         if (pEnd == Buf->buf)
 791                 return 0;
 792         if ((pEnd != NULL) && (pEnd == Buf->buf + Buf->BufUsed))
 793                 return 1;
 794         if (Buf->buf == pEnd)
 795                 return 0;
 796         return 0;
 797 }
 798
 799 /**
 800  * @ingroup StrBuf_Filler
 801  * @brief modifies a Single char of the Buf
 802  * You can point to it via char* or a zero-based integer
 803  * @param Buf The buffer to manipulate
 804  * @param ptr char* to zero; use NULL if unused
 805  * @param nThChar zero based pointer into the string; use -1 if unused
 806  * @param PeekValue The Character to place into the position
 807  */
 808 long StrBufPeek(StrBuf *Buf, const char* ptr, long nThChar, char PeekValue)
 809 {
 810         if (Buf == NULL)
 811                 return -1;
 812         if (ptr != NULL)
 813                 nThChar = ptr - Buf->buf;
 814         if ((nThChar < 0) || (nThChar > Buf->BufUsed))
 815                 return -1;
 816         Buf->buf[nThChar] = PeekValue;
 817         return nThChar;
 818 }
 819
 820 /**
 821  * @ingroup StrBuf_Filler
 822  * @brief modifies a range of chars of the Buf
 823  * You can point to it via char* or a zero-based integer
 824  * @param Buf The buffer to manipulate
 825  * @param ptr char* to zero; use NULL if unused
 826  * @param nThChar zero based pointer into the string; use -1 if unused
 827  * @param nChars how many chars are to be flushed?
 828  * @param PookValue The Character to place into that area
 829  */
 830 long StrBufPook(StrBuf *Buf, const char* ptr, long nThChar, long nChars, char PookValue)
 831 {
 832         if (Buf == NULL)
 833                 return -1;
 834         if (ptr != NULL)
 835                 nThChar = ptr - Buf->buf;
 836         if ((nThChar < 0) || (nThChar > Buf->BufUsed))
 837                 return -1;
 838         if (nThChar + nChars > Buf->BufUsed)
 839                 nChars =  Buf->BufUsed - nThChar;
 840
 841         memset(Buf->buf + nThChar, PookValue, nChars);
 842         /* just to be shure... */
 843         Buf->buf[Buf->BufUsed] = 0;
 844         return nChars;
 845 }
 846
 847 /**
 848  * @ingroup StrBuf_Filler
 849  * @brief Append a StringBuffer to the buffer
 850  * @param Buf Buffer to modify
 851  * @param AppendBuf Buffer to copy at the end of our buffer
 852  * @param Offset Should we start copying from an offset?
 853  */
 854 void StrBufAppendBuf(StrBuf *Buf, const StrBuf *AppendBuf, unsigned long Offset)
 855 {
 856         if ((AppendBuf == NULL) || (AppendBuf->buf == NULL) ||
 857             (Buf == NULL) || (Buf->buf == NULL))
 858                 return;
 859
 860         if (Buf->BufSize - Offset < AppendBuf->BufUsed + Buf->BufUsed + 1)
 861                 IncreaseBuf(Buf,
 862                             (Buf->BufUsed > 0),
 863                             AppendBuf->BufUsed + Buf->BufUsed);
 864
 865         memcpy(Buf->buf + Buf->BufUsed,
 866                AppendBuf->buf + Offset,
 867                AppendBuf->BufUsed - Offset);
 868         Buf->BufUsed += AppendBuf->BufUsed - Offset;
 869         Buf->buf[Buf->BufUsed] = '\0';
 870 }
 871
 872
 873 /**
 874  * @ingroup StrBuf_Filler
 875  * @brief Append a C-String to the buffer
 876  * @param Buf Buffer to modify
 877  * @param AppendBuf Buffer to copy at the end of our buffer
 878  * @param AppendSize number of bytes to copy; set to -1 if we should count it in advance
 879  * @param Offset Should we start copying from an offset?
 880  */
 881 void StrBufAppendBufPlain(StrBuf *Buf, const char *AppendBuf, long AppendSize, unsigned long Offset)
 882 {
 883         long aps;
 884         long BufSizeRequired;
 885
 886         if ((AppendBuf == NULL) || (Buf == NULL))
 887                 return;
 888
 889         if (AppendSize < 0 )
 890                 aps = strlen(AppendBuf + Offset);
 891         else
 892                 aps = AppendSize - Offset;
 893
 894         BufSizeRequired = Buf->BufUsed + aps + 1;
 895         if (Buf->BufSize <= BufSizeRequired)
 896                 IncreaseBuf(Buf, (Buf->BufUsed > 0), BufSizeRequired);
 897
 898         memcpy(Buf->buf + Buf->BufUsed,
 899                AppendBuf + Offset,
 900                aps);
 901         Buf->BufUsed += aps;
 902         Buf->buf[Buf->BufUsed] = '\0';
 903 }
 904
 905 /**
 906  * @ingroup StrBuf_Filler
 907  * @brief sprintf like function appending the formated string to the buffer
 908  * vsnprintf version to wrap into own calls
 909  * @param Buf Buffer to extend by format and Params
 910  * @param format printf alike format to add
 911  * @param ap va_list containing the items for format
 912  */
 913 void StrBufVAppendPrintf(StrBuf *Buf, const char *format, va_list ap)
 914 {
 915         va_list apl;
 916         size_t BufSize;
 917         size_t nWritten;
 918         size_t Offset;
 919         size_t newused;
 920
 921         if ((Buf == NULL)  || (format == NULL))
 922                 return;
 923
 924         BufSize = Buf->BufSize;
 925         nWritten = Buf->BufSize + 1;
 926         Offset = Buf->BufUsed;
 927         newused = Offset + nWritten;
 928
 929         while (newused >= BufSize) {
 930                 va_copy(apl, ap);
 931                 nWritten = vsnprintf(Buf->buf + Offset,
 932                                      Buf->BufSize - Offset,
 933                                      format, apl);
 934                 va_end(apl);
 935                 newused = Offset + nWritten;
 936                 if (newused >= Buf->BufSize) {
 937                         if (IncreaseBuf(Buf, 1, newused) == -1)
 938                                 return; /* TODO: error handling? */
 939                         newused = Buf->BufSize + 1;
 940                 }
 941                 else {
 942                         Buf->BufUsed = Offset + nWritten;
 943                         BufSize = Buf->BufSize;
 944                 }
 945
 946         }
 947 }
 948
 949 /**
 950  * @ingroup StrBuf_Filler
 951  * @brief sprintf like function appending the formated string to the buffer
 952  * @param Buf Buffer to extend by format and Params
 953  * @param format printf alike format to add
 954  */
 955 void StrBufAppendPrintf(StrBuf *Buf, const char *format, ...)
 956 {
 957         size_t BufSize;
 958         size_t nWritten;
 959         size_t Offset;
 960         size_t newused;
 961         va_list arg_ptr;
 962
 963         if ((Buf == NULL)  || (format == NULL))
 964                 return;
 965
 966         BufSize = Buf->BufSize;
 967         nWritten = Buf->BufSize + 1;
 968         Offset = Buf->BufUsed;
 969         newused = Offset + nWritten;
 970
 971         while (newused >= BufSize) {
 972                 va_start(arg_ptr, format);
 973                 nWritten = vsnprintf(Buf->buf + Buf->BufUsed,
 974                                      Buf->BufSize - Buf->BufUsed,
 975                                      format, arg_ptr);
 976                 va_end(arg_ptr);
 977                 newused = Buf->BufUsed + nWritten;
 978                 if (newused >= Buf->BufSize) {
 979                         if (IncreaseBuf(Buf, 1, newused) == -1)
 980                                 return; /* TODO: error handling? */
 981                         newused = Buf->BufSize + 1;
 982                 }
 983                 else {
 984                         Buf->BufUsed += nWritten;
 985                         BufSize = Buf->BufSize;
 986                 }
 987
 988         }
 989 }
 990
 991 /**
 992  * @ingroup StrBuf_Filler
 993  * @brief sprintf like function putting the formated string into the buffer
 994  * @param Buf Buffer to extend by format and Parameters
 995  * @param format printf alike format to add
 996  */
 997 void StrBufPrintf(StrBuf *Buf, const char *format, ...)
 998 {
 999         size_t nWritten;
1000         va_list arg_ptr;
1001
1002         if ((Buf == NULL)  || (format == NULL))
1003                 return;
1004
1005         nWritten = Buf->BufSize + 1;
1006         while (nWritten >= Buf->BufSize) {
1007                 va_start(arg_ptr, format);
1008                 nWritten = vsnprintf(Buf->buf, Buf->BufSize, format, arg_ptr);
1009                 va_end(arg_ptr);
1010                 if (nWritten >= Buf->BufSize) {
1011                         if (IncreaseBuf(Buf, 0, 0) == -1)
1012                                 return; /* TODO: error handling? */
1013                         nWritten = Buf->BufSize + 1;
1014                         continue;
1015                 }
1016                 Buf->BufUsed = nWritten ;
1017         }
1018 }
1019
1020 /**
1021  * @ingroup StrBuf_Filler
1022  * @brief Callback for cURL to append the webserver reply to a buffer
1023  * @param ptr pre-defined by the cURL API; see man 3 curl for mre info
1024  * @param size pre-defined by the cURL API; see man 3 curl for mre info
1025  * @param nmemb pre-defined by the cURL API; see man 3 curl for mre info
1026  * @param stream pre-defined by the cURL API; see man 3 curl for mre info
1027  */
1028 size_t CurlFillStrBuf_callback(void *ptr, size_t size, size_t nmemb, void *stream)
1029 {
1030
1031         StrBuf *Target;
1032
1033         Target = stream;
1034         if (ptr == NULL)
1035                 return 0;
1036
1037         StrBufAppendBufPlain(Target, ptr, size * nmemb, 0);
1038         return size * nmemb;
1039 }
1040
1041
1042 /**
1043  * @ingroup StrBuf
1044  * @brief extracts a substring from Source into dest
1045  * @param dest buffer to place substring into
1046  * @param Source string to copy substring from
1047  * @param Offset chars to skip from start
1048  * @param nChars number of chars to copy
1049  * @returns the number of chars copied; may be different from nChars due to the size of Source
1050  */
1051 int StrBufSub(StrBuf *dest, const StrBuf *Source, unsigned long Offset, size_t nChars)
1052 {
1053         size_t NCharsRemain;
1054         if (Offset > Source->BufUsed)
1055         {
1056                 if (dest != NULL)
1057                         FlushStrBuf(dest);
1058                 return 0;
1059         }
1060         if (Offset + nChars < Source->BufUsed)
1061         {
1062                 if ((nChars >= dest->BufSize) &&
1063                     (IncreaseBuf(dest, 0, nChars + 1) == -1))
1064                         return 0;
1065                 memcpy(dest->buf, Source->buf + Offset, nChars);
1066                 dest->BufUsed = nChars;
1067                 dest->buf[dest->BufUsed] = '\0';
1068                 return nChars;
1069         }
1070         NCharsRemain = Source->BufUsed - Offset;
1071         if ((NCharsRemain  >= dest->BufSize) &&
1072             (IncreaseBuf(dest, 0, NCharsRemain + 1) == -1))
1073                 return 0;
1074         memcpy(dest->buf, Source->buf + Offset, NCharsRemain);
1075         dest->BufUsed = NCharsRemain;
1076         dest->buf[dest->BufUsed] = '\0';
1077         return NCharsRemain;
1078 }
1079
1080 /**
1081  * @ingroup StrBuf
1082  * @brief Cut nChars from the start of the string
1083  * @param Buf Buffer to modify
1084  * @param nChars how many chars should be skipped?
1085  */
1086 void StrBufCutLeft(StrBuf *Buf, int nChars)
1087 {
1088         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1089         if (nChars >= Buf->BufUsed) {
1090                 FlushStrBuf(Buf);
1091                 return;
1092         }
1093         memmove(Buf->buf, Buf->buf + nChars, Buf->BufUsed - nChars);
1094         Buf->BufUsed -= nChars;
1095         Buf->buf[Buf->BufUsed] = '\0';
1096 }
1097
1098 /**
1099  * @ingroup StrBuf
1100  * @brief Cut the trailing n Chars from the string
1101  * @param Buf Buffer to modify
1102  * @param nChars how many chars should be trunkated?
1103  */
1104 void StrBufCutRight(StrBuf *Buf, int nChars)
1105 {
1106         if ((Buf == NULL) || (Buf->BufUsed == 0) || (Buf->buf == NULL))
1107                 return;
1108
1109         if (nChars >= Buf->BufUsed) {
1110                 FlushStrBuf(Buf);
1111                 return;
1112         }
1113         Buf->BufUsed -= nChars;
1114         Buf->buf[Buf->BufUsed] = '\0';
1115 }
1116
1117 /**
1118  * @ingroup StrBuf
1119  * @brief Cut the string after n Chars
1120  * @param Buf Buffer to modify
1121  * @param AfternChars after how many chars should we trunkate the string?
1122  * @param At if non-null and points inside of our string, cut it there.
1123  */
1124 void StrBufCutAt(StrBuf *Buf, int AfternChars, const char *At)
1125 {
1126         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1127         if (At != NULL){
1128                 AfternChars = At - Buf->buf;
1129         }
1130
1131         if ((AfternChars < 0) || (AfternChars >= Buf->BufUsed))
1132                 return;
1133         Buf->BufUsed = AfternChars;
1134         Buf->buf[Buf->BufUsed] = '\0';
1135 }
1136
1137
1138 /**
1139  * @ingroup StrBuf
1140  * @brief Strip leading and trailing spaces from a string; with premeasured and adjusted length.
1141  * @param Buf the string to modify
1142  */
1143 void StrBufTrim(StrBuf *Buf)
1144 {
1145         int delta = 0;
1146         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1147
1148         while ((Buf->BufUsed > 0) &&
1149                isspace(Buf->buf[Buf->BufUsed - 1]))
1150         {
1151                 Buf->BufUsed --;
1152         }
1153         Buf->buf[Buf->BufUsed] = '\0';
1154
1155         if (Buf->BufUsed == 0) return;
1156
1157         while ((Buf->BufUsed > delta) && (isspace(Buf->buf[delta]))){
1158                 delta ++;
1159         }
1160         if (delta > 0) StrBufCutLeft(Buf, delta);
1161 }
1162 /**
1163  * @ingroup StrBuf
1164  * @brief changes all spaces in the string  (tab, linefeed...) to Blank (0x20)
1165  * @param Buf the string to modify
1166  */
1167 void StrBufSpaceToBlank(StrBuf *Buf)
1168 {
1169         char *pche, *pch;
1170
1171         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1172
1173         pch = Buf->buf;
1174         pche = pch + Buf->BufUsed;
1175         while (pch < pche)
1176         {
1177                 if (isspace(*pch))
1178                         *pch = ' ';
1179                 pch ++;
1180         }
1181 }
1182
1183 void StrBufStripAllBut(StrBuf *Buf, char leftboundary, char rightboundary)
1184 {
1185         const char *pLeft;
1186         const char *pRight;
1187
1188         if ((Buf == NULL) || (Buf->buf == NULL)) {
1189                 return;
1190         }
1191
1192         pRight = strchr(Buf->buf, rightboundary);
1193         if (pRight != NULL) {
1194                 StrBufCutAt(Buf, 0, pRight);
1195         }
1196
1197         pLeft = strrchr(ChrPtr(Buf), leftboundary);
1198         if (pLeft != NULL) {
1199                 StrBufCutLeft(Buf, pLeft - Buf->buf + 1);
1200         }
1201 }
1202
1203
1204 /**
1205  * @ingroup StrBuf_Filler
1206  * @brief uppercase the contents of a buffer
1207  * @param Buf the buffer to translate
1208  */
1209 void StrBufUpCase(StrBuf *Buf)
1210 {
1211         char *pch, *pche;
1212
1213         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1214
1215         pch = Buf->buf;
1216         pche = pch + Buf->BufUsed;
1217         while (pch < pche) {
1218                 *pch = toupper(*pch);
1219                 pch ++;
1220         }
1221 }
1222
1223
1224 /**
1225  * @ingroup StrBuf_Filler
1226  * @brief lowercase the contents of a buffer
1227  * @param Buf the buffer to translate
1228  */
1229 void StrBufLowerCase(StrBuf *Buf)
1230 {
1231         char *pch, *pche;
1232
1233         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1234
1235         pch = Buf->buf;
1236         pche = pch + Buf->BufUsed;
1237         while (pch < pche) {
1238                 *pch = tolower(*pch);
1239                 pch ++;
1240         }
1241 }
1242
1243
1244 /*******************************************************************************
1245  *           a tokenizer that kills, maims, and destroys                       *
1246  *******************************************************************************/
1247
1248 /**
1249  * @ingroup StrBuf_Tokenizer
1250  * @brief Replace a token at a given place with a given length by another token with given length
1251  * @param Buf String where to work on
1252  * @param where where inside of the Buf is the search-token
1253  * @param HowLong How long is the token to be replaced
1254  * @param Repl Token to insert at 'where'
1255  * @param ReplLen Length of repl
1256  * @returns -1 if fail else length of resulting Buf
1257  */
1258 int StrBufReplaceToken(StrBuf *Buf, long where, long HowLong,
1259                        const char *Repl, long ReplLen)
1260 {
1261
1262         if ((Buf == NULL) ||
1263             (where > Buf->BufUsed) ||
1264             (where + HowLong > Buf->BufUsed))
1265                 return -1;
1266
1267         if (where + ReplLen - HowLong > Buf->BufSize)
1268                 if (IncreaseBuf(Buf, 1, Buf->BufUsed + ReplLen) < 0)
1269                         return -1;
1270
1271         memmove(Buf->buf + where + ReplLen,
1272                 Buf->buf + where + HowLong,
1273                 Buf->BufUsed - where - HowLong);
1274
1275         memcpy(Buf->buf + where,
1276                Repl, ReplLen);
1277
1278         Buf->BufUsed += ReplLen - HowLong;
1279
1280         return Buf->BufUsed;
1281 }
1282
1283 /**
1284  * @ingroup StrBuf_Tokenizer
1285  * @brief Counts the numbmer of tokens in a buffer
1286  * @param source String to count tokens in
1287  * @param tok    Tokenizer char to count
1288  * @returns numbers of tokenizer chars found
1289  */
1290 int StrBufNum_tokens(const StrBuf *source, char tok)
1291 {
1292         char *pch, *pche;
1293         long NTokens;
1294         if ((source == NULL) || (source->BufUsed == 0))
1295                 return 0;
1296         if ((source->BufUsed == 1) && (*source->buf == tok))
1297                 return 2;
1298         NTokens = 1;
1299         pch = source->buf;
1300         pche = pch + source->BufUsed;
1301         while (pch < pche)
1302         {
1303                 if (*pch == tok)
1304                         NTokens ++;
1305                 pch ++;
1306         }
1307         return NTokens;
1308 }
1309
1310 /**
1311  * @ingroup StrBuf_Tokenizer
1312  * @brief a string tokenizer
1313  * @param Source StringBuffer to read into
1314  * @param parmnum n'th Parameter to remove
1315  * @param separator tokenizer character
1316  * @returns -1 if not found, else length of token.
1317  */
1318 int StrBufRemove_token(StrBuf *Source, int parmnum, char separator)
1319 {
1320         int ReducedBy;
1321         char *d, *s, *end;              /* dest, source */
1322         int count = 0;
1323
1324         /* Find desired @parameter */
1325         end = Source->buf + Source->BufUsed;
1326         d = Source->buf;
1327         while ((d <= end) &&
1328                (count < parmnum))
1329         {
1330                 /* End of string, bail! */
1331                 if (!*d) {
1332                         d = NULL;
1333                         break;
1334                 }
1335                 if (*d == separator) {
1336                         count++;
1337                 }
1338                 d++;
1339         }
1340         if ((d == NULL) || (d >= end))
1341                 return 0;               /* @Parameter not found */
1342
1343         /* Find next @parameter */
1344         s = d;
1345         while ((s <= end) &&
1346                (*s && *s != separator))
1347         {
1348                 s++;
1349         }
1350         if (*s == separator)
1351                 s++;
1352         ReducedBy = d - s;
1353
1354         /* Hack and slash */
1355         if (s >= end) {
1356                 return 0;
1357         }
1358         else if (*s) {
1359                 memmove(d, s, Source->BufUsed - (s - Source->buf));
1360                 Source->BufUsed += ReducedBy;
1361                 Source->buf[Source->BufUsed] = '\0';
1362         }
1363         else if (d == Source->buf) {
1364                 *d = 0;
1365                 Source->BufUsed = 0;
1366         }
1367         else {
1368                 *--d = '\0';
1369                 Source->BufUsed += ReducedBy;
1370         }
1371         /*
1372         while (*s) {
1373                 *d++ = *s++;
1374         }
1375         *d = 0;
1376         */
1377         return ReducedBy;
1378 }
1379
1380 int StrBufExtract_tokenFromStr(StrBuf *dest, const char *Source, long SourceLen, int parmnum, char separator)
1381 {
1382         const StrBuf Temp = {
1383                 (char*)Source,
1384                 SourceLen,
1385                 SourceLen,
1386                 1
1387 #ifdef SIZE_DEBUG
1388                 ,
1389                 0,
1390                 "",
1391                 ""
1392 #endif
1393         };
1394
1395         return StrBufExtract_token(dest, &Temp, parmnum, separator);
1396 }
1397
1398 /**
1399  * @ingroup StrBuf_Tokenizer
1400  * @brief a string tokenizer
1401  * @param dest Destination StringBuffer
1402  * @param Source StringBuffer to read into
1403  * @param parmnum n'th Parameter to extract
1404  * @param separator tokenizer character
1405  * @returns -1 if not found, else length of token.
1406  */
1407 int StrBufExtract_token(StrBuf *dest, const StrBuf *Source, int parmnum, char separator)
1408 {
1409         const char *s, *e;              //* source * /
1410         int len = 0;                    //* running total length of extracted string * /
1411         int current_token = 0;          //* token currently being processed * /
1412
1413         if (dest != NULL) {
1414                 dest->buf[0] = '\0';
1415                 dest->BufUsed = 0;
1416         }
1417         else
1418                 return(-1);
1419
1420         if ((Source == NULL) || (Source->BufUsed ==0)) {
1421                 return(-1);
1422         }
1423         s = Source->buf;
1424         e = s + Source->BufUsed;
1425
1426         //cit_backtrace();
1427         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1428
1429         while ((s < e) && !IsEmptyStr(s)) {
1430                 if (*s == separator) {
1431                         ++current_token;
1432                 }
1433                 if (len >= dest->BufSize) {
1434                         dest->BufUsed = len;
1435                         if (IncreaseBuf(dest, 1, -1) < 0) {
1436                                 dest->BufUsed --;
1437                                 break;
1438                         }
1439                 }
1440                 if ( (current_token == parmnum) &&
1441                      (*s != separator)) {
1442                         dest->buf[len] = *s;
1443                         ++len;
1444                 }
1445                 else if (current_token > parmnum) {
1446                         break;
1447                 }
1448                 ++s;
1449         }
1450
1451         dest->buf[len] = '\0';
1452         dest->BufUsed = len;
1453
1454         if (current_token < parmnum) {
1455                 //lprintf (CTDL_DEBUG,"test <!: %s\n", dest);
1456                 return(-1);
1457         }
1458         //lprintf (CTDL_DEBUG,"test <: %d; %s\n", len, dest);
1459         return(len);
1460 }
1461
1462
1463
1464
1465
1466 /**
1467  * @ingroup StrBuf_Tokenizer
1468  * @brief a string tokenizer to fetch an integer
1469  * @param Source String containing tokens
1470  * @param parmnum n'th Parameter to extract
1471  * @param separator tokenizer character
1472  * @returns 0 if not found, else integer representation of the token
1473  */
1474 int StrBufExtract_int(const StrBuf* Source, int parmnum, char separator)
1475 {
1476         StrBuf tmp;
1477         char buf[64];
1478
1479         tmp.buf = buf;
1480         buf[0] = '\0';
1481         tmp.BufSize = 64;
1482         tmp.BufUsed = 0;
1483         tmp.ConstBuf = 1;
1484         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1485                 return(atoi(buf));
1486         else
1487                 return 0;
1488 }
1489
1490 /**
1491  * @ingroup StrBuf_Tokenizer
1492  * @brief a string tokenizer to fetch a long integer
1493  * @param Source String containing tokens
1494  * @param parmnum n'th Parameter to extract
1495  * @param separator tokenizer character
1496  * @returns 0 if not found, else long integer representation of the token
1497  */
1498 long StrBufExtract_long(const StrBuf* Source, int parmnum, char separator)
1499 {
1500         StrBuf tmp;
1501         char buf[64];
1502
1503         tmp.buf = buf;
1504         buf[0] = '\0';
1505         tmp.BufSize = 64;
1506         tmp.BufUsed = 0;
1507         tmp.ConstBuf = 1;
1508         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1509                 return(atoi(buf));
1510         else
1511                 return 0;
1512 }
1513
1514
1515 /**
1516  * @ingroup StrBuf_Tokenizer
1517  * @brief a string tokenizer to fetch an unsigned long
1518  * @param Source String containing tokens
1519  * @param parmnum n'th Parameter to extract
1520  * @param separator tokenizer character
1521  * @returns 0 if not found, else unsigned long representation of the token
1522  */
1523 unsigned long StrBufExtract_unsigned_long(const StrBuf* Source, int parmnum, char separator)
1524 {
1525         StrBuf tmp;
1526         char buf[64];
1527         char *pnum;
1528
1529         tmp.buf = buf;
1530         buf[0] = '\0';
1531         tmp.BufSize = 64;
1532         tmp.BufUsed = 0;
1533         tmp.ConstBuf = 1;
1534         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0) {
1535                 pnum = &buf[0];
1536                 if (*pnum == '-')
1537                         pnum ++;
1538                 return (unsigned long) atol(pnum);
1539         }
1540         else
1541                 return 0;
1542 }
1543
1544
1545
1546 /**
1547  * @ingroup StrBuf_NextTokenizer
1548  * @brief a string tokenizer; Bounds checker
1549  *  function to make shure whether StrBufExtract_NextToken and friends have reached the end of the string.
1550  * @param Source our tokenbuffer
1551  * @param pStart the token iterator pointer to inspect
1552  * @returns whether the revolving pointer is inside of the search range
1553  */
1554 int StrBufHaveNextToken(const StrBuf *Source, const char **pStart)
1555 {
1556         if ((Source == NULL) ||
1557             (*pStart == StrBufNOTNULL) ||
1558             (Source->BufUsed == 0))
1559         {
1560                 return 0;
1561         }
1562         if (*pStart == NULL)
1563         {
1564                 return 1;
1565         }
1566         else if (*pStart > Source->buf + Source->BufUsed)
1567         {
1568                 return 0;
1569         }
1570         else if (*pStart <= Source->buf)
1571         {
1572                 return 0;
1573         }
1574
1575         return 1;
1576 }
1577
1578 /**
1579  * @ingroup StrBuf_NextTokenizer
1580  * @brief a string tokenizer
1581  * @param dest Destination StringBuffer
1582  * @param Source StringBuffer to read into
1583  * @param pStart pointer to the end of the last token. Feed with NULL on start.
1584  * @param separator tokenizer
1585  * @returns -1 if not found, else length of token.
1586  */
1587 int StrBufExtract_NextToken(StrBuf *dest, const StrBuf *Source, const char **pStart, char separator)
1588 {
1589         const char *s;          /* source */
1590         const char *EndBuffer;  /* end stop of source buffer */
1591         int current_token = 0;  /* token currently being processed */
1592         int len = 0;            /* running total length of extracted string */
1593
1594         if ((Source          == NULL) ||
1595             (Source->BufUsed == 0)      )
1596         {
1597                 *pStart = StrBufNOTNULL;
1598                 if (dest != NULL)
1599                         FlushStrBuf(dest);
1600                 return -1;
1601         }
1602
1603         EndBuffer = Source->buf + Source->BufUsed;
1604
1605         if (dest != NULL)
1606         {
1607                 dest->buf[0] = '\0';
1608                 dest->BufUsed = 0;
1609         }
1610         else
1611         {
1612                 *pStart = EndBuffer + 1;
1613                 return -1;
1614         }
1615
1616         if (*pStart == NULL)
1617         {
1618                 *pStart = Source->buf; /* we're starting to examine this buffer. */
1619         }
1620         else if ((*pStart < Source->buf) ||
1621                  (*pStart > EndBuffer  )   )
1622         {
1623                 return -1; /* no more tokens to find. */
1624         }
1625
1626         s = *pStart;
1627         /* start to find the next token */
1628         while ((s <= EndBuffer)      &&
1629                (current_token == 0) )
1630         {
1631                 if (*s == separator)
1632                 {
1633                         /* we found the next token */
1634                         ++current_token;
1635                 }
1636
1637                 if (len >= dest->BufSize)
1638                 {
1639                         /* our Dest-buffer isn't big enough, increase it. */
1640                         dest->BufUsed = len;
1641
1642                         if (IncreaseBuf(dest, 1, -1) < 0) {
1643                                 /* WHUT? no more mem? bail out. */
1644                                 s = EndBuffer;
1645                                 dest->BufUsed --;
1646                                 break;
1647                         }
1648                 }
1649
1650                 if ( (current_token == 0 ) &&   /* are we in our target token? */
1651                      (!IsEmptyStr(s)     ) &&
1652                      (separator     != *s)    ) /* don't copy the token itself */
1653                 {
1654                         dest->buf[len] = *s;    /* Copy the payload */
1655                         ++len;                  /* remember the bigger size. */
1656                 }
1657
1658                 ++s;
1659         }
1660
1661         /* did we reach the end? */
1662         if ((s > EndBuffer)) {
1663                 EndBuffer = StrBufNOTNULL;
1664                 *pStart = EndBuffer;
1665         }
1666         else {
1667                 *pStart = s;  /* remember the position for the next run */
1668         }
1669
1670         /* sanitize our extracted token */
1671         dest->buf[len] = '\0';
1672         dest->BufUsed  = len;
1673
1674         return (len);
1675 }
1676
1677
1678 /**
1679  * @ingroup StrBuf_NextTokenizer
1680  * @brief a string tokenizer
1681  * @param Source StringBuffer to read from
1682  * @param pStart pointer to the end of the last token. Feed with NULL.
1683  * @param separator tokenizer character
1684  * @param nTokens number of tokens to fastforward over
1685  * @returns -1 if not found, else length of token.
1686  */
1687 int StrBufSkip_NTokenS(const StrBuf *Source, const char **pStart, char separator, int nTokens)
1688 {
1689         const char *s, *EndBuffer;      //* source * /
1690         int len = 0;                    //* running total length of extracted string * /
1691         int current_token = 0;          //* token currently being processed * /
1692
1693         if ((Source == NULL) ||
1694             (Source->BufUsed ==0)) {
1695                 return(-1);
1696         }
1697         if (nTokens == 0)
1698                 return Source->BufUsed;
1699
1700         if (*pStart == NULL)
1701                 *pStart = Source->buf;
1702
1703         EndBuffer = Source->buf + Source->BufUsed;
1704
1705         if ((*pStart < Source->buf) ||
1706             (*pStart >  EndBuffer)) {
1707                 return (-1);
1708         }
1709
1710
1711         s = *pStart;
1712
1713         //cit_backtrace();
1714         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1715
1716         while ((s < EndBuffer) && !IsEmptyStr(s)) {
1717                 if (*s == separator) {
1718                         ++current_token;
1719                 }
1720                 if (current_token >= nTokens) {
1721                         break;
1722                 }
1723                 ++s;
1724         }
1725         *pStart = s;
1726         (*pStart) ++;
1727
1728         return(len);
1729 }
1730
1731 /**
1732  * @ingroup StrBuf_NextTokenizer
1733  * @brief a string tokenizer to fetch an integer
1734  * @param Source StringBuffer to read from
1735  * @param pStart Cursor on the tokenstring
1736  * @param separator tokenizer character
1737  * @returns 0 if not found, else integer representation of the token
1738  */
1739 int StrBufExtractNext_int(const StrBuf* Source, const char **pStart, char separator)
1740 {
1741         StrBuf tmp;
1742         char buf[64];
1743
1744         tmp.buf = buf;
1745         buf[0] = '\0';
1746         tmp.BufSize = 64;
1747         tmp.BufUsed = 0;
1748         tmp.ConstBuf = 1;
1749         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1750                 return(atoi(buf));
1751         else
1752                 return 0;
1753 }
1754
1755 /**
1756  * @ingroup StrBuf_NextTokenizer
1757  * @brief a string tokenizer to fetch a long integer
1758  * @param Source StringBuffer to read from
1759  * @param pStart Cursor on the tokenstring
1760  * @param separator tokenizer character
1761  * @returns 0 if not found, else long integer representation of the token
1762  */
1763 long StrBufExtractNext_long(const StrBuf* Source, const char **pStart, char separator)
1764 {
1765         StrBuf tmp;
1766         char buf[64];
1767
1768         tmp.buf = buf;
1769         buf[0] = '\0';
1770         tmp.BufSize = 64;
1771         tmp.BufUsed = 0;
1772         tmp.ConstBuf = 1;
1773         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1774                 return(atoi(buf));
1775         else
1776                 return 0;
1777 }
1778
1779
1780 /**
1781  * @ingroup StrBuf_NextTokenizer
1782  * @brief a string tokenizer to fetch an unsigned long
1783  * @param Source StringBuffer to read from
1784  * @param pStart Cursor on the tokenstring
1785  * @param separator tokenizer character
1786  * @returns 0 if not found, else unsigned long representation of the token
1787  */
1788 unsigned long StrBufExtractNext_unsigned_long(const StrBuf* Source, const char **pStart, char separator)
1789 {
1790         StrBuf tmp;
1791         char buf[64];
1792         char *pnum;
1793
1794         tmp.buf = buf;
1795         buf[0] = '\0';
1796         tmp.BufSize = 64;
1797         tmp.BufUsed = 0;
1798         tmp.ConstBuf = 1;
1799         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0) {
1800                 pnum = &buf[0];
1801                 if (*pnum == '-')
1802                         pnum ++;
1803                 return (unsigned long) atol(pnum);
1804         }
1805         else
1806                 return 0;
1807 }
1808
1809
1810
1811
1812
1813 /*******************************************************************************
1814  *                             Escape Appending                                *
1815  *******************************************************************************/
1816
1817 /**
1818  * @ingroup StrBuf_DeEnCoder
1819  * @brief Escape a string for feeding out as a URL while appending it to a Buffer
1820  * @param OutBuf the output buffer
1821  * @param In Buffer to encode
1822  * @param PlainIn way in from plain old c strings
1823  */
1824 void StrBufUrlescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
1825 {
1826         const char *pch, *pche;
1827         char *pt, *pte;
1828         int len;
1829
1830         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1831                 return;
1832         if (PlainIn != NULL) {
1833                 len = strlen(PlainIn);
1834                 pch = PlainIn;
1835                 pche = pch + len;
1836         }
1837         else {
1838                 pch = In->buf;
1839                 pche = pch + In->BufUsed;
1840                 len = In->BufUsed;
1841         }
1842
1843         if (len == 0)
1844                 return;
1845
1846         pt = OutBuf->buf + OutBuf->BufUsed;
1847         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1848
1849         while (pch < pche) {
1850                 if (pt >= pte) {
1851                         IncreaseBuf(OutBuf, 1, -1);
1852                         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1853                         pt = OutBuf->buf + OutBuf->BufUsed;
1854                 }
1855
1856                 if((*pch >= 'a' && *pch <= 'z') ||
1857                    (*pch >= '@' && *pch <= 'Z') || /* @ A-Z */
1858                    (*pch >= '0' && *pch <= ':') || /* 0-9 : */
1859                    (*pch == '!') || (*pch == '_') ||
1860                    (*pch == ',') || (*pch == '.'))
1861                 {
1862                         *(pt++) = *(pch++);
1863                         OutBuf->BufUsed++;
1864                 }
1865                 else {
1866                         *pt = '%';
1867                         *(pt + 1) = HexList[(unsigned char)*pch][0];
1868                         *(pt + 2) = HexList[(unsigned char)*pch][1];
1869                         pt += 3;
1870                         OutBuf->BufUsed += 3;
1871                         pch ++;
1872                 }
1873         }
1874         *pt = '\0';
1875 }
1876
1877 /**
1878  * @ingroup StrBuf_DeEnCoder
1879  * @brief Escape a string for feeding out as a the username/password part of an URL while appending it to a Buffer
1880  * @param OutBuf the output buffer
1881  * @param In Buffer to encode
1882  * @param PlainIn way in from plain old c strings
1883  */
1884 void StrBufUrlescUPAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
1885 {
1886         const char *pch, *pche;
1887         char *pt, *pte;
1888         int len;
1889
1890         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1891                 return;
1892         if (PlainIn != NULL) {
1893                 len = strlen(PlainIn);
1894                 pch = PlainIn;
1895                 pche = pch + len;
1896         }
1897         else {
1898                 pch = In->buf;
1899                 pche = pch + In->BufUsed;
1900                 len = In->BufUsed;
1901         }
1902
1903         if (len == 0)
1904                 return;
1905
1906         pt = OutBuf->buf + OutBuf->BufUsed;
1907         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1908
1909         while (pch < pche) {
1910                 if (pt >= pte) {
1911                         IncreaseBuf(OutBuf, 1, -1);
1912                         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1913                         pt = OutBuf->buf + OutBuf->BufUsed;
1914                 }
1915
1916                 if((*pch >= 'a' && *pch <= 'z') ||
1917                    (*pch >= 'A' && *pch <= 'Z') || /* A-Z */
1918                    (*pch >= '0' && *pch <= ':') || /* 0-9 : */
1919                    (*pch == '!') || (*pch == '_') ||
1920                    (*pch == ',') || (*pch == '.'))
1921                 {
1922                         *(pt++) = *(pch++);
1923                         OutBuf->BufUsed++;
1924                 }
1925                 else {
1926                         *pt = '%';
1927                         *(pt + 1) = HexList[(unsigned char)*pch][0];
1928                         *(pt + 2) = HexList[(unsigned char)*pch][1];
1929                         pt += 3;
1930                         OutBuf->BufUsed += 3;
1931                         pch ++;
1932                 }
1933         }
1934         *pt = '\0';
1935 }
1936
1937 /**
1938  * @ingroup StrBuf_DeEnCoder
1939  * @brief append a string with characters having a special meaning in xml encoded to the buffer
1940  * @param OutBuf the output buffer
1941  * @param In Buffer to encode
1942  * @param PlainIn way in from plain old c strings
1943  * @param PlainInLen way in from plain old c strings; maybe you've got binary data or know the length?
1944  * @param OverrideLowChars should chars < 0x20 be replaced by _ or escaped as xml entity?
1945  */
1946 void StrBufXMLEscAppend(StrBuf *OutBuf,
1947                         const StrBuf *In,
1948                         const char *PlainIn,
1949                         long PlainInLen,
1950                         int OverrideLowChars)
1951 {
1952         const char *pch, *pche;
1953         char *pt, *pte;
1954         int IsUtf8Sequence;
1955         int len;
1956
1957         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1958                 return;
1959         if (PlainIn != NULL) {
1960                 if (PlainInLen < 0)
1961                         len = strlen((const char*)PlainIn);
1962                 else
1963                         len = PlainInLen;
1964                 pch = PlainIn;
1965                 pche = pch + len;
1966         }
1967         else {
1968                 pch = (const char*)In->buf;
1969                 pche = pch + In->BufUsed;
1970                 len = In->BufUsed;
1971         }
1972
1973         if (len == 0)
1974                 return;
1975
1976         pt = OutBuf->buf + OutBuf->BufUsed;
1977         /**< we max append 6 chars at once plus the \0 */
1978         pte = OutBuf->buf + OutBuf->BufSize - 6;
1979
1980         while (pch < pche) {
1981                 if (pt >= pte) {
1982                         OutBuf->BufUsed = pt - OutBuf->buf;
1983                         IncreaseBuf(OutBuf, 1, -1);
1984                         pte = OutBuf->buf + OutBuf->BufSize - 6;
1985                         /**< we max append 3 chars at once plus the \0 */
1986
1987                         pt = OutBuf->buf + OutBuf->BufUsed;
1988                 }
1989
1990                 if (*pch == '<') {
1991                         memcpy(pt, HKEY("&lt;"));
1992                         pt += 4;
1993                         pch ++;
1994                 }
1995                 else if (*pch == '>') {
1996                         memcpy(pt, HKEY("&gt;"));
1997                         pt += 4;
1998                         pch ++;
1999                 }
2000                 else if (*pch == '&') {
2001                         memcpy(pt, HKEY("&amp;"));
2002                         pt += 5;
2003                         pch++;
2004                 }
2005                 else if ((*pch >= 0x20) && (*pch <= 0x7F)) {
2006                         *pt = *pch;
2007                         pt++; pch++;
2008                 }
2009                 else {
2010                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(pch, pche);
2011                         if (IsUtf8Sequence)
2012                         {
2013                                 while ((IsUtf8Sequence > 0) &&
2014                                        (pch < pche))
2015                                 {
2016                                         *pt = *pch;
2017                                         pt ++;
2018                                         pch ++;
2019                                         --IsUtf8Sequence;
2020                                 }
2021                         }
2022                         else
2023                         {
2024                                 *pt = '&';
2025                                 pt++;
2026                                 *pt = HexList[*(unsigned char*)pch][0];
2027                                 pt ++;
2028                                 *pt = HexList[*(unsigned char*)pch][1];
2029                                 pt ++; pch ++;
2030                                 *pt = '&';
2031                                 pt++;
2032                                 pch ++;
2033                         }
2034                 }
2035         }
2036         *pt = '\0';
2037         OutBuf->BufUsed = pt - OutBuf->buf;
2038 }
2039
2040
2041 /**
2042  * @ingroup StrBuf_DeEnCoder
2043  * @brief append a string in hex encoding to the buffer
2044  * @param OutBuf the output buffer
2045  * @param In Buffer to encode
2046  * @param PlainIn way in from plain old c strings
2047  * @param PlainInLen way in from plain old c strings; maybe you've got binary data or know the length?
2048  */
2049 void StrBufHexEscAppend(StrBuf *OutBuf, const StrBuf *In, const unsigned char *PlainIn, long PlainInLen)
2050 {
2051         const unsigned char *pch, *pche;
2052         char *pt, *pte;
2053         int len;
2054
2055         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
2056                 return;
2057         if (PlainIn != NULL) {
2058                 if (PlainInLen < 0)
2059                         len = strlen((const char*)PlainIn);
2060                 else
2061                         len = PlainInLen;
2062                 pch = PlainIn;
2063                 pche = pch + len;
2064         }
2065         else {
2066                 pch = (const unsigned char*)In->buf;
2067                 pche = pch + In->BufUsed;
2068                 len = In->BufUsed;
2069         }
2070
2071         if (len == 0)
2072                 return;
2073
2074         pt = OutBuf->buf + OutBuf->BufUsed;
2075         pte = OutBuf->buf + OutBuf->BufSize - 3; /**< we max append 3 chars at once plus the \0 */
2076
2077         while (pch < pche) {
2078                 if (pt >= pte) {
2079                         IncreaseBuf(OutBuf, 1, -1);
2080                         pte = OutBuf->buf + OutBuf->BufSize - 3; /**< we max append 3 chars at once plus the \0 */
2081                         pt = OutBuf->buf + OutBuf->BufUsed;
2082                 }
2083
2084                 *pt = HexList[*pch][0];
2085                 pt ++;
2086                 *pt = HexList[*pch][1];
2087                 pt ++; pch ++; OutBuf->BufUsed += 2;
2088         }
2089         *pt = '\0';
2090 }
2091
2092 void StrBufBase64Append(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn, long PlainInLen, int linebreaks)
2093 {
2094         const char *pch;
2095         char *pt;
2096         int len;
2097         long ExpectLen;
2098
2099         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
2100                 return;
2101         if (PlainIn != NULL) {
2102                 if (PlainInLen < 0)
2103                         len = strlen(PlainIn);
2104                 else
2105                         len = PlainInLen;
2106                 pch = PlainIn;
2107         }
2108         else {
2109                 pch = In->buf;
2110                 len = In->BufUsed;
2111         }
2112
2113         if (len == 0)
2114                 return;
2115
2116         ExpectLen = ((len * 134) / 100) + OutBuf->BufUsed;
2117
2118         if (ExpectLen > OutBuf->BufSize)
2119                 if (IncreaseBuf(OutBuf, 1, ExpectLen) < ExpectLen)
2120                         return;
2121
2122         pt = OutBuf->buf + OutBuf->BufUsed;
2123
2124         len = CtdlEncodeBase64(pt, pch, len, linebreaks);
2125
2126         pt += len;
2127         OutBuf->BufUsed += len;
2128         *pt = '\0';
2129 }
2130
2131 /**
2132  * @ingroup StrBuf_DeEnCoder
2133  * @brief append a string in hex encoding to the buffer
2134  * @param OutBuf the output buffer
2135  * @param In Buffer to encode
2136  * @param PlainIn way in from plain old c strings
2137  */
2138 void StrBufHexescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
2139 {
2140         StrBufHexEscAppend(OutBuf, In, (const unsigned char*) PlainIn, -1);
2141 }
2142
2143 /**
2144  * @ingroup StrBuf_DeEnCoder
2145  * @brief Append a string, escaping characters which have meaning in HTML.
2146  *
2147  * @param Target        target buffer
2148  * @param Source        source buffer; set to NULL if you just have a C-String
2149  * @param PlainIn       Plain-C string to append; set to NULL if unused
2150  * @param nbsp          If nonzero, spaces are converted to non-breaking spaces.
2151  * @param nolinebreaks  if set to 1, linebreaks are removed from the string.
2152  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
2153  */
2154 long StrEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
2155 {
2156         const char *aptr, *eiptr;
2157         char *bptr, *eptr;
2158         long len;
2159
2160         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2161                 return -1;
2162
2163         if (PlainIn != NULL) {
2164                 aptr = PlainIn;
2165                 len = strlen(PlainIn);
2166                 eiptr = aptr + len;
2167         }
2168         else {
2169                 aptr = Source->buf;
2170                 eiptr = aptr + Source->BufUsed;
2171                 len = Source->BufUsed;
2172         }
2173
2174         if (len == 0)
2175                 return -1;
2176
2177         bptr = Target->buf + Target->BufUsed;
2178         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2179
2180         while (aptr < eiptr){
2181                 if(bptr >= eptr) {
2182                         IncreaseBuf(Target, 1, -1);
2183                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2184                         bptr = Target->buf + Target->BufUsed;
2185                 }
2186                 if (*aptr == '<') {
2187                         memcpy(bptr, "&lt;", 4);
2188                         bptr += 4;
2189                         Target->BufUsed += 4;
2190                 }
2191                 else if (*aptr == '>') {
2192                         memcpy(bptr, "&gt;", 4);
2193                         bptr += 4;
2194                         Target->BufUsed += 4;
2195                 }
2196                 else if (*aptr == '&') {
2197                         memcpy(bptr, "&amp;", 5);
2198                         bptr += 5;
2199                         Target->BufUsed += 5;
2200                 }
2201                 else if (*aptr == '"') {
2202                         memcpy(bptr, "&quot;", 6);
2203                         bptr += 6;
2204                         Target->BufUsed += 6;
2205                 }
2206                 else if (*aptr == '\'') {
2207                         memcpy(bptr, "&#39;", 5);
2208                         bptr += 5;
2209                         Target->BufUsed += 5;
2210                 }
2211                 else if (*aptr == LB) {
2212                         *bptr = '<';
2213                         bptr ++;
2214                         Target->BufUsed ++;
2215                 }
2216                 else if (*aptr == RB) {
2217                         *bptr = '>';
2218                         bptr ++;
2219                         Target->BufUsed ++;
2220                 }
2221                 else if (*aptr == QU) {
2222                         *bptr ='"';
2223                         bptr ++;
2224                         Target->BufUsed ++;
2225                 }
2226                 else if ((*aptr == 32) && (nbsp == 1)) {
2227                         memcpy(bptr, "&nbsp;", 6);
2228                         bptr += 6;
2229                         Target->BufUsed += 6;
2230                 }
2231                 else if ((*aptr == '\n') && (nolinebreaks == 1)) {
2232                         *bptr='\0';     /* nothing */
2233                 }
2234                 else if ((*aptr == '\n') && (nolinebreaks == 2)) {
2235                         memcpy(bptr, "&lt;br/&gt;", 11);
2236                         bptr += 11;
2237                         Target->BufUsed += 11;
2238                 }
2239
2240
2241                 else if ((*aptr == '\r') && (nolinebreaks != 0)) {
2242                         *bptr='\0';     /* nothing */
2243                 }
2244                 else{
2245                         *bptr = *aptr;
2246                         bptr++;
2247                         Target->BufUsed ++;
2248                 }
2249                 aptr ++;
2250         }
2251         *bptr = '\0';
2252         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
2253                 return -1;
2254         return Target->BufUsed;
2255 }
2256
2257 /**
2258  * @ingroup StrBuf_DeEnCoder
2259  * @brief Append a string, escaping characters which have meaning in HTML.
2260  * Converts linebreaks into blanks; escapes single quotes
2261  * @param Target        target buffer
2262  * @param Source        source buffer; set to NULL if you just have a C-String
2263  * @param PlainIn       Plain-C string to append; set to NULL if unused
2264  */
2265 void StrMsgEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2266 {
2267         const char *aptr, *eiptr;
2268         char *tptr, *eptr;
2269         long len;
2270
2271         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2272                 return ;
2273
2274         if (PlainIn != NULL) {
2275                 aptr = PlainIn;
2276                 len = strlen(PlainIn);
2277                 eiptr = aptr + len;
2278         }
2279         else {
2280                 aptr = Source->buf;
2281                 eiptr = aptr + Source->BufUsed;
2282                 len = Source->BufUsed;
2283         }
2284
2285         if (len == 0)
2286                 return;
2287
2288         eptr = Target->buf + Target->BufSize - 8;
2289         tptr = Target->buf + Target->BufUsed;
2290
2291         while (aptr < eiptr){
2292                 if(tptr >= eptr) {
2293                         IncreaseBuf(Target, 1, -1);
2294                         eptr = Target->buf + Target->BufSize - 8;
2295                         tptr = Target->buf + Target->BufUsed;
2296                 }
2297
2298                 if (*aptr == '\n') {
2299                         *tptr = ' ';
2300                         Target->BufUsed++;
2301                 }
2302                 else if (*aptr == '\r') {
2303                         *tptr = ' ';
2304                         Target->BufUsed++;
2305                 }
2306                 else if (*aptr == '\'') {
2307                         *(tptr++) = '&';
2308                         *(tptr++) = '#';
2309                         *(tptr++) = '3';
2310                         *(tptr++) = '9';
2311                         *tptr = ';';
2312                         Target->BufUsed += 5;
2313                 } else {
2314                         *tptr = *aptr;
2315                         Target->BufUsed++;
2316                 }
2317                 tptr++; aptr++;
2318         }
2319         *tptr = '\0';
2320 }
2321
2322
2323
2324 /**
2325  * @ingroup StrBuf_DeEnCoder
2326  * @brief Append a string, escaping characters which have meaning in ICAL.
2327  * [\n,]
2328  * @param Target        target buffer
2329  * @param Source        source buffer; set to NULL if you just have a C-String
2330  * @param PlainIn       Plain-C string to append; set to NULL if unused
2331  */
2332 void StrIcalEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2333 {
2334         const char *aptr, *eiptr;
2335         char *tptr, *eptr;
2336         long len;
2337
2338         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2339                 return ;
2340
2341         if (PlainIn != NULL) {
2342                 aptr = PlainIn;
2343                 len = strlen(PlainIn);
2344                 eiptr = aptr + len;
2345         }
2346         else {
2347                 aptr = Source->buf;
2348                 eiptr = aptr + Source->BufUsed;
2349                 len = Source->BufUsed;
2350         }
2351
2352         if (len == 0)
2353                 return;
2354
2355         eptr = Target->buf + Target->BufSize - 8;
2356         tptr = Target->buf + Target->BufUsed;
2357
2358         while (aptr < eiptr){
2359                 if(tptr + 3 >= eptr) {
2360                         IncreaseBuf(Target, 1, -1);
2361                         eptr = Target->buf + Target->BufSize - 8;
2362                         tptr = Target->buf + Target->BufUsed;
2363                 }
2364
2365                 if (*aptr == '\n') {
2366                         *tptr = '\\';
2367                         Target->BufUsed++;
2368                         tptr++;
2369                         *tptr = 'n';
2370                         Target->BufUsed++;
2371                 }
2372                 else if (*aptr == '\r') {
2373                         *tptr = '\\';
2374                         Target->BufUsed++;
2375                         tptr++;
2376                         *tptr = 'r';
2377                         Target->BufUsed++;
2378                 }
2379                 else if (*aptr == ',') {
2380                         *tptr = '\\';
2381                         Target->BufUsed++;
2382                         tptr++;
2383                         *tptr = ',';
2384                         Target->BufUsed++;
2385                 } else {
2386                         *tptr = *aptr;
2387                         Target->BufUsed++;
2388                 }
2389                 tptr++; aptr++;
2390         }
2391         *tptr = '\0';
2392 }
2393
2394 /**
2395  * @ingroup StrBuf_DeEnCoder
2396  * @brief Append a string, escaping characters which have meaning in JavaScript strings .
2397  *
2398  * @param Target        target buffer
2399  * @param Source        source buffer; set to NULL if you just have a C-String
2400  * @param PlainIn       Plain-C string to append; set to NULL if unused
2401  * @returns size of result or -1
2402  */
2403 long StrECMAEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2404 {
2405         const char *aptr, *eiptr;
2406         char *bptr, *eptr;
2407         long len;
2408         int IsUtf8Sequence;
2409
2410         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2411                 return -1;
2412
2413         if (PlainIn != NULL) {
2414                 aptr = PlainIn;
2415                 len = strlen(PlainIn);
2416                 eiptr = aptr + len;
2417         }
2418         else {
2419                 aptr = Source->buf;
2420                 eiptr = aptr + Source->BufUsed;
2421                 len = Source->BufUsed;
2422         }
2423
2424         if (len == 0)
2425                 return -1;
2426
2427         bptr = Target->buf + Target->BufUsed;
2428         eptr = Target->buf + Target->BufSize - 7; /* our biggest unit to put in...  */
2429
2430         while (aptr < eiptr){
2431                 if(bptr >= eptr) {
2432                         IncreaseBuf(Target, 1, -1);
2433                         eptr = Target->buf + Target->BufSize - 7; /* our biggest unit to put in...  */
2434                         bptr = Target->buf + Target->BufUsed;
2435                 }
2436                 switch (*aptr) {
2437                 case '\n':
2438                         memcpy(bptr, HKEY("\\n"));
2439                         bptr += 2;
2440                         Target->BufUsed += 2;
2441                         break;
2442                 case '\r':
2443                         memcpy(bptr, HKEY("\\r"));
2444                         bptr += 2;
2445                         Target->BufUsed += 2;
2446                         break;
2447                 case '"':
2448                         *bptr = '\\';
2449                         bptr ++;
2450                         *bptr = '"';
2451                         bptr ++;
2452                         Target->BufUsed += 2;
2453                         break;
2454                 case '\\':
2455                         if ((*(aptr + 1) == 'u') &&
2456                             isxdigit(*(aptr + 2)) &&
2457                             isxdigit(*(aptr + 3)) &&
2458                             isxdigit(*(aptr + 4)) &&
2459                             isxdigit(*(aptr + 5)))
2460                         { /* oh, a unicode escaper. let it pass through. */
2461                                 memcpy(bptr, aptr, 6);
2462                                 aptr += 5;
2463                                 bptr +=6;
2464                                 Target->BufUsed += 6;
2465                         }
2466                         else
2467                         {
2468                                 *bptr = '\\';
2469                                 bptr ++;
2470                                 *bptr = '\\';
2471                                 bptr ++;
2472                                 Target->BufUsed += 2;
2473                         }
2474                         break;
2475                 case '\b':
2476                         *bptr = '\\';
2477                         bptr ++;
2478                         *bptr = 'b';
2479                         bptr ++;
2480                         Target->BufUsed += 2;
2481                         break;
2482                 case '\f':
2483                         *bptr = '\\';
2484                         bptr ++;
2485                         *bptr = 'f';
2486                         bptr ++;
2487                         Target->BufUsed += 2;
2488                         break;
2489                 case '\t':
2490                         *bptr = '\\';
2491                         bptr ++;
2492                         *bptr = 't';
2493                         bptr ++;
2494                         Target->BufUsed += 2;
2495                         break;
2496                 default:
2497                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(aptr, eiptr);
2498                         while (IsUtf8Sequence > 0){
2499                                 *bptr = *aptr;
2500                                 Target->BufUsed ++;
2501                                 if (--IsUtf8Sequence)
2502                                         aptr++;
2503                                 bptr++;
2504                         }
2505                 }
2506                 aptr ++;
2507         }
2508         *bptr = '\0';
2509         if ((bptr == eptr - 1 ) && !IsEmptyStr(aptr) )
2510                 return -1;
2511         return Target->BufUsed;
2512 }
2513
2514 /**
2515  * @ingroup StrBuf_DeEnCoder
2516  * @brief Append a string, escaping characters which have meaning in HTML + json.
2517  *
2518  * @param Target        target buffer
2519  * @param Source        source buffer; set to NULL if you just have a C-String
2520  * @param PlainIn       Plain-C string to append; set to NULL if unused
2521  * @param nbsp          If nonzero, spaces are converted to non-breaking spaces.
2522  * @param nolinebreaks  if set to 1, linebreaks are removed from the string.
2523  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
2524  */
2525 long StrHtmlEcmaEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
2526 {
2527         const char *aptr, *eiptr;
2528         char *bptr, *eptr;
2529         long len;
2530         int IsUtf8Sequence = 0;
2531
2532         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2533                 return -1;
2534
2535         if (PlainIn != NULL) {
2536                 aptr = PlainIn;
2537                 len = strlen(PlainIn);
2538                 eiptr = aptr + len;
2539         }
2540         else {
2541                 aptr = Source->buf;
2542                 eiptr = aptr + Source->BufUsed;
2543                 len = Source->BufUsed;
2544         }
2545
2546         if (len == 0)
2547                 return -1;
2548
2549         bptr = Target->buf + Target->BufUsed;
2550         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2551
2552         while (aptr < eiptr){
2553                 if(bptr >= eptr) {
2554                         IncreaseBuf(Target, 1, -1);
2555                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2556                         bptr = Target->buf + Target->BufUsed;
2557                 }
2558                 switch (*aptr) {
2559                 case '<':
2560                         memcpy(bptr, HKEY("&lt;"));
2561                         bptr += 4;
2562                         Target->BufUsed += 4;
2563                         break;
2564                 case '>':
2565                         memcpy(bptr, HKEY("&gt;"));
2566                         bptr += 4;
2567                         Target->BufUsed += 4;
2568                         break;
2569                 case '&':
2570                         memcpy(bptr, HKEY("&amp;"));
2571                         bptr += 5;
2572                         Target->BufUsed += 5;
2573                         break;
2574                 case LB:
2575                         *bptr = '<';
2576                         bptr ++;
2577                         Target->BufUsed ++;
2578                         break;
2579                 case RB:
2580                         *bptr = '>';
2581                         bptr ++;
2582                         Target->BufUsed ++;
2583                         break;
2584                 case '\n':
2585                         switch (nolinebreaks) {
2586                         case 1:
2587                                 *bptr='\0';     /* nothing */
2588                                 break;
2589                         case 2:
2590                                 memcpy(bptr, HKEY("&lt;br/&gt;"));
2591                                 bptr += 11;
2592                                 Target->BufUsed += 11;
2593                                 break;
2594                         default:
2595                                 memcpy(bptr, HKEY("\\n"));
2596                                 bptr += 2;
2597                                 Target->BufUsed += 2;
2598                         }
2599                         break;
2600                 case '\r':
2601                         switch (nolinebreaks) {
2602                         case 1:
2603                         case 2:
2604                                 *bptr='\0';     /* nothing */
2605                                 break;
2606                         default:
2607                                 memcpy(bptr, HKEY("\\r"));
2608                                 bptr += 2;
2609                                 Target->BufUsed += 2;
2610                                 break;
2611                         }
2612                         break;
2613                 case '"':
2614                 case QU:
2615                         *bptr = '\\';
2616                         bptr ++;
2617                         *bptr = '"';
2618                         bptr ++;
2619                         Target->BufUsed += 2;
2620                         break;
2621                 case '\\':
2622                         if ((*(aptr + 1) == 'u') &&
2623                             isxdigit(*(aptr + 2)) &&
2624                             isxdigit(*(aptr + 3)) &&
2625                             isxdigit(*(aptr + 4)) &&
2626                             isxdigit(*(aptr + 5)))
2627                         { /* oh, a unicode escaper. let it pass through. */
2628                                 memcpy(bptr, aptr, 6);
2629                                 aptr += 5;
2630                                 bptr +=6;
2631                                 Target->BufUsed += 6;
2632                         }
2633                         else
2634                         {
2635                                 *bptr = '\\';
2636                                 bptr ++;
2637                                 *bptr = '\\';
2638                                 bptr ++;
2639                                 Target->BufUsed += 2;
2640                         }
2641                         break;
2642                 case '\b':
2643                         *bptr = '\\';
2644                         bptr ++;
2645                         *bptr = 'b';
2646                         bptr ++;
2647                         Target->BufUsed += 2;
2648                         break;
2649                 case '\f':
2650                         *bptr = '\\';
2651                         bptr ++;
2652                         *bptr = 'f';
2653                         bptr ++;
2654                         Target->BufUsed += 2;
2655                         break;
2656                 case '\t':
2657                         *bptr = '\\';
2658                         bptr ++;
2659                         *bptr = 't';
2660                         bptr ++;
2661                         Target->BufUsed += 2;
2662                         break;
2663                 case  32:
2664                         if (nbsp == 1) {
2665                                 memcpy(bptr, HKEY("&nbsp;"));
2666                                 bptr += 6;
2667                                 Target->BufUsed += 6;
2668                                 break;
2669                         }
2670                 default:
2671                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(aptr, eiptr);
2672                         while (IsUtf8Sequence > 0){
2673                                 *bptr = *aptr;
2674                                 Target->BufUsed ++;
2675                                 if (--IsUtf8Sequence)
2676                                         aptr++;
2677                                 bptr++;
2678                         }
2679                 }
2680                 aptr ++;
2681         }
2682         *bptr = '\0';
2683         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
2684                 return -1;
2685         return Target->BufUsed;
2686 }
2687
2688
2689 /**
2690  * @ingroup StrBuf_DeEnCoder
2691  * @brief replace all non-Ascii characters by another
2692  * @param Buf buffer to inspect
2693  * @param repl charater to stamp over non ascii chars
2694  */
2695 void StrBufAsciify(StrBuf *Buf, const char repl)
2696 {
2697         long offset;
2698
2699         for (offset = 0; offset < Buf->BufUsed; offset ++)
2700                 if (!isascii(Buf->buf[offset]))
2701                         Buf->buf[offset] = repl;
2702
2703 }
2704
2705 /**
2706  * @ingroup StrBuf_DeEnCoder
2707  * @brief unhide special chars hidden to the HTML escaper
2708  * @param target buffer to put the unescaped string in
2709  * @param source buffer to unescape
2710  */
2711 void StrBufEUid_unescapize(StrBuf *target, const StrBuf *source)
2712 {
2713         int a, b, len;
2714         char hex[3];
2715
2716         if ((source == NULL) || (target == NULL) || (target->buf == NULL))
2717         {
2718                 return;
2719         }
2720
2721         if (target != NULL)
2722                 FlushStrBuf(target);
2723
2724         len = source->BufUsed;
2725         for (a = 0; a < len; ++a) {
2726                 if (target->BufUsed >= target->BufSize)
2727                         IncreaseBuf(target, 1, -1);
2728
2729                 if (source->buf[a] == '=') {
2730                         hex[0] = source->buf[a + 1];
2731                         hex[1] = source->buf[a + 2];
2732                         hex[2] = 0;
2733                         b = 0;
2734                         sscanf(hex, "%02x", &b);
2735                         target->buf[target->BufUsed] = b;
2736                         target->buf[++target->BufUsed] = 0;
2737                         a += 2;
2738                 }
2739                 else {
2740                         target->buf[target->BufUsed] = source->buf[a];
2741                         target->buf[++target->BufUsed] = 0;
2742                 }
2743         }
2744 }
2745
2746
2747 /**
2748  * @ingroup StrBuf_DeEnCoder
2749  * @brief hide special chars from the HTML escapers and friends
2750  * @param target buffer to put the escaped string in
2751  * @param source buffer to escape
2752  */
2753 void StrBufEUid_escapize(StrBuf *target, const StrBuf *source)
2754 {
2755         int i, len;
2756
2757         if (target != NULL)
2758                 FlushStrBuf(target);
2759
2760         if ((source == NULL) || (target == NULL) || (target->buf == NULL))
2761         {
2762                 return;
2763         }
2764
2765         len = source->BufUsed;
2766         for (i=0; i<len; ++i) {
2767                 if (target->BufUsed + 4 >= target->BufSize)
2768                         IncreaseBuf(target, 1, -1);
2769                 if ( (isalnum(source->buf[i])) ||
2770                      (source->buf[i]=='-') ||
2771                      (source->buf[i]=='_') ) {
2772                         target->buf[target->BufUsed++] = source->buf[i];
2773                 }
2774                 else {
2775                         sprintf(&target->buf[target->BufUsed],
2776                                 "=%02X",
2777                                 (0xFF &source->buf[i]));
2778                         target->BufUsed += 3;
2779                 }
2780         }
2781         target->buf[target->BufUsed + 1] = '\0';
2782 }
2783
2784
2785 /*******************************************************************************
2786  *                      Quoted Printable de/encoding                           *
2787  *******************************************************************************/
2788
2789 /**
2790  * @ingroup StrBuf_DeEnCoder
2791  * @brief decode a buffer from base 64 encoding; destroys original
2792  * @param Buf Buffor to transform
2793  */
2794 int StrBufDecodeBase64(StrBuf *Buf)
2795 {
2796         char *xferbuf;
2797         size_t siz;
2798
2799         if (Buf == NULL)
2800                 return -1;
2801
2802         xferbuf = (char*) malloc(Buf->BufSize);
2803         if (xferbuf == NULL)
2804                 return -1;
2805
2806         *xferbuf = '\0';
2807         siz = CtdlDecodeBase64(xferbuf,
2808                                Buf->buf,
2809                                Buf->BufUsed);
2810         free(Buf->buf);
2811         Buf->buf = xferbuf;
2812         Buf->BufUsed = siz;
2813
2814         Buf->buf[Buf->BufUsed] = '\0';
2815         return siz;
2816 }
2817
2818 /**
2819  * @ingroup StrBuf_DeEnCoder
2820  * @brief decode a buffer from base 64 encoding; expects targetbuffer
2821  * @param BufIn Buffor to transform
2822  * @param BufOut Buffer to put result into
2823  */
2824 int StrBufDecodeBase64To(const StrBuf *BufIn, StrBuf *BufOut)
2825 {
2826         if ((BufIn == NULL) || (BufOut == NULL))
2827                 return -1;
2828
2829         if (BufOut->BufSize < BufIn->BufUsed)
2830                 IncreaseBuf(BufOut, 0, BufIn->BufUsed);
2831
2832         BufOut->BufUsed = CtdlDecodeBase64(BufOut->buf,
2833                                            BufIn->buf,
2834                                            BufIn->BufUsed);
2835         return BufOut->BufUsed;
2836 }
2837
2838 typedef struct __z_enc_stream {
2839         StrBuf OutBuf;
2840         z_stream zstream;
2841 } z_enc_stream;
2842
2843 vStreamT *StrBufNewStreamContext(eStreamType type, const char **Err)
2844 {
2845         base64_decodestate *state;;
2846         *Err = NULL;
2847
2848         switch (type)
2849         {
2850         case eBase64Decode:
2851         case eBase64Encode:
2852                 state = (base64_decodestate*) malloc(sizeof(base64_decodestate));
2853                 base64_init_decodestate(state);
2854                 return (vStreamT*) state;
2855                 break;
2856         case eZLibDecode:
2857         {
2858
2859                 z_enc_stream *stream;
2860                 int err;
2861
2862                 stream = (z_enc_stream *) malloc(sizeof(z_enc_stream));
2863                 memset(stream, 0, sizeof(z_enc_stream));
2864                 stream->OutBuf.BufSize = 4*SIZ; /// TODO 64
2865                 stream->OutBuf.buf = (char*)malloc(stream->OutBuf.BufSize);
2866
2867                 err = inflateInit(&stream->zstream);
2868
2869                 if (err != Z_OK) {
2870                         StrBufDestroyStreamContext(type, (vStreamT**) &stream, Err);
2871                         *Err = zError(err);
2872                         return NULL;
2873                 }
2874                 return (vStreamT*) stream;
2875
2876         }
2877         case eZLibEncode:
2878         {
2879                 z_enc_stream *stream;
2880                 int err;
2881
2882                 stream = (z_enc_stream *) malloc(sizeof(z_enc_stream));
2883                 memset(stream, 0, sizeof(z_enc_stream));
2884                 stream->OutBuf.BufSize = 4*SIZ; /// todo 64
2885                 stream->OutBuf.buf = (char*)malloc(stream->OutBuf.BufSize);
2886                 /* write gzip header */
2887                 stream->OutBuf.BufUsed = snprintf
2888                         (stream->OutBuf.buf,
2889                          stream->OutBuf.BufSize,
2890                          "%c%c%c%c%c%c%c%c%c%c",
2891                          gz_magic[0], gz_magic[1], Z_DEFLATED,
2892                          0 /*flags */ , 0, 0, 0, 0 /*time */ , 0 /* xflags */ ,
2893                          OS_CODE);
2894
2895                 err = deflateInit2(&stream->zstream,
2896                                    ZLibCompressionRatio,
2897                                    Z_DEFLATED,
2898                                    -MAX_WBITS,
2899                                    DEF_MEM_LEVEL,
2900                                    Z_DEFAULT_STRATEGY);
2901                 if (err != Z_OK) {
2902                         StrBufDestroyStreamContext(type, (vStreamT**) &stream, Err);
2903                         *Err = zError(err);
2904                         return NULL;
2905                 }
2906                 return (vStreamT*) stream;
2907         }
2908         case eEmtyCodec:
2909                 /// TODO
2910
2911                 break;
2912         }
2913         return NULL;
2914 }
2915
2916 int StrBufDestroyStreamContext(eStreamType type, vStreamT **vStream, const char **Err)
2917 {
2918         int err;
2919         int rc = 0;
2920         *Err = NULL;
2921
2922         if ((vStream == NULL) || (*vStream==NULL)) {
2923                 *Err = strerror(EINVAL);
2924                 return EINVAL;
2925         }
2926         switch (type)
2927         {
2928         case eBase64Encode:
2929         case eBase64Decode:
2930                 free(*vStream);
2931                 *vStream = NULL;
2932                 break;
2933         case eZLibDecode:
2934         {
2935                 z_enc_stream *stream = (z_enc_stream *)*vStream;
2936                 (void)inflateEnd(&stream->zstream);
2937                 free(stream->OutBuf.buf);
2938                 free(stream);
2939         }
2940         case eZLibEncode:
2941         {
2942                 z_enc_stream *stream = (z_enc_stream *)*vStream;
2943                 err = deflateEnd(&stream->zstream);
2944                 if (err != Z_OK) {
2945                         *Err = zError(err);
2946                         rc = -1;
2947                 }
2948                 free(stream->OutBuf.buf);
2949                 free(stream);
2950                 *vStream = NULL;
2951                 break;
2952         }
2953         case eEmtyCodec:
2954                 break; /// TODO
2955         }
2956         return rc;
2957 }
2958
2959 int StrBufStreamTranscode(eStreamType type, IOBuffer *Target, IOBuffer *In, const char* pIn, long pInLen, vStreamT *vStream, int LastChunk, const char **Err)
2960 {
2961         int rc = 0;
2962         switch (type)
2963         {
2964         case eBase64Encode:
2965         {
2966                 /// TODO
2967 /*
2968                 // base64_decodestate *state = (base64_decodestate*)vStream;
2969                 long ExpectLen;
2970
2971                 if (In != NULL)
2972                 {
2973                         pIn = In->buf;
2974                         pInLen = In->BufUsed;
2975                 }
2976                 if ((In == NULL) || (vStream == NULL))
2977                         return;
2978
2979                 ExpectLen = (pInLen / 4) * 3;
2980
2981                 if (Target->BufSize - Target->BufUsed < ExpectLen)
2982                 {
2983                         IncreaseBuf(Target, 1, Target->BufUsed + ExpectLen + 1);
2984                 }
2985
2986                 ////    ExpectLen = base64_encode_block(pIn, pInLen, Target->buf + Target->BufUsed, state);
2987                 Target->BufUsed += ExpectLen;
2988                 Target->buf[Target->BufUsed] = '\0';
2989 */
2990         }
2991         break;
2992         case eBase64Decode:
2993         {
2994 /*
2995                 base64_decodestate *state = (base64_decodestate *)vStream;
2996                 long ExpectLen;
2997
2998                 if (In != NULL)
2999                 {
3000                         pIn = In->buf;
3001                         pInLen = In->BufUsed;
3002                 }
3003                 if ((pIn == NULL) || (vStream == NULL))
3004                         return;
3005
3006                 ExpectLen = (pInLen / 4) * 3;
3007
3008                 if (Target->BufSize - Target->BufUsed < ExpectLen)
3009                 {
3010                         IncreaseBuf(Target, 1, Target->BufUsed + ExpectLen + 1);
3011                 }
3012
3013                 ExpectLen = base64_decode_block(pIn, pInLen, Target->buf + Target->BufUsed, state);
3014                 Target->BufUsed += ExpectLen;
3015                 Target->buf[Target->BufUsed] = '\0';
3016 */
3017         }
3018         break;
3019         case eZLibEncode:
3020         {
3021                 z_enc_stream *stream = (z_enc_stream *)vStream;
3022                 int org_outbuf_len = stream->OutBuf.BufUsed;
3023                 int err;
3024                 unsigned int chunkavail;
3025
3026                 if (In->ReadWritePointer != NULL)
3027                 {
3028                         stream->zstream.next_in = (Bytef *) In->ReadWritePointer;
3029                         stream->zstream.avail_in = (uInt) In->Buf->BufUsed -
3030                                 (In->ReadWritePointer - In->Buf->buf);
3031                 }
3032                 else
3033                 {
3034                         stream->zstream.next_in = (Bytef *) In->Buf->buf;
3035                         stream->zstream.avail_in = (uInt) In->Buf->BufUsed;
3036                 }
3037
3038                 stream->zstream.next_out = (unsigned char*)stream->OutBuf.buf + stream->OutBuf.BufUsed;
3039                 stream->zstream.avail_out = chunkavail = (uInt) stream->OutBuf.BufSize - stream->OutBuf.BufUsed;
3040
3041                 err = deflate(&stream->zstream,  (LastChunk) ? Z_FINISH : Z_NO_FLUSH);
3042
3043                 stream->OutBuf.BufUsed += (chunkavail - stream->zstream.avail_out);
3044
3045                 if (Target &&
3046                     (LastChunk ||
3047                      (stream->OutBuf.BufUsed != org_outbuf_len)
3048                             ))
3049                 {
3050                         SwapBuffers(Target->Buf, &stream->OutBuf);
3051                 }
3052
3053                 if (stream->zstream.avail_in == 0)
3054                 {
3055                         FlushStrBuf(In->Buf);
3056                         In->ReadWritePointer = NULL;
3057                 }
3058                 else
3059                 {
3060                         if (stream->zstream.avail_in < 64)
3061                         {
3062                                 memmove(In->Buf->buf,
3063                                         In->Buf->buf + In->Buf->BufUsed - stream->zstream.avail_in,
3064                                         stream->zstream.avail_in);
3065
3066                                 In->Buf->BufUsed = stream->zstream.avail_in;
3067                                 In->Buf->buf[In->Buf->BufUsed] = '\0';
3068                         }
3069                         else
3070                         {
3071
3072                                 In->ReadWritePointer = In->Buf->buf +
3073                                         (In->Buf->BufUsed - stream->zstream.avail_in);
3074                         }
3075                 }
3076                 rc = (LastChunk && (err != Z_FINISH));
3077                 if (!rc && (err != Z_OK)) {
3078                         *Err = zError(err);
3079                 }
3080
3081         }
3082         break;
3083         case eZLibDecode: {
3084                 z_enc_stream *stream = (z_enc_stream *)vStream;
3085                 int org_outbuf_len = stream->zstream.total_out;
3086                 int err;
3087
3088                 if ((stream->zstream.avail_out != 0) && (stream->zstream.next_in != NULL)) {
3089                         if (In->ReadWritePointer != NULL)
3090                         {
3091                                 stream->zstream.next_in = (Bytef *) In->ReadWritePointer;
3092                                 stream->zstream.avail_in = (uInt) In->Buf->BufUsed -
3093                                         (In->ReadWritePointer - In->Buf->buf);
3094                         }
3095                         else
3096                         {
3097                                 stream->zstream.next_in = (Bytef *) In->Buf->buf;
3098                                 stream->zstream.avail_in = (uInt) In->Buf->BufUsed;
3099                         }
3100                 }
3101
3102                 stream->zstream.next_out = (unsigned char*)stream->OutBuf.buf + stream->OutBuf.BufUsed;
3103                 stream->zstream.avail_out = (uInt) stream->OutBuf.BufSize - stream->OutBuf.BufUsed;
3104
3105                 err = inflate(&stream->zstream, Z_NO_FLUSH);
3106
3107                 ///assert(ret != Z_STREAM_ERROR);  /* state not clobbered * /
3108                 switch (err) {
3109                 case Z_NEED_DICT:
3110                         err = Z_DATA_ERROR;     /* and fall through */
3111
3112                 case Z_DATA_ERROR:
3113                         *Err = zError(err);
3114                 case Z_MEM_ERROR:
3115                         (void)inflateEnd(&stream->zstream);
3116                         return err;
3117                 }
3118
3119                 stream->OutBuf.BufUsed += stream->zstream.total_out + org_outbuf_len;
3120
3121                 if (Target) SwapBuffers(Target->Buf, &stream->OutBuf);
3122
3123                 if (stream->zstream.avail_in == 0)
3124                 {
3125                         FlushStrBuf(In->Buf);
3126                         In->ReadWritePointer = NULL;
3127                 }
3128                 else
3129                 {
3130                         if (stream->zstream.avail_in < 64)
3131                         {
3132                                 memmove(In->Buf->buf,
3133                                         In->Buf->buf + In->Buf->BufUsed - stream->zstream.avail_in,
3134                                         stream->zstream.avail_in);
3135
3136                                 In->Buf->BufUsed = stream->zstream.avail_in;
3137                                 In->Buf->buf[In->Buf->BufUsed] = '\0';
3138                         }
3139                         else
3140                         {
3141
3142                                 In->ReadWritePointer = In->Buf->buf +
3143                                         (In->Buf->BufUsed - stream->zstream.avail_in);
3144                         }
3145                 }
3146         }
3147                 break;
3148         case eEmtyCodec: {
3149
3150         }
3151                 break; /// TODO
3152         }
3153         return rc;
3154 }
3155
3156 /**
3157  * @ingroup StrBuf_DeEnCoder
3158  * @brief decode a buffer from base 64 encoding; destroys original
3159  * @param Buf Buffor to transform
3160  */
3161 int StrBufDecodeHex(StrBuf *Buf)
3162 {
3163         unsigned int ch;
3164         char *pch, *pche, *pchi;
3165
3166         if (Buf == NULL) return -1;
3167
3168         pch = pchi = Buf->buf;
3169         pche = pch + Buf->BufUsed;
3170
3171         while (pchi < pche){
3172                 ch = decode_hex(pchi);
3173                 *pch = ch;
3174                 pch ++;
3175                 pchi += 2;
3176         }
3177
3178         *pch = '\0';
3179         Buf->BufUsed = pch - Buf->buf;
3180         return Buf->BufUsed;
3181 }
3182
3183 /**
3184  * @ingroup StrBuf_DeEnCoder
3185  * @brief replace all chars >0x20 && < 0x7F with Mute
3186  * @param Mute char to put over invalid chars
3187  * @param Buf Buffor to transform
3188  */
3189 int StrBufSanitizeAscii(StrBuf *Buf, const char Mute)
3190 {
3191         unsigned char *pch;
3192
3193         if (Buf == NULL) return -1;
3194         pch = (unsigned char *)Buf->buf;
3195         while (pch < (unsigned char *)Buf->buf + Buf->BufUsed) {
3196                 if ((*pch < 0x20) || (*pch > 0x7F))
3197                         *pch = Mute;
3198                 pch ++;
3199         }
3200         return Buf->BufUsed;
3201 }
3202
3203
3204 /**
3205  * @ingroup StrBuf_DeEnCoder
3206  * @brief remove escaped strings from i.e. the url string (like %20 for blanks)
3207  * @param Buf Buffer to translate
3208  * @param StripBlanks Reduce several blanks to one?
3209  */
3210 long StrBufUnescape(StrBuf *Buf, int StripBlanks)
3211 {
3212         int a, b;
3213         char hex[3];
3214         long len;
3215
3216         if (Buf == NULL)
3217                 return -1;
3218
3219         while ((Buf->BufUsed > 0) && (isspace(Buf->buf[Buf->BufUsed - 1]))){
3220                 Buf->buf[Buf->BufUsed - 1] = '\0';
3221                 Buf->BufUsed --;
3222         }
3223
3224         a = 0;
3225         while (a < Buf->BufUsed) {
3226                 if (Buf->buf[a] == '+')
3227                         Buf->buf[a] = ' ';
3228                 else if (Buf->buf[a] == '%') {
3229                         /* don't let % chars through, rather truncate the input. */
3230                         if (a + 2 > Buf->BufUsed) {
3231                                 Buf->buf[a] = '\0';
3232                                 Buf->BufUsed = a;
3233                         }
3234                         else {
3235                                 hex[0] = Buf->buf[a + 1];
3236                                 hex[1] = Buf->buf[a + 2];
3237                                 hex[2] = 0;
3238                                 b = 0;
3239                                 sscanf(hex, "%02x", &b);
3240                                 Buf->buf[a] = (char) b;
3241                                 len = Buf->BufUsed - a - 2;
3242                                 if (len > 0)
3243                                         memmove(&Buf->buf[a + 1], &Buf->buf[a + 3], len);
3244
3245                                 Buf->BufUsed -=2;
3246                         }
3247                 }
3248                 a++;
3249         }
3250         return a;
3251 }
3252
3253
3254 /**
3255  * @ingroup StrBuf_DeEnCoder
3256  * @brief       RFC2047-encode a header field if necessary.
3257  *              If no non-ASCII characters are found, the string
3258  *              will be copied verbatim without encoding.
3259  *
3260  * @param       target          Target buffer.
3261  * @param       source          Source string to be encoded.
3262  * @returns     encoded length; -1 if non success.
3263  */
3264 int StrBufRFC2047encode(StrBuf **target, const StrBuf *source)
3265 {
3266         const char headerStr[] = "=?UTF-8?Q?";
3267         int need_to_encode = 0;
3268         int i = 0;
3269         unsigned char ch;
3270
3271         if ((source == NULL) ||
3272             (target == NULL))
3273             return -1;
3274
3275         while ((i < source->BufUsed) &&
3276                (!IsEmptyStr (&source->buf[i])) &&
3277                (need_to_encode == 0)) {
3278                 if (((unsigned char) source->buf[i] < 32) ||
3279                     ((unsigned char) source->buf[i] > 126)) {
3280                         need_to_encode = 1;
3281                 }
3282                 i++;
3283         }
3284
3285         if (!need_to_encode) {
3286                 if (*target == NULL) {
3287                         *target = NewStrBufPlain(source->buf, source->BufUsed);
3288                 }
3289                 else {
3290                         FlushStrBuf(*target);
3291                         StrBufAppendBuf(*target, source, 0);
3292                 }
3293                 if (*target != 0)
3294                         return (*target)->BufUsed;
3295                 else
3296                         return 0;
3297         }
3298         if (*target == NULL)
3299                 *target = NewStrBufPlain(NULL, sizeof(headerStr) + source->BufUsed * 2);
3300         else if (sizeof(headerStr) + source->BufUsed >= (*target)->BufSize)
3301                 IncreaseBuf(*target, sizeof(headerStr) + source->BufUsed, 0);
3302         memcpy ((*target)->buf, headerStr, sizeof(headerStr) - 1);
3303         (*target)->BufUsed = sizeof(headerStr) - 1;
3304         for (i=0; (i < source->BufUsed); ++i) {
3305                 if ((*target)->BufUsed + 4 >= (*target)->BufSize)
3306                         IncreaseBuf(*target, 1, 0);
3307                 ch = (unsigned char) source->buf[i];
3308                 if ((ch  <  32) ||
3309                     (ch  > 126) ||
3310                     (ch == '=') ||
3311                     (ch == '?') ||
3312                     (ch == '_') ||
3313                     (ch == '[') ||
3314                     (ch == ']')   )
3315                 {
3316                         sprintf(&(*target)->buf[(*target)->BufUsed], "=%02X", ch);
3317                         (*target)->BufUsed += 3;
3318                 }
3319                 else {
3320                         if (ch == ' ')
3321                                 (*target)->buf[(*target)->BufUsed] = '_';
3322                         else
3323                                 (*target)->buf[(*target)->BufUsed] = ch;
3324                         (*target)->BufUsed++;
3325                 }
3326         }
3327
3328         if ((*target)->BufUsed + 4 >= (*target)->BufSize)
3329                 IncreaseBuf(*target, 1, 0);
3330
3331         (*target)->buf[(*target)->BufUsed++] = '?';
3332         (*target)->buf[(*target)->BufUsed++] = '=';
3333         (*target)->buf[(*target)->BufUsed] = '\0';
3334         return (*target)->BufUsed;;
3335 }
3336
3337 /**
3338  * @ingroup StrBuf_DeEnCoder
3339  * @brief       Quoted-Printable encode a message; make it < 80 columns width.
3340  * @param       source          Source string to be encoded.
3341  * @returns     buffer with encoded message.
3342  */
3343 StrBuf *StrBufRFC2047encodeMessage(const StrBuf *EncodeMe)
3344 {
3345         StrBuf *OutBuf;
3346         char *Optr, *OEptr;
3347         const char *ptr, *eptr;
3348         unsigned char ch;
3349         int LinePos;
3350
3351         OutBuf = NewStrBufPlain(NULL, StrLength(EncodeMe) * 4);
3352         Optr = OutBuf->buf;
3353         OEptr = OutBuf->buf + OutBuf->BufSize;
3354         ptr = EncodeMe->buf;
3355         eptr = EncodeMe->buf + EncodeMe->BufUsed;
3356         LinePos = 0;
3357
3358         while (ptr < eptr)
3359         {
3360                 if (Optr + 4 >= OEptr)
3361                 {
3362                         long Offset;
3363                         Offset = Optr - OutBuf->buf;
3364                         OutBuf->BufUsed = Optr - OutBuf->buf;
3365                         IncreaseBuf(OutBuf, 1, 0);
3366                         Optr = OutBuf->buf + Offset;
3367                         OEptr = OutBuf->buf + OutBuf->BufSize;
3368                 }
3369                 if (*ptr == '\r')
3370                 {
3371                         /* ignore carriage returns */
3372                         ptr ++;
3373                 }
3374                 else if (*ptr == '\n') {
3375                         /* hard line break */
3376                         memcpy(Optr, HKEY("=0A"));
3377                         Optr += 3;
3378                         LinePos += 3;
3379                         ptr ++;
3380                 }
3381                 else if (( (*ptr >= 32) && (*ptr <= 60) ) ||
3382                          ( (*ptr >= 62) && (*ptr <= 126) ))
3383                 {
3384                         *Optr = *ptr;
3385                         Optr ++;
3386                         ptr ++;
3387                         LinePos ++;
3388                 }
3389                 else {
3390                         ch = *ptr;
3391                         *Optr = '=';
3392                         Optr ++;
3393                         *Optr = HexList[ch][0];
3394                         Optr ++;
3395                         *Optr = HexList[ch][1];
3396                         Optr ++;
3397                         LinePos += 3;
3398                         ptr ++;
3399                 }
3400
3401                 if (LinePos > 72) {
3402                         /* soft line break */
3403                         if (isspace(*(Optr - 1))) {
3404                                 ch = *(Optr - 1);
3405                                 Optr --;
3406                                 *Optr = '=';
3407                                 Optr ++;
3408                                 *Optr = HexList[ch][0];
3409                                 Optr ++;
3410                                 *Optr = HexList[ch][1];
3411                                 Optr ++;
3412                                 LinePos += 3;
3413                         }
3414                         *Optr = '=';
3415                         Optr ++;
3416                         *Optr = '\n';
3417                         Optr ++;
3418                         LinePos = 0;
3419                 }
3420         }
3421         *Optr = '\0';
3422         OutBuf->BufUsed = Optr - OutBuf->buf;
3423
3424         return OutBuf;
3425 }
3426
3427
3428 static void AddRecipient(StrBuf *Target,
3429                          StrBuf *UserName,
3430                          StrBuf *EmailAddress,
3431                          StrBuf *EncBuf)
3432 {
3433         int QuoteMe = 0;
3434
3435         if (StrLength(Target) > 0) StrBufAppendBufPlain(Target, HKEY(", "), 0);
3436         if (strchr(ChrPtr(UserName), ',') != NULL) QuoteMe = 1;
3437
3438         if (QuoteMe)  StrBufAppendBufPlain(Target, HKEY("\""), 0);
3439         StrBufRFC2047encode(&EncBuf, UserName);
3440         StrBufAppendBuf(Target, EncBuf, 0);
3441         if (QuoteMe)  StrBufAppendBufPlain(Target, HKEY("\" "), 0);
3442         else          StrBufAppendBufPlain(Target, HKEY(" "), 0);
3443
3444         if (StrLength(EmailAddress) > 0){
3445                 StrBufAppendBufPlain(Target, HKEY("<"), 0);
3446                 StrBufAppendBuf(Target, EmailAddress, 0); /* TODO: what about IDN???? */
3447                 StrBufAppendBufPlain(Target, HKEY(">"), 0);
3448         }
3449 }
3450
3451
3452 /**
3453  * \brief QP encode parts of an email TO/CC/BCC vector, and strip/filter invalid parts
3454  * \param Recp Source list of email recipients
3455  * \param UserName Temporary buffer for internal use; Please provide valid buffer.
3456  * \param EmailAddress Temporary buffer for internal use; Please provide valid buffer.
3457  * \param EncBuf Temporary buffer for internal use; Please provide valid buffer.
3458  * \returns encoded & sanitized buffer with the contents of Recp; Caller owns this memory.
3459  */
3460 StrBuf *StrBufSanitizeEmailRecipientVector(const StrBuf *Recp,
3461                                            StrBuf *UserName,
3462                                            StrBuf *EmailAddress,
3463                                            StrBuf *EncBuf)
3464 {
3465         StrBuf *Target;
3466         const char *pch, *pche;
3467         const char *UserStart, *UserEnd, *EmailStart, *EmailEnd, *At;
3468
3469         if ((Recp == NULL) || (StrLength(Recp) == 0))
3470                 return NULL;
3471
3472         pch = ChrPtr(Recp);
3473         pche = pch + StrLength(Recp);
3474
3475         if (!CheckEncode(pch, -1, pche))
3476                 return NewStrBufDup(Recp);
3477
3478         Target = NewStrBufPlain(NULL, StrLength(Recp));
3479
3480         while ((pch != NULL) && (pch < pche))
3481         {
3482                 while (isspace(*pch)) pch++;
3483                 UserEnd = EmailStart = EmailEnd = NULL;
3484
3485                 if ((*pch == '"') || (*pch == '\'')) {
3486                         UserStart = pch + 1;
3487
3488                         UserEnd = strchr(UserStart, *pch);
3489                         if (UserEnd == NULL)
3490                                 break; ///TODO: Userfeedback??
3491                         EmailStart = UserEnd + 1;
3492                         while (isspace(*EmailStart))
3493                                 EmailStart++;
3494                         if (UserEnd == UserStart) {
3495                                 UserStart = UserEnd = NULL;
3496                         }
3497
3498                         if (*EmailStart == '<') {
3499                                 EmailStart++;
3500                                 EmailEnd = strchr(EmailStart, '>');
3501                                 if (EmailEnd == NULL)
3502                                         EmailEnd = strchr(EmailStart, ',');
3503
3504                         }
3505                         else {
3506                                 EmailEnd = strchr(EmailStart, ',');
3507                         }
3508                         if (EmailEnd == NULL)
3509                                 EmailEnd = pche;
3510                         pch = EmailEnd + 1;
3511                 }
3512                 else {
3513                         int gt = 0;
3514                         UserStart = pch;
3515                         EmailEnd = strchr(UserStart, ',');
3516                         if (EmailEnd == NULL) {
3517                                 EmailEnd = strchr(pch, '>');
3518                                 pch = NULL;
3519                                 if (EmailEnd != NULL) {
3520                                         gt = 1;
3521                                 }
3522                                 else {
3523                                         EmailEnd = pche;
3524                                 }
3525                         }
3526                         else {
3527
3528                                 pch = EmailEnd + 1;
3529                                 while ((EmailEnd > UserStart) && !gt &&
3530                                        ((*EmailEnd == ',') ||
3531                                         (*EmailEnd == '>') ||
3532                                         (isspace(*EmailEnd))))
3533                                 {
3534                                         if (*EmailEnd == '>')
3535                                                 gt = 1;
3536                                         else
3537                                                 EmailEnd--;
3538                                 }
3539                                 if (EmailEnd == UserStart)
3540                                         break;
3541                         }
3542                         if (gt) {
3543                                 EmailStart = strchr(UserStart, '<');
3544                                 if ((EmailStart == NULL) || (EmailStart > EmailEnd))
3545                                         break;
3546                                 UserEnd = EmailStart;
3547
3548                                 while ((UserEnd > UserStart) &&
3549                                        isspace (*(UserEnd - 1)))
3550                                         UserEnd --;
3551                                 EmailStart ++;
3552                                 if (UserStart >= UserEnd)
3553                                         UserStart = UserEnd = NULL;
3554                         }
3555                         else { /* this is a local recipient... no domain, just a realname */
3556                                 EmailStart = UserStart;
3557                                 At = strchr(EmailStart, '@');
3558                                 if (At == NULL) {
3559                                         UserEnd = EmailEnd;
3560                                         EmailEnd = NULL;
3561                                 }
3562                                 else {
3563                                         EmailStart = UserStart;
3564                                         UserStart = NULL;
3565                                 }
3566                         }
3567                 }
3568
3569                 if ((UserStart != NULL) && (UserEnd != NULL))
3570                         StrBufPlain(UserName, UserStart, UserEnd - UserStart);
3571                 else if ((UserStart != NULL) && (UserEnd == NULL))
3572                         StrBufPlain(UserName, UserStart, UserEnd - UserStart);
3573                 else
3574                         FlushStrBuf(UserName);
3575
3576                 if ((EmailStart != NULL) && (EmailEnd != NULL))
3577                         StrBufPlain(EmailAddress, EmailStart, EmailEnd - EmailStart);
3578                 else if ((EmailStart != NULL) && (EmailEnd == NULL))
3579                         StrBufPlain(EmailAddress, EmailStart, EmailEnd - pche);
3580                 else
3581                         FlushStrBuf(EmailAddress);
3582
3583                 AddRecipient(Target, UserName, EmailAddress, EncBuf);
3584
3585                 if (pch == NULL)
3586                         break;
3587
3588                 if ((pch != NULL) && (*pch == ','))
3589                         pch ++;
3590                 if (pch != NULL) while (isspace(*pch))
3591                         pch ++;
3592         }
3593         return Target;
3594 }
3595
3596
3597 /**
3598  * @ingroup StrBuf
3599  * @brief replaces all occurances of 'search' by 'replace'
3600  * @param buf Buffer to modify
3601  * @param search character to search
3602  * @param replace character to replace search by
3603  */
3604 void StrBufReplaceChars(StrBuf *buf, char search, char replace)
3605 {
3606         long i;
3607         if (buf == NULL)
3608                 return;
3609         for (i=0; i<buf->BufUsed; i++)
3610                 if (buf->buf[i] == search)
3611                         buf->buf[i] = replace;
3612
3613 }
3614
3615 /**
3616  * @ingroup StrBuf
3617  * @brief removes all \\r s from the string, or replaces them with \n if its not a combination of both.
3618  * @param buf Buffer to modify
3619  */
3620 void StrBufToUnixLF(StrBuf *buf)
3621 {
3622         char *pche, *pchS, *pchT;
3623         if (buf == NULL)
3624                 return;
3625
3626         pche = buf->buf + buf->BufUsed;
3627         pchS = pchT = buf->buf;
3628         while (pchS < pche)
3629         {
3630                 if (*pchS == '\r')
3631                 {
3632                         pchS ++;
3633                         if (*pchS != '\n') {
3634                                 *pchT = '\n';
3635                                 pchT++;
3636                         }
3637                 }
3638                 *pchT = *pchS;
3639                 pchT++; pchS++;
3640         }
3641         *pchT = '\0';
3642         buf->BufUsed = pchT - buf->buf;
3643 }
3644
3645
3646 /*******************************************************************************
3647  *                 Iconv Wrapper; RFC822 de/encoding                           *
3648  *******************************************************************************/
3649
3650 /**
3651  * @ingroup StrBuf_DeEnCoder
3652  * @brief Wrapper around iconv_open()
3653  * Our version adds aliases for non-standard Microsoft charsets
3654  * such as 'MS950', aliasing them to names like 'CP950'
3655  *
3656  * @param tocode        Target encoding
3657  * @param fromcode      Source encoding
3658  * @param pic           anonimized pointer to iconv struct
3659  */
3660 void  ctdl_iconv_open(const char *tocode, const char *fromcode, void *pic)
3661 {
3662 #ifdef HAVE_ICONV
3663         iconv_t ic = (iconv_t)(-1) ;
3664         ic = iconv_open(tocode, fromcode);
3665         if (ic == (iconv_t)(-1) ) {
3666                 char alias_fromcode[64];
3667                 if ( (strlen(fromcode) == 5) && (!strncasecmp(fromcode, "MS", 2)) ) {
3668                         safestrncpy(alias_fromcode, fromcode, sizeof alias_fromcode);
3669                         alias_fromcode[0] = 'C';
3670                         alias_fromcode[1] = 'P';
3671                         ic = iconv_open(tocode, alias_fromcode);
3672                 }
3673         }
3674         *(iconv_t *)pic = ic;
3675 #endif
3676 }
3677
3678
3679 /**
3680  * @ingroup StrBuf_DeEnCoder
3681  * @brief find one chunk of a RFC822 encoded string
3682  * @param Buffer where to search
3683  * @param bptr where to start searching
3684  * @returns found position, NULL if none.
3685  */
3686 static inline const char *FindNextEnd (const StrBuf *Buf, const char *bptr)
3687 {
3688         const char * end;
3689         /* Find the next ?Q? */
3690         if (Buf->BufUsed - (bptr - Buf->buf)  < 6)
3691                 return NULL;
3692
3693         end = strchr(bptr + 2, '?');
3694
3695         if (end == NULL)
3696                 return NULL;
3697
3698         if ((Buf->BufUsed - (end - Buf->buf) > 3) &&
3699             (((*(end + 1) == 'B') || (*(end + 1) == 'Q')) ||
3700              ((*(end + 1) == 'b') || (*(end + 1) == 'q'))) &&
3701             (*(end + 2) == '?')) {
3702                 /* skip on to the end of the cluster, the next ?= */
3703                 end = strstr(end + 3, "?=");
3704         }
3705         else
3706                 /* sort of half valid encoding, try to find an end. */
3707                 end = strstr(bptr, "?=");
3708         return end;
3709 }
3710
3711
3712
3713 /**
3714  * @ingroup StrBuf_DeEnCoder
3715  * @brief convert one buffer according to the preselected iconv pointer PIC
3716  * @param ConvertBuf buffer we need to translate
3717  * @param TmpBuf To share a workbuffer over several iterations. prepare to have it filled with useless stuff afterwards.
3718  * @param pic Pointer to the iconv-session Object
3719  */
3720 void StrBufConvert(StrBuf *ConvertBuf, StrBuf *TmpBuf, void *pic)
3721 {
3722 #ifdef HAVE_ICONV
3723         long trycount = 0;
3724         size_t siz;
3725         iconv_t ic;
3726         char *ibuf;                     /**< Buffer of characters to be converted */
3727         char *obuf;                     /**< Buffer for converted characters */
3728         size_t ibuflen;                 /**< Length of input buffer */
3729         size_t obuflen;                 /**< Length of output buffer */
3730
3731
3732         if ((ConvertBuf == NULL) || (TmpBuf == NULL))
3733                 return;
3734
3735         /* since we're converting to utf-8, one glyph may take up to 6 bytes */
3736         if (ConvertBuf->BufUsed * 6 >= TmpBuf->BufSize)
3737                 IncreaseBuf(TmpBuf, 0, ConvertBuf->BufUsed * 6);
3738 TRYAGAIN:
3739         ic = *(iconv_t*)pic;
3740         ibuf = ConvertBuf->buf;
3741         ibuflen = ConvertBuf->BufUsed;
3742         obuf = TmpBuf->buf;
3743         obuflen = TmpBuf->BufSize;
3744
3745         siz = iconv(ic, &ibuf, &ibuflen, &obuf, &obuflen);
3746
3747         if (siz < 0) {
3748                 if (errno == E2BIG) {
3749                         trycount ++;
3750                         IncreaseBuf(TmpBuf, 0, 0);
3751                         if (trycount < 5)
3752                                 goto TRYAGAIN;
3753
3754                 }
3755                 else if (errno == EILSEQ){
3756                         /* hm, invalid utf8 sequence... what to do now? */
3757                         /* An invalid multibyte sequence has been encountered in the input */
3758                 }
3759                 else if (errno == EINVAL) {
3760                         /* An incomplete multibyte sequence has been encountered in the input. */
3761                 }
3762
3763                 FlushStrBuf(TmpBuf);
3764         }
3765         else {
3766                 TmpBuf->BufUsed = TmpBuf->BufSize - obuflen;
3767                 TmpBuf->buf[TmpBuf->BufUsed] = '\0';
3768
3769                 /* little card game: wheres the red lady? */
3770                 SwapBuffers(ConvertBuf, TmpBuf);
3771                 FlushStrBuf(TmpBuf);
3772         }
3773 #endif
3774 }
3775
3776
3777 /**
3778  * @ingroup StrBuf_DeEnCoder
3779  * @brief catches one RFC822 encoded segment, and decodes it.
3780  * @param Target buffer to fill with result
3781  * @param DecodeMe buffer with stuff to process
3782  * @param SegmentStart points to our current segment in DecodeMe
3783  * @param SegmentEnd Points to the end of our current segment in DecodeMe
3784  * @param ConvertBuf Workbuffer shared between several iterations. Random content; needs to be valid
3785  * @param ConvertBuf2 Workbuffer shared between several iterations. Random content; needs to be valid
3786  * @param FoundCharset Characterset to default decoding to; if we find another we will overwrite it.
3787  */
3788 inline static void DecodeSegment(StrBuf *Target,
3789                                  const StrBuf *DecodeMe,
3790                                  const char *SegmentStart,
3791                                  const char *SegmentEnd,
3792                                  StrBuf *ConvertBuf,
3793                                  StrBuf *ConvertBuf2,
3794                                  StrBuf *FoundCharset)
3795 {
3796         StrBuf StaticBuf;
3797         char charset[128];
3798         char encoding[16];
3799 #ifdef HAVE_ICONV
3800         iconv_t ic = (iconv_t)(-1);
3801 #else
3802         void *ic = NULL;
3803 #endif
3804         /* Now we handle foreign character sets properly encoded
3805          * in RFC2047 format.
3806          */
3807         StaticBuf.buf = (char*) SegmentStart; /*< it will just be read there... */
3808         StaticBuf.BufUsed = SegmentEnd - SegmentStart;
3809         StaticBuf.BufSize = DecodeMe->BufSize - (SegmentStart - DecodeMe->buf);
3810         extract_token(charset, SegmentStart, 1, '?', sizeof charset);
3811         if (FoundCharset != NULL) {
3812                 FlushStrBuf(FoundCharset);
3813                 StrBufAppendBufPlain(FoundCharset, charset, -1, 0);
3814         }
3815         extract_token(encoding, SegmentStart, 2, '?', sizeof encoding);
3816         StrBufExtract_token(ConvertBuf, &StaticBuf, 3, '?');
3817
3818         *encoding = toupper(*encoding);
3819         if (*encoding == 'B') { /**< base64 */
3820                 if (ConvertBuf2->BufSize < ConvertBuf->BufUsed)
3821                         IncreaseBuf(ConvertBuf2, 0, ConvertBuf->BufUsed);
3822                 ConvertBuf2->BufUsed = CtdlDecodeBase64(ConvertBuf2->buf,
3823                                                         ConvertBuf->buf,
3824                                                         ConvertBuf->BufUsed);
3825         }
3826         else if (*encoding == 'Q') {    /**< quoted-printable */
3827                 long pos;
3828
3829                 pos = 0;
3830                 while (pos < ConvertBuf->BufUsed)
3831                 {
3832                         if (ConvertBuf->buf[pos] == '_')
3833                                 ConvertBuf->buf[pos] = ' ';
3834                         pos++;
3835                 }
3836
3837                 if (ConvertBuf2->BufSize < ConvertBuf->BufUsed)
3838                         IncreaseBuf(ConvertBuf2, 0, ConvertBuf->BufUsed);
3839
3840                 ConvertBuf2->BufUsed = CtdlDecodeQuotedPrintable(
3841                         ConvertBuf2->buf,
3842                         ConvertBuf->buf,
3843                         ConvertBuf->BufUsed);
3844         }
3845         else {
3846                 StrBufAppendBuf(ConvertBuf2, ConvertBuf, 0);
3847         }
3848 #ifdef HAVE_ICONV
3849         ctdl_iconv_open("UTF-8", charset, &ic);
3850         if (ic != (iconv_t)(-1) ) {
3851 #endif
3852                 StrBufConvert(ConvertBuf2, ConvertBuf, &ic);
3853                 StrBufAppendBuf(Target, ConvertBuf2, 0);
3854 #ifdef HAVE_ICONV
3855                 iconv_close(ic);
3856         }
3857         else {
3858                 StrBufAppendBufPlain(Target, HKEY("(unreadable)"), 0);
3859         }
3860 #endif
3861 }
3862
3863 /**
3864  * @ingroup StrBuf_DeEnCoder
3865  * @brief Handle subjects with RFC2047 encoding such as: [deprecated old syntax!]
3866  * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
3867  * @param Target where to put the decoded string to
3868  * @param DecodeMe buffer with encoded string
3869  * @param DefaultCharset if we don't find one, which should we use?
3870  * @param FoundCharset overrides DefaultCharset if non-empty; If we find a charset inside of the string,
3871  *        put it here for later use where no string might be known.
3872  */
3873 void StrBuf_RFC822_to_Utf8(StrBuf *Target, const StrBuf *DecodeMe, const StrBuf* DefaultCharset, StrBuf *FoundCharset)
3874 {
3875         StrBuf *ConvertBuf;
3876         StrBuf *ConvertBuf2;
3877         ConvertBuf = NewStrBufPlain(NULL, StrLength(DecodeMe));
3878         ConvertBuf2 = NewStrBufPlain(NULL, StrLength(DecodeMe));
3879
3880         StrBuf_RFC822_2_Utf8(Target,
3881                              DecodeMe,
3882                              DefaultCharset,
3883                              FoundCharset,
3884                              ConvertBuf,
3885                              ConvertBuf2);
3886         FreeStrBuf(&ConvertBuf);
3887         FreeStrBuf(&ConvertBuf2);
3888 }
3889
3890 /**
3891  * @ingroup StrBuf_DeEnCoder
3892  * @brief Handle subjects with RFC2047 encoding such as:
3893  * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
3894  * @param Target where to put the decoded string to
3895  * @param DecodeMe buffer with encoded string
3896  * @param DefaultCharset if we don't find one, which should we use?
3897  * @param FoundCharset overrides DefaultCharset if non-empty; If we find a charset inside of the string,
3898  *        put it here for later use where no string might be known.
3899  * @param ConvertBuf workbuffer. feed in, you shouldn't care about its content.
3900  * @param ConvertBuf2 workbuffer. feed in, you shouldn't care about its content.
3901  */
3902 void StrBuf_RFC822_2_Utf8(StrBuf *Target,
3903                           const StrBuf *DecodeMe,
3904                           const StrBuf* DefaultCharset,
3905                           StrBuf *FoundCharset,
3906                           StrBuf *ConvertBuf,
3907                           StrBuf *ConvertBuf2)
3908 {
3909         StrBuf *DecodedInvalidBuf = NULL;
3910         const StrBuf *DecodeMee = DecodeMe;
3911         const char *start, *end, *next, *nextend, *ptr = NULL;
3912 #ifdef HAVE_ICONV
3913         iconv_t ic = (iconv_t)(-1) ;
3914 #endif
3915         const char *eptr;
3916         int passes = 0;
3917         int i;
3918         int illegal_non_rfc2047_encoding = 0;
3919
3920
3921         if (DecodeMe == NULL)
3922                 return;
3923         /* Sometimes, badly formed messages contain strings which were simply
3924          *  written out directly in some foreign character set instead of
3925          *  using RFC2047 encoding.  This is illegal but we will attempt to
3926          *  handle it anyway by converting from a user-specified default
3927          *  charset to UTF-8 if we see any nonprintable characters.
3928          */
3929
3930         for (i=0; i<DecodeMe->BufUsed; ++i) {
3931                 if ((DecodeMe->buf[i] < 32) || (DecodeMe->buf[i] > 126)) {
3932                         illegal_non_rfc2047_encoding = 1;
3933                         break;
3934                 }
3935         }
3936
3937         if ((illegal_non_rfc2047_encoding) &&
3938             (strcasecmp(ChrPtr(DefaultCharset), "UTF-8")) &&
3939             (strcasecmp(ChrPtr(DefaultCharset), "us-ascii")) )
3940         {
3941 #ifdef HAVE_ICONV
3942                 ctdl_iconv_open("UTF-8", ChrPtr(DefaultCharset), &ic);
3943                 if (ic != (iconv_t)(-1) ) {
3944                         DecodedInvalidBuf = NewStrBufDup(DecodeMe);
3945                         StrBufConvert(DecodedInvalidBuf, ConvertBuf, &ic);///TODO: don't void const?
3946                         DecodeMee = DecodedInvalidBuf;
3947                         iconv_close(ic);
3948                 }
3949 #endif
3950         }
3951
3952         /* pre evaluate the first pair */
3953         end = NULL;
3954         start = strstr(DecodeMee->buf, "=?");
3955         eptr = DecodeMee->buf + DecodeMee->BufUsed;
3956         if (start != NULL)
3957                 end = FindNextEnd (DecodeMee, start + 2);
3958         else {
3959                 StrBufAppendBuf(Target, DecodeMee, 0);
3960                 FreeStrBuf(&DecodedInvalidBuf);
3961                 return;
3962         }
3963
3964
3965         if (start != DecodeMee->buf) {
3966                 long nFront;
3967
3968                 nFront = start - DecodeMee->buf;
3969                 StrBufAppendBufPlain(Target, DecodeMee->buf, nFront, 0);
3970         }
3971         /*
3972          * Since spammers will go to all sorts of absurd lengths to get their
3973          * messages through, there are LOTS of corrupt headers out there.
3974          * So, prevent a really badly formed RFC2047 header from throwing
3975          * this function into an infinite loop.
3976          */
3977         while ((start != NULL) &&
3978                (end != NULL) &&
3979                (start < eptr) &&
3980                (end < eptr) &&
3981                (passes < 20))
3982         {
3983                 passes++;
3984                 DecodeSegment(Target,
3985                               DecodeMee,
3986                               start,
3987                               end,
3988                               ConvertBuf,
3989                               ConvertBuf2,
3990                               FoundCharset);
3991
3992                 next = strstr(end, "=?");
3993                 nextend = NULL;
3994                 if ((next != NULL) &&
3995                     (next < eptr))
3996                         nextend = FindNextEnd(DecodeMee, next);
3997                 if (nextend == NULL)
3998                         next = NULL;
3999
4000                 /* did we find two partitions */
4001                 if ((next != NULL) &&
4002                     ((next - end) > 2))
4003                 {
4004                         ptr = end + 2;
4005                         while ((ptr < next) &&
4006                                (isspace(*ptr) ||
4007                                 (*ptr == '\r') ||
4008                                 (*ptr == '\n') ||
4009                                 (*ptr == '\t')))
4010                                 ptr ++;
4011                         /*
4012                          * did we find a gab just filled with blanks?
4013                          * if not, copy its stuff over.
4014                          */
4015                         if (ptr != next)
4016                         {
4017                                 StrBufAppendBufPlain(Target,
4018                                                      end + 2,
4019                                                      next - end - 2,
4020                                                      0);
4021                         }
4022                 }
4023                 /* our next-pair is our new first pair now. */
4024                 ptr = end + 2;
4025                 start = next;
4026                 end = nextend;
4027         }
4028         end = ptr;
4029         nextend = DecodeMee->buf + DecodeMee->BufUsed;
4030         if ((end != NULL) && (end < nextend)) {
4031                 ptr = end;
4032                 while ( (ptr < nextend) &&
4033                         (isspace(*ptr) ||
4034                          (*ptr == '\r') ||
4035                          (*ptr == '\n') ||
4036                          (*ptr == '\t')))
4037                         ptr ++;
4038                 if (ptr < nextend)
4039                         StrBufAppendBufPlain(Target, end, nextend - end, 0);
4040         }
4041         FreeStrBuf(&DecodedInvalidBuf);
4042 }
4043
4044 /*******************************************************************************
4045  *                   Manipulating UTF-8 Strings                                *
4046  *******************************************************************************/
4047
4048 /**
4049  * @ingroup StrBuf
4050  * @brief evaluate the length of an utf8 special character sequence
4051  * @param Char the character to examine
4052  * @returns width of utf8 chars in bytes; if the sequence is broken 0 is returned; 1 if its simply ASCII.
4053  */
4054 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE)
4055 {
4056         int n = 0;
4057         unsigned char test = (1<<7);
4058
4059         if ((*CharS & 0xC0) != 0xC0)
4060                 return 1;
4061
4062         while ((n < 8) &&
4063                ((test & ((unsigned char)*CharS)) != 0))
4064         {
4065                 test = test >> 1;
4066                 n ++;
4067         }
4068         if ((n > 6) || ((CharE - CharS) < n))
4069                 n = 0;
4070         return n;
4071 }
4072
4073 /**
4074  * @ingroup StrBuf
4075  * @brief detect whether this char starts an utf-8 encoded char
4076  * @param Char character to inspect
4077  * @returns yes or no
4078  */
4079 static inline int Ctdl_IsUtf8SequenceStart(const char Char)
4080 {
4081 /** 11??.???? indicates an UTF8 Sequence. */
4082         return ((Char & 0xC0) == 0xC0);
4083 }
4084
4085 /**
4086  * @ingroup StrBuf
4087  * @brief measure the number of glyphs in an UTF8 string...
4088  * @param Buf string to measure
4089  * @returns the number of glyphs in Buf
4090  */
4091 long StrBuf_Utf8StrLen(StrBuf *Buf)
4092 {
4093         int n = 0;
4094         int m = 0;
4095         char *aptr, *eptr;
4096
4097         if ((Buf == NULL) || (Buf->BufUsed == 0))
4098                 return 0;
4099         aptr = Buf->buf;
4100         eptr = Buf->buf + Buf->BufUsed;
4101         while ((aptr < eptr) && (*aptr != '\0')) {
4102                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
4103                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
4104                         while ((aptr < eptr) && (*aptr++ != '\0')&& (m-- > 0) );
4105                         n ++;
4106                 }
4107                 else {
4108                         n++;
4109                         aptr++;
4110                 }
4111         }
4112         return n;
4113 }
4114
4115 /**
4116  * @ingroup StrBuf
4117  * @brief cuts a string after maxlen glyphs
4118  * @param Buf string to cut to maxlen glyphs
4119  * @param maxlen how long may the string become?
4120  * @returns current length of the string
4121  */
4122 long StrBuf_Utf8StrCut(StrBuf *Buf, int maxlen)
4123 {
4124         char *aptr, *eptr;
4125         int n = 0, m = 0;
4126
4127         aptr = Buf->buf;
4128         eptr = Buf->buf + Buf->BufUsed;
4129         while ((aptr < eptr) && (*aptr != '\0')) {
4130                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
4131                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
4132                         while ((*aptr++ != '\0') && (m-- > 0));
4133                         n ++;
4134                 }
4135                 else {
4136                         n++;
4137                         aptr++;
4138                 }
4139                 if (n > maxlen) {
4140                         *aptr = '\0';
4141                         Buf->BufUsed = aptr - Buf->buf;
4142                         return Buf->BufUsed;
4143                 }
4144         }
4145         return Buf->BufUsed;
4146
4147 }
4148
4149
4150
4151
4152
4153 /*******************************************************************************
4154  *                               wrapping ZLib                                 *
4155  *******************************************************************************/
4156
4157 /**
4158  * @ingroup StrBuf_DeEnCoder
4159  * @brief uses the same calling syntax as compress2(), but it
4160  *   creates a stream compatible with HTTP "Content-encoding: gzip"
4161  * @param dest compressed buffer
4162  * @param destLen length of the compresed data
4163  * @param source source to encode
4164  * @param sourceLen length of source to encode
4165  * @param level compression level
4166  */
4167 #ifdef HAVE_ZLIB
4168 int ZEXPORT compress_gzip(Bytef * dest,
4169                           size_t * destLen,
4170                           const Bytef * source,
4171                           uLong sourceLen,
4172                           int level)
4173 {
4174         /* write gzip header */
4175         snprintf((char *) dest, *destLen,
4176                  "%c%c%c%c%c%c%c%c%c%c",
4177                  gz_magic[0], gz_magic[1], Z_DEFLATED,
4178                  0 /*flags */ , 0, 0, 0, 0 /*time */ , 0 /* xflags */ ,
4179                  OS_CODE);
4180
4181         /* normal deflate */
4182         z_stream stream;
4183         int err;
4184         stream.next_in = (Bytef *) source;
4185         stream.avail_in = (uInt) sourceLen;
4186         stream.next_out = dest + 10L;   // after header
4187         stream.avail_out = (uInt) * destLen;
4188         if ((uLong) stream.avail_out != *destLen)
4189                 return Z_BUF_ERROR;
4190
4191         stream.zalloc = (alloc_func) 0;
4192         stream.zfree = (free_func) 0;
4193         stream.opaque = (voidpf) 0;
4194
4195         err = deflateInit2(&stream, level, Z_DEFLATED, -MAX_WBITS,
4196                            DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
4197         if (err != Z_OK)
4198                 return err;
4199
4200         err = deflate(&stream, Z_FINISH);
4201         if (err != Z_STREAM_END) {
4202                 deflateEnd(&stream);
4203                 return err == Z_OK ? Z_BUF_ERROR : err;
4204         }
4205         *destLen = stream.total_out + 10L;
4206
4207         /* write CRC and Length */
4208         uLong crc = crc32(0L, source, sourceLen);
4209         int n;
4210         for (n = 0; n < 4; ++n, ++*destLen) {
4211                 dest[*destLen] = (int) (crc & 0xff);
4212                 crc >>= 8;
4213         }
4214         uLong len = stream.total_in;
4215         for (n = 0; n < 4; ++n, ++*destLen) {
4216                 dest[*destLen] = (int) (len & 0xff);
4217                 len >>= 8;
4218         }
4219         err = deflateEnd(&stream);
4220         return err;
4221 }
4222 #endif
4223
4224
4225 /**
4226  * @ingroup StrBuf_DeEnCoder
4227  * @brief compress the buffer with gzip
4228  * Attention! If you feed this a Const String, you must maintain the uncompressed buffer yourself!
4229  * @param Buf buffer whose content is to be gzipped
4230  */
4231 int CompressBuffer(StrBuf *Buf)
4232 {
4233 #ifdef HAVE_ZLIB
4234         char *compressed_data = NULL;
4235         size_t compressed_len, bufsize;
4236         int i = 0;
4237
4238         bufsize = compressed_len = Buf->BufUsed +  (Buf->BufUsed / 100) + 100;
4239         compressed_data = malloc(compressed_len);
4240
4241         if (compressed_data == NULL)
4242                 return -1;
4243         /* Flush some space after the used payload so valgrind shuts up... */
4244         while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
4245                 Buf->buf[Buf->BufUsed + i++] = '\0';
4246         if (compress_gzip((Bytef *) compressed_data,
4247                           &compressed_len,
4248                           (Bytef *) Buf->buf,
4249                           (uLongf) Buf->BufUsed, Z_BEST_SPEED) == Z_OK) {
4250                 if (!Buf->ConstBuf)
4251                         free(Buf->buf);
4252                 Buf->buf = compressed_data;
4253                 Buf->BufUsed = compressed_len;
4254                 Buf->BufSize = bufsize;
4255                 /* Flush some space after the used payload so valgrind shuts up... */
4256                 i = 0;
4257                 while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
4258                         Buf->buf[Buf->BufUsed + i++] = '\0';
4259                 return 1;
4260         } else {
4261                 free(compressed_data);
4262         }
4263 #endif  /* HAVE_ZLIB */
4264         return 0;
4265 }
4266
4267 /*******************************************************************************
4268  *           File I/O; Callbacks to libevent                                   *
4269  *******************************************************************************/
4270
4271 long StrBuf_read_one_chunk_callback (int fd, short event, IOBuffer *FB)
4272 {
4273         long bufremain = 0;
4274         int n;
4275
4276         if ((FB == NULL) || (FB->Buf == NULL))
4277                 return -1;
4278
4279         /*
4280          * check whether the read pointer is somewhere in a range
4281          * where a cut left is inexpensive
4282          */
4283
4284         if (FB->ReadWritePointer != NULL)
4285         {
4286                 long already_read;
4287
4288                 already_read = FB->ReadWritePointer - FB->Buf->buf;
4289                 bufremain = FB->Buf->BufSize - FB->Buf->BufUsed - 1;
4290
4291                 if (already_read != 0) {
4292                         long unread;
4293
4294                         unread = FB->Buf->BufUsed - already_read;
4295
4296                         /* else nothing to compact... */
4297                         if (unread == 0) {
4298                                 FB->ReadWritePointer = FB->Buf->buf;
4299                                 bufremain = FB->Buf->BufSize;
4300                         }
4301                         else if ((unread < 64) ||
4302                                  (bufremain < already_read))
4303                         {
4304                                 /*
4305                                  * if its just a tiny bit remaining, or we run out of space...
4306                                  * lets tidy up.
4307                                  */
4308                                 FB->Buf->BufUsed = unread;
4309                                 if (unread < already_read)
4310                                         memcpy(FB->Buf->buf, FB->ReadWritePointer, unread);
4311                                 else
4312                                         memmove(FB->Buf->buf, FB->ReadWritePointer, unread);
4313                                 FB->ReadWritePointer = FB->Buf->buf;
4314                                 bufremain = FB->Buf->BufSize - unread - 1;
4315                         }
4316                         else if (bufremain < (FB->Buf->BufSize / 10))
4317                         {
4318                                 /* get a bigger buffer */
4319
4320                                 IncreaseBuf(FB->Buf, 0, FB->Buf->BufUsed + 1);
4321
4322                                 FB->ReadWritePointer = FB->Buf->buf + unread;
4323
4324                                 bufremain = FB->Buf->BufSize - unread - 1;
4325 /*TODO: special increase function that won't copy the already read! */
4326                         }
4327                 }
4328                 else if (bufremain < 10) {
4329                         IncreaseBuf(FB->Buf, 1, FB->Buf->BufUsed + 10);
4330
4331                         FB->ReadWritePointer = FB->Buf->buf;
4332
4333                         bufremain = FB->Buf->BufSize - FB->Buf->BufUsed - 1;
4334                 }
4335
4336         }
4337         else {
4338                 FB->ReadWritePointer = FB->Buf->buf;
4339                 bufremain = FB->Buf->BufSize - 1;
4340         }
4341
4342         n = read(fd, FB->Buf->buf + FB->Buf->BufUsed, bufremain);
4343
4344         if (n > 0) {
4345                 FB->Buf->BufUsed += n;
4346                 FB->Buf->buf[FB->Buf->BufUsed] = '\0';
4347         }
4348         return n;
4349 }
4350
4351 int StrBuf_write_one_chunk_callback(int fd, short event, IOBuffer *FB)
4352 {
4353         long WriteRemain;
4354         int n;
4355
4356         if ((FB == NULL) || (FB->Buf == NULL))
4357                 return -1;
4358
4359         if (FB->ReadWritePointer != NULL)
4360         {
4361                 WriteRemain = FB->Buf->BufUsed -
4362                         (FB->ReadWritePointer -
4363                          FB->Buf->buf);
4364         }
4365         else {
4366                 FB->ReadWritePointer = FB->Buf->buf;
4367                 WriteRemain = FB->Buf->BufUsed;
4368         }
4369
4370         n = write(fd, FB->ReadWritePointer, WriteRemain);
4371         if (n > 0) {
4372                 FB->ReadWritePointer += n;
4373
4374                 if (FB->ReadWritePointer ==
4375                     FB->Buf->buf + FB->Buf->BufUsed)
4376                 {
4377                         FlushStrBuf(FB->Buf);
4378                         FB->ReadWritePointer = NULL;
4379                         return 0;
4380                 }
4381         // check whether we've got something to write
4382         // get the maximum chunk plus the pointer we can send
4383         // write whats there
4384         // if not all was sent, remember the send pointer for the next time
4385                 return FB->ReadWritePointer - FB->Buf->buf + FB->Buf->BufUsed;
4386         }
4387         return n;
4388 }
4389
4390 /**
4391  * @ingroup StrBuf_IO
4392  * @brief extract a "next line" from Buf; Ptr to persist across several iterations
4393  * @param LineBuf your line will be copied here.
4394  * @param FB BLOB with lines of text...
4395  * @param Ptr moved arround to keep the next-line across several iterations
4396  *        has to be &NULL on start; will be &NotNULL on end of buffer
4397  * @returns size of copied buffer
4398  */
4399 eReadState StrBufChunkSipLine(StrBuf *LineBuf, IOBuffer *FB)
4400 {
4401         const char *aptr, *ptr, *eptr;
4402         char *optr, *xptr;
4403
4404         if ((FB == NULL) || (LineBuf == NULL) || (LineBuf->buf == NULL))
4405                 return eReadFail;
4406
4407
4408         if ((FB->Buf == NULL) || (FB->ReadWritePointer == StrBufNOTNULL)) {
4409                 FB->ReadWritePointer = StrBufNOTNULL;
4410                 return eReadFail;
4411         }
4412
4413         FlushStrBuf(LineBuf);
4414         if (FB->ReadWritePointer == NULL)
4415                 ptr = aptr = FB->Buf->buf;
4416         else
4417                 ptr = aptr = FB->ReadWritePointer;
4418
4419         optr = LineBuf->buf;
4420         eptr = FB->Buf->buf + FB->Buf->BufUsed;
4421         xptr = LineBuf->buf + LineBuf->BufSize - 1;
4422
4423         while ((ptr <= eptr) &&
4424                (*ptr != '\n') &&
4425                (*ptr != '\r') )
4426         {
4427                 *optr = *ptr;
4428                 optr++; ptr++;
4429                 if (optr == xptr) {
4430                         LineBuf->BufUsed = optr - LineBuf->buf;
4431                         IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
4432                         optr = LineBuf->buf + LineBuf->BufUsed;
4433                         xptr = LineBuf->buf + LineBuf->BufSize - 1;
4434                 }
4435         }
4436
4437         if (ptr >= eptr) {
4438                 if (optr > LineBuf->buf)
4439                         optr --;
4440                 if ((*(ptr - 1) != '\r') && (*(ptr - 1) != '\n')) {
4441                         LineBuf->BufUsed = optr - LineBuf->buf;
4442                         *optr = '\0';
4443                         if ((FB->ReadWritePointer != NULL) &&
4444                             (FB->ReadWritePointer != FB->Buf->buf))
4445                         {
4446                                 /* Ok, the client application read all the data
4447                                    it was interested in so far. Since there is more to read,
4448                                    we now shrink the buffer, and move the rest over.
4449                                 */
4450                                 StrBufCutLeft(FB->Buf,
4451                                               FB->ReadWritePointer - FB->Buf->buf);
4452                                 FB->ReadWritePointer = FB->Buf->buf;
4453                         }
4454                         return eMustReadMore;
4455                 }
4456         }
4457         LineBuf->BufUsed = optr - LineBuf->buf;
4458         *optr = '\0';
4459         if ((ptr <= eptr) && (*ptr == '\r'))
4460                 ptr ++;
4461         if ((ptr <= eptr) && (*ptr == '\n'))
4462                 ptr ++;
4463
4464         if (ptr < eptr) {
4465                 FB->ReadWritePointer = ptr;
4466         }
4467         else {
4468                 FlushStrBuf(FB->Buf);
4469                 FB->ReadWritePointer = NULL;
4470         }
4471
4472         return eReadSuccess;
4473 }
4474
4475 /**
4476  * @ingroup StrBuf_CHUNKED_IO
4477  * @brief check whether the chunk-buffer has more data waiting or not.
4478  * @param FB Chunk-Buffer to inspect
4479  */
4480 eReadState StrBufCheckBuffer(IOBuffer *FB)
4481 {
4482         if (FB == NULL)
4483                 return eReadFail;
4484         if (FB->Buf->BufUsed == 0)
4485                 return eReadSuccess;
4486         if (FB->ReadWritePointer == NULL)
4487                 return eBufferNotEmpty;
4488         if (FB->Buf->buf + FB->Buf->BufUsed > FB->ReadWritePointer)
4489                 return eBufferNotEmpty;
4490         return eReadSuccess;
4491 }
4492
4493 long IOBufferStrLength(IOBuffer *FB)
4494 {
4495         if ((FB == NULL) || (FB->Buf == NULL))
4496                 return 0;
4497         if (FB->ReadWritePointer == NULL)
4498                 return StrLength(FB->Buf);
4499
4500         return StrLength(FB->Buf) - (FB->ReadWritePointer - FB->Buf->buf);
4501 }
4502
4503 inline static void FDIOBufferFlush(FDIOBuffer *FDB)
4504 {
4505         memset(FDB, 0, sizeof(FDIOBuffer));
4506         FDB->OtherFD = -1;
4507         FDB->SplicePipe[0] = -1;
4508         FDB->SplicePipe[1] = -1;
4509 }
4510
4511 void FDIOBufferInit(FDIOBuffer *FDB, IOBuffer *IO, int FD, long TotalSendSize)
4512 {
4513         FDIOBufferFlush(FDB);
4514
4515         FDB->TotalSendSize = TotalSendSize;
4516         if (TotalSendSize > 0)
4517                 FDB->ChunkSize = TotalSendSize;
4518         else
4519         {
4520                 TotalSendSize = SIZ * 10;
4521                 FDB->ChunkSize = TotalSendSize;
4522         }
4523         FDB->IOB = IO;
4524
4525 #ifdef LINUX_SPLICE
4526         if (EnableSplice)
4527                 pipe(FDB->SplicePipe);
4528         else
4529 #endif
4530                 FDB->ChunkBuffer = NewStrBufPlain(NULL, TotalSendSize+ 1);
4531
4532         FDB->OtherFD = FD;
4533 }
4534
4535 void FDIOBufferDelete(FDIOBuffer *FDB)
4536 {
4537 #ifdef LINUX_SPLICE
4538         if (EnableSplice)
4539         {
4540                 if (FDB->SplicePipe[0] > 0)
4541                         close(FDB->SplicePipe[0]);
4542                 if (FDB->SplicePipe[1] > 0)
4543                         close(FDB->SplicePipe[1]);
4544         }
4545         else
4546 #endif
4547                 FreeStrBuf(&FDB->ChunkBuffer);
4548
4549         if (FDB->OtherFD > 0)
4550                 close(FDB->OtherFD);
4551         FDIOBufferFlush(FDB);
4552 }
4553
4554 int FileSendChunked(FDIOBuffer *FDB, const char **Err)
4555 {
4556         ssize_t sent, pipesize;
4557
4558         if (FDB->TotalSendSize > 0)
4559         {
4560 #ifdef LINUX_SPLICE
4561                 if (EnableSplice)
4562                 {
4563                         if (FDB->PipeSize == 0)
4564                         {
4565                                 pipesize = splice(FDB->OtherFD,
4566                                                   &FDB->TotalSentAlready,
4567                                                   FDB->SplicePipe[1],
4568                                                   NULL,
4569                                                   FDB->ChunkSendRemain,
4570                                                   SPLICE_F_MOVE);
4571
4572                                 if (pipesize == -1)
4573                                 {
4574                                         *Err = strerror(errno);
4575                                         return pipesize;
4576                                 }
4577                                 FDB->PipeSize = pipesize;
4578                         }
4579                         sent =  splice(FDB->SplicePipe[0],
4580                                        NULL,
4581                                        FDB->IOB->fd,
4582                                        NULL,
4583                                        FDB->PipeSize,
4584                                        SPLICE_F_MORE | SPLICE_F_MOVE | SPLICE_F_NONBLOCK);
4585                         if (sent == -1)
4586                         {
4587                                 *Err = strerror(errno);
4588                                 return sent;
4589                         }
4590                         FDB->PipeSize -= sent;
4591                         FDB->ChunkSendRemain -= sent;
4592                         return sent;
4593                 }
4594                 else
4595 #endif
4596                 {
4597                         char *pRead;
4598                         long nRead = 0;
4599
4600                         pRead = FDB->ChunkBuffer->buf;
4601                         while ((FDB->ChunkBuffer->BufUsed < FDB->TotalSendSize) && (nRead >= 0))
4602                         {
4603                                 nRead = read(FDB->OtherFD, pRead, FDB->TotalSendSize - FDB->ChunkBuffer->BufUsed);
4604                                 if (nRead > 0) {
4605                                         FDB->ChunkBuffer->BufUsed += nRead;
4606                                         FDB->ChunkBuffer->buf[FDB->ChunkBuffer->BufUsed] = '\0';
4607                                 }
4608                                 else if (nRead == 0) {}
4609                                 else return nRead;
4610                         }
4611
4612                         nRead = write(FDB->IOB->fd,
4613                                       FDB->ChunkBuffer->buf     + FDB->TotalSentAlready,
4614                                       FDB->ChunkBuffer->BufUsed - FDB->TotalSentAlready);
4615
4616                         if (nRead >= 0) {
4617                                 FDB->TotalSentAlready += nRead;
4618                                 FDB->ChunkSendRemain -= nRead;
4619                                 return FDB->ChunkSendRemain;
4620                         }
4621                         else {
4622                                 return nRead;
4623                         }
4624                 }
4625         }
4626         else
4627         {
4628 #ifdef LINUX_SPLICE
4629                 if (EnableSplice)
4630                 {
4631                         if (FDB->PipeSize == 0)
4632                         {
4633                                 pipesize = splice(FDB->OtherFD,
4634                                                   &FDB->TotalSentAlready,
4635                                                   FDB->SplicePipe[1],
4636                                                   NULL,
4637                                                   SIZ * 10,
4638                                                   SPLICE_F_MOVE);
4639
4640                                 if (pipesize == -1)
4641                                 {
4642                                         *Err = strerror(errno);
4643                                         return pipesize;
4644                                 }
4645                                 FDB->PipeSize = pipesize;
4646                                 if (pipesize == 0)
4647                                         return -1;
4648                         }
4649                         sent =  splice(FDB->SplicePipe[0],
4650                                        NULL,
4651                                        FDB->IOB->fd,
4652                                        NULL,
4653                                        FDB->PipeSize,
4654                                        SPLICE_F_MORE | SPLICE_F_MOVE | SPLICE_F_NONBLOCK);
4655                         if (sent == -1)
4656                         {
4657                                 *Err = strerror(errno);
4658                                 return sent;
4659                         }
4660                         FDB->PipeSize -= sent;
4661                         FDB->ChunkSendRemain -= sent;
4662                         return sent;
4663                 }
4664                 else
4665 #endif
4666                 {
4667                         char *pRead;
4668                         long nRead = 0;
4669
4670                         pRead = FDB->ChunkBuffer->buf;
4671                         while ((FDB->ChunkSendRemain == 0) &&
4672                                (FDB->ChunkBuffer->BufUsed < FDB->ChunkBuffer->BufSize) &&
4673                                (nRead >= 0))
4674                         {
4675                                 FDB->TotalSentAlready = 0;
4676                                 nRead = read(FDB->OtherFD, pRead, FDB->ChunkBuffer->BufSize - FDB->ChunkBuffer->BufUsed);
4677                                 if (nRead > 0) {
4678                                         FDB->ChunkBuffer->BufUsed += nRead;
4679                                         FDB->ChunkBuffer->buf[FDB->ChunkBuffer->BufUsed] = '\0';
4680                                         FDB->ChunkSendRemain += nRead;
4681                                 }
4682                                 else if (nRead == 0)
4683                                 {
4684                                         return -1;
4685                                 }
4686                                 else
4687                                 {
4688                                         *Err = strerror(errno);
4689                                         return nRead;
4690                                 }
4691                         }
4692
4693                         nRead = write(FDB->IOB->fd,
4694                                       FDB->ChunkBuffer->buf     + FDB->TotalSentAlready,
4695                                       FDB->ChunkBuffer->BufUsed - FDB->TotalSentAlready);
4696
4697                         if (nRead >= 0) {
4698                                 FDB->TotalSentAlready += nRead;
4699                                 FDB->ChunkSendRemain -= nRead;
4700                                 if (FDB->ChunkSendRemain == 0)
4701                                 {
4702                                         FDB->ChunkBuffer->BufUsed = 0;
4703                                         FDB->TotalSentAlready = 0;
4704                                 }
4705                                 return FDB->ChunkSendRemain;
4706                         }
4707                         else {
4708                                 return nRead;
4709                         }
4710                 }
4711         }
4712 }
4713
4714 int FileRecvChunked(FDIOBuffer *FDB, const char **Err)
4715 {
4716         ssize_t sent, pipesize;
4717
4718 #ifdef LINUX_SPLICE
4719         if (EnableSplice)
4720         {
4721                 if (FDB->PipeSize == 0)
4722                 {
4723                         pipesize = splice(FDB->IOB->fd,
4724                                           NULL,
4725                                           FDB->SplicePipe[1],
4726                                           NULL,
4727                                           FDB->ChunkSendRemain,
4728                                           SPLICE_F_MORE | SPLICE_F_MOVE|SPLICE_F_NONBLOCK);
4729
4730                         if (pipesize == -1)
4731                         {
4732                                 *Err = strerror(errno);
4733                                 return pipesize;
4734                         }
4735                         FDB->PipeSize = pipesize;
4736                 }
4737
4738                 sent = splice(FDB->SplicePipe[0],
4739                               NULL,
4740                               FDB->OtherFD,
4741                               &FDB->TotalSentAlready,
4742                               FDB->PipeSize,
4743                               SPLICE_F_MORE | SPLICE_F_MOVE);
4744
4745                 if (sent == -1)
4746                 {
4747                         *Err = strerror(errno);
4748                         return sent;
4749                 }
4750                 FDB->PipeSize -= sent;
4751                 FDB->ChunkSendRemain -= sent;
4752                 return sent;
4753         }
4754         else
4755 #endif
4756         {
4757                 sent = read(FDB->IOB->fd, FDB->ChunkBuffer->buf, FDB->ChunkSendRemain);
4758                 if (sent > 0) {
4759                         int nWritten = 0;
4760                         int rc;
4761
4762                         FDB->ChunkBuffer->BufUsed = sent;
4763
4764                         while (nWritten < FDB->ChunkBuffer->BufUsed) {
4765                                 rc =  write(FDB->OtherFD, FDB->ChunkBuffer->buf + nWritten, FDB->ChunkBuffer->BufUsed - nWritten);
4766                                 if (rc < 0) {
4767                                         *Err = strerror(errno);
4768                                         return rc;
4769                                 }
4770                                 nWritten += rc;
4771
4772                         }
4773                         FDB->ChunkBuffer->BufUsed = 0;
4774                         FDB->TotalSentAlready += sent;
4775                         FDB->ChunkSendRemain -= sent;
4776                         return FDB->ChunkSendRemain;
4777                 }
4778                 else if (sent < 0) {
4779                         *Err = strerror(errno);
4780                         return sent;
4781                 }
4782                 return 0;
4783         }
4784 }
4785
4786 int FileMoveChunked(FDIOBuffer *FDB, const char **Err)
4787 {
4788         ssize_t sent, pipesize;
4789
4790 #ifdef LINUX_SPLICE
4791         if (EnableSplice)
4792         {
4793                 if (FDB->PipeSize == 0)
4794                 {
4795                         pipesize = splice(FDB->IOB->fd,
4796                                           &FDB->TotalReadAlready,
4797                                           FDB->SplicePipe[1],
4798                                           NULL,
4799                                           FDB->ChunkSendRemain,
4800                                           SPLICE_F_MORE | SPLICE_F_MOVE|SPLICE_F_NONBLOCK);
4801
4802                         if (pipesize == -1)
4803                         {
4804                                 *Err = strerror(errno);
4805                                 return pipesize;
4806                         }
4807                         FDB->PipeSize = pipesize;
4808                 }
4809
4810                 sent = splice(FDB->SplicePipe[0],
4811                               NULL,
4812                               FDB->OtherFD,
4813                               &FDB->TotalSentAlready,
4814                               FDB->PipeSize,
4815                               SPLICE_F_MORE | SPLICE_F_MOVE);
4816
4817                 if (sent == -1)
4818                 {
4819                         *Err = strerror(errno);
4820                         return sent;
4821                 }
4822                 FDB->PipeSize -= sent;
4823                 FDB->ChunkSendRemain -= sent;
4824                 return sent;
4825         }
4826         else
4827 #endif
4828         {
4829                 sent = read(FDB->IOB->fd, FDB->ChunkBuffer->buf, FDB->ChunkSendRemain);
4830                 if (sent > 0) {
4831                         int nWritten = 0;
4832                         int rc;
4833
4834                         FDB->ChunkBuffer->BufUsed = sent;
4835
4836                         while (nWritten < FDB->ChunkBuffer->BufUsed) {
4837                                 rc =  write(FDB->OtherFD, FDB->ChunkBuffer->buf + nWritten, FDB->ChunkBuffer->BufUsed - nWritten);
4838                                 if (rc < 0) {
4839                                         *Err = strerror(errno);
4840                                         return rc;
4841                                 }
4842                                 nWritten += rc;
4843
4844                         }
4845                         FDB->ChunkBuffer->BufUsed = 0;
4846                         FDB->TotalSentAlready += sent;
4847                         FDB->ChunkSendRemain -= sent;
4848                         return FDB->ChunkSendRemain;
4849                 }
4850                 else if (sent < 0) {
4851                         *Err = strerror(errno);
4852                         return sent;
4853                 }
4854                 return 0;
4855         }
4856 }
4857
4858 eReadState WriteIOBAlreadyRead(FDIOBuffer *FDB, const char **Error)
4859 {
4860         int IsNonBlock;
4861         int fdflags;
4862         long rlen;
4863         long should_write;
4864         int nSuccessLess = 0;
4865         struct timeval tv;
4866         fd_set rfds;
4867
4868         fdflags = fcntl(FDB->OtherFD, F_GETFL);
4869         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
4870
4871         while ((FDB->IOB->ReadWritePointer - FDB->IOB->Buf->buf < FDB->IOB->Buf->BufUsed) &&
4872                (FDB->ChunkSendRemain > 0))
4873         {
4874                 if (IsNonBlock){
4875                         tv.tv_sec = 1; /* selectresolution; */
4876                         tv.tv_usec = 0;
4877
4878                         FD_ZERO(&rfds);
4879                         FD_SET(FDB->OtherFD, &rfds);
4880                         if (select(FDB->OtherFD + 1, NULL, &rfds, NULL, &tv) == -1) {
4881                                 *Error = strerror(errno);
4882                                 return eReadFail;
4883                         }
4884                 }
4885                 if (IsNonBlock && !  FD_ISSET(FDB->OtherFD, &rfds)) {
4886                         nSuccessLess ++;
4887                         continue;
4888                 }
4889
4890                 should_write = FDB->IOB->Buf->BufUsed -
4891                         (FDB->IOB->ReadWritePointer - FDB->IOB->Buf->buf);
4892                 if (should_write > FDB->ChunkSendRemain)
4893                         should_write = FDB->ChunkSendRemain;
4894
4895                 rlen = write(FDB->OtherFD,
4896                              FDB->IOB->ReadWritePointer,
4897                              should_write);
4898                 if (rlen < 1) {
4899                         *Error = strerror(errno);
4900
4901                         return eReadFail;
4902                 }
4903                 FDB->TotalSentAlready += rlen;
4904                 FDB->IOB->ReadWritePointer += rlen;
4905                 FDB->ChunkSendRemain -= rlen;
4906         }
4907         if (FDB->IOB->ReadWritePointer >= FDB->IOB->Buf->buf + FDB->IOB->Buf->BufUsed)
4908         {
4909                 FlushStrBuf(FDB->IOB->Buf);
4910                 FDB->IOB->ReadWritePointer = NULL;
4911         }
4912
4913         if (FDB->ChunkSendRemain == 0)
4914                 return eReadSuccess;
4915         else
4916                 return eMustReadMore;
4917 }
4918
4919 /*******************************************************************************
4920  *           File I/O; Prefer buffered read since its faster!                  *
4921  *******************************************************************************/
4922
4923 /**
4924  * @ingroup StrBuf_IO
4925  * @brief Read a line from socket
4926  * flushes and closes the FD on error
4927  * @param buf the buffer to get the input to
4928  * @param fd pointer to the filedescriptor to read
4929  * @param append Append to an existing string or replace?
4930  * @param Error strerror() on error
4931  * @returns numbers of chars read
4932  */
4933 int StrBufTCP_read_line(StrBuf *buf, int *fd, int append, const char **Error)
4934 {
4935         int len, rlen, slen;
4936
4937         if ((buf == NULL) || (buf->buf == NULL)) {
4938                 *Error = strerror(EINVAL);
4939                 return -1;
4940         }
4941
4942         if (!append)
4943                 FlushStrBuf(buf);
4944
4945         slen = len = buf->BufUsed;
4946         while (1) {
4947                 rlen = read(*fd, &buf->buf[len], 1);
4948                 if (rlen < 1) {
4949                         *Error = strerror(errno);
4950
4951                         close(*fd);
4952                         *fd = -1;
4953
4954                         return -1;
4955                 }
4956                 if (buf->buf[len] == '\n')
4957                         break;
4958                 if (buf->buf[len] != '\r')
4959                         len ++;
4960                 if (len + 2 >= buf->BufSize) {
4961                         buf->BufUsed = len;
4962                         buf->buf[len+1] = '\0';
4963                         IncreaseBuf(buf, 1, -1);
4964                 }
4965         }
4966         buf->BufUsed = len;
4967         buf->buf[len] = '\0';
4968         return len - slen;
4969 }
4970
4971 /**
4972  * @ingroup StrBuf_BufferedIO
4973  * @brief Read a line from socket
4974  * flushes and closes the FD on error
4975  * @param Line the line to read from the fd / I/O Buffer
4976  * @param buf the buffer to get the input to
4977  * @param fd pointer to the filedescriptor to read
4978  * @param timeout number of successless selects until we bail out
4979  * @param selectresolution how long to wait on each select
4980  * @param Error strerror() on error
4981  * @returns numbers of chars read
4982  */
4983 int StrBufTCP_read_buffered_line(StrBuf *Line,
4984                                  StrBuf *buf,
4985                                  int *fd,
4986                                  int timeout,
4987                                  int selectresolution,
4988                                  const char **Error)
4989 {
4990         int len, rlen;
4991         int nSuccessLess = 0;
4992         fd_set rfds;
4993         char *pch = NULL;
4994         int fdflags;
4995         int IsNonBlock;
4996         struct timeval tv;
4997
4998         if (buf->BufUsed > 0) {
4999                 pch = strchr(buf->buf, '\n');
5000                 if (pch != NULL) {
5001                         rlen = 0;
5002                         len = pch - buf->buf;
5003                         if (len > 0 && (*(pch - 1) == '\r') )
5004                                 rlen ++;
5005                         StrBufSub(Line, buf, 0, len - rlen);
5006                         StrBufCutLeft(buf, len + 1);
5007                         return len - rlen;
5008                 }
5009         }
5010
5011         if (buf->BufSize - buf->BufUsed < 10)
5012                 IncreaseBuf(buf, 1, -1);
5013
5014         fdflags = fcntl(*fd, F_GETFL);
5015         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
5016
5017         while ((nSuccessLess < timeout) && (pch == NULL)) {
5018                 if (IsNonBlock){
5019                         tv.tv_sec = selectresolution;
5020                         tv.tv_usec = 0;
5021
5022                         FD_ZERO(&rfds);
5023                         FD_SET(*fd, &rfds);
5024                         if (select(*fd + 1, NULL, &rfds, NULL, &tv) == -1) {
5025                                 *Error = strerror(errno);
5026                                 close (*fd);
5027                                 *fd = -1;
5028                                 return -1;
5029                         }
5030                 }
5031                 if (IsNonBlock && !  FD_ISSET(*fd, &rfds)) {
5032                         nSuccessLess ++;
5033                         continue;
5034                 }
5035                 rlen = read(*fd,
5036                             &buf->buf[buf->BufUsed],
5037                             buf->BufSize - buf->BufUsed - 1);
5038                 if (rlen < 1) {
5039                         *Error = strerror(errno);
5040                         close(*fd);
5041                         *fd = -1;
5042                         return -1;
5043                 }
5044                 else if (rlen > 0) {
5045                         nSuccessLess = 0;
5046                         buf->BufUsed += rlen;
5047                         buf->buf[buf->BufUsed] = '\0';
5048                         pch = strchr(buf->buf, '\n');
5049                         if ((pch == NULL) &&
5050                             (buf->BufUsed + 10 > buf->BufSize) &&
5051                             (IncreaseBuf(buf, 1, -1) == -1))
5052                                 return -1;
5053                         continue;
5054                 }
5055
5056         }
5057         if (pch != NULL) {
5058                 rlen = 0;
5059                 len = pch - buf->buf;
5060                 if (len > 0 && (*(pch - 1) == '\r') )
5061                         rlen ++;
5062                 StrBufSub(Line, buf, 0, len - rlen);
5063                 StrBufCutLeft(buf, len + 1);
5064                 return len - rlen;
5065         }
5066         return -1;
5067
5068 }
5069
5070 static const char *ErrRBLF_PreConditionFailed="StrBufTCP_read_buffered_line_fast: Wrong arguments or invalid Filedescriptor";
5071 static const char *ErrRBLF_SelectFailed="StrBufTCP_read_buffered_line_fast: Select failed without reason";
5072 static const char *ErrRBLF_NotEnoughSentFromServer="StrBufTCP_read_buffered_line_fast: No complete line was sent from peer";
5073 /**
5074  * @ingroup StrBuf_BufferedIO
5075  * @brief Read a line from socket
5076  * flushes and closes the FD on error
5077  * @param Line where to append our Line read from the fd / I/O Buffer;
5078  * @param IOBuf the buffer to get the input to; lifetime pair to FD
5079  * @param Pos pointer to the current read position, should be NULL initialized on opening the FD it belongs to.!
5080  * @param fd pointer to the filedescriptor to read
5081  * @param timeout number of successless selects until we bail out
5082  * @param selectresolution how long to wait on each select
5083  * @param Error strerror() on error
5084  * @returns numbers of chars read or -1 in case of error. "\n" will become 0
5085  */
5086 int StrBufTCP_read_buffered_line_fast(StrBuf *Line,
5087                                       StrBuf *IOBuf,
5088                                       const char **Pos,
5089                                       int *fd,
5090                                       int timeout,
5091                                       int selectresolution,
5092                                       const char **Error)
5093 {
5094         const char *pche = NULL;
5095         const char *pos = NULL;
5096         const char *pLF;
5097         int len, rlen, retlen;
5098         int nSuccessLess = 0;
5099         fd_set rfds;
5100         const char *pch = NULL;
5101         int fdflags;
5102         int IsNonBlock;
5103         struct timeval tv;
5104
5105         retlen = 0;
5106         if ((Line == NULL) ||
5107             (Pos == NULL) ||
5108             (IOBuf == NULL) ||
5109             (*fd == -1))
5110         {
5111                 if (Pos != NULL)
5112                         *Pos = NULL;
5113                 *Error = ErrRBLF_PreConditionFailed;
5114                 return -1;
5115         }
5116
5117         pos = *Pos;
5118         if ((IOBuf->BufUsed > 0) &&
5119             (pos != NULL) &&
5120             (pos < IOBuf->buf + IOBuf->BufUsed))
5121         {
5122                 char *pcht;
5123
5124                 pche = IOBuf->buf + IOBuf->BufUsed;
5125                 pch = pos;
5126                 pcht = Line->buf;
5127
5128                 while ((pch < pche) && (*pch != '\n'))
5129                 {
5130                         if (Line->BufUsed + 10 > Line->BufSize)
5131                         {
5132                                 long apos;
5133                                 apos = pcht - Line->buf;
5134                                 *pcht = '\0';
5135                                 IncreaseBuf(Line, 1, -1);
5136                                 pcht = Line->buf + apos;
5137                         }
5138                         *pcht++ = *pch++;
5139                         Line->BufUsed++;
5140                         retlen++;
5141                 }
5142
5143                 len = pch - pos;
5144                 if (len > 0 && (*(pch - 1) == '\r') )
5145                 {
5146                         retlen--;
5147                         len --;
5148                         pcht --;
5149                         Line->BufUsed --;
5150                 }
5151                 *pcht = '\0';
5152
5153                 if ((pch >= pche) || (*pch == '\0'))
5154                 {
5155                         FlushStrBuf(IOBuf);
5156                         *Pos = NULL;
5157                         pch = NULL;
5158                         pos = 0;
5159                 }
5160
5161                 if ((pch != NULL) &&
5162                     (pch <= pche))
5163                 {
5164                         if (pch + 1 >= pche) {
5165                                 *Pos = NULL;
5166                                 FlushStrBuf(IOBuf);
5167                         }
5168                         else
5169                                 *Pos = pch + 1;
5170
5171                         return retlen;
5172                 }
5173                 else
5174                         FlushStrBuf(IOBuf);
5175         }
5176
5177         /* If we come here, Pos is Unset since we read everything into Line, and now go for more. */
5178
5179         if (IOBuf->BufSize - IOBuf->BufUsed < 10)
5180                 IncreaseBuf(IOBuf, 1, -1);
5181
5182         fdflags = fcntl(*fd, F_GETFL);
5183         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
5184
5185         pLF = NULL;
5186         while ((nSuccessLess < timeout) &&
5187                (pLF == NULL) &&
5188                (*fd != -1)) {
5189                 if (IsNonBlock)
5190                 {
5191                         tv.tv_sec = 1;
5192                         tv.tv_usec = 0;
5193
5194                         FD_ZERO(&rfds);
5195                         FD_SET(*fd, &rfds);
5196                         if (select((*fd) + 1, &rfds, NULL, NULL, &tv) == -1) {
5197                                 *Error = strerror(errno);
5198                                 close (*fd);
5199                                 *fd = -1;
5200                                 if (*Error == NULL)
5201                                         *Error = ErrRBLF_SelectFailed;
5202                                 return -1;
5203                         }
5204                         if (! FD_ISSET(*fd, &rfds) != 0) {
5205                                 nSuccessLess ++;
5206                                 continue;
5207                         }
5208                 }
5209                 rlen = read(*fd,
5210                             &IOBuf->buf[IOBuf->BufUsed],
5211                             IOBuf->BufSize - IOBuf->BufUsed - 1);
5212                 if (rlen < 1) {
5213                         *Error = strerror(errno);
5214                         close(*fd);
5215                         *fd = -1;
5216                         return -1;
5217                 }
5218                 else if (rlen > 0) {
5219                         nSuccessLess = 0;
5220                         pLF = IOBuf->buf + IOBuf->BufUsed;
5221                         IOBuf->BufUsed += rlen;
5222                         IOBuf->buf[IOBuf->BufUsed] = '\0';
5223
5224                         pche = IOBuf->buf + IOBuf->BufUsed;
5225
5226                         while ((pLF < pche) && (*pLF != '\n'))
5227                                 pLF ++;
5228                         if ((pLF >= pche) || (*pLF == '\0'))
5229                                 pLF = NULL;
5230
5231                         if (IOBuf->BufUsed + 10 > IOBuf->BufSize)
5232                         {
5233                                 long apos = 0;
5234
5235                                 if (pLF != NULL) apos = pLF - IOBuf->buf;
5236                                 IncreaseBuf(IOBuf, 1, -1);
5237                                 if (pLF != NULL) pLF = IOBuf->buf + apos;
5238                         }
5239
5240                         continue;
5241                 }
5242                 else
5243                 {
5244                         nSuccessLess++;
5245                 }
5246         }
5247         *Pos = NULL;
5248         if (pLF != NULL) {
5249                 pos = IOBuf->buf;
5250                 len = pLF - pos;
5251                 if (len > 0 && (*(pLF - 1) == '\r') )
5252                         len --;
5253                 StrBufAppendBufPlain(Line, ChrPtr(IOBuf), len, 0);
5254                 if (pLF + 1 >= IOBuf->buf + IOBuf->BufUsed)
5255                 {
5256                         FlushStrBuf(IOBuf);
5257                 }
5258                 else
5259                         *Pos = pLF + 1;
5260                 return retlen + len;
5261         }
5262         *Error = ErrRBLF_NotEnoughSentFromServer;
5263         return -1;
5264
5265 }
5266
5267 static const char *ErrRBLF_BLOBPreConditionFailed="StrBufReadBLOB: Wrong arguments or invalid Filedescriptor";
5268 /**
5269  * @ingroup StrBuf_IO
5270  * @brief Input binary data from socket
5271  * flushes and closes the FD on error
5272  * @param Buf the buffer to get the input to
5273  * @param fd pointer to the filedescriptor to read
5274  * @param append Append to an existing string or replace?
5275  * @param nBytes the maximal number of bytes to read
5276  * @param Error strerror() on error
5277  * @returns numbers of chars read
5278  */
5279 int StrBufReadBLOB(StrBuf *Buf, int *fd, int append, long nBytes, const char **Error)
5280 {
5281         int fdflags;
5282         int rlen;
5283         int nSuccessLess;
5284         int nRead = 0;
5285         char *ptr;
5286         int IsNonBlock;
5287         struct timeval tv;
5288         fd_set rfds;
5289
5290         if ((Buf == NULL) || (Buf->buf == NULL) || (*fd == -1))
5291         {
5292                 *Error = ErrRBLF_BLOBPreConditionFailed;
5293                 return -1;
5294         }
5295         if (!append)
5296                 FlushStrBuf(Buf);
5297         if (Buf->BufUsed + nBytes >= Buf->BufSize)
5298                 IncreaseBuf(Buf, 1, Buf->BufUsed + nBytes);
5299
5300         ptr = Buf->buf + Buf->BufUsed;
5301
5302         fdflags = fcntl(*fd, F_GETFL);
5303         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
5304         nSuccessLess = 0;
5305         while ((nRead < nBytes) &&
5306                (*fd != -1))
5307         {
5308                 if (IsNonBlock)
5309                 {
5310                         tv.tv_sec = 1;
5311                         tv.tv_usec = 0;
5312
5313                         FD_ZERO(&rfds);
5314                         FD_SET(*fd, &rfds);
5315                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
5316                                 *Error = strerror(errno);
5317                                 close (*fd);
5318                                 *fd = -1;
5319                                 if (*Error == NULL)
5320                                         *Error = ErrRBLF_SelectFailed;
5321                                 return -1;
5322                         }
5323                         if (! FD_ISSET(*fd, &rfds) != 0) {
5324                                 nSuccessLess ++;
5325                                 continue;
5326                         }
5327                 }
5328
5329                 if ((rlen = read(*fd,
5330                                  ptr,
5331                                  nBytes - nRead)) == -1) {
5332                         close(*fd);
5333                         *fd = -1;
5334                         *Error = strerror(errno);
5335                         return rlen;
5336                 }
5337                 nRead += rlen;
5338                 ptr += rlen;
5339                 Buf->BufUsed += rlen;
5340         }
5341         Buf->buf[Buf->BufUsed] = '\0';
5342         return nRead;
5343 }
5344
5345 const char *ErrRBB_BLOBFPreConditionFailed = "StrBufReadBLOBBuffered: to many selects; aborting.";
5346 const char *ErrRBB_too_many_selects        = "StrBufReadBLOBBuffered: to many selects; aborting.";
5347 /**
5348  * @ingroup StrBuf_BufferedIO
5349  * @brief Input binary data from socket
5350  * flushes and closes the FD on error
5351  * @param Blob put binary thing here
5352  * @param IOBuf the buffer to get the input to
5353  * @param Pos offset inside of IOBuf
5354  * @param fd pointer to the filedescriptor to read
5355  * @param append Append to an existing string or replace?
5356  * @param nBytes the maximal number of bytes to read
5357  * @param check whether we should search for '000\n' terminators in case of timeouts
5358  * @param Error strerror() on error
5359  * @returns numbers of chars read
5360  */
5361 int StrBufReadBLOBBuffered(StrBuf *Blob,
5362                            StrBuf *IOBuf,
5363                            const char **Pos,
5364                            int *fd,
5365                            int append,
5366                            long nBytes,
5367                            int check,
5368                            const char **Error)
5369 {
5370         const char *pos;
5371         int fdflags;
5372         int rlen = 0;
5373         int nRead = 0;
5374         int nAlreadyRead = 0;
5375         int IsNonBlock;
5376         char *ptr;
5377         fd_set rfds;
5378         struct timeval tv;
5379         int nSuccessLess = 0;
5380         int MaxTries;
5381
5382         if ((Blob == NULL)  ||
5383             (*fd == -1)     ||
5384             (IOBuf == NULL) ||
5385             (Pos == NULL))
5386         {
5387                 if (Pos != NULL)
5388                         *Pos = NULL;
5389                 *Error = ErrRBB_BLOBFPreConditionFailed;
5390                 return -1;
5391         }
5392
5393         if (!append)
5394                 FlushStrBuf(Blob);
5395         if (Blob->BufUsed + nBytes >= Blob->BufSize)
5396                 IncreaseBuf(Blob, append, Blob->BufUsed + nBytes);
5397
5398         pos = *Pos;
5399
5400         if (pos != NULL)
5401                 rlen = pos - IOBuf->buf;
5402         rlen = IOBuf->BufUsed - rlen;
5403
5404
5405         if ((IOBuf->BufUsed > 0) &&
5406             (pos != NULL) &&
5407             (pos < IOBuf->buf + IOBuf->BufUsed))
5408         {
5409                 if (rlen < nBytes) {
5410                         memcpy(Blob->buf + Blob->BufUsed, pos, rlen);
5411                         Blob->BufUsed += rlen;
5412                         Blob->buf[Blob->BufUsed] = '\0';
5413                         nAlreadyRead = nRead = rlen;
5414                         *Pos = NULL;
5415                 }
5416                 if (rlen >= nBytes) {
5417                         memcpy(Blob->buf + Blob->BufUsed, pos, nBytes);
5418                         Blob->BufUsed += nBytes;
5419                         Blob->buf[Blob->BufUsed] = '\0';
5420                         if (rlen == nBytes) {
5421                                 *Pos = NULL;
5422                                 FlushStrBuf(IOBuf);
5423                         }
5424                         else
5425                                 *Pos += nBytes;
5426                         return nBytes;
5427                 }
5428         }
5429
5430         FlushStrBuf(IOBuf);
5431         *Pos = NULL;
5432         if (IOBuf->BufSize < nBytes - nRead)
5433                 IncreaseBuf(IOBuf, 0, nBytes - nRead);
5434         ptr = IOBuf->buf;
5435
5436         fdflags = fcntl(*fd, F_GETFL);
5437         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
5438         if (IsNonBlock)
5439                 MaxTries =   1000;
5440         else
5441                 MaxTries = 100000;
5442
5443         nBytes -= nRead;
5444         nRead = 0;
5445         while ((nSuccessLess < MaxTries) &&
5446                (nRead < nBytes) &&
5447                (*fd != -1)) {
5448                 if (IsNonBlock)
5449                 {
5450                         tv.tv_sec = 1;
5451                         tv.tv_usec = 0;
5452
5453                         FD_ZERO(&rfds);
5454                         FD_SET(*fd, &rfds);
5455                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
5456                                 *Error = strerror(errno);
5457                                 close (*fd);
5458                                 *fd = -1;
5459                                 if (*Error == NULL)
5460                                         *Error = ErrRBLF_SelectFailed;
5461                                 return -1;
5462                         }
5463                         if (! FD_ISSET(*fd, &rfds) != 0) {
5464                                 nSuccessLess ++;
5465                                 continue;
5466                         }
5467                 }
5468                 rlen = read(*fd,
5469                             ptr,
5470                             IOBuf->BufSize - (ptr - IOBuf->buf));
5471                 if (rlen == -1) {
5472                         close(*fd);
5473                         *fd = -1;
5474                         *Error = strerror(errno);
5475                         return rlen;
5476                 }
5477                 else if (rlen == 0){
5478                         if ((check == NNN_TERM) &&
5479                             (nRead > 5) &&
5480                             (strncmp(IOBuf->buf + IOBuf->BufUsed - 5, "\n000\n", 5) == 0))
5481                         {
5482                                 StrBufPlain(Blob, HKEY("\n000\n"));
5483                                 StrBufCutRight(Blob, 5);
5484                                 return Blob->BufUsed;
5485                         }
5486                         else if (!IsNonBlock)
5487                                 nSuccessLess ++;
5488                         else if (nSuccessLess > MaxTries) {
5489                                 FlushStrBuf(IOBuf);
5490                                 *Error = ErrRBB_too_many_selects;
5491                                 return -1;
5492                         }
5493                 }
5494                 else if (rlen > 0) {
5495                         nSuccessLess = 0;
5496                         nRead += rlen;
5497                         ptr += rlen;
5498                         IOBuf->BufUsed += rlen;
5499                 }
5500         }
5501         if (nSuccessLess >= MaxTries) {
5502                 FlushStrBuf(IOBuf);
5503                 *Error = ErrRBB_too_many_selects;
5504                 return -1;
5505         }
5506
5507         if (nRead > nBytes) {
5508                 *Pos = IOBuf->buf + nBytes;
5509         }
5510         Blob->buf[Blob->BufUsed] = '\0';
5511         StrBufAppendBufPlain(Blob, IOBuf->buf, nBytes, 0);
5512         if (*Pos == NULL) {
5513                 FlushStrBuf(IOBuf);
5514         }
5515         return nRead + nAlreadyRead;
5516 }
5517
5518 /**
5519  * @ingroup StrBuf_IO
5520  * @brief extract a "next line" from Buf; Ptr to persist across several iterations
5521  * @param LineBuf your line will be copied here.
5522  * @param Buf BLOB with lines of text...
5523  * @param Ptr moved arround to keep the next-line across several iterations
5524  *        has to be &NULL on start; will be &NotNULL on end of buffer
5525  * @returns size of remaining buffer
5526  */
5527 int StrBufSipLine(StrBuf *LineBuf, const StrBuf *Buf, const char **Ptr)
5528 {
5529         const char *aptr, *ptr, *eptr;
5530         char *optr, *xptr;
5531
5532         if ((Buf == NULL) ||
5533             (*Ptr == StrBufNOTNULL) ||
5534             (LineBuf == NULL)||
5535             (LineBuf->buf == NULL))
5536         {
5537                 *Ptr = StrBufNOTNULL;
5538                 return 0;
5539         }
5540
5541         FlushStrBuf(LineBuf);
5542         if (*Ptr==NULL)
5543                 ptr = aptr = Buf->buf;
5544         else
5545                 ptr = aptr = *Ptr;
5546
5547         optr = LineBuf->buf;
5548         eptr = Buf->buf + Buf->BufUsed;
5549         xptr = LineBuf->buf + LineBuf->BufSize - 1;
5550
5551         while ((ptr <= eptr) &&
5552                (*ptr != '\n') &&
5553                (*ptr != '\r') )
5554         {
5555                 *optr = *ptr;
5556                 optr++; ptr++;
5557                 if (optr == xptr) {
5558                         LineBuf->BufUsed = optr - LineBuf->buf;
5559                         IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
5560                         optr = LineBuf->buf + LineBuf->BufUsed;
5561                         xptr = LineBuf->buf + LineBuf->BufSize - 1;
5562                 }
5563         }
5564
5565         if ((ptr >= eptr) && (optr > LineBuf->buf))
5566                 optr --;
5567         LineBuf->BufUsed = optr - LineBuf->buf;
5568         *optr = '\0';
5569         if ((ptr <= eptr) && (*ptr == '\r'))
5570                 ptr ++;
5571         if ((ptr <= eptr) && (*ptr == '\n'))
5572                 ptr ++;
5573
5574         if (ptr < eptr) {
5575                 *Ptr = ptr;
5576         }
5577         else {
5578                 *Ptr = StrBufNOTNULL;
5579         }
5580
5581         return Buf->BufUsed - (ptr - Buf->buf);
5582 }
5583
5584
5585 /**
5586  * @ingroup StrBuf_IO
5587  * @brief removes double slashes from pathnames
5588  * @param Dir directory string to filter
5589  * @param RemoveTrailingSlash allows / disallows trailing slashes
5590  */
5591 void StrBufStripSlashes(StrBuf *Dir, int RemoveTrailingSlash)
5592 {
5593         char *a, *b;
5594
5595         a = b = Dir->buf;
5596
5597         while (!IsEmptyStr(a)) {
5598                 if (*a == '/') {
5599                         while (*a == '/')
5600                                 a++;
5601                         *b = '/';
5602                         b++;
5603                 }
5604                 else {
5605                         *b = *a;
5606                         b++; a++;
5607                 }
5608         }
5609         if ((RemoveTrailingSlash) &&
5610             (b > Dir->buf) &&
5611             (*(b - 1) == '/')){
5612                 b--;
5613         }
5614         *b = '\0';
5615         Dir->BufUsed = b - Dir->buf;
5616 }
5617
5618