libcitadel/lib/stringbuf.c

   1 /*
   2  * Copyright (c) 1987-2013 by the citadel.org team
   3  *
   4  * This program is open source software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 3 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #define _GNU_SOURCE
  20 #include "sysdep.h"
  21 #include <ctype.h>
  22 #include <errno.h>
  23 #include <string.h>
  24 #include <unistd.h>
  25 #include <string.h>
  26 #include <stdio.h>
  27 #include <sys/select.h>
  28 #include <fcntl.h>
  29 #include <sys/types.h>
  30 #define SHOW_ME_VAPPEND_PRINTF
  31 #include <stdarg.h>
  32
  33 #include "libcitadel.h"
  34
  35 #include "b64/cencode.h"
  36 #include "b64/cdecode.h"
  37
  38 #ifdef HAVE_ICONV
  39 #include <iconv.h>
  40 #endif
  41
  42 #ifdef HAVE_BACKTRACE
  43 #include <execinfo.h>
  44 #endif
  45
  46 #ifdef UNDEF_MEMCPY
  47 #undef memcpy
  48 #endif
  49
  50 #ifdef HAVE_ZLIB
  51 #include <zlib.h>
  52 int ZEXPORT compress_gzip(Bytef * dest, size_t * destLen,
  53                           const Bytef * source, uLong sourceLen, int level);
  54 #endif
  55 int BaseStrBufSize = 64;
  56 int EnableSplice = 0;
  57 int ZLibCompressionRatio = -1; /* defaults to 6 */
  58 #ifdef HAVE_ZLIB
  59 #define DEF_MEM_LEVEL 8 /*< memlevel??? */
  60 #define OS_CODE 0x03    /*< unix */
  61 const int gz_magic[2] = { 0x1f, 0x8b }; /* gzip magic header */
  62 #endif
  63
  64 const char *StrBufNOTNULL = ((char*) NULL) - 1;
  65
  66 const char HexList[256][3] = {
  67         "00","01","02","03","04","05","06","07","08","09","0A","0B","0C","0D","0E","0F",
  68         "10","11","12","13","14","15","16","17","18","19","1A","1B","1C","1D","1E","1F",
  69         "20","21","22","23","24","25","26","27","28","29","2A","2B","2C","2D","2E","2F",
  70         "30","31","32","33","34","35","36","37","38","39","3A","3B","3C","3D","3E","3F",
  71         "40","41","42","43","44","45","46","47","48","49","4A","4B","4C","4D","4E","4F",
  72         "50","51","52","53","54","55","56","57","58","59","5A","5B","5C","5D","5E","5F",
  73         "60","61","62","63","64","65","66","67","68","69","6A","6B","6C","6D","6E","6F",
  74         "70","71","72","73","74","75","76","77","78","79","7A","7B","7C","7D","7E","7F",
  75         "80","81","82","83","84","85","86","87","88","89","8A","8B","8C","8D","8E","8F",
  76         "90","91","92","93","94","95","96","97","98","99","9A","9B","9C","9D","9E","9F",
  77         "A0","A1","A2","A3","A4","A5","A6","A7","A8","A9","AA","AB","AC","AD","AE","AF",
  78         "B0","B1","B2","B3","B4","B5","B6","B7","B8","B9","BA","BB","BC","BD","BE","BF",
  79         "C0","C1","C2","C3","C4","C5","C6","C7","C8","C9","CA","CB","CC","CD","CE","CF",
  80         "D0","D1","D2","D3","D4","D5","D6","D7","D8","D9","DA","DB","DC","DD","DE","DF",
  81         "E0","E1","E2","E3","E4","E5","E6","E7","E8","E9","EA","EB","EC","ED","EE","EF",
  82         "F0","F1","F2","F3","F4","F5","F6","F7","F8","F9","FA","FB","FC","FD","FE","FF"};
  83
  84 /**
  85  * @defgroup StrBuf Stringbuffer, A class for manipulating strings with dynamic buffers
  86  * StrBuf is a versatile class, aiding the handling of dynamic strings
  87  *  * reduce de/reallocations
  88  *  * reduce the need to remeasure it
  89  *  * reduce scanning over the string (in @ref StrBuf_NextTokenizer "Tokenizers")
  90  *  * allow asyncroneous IO for line and Blob based operations
  91  *  * reduce the use of memove in those
  92  *  * Quick filling in several operations with append functions
  93  */
  94
  95 /**
  96  * @defgroup StrBuf_DeConstructors Create/Destroy StrBufs
  97  * @ingroup StrBuf
  98  */
  99
 100 /**
 101  * @defgroup StrBuf_Cast Cast operators to interact with char* based code
 102  * @ingroup StrBuf
 103  * use these operators to interfere with code demanding char*;
 104  * if you need to own the content, smash me. Avoid, since we loose the length information.
 105  */
 106
 107 /**
 108  * @defgroup StrBuf_Filler Create/Replace/Append Content into a StrBuf
 109  * @ingroup StrBuf
 110  * operations to get your Strings into a StrBuf, manipulating them, or appending
 111  */
 112 /**
 113  * @defgroup StrBuf_NextTokenizer Fast tokenizer to pull tokens in sequence
 114  * @ingroup StrBuf
 115  * Quick tokenizer; demands of the user to pull its tokens in sequence
 116  */
 117
 118 /**
 119  * @defgroup StrBuf_Tokenizer tokenizer Functions; Slow ones.
 120  * @ingroup StrBuf
 121  * versatile tokenizer; random access to tokens, but slower; Prefer the @ref StrBuf_NextTokenizer "Next Tokenizer"
 122  */
 123
 124 /**
 125  * @defgroup StrBuf_BufferedIO Buffered IO with Asynchroneous reads and no unneeded memmoves (the fast ones)
 126  * @ingroup StrBuf
 127  * File IO to fill StrBufs; Works with work-buffer shared across several calls;
 128  * External Cursor to maintain the current read position inside of the buffer
 129  * the non-fast ones will use memove to keep the start of the buffer the read buffer (which is slower)
 130  */
 131
 132 /**
 133  * @defgroup StrBuf_IO FileIO; Prefer @ref StrBuf_BufferedIO
 134  * @ingroup StrBuf
 135  * Slow I/O; avoid.
 136  */
 137
 138 /**
 139  * @defgroup StrBuf_DeEnCoder functions to translate the contents of a buffer
 140  * @ingroup StrBuf
 141  * these functions translate the content of a buffer into another representation;
 142  * some are combined Fillers and encoders
 143  */
 144
 145 /**
 146  * Private Structure for the Stringbuffer
 147  */
 148 struct StrBuf {
 149         char *buf;         /**< the pointer to the dynamic buffer */
 150         long BufSize;      /**< how many spcae do we optain */
 151         long BufUsed;      /**< StNumber of Chars used excluding the trailing \\0 */
 152         int ConstBuf;      /**< are we just a wrapper arround a static buffer and musn't we be changed? */
 153 #ifdef SIZE_DEBUG
 154         long nIncreases;   /**< for profiling; cound how many times we needed more */
 155         char bt [SIZ];     /**< Stacktrace of last increase */
 156         char bt_lastinc [SIZ]; /**< How much did we increase last time? */
 157 #endif
 158 };
 159
 160
 161 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE);
 162 static inline int Ctdl_IsUtf8SequenceStart(const char Char);
 163
 164 #ifdef SIZE_DEBUG
 165 #ifdef HAVE_BACKTRACE
 166 static void StrBufBacktrace(StrBuf *Buf, int which)
 167 {
 168         int n;
 169         char *pstart, *pch;
 170         void *stack_frames[50];
 171         size_t size, i;
 172         char **strings;
 173
 174         if (which)
 175                 pstart = pch = Buf->bt;
 176         else
 177                 pstart = pch = Buf->bt_lastinc;
 178         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
 179         strings = backtrace_symbols(stack_frames, size);
 180         for (i = 0; i < size; i++) {
 181                 if (strings != NULL)
 182                         n = snprintf(pch, SIZ - (pch - pstart), "%s\\n", strings[i]);
 183                 else
 184                         n = snprintf(pch, SIZ - (pch - pstart), "%p\\n", stack_frames[i]);
 185                 pch += n;
 186         }
 187         free(strings);
 188
 189
 190 }
 191 #endif
 192
 193 void dbg_FreeStrBuf(StrBuf *FreeMe, char *FromWhere)
 194 {
 195         if (hFreeDbglog == -1){
 196                 pid_t pid = getpid();
 197                 char path [SIZ];
 198                 snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
 199                 hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
 200         }
 201         if ((*FreeMe)->nIncreases > 0)
 202         {
 203                 char buf[SIZ * 3];
 204                 long n;
 205                 n = snprintf(buf, SIZ * 3, "%c+|%ld|%ld|%ld|%s|%s|\n",
 206                              FromWhere,
 207                              (*FreeMe)->nIncreases,
 208                              (*FreeMe)->BufUsed,
 209                              (*FreeMe)->BufSize,
 210                              (*FreeMe)->bt,
 211                              (*FreeMe)->bt_lastinc);
 212                 n = write(hFreeDbglog, buf, n);
 213         }
 214         else
 215         {
 216                 char buf[128];
 217                 long n;
 218                 n = snprintf(buf, 128, "%c_|0|%ld%ld|\n",
 219                              FromWhere,
 220                              (*FreeMe)->BufUsed,
 221                              (*FreeMe)->BufSize);
 222                 n = write(hFreeDbglog, buf, n);
 223         }
 224 }
 225
 226 void dbg_IncreaseBuf(StrBuf *IncMe)
 227 {
 228         Buf->nIncreases++;
 229 #ifdef HAVE_BACKTRACE
 230         StrBufBacktrace(Buf, 1);
 231 #endif
 232 }
 233
 234 void dbg_Init(StrBuf *Buf)
 235 {
 236         Buf->nIncreases = 0;
 237         Buf->bt[0] = '\0';
 238         Buf->bt_lastinc[0] = '\0';
 239 #ifdef HAVE_BACKTRACE
 240         StrBufBacktrace(Buf, 0);
 241 #endif
 242 }
 243
 244 #else
 245 /* void it... */
 246 #define dbg_FreeStrBuf(a, b)
 247 #define dbg_IncreaseBuf(a)
 248 #define dbg_Init(a)
 249
 250 #endif
 251
 252 /**
 253  * @ingroup StrBuf
 254  * @brief swaps the contents of two StrBufs
 255  * this is to be used to have cheap switched between a work-buffer and a target buffer
 256  * @param A First one
 257  * @param B second one
 258  */
 259 static inline void SwapBuffers(StrBuf *A, StrBuf *B)
 260 {
 261         StrBuf C;
 262
 263         memcpy(&C, A, sizeof(*A));
 264         memcpy(A, B, sizeof(*B));
 265         memcpy(B, &C, sizeof(C));
 266
 267 }
 268
 269 /**
 270  * @ingroup StrBuf_Cast
 271  * @brief Cast operator to Plain String
 272  * @note if the buffer is altered by StrBuf operations, this pointer may become
 273  *  invalid. So don't lean on it after altering the buffer!
 274  *  Since this operation is considered cheap, rather call it often than risking
 275  *  your pointer to become invalid!
 276  * @param Str the string we want to get the c-string representation for
 277  * @returns the Pointer to the Content. Don't mess with it!
 278  */
 279 inline const char *ChrPtr(const StrBuf *Str)
 280 {
 281         if (Str == NULL)
 282                 return "";
 283         return Str->buf;
 284 }
 285
 286 /**
 287  * @ingroup StrBuf_Cast
 288  * @brief since we know strlen()'s result, provide it here.
 289  * @param Str the string to return the length to
 290  * @returns contentlength of the buffer
 291  */
 292 inline int StrLength(const StrBuf *Str)
 293 {
 294         return (Str != NULL) ? Str->BufUsed : 0;
 295 }
 296
 297 /**
 298  * @ingroup StrBuf_DeConstructors
 299  * @brief local utility function to resize the buffer
 300  * @param Buf the buffer whichs storage we should increase
 301  * @param KeepOriginal should we copy the original buffer or just start over with a new one
 302  * @param DestSize what should fit in after?
 303  */
 304 static int IncreaseBuf(StrBuf *Buf, int KeepOriginal, int DestSize)
 305 {
 306         char *NewBuf;
 307         size_t NewSize = Buf->BufSize * 2;
 308
 309         if (Buf->ConstBuf)
 310                 return -1;
 311
 312         if (DestSize > 0)
 313                 while ((NewSize <= DestSize) && (NewSize != 0))
 314                         NewSize *= 2;
 315
 316         if (NewSize == 0)
 317                 return -1;
 318
 319         NewBuf= (char*) malloc(NewSize);
 320         if (NewBuf == NULL)
 321                 return -1;
 322
 323         if (KeepOriginal && (Buf->BufUsed > 0))
 324         {
 325                 memcpy(NewBuf, Buf->buf, Buf->BufUsed);
 326         }
 327         else
 328         {
 329                 NewBuf[0] = '\0';
 330                 Buf->BufUsed = 0;
 331         }
 332         free (Buf->buf);
 333         Buf->buf = NewBuf;
 334         Buf->BufSize = NewSize;
 335
 336         dbg_IncreaseBuf(Buf);
 337
 338         return Buf->BufSize;
 339 }
 340
 341 /**
 342  * @ingroup StrBuf_DeConstructors
 343  * @brief shrink / increase an _EMPTY_ buffer to NewSize. Buffercontent is thoroughly ignored and flushed.
 344  * @param Buf Buffer to shrink (has to be empty)
 345  * @param ThreshHold if the buffer is bigger then this, its readjusted
 346  * @param NewSize if we Shrink it, how big are we going to be afterwards?
 347  */
 348 void ReAdjustEmptyBuf(StrBuf *Buf, long ThreshHold, long NewSize)
 349 {
 350         if ((Buf != NULL) &&
 351             (Buf->BufUsed == 0) &&
 352             (Buf->BufSize < ThreshHold)) {
 353                 free(Buf->buf);
 354                 Buf->buf = (char*) malloc(NewSize);
 355                 Buf->BufUsed = 0;
 356                 Buf->BufSize = NewSize;
 357         }
 358 }
 359
 360 /**
 361  * @ingroup StrBuf_DeConstructors
 362  * @brief shrink long term buffers to their real size so they don't waste memory
 363  * @param Buf buffer to shrink
 364  * @param Force if not set, will just executed if the buffer is much to big; set for lifetime strings
 365  * @returns physical size of the buffer
 366  */
 367 long StrBufShrinkToFit(StrBuf *Buf, int Force)
 368 {
 369         if (Buf == NULL)
 370                 return -1;
 371         if (Force ||
 372             (Buf->BufUsed + (Buf->BufUsed / 3) > Buf->BufSize))
 373         {
 374                 char *TmpBuf;
 375
 376                 TmpBuf = (char*) malloc(Buf->BufUsed + 1);
 377                 if (TmpBuf == NULL)
 378                         return -1;
 379
 380                 memcpy (TmpBuf, Buf->buf, Buf->BufUsed + 1);
 381                 Buf->BufSize = Buf->BufUsed + 1;
 382                 free(Buf->buf);
 383                 Buf->buf = TmpBuf;
 384         }
 385         return Buf->BufUsed;
 386 }
 387
 388 /**
 389  * @ingroup StrBuf_DeConstructors
 390  * @brief Allocate a new buffer with default buffer size
 391  * @returns the new stringbuffer
 392  */
 393 StrBuf* NewStrBuf(void)
 394 {
 395         StrBuf *NewBuf;
 396
 397         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 398         if (NewBuf == NULL)
 399                 return NULL;
 400
 401         NewBuf->buf = (char*) malloc(BaseStrBufSize);
 402         if (NewBuf->buf == NULL)
 403         {
 404                 free(NewBuf);
 405                 return NULL;
 406         }
 407         NewBuf->buf[0] = '\0';
 408         NewBuf->BufSize = BaseStrBufSize;
 409         NewBuf->BufUsed = 0;
 410         NewBuf->ConstBuf = 0;
 411
 412         dbg_Init (NewBuf);
 413
 414         return NewBuf;
 415 }
 416
 417 /**
 418  * @ingroup StrBuf_DeConstructors
 419  * @brief Copy Constructor; returns a duplicate of CopyMe
 420  * @param CopyMe Buffer to faxmilate
 421  * @returns the new stringbuffer
 422  */
 423 StrBuf* NewStrBufDup(const StrBuf *CopyMe)
 424 {
 425         StrBuf *NewBuf;
 426
 427         if (CopyMe == NULL)
 428                 return NewStrBuf();
 429
 430         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 431         if (NewBuf == NULL)
 432                 return NULL;
 433
 434         NewBuf->buf = (char*) malloc(CopyMe->BufSize);
 435         if (NewBuf->buf == NULL)
 436         {
 437                 free(NewBuf);
 438                 return NULL;
 439         }
 440
 441         memcpy(NewBuf->buf, CopyMe->buf, CopyMe->BufUsed + 1);
 442         NewBuf->BufUsed = CopyMe->BufUsed;
 443         NewBuf->BufSize = CopyMe->BufSize;
 444         NewBuf->ConstBuf = 0;
 445
 446         dbg_Init(NewBuf);
 447
 448         return NewBuf;
 449 }
 450
 451 /**
 452  * @ingroup StrBuf_DeConstructors
 453  * @brief Copy Constructor; CreateRelpaceMe will contain CopyFlushMe afterwards.
 454  * @param NoMe if non-NULL, we will use that buffer as value; KeepOriginal will abused as len.
 455  * @param CopyFlushMe Buffer to faxmilate if KeepOriginal, or to move into CreateRelpaceMe if !KeepOriginal.
 456  * @param CreateRelpaceMe If NULL, will be created, else Flushed and filled CopyFlushMe
 457  * @param KeepOriginal should CopyFlushMe remain intact? or may we Steal its buffer?
 458  * @returns the new stringbuffer
 459  */
 460 void NewStrBufDupAppendFlush(StrBuf **CreateRelpaceMe, StrBuf *CopyFlushMe, const char *NoMe, int KeepOriginal)
 461 {
 462         StrBuf *NewBuf;
 463
 464         if (CreateRelpaceMe == NULL)
 465                 return;
 466
 467         if (NoMe != NULL)
 468         {
 469                 if (*CreateRelpaceMe != NULL)
 470                         StrBufPlain(*CreateRelpaceMe, NoMe, KeepOriginal);
 471                 else
 472                         *CreateRelpaceMe = NewStrBufPlain(NoMe, KeepOriginal);
 473                 return;
 474         }
 475
 476         if (CopyFlushMe == NULL)
 477         {
 478                 if (*CreateRelpaceMe != NULL)
 479                         FlushStrBuf(*CreateRelpaceMe);
 480                 else
 481                         *CreateRelpaceMe = NewStrBuf();
 482                 return;
 483         }
 484
 485         /*
 486          * Randomly Chosen: bigger than 64 chars is cheaper to swap the buffers instead of copying.
 487          * else *CreateRelpaceMe may use more memory than needed in a longer term, CopyFlushMe might
 488          * be a big IO-Buffer...
 489          */
 490         if (KeepOriginal || (StrLength(CopyFlushMe) < 256))
 491         {
 492                 if (*CreateRelpaceMe == NULL)
 493                 {
 494                         *CreateRelpaceMe = NewBuf = NewStrBufPlain(NULL, CopyFlushMe->BufUsed);
 495                         dbg_Init(NewBuf);
 496                 }
 497                 else
 498                 {
 499                         NewBuf = *CreateRelpaceMe;
 500                         FlushStrBuf(NewBuf);
 501                 }
 502                 StrBufAppendBuf(NewBuf, CopyFlushMe, 0);
 503         }
 504         else
 505         {
 506                 if (*CreateRelpaceMe == NULL)
 507                 {
 508                         *CreateRelpaceMe = NewBuf = NewStrBufPlain(NULL, CopyFlushMe->BufUsed);
 509                         dbg_Init(NewBuf);
 510                 }
 511                 else
 512                         NewBuf = *CreateRelpaceMe;
 513                 SwapBuffers (NewBuf, CopyFlushMe);
 514         }
 515         if (!KeepOriginal)
 516                 FlushStrBuf(CopyFlushMe);
 517         return;
 518 }
 519
 520 /**
 521  * @ingroup StrBuf_DeConstructors
 522  * @brief create a new Buffer using an existing c-string
 523  * this function should also be used if you want to pre-suggest
 524  * the buffer size to allocate in conjunction with ptr == NULL
 525  * @param ptr the c-string to copy; may be NULL to create a blank instance
 526  * @param nChars How many chars should we copy; -1 if we should measure the length ourselves
 527  * @returns the new stringbuffer
 528  */
 529 StrBuf* NewStrBufPlain(const char* ptr, int nChars)
 530 {
 531         StrBuf *NewBuf;
 532         size_t Siz = BaseStrBufSize;
 533         size_t CopySize;
 534
 535         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 536         if (NewBuf == NULL)
 537                 return NULL;
 538
 539         if (nChars < 0)
 540                 CopySize = strlen((ptr != NULL)?ptr:"");
 541         else
 542                 CopySize = nChars;
 543
 544         while ((Siz <= CopySize) && (Siz != 0))
 545                 Siz *= 2;
 546
 547         if (Siz == 0)
 548         {
 549                 free(NewBuf);
 550                 return NULL;
 551         }
 552
 553         NewBuf->buf = (char*) malloc(Siz);
 554         if (NewBuf->buf == NULL)
 555         {
 556                 free(NewBuf);
 557                 return NULL;
 558         }
 559         NewBuf->BufSize = Siz;
 560         if (ptr != NULL) {
 561                 memcpy(NewBuf->buf, ptr, CopySize);
 562                 NewBuf->buf[CopySize] = '\0';
 563                 NewBuf->BufUsed = CopySize;
 564         }
 565         else {
 566                 NewBuf->buf[0] = '\0';
 567                 NewBuf->BufUsed = 0;
 568         }
 569         NewBuf->ConstBuf = 0;
 570
 571         dbg_Init(NewBuf);
 572
 573         return NewBuf;
 574 }
 575
 576 /**
 577  * @ingroup StrBuf_DeConstructors
 578  * @brief Set an existing buffer from a c-string
 579  * @param Buf buffer to load
 580  * @param ptr c-string to put into
 581  * @param nChars set to -1 if we should work 0-terminated
 582  * @returns the new length of the string
 583  */
 584 int StrBufPlain(StrBuf *Buf, const char* ptr, int nChars)
 585 {
 586         size_t Siz;
 587         size_t CopySize;
 588
 589         if (Buf == NULL)
 590                 return -1;
 591         if (ptr == NULL) {
 592                 FlushStrBuf(Buf);
 593                 return -1;
 594         }
 595
 596         Siz = Buf->BufSize;
 597
 598         if (nChars < 0)
 599                 CopySize = strlen(ptr);
 600         else
 601                 CopySize = nChars;
 602
 603         while ((Siz <= CopySize) && (Siz != 0))
 604                 Siz *= 2;
 605
 606         if (Siz == 0) {
 607                 FlushStrBuf(Buf);
 608                 return -1;
 609         }
 610
 611         if (Siz != Buf->BufSize)
 612                 IncreaseBuf(Buf, 0, Siz);
 613         memcpy(Buf->buf, ptr, CopySize);
 614         Buf->buf[CopySize] = '\0';
 615         Buf->BufUsed = CopySize;
 616         Buf->ConstBuf = 0;
 617         return CopySize;
 618 }
 619
 620
 621 /**
 622  * @ingroup StrBuf_DeConstructors
 623  * @brief use strbuf as wrapper for a string constant for easy handling
 624  * @param StringConstant a string to wrap
 625  * @param SizeOfStrConstant should be sizeof(StringConstant)-1
 626  */
 627 StrBuf* _NewConstStrBuf(const char* StringConstant, size_t SizeOfStrConstant)
 628 {
 629         StrBuf *NewBuf;
 630
 631         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 632         if (NewBuf == NULL)
 633                 return NULL;
 634         NewBuf->buf = (char*) StringConstant;
 635         NewBuf->BufSize = SizeOfStrConstant;
 636         NewBuf->BufUsed = SizeOfStrConstant;
 637         NewBuf->ConstBuf = 1;
 638
 639         dbg_Init(NewBuf);
 640
 641         return NewBuf;
 642 }
 643
 644
 645 /**
 646  * @ingroup StrBuf_DeConstructors
 647  * @brief flush the content of a Buf; keep its struct
 648  * @param buf Buffer to flush
 649  */
 650 int FlushStrBuf(StrBuf *buf)
 651 {
 652         if ((buf == NULL) || (buf->buf == NULL))
 653                 return -1;
 654         if (buf->ConstBuf)
 655                 return -1;
 656         buf->buf[0] ='\0';
 657         buf->BufUsed = 0;
 658         return 0;
 659 }
 660
 661 /**
 662  * @ingroup StrBuf_DeConstructors
 663  * @brief wipe the content of a Buf thoroughly (overwrite it -> expensive); keep its struct
 664  * @param buf Buffer to wipe
 665  */
 666 int FLUSHStrBuf(StrBuf *buf)
 667 {
 668         if (buf == NULL)
 669                 return -1;
 670         if (buf->ConstBuf)
 671                 return -1;
 672         if (buf->BufUsed > 0) {
 673                 memset(buf->buf, 0, buf->BufUsed);
 674                 buf->BufUsed = 0;
 675         }
 676         return 0;
 677 }
 678
 679 #ifdef SIZE_DEBUG
 680 int hFreeDbglog = -1;
 681 #endif
 682 /**
 683  * @ingroup StrBuf_DeConstructors
 684  * @brief Release a Buffer
 685  * Its a double pointer, so it can NULL your pointer
 686  * so fancy SIG11 appear instead of random results
 687  * @param FreeMe Pointer Pointer to the buffer to free
 688  */
 689 void FreeStrBuf (StrBuf **FreeMe)
 690 {
 691         if (*FreeMe == NULL)
 692                 return;
 693
 694         dbg_FreeStrBuf(FreeMe, 'F');
 695
 696         if (!(*FreeMe)->ConstBuf)
 697                 free((*FreeMe)->buf);
 698         free(*FreeMe);
 699         *FreeMe = NULL;
 700 }
 701
 702 /**
 703  * @ingroup StrBuf_DeConstructors
 704  * @brief flatten a Buffer to the Char * we return
 705  * Its a double pointer, so it can NULL your pointer
 706  * so fancy SIG11 appear instead of random results
 707  * The Callee then owns the buffer and is responsible for freeing it.
 708  * @param SmashMe Pointer Pointer to the buffer to release Buf from and free
 709  * @returns the pointer of the buffer; Callee owns the memory thereafter.
 710  */
 711 char *SmashStrBuf (StrBuf **SmashMe)
 712 {
 713         char *Ret;
 714
 715         if ((SmashMe == NULL) || (*SmashMe == NULL))
 716                 return NULL;
 717
 718         dbg_FreeStrBuf(SmashMe, 'S');
 719
 720         Ret = (*SmashMe)->buf;
 721         free(*SmashMe);
 722         *SmashMe = NULL;
 723         return Ret;
 724 }
 725
 726 /**
 727  * @ingroup StrBuf_DeConstructors
 728  * @brief Release the buffer
 729  * If you want put your StrBuf into a Hash, use this as Destructor.
 730  * @param VFreeMe untyped pointer to a StrBuf. be shure to do the right thing [TM]
 731  */
 732 void HFreeStrBuf (void *VFreeMe)
 733 {
 734         StrBuf *FreeMe = (StrBuf*)VFreeMe;
 735         if (FreeMe == NULL)
 736                 return;
 737
 738         dbg_FreeStrBuf(SmashMe, 'H');
 739
 740         if (!FreeMe->ConstBuf)
 741                 free(FreeMe->buf);
 742         free(FreeMe);
 743 }
 744
 745
 746 /*******************************************************************************
 747  *                      Simple string transformations                          *
 748  *******************************************************************************/
 749
 750 /**
 751  * @ingroup StrBuf
 752  * @brief Wrapper around atol
 753  */
 754 long StrTol(const StrBuf *Buf)
 755 {
 756         if (Buf == NULL)
 757                 return 0;
 758         if(Buf->BufUsed > 0)
 759                 return atol(Buf->buf);
 760         else
 761                 return 0;
 762 }
 763
 764 /**
 765  * @ingroup StrBuf
 766  * @brief Wrapper around atoi
 767  */
 768 int StrToi(const StrBuf *Buf)
 769 {
 770         if (Buf == NULL)
 771                 return 0;
 772         if (Buf->BufUsed > 0)
 773                 return atoi(Buf->buf);
 774         else
 775                 return 0;
 776 }
 777
 778 /**
 779  * @ingroup StrBuf
 780  * @brief Checks to see if the string is a pure number
 781  * @param Buf The buffer to inspect
 782  * @returns 1 if its a pure number, 0, if not.
 783  */
 784 int StrBufIsNumber(const StrBuf *Buf) {
 785         char * pEnd;
 786         if ((Buf == NULL) || (Buf->BufUsed == 0)) {
 787                 return 0;
 788         }
 789         strtoll(Buf->buf, &pEnd, 10);
 790         if (pEnd == Buf->buf)
 791                 return 0;
 792         if ((pEnd != NULL) && (pEnd == Buf->buf + Buf->BufUsed))
 793                 return 1;
 794         if (Buf->buf == pEnd)
 795                 return 0;
 796         return 0;
 797 }
 798
 799 /**
 800  * @ingroup StrBuf_Filler
 801  * @brief modifies a Single char of the Buf
 802  * You can point to it via char* or a zero-based integer
 803  * @param Buf The buffer to manipulate
 804  * @param ptr char* to zero; use NULL if unused
 805  * @param nThChar zero based pointer into the string; use -1 if unused
 806  * @param PeekValue The Character to place into the position
 807  */
 808 long StrBufPeek(StrBuf *Buf, const char* ptr, long nThChar, char PeekValue)
 809 {
 810         if (Buf == NULL)
 811                 return -1;
 812         if (ptr != NULL)
 813                 nThChar = ptr - Buf->buf;
 814         if ((nThChar < 0) || (nThChar > Buf->BufUsed))
 815                 return -1;
 816         Buf->buf[nThChar] = PeekValue;
 817         return nThChar;
 818 }
 819
 820 /**
 821  * @ingroup StrBuf_Filler
 822  * @brief modifies a range of chars of the Buf
 823  * You can point to it via char* or a zero-based integer
 824  * @param Buf The buffer to manipulate
 825  * @param ptr char* to zero; use NULL if unused
 826  * @param nThChar zero based pointer into the string; use -1 if unused
 827  * @param nChars how many chars are to be flushed?
 828  * @param PookValue The Character to place into that area
 829  */
 830 long StrBufPook(StrBuf *Buf, const char* ptr, long nThChar, long nChars, char PookValue)
 831 {
 832         if (Buf == NULL)
 833                 return -1;
 834         if (ptr != NULL)
 835                 nThChar = ptr - Buf->buf;
 836         if ((nThChar < 0) || (nThChar > Buf->BufUsed))
 837                 return -1;
 838         if (nThChar + nChars > Buf->BufUsed)
 839                 nChars =  Buf->BufUsed - nThChar;
 840
 841         memset(Buf->buf + nThChar, PookValue, nChars);
 842         /* just to be shure... */
 843         Buf->buf[Buf->BufUsed] = 0;
 844         return nChars;
 845 }
 846
 847 /**
 848  * @ingroup StrBuf_Filler
 849  * @brief Append a StringBuffer to the buffer
 850  * @param Buf Buffer to modify
 851  * @param AppendBuf Buffer to copy at the end of our buffer
 852  * @param Offset Should we start copying from an offset?
 853  */
 854 void StrBufAppendBuf(StrBuf *Buf, const StrBuf *AppendBuf, unsigned long Offset)
 855 {
 856         if ((AppendBuf == NULL) || (AppendBuf->buf == NULL) ||
 857             (Buf == NULL) || (Buf->buf == NULL))
 858                 return;
 859
 860         if (Buf->BufSize - Offset < AppendBuf->BufUsed + Buf->BufUsed + 1)
 861                 IncreaseBuf(Buf,
 862                             (Buf->BufUsed > 0),
 863                             AppendBuf->BufUsed + Buf->BufUsed);
 864
 865         memcpy(Buf->buf + Buf->BufUsed,
 866                AppendBuf->buf + Offset,
 867                AppendBuf->BufUsed - Offset);
 868         Buf->BufUsed += AppendBuf->BufUsed - Offset;
 869         Buf->buf[Buf->BufUsed] = '\0';
 870 }
 871
 872
 873 /**
 874  * @ingroup StrBuf_Filler
 875  * @brief Append a C-String to the buffer
 876  * @param Buf Buffer to modify
 877  * @param AppendBuf Buffer to copy at the end of our buffer
 878  * @param AppendSize number of bytes to copy; set to -1 if we should count it in advance
 879  * @param Offset Should we start copying from an offset?
 880  */
 881 void StrBufAppendBufPlain(StrBuf *Buf, const char *AppendBuf, long AppendSize, unsigned long Offset)
 882 {
 883         long aps;
 884         long BufSizeRequired;
 885
 886         if ((AppendBuf == NULL) || (Buf == NULL))
 887                 return;
 888
 889         if (AppendSize < 0 )
 890                 aps = strlen(AppendBuf + Offset);
 891         else
 892                 aps = AppendSize - Offset;
 893
 894         BufSizeRequired = Buf->BufUsed + aps + 1;
 895         if (Buf->BufSize <= BufSizeRequired)
 896                 IncreaseBuf(Buf, (Buf->BufUsed > 0), BufSizeRequired);
 897
 898         memcpy(Buf->buf + Buf->BufUsed,
 899                AppendBuf + Offset,
 900                aps);
 901         Buf->BufUsed += aps;
 902         Buf->buf[Buf->BufUsed] = '\0';
 903 }
 904
 905 /**
 906  * @ingroup StrBuf_Filler
 907  * @brief sprintf like function appending the formated string to the buffer
 908  * vsnprintf version to wrap into own calls
 909  * @param Buf Buffer to extend by format and Params
 910  * @param format printf alike format to add
 911  * @param ap va_list containing the items for format
 912  */
 913 void StrBufVAppendPrintf(StrBuf *Buf, const char *format, va_list ap)
 914 {
 915         va_list apl;
 916         size_t BufSize;
 917         size_t nWritten;
 918         size_t Offset;
 919         size_t newused;
 920
 921         if ((Buf == NULL)  || (format == NULL))
 922                 return;
 923
 924         BufSize = Buf->BufSize;
 925         nWritten = Buf->BufSize + 1;
 926         Offset = Buf->BufUsed;
 927         newused = Offset + nWritten;
 928
 929         while (newused >= BufSize) {
 930                 va_copy(apl, ap);
 931                 nWritten = vsnprintf(Buf->buf + Offset,
 932                                      Buf->BufSize - Offset,
 933                                      format, apl);
 934                 va_end(apl);
 935                 newused = Offset + nWritten;
 936                 if (newused >= Buf->BufSize) {
 937                         if (IncreaseBuf(Buf, 1, newused) == -1)
 938                                 return; /* TODO: error handling? */
 939                         newused = Buf->BufSize + 1;
 940                 }
 941                 else {
 942                         Buf->BufUsed = Offset + nWritten;
 943                         BufSize = Buf->BufSize;
 944                 }
 945
 946         }
 947 }
 948
 949 /**
 950  * @ingroup StrBuf_Filler
 951  * @brief sprintf like function appending the formated string to the buffer
 952  * @param Buf Buffer to extend by format and Params
 953  * @param format printf alike format to add
 954  */
 955 void StrBufAppendPrintf(StrBuf *Buf, const char *format, ...)
 956 {
 957         size_t BufSize;
 958         size_t nWritten;
 959         size_t Offset;
 960         size_t newused;
 961         va_list arg_ptr;
 962
 963         if ((Buf == NULL)  || (format == NULL))
 964                 return;
 965
 966         BufSize = Buf->BufSize;
 967         nWritten = Buf->BufSize + 1;
 968         Offset = Buf->BufUsed;
 969         newused = Offset + nWritten;
 970
 971         while (newused >= BufSize) {
 972                 va_start(arg_ptr, format);
 973                 nWritten = vsnprintf(Buf->buf + Buf->BufUsed,
 974                                      Buf->BufSize - Buf->BufUsed,
 975                                      format, arg_ptr);
 976                 va_end(arg_ptr);
 977                 newused = Buf->BufUsed + nWritten;
 978                 if (newused >= Buf->BufSize) {
 979                         if (IncreaseBuf(Buf, 1, newused) == -1)
 980                                 return; /* TODO: error handling? */
 981                         newused = Buf->BufSize + 1;
 982                 }
 983                 else {
 984                         Buf->BufUsed += nWritten;
 985                         BufSize = Buf->BufSize;
 986                 }
 987
 988         }
 989 }
 990
 991 /**
 992  * @ingroup StrBuf_Filler
 993  * @brief sprintf like function putting the formated string into the buffer
 994  * @param Buf Buffer to extend by format and Parameters
 995  * @param format printf alike format to add
 996  */
 997 void StrBufPrintf(StrBuf *Buf, const char *format, ...)
 998 {
 999         size_t nWritten;
1000         va_list arg_ptr;
1001
1002         if ((Buf == NULL)  || (format == NULL))
1003                 return;
1004
1005         nWritten = Buf->BufSize + 1;
1006         while (nWritten >= Buf->BufSize) {
1007                 va_start(arg_ptr, format);
1008                 nWritten = vsnprintf(Buf->buf, Buf->BufSize, format, arg_ptr);
1009                 va_end(arg_ptr);
1010                 if (nWritten >= Buf->BufSize) {
1011                         if (IncreaseBuf(Buf, 0, 0) == -1)
1012                                 return; /* TODO: error handling? */
1013                         nWritten = Buf->BufSize + 1;
1014                         continue;
1015                 }
1016                 Buf->BufUsed = nWritten ;
1017         }
1018 }
1019
1020 /**
1021  * @ingroup StrBuf_Filler
1022  * @brief Callback for cURL to append the webserver reply to a buffer
1023  * @param ptr pre-defined by the cURL API; see man 3 curl for mre info
1024  * @param size pre-defined by the cURL API; see man 3 curl for mre info
1025  * @param nmemb pre-defined by the cURL API; see man 3 curl for mre info
1026  * @param stream pre-defined by the cURL API; see man 3 curl for mre info
1027  */
1028 size_t CurlFillStrBuf_callback(void *ptr, size_t size, size_t nmemb, void *stream)
1029 {
1030
1031         StrBuf *Target;
1032
1033         Target = stream;
1034         if (ptr == NULL)
1035                 return 0;
1036
1037         StrBufAppendBufPlain(Target, ptr, size * nmemb, 0);
1038         return size * nmemb;
1039 }
1040
1041
1042 /**
1043  * @ingroup StrBuf
1044  * @brief extracts a substring from Source into dest
1045  * @param dest buffer to place substring into
1046  * @param Source string to copy substring from
1047  * @param Offset chars to skip from start
1048  * @param nChars number of chars to copy
1049  * @returns the number of chars copied; may be different from nChars due to the size of Source
1050  */
1051 int StrBufSub(StrBuf *dest, const StrBuf *Source, unsigned long Offset, size_t nChars)
1052 {
1053         size_t NCharsRemain;
1054         if (Offset > Source->BufUsed)
1055         {
1056                 if (dest != NULL)
1057                         FlushStrBuf(dest);
1058                 return 0;
1059         }
1060         if (Offset + nChars < Source->BufUsed)
1061         {
1062                 if ((nChars >= dest->BufSize) &&
1063                     (IncreaseBuf(dest, 0, nChars + 1) == -1))
1064                         return 0;
1065                 memcpy(dest->buf, Source->buf + Offset, nChars);
1066                 dest->BufUsed = nChars;
1067                 dest->buf[dest->BufUsed] = '\0';
1068                 return nChars;
1069         }
1070         NCharsRemain = Source->BufUsed - Offset;
1071         if ((NCharsRemain  >= dest->BufSize) &&
1072             (IncreaseBuf(dest, 0, NCharsRemain + 1) == -1))
1073                 return 0;
1074         memcpy(dest->buf, Source->buf + Offset, NCharsRemain);
1075         dest->BufUsed = NCharsRemain;
1076         dest->buf[dest->BufUsed] = '\0';
1077         return NCharsRemain;
1078 }
1079
1080 /**
1081  * @ingroup StrBuf
1082  * @brief Cut nChars from the start of the string
1083  * @param Buf Buffer to modify
1084  * @param nChars how many chars should be skipped?
1085  */
1086 void StrBufCutLeft(StrBuf *Buf, int nChars)
1087 {
1088         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1089         if (nChars >= Buf->BufUsed) {
1090                 FlushStrBuf(Buf);
1091                 return;
1092         }
1093         memmove(Buf->buf, Buf->buf + nChars, Buf->BufUsed - nChars);
1094         Buf->BufUsed -= nChars;
1095         Buf->buf[Buf->BufUsed] = '\0';
1096 }
1097
1098 /**
1099  * @ingroup StrBuf
1100  * @brief Cut the trailing n Chars from the string
1101  * @param Buf Buffer to modify
1102  * @param nChars how many chars should be trunkated?
1103  */
1104 void StrBufCutRight(StrBuf *Buf, int nChars)
1105 {
1106         if ((Buf == NULL) || (Buf->BufUsed == 0) || (Buf->buf == NULL))
1107                 return;
1108
1109         if (nChars >= Buf->BufUsed) {
1110                 FlushStrBuf(Buf);
1111                 return;
1112         }
1113         Buf->BufUsed -= nChars;
1114         Buf->buf[Buf->BufUsed] = '\0';
1115 }
1116
1117 /**
1118  * @ingroup StrBuf
1119  * @brief Cut the string after n Chars
1120  * @param Buf Buffer to modify
1121  * @param AfternChars after how many chars should we trunkate the string?
1122  * @param At if non-null and points inside of our string, cut it there.
1123  */
1124 void StrBufCutAt(StrBuf *Buf, int AfternChars, const char *At)
1125 {
1126         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1127         if (At != NULL){
1128                 AfternChars = At - Buf->buf;
1129         }
1130
1131         if ((AfternChars < 0) || (AfternChars >= Buf->BufUsed))
1132                 return;
1133         Buf->BufUsed = AfternChars;
1134         Buf->buf[Buf->BufUsed] = '\0';
1135 }
1136
1137
1138 /**
1139  * @ingroup StrBuf
1140  * @brief Strip leading and trailing spaces from a string; with premeasured and adjusted length.
1141  * @param Buf the string to modify
1142  */
1143 void StrBufTrim(StrBuf *Buf)
1144 {
1145         int delta = 0;
1146         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1147
1148         while ((Buf->BufUsed > 0) &&
1149                isspace(Buf->buf[Buf->BufUsed - 1]))
1150         {
1151                 Buf->BufUsed --;
1152         }
1153         Buf->buf[Buf->BufUsed] = '\0';
1154
1155         if (Buf->BufUsed == 0) return;
1156
1157         while ((Buf->BufUsed > delta) && (isspace(Buf->buf[delta]))){
1158                 delta ++;
1159         }
1160         if (delta > 0) StrBufCutLeft(Buf, delta);
1161 }
1162 /**
1163  * @ingroup StrBuf
1164  * @brief changes all spaces in the string  (tab, linefeed...) to Blank (0x20)
1165  * @param Buf the string to modify
1166  */
1167 void StrBufSpaceToBlank(StrBuf *Buf)
1168 {
1169         char *pche, *pch;
1170
1171         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1172
1173         pch = Buf->buf;
1174         pche = pch + Buf->BufUsed;
1175         while (pch < pche)
1176         {
1177                 if (isspace(*pch))
1178                         *pch = ' ';
1179                 pch ++;
1180         }
1181 }
1182
1183 void StrBufStripAllBut(StrBuf *Buf, char leftboundary, char rightboundary)
1184 {
1185         const char *pLeft;
1186         const char *pRight;
1187
1188         if ((Buf == NULL) || (Buf->buf == NULL)) {
1189                 return;
1190         }
1191
1192         pRight = strchr(Buf->buf, rightboundary);
1193         if (pRight != NULL) {
1194                 StrBufCutAt(Buf, 0, pRight);
1195         }
1196
1197         pLeft = strrchr(ChrPtr(Buf), leftboundary);
1198         if (pLeft != NULL) {
1199                 StrBufCutLeft(Buf, pLeft - Buf->buf + 1);
1200         }
1201 }
1202
1203
1204 /**
1205  * @ingroup StrBuf_Filler
1206  * @brief uppercase the contents of a buffer
1207  * @param Buf the buffer to translate
1208  */
1209 void StrBufUpCase(StrBuf *Buf)
1210 {
1211         char *pch, *pche;
1212
1213         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1214
1215         pch = Buf->buf;
1216         pche = pch + Buf->BufUsed;
1217         while (pch < pche) {
1218                 *pch = toupper(*pch);
1219                 pch ++;
1220         }
1221 }
1222
1223
1224 /**
1225  * @ingroup StrBuf_Filler
1226  * @brief lowercase the contents of a buffer
1227  * @param Buf the buffer to translate
1228  */
1229 void StrBufLowerCase(StrBuf *Buf)
1230 {
1231         char *pch, *pche;
1232
1233         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1234
1235         pch = Buf->buf;
1236         pche = pch + Buf->BufUsed;
1237         while (pch < pche) {
1238                 *pch = tolower(*pch);
1239                 pch ++;
1240         }
1241 }
1242
1243
1244 /*******************************************************************************
1245  *           a tokenizer that kills, maims, and destroys                       *
1246  *******************************************************************************/
1247
1248 /**
1249  * @ingroup StrBuf_Tokenizer
1250  * @brief Replace a token at a given place with a given length by another token with given length
1251  * @param Buf String where to work on
1252  * @param where where inside of the Buf is the search-token
1253  * @param HowLong How long is the token to be replaced
1254  * @param Repl Token to insert at 'where'
1255  * @param ReplLen Length of repl
1256  * @returns -1 if fail else length of resulting Buf
1257  */
1258 int StrBufReplaceToken(StrBuf *Buf, long where, long HowLong,
1259                        const char *Repl, long ReplLen)
1260 {
1261
1262         if ((Buf == NULL) ||
1263             (where > Buf->BufUsed) ||
1264             (where + HowLong > Buf->BufUsed))
1265                 return -1;
1266
1267         if (where + ReplLen - HowLong > Buf->BufSize)
1268                 if (IncreaseBuf(Buf, 1, Buf->BufUsed + ReplLen) < 0)
1269                         return -1;
1270
1271         memmove(Buf->buf + where + ReplLen,
1272                 Buf->buf + where + HowLong,
1273                 Buf->BufUsed - where - HowLong);
1274
1275         memcpy(Buf->buf + where,
1276                Repl, ReplLen);
1277
1278         Buf->BufUsed += ReplLen - HowLong;
1279
1280         return Buf->BufUsed;
1281 }
1282
1283 /**
1284  * @ingroup StrBuf_Tokenizer
1285  * @brief Counts the numbmer of tokens in a buffer
1286  * @param source String to count tokens in
1287  * @param tok    Tokenizer char to count
1288  * @returns numbers of tokenizer chars found
1289  */
1290 int StrBufNum_tokens(const StrBuf *source, char tok)
1291 {
1292         char *pch, *pche;
1293         long NTokens;
1294         if ((source == NULL) || (source->BufUsed == 0))
1295                 return 0;
1296         if ((source->BufUsed == 1) && (*source->buf == tok))
1297                 return 2;
1298         NTokens = 1;
1299         pch = source->buf;
1300         pche = pch + source->BufUsed;
1301         while (pch < pche)
1302         {
1303                 if (*pch == tok)
1304                         NTokens ++;
1305                 pch ++;
1306         }
1307         return NTokens;
1308 }
1309
1310 /**
1311  * @ingroup StrBuf_Tokenizer
1312  * @brief a string tokenizer
1313  * @param Source StringBuffer to read into
1314  * @param parmnum n'th Parameter to remove
1315  * @param separator tokenizer character
1316  * @returns -1 if not found, else length of token.
1317  */
1318 int StrBufRemove_token(StrBuf *Source, int parmnum, char separator)
1319 {
1320         int ReducedBy;
1321         char *d, *s, *end;              /* dest, source */
1322         int count = 0;
1323
1324         /* Find desired @parameter */
1325         end = Source->buf + Source->BufUsed;
1326         d = Source->buf;
1327         while ((d <= end) &&
1328                (count < parmnum))
1329         {
1330                 /* End of string, bail! */
1331                 if (!*d) {
1332                         d = NULL;
1333                         break;
1334                 }
1335                 if (*d == separator) {
1336                         count++;
1337                 }
1338                 d++;
1339         }
1340         if ((d == NULL) || (d >= end))
1341                 return 0;               /* @Parameter not found */
1342
1343         /* Find next @parameter */
1344         s = d;
1345         while ((s <= end) &&
1346                (*s && *s != separator))
1347         {
1348                 s++;
1349         }
1350         if (*s == separator)
1351                 s++;
1352         ReducedBy = d - s;
1353
1354         /* Hack and slash */
1355         if (s >= end) {
1356                 return 0;
1357         }
1358         else if (*s) {
1359                 memmove(d, s, Source->BufUsed - (s - Source->buf));
1360                 Source->BufUsed += ReducedBy;
1361                 Source->buf[Source->BufUsed] = '\0';
1362         }
1363         else if (d == Source->buf) {
1364                 *d = 0;
1365                 Source->BufUsed = 0;
1366         }
1367         else {
1368                 *--d = '\0';
1369                 Source->BufUsed += ReducedBy;
1370         }
1371         /*
1372         while (*s) {
1373                 *d++ = *s++;
1374         }
1375         *d = 0;
1376         */
1377         return ReducedBy;
1378 }
1379
1380 int StrBufExtract_tokenFromStr(StrBuf *dest, const char *Source, long SourceLen, int parmnum, char separator)
1381 {
1382         const StrBuf Temp = {
1383                 (char*)Source,
1384                 SourceLen,
1385                 SourceLen,
1386                 1
1387 #ifdef SIZE_DEBUG
1388                 ,
1389                 0,
1390                 "",
1391                 ""
1392 #endif
1393         };
1394
1395         return StrBufExtract_token(dest, &Temp, parmnum, separator);
1396 }
1397
1398 /**
1399  * @ingroup StrBuf_Tokenizer
1400  * @brief a string tokenizer
1401  * @param dest Destination StringBuffer
1402  * @param Source StringBuffer to read into
1403  * @param parmnum n'th Parameter to extract
1404  * @param separator tokenizer character
1405  * @returns -1 if not found, else length of token.
1406  */
1407 int StrBufExtract_token(StrBuf *dest, const StrBuf *Source, int parmnum, char separator)
1408 {
1409         const char *s, *e;              //* source * /
1410         int len = 0;                    //* running total length of extracted string * /
1411         int current_token = 0;          //* token currently being processed * /
1412
1413         if (dest != NULL) {
1414                 dest->buf[0] = '\0';
1415                 dest->BufUsed = 0;
1416         }
1417         else
1418                 return(-1);
1419
1420         if ((Source == NULL) || (Source->BufUsed ==0)) {
1421                 return(-1);
1422         }
1423         s = Source->buf;
1424         e = s + Source->BufUsed;
1425
1426         //cit_backtrace();
1427         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1428
1429         while ((s < e) && !IsEmptyStr(s)) {
1430                 if (*s == separator) {
1431                         ++current_token;
1432                 }
1433                 if (len >= dest->BufSize) {
1434                         dest->BufUsed = len;
1435                         if (IncreaseBuf(dest, 1, -1) < 0) {
1436                                 dest->BufUsed --;
1437                                 break;
1438                         }
1439                 }
1440                 if ( (current_token == parmnum) &&
1441                      (*s != separator)) {
1442                         dest->buf[len] = *s;
1443                         ++len;
1444                 }
1445                 else if (current_token > parmnum) {
1446                         break;
1447                 }
1448                 ++s;
1449         }
1450
1451         dest->buf[len] = '\0';
1452         dest->BufUsed = len;
1453
1454         if (current_token < parmnum) {
1455                 //lprintf (CTDL_DEBUG,"test <!: %s\n", dest);
1456                 return(-1);
1457         }
1458         //lprintf (CTDL_DEBUG,"test <: %d; %s\n", len, dest);
1459         return(len);
1460 }
1461
1462
1463
1464
1465
1466 /**
1467  * @ingroup StrBuf_Tokenizer
1468  * @brief a string tokenizer to fetch an integer
1469  * @param Source String containing tokens
1470  * @param parmnum n'th Parameter to extract
1471  * @param separator tokenizer character
1472  * @returns 0 if not found, else integer representation of the token
1473  */
1474 int StrBufExtract_int(const StrBuf* Source, int parmnum, char separator)
1475 {
1476         StrBuf tmp;
1477         char buf[64];
1478
1479         tmp.buf = buf;
1480         buf[0] = '\0';
1481         tmp.BufSize = 64;
1482         tmp.BufUsed = 0;
1483         tmp.ConstBuf = 1;
1484         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1485                 return(atoi(buf));
1486         else
1487                 return 0;
1488 }
1489
1490 /**
1491  * @ingroup StrBuf_Tokenizer
1492  * @brief a string tokenizer to fetch a long integer
1493  * @param Source String containing tokens
1494  * @param parmnum n'th Parameter to extract
1495  * @param separator tokenizer character
1496  * @returns 0 if not found, else long integer representation of the token
1497  */
1498 long StrBufExtract_long(const StrBuf* Source, int parmnum, char separator)
1499 {
1500         StrBuf tmp;
1501         char buf[64];
1502
1503         tmp.buf = buf;
1504         buf[0] = '\0';
1505         tmp.BufSize = 64;
1506         tmp.BufUsed = 0;
1507         tmp.ConstBuf = 1;
1508         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1509                 return(atoi(buf));
1510         else
1511                 return 0;
1512 }
1513
1514
1515 /**
1516  * @ingroup StrBuf_Tokenizer
1517  * @brief a string tokenizer to fetch an unsigned long
1518  * @param Source String containing tokens
1519  * @param parmnum n'th Parameter to extract
1520  * @param separator tokenizer character
1521  * @returns 0 if not found, else unsigned long representation of the token
1522  */
1523 unsigned long StrBufExtract_unsigned_long(const StrBuf* Source, int parmnum, char separator)
1524 {
1525         StrBuf tmp;
1526         char buf[64];
1527         char *pnum;
1528
1529         tmp.buf = buf;
1530         buf[0] = '\0';
1531         tmp.BufSize = 64;
1532         tmp.BufUsed = 0;
1533         tmp.ConstBuf = 1;
1534         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0) {
1535                 pnum = &buf[0];
1536                 if (*pnum == '-')
1537                         pnum ++;
1538                 return (unsigned long) atol(pnum);
1539         }
1540         else
1541                 return 0;
1542 }
1543
1544
1545
1546 /**
1547  * @ingroup StrBuf_NextTokenizer
1548  * @brief a string tokenizer; Bounds checker
1549  *  function to make shure whether StrBufExtract_NextToken and friends have reached the end of the string.
1550  * @param Source our tokenbuffer
1551  * @param pStart the token iterator pointer to inspect
1552  * @returns whether the revolving pointer is inside of the search range
1553  */
1554 int StrBufHaveNextToken(const StrBuf *Source, const char **pStart)
1555 {
1556         if ((Source == NULL) ||
1557             (*pStart == StrBufNOTNULL) ||
1558             (Source->BufUsed == 0))
1559         {
1560                 return 0;
1561         }
1562         if (*pStart == NULL)
1563         {
1564                 return 1;
1565         }
1566         else if (*pStart > Source->buf + Source->BufUsed)
1567         {
1568                 return 0;
1569         }
1570         else if (*pStart <= Source->buf)
1571         {
1572                 return 0;
1573         }
1574
1575         return 1;
1576 }
1577
1578 /**
1579  * @ingroup StrBuf_NextTokenizer
1580  * @brief a string tokenizer
1581  * @param dest Destination StringBuffer
1582  * @param Source StringBuffer to read into
1583  * @param pStart pointer to the end of the last token. Feed with NULL on start.
1584  * @param separator tokenizer
1585  * @returns -1 if not found, else length of token.
1586  */
1587 int StrBufExtract_NextToken(StrBuf *dest, const StrBuf *Source, const char **pStart, char separator)
1588 {
1589         const char *s;          /* source */
1590         const char *EndBuffer;  /* end stop of source buffer */
1591         int current_token = 0;  /* token currently being processed */
1592         int len = 0;            /* running total length of extracted string */
1593
1594         if ((Source          == NULL) ||
1595             (Source->BufUsed == 0)      )
1596         {
1597                 *pStart = StrBufNOTNULL;
1598                 if (dest != NULL)
1599                         FlushStrBuf(dest);
1600                 return -1;
1601         }
1602
1603         EndBuffer = Source->buf + Source->BufUsed;
1604
1605         if (dest != NULL)
1606         {
1607                 dest->buf[0] = '\0';
1608                 dest->BufUsed = 0;
1609         }
1610         else
1611         {
1612                 *pStart = EndBuffer + 1;
1613                 return -1;
1614         }
1615
1616         if (*pStart == NULL)
1617         {
1618                 *pStart = Source->buf; /* we're starting to examine this buffer. */
1619         }
1620         else if ((*pStart < Source->buf) ||
1621                  (*pStart > EndBuffer  )   )
1622         {
1623                 return -1; /* no more tokens to find. */
1624         }
1625
1626         s = *pStart;
1627         /* start to find the next token */
1628         while ((s <= EndBuffer)      &&
1629                (current_token == 0) )
1630         {
1631                 if (*s == separator)
1632                 {
1633                         /* we found the next token */
1634                         ++current_token;
1635                 }
1636
1637                 if (len >= dest->BufSize)
1638                 {
1639                         /* our Dest-buffer isn't big enough, increase it. */
1640                         dest->BufUsed = len;
1641
1642                         if (IncreaseBuf(dest, 1, -1) < 0) {
1643                                 /* WHUT? no more mem? bail out. */
1644                                 s = EndBuffer;
1645                                 dest->BufUsed --;
1646                                 break;
1647                         }
1648                 }
1649
1650                 if ( (current_token == 0 ) &&   /* are we in our target token? */
1651                      (!IsEmptyStr(s)     ) &&
1652                      (separator     != *s)    ) /* don't copy the token itself */
1653                 {
1654                         dest->buf[len] = *s;    /* Copy the payload */
1655                         ++len;                  /* remember the bigger size. */
1656                 }
1657
1658                 ++s;
1659         }
1660
1661         /* did we reach the end? */
1662         if ((s > EndBuffer)) {
1663                 EndBuffer = StrBufNOTNULL;
1664                 *pStart = EndBuffer;
1665         }
1666         else {
1667                 *pStart = s;  /* remember the position for the next run */
1668         }
1669
1670         /* sanitize our extracted token */
1671         dest->buf[len] = '\0';
1672         dest->BufUsed  = len;
1673
1674         return (len);
1675 }
1676
1677
1678 /**
1679  * @ingroup StrBuf_NextTokenizer
1680  * @brief a string tokenizer
1681  * @param Source StringBuffer to read from
1682  * @param pStart pointer to the end of the last token. Feed with NULL.
1683  * @param separator tokenizer character
1684  * @param nTokens number of tokens to fastforward over
1685  * @returns -1 if not found, else length of token.
1686  */
1687 int StrBufSkip_NTokenS(const StrBuf *Source, const char **pStart, char separator, int nTokens)
1688 {
1689         const char *s, *EndBuffer;      //* source * /
1690         int len = 0;                    //* running total length of extracted string * /
1691         int current_token = 0;          //* token currently being processed * /
1692
1693         if ((Source == NULL) ||
1694             (Source->BufUsed ==0)) {
1695                 return(-1);
1696         }
1697         if (nTokens == 0)
1698                 return Source->BufUsed;
1699
1700         if (*pStart == NULL)
1701                 *pStart = Source->buf;
1702
1703         EndBuffer = Source->buf + Source->BufUsed;
1704
1705         if ((*pStart < Source->buf) ||
1706             (*pStart >  EndBuffer)) {
1707                 return (-1);
1708         }
1709
1710
1711         s = *pStart;
1712
1713         //cit_backtrace();
1714         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1715
1716         while ((s < EndBuffer) && !IsEmptyStr(s)) {
1717                 if (*s == separator) {
1718                         ++current_token;
1719                 }
1720                 if (current_token >= nTokens) {
1721                         break;
1722                 }
1723                 ++s;
1724         }
1725         *pStart = s;
1726         (*pStart) ++;
1727
1728         return(len);
1729 }
1730
1731 /**
1732  * @ingroup StrBuf_NextTokenizer
1733  * @brief a string tokenizer to fetch an integer
1734  * @param Source StringBuffer to read from
1735  * @param pStart Cursor on the tokenstring
1736  * @param separator tokenizer character
1737  * @returns 0 if not found, else integer representation of the token
1738  */
1739 int StrBufExtractNext_int(const StrBuf* Source, const char **pStart, char separator)
1740 {
1741         StrBuf tmp;
1742         char buf[64];
1743
1744         tmp.buf = buf;
1745         buf[0] = '\0';
1746         tmp.BufSize = 64;
1747         tmp.BufUsed = 0;
1748         tmp.ConstBuf = 1;
1749         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1750                 return(atoi(buf));
1751         else
1752                 return 0;
1753 }
1754
1755 /**
1756  * @ingroup StrBuf_NextTokenizer
1757  * @brief a string tokenizer to fetch a long integer
1758  * @param Source StringBuffer to read from
1759  * @param pStart Cursor on the tokenstring
1760  * @param separator tokenizer character
1761  * @returns 0 if not found, else long integer representation of the token
1762  */
1763 long StrBufExtractNext_long(const StrBuf* Source, const char **pStart, char separator)
1764 {
1765         StrBuf tmp;
1766         char buf[64];
1767
1768         tmp.buf = buf;
1769         buf[0] = '\0';
1770         tmp.BufSize = 64;
1771         tmp.BufUsed = 0;
1772         tmp.ConstBuf = 1;
1773         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1774                 return(atoi(buf));
1775         else
1776                 return 0;
1777 }
1778
1779
1780 /**
1781  * @ingroup StrBuf_NextTokenizer
1782  * @brief a string tokenizer to fetch an unsigned long
1783  * @param Source StringBuffer to read from
1784  * @param pStart Cursor on the tokenstring
1785  * @param separator tokenizer character
1786  * @returns 0 if not found, else unsigned long representation of the token
1787  */
1788 unsigned long StrBufExtractNext_unsigned_long(const StrBuf* Source, const char **pStart, char separator)
1789 {
1790         StrBuf tmp;
1791         char buf[64];
1792         char *pnum;
1793
1794         tmp.buf = buf;
1795         buf[0] = '\0';
1796         tmp.BufSize = 64;
1797         tmp.BufUsed = 0;
1798         tmp.ConstBuf = 1;
1799         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0) {
1800                 pnum = &buf[0];
1801                 if (*pnum == '-')
1802                         pnum ++;
1803                 return (unsigned long) atol(pnum);
1804         }
1805         else
1806                 return 0;
1807 }
1808
1809
1810
1811
1812
1813 /*******************************************************************************
1814  *                             Escape Appending                                *
1815  *******************************************************************************/
1816
1817 /**
1818  * @ingroup StrBuf_DeEnCoder
1819  * @brief Escape a string for feeding out as a URL while appending it to a Buffer
1820  * @param OutBuf the output buffer
1821  * @param In Buffer to encode
1822  * @param PlainIn way in from plain old c strings
1823  */
1824 void StrBufUrlescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
1825 {
1826         const char *pch, *pche;
1827         char *pt, *pte;
1828         int len;
1829
1830         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1831                 return;
1832         if (PlainIn != NULL) {
1833                 len = strlen(PlainIn);
1834                 pch = PlainIn;
1835                 pche = pch + len;
1836         }
1837         else {
1838                 pch = In->buf;
1839                 pche = pch + In->BufUsed;
1840                 len = In->BufUsed;
1841         }
1842
1843         if (len == 0)
1844                 return;
1845
1846         pt = OutBuf->buf + OutBuf->BufUsed;
1847         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1848
1849         while (pch < pche) {
1850                 if (pt >= pte) {
1851                         IncreaseBuf(OutBuf, 1, -1);
1852                         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1853                         pt = OutBuf->buf + OutBuf->BufUsed;
1854                 }
1855
1856                 if((*pch >= 'a' && *pch <= 'z') ||
1857                    (*pch >= '@' && *pch <= 'Z') || /* @ A-Z */
1858                    (*pch >= '0' && *pch <= ':') || /* 0-9 : */
1859                    (*pch == '!') || (*pch == '_') ||
1860                    (*pch == ',') || (*pch == '.'))
1861                 {
1862                         *(pt++) = *(pch++);
1863                         OutBuf->BufUsed++;
1864                 }
1865                 else {
1866                         *pt = '%';
1867                         *(pt + 1) = HexList[(unsigned char)*pch][0];
1868                         *(pt + 2) = HexList[(unsigned char)*pch][1];
1869                         pt += 3;
1870                         OutBuf->BufUsed += 3;
1871                         pch ++;
1872                 }
1873         }
1874         *pt = '\0';
1875 }
1876
1877 /**
1878  * @ingroup StrBuf_DeEnCoder
1879  * @brief Escape a string for feeding out as a the username/password part of an URL while appending it to a Buffer
1880  * @param OutBuf the output buffer
1881  * @param In Buffer to encode
1882  * @param PlainIn way in from plain old c strings
1883  */
1884 void StrBufUrlescUPAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
1885 {
1886         const char *pch, *pche;
1887         char *pt, *pte;
1888         int len;
1889
1890         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1891                 return;
1892         if (PlainIn != NULL) {
1893                 len = strlen(PlainIn);
1894                 pch = PlainIn;
1895                 pche = pch + len;
1896         }
1897         else {
1898                 pch = In->buf;
1899                 pche = pch + In->BufUsed;
1900                 len = In->BufUsed;
1901         }
1902
1903         if (len == 0)
1904                 return;
1905
1906         pt = OutBuf->buf + OutBuf->BufUsed;
1907         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1908
1909         while (pch < pche) {
1910                 if (pt >= pte) {
1911                         IncreaseBuf(OutBuf, 1, -1);
1912                         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1913                         pt = OutBuf->buf + OutBuf->BufUsed;
1914                 }
1915
1916                 if((*pch >= 'a' && *pch <= 'z') ||
1917                    (*pch >= 'A' && *pch <= 'Z') || /* A-Z */
1918                    (*pch >= '0' && *pch <= ':') || /* 0-9 : */
1919                    (*pch == '!') || (*pch == '_') ||
1920                    (*pch == ',') || (*pch == '.'))
1921                 {
1922                         *(pt++) = *(pch++);
1923                         OutBuf->BufUsed++;
1924                 }
1925                 else {
1926                         *pt = '%';
1927                         *(pt + 1) = HexList[(unsigned char)*pch][0];
1928                         *(pt + 2) = HexList[(unsigned char)*pch][1];
1929                         pt += 3;
1930                         OutBuf->BufUsed += 3;
1931                         pch ++;
1932                 }
1933         }
1934         *pt = '\0';
1935 }
1936
1937 /**
1938  * @ingroup StrBuf_DeEnCoder
1939  * @brief append a string with characters having a special meaning in xml encoded to the buffer
1940  * @param OutBuf the output buffer
1941  * @param In Buffer to encode
1942  * @param PlainIn way in from plain old c strings
1943  * @param PlainInLen way in from plain old c strings; maybe you've got binary data or know the length?
1944  * @param OverrideLowChars should chars < 0x20 be replaced by _ or escaped as xml entity?
1945  */
1946 void StrBufXMLEscAppend(StrBuf *OutBuf,
1947                         const StrBuf *In,
1948                         const char *PlainIn,
1949                         long PlainInLen,
1950                         int OverrideLowChars)
1951 {
1952         const char *pch, *pche;
1953         char *pt, *pte;
1954         int IsUtf8Sequence;
1955         int len;
1956
1957         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1958                 return;
1959         if (PlainIn != NULL) {
1960                 if (PlainInLen < 0)
1961                         len = strlen((const char*)PlainIn);
1962                 else
1963                         len = PlainInLen;
1964                 pch = PlainIn;
1965                 pche = pch + len;
1966         }
1967         else {
1968                 pch = (const char*)In->buf;
1969                 pche = pch + In->BufUsed;
1970                 len = In->BufUsed;
1971         }
1972
1973         if (len == 0)
1974                 return;
1975
1976         pt = OutBuf->buf + OutBuf->BufUsed;
1977         /**< we max append 6 chars at once plus the \0 */
1978         pte = OutBuf->buf + OutBuf->BufSize - 6;
1979
1980         while (pch < pche) {
1981                 if (pt >= pte) {
1982                         OutBuf->BufUsed = pt - OutBuf->buf;
1983                         IncreaseBuf(OutBuf, 1, -1);
1984                         pte = OutBuf->buf + OutBuf->BufSize - 6;
1985                         /**< we max append 3 chars at once plus the \0 */
1986
1987                         pt = OutBuf->buf + OutBuf->BufUsed;
1988                 }
1989
1990                 if (*pch == '<') {
1991                         memcpy(pt, HKEY("&lt;"));
1992                         pt += 4;
1993                         pch ++;
1994                 }
1995                 else if (*pch == '>') {
1996                         memcpy(pt, HKEY("&gt;"));
1997                         pt += 4;
1998                         pch ++;
1999                 }
2000                 else if (*pch == '&') {
2001                         memcpy(pt, HKEY("&amp;"));
2002                         pt += 5;
2003                         pch++;
2004                 }
2005                 else if ((*pch >= 0x20) && (*pch <= 0x7F)) {
2006                         *pt = *pch;
2007                         pt++; pch++;
2008                 }
2009                 else if (*pch < 0x20) {
2010                         /* we probably shouldn't be doing this */
2011                         if (OverrideLowChars)
2012                         {
2013                                 *pt = '_';
2014                                 pt ++;
2015                                 pch ++;
2016                         }
2017                         else
2018                         {
2019                                 *pt = '&';
2020                                 pt++;
2021                                 *pt = HexList[*(unsigned char*)pch][0];
2022                                 pt ++;
2023                                 *pt = HexList[*(unsigned char*)pch][1];
2024                                 pt ++; pch ++;
2025                                 *pt = '&';
2026                                 pt++;
2027                                 pch ++;
2028                         }
2029                 }
2030                 else {
2031                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(pch, pche);
2032                         if (IsUtf8Sequence)
2033                         {
2034                                 while (IsUtf8Sequence > 0){
2035                                         *pt = *pch;
2036                                         pt ++;
2037                                         pch ++;
2038                                         --IsUtf8Sequence;
2039                                 }
2040                         }
2041                         else
2042                         {
2043                                 *pt = '&';
2044                                 pt++;
2045                                 *pt = HexList[*(unsigned char*)pch][0];
2046                                 pt ++;
2047                                 *pt = HexList[*(unsigned char*)pch][1];
2048                                 pt ++; pch ++;
2049                                 *pt = '&';
2050                                 pt++;
2051                                 pch ++;
2052                         }
2053                 }
2054         }
2055         *pt = '\0';
2056         OutBuf->BufUsed = pt - OutBuf->buf;
2057 }
2058
2059
2060 /**
2061  * @ingroup StrBuf_DeEnCoder
2062  * @brief append a string in hex encoding to the buffer
2063  * @param OutBuf the output buffer
2064  * @param In Buffer to encode
2065  * @param PlainIn way in from plain old c strings
2066  * @param PlainInLen way in from plain old c strings; maybe you've got binary data or know the length?
2067  */
2068 void StrBufHexEscAppend(StrBuf *OutBuf, const StrBuf *In, const unsigned char *PlainIn, long PlainInLen)
2069 {
2070         const unsigned char *pch, *pche;
2071         char *pt, *pte;
2072         int len;
2073
2074         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
2075                 return;
2076         if (PlainIn != NULL) {
2077                 if (PlainInLen < 0)
2078                         len = strlen((const char*)PlainIn);
2079                 else
2080                         len = PlainInLen;
2081                 pch = PlainIn;
2082                 pche = pch + len;
2083         }
2084         else {
2085                 pch = (const unsigned char*)In->buf;
2086                 pche = pch + In->BufUsed;
2087                 len = In->BufUsed;
2088         }
2089
2090         if (len == 0)
2091                 return;
2092
2093         pt = OutBuf->buf + OutBuf->BufUsed;
2094         pte = OutBuf->buf + OutBuf->BufSize - 3; /**< we max append 3 chars at once plus the \0 */
2095
2096         while (pch < pche) {
2097                 if (pt >= pte) {
2098                         IncreaseBuf(OutBuf, 1, -1);
2099                         pte = OutBuf->buf + OutBuf->BufSize - 3; /**< we max append 3 chars at once plus the \0 */
2100                         pt = OutBuf->buf + OutBuf->BufUsed;
2101                 }
2102
2103                 *pt = HexList[*pch][0];
2104                 pt ++;
2105                 *pt = HexList[*pch][1];
2106                 pt ++; pch ++; OutBuf->BufUsed += 2;
2107         }
2108         *pt = '\0';
2109 }
2110
2111 void StrBufBase64Append(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn, long PlainInLen, int linebreaks)
2112 {
2113         const char *pch;
2114         char *pt;
2115         int len;
2116         long ExpectLen;
2117
2118         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
2119                 return;
2120         if (PlainIn != NULL) {
2121                 if (PlainInLen < 0)
2122                         len = strlen(PlainIn);
2123                 else
2124                         len = PlainInLen;
2125                 pch = PlainIn;
2126         }
2127         else {
2128                 pch = In->buf;
2129                 len = In->BufUsed;
2130         }
2131
2132         if (len == 0)
2133                 return;
2134
2135         ExpectLen = ((len * 134) / 100) + OutBuf->BufUsed;
2136
2137         if (ExpectLen > OutBuf->BufSize)
2138                 if (IncreaseBuf(OutBuf, 1, ExpectLen) < ExpectLen)
2139                         return;
2140
2141         pt = OutBuf->buf + OutBuf->BufUsed;
2142
2143         len = CtdlEncodeBase64(pt, pch, len, linebreaks);
2144
2145         pt += len;
2146         OutBuf->BufUsed += len;
2147         *pt = '\0';
2148 }
2149
2150 /**
2151  * @ingroup StrBuf_DeEnCoder
2152  * @brief append a string in hex encoding to the buffer
2153  * @param OutBuf the output buffer
2154  * @param In Buffer to encode
2155  * @param PlainIn way in from plain old c strings
2156  */
2157 void StrBufHexescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
2158 {
2159         StrBufHexEscAppend(OutBuf, In, (const unsigned char*) PlainIn, -1);
2160 }
2161
2162 /**
2163  * @ingroup StrBuf_DeEnCoder
2164  * @brief Append a string, escaping characters which have meaning in HTML.
2165  *
2166  * @param Target        target buffer
2167  * @param Source        source buffer; set to NULL if you just have a C-String
2168  * @param PlainIn       Plain-C string to append; set to NULL if unused
2169  * @param nbsp          If nonzero, spaces are converted to non-breaking spaces.
2170  * @param nolinebreaks  if set to 1, linebreaks are removed from the string.
2171  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
2172  */
2173 long StrEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
2174 {
2175         const char *aptr, *eiptr;
2176         char *bptr, *eptr;
2177         long len;
2178
2179         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2180                 return -1;
2181
2182         if (PlainIn != NULL) {
2183                 aptr = PlainIn;
2184                 len = strlen(PlainIn);
2185                 eiptr = aptr + len;
2186         }
2187         else {
2188                 aptr = Source->buf;
2189                 eiptr = aptr + Source->BufUsed;
2190                 len = Source->BufUsed;
2191         }
2192
2193         if (len == 0)
2194                 return -1;
2195
2196         bptr = Target->buf + Target->BufUsed;
2197         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2198
2199         while (aptr < eiptr){
2200                 if(bptr >= eptr) {
2201                         IncreaseBuf(Target, 1, -1);
2202                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2203                         bptr = Target->buf + Target->BufUsed;
2204                 }
2205                 if (*aptr == '<') {
2206                         memcpy(bptr, "&lt;", 4);
2207                         bptr += 4;
2208                         Target->BufUsed += 4;
2209                 }
2210                 else if (*aptr == '>') {
2211                         memcpy(bptr, "&gt;", 4);
2212                         bptr += 4;
2213                         Target->BufUsed += 4;
2214                 }
2215                 else if (*aptr == '&') {
2216                         memcpy(bptr, "&amp;", 5);
2217                         bptr += 5;
2218                         Target->BufUsed += 5;
2219                 }
2220                 else if (*aptr == '"') {
2221                         memcpy(bptr, "&quot;", 6);
2222                         bptr += 6;
2223                         Target->BufUsed += 6;
2224                 }
2225                 else if (*aptr == '\'') {
2226                         memcpy(bptr, "&#39;", 5);
2227                         bptr += 5;
2228                         Target->BufUsed += 5;
2229                 }
2230                 else if (*aptr == LB) {
2231                         *bptr = '<';
2232                         bptr ++;
2233                         Target->BufUsed ++;
2234                 }
2235                 else if (*aptr == RB) {
2236                         *bptr = '>';
2237                         bptr ++;
2238                         Target->BufUsed ++;
2239                 }
2240                 else if (*aptr == QU) {
2241                         *bptr ='"';
2242                         bptr ++;
2243                         Target->BufUsed ++;
2244                 }
2245                 else if ((*aptr == 32) && (nbsp == 1)) {
2246                         memcpy(bptr, "&nbsp;", 6);
2247                         bptr += 6;
2248                         Target->BufUsed += 6;
2249                 }
2250                 else if ((*aptr == '\n') && (nolinebreaks == 1)) {
2251                         *bptr='\0';     /* nothing */
2252                 }
2253                 else if ((*aptr == '\n') && (nolinebreaks == 2)) {
2254                         memcpy(bptr, "&lt;br/&gt;", 11);
2255                         bptr += 11;
2256                         Target->BufUsed += 11;
2257                 }
2258
2259
2260                 else if ((*aptr == '\r') && (nolinebreaks != 0)) {
2261                         *bptr='\0';     /* nothing */
2262                 }
2263                 else{
2264                         *bptr = *aptr;
2265                         bptr++;
2266                         Target->BufUsed ++;
2267                 }
2268                 aptr ++;
2269         }
2270         *bptr = '\0';
2271         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
2272                 return -1;
2273         return Target->BufUsed;
2274 }
2275
2276 /**
2277  * @ingroup StrBuf_DeEnCoder
2278  * @brief Append a string, escaping characters which have meaning in HTML.
2279  * Converts linebreaks into blanks; escapes single quotes
2280  * @param Target        target buffer
2281  * @param Source        source buffer; set to NULL if you just have a C-String
2282  * @param PlainIn       Plain-C string to append; set to NULL if unused
2283  */
2284 void StrMsgEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2285 {
2286         const char *aptr, *eiptr;
2287         char *tptr, *eptr;
2288         long len;
2289
2290         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2291                 return ;
2292
2293         if (PlainIn != NULL) {
2294                 aptr = PlainIn;
2295                 len = strlen(PlainIn);
2296                 eiptr = aptr + len;
2297         }
2298         else {
2299                 aptr = Source->buf;
2300                 eiptr = aptr + Source->BufUsed;
2301                 len = Source->BufUsed;
2302         }
2303
2304         if (len == 0)
2305                 return;
2306
2307         eptr = Target->buf + Target->BufSize - 8;
2308         tptr = Target->buf + Target->BufUsed;
2309
2310         while (aptr < eiptr){
2311                 if(tptr >= eptr) {
2312                         IncreaseBuf(Target, 1, -1);
2313                         eptr = Target->buf + Target->BufSize - 8;
2314                         tptr = Target->buf + Target->BufUsed;
2315                 }
2316
2317                 if (*aptr == '\n') {
2318                         *tptr = ' ';
2319                         Target->BufUsed++;
2320                 }
2321                 else if (*aptr == '\r') {
2322                         *tptr = ' ';
2323                         Target->BufUsed++;
2324                 }
2325                 else if (*aptr == '\'') {
2326                         *(tptr++) = '&';
2327                         *(tptr++) = '#';
2328                         *(tptr++) = '3';
2329                         *(tptr++) = '9';
2330                         *tptr = ';';
2331                         Target->BufUsed += 5;
2332                 } else {
2333                         *tptr = *aptr;
2334                         Target->BufUsed++;
2335                 }
2336                 tptr++; aptr++;
2337         }
2338         *tptr = '\0';
2339 }
2340
2341
2342
2343 /**
2344  * @ingroup StrBuf_DeEnCoder
2345  * @brief Append a string, escaping characters which have meaning in ICAL.
2346  * [\n,]
2347  * @param Target        target buffer
2348  * @param Source        source buffer; set to NULL if you just have a C-String
2349  * @param PlainIn       Plain-C string to append; set to NULL if unused
2350  */
2351 void StrIcalEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2352 {
2353         const char *aptr, *eiptr;
2354         char *tptr, *eptr;
2355         long len;
2356
2357         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2358                 return ;
2359
2360         if (PlainIn != NULL) {
2361                 aptr = PlainIn;
2362                 len = strlen(PlainIn);
2363                 eiptr = aptr + len;
2364         }
2365         else {
2366                 aptr = Source->buf;
2367                 eiptr = aptr + Source->BufUsed;
2368                 len = Source->BufUsed;
2369         }
2370
2371         if (len == 0)
2372                 return;
2373
2374         eptr = Target->buf + Target->BufSize - 8;
2375         tptr = Target->buf + Target->BufUsed;
2376
2377         while (aptr < eiptr){
2378                 if(tptr + 3 >= eptr) {
2379                         IncreaseBuf(Target, 1, -1);
2380                         eptr = Target->buf + Target->BufSize - 8;
2381                         tptr = Target->buf + Target->BufUsed;
2382                 }
2383
2384                 if (*aptr == '\n') {
2385                         *tptr = '\\';
2386                         Target->BufUsed++;
2387                         tptr++;
2388                         *tptr = 'n';
2389                         Target->BufUsed++;
2390                 }
2391                 else if (*aptr == '\r') {
2392                         *tptr = '\\';
2393                         Target->BufUsed++;
2394                         tptr++;
2395                         *tptr = 'r';
2396                         Target->BufUsed++;
2397                 }
2398                 else if (*aptr == ',') {
2399                         *tptr = '\\';
2400                         Target->BufUsed++;
2401                         tptr++;
2402                         *tptr = ',';
2403                         Target->BufUsed++;
2404                 } else {
2405                         *tptr = *aptr;
2406                         Target->BufUsed++;
2407                 }
2408                 tptr++; aptr++;
2409         }
2410         *tptr = '\0';
2411 }
2412
2413 /**
2414  * @ingroup StrBuf_DeEnCoder
2415  * @brief Append a string, escaping characters which have meaning in JavaScript strings .
2416  *
2417  * @param Target        target buffer
2418  * @param Source        source buffer; set to NULL if you just have a C-String
2419  * @param PlainIn       Plain-C string to append; set to NULL if unused
2420  * @returns size of result or -1
2421  */
2422 long StrECMAEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2423 {
2424         const char *aptr, *eiptr;
2425         char *bptr, *eptr;
2426         long len;
2427         int IsUtf8Sequence;
2428
2429         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2430                 return -1;
2431
2432         if (PlainIn != NULL) {
2433                 aptr = PlainIn;
2434                 len = strlen(PlainIn);
2435                 eiptr = aptr + len;
2436         }
2437         else {
2438                 aptr = Source->buf;
2439                 eiptr = aptr + Source->BufUsed;
2440                 len = Source->BufUsed;
2441         }
2442
2443         if (len == 0)
2444                 return -1;
2445
2446         bptr = Target->buf + Target->BufUsed;
2447         eptr = Target->buf + Target->BufSize - 7; /* our biggest unit to put in...  */
2448
2449         while (aptr < eiptr){
2450                 if(bptr >= eptr) {
2451                         IncreaseBuf(Target, 1, -1);
2452                         eptr = Target->buf + Target->BufSize - 7; /* our biggest unit to put in...  */
2453                         bptr = Target->buf + Target->BufUsed;
2454                 }
2455                 switch (*aptr) {
2456                 case '\n':
2457                         memcpy(bptr, HKEY("\\n"));
2458                         bptr += 2;
2459                         Target->BufUsed += 2;
2460                         break;
2461                 case '\r':
2462                         memcpy(bptr, HKEY("\\r"));
2463                         bptr += 2;
2464                         Target->BufUsed += 2;
2465                         break;
2466                 case '"':
2467                         *bptr = '\\';
2468                         bptr ++;
2469                         *bptr = '"';
2470                         bptr ++;
2471                         Target->BufUsed += 2;
2472                         break;
2473                 case '\\':
2474                         if ((*(aptr + 1) == 'u') &&
2475                             isxdigit(*(aptr + 2)) &&
2476                             isxdigit(*(aptr + 3)) &&
2477                             isxdigit(*(aptr + 4)) &&
2478                             isxdigit(*(aptr + 5)))
2479                         { /* oh, a unicode escaper. let it pass through. */
2480                                 memcpy(bptr, aptr, 6);
2481                                 aptr += 5;
2482                                 bptr +=6;
2483                                 Target->BufUsed += 6;
2484                         }
2485                         else
2486                         {
2487                                 *bptr = '\\';
2488                                 bptr ++;
2489                                 *bptr = '\\';
2490                                 bptr ++;
2491                                 Target->BufUsed += 2;
2492                         }
2493                         break;
2494                 case '\b':
2495                         *bptr = '\\';
2496                         bptr ++;
2497                         *bptr = 'b';
2498                         bptr ++;
2499                         Target->BufUsed += 2;
2500                         break;
2501                 case '\f':
2502                         *bptr = '\\';
2503                         bptr ++;
2504                         *bptr = 'f';
2505                         bptr ++;
2506                         Target->BufUsed += 2;
2507                         break;
2508                 case '\t':
2509                         *bptr = '\\';
2510                         bptr ++;
2511                         *bptr = 't';
2512                         bptr ++;
2513                         Target->BufUsed += 2;
2514                         break;
2515                 default:
2516                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(aptr, eiptr);
2517                         while (IsUtf8Sequence > 0){
2518                                 *bptr = *aptr;
2519                                 Target->BufUsed ++;
2520                                 if (--IsUtf8Sequence)
2521                                         aptr++;
2522                                 bptr++;
2523                         }
2524                 }
2525                 aptr ++;
2526         }
2527         *bptr = '\0';
2528         if ((bptr == eptr - 1 ) && !IsEmptyStr(aptr) )
2529                 return -1;
2530         return Target->BufUsed;
2531 }
2532
2533 /**
2534  * @ingroup StrBuf_DeEnCoder
2535  * @brief Append a string, escaping characters which have meaning in HTML + json.
2536  *
2537  * @param Target        target buffer
2538  * @param Source        source buffer; set to NULL if you just have a C-String
2539  * @param PlainIn       Plain-C string to append; set to NULL if unused
2540  * @param nbsp          If nonzero, spaces are converted to non-breaking spaces.
2541  * @param nolinebreaks  if set to 1, linebreaks are removed from the string.
2542  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
2543  */
2544 long StrHtmlEcmaEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
2545 {
2546         const char *aptr, *eiptr;
2547         char *bptr, *eptr;
2548         long len;
2549         int IsUtf8Sequence = 0;
2550
2551         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2552                 return -1;
2553
2554         if (PlainIn != NULL) {
2555                 aptr = PlainIn;
2556                 len = strlen(PlainIn);
2557                 eiptr = aptr + len;
2558         }
2559         else {
2560                 aptr = Source->buf;
2561                 eiptr = aptr + Source->BufUsed;
2562                 len = Source->BufUsed;
2563         }
2564
2565         if (len == 0)
2566                 return -1;
2567
2568         bptr = Target->buf + Target->BufUsed;
2569         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2570
2571         while (aptr < eiptr){
2572                 if(bptr >= eptr) {
2573                         IncreaseBuf(Target, 1, -1);
2574                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2575                         bptr = Target->buf + Target->BufUsed;
2576                 }
2577                 switch (*aptr) {
2578                 case '<':
2579                         memcpy(bptr, HKEY("&lt;"));
2580                         bptr += 4;
2581                         Target->BufUsed += 4;
2582                         break;
2583                 case '>':
2584                         memcpy(bptr, HKEY("&gt;"));
2585                         bptr += 4;
2586                         Target->BufUsed += 4;
2587                         break;
2588                 case '&':
2589                         memcpy(bptr, HKEY("&amp;"));
2590                         bptr += 5;
2591                         Target->BufUsed += 5;
2592                         break;
2593                 case LB:
2594                         *bptr = '<';
2595                         bptr ++;
2596                         Target->BufUsed ++;
2597                         break;
2598                 case RB:
2599                         *bptr = '>';
2600                         bptr ++;
2601                         Target->BufUsed ++;
2602                         break;
2603                 case '\n':
2604                         switch (nolinebreaks) {
2605                         case 1:
2606                                 *bptr='\0';     /* nothing */
2607                                 break;
2608                         case 2:
2609                                 memcpy(bptr, HKEY("&lt;br/&gt;"));
2610                                 bptr += 11;
2611                                 Target->BufUsed += 11;
2612                                 break;
2613                         default:
2614                                 memcpy(bptr, HKEY("\\n"));
2615                                 bptr += 2;
2616                                 Target->BufUsed += 2;
2617                         }
2618                         break;
2619                 case '\r':
2620                         switch (nolinebreaks) {
2621                         case 1:
2622                         case 2:
2623                                 *bptr='\0';     /* nothing */
2624                                 break;
2625                         default:
2626                                 memcpy(bptr, HKEY("\\r"));
2627                                 bptr += 2;
2628                                 Target->BufUsed += 2;
2629                                 break;
2630                         }
2631                         break;
2632                 case '"':
2633                 case QU:
2634                         *bptr = '\\';
2635                         bptr ++;
2636                         *bptr = '"';
2637                         bptr ++;
2638                         Target->BufUsed += 2;
2639                         break;
2640                 case '\\':
2641                         if ((*(aptr + 1) == 'u') &&
2642                             isxdigit(*(aptr + 2)) &&
2643                             isxdigit(*(aptr + 3)) &&
2644                             isxdigit(*(aptr + 4)) &&
2645                             isxdigit(*(aptr + 5)))
2646                         { /* oh, a unicode escaper. let it pass through. */
2647                                 memcpy(bptr, aptr, 6);
2648                                 aptr += 5;
2649                                 bptr +=6;
2650                                 Target->BufUsed += 6;
2651                         }
2652                         else
2653                         {
2654                                 *bptr = '\\';
2655                                 bptr ++;
2656                                 *bptr = '\\';
2657                                 bptr ++;
2658                                 Target->BufUsed += 2;
2659                         }
2660                         break;
2661                 case '\b':
2662                         *bptr = '\\';
2663                         bptr ++;
2664                         *bptr = 'b';
2665                         bptr ++;
2666                         Target->BufUsed += 2;
2667                         break;
2668                 case '\f':
2669                         *bptr = '\\';
2670                         bptr ++;
2671                         *bptr = 'f';
2672                         bptr ++;
2673                         Target->BufUsed += 2;
2674                         break;
2675                 case '\t':
2676                         *bptr = '\\';
2677                         bptr ++;
2678                         *bptr = 't';
2679                         bptr ++;
2680                         Target->BufUsed += 2;
2681                         break;
2682                 case  32:
2683                         if (nbsp == 1) {
2684                                 memcpy(bptr, HKEY("&nbsp;"));
2685                                 bptr += 6;
2686                                 Target->BufUsed += 6;
2687                                 break;
2688                         }
2689                 default:
2690                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(aptr, eiptr);
2691                         while (IsUtf8Sequence > 0){
2692                                 *bptr = *aptr;
2693                                 Target->BufUsed ++;
2694                                 if (--IsUtf8Sequence)
2695                                         aptr++;
2696                                 bptr++;
2697                         }
2698                 }
2699                 aptr ++;
2700         }
2701         *bptr = '\0';
2702         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
2703                 return -1;
2704         return Target->BufUsed;
2705 }
2706
2707
2708 /**
2709  * @ingroup StrBuf_DeEnCoder
2710  * @brief replace all non-Ascii characters by another
2711  * @param Buf buffer to inspect
2712  * @param repl charater to stamp over non ascii chars
2713  */
2714 void StrBufAsciify(StrBuf *Buf, const char repl)
2715 {
2716         long offset;
2717
2718         for (offset = 0; offset < Buf->BufUsed; offset ++)
2719                 if (!isascii(Buf->buf[offset]))
2720                         Buf->buf[offset] = repl;
2721
2722 }
2723
2724 /**
2725  * @ingroup StrBuf_DeEnCoder
2726  * @brief unhide special chars hidden to the HTML escaper
2727  * @param target buffer to put the unescaped string in
2728  * @param source buffer to unescape
2729  */
2730 void StrBufEUid_unescapize(StrBuf *target, const StrBuf *source)
2731 {
2732         int a, b, len;
2733         char hex[3];
2734
2735         if ((source == NULL) || (target == NULL) || (target->buf == NULL))
2736         {
2737                 return;
2738         }
2739
2740         if (target != NULL)
2741                 FlushStrBuf(target);
2742
2743         len = source->BufUsed;
2744         for (a = 0; a < len; ++a) {
2745                 if (target->BufUsed >= target->BufSize)
2746                         IncreaseBuf(target, 1, -1);
2747
2748                 if (source->buf[a] == '=') {
2749                         hex[0] = source->buf[a + 1];
2750                         hex[1] = source->buf[a + 2];
2751                         hex[2] = 0;
2752                         b = 0;
2753                         sscanf(hex, "%02x", &b);
2754                         target->buf[target->BufUsed] = b;
2755                         target->buf[++target->BufUsed] = 0;
2756                         a += 2;
2757                 }
2758                 else {
2759                         target->buf[target->BufUsed] = source->buf[a];
2760                         target->buf[++target->BufUsed] = 0;
2761                 }
2762         }
2763 }
2764
2765
2766 /**
2767  * @ingroup StrBuf_DeEnCoder
2768  * @brief hide special chars from the HTML escapers and friends
2769  * @param target buffer to put the escaped string in
2770  * @param source buffer to escape
2771  */
2772 void StrBufEUid_escapize(StrBuf *target, const StrBuf *source)
2773 {
2774         int i, len;
2775
2776         if (target != NULL)
2777                 FlushStrBuf(target);
2778
2779         if ((source == NULL) || (target == NULL) || (target->buf == NULL))
2780         {
2781                 return;
2782         }
2783
2784         len = source->BufUsed;
2785         for (i=0; i<len; ++i) {
2786                 if (target->BufUsed + 4 >= target->BufSize)
2787                         IncreaseBuf(target, 1, -1);
2788                 if ( (isalnum(source->buf[i])) ||
2789                      (source->buf[i]=='-') ||
2790                      (source->buf[i]=='_') ) {
2791                         target->buf[target->BufUsed++] = source->buf[i];
2792                 }
2793                 else {
2794                         sprintf(&target->buf[target->BufUsed],
2795                                 "=%02X",
2796                                 (0xFF &source->buf[i]));
2797                         target->BufUsed += 3;
2798                 }
2799         }
2800         target->buf[target->BufUsed + 1] = '\0';
2801 }
2802
2803
2804 /*******************************************************************************
2805  *                      Quoted Printable de/encoding                           *
2806  *******************************************************************************/
2807
2808 /**
2809  * @ingroup StrBuf_DeEnCoder
2810  * @brief decode a buffer from base 64 encoding; destroys original
2811  * @param Buf Buffor to transform
2812  */
2813 int StrBufDecodeBase64(StrBuf *Buf)
2814 {
2815         char *xferbuf;
2816         size_t siz;
2817
2818         if (Buf == NULL)
2819                 return -1;
2820
2821         xferbuf = (char*) malloc(Buf->BufSize);
2822         if (xferbuf == NULL)
2823                 return -1;
2824
2825         *xferbuf = '\0';
2826         siz = CtdlDecodeBase64(xferbuf,
2827                                Buf->buf,
2828                                Buf->BufUsed);
2829         free(Buf->buf);
2830         Buf->buf = xferbuf;
2831         Buf->BufUsed = siz;
2832
2833         Buf->buf[Buf->BufUsed] = '\0';
2834         return siz;
2835 }
2836
2837 /**
2838  * @ingroup StrBuf_DeEnCoder
2839  * @brief decode a buffer from base 64 encoding; expects targetbuffer
2840  * @param BufIn Buffor to transform
2841  * @param BufOut Buffer to put result into
2842  */
2843 int StrBufDecodeBase64To(const StrBuf *BufIn, StrBuf *BufOut)
2844 {
2845         if ((BufIn == NULL) || (BufOut == NULL))
2846                 return -1;
2847
2848         if (BufOut->BufSize < BufIn->BufUsed)
2849                 IncreaseBuf(BufOut, BufIn->BufUsed, 0);
2850
2851         BufOut->BufUsed = CtdlDecodeBase64(BufOut->buf,
2852                                            BufIn->buf,
2853                                            BufIn->BufUsed);
2854         return BufOut->BufUsed;
2855 }
2856
2857 typedef struct __z_enc_stream {
2858         StrBuf OutBuf;
2859         z_stream zstream;
2860 } z_enc_stream;
2861
2862 vStreamT *StrBufNewStreamContext(eStreamType type, const char **Err)
2863 {
2864         base64_decodestate *state;;
2865         *Err = NULL;
2866
2867         switch (type)
2868         {
2869         case eBase64Decode:
2870         case eBase64Encode:
2871                 state = (base64_decodestate*) malloc(sizeof(base64_decodestate));
2872                 base64_init_decodestate(state);
2873                 return (vStreamT*) state;
2874                 break;
2875         case eZLibDecode:
2876         {
2877
2878                 z_enc_stream *stream;
2879                 int err;
2880
2881                 stream = (z_enc_stream *) malloc(sizeof(z_enc_stream));
2882                 memset(stream, 0, sizeof(z_enc_stream));
2883                 stream->OutBuf.BufSize = 4*SIZ; /// TODO 64
2884                 stream->OutBuf.buf = (char*)malloc(stream->OutBuf.BufSize);
2885
2886                 err = inflateInit(&stream->zstream);
2887
2888                 if (err != Z_OK) {
2889                         StrBufDestroyStreamContext(type, (vStreamT**) &stream, Err);
2890                         *Err = zError(err);
2891                         return NULL;
2892                 }
2893                 return (vStreamT*) stream;
2894
2895         }
2896         case eZLibEncode:
2897         {
2898                 z_enc_stream *stream;
2899                 int err;
2900
2901                 stream = (z_enc_stream *) malloc(sizeof(z_enc_stream));
2902                 memset(stream, 0, sizeof(z_enc_stream));
2903                 stream->OutBuf.BufSize = 4*SIZ; /// todo 64
2904                 stream->OutBuf.buf = (char*)malloc(stream->OutBuf.BufSize);
2905                 /* write gzip header */
2906                 stream->OutBuf.BufUsed = snprintf
2907                         (stream->OutBuf.buf,
2908                          stream->OutBuf.BufSize,
2909                          "%c%c%c%c%c%c%c%c%c%c",
2910                          gz_magic[0], gz_magic[1], Z_DEFLATED,
2911                          0 /*flags */ , 0, 0, 0, 0 /*time */ , 0 /* xflags */ ,
2912                          OS_CODE);
2913
2914                 err = deflateInit2(&stream->zstream,
2915                                    ZLibCompressionRatio,
2916                                    Z_DEFLATED,
2917                                    -MAX_WBITS,
2918                                    DEF_MEM_LEVEL,
2919                                    Z_DEFAULT_STRATEGY);
2920                 if (err != Z_OK) {
2921                         StrBufDestroyStreamContext(type, (vStreamT**) &stream, Err);
2922                         *Err = zError(err);
2923                         return NULL;
2924                 }
2925                 return (vStreamT*) stream;
2926         }
2927         case eEmtyCodec:
2928                 /// TODO
2929
2930                 break;
2931         }
2932         return NULL;
2933 }
2934
2935 int StrBufDestroyStreamContext(eStreamType type, vStreamT **vStream, const char **Err)
2936 {
2937         int err;
2938         int rc = 0;
2939         *Err = NULL;
2940
2941         if ((vStream == NULL) || (*vStream==NULL)) {
2942                 *Err = strerror(EINVAL);
2943                 return EINVAL;
2944         }
2945         switch (type)
2946         {
2947         case eBase64Encode:
2948         case eBase64Decode:
2949                 free(*vStream);
2950                 *vStream = NULL;
2951                 break;
2952         case eZLibDecode:
2953         {
2954                 z_enc_stream *stream = (z_enc_stream *)*vStream;
2955                 (void)inflateEnd(&stream->zstream);
2956                 free(stream->OutBuf.buf);
2957                 free(stream);
2958         }
2959         case eZLibEncode:
2960         {
2961                 z_enc_stream *stream = (z_enc_stream *)*vStream;
2962                 err = deflateEnd(&stream->zstream);
2963                 if (err != Z_OK) {
2964                         *Err = zError(err);
2965                         rc = -1;
2966                 }
2967                 free(stream->OutBuf.buf);
2968                 free(stream);
2969                 *vStream = NULL;
2970                 break;
2971         }
2972         case eEmtyCodec:
2973                 break; /// TODO
2974         }
2975         return rc;
2976 }
2977
2978 int StrBufStreamTranscode(eStreamType type, IOBuffer *Target, IOBuffer *In, const char* pIn, long pInLen, vStreamT *vStream, int LastChunk, const char **Err)
2979 {
2980         int rc = 0;
2981         switch (type)
2982         {
2983         case eBase64Encode:
2984         {
2985                 /// TODO
2986 /*
2987                 // base64_decodestate *state = (base64_decodestate*)vStream;
2988                 long ExpectLen;
2989
2990                 if (In != NULL)
2991                 {
2992                         pIn = In->buf;
2993                         pInLen = In->BufUsed;
2994                 }
2995                 if ((In == NULL) || (vStream == NULL))
2996                         return;
2997
2998                 ExpectLen = (pInLen / 4) * 3;
2999
3000                 if (Target->BufSize - Target->BufUsed < ExpectLen)
3001                 {
3002                         IncreaseBuf(Target, 1, Target->BufUsed + ExpectLen + 1);
3003                 }
3004
3005                 ////    ExpectLen = base64_encode_block(pIn, pInLen, Target->buf + Target->BufUsed, state);
3006                 Target->BufUsed += ExpectLen;
3007                 Target->buf[Target->BufUsed] = '\0';
3008 */
3009         }
3010         break;
3011         case eBase64Decode:
3012         {
3013 /*
3014                 base64_decodestate *state = (base64_decodestate *)vStream;
3015                 long ExpectLen;
3016
3017                 if (In != NULL)
3018                 {
3019                         pIn = In->buf;
3020                         pInLen = In->BufUsed;
3021                 }
3022                 if ((pIn == NULL) || (vStream == NULL))
3023                         return;
3024
3025                 ExpectLen = (pInLen / 4) * 3;
3026
3027                 if (Target->BufSize - Target->BufUsed < ExpectLen)
3028                 {
3029                         IncreaseBuf(Target, 1, Target->BufUsed + ExpectLen + 1);
3030                 }
3031
3032                 ExpectLen = base64_decode_block(pIn, pInLen, Target->buf + Target->BufUsed, state);
3033                 Target->BufUsed += ExpectLen;
3034                 Target->buf[Target->BufUsed] = '\0';
3035 */
3036         }
3037         break;
3038         case eZLibEncode:
3039         {
3040                 z_enc_stream *stream = (z_enc_stream *)vStream;
3041                 int org_outbuf_len = stream->OutBuf.BufUsed;
3042                 int err;
3043                 unsigned int chunkavail;
3044
3045                 if (In->ReadWritePointer != NULL)
3046                 {
3047                         stream->zstream.next_in = (Bytef *) In->ReadWritePointer;
3048                         stream->zstream.avail_in = (uInt) In->Buf->BufUsed -
3049                                 (In->ReadWritePointer - In->Buf->buf);
3050                 }
3051                 else
3052                 {
3053                         stream->zstream.next_in = (Bytef *) In->Buf->buf;
3054                         stream->zstream.avail_in = (uInt) In->Buf->BufUsed;
3055                 }
3056
3057                 stream->zstream.next_out = (unsigned char*)stream->OutBuf.buf + stream->OutBuf.BufUsed;
3058                 stream->zstream.avail_out = chunkavail = (uInt) stream->OutBuf.BufSize - stream->OutBuf.BufUsed;
3059
3060                 err = deflate(&stream->zstream,  (LastChunk) ? Z_FINISH : Z_NO_FLUSH);
3061
3062                 stream->OutBuf.BufUsed += (chunkavail - stream->zstream.avail_out);
3063
3064                 if (Target &&
3065                     (LastChunk ||
3066                      (stream->OutBuf.BufUsed != org_outbuf_len)
3067                             ))
3068                 {
3069                         SwapBuffers(Target->Buf, &stream->OutBuf);
3070                 }
3071
3072                 if (stream->zstream.avail_in == 0)
3073                 {
3074                         FlushStrBuf(In->Buf);
3075                         In->ReadWritePointer = NULL;
3076                 }
3077                 else
3078                 {
3079                         if (stream->zstream.avail_in < 64)
3080                         {
3081                                 memmove(In->Buf->buf,
3082                                         In->Buf->buf + In->Buf->BufUsed - stream->zstream.avail_in,
3083                                         stream->zstream.avail_in);
3084
3085                                 In->Buf->BufUsed = stream->zstream.avail_in;
3086                                 In->Buf->buf[In->Buf->BufUsed] = '\0';
3087                         }
3088                         else
3089                         {
3090
3091                                 In->ReadWritePointer = In->Buf->buf +
3092                                         (In->Buf->BufUsed - stream->zstream.avail_in);
3093                         }
3094                 }
3095                 rc = (LastChunk && (err != Z_FINISH));
3096                 if (!rc && (err != Z_OK)) {
3097                         *Err = zError(err);
3098                 }
3099
3100         }
3101         break;
3102         case eZLibDecode: {
3103                 z_enc_stream *stream = (z_enc_stream *)vStream;
3104                 int org_outbuf_len = stream->zstream.total_out;
3105                 int err;
3106
3107                 if ((stream->zstream.avail_out != 0) && (stream->zstream.next_in != NULL)) {
3108                         if (In->ReadWritePointer != NULL)
3109                         {
3110                                 stream->zstream.next_in = (Bytef *) In->ReadWritePointer;
3111                                 stream->zstream.avail_in = (uInt) In->Buf->BufUsed -
3112                                         (In->ReadWritePointer - In->Buf->buf);
3113                         }
3114                         else
3115                         {
3116                                 stream->zstream.next_in = (Bytef *) In->Buf->buf;
3117                                 stream->zstream.avail_in = (uInt) In->Buf->BufUsed;
3118                         }
3119                 }
3120
3121                 stream->zstream.next_out = (unsigned char*)stream->OutBuf.buf + stream->OutBuf.BufUsed;
3122                 stream->zstream.avail_out = (uInt) stream->OutBuf.BufSize - stream->OutBuf.BufUsed;
3123
3124                 err = inflate(&stream->zstream, Z_NO_FLUSH);
3125
3126                 ///assert(ret != Z_STREAM_ERROR);  /* state not clobbered * /
3127                 switch (err) {
3128                 case Z_NEED_DICT:
3129                         err = Z_DATA_ERROR;     /* and fall through */
3130
3131                 case Z_DATA_ERROR:
3132                         *Err = zError(err);
3133                 case Z_MEM_ERROR:
3134                         (void)inflateEnd(&stream->zstream);
3135                         return err;
3136                 }
3137
3138                 stream->OutBuf.BufUsed += stream->zstream.total_out + org_outbuf_len;
3139
3140                 if (Target) SwapBuffers(Target->Buf, &stream->OutBuf);
3141
3142                 if (stream->zstream.avail_in == 0)
3143                 {
3144                         FlushStrBuf(In->Buf);
3145                         In->ReadWritePointer = NULL;
3146                 }
3147                 else
3148                 {
3149                         if (stream->zstream.avail_in < 64)
3150                         {
3151                                 memmove(In->Buf->buf,
3152                                         In->Buf->buf + In->Buf->BufUsed - stream->zstream.avail_in,
3153                                         stream->zstream.avail_in);
3154
3155                                 In->Buf->BufUsed = stream->zstream.avail_in;
3156                                 In->Buf->buf[In->Buf->BufUsed] = '\0';
3157                         }
3158                         else
3159                         {
3160
3161                                 In->ReadWritePointer = In->Buf->buf +
3162                                         (In->Buf->BufUsed - stream->zstream.avail_in);
3163                         }
3164                 }
3165         }
3166                 break;
3167         case eEmtyCodec: {
3168
3169         }
3170                 break; /// TODO
3171         }
3172         return rc;
3173 }
3174
3175 /**
3176  * @ingroup StrBuf_DeEnCoder
3177  * @brief decode a buffer from base 64 encoding; destroys original
3178  * @param Buf Buffor to transform
3179  */
3180 int StrBufDecodeHex(StrBuf *Buf)
3181 {
3182         unsigned int ch;
3183         char *pch, *pche, *pchi;
3184
3185         if (Buf == NULL) return -1;
3186
3187         pch = pchi = Buf->buf;
3188         pche = pch + Buf->BufUsed;
3189
3190         while (pchi < pche){
3191                 ch = decode_hex(pchi);
3192                 *pch = ch;
3193                 pch ++;
3194                 pchi += 2;
3195         }
3196
3197         *pch = '\0';
3198         Buf->BufUsed = pch - Buf->buf;
3199         return Buf->BufUsed;
3200 }
3201
3202 /**
3203  * @ingroup StrBuf_DeEnCoder
3204  * @brief replace all chars >0x20 && < 0x7F with Mute
3205  * @param Mute char to put over invalid chars
3206  * @param Buf Buffor to transform
3207  */
3208 int StrBufSanitizeAscii(StrBuf *Buf, const char Mute)
3209 {
3210         unsigned char *pch;
3211
3212         if (Buf == NULL) return -1;
3213         pch = (unsigned char *)Buf->buf;
3214         while (pch < (unsigned char *)Buf->buf + Buf->BufUsed) {
3215                 if ((*pch < 0x20) || (*pch > 0x7F))
3216                         *pch = Mute;
3217                 pch ++;
3218         }
3219         return Buf->BufUsed;
3220 }
3221
3222
3223 /**
3224  * @ingroup StrBuf_DeEnCoder
3225  * @brief remove escaped strings from i.e. the url string (like %20 for blanks)
3226  * @param Buf Buffer to translate
3227  * @param StripBlanks Reduce several blanks to one?
3228  */
3229 long StrBufUnescape(StrBuf *Buf, int StripBlanks)
3230 {
3231         int a, b;
3232         char hex[3];
3233         long len;
3234
3235         if (Buf == NULL)
3236                 return -1;
3237
3238         while ((Buf->BufUsed > 0) && (isspace(Buf->buf[Buf->BufUsed - 1]))){
3239                 Buf->buf[Buf->BufUsed - 1] = '\0';
3240                 Buf->BufUsed --;
3241         }
3242
3243         a = 0;
3244         while (a < Buf->BufUsed) {
3245                 if (Buf->buf[a] == '+')
3246                         Buf->buf[a] = ' ';
3247                 else if (Buf->buf[a] == '%') {
3248                         /* don't let % chars through, rather truncate the input. */
3249                         if (a + 2 > Buf->BufUsed) {
3250                                 Buf->buf[a] = '\0';
3251                                 Buf->BufUsed = a;
3252                         }
3253                         else {
3254                                 hex[0] = Buf->buf[a + 1];
3255                                 hex[1] = Buf->buf[a + 2];
3256                                 hex[2] = 0;
3257                                 b = 0;
3258                                 sscanf(hex, "%02x", &b);
3259                                 Buf->buf[a] = (char) b;
3260                                 len = Buf->BufUsed - a - 2;
3261                                 if (len > 0)
3262                                         memmove(&Buf->buf[a + 1], &Buf->buf[a + 3], len);
3263
3264                                 Buf->BufUsed -=2;
3265                         }
3266                 }
3267                 a++;
3268         }
3269         return a;
3270 }
3271
3272
3273 /**
3274  * @ingroup StrBuf_DeEnCoder
3275  * @brief       RFC2047-encode a header field if necessary.
3276  *              If no non-ASCII characters are found, the string
3277  *              will be copied verbatim without encoding.
3278  *
3279  * @param       target          Target buffer.
3280  * @param       source          Source string to be encoded.
3281  * @returns     encoded length; -1 if non success.
3282  */
3283 int StrBufRFC2047encode(StrBuf **target, const StrBuf *source)
3284 {
3285         const char headerStr[] = "=?UTF-8?Q?";
3286         int need_to_encode = 0;
3287         int i = 0;
3288         unsigned char ch;
3289
3290         if ((source == NULL) ||
3291             (target == NULL))
3292             return -1;
3293
3294         while ((i < source->BufUsed) &&
3295                (!IsEmptyStr (&source->buf[i])) &&
3296                (need_to_encode == 0)) {
3297                 if (((unsigned char) source->buf[i] < 32) ||
3298                     ((unsigned char) source->buf[i] > 126)) {
3299                         need_to_encode = 1;
3300                 }
3301                 i++;
3302         }
3303
3304         if (!need_to_encode) {
3305                 if (*target == NULL) {
3306                         *target = NewStrBufPlain(source->buf, source->BufUsed);
3307                 }
3308                 else {
3309                         FlushStrBuf(*target);
3310                         StrBufAppendBuf(*target, source, 0);
3311                 }
3312                 if (*target != 0)
3313                         return (*target)->BufUsed;
3314                 else
3315                         return 0;
3316         }
3317         if (*target == NULL)
3318                 *target = NewStrBufPlain(NULL, sizeof(headerStr) + source->BufUsed * 2);
3319         else if (sizeof(headerStr) + source->BufUsed >= (*target)->BufSize)
3320                 IncreaseBuf(*target, sizeof(headerStr) + source->BufUsed, 0);
3321         memcpy ((*target)->buf, headerStr, sizeof(headerStr) - 1);
3322         (*target)->BufUsed = sizeof(headerStr) - 1;
3323         for (i=0; (i < source->BufUsed); ++i) {
3324                 if ((*target)->BufUsed + 4 >= (*target)->BufSize)
3325                         IncreaseBuf(*target, 1, 0);
3326                 ch = (unsigned char) source->buf[i];
3327                 if ((ch  <  32) ||
3328                     (ch  > 126) ||
3329                     (ch == '=') ||
3330                     (ch == '?') ||
3331                     (ch == '_') ||
3332                     (ch == '[') ||
3333                     (ch == ']')   )
3334                 {
3335                         sprintf(&(*target)->buf[(*target)->BufUsed], "=%02X", ch);
3336                         (*target)->BufUsed += 3;
3337                 }
3338                 else {
3339                         if (ch == ' ')
3340                                 (*target)->buf[(*target)->BufUsed] = '_';
3341                         else
3342                                 (*target)->buf[(*target)->BufUsed] = ch;
3343                         (*target)->BufUsed++;
3344                 }
3345         }
3346
3347         if ((*target)->BufUsed + 4 >= (*target)->BufSize)
3348                 IncreaseBuf(*target, 1, 0);
3349
3350         (*target)->buf[(*target)->BufUsed++] = '?';
3351         (*target)->buf[(*target)->BufUsed++] = '=';
3352         (*target)->buf[(*target)->BufUsed] = '\0';
3353         return (*target)->BufUsed;;
3354 }
3355
3356 /**
3357  * @ingroup StrBuf_DeEnCoder
3358  * @brief       Quoted-Printable encode a message; make it < 80 columns width.
3359  * @param       source          Source string to be encoded.
3360  * @returns     buffer with encoded message.
3361  */
3362 StrBuf *StrBufRFC2047encodeMessage(const StrBuf *EncodeMe)
3363 {
3364         StrBuf *OutBuf;
3365         char *Optr, *OEptr;
3366         const char *ptr, *eptr;
3367         unsigned char ch;
3368         int LinePos;
3369
3370         OutBuf = NewStrBufPlain(NULL, StrLength(EncodeMe) * 4);
3371         Optr = OutBuf->buf;
3372         OEptr = OutBuf->buf + OutBuf->BufSize;
3373         ptr = EncodeMe->buf;
3374         eptr = EncodeMe->buf + EncodeMe->BufUsed;
3375         LinePos = 0;
3376
3377         while (ptr < eptr)
3378         {
3379                 if (Optr + 4 >= OEptr)
3380                 {
3381                         long Offset;
3382                         Offset = Optr - OutBuf->buf;
3383                         OutBuf->BufUsed = Optr - OutBuf->buf;
3384                         IncreaseBuf(OutBuf, 1, 0);
3385                         Optr = OutBuf->buf + Offset;
3386                         OEptr = OutBuf->buf + OutBuf->BufSize;
3387                 }
3388                 if (*ptr == '\r')
3389                 {
3390                         /* ignore carriage returns */
3391                         ptr ++;
3392                 }
3393                 else if (*ptr == '\n') {
3394                         /* hard line break */
3395                         memcpy(Optr, HKEY("=0A"));
3396                         Optr += 3;
3397                         LinePos += 3;
3398                         ptr ++;
3399                 }
3400                 else if (( (*ptr >= 32) && (*ptr <= 60) ) ||
3401                          ( (*ptr >= 62) && (*ptr <= 126) ))
3402                 {
3403                         *Optr = *ptr;
3404                         Optr ++;
3405                         ptr ++;
3406                         LinePos ++;
3407                 }
3408                 else {
3409                         ch = *ptr;
3410                         *Optr = '=';
3411                         Optr ++;
3412                         *Optr = HexList[ch][0];
3413                         Optr ++;
3414                         *Optr = HexList[ch][1];
3415                         Optr ++;
3416                         LinePos += 3;
3417                         ptr ++;
3418                 }
3419
3420                 if (LinePos > 72) {
3421                         /* soft line break */
3422                         if (isspace(*(Optr - 1))) {
3423                                 ch = *(Optr - 1);
3424                                 Optr --;
3425                                 *Optr = '=';
3426                                 Optr ++;
3427                                 *Optr = HexList[ch][0];
3428                                 Optr ++;
3429                                 *Optr = HexList[ch][1];
3430                                 Optr ++;
3431                                 LinePos += 3;
3432                         }
3433                         *Optr = '=';
3434                         Optr ++;
3435                         *Optr = '\n';
3436                         Optr ++;
3437                         LinePos = 0;
3438                 }
3439         }
3440         *Optr = '\0';
3441         OutBuf->BufUsed = Optr - OutBuf->buf;
3442
3443         return OutBuf;
3444 }
3445
3446
3447 static void AddRecipient(StrBuf *Target,
3448                          StrBuf *UserName,
3449                          StrBuf *EmailAddress,
3450                          StrBuf *EncBuf)
3451 {
3452         int QuoteMe = 0;
3453
3454         if (StrLength(Target) > 0) StrBufAppendBufPlain(Target, HKEY(", "), 0);
3455         if (strchr(ChrPtr(UserName), ',') != NULL) QuoteMe = 1;
3456
3457         if (QuoteMe)  StrBufAppendBufPlain(Target, HKEY("\""), 0);
3458         StrBufRFC2047encode(&EncBuf, UserName);
3459         StrBufAppendBuf(Target, EncBuf, 0);
3460         if (QuoteMe)  StrBufAppendBufPlain(Target, HKEY("\" "), 0);
3461         else          StrBufAppendBufPlain(Target, HKEY(" "), 0);
3462
3463         if (StrLength(EmailAddress) > 0){
3464                 StrBufAppendBufPlain(Target, HKEY("<"), 0);
3465                 StrBufAppendBuf(Target, EmailAddress, 0); /* TODO: what about IDN???? */
3466                 StrBufAppendBufPlain(Target, HKEY(">"), 0);
3467         }
3468 }
3469
3470
3471 /**
3472  * \brief QP encode parts of an email TO/CC/BCC vector, and strip/filter invalid parts
3473  * \param Recp Source list of email recipients
3474  * \param UserName Temporary buffer for internal use; Please provide valid buffer.
3475  * \param EmailAddress Temporary buffer for internal use; Please provide valid buffer.
3476  * \param EncBuf Temporary buffer for internal use; Please provide valid buffer.
3477  * \returns encoded & sanitized buffer with the contents of Recp; Caller owns this memory.
3478  */
3479 StrBuf *StrBufSanitizeEmailRecipientVector(const StrBuf *Recp,
3480                                            StrBuf *UserName,
3481                                            StrBuf *EmailAddress,
3482                                            StrBuf *EncBuf)
3483 {
3484         StrBuf *Target;
3485         const char *pch, *pche;
3486         const char *UserStart, *UserEnd, *EmailStart, *EmailEnd, *At;
3487
3488         if ((Recp == NULL) || (StrLength(Recp) == 0))
3489                 return NULL;
3490
3491         pch = ChrPtr(Recp);
3492         pche = pch + StrLength(Recp);
3493
3494         if (!CheckEncode(pch, -1, pche))
3495                 return NewStrBufDup(Recp);
3496
3497         Target = NewStrBufPlain(NULL, StrLength(Recp));
3498
3499         while ((pch != NULL) && (pch < pche))
3500         {
3501                 while (isspace(*pch)) pch++;
3502                 UserEnd = EmailStart = EmailEnd = NULL;
3503
3504                 if ((*pch == '"') || (*pch == '\'')) {
3505                         UserStart = pch + 1;
3506
3507                         UserEnd = strchr(UserStart, *pch);
3508                         if (UserEnd == NULL)
3509                                 break; ///TODO: Userfeedback??
3510                         EmailStart = UserEnd + 1;
3511                         while (isspace(*EmailStart))
3512                                 EmailStart++;
3513                         if (UserEnd == UserStart) {
3514                                 UserStart = UserEnd = NULL;
3515                         }
3516
3517                         if (*EmailStart == '<') {
3518                                 EmailStart++;
3519                                 EmailEnd = strchr(EmailStart, '>');
3520                                 if (EmailEnd == NULL)
3521                                         EmailEnd = strchr(EmailStart, ',');
3522
3523                         }
3524                         else {
3525                                 EmailEnd = strchr(EmailStart, ',');
3526                         }
3527                         if (EmailEnd == NULL)
3528                                 EmailEnd = pche;
3529                         pch = EmailEnd + 1;
3530                 }
3531                 else {
3532                         int gt = 0;
3533                         UserStart = pch;
3534                         EmailEnd = strchr(UserStart, ',');
3535                         if (EmailEnd == NULL) {
3536                                 EmailEnd = strchr(pch, '>');
3537                                 pch = NULL;
3538                                 if (EmailEnd != NULL) {
3539                                         gt = 1;
3540                                 }
3541                                 else {
3542                                         EmailEnd = pche;
3543                                 }
3544                         }
3545                         else {
3546
3547                                 pch = EmailEnd + 1;
3548                                 while ((EmailEnd > UserStart) && !gt &&
3549                                        ((*EmailEnd == ',') ||
3550                                         (*EmailEnd == '>') ||
3551                                         (isspace(*EmailEnd))))
3552                                 {
3553                                         if (*EmailEnd == '>')
3554                                                 gt = 1;
3555                                         else
3556                                                 EmailEnd--;
3557                                 }
3558                                 if (EmailEnd == UserStart)
3559                                         break;
3560                         }
3561                         if (gt) {
3562                                 EmailStart = strchr(UserStart, '<');
3563                                 if ((EmailStart == NULL) || (EmailStart > EmailEnd))
3564                                         break;
3565                                 UserEnd = EmailStart;
3566
3567                                 while ((UserEnd > UserStart) &&
3568                                        isspace (*(UserEnd - 1)))
3569                                         UserEnd --;
3570                                 EmailStart ++;
3571                                 if (UserStart >= UserEnd)
3572                                         UserStart = UserEnd = NULL;
3573                         }
3574                         else { /* this is a local recipient... no domain, just a realname */
3575                                 EmailStart = UserStart;
3576                                 At = strchr(EmailStart, '@');
3577                                 if (At == NULL) {
3578                                         UserEnd = EmailEnd;
3579                                         EmailEnd = NULL;
3580                                 }
3581                                 else {
3582                                         EmailStart = UserStart;
3583                                         UserStart = NULL;
3584                                 }
3585                         }
3586                 }
3587
3588                 if ((UserStart != NULL) && (UserEnd != NULL))
3589                         StrBufPlain(UserName, UserStart, UserEnd - UserStart);
3590                 else if ((UserStart != NULL) && (UserEnd == NULL))
3591                         StrBufPlain(UserName, UserStart, UserEnd - UserStart);
3592                 else
3593                         FlushStrBuf(UserName);
3594
3595                 if ((EmailStart != NULL) && (EmailEnd != NULL))
3596                         StrBufPlain(EmailAddress, EmailStart, EmailEnd - EmailStart);
3597                 else if ((EmailStart != NULL) && (EmailEnd == NULL))
3598                         StrBufPlain(EmailAddress, EmailStart, EmailEnd - pche);
3599                 else
3600                         FlushStrBuf(EmailAddress);
3601
3602                 AddRecipient(Target, UserName, EmailAddress, EncBuf);
3603
3604                 if (pch == NULL)
3605                         break;
3606
3607                 if ((pch != NULL) && (*pch == ','))
3608                         pch ++;
3609                 if (pch != NULL) while (isspace(*pch))
3610                         pch ++;
3611         }
3612         return Target;
3613 }
3614
3615
3616 /**
3617  * @ingroup StrBuf
3618  * @brief replaces all occurances of 'search' by 'replace'
3619  * @param buf Buffer to modify
3620  * @param search character to search
3621  * @param replace character to replace search by
3622  */
3623 void StrBufReplaceChars(StrBuf *buf, char search, char replace)
3624 {
3625         long i;
3626         if (buf == NULL)
3627                 return;
3628         for (i=0; i<buf->BufUsed; i++)
3629                 if (buf->buf[i] == search)
3630                         buf->buf[i] = replace;
3631
3632 }
3633
3634 /**
3635  * @ingroup StrBuf
3636  * @brief removes all \\r s from the string, or replaces them with \n if its not a combination of both.
3637  * @param buf Buffer to modify
3638  */
3639 void StrBufToUnixLF(StrBuf *buf)
3640 {
3641         char *pche, *pchS, *pchT;
3642         if (buf == NULL)
3643                 return;
3644
3645         pche = buf->buf + buf->BufUsed;
3646         pchS = pchT = buf->buf;
3647         while (pchS < pche)
3648         {
3649                 if (*pchS == '\r')
3650                 {
3651                         pchS ++;
3652                         if (*pchS != '\n') {
3653                                 *pchT = '\n';
3654                                 pchT++;
3655                         }
3656                 }
3657                 *pchT = *pchS;
3658                 pchT++; pchS++;
3659         }
3660         *pchT = '\0';
3661         buf->BufUsed = pchT - buf->buf;
3662 }
3663
3664
3665 /*******************************************************************************
3666  *                 Iconv Wrapper; RFC822 de/encoding                           *
3667  *******************************************************************************/
3668
3669 /**
3670  * @ingroup StrBuf_DeEnCoder
3671  * @brief Wrapper around iconv_open()
3672  * Our version adds aliases for non-standard Microsoft charsets
3673  * such as 'MS950', aliasing them to names like 'CP950'
3674  *
3675  * @param tocode        Target encoding
3676  * @param fromcode      Source encoding
3677  * @param pic           anonimized pointer to iconv struct
3678  */
3679 void  ctdl_iconv_open(const char *tocode, const char *fromcode, void *pic)
3680 {
3681 #ifdef HAVE_ICONV
3682         iconv_t ic = (iconv_t)(-1) ;
3683         ic = iconv_open(tocode, fromcode);
3684         if (ic == (iconv_t)(-1) ) {
3685                 char alias_fromcode[64];
3686                 if ( (strlen(fromcode) == 5) && (!strncasecmp(fromcode, "MS", 2)) ) {
3687                         safestrncpy(alias_fromcode, fromcode, sizeof alias_fromcode);
3688                         alias_fromcode[0] = 'C';
3689                         alias_fromcode[1] = 'P';
3690                         ic = iconv_open(tocode, alias_fromcode);
3691                 }
3692         }
3693         *(iconv_t *)pic = ic;
3694 #endif
3695 }
3696
3697
3698 /**
3699  * @ingroup StrBuf_DeEnCoder
3700  * @brief find one chunk of a RFC822 encoded string
3701  * @param Buffer where to search
3702  * @param bptr where to start searching
3703  * @returns found position, NULL if none.
3704  */
3705 static inline const char *FindNextEnd (const StrBuf *Buf, const char *bptr)
3706 {
3707         const char * end;
3708         /* Find the next ?Q? */
3709         if (Buf->BufUsed - (bptr - Buf->buf)  < 6)
3710                 return NULL;
3711
3712         end = strchr(bptr + 2, '?');
3713
3714         if (end == NULL)
3715                 return NULL;
3716
3717         if ((Buf->BufUsed - (end - Buf->buf) > 3) &&
3718             (((*(end + 1) == 'B') || (*(end + 1) == 'Q')) ||
3719              ((*(end + 1) == 'b') || (*(end + 1) == 'q'))) &&
3720             (*(end + 2) == '?')) {
3721                 /* skip on to the end of the cluster, the next ?= */
3722                 end = strstr(end + 3, "?=");
3723         }
3724         else
3725                 /* sort of half valid encoding, try to find an end. */
3726                 end = strstr(bptr, "?=");
3727         return end;
3728 }
3729
3730
3731
3732 /**
3733  * @ingroup StrBuf_DeEnCoder
3734  * @brief convert one buffer according to the preselected iconv pointer PIC
3735  * @param ConvertBuf buffer we need to translate
3736  * @param TmpBuf To share a workbuffer over several iterations. prepare to have it filled with useless stuff afterwards.
3737  * @param pic Pointer to the iconv-session Object
3738  */
3739 void StrBufConvert(StrBuf *ConvertBuf, StrBuf *TmpBuf, void *pic)
3740 {
3741 #ifdef HAVE_ICONV
3742         long trycount = 0;
3743         size_t siz;
3744         iconv_t ic;
3745         char *ibuf;                     /**< Buffer of characters to be converted */
3746         char *obuf;                     /**< Buffer for converted characters */
3747         size_t ibuflen;                 /**< Length of input buffer */
3748         size_t obuflen;                 /**< Length of output buffer */
3749
3750
3751         if ((ConvertBuf == NULL) || (TmpBuf == NULL))
3752                 return;
3753
3754         /* since we're converting to utf-8, one glyph may take up to 6 bytes */
3755         if (ConvertBuf->BufUsed * 6 >= TmpBuf->BufSize)
3756                 IncreaseBuf(TmpBuf, 0, ConvertBuf->BufUsed * 6);
3757 TRYAGAIN:
3758         ic = *(iconv_t*)pic;
3759         ibuf = ConvertBuf->buf;
3760         ibuflen = ConvertBuf->BufUsed;
3761         obuf = TmpBuf->buf;
3762         obuflen = TmpBuf->BufSize;
3763
3764         siz = iconv(ic, &ibuf, &ibuflen, &obuf, &obuflen);
3765
3766         if (siz < 0) {
3767                 if (errno == E2BIG) {
3768                         trycount ++;
3769                         IncreaseBuf(TmpBuf, 0, 0);
3770                         if (trycount < 5)
3771                                 goto TRYAGAIN;
3772
3773                 }
3774                 else if (errno == EILSEQ){
3775                         /* hm, invalid utf8 sequence... what to do now? */
3776                         /* An invalid multibyte sequence has been encountered in the input */
3777                 }
3778                 else if (errno == EINVAL) {
3779                         /* An incomplete multibyte sequence has been encountered in the input. */
3780                 }
3781
3782                 FlushStrBuf(TmpBuf);
3783         }
3784         else {
3785                 TmpBuf->BufUsed = TmpBuf->BufSize - obuflen;
3786                 TmpBuf->buf[TmpBuf->BufUsed] = '\0';
3787
3788                 /* little card game: wheres the red lady? */
3789                 SwapBuffers(ConvertBuf, TmpBuf);
3790                 FlushStrBuf(TmpBuf);
3791         }
3792 #endif
3793 }
3794
3795
3796 /**
3797  * @ingroup StrBuf_DeEnCoder
3798  * @brief catches one RFC822 encoded segment, and decodes it.
3799  * @param Target buffer to fill with result
3800  * @param DecodeMe buffer with stuff to process
3801  * @param SegmentStart points to our current segment in DecodeMe
3802  * @param SegmentEnd Points to the end of our current segment in DecodeMe
3803  * @param ConvertBuf Workbuffer shared between several iterations. Random content; needs to be valid
3804  * @param ConvertBuf2 Workbuffer shared between several iterations. Random content; needs to be valid
3805  * @param FoundCharset Characterset to default decoding to; if we find another we will overwrite it.
3806  */
3807 inline static void DecodeSegment(StrBuf *Target,
3808                                  const StrBuf *DecodeMe,
3809                                  const char *SegmentStart,
3810                                  const char *SegmentEnd,
3811                                  StrBuf *ConvertBuf,
3812                                  StrBuf *ConvertBuf2,
3813                                  StrBuf *FoundCharset)
3814 {
3815         StrBuf StaticBuf;
3816         char charset[128];
3817         char encoding[16];
3818 #ifdef HAVE_ICONV
3819         iconv_t ic = (iconv_t)(-1);
3820 #else
3821         void *ic = NULL;
3822 #endif
3823         /* Now we handle foreign character sets properly encoded
3824          * in RFC2047 format.
3825          */
3826         StaticBuf.buf = (char*) SegmentStart; /*< it will just be read there... */
3827         StaticBuf.BufUsed = SegmentEnd - SegmentStart;
3828         StaticBuf.BufSize = DecodeMe->BufSize - (SegmentStart - DecodeMe->buf);
3829         extract_token(charset, SegmentStart, 1, '?', sizeof charset);
3830         if (FoundCharset != NULL) {
3831                 FlushStrBuf(FoundCharset);
3832                 StrBufAppendBufPlain(FoundCharset, charset, -1, 0);
3833         }
3834         extract_token(encoding, SegmentStart, 2, '?', sizeof encoding);
3835         StrBufExtract_token(ConvertBuf, &StaticBuf, 3, '?');
3836
3837         *encoding = toupper(*encoding);
3838         if (*encoding == 'B') { /**< base64 */
3839                 if (ConvertBuf2->BufSize < ConvertBuf->BufUsed)
3840                         IncreaseBuf(ConvertBuf2, 0, ConvertBuf->BufUsed);
3841                 ConvertBuf2->BufUsed = CtdlDecodeBase64(ConvertBuf2->buf,
3842                                                         ConvertBuf->buf,
3843                                                         ConvertBuf->BufUsed);
3844         }
3845         else if (*encoding == 'Q') {    /**< quoted-printable */
3846                 long pos;
3847
3848                 pos = 0;
3849                 while (pos < ConvertBuf->BufUsed)
3850                 {
3851                         if (ConvertBuf->buf[pos] == '_')
3852                                 ConvertBuf->buf[pos] = ' ';
3853                         pos++;
3854                 }
3855
3856                 if (ConvertBuf2->BufSize < ConvertBuf->BufUsed)
3857                         IncreaseBuf(ConvertBuf2, 0, ConvertBuf->BufUsed);
3858
3859                 ConvertBuf2->BufUsed = CtdlDecodeQuotedPrintable(
3860                         ConvertBuf2->buf,
3861                         ConvertBuf->buf,
3862                         ConvertBuf->BufUsed);
3863         }
3864         else {
3865                 StrBufAppendBuf(ConvertBuf2, ConvertBuf, 0);
3866         }
3867 #ifdef HAVE_ICONV
3868         ctdl_iconv_open("UTF-8", charset, &ic);
3869         if (ic != (iconv_t)(-1) ) {
3870 #endif
3871                 StrBufConvert(ConvertBuf2, ConvertBuf, &ic);
3872                 StrBufAppendBuf(Target, ConvertBuf2, 0);
3873 #ifdef HAVE_ICONV
3874                 iconv_close(ic);
3875         }
3876         else {
3877                 StrBufAppendBufPlain(Target, HKEY("(unreadable)"), 0);
3878         }
3879 #endif
3880 }
3881
3882 /**
3883  * @ingroup StrBuf_DeEnCoder
3884  * @brief Handle subjects with RFC2047 encoding such as: [deprecated old syntax!]
3885  * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
3886  * @param Target where to put the decoded string to
3887  * @param DecodeMe buffer with encoded string
3888  * @param DefaultCharset if we don't find one, which should we use?
3889  * @param FoundCharset overrides DefaultCharset if non-empty; If we find a charset inside of the string,
3890  *        put it here for later use where no string might be known.
3891  */
3892 void StrBuf_RFC822_to_Utf8(StrBuf *Target, const StrBuf *DecodeMe, const StrBuf* DefaultCharset, StrBuf *FoundCharset)
3893 {
3894         StrBuf *ConvertBuf;
3895         StrBuf *ConvertBuf2;
3896         ConvertBuf = NewStrBufPlain(NULL, StrLength(DecodeMe));
3897         ConvertBuf2 = NewStrBufPlain(NULL, StrLength(DecodeMe));
3898
3899         StrBuf_RFC822_2_Utf8(Target,
3900                              DecodeMe,
3901                              DefaultCharset,
3902                              FoundCharset,
3903                              ConvertBuf,
3904                              ConvertBuf2);
3905         FreeStrBuf(&ConvertBuf);
3906         FreeStrBuf(&ConvertBuf2);
3907 }
3908
3909 /**
3910  * @ingroup StrBuf_DeEnCoder
3911  * @brief Handle subjects with RFC2047 encoding such as:
3912  * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
3913  * @param Target where to put the decoded string to
3914  * @param DecodeMe buffer with encoded string
3915  * @param DefaultCharset if we don't find one, which should we use?
3916  * @param FoundCharset overrides DefaultCharset if non-empty; If we find a charset inside of the string,
3917  *        put it here for later use where no string might be known.
3918  * @param ConvertBuf workbuffer. feed in, you shouldn't care about its content.
3919  * @param ConvertBuf2 workbuffer. feed in, you shouldn't care about its content.
3920  */
3921 void StrBuf_RFC822_2_Utf8(StrBuf *Target,
3922                           const StrBuf *DecodeMe,
3923                           const StrBuf* DefaultCharset,
3924                           StrBuf *FoundCharset,
3925                           StrBuf *ConvertBuf,
3926                           StrBuf *ConvertBuf2)
3927 {
3928         StrBuf *DecodedInvalidBuf = NULL;
3929         const StrBuf *DecodeMee = DecodeMe;
3930         const char *start, *end, *next, *nextend, *ptr = NULL;
3931 #ifdef HAVE_ICONV
3932         iconv_t ic = (iconv_t)(-1) ;
3933 #endif
3934         const char *eptr;
3935         int passes = 0;
3936         int i;
3937         int illegal_non_rfc2047_encoding = 0;
3938
3939
3940         if (DecodeMe == NULL)
3941                 return;
3942         /* Sometimes, badly formed messages contain strings which were simply
3943          *  written out directly in some foreign character set instead of
3944          *  using RFC2047 encoding.  This is illegal but we will attempt to
3945          *  handle it anyway by converting from a user-specified default
3946          *  charset to UTF-8 if we see any nonprintable characters.
3947          */
3948
3949         for (i=0; i<DecodeMe->BufUsed; ++i) {
3950                 if ((DecodeMe->buf[i] < 32) || (DecodeMe->buf[i] > 126)) {
3951                         illegal_non_rfc2047_encoding = 1;
3952                         break;
3953                 }
3954         }
3955
3956         if ((illegal_non_rfc2047_encoding) &&
3957             (strcasecmp(ChrPtr(DefaultCharset), "UTF-8")) &&
3958             (strcasecmp(ChrPtr(DefaultCharset), "us-ascii")) )
3959         {
3960 #ifdef HAVE_ICONV
3961                 ctdl_iconv_open("UTF-8", ChrPtr(DefaultCharset), &ic);
3962                 if (ic != (iconv_t)(-1) ) {
3963                         DecodedInvalidBuf = NewStrBufDup(DecodeMe);
3964                         StrBufConvert(DecodedInvalidBuf, ConvertBuf, &ic);///TODO: don't void const?
3965                         DecodeMee = DecodedInvalidBuf;
3966                         iconv_close(ic);
3967                 }
3968 #endif
3969         }
3970
3971         /* pre evaluate the first pair */
3972         end = NULL;
3973         start = strstr(DecodeMee->buf, "=?");
3974         eptr = DecodeMee->buf + DecodeMee->BufUsed;
3975         if (start != NULL)
3976                 end = FindNextEnd (DecodeMee, start + 2);
3977         else {
3978                 StrBufAppendBuf(Target, DecodeMee, 0);
3979                 FreeStrBuf(&DecodedInvalidBuf);
3980                 return;
3981         }
3982
3983
3984         if (start != DecodeMee->buf) {
3985                 long nFront;
3986
3987                 nFront = start - DecodeMee->buf;
3988                 StrBufAppendBufPlain(Target, DecodeMee->buf, nFront, 0);
3989         }
3990         /*
3991          * Since spammers will go to all sorts of absurd lengths to get their
3992          * messages through, there are LOTS of corrupt headers out there.
3993          * So, prevent a really badly formed RFC2047 header from throwing
3994          * this function into an infinite loop.
3995          */
3996         while ((start != NULL) &&
3997                (end != NULL) &&
3998                (start < eptr) &&
3999                (end < eptr) &&
4000                (passes < 20))
4001         {
4002                 passes++;
4003                 DecodeSegment(Target,
4004                               DecodeMee,
4005                               start,
4006                               end,
4007                               ConvertBuf,
4008                               ConvertBuf2,
4009                               FoundCharset);
4010
4011                 next = strstr(end, "=?");
4012                 nextend = NULL;
4013                 if ((next != NULL) &&
4014                     (next < eptr))
4015                         nextend = FindNextEnd(DecodeMee, next);
4016                 if (nextend == NULL)
4017                         next = NULL;
4018
4019                 /* did we find two partitions */
4020                 if ((next != NULL) &&
4021                     ((next - end) > 2))
4022                 {
4023                         ptr = end + 2;
4024                         while ((ptr < next) &&
4025                                (isspace(*ptr) ||
4026                                 (*ptr == '\r') ||
4027                                 (*ptr == '\n') ||
4028                                 (*ptr == '\t')))
4029                                 ptr ++;
4030                         /*
4031                          * did we find a gab just filled with blanks?
4032                          * if not, copy its stuff over.
4033                          */
4034                         if (ptr != next)
4035                         {
4036                                 StrBufAppendBufPlain(Target,
4037                                                      end + 2,
4038                                                      next - end - 2,
4039                                                      0);
4040                         }
4041                 }
4042                 /* our next-pair is our new first pair now. */
4043                 ptr = end + 2;
4044                 start = next;
4045                 end = nextend;
4046         }
4047         end = ptr;
4048         nextend = DecodeMee->buf + DecodeMee->BufUsed;
4049         if ((end != NULL) && (end < nextend)) {
4050                 ptr = end;
4051                 while ( (ptr < nextend) &&
4052                         (isspace(*ptr) ||
4053                          (*ptr == '\r') ||
4054                          (*ptr == '\n') ||
4055                          (*ptr == '\t')))
4056                         ptr ++;
4057                 if (ptr < nextend)
4058                         StrBufAppendBufPlain(Target, end, nextend - end, 0);
4059         }
4060         FreeStrBuf(&DecodedInvalidBuf);
4061 }
4062
4063 /*******************************************************************************
4064  *                   Manipulating UTF-8 Strings                                *
4065  *******************************************************************************/
4066
4067 /**
4068  * @ingroup StrBuf
4069  * @brief evaluate the length of an utf8 special character sequence
4070  * @param Char the character to examine
4071  * @returns width of utf8 chars in bytes; if the sequence is broken 0 is returned; 1 if its simply ASCII.
4072  */
4073 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE)
4074 {
4075         int n = 0;
4076         unsigned char test = (1<<7);
4077
4078         if ((*CharS & 0xC0) != 0xC0)
4079                 return 1;
4080
4081         while ((n < 8) &&
4082                ((test & ((unsigned char)*CharS)) != 0))
4083         {
4084                 test = test >> 1;
4085                 n ++;
4086         }
4087         if ((n > 6) || ((CharE - CharS) < n))
4088                 n = 0;
4089         return n;
4090 }
4091
4092 /**
4093  * @ingroup StrBuf
4094  * @brief detect whether this char starts an utf-8 encoded char
4095  * @param Char character to inspect
4096  * @returns yes or no
4097  */
4098 static inline int Ctdl_IsUtf8SequenceStart(const char Char)
4099 {
4100 /** 11??.???? indicates an UTF8 Sequence. */
4101         return ((Char & 0xC0) == 0xC0);
4102 }
4103
4104 /**
4105  * @ingroup StrBuf
4106  * @brief measure the number of glyphs in an UTF8 string...
4107  * @param Buf string to measure
4108  * @returns the number of glyphs in Buf
4109  */
4110 long StrBuf_Utf8StrLen(StrBuf *Buf)
4111 {
4112         int n = 0;
4113         int m = 0;
4114         char *aptr, *eptr;
4115
4116         if ((Buf == NULL) || (Buf->BufUsed == 0))
4117                 return 0;
4118         aptr = Buf->buf;
4119         eptr = Buf->buf + Buf->BufUsed;
4120         while ((aptr < eptr) && (*aptr != '\0')) {
4121                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
4122                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
4123                         while ((aptr < eptr) && (*aptr++ != '\0')&& (m-- > 0) );
4124                         n ++;
4125                 }
4126                 else {
4127                         n++;
4128                         aptr++;
4129                 }
4130         }
4131         return n;
4132 }
4133
4134 /**
4135  * @ingroup StrBuf
4136  * @brief cuts a string after maxlen glyphs
4137  * @param Buf string to cut to maxlen glyphs
4138  * @param maxlen how long may the string become?
4139  * @returns current length of the string
4140  */
4141 long StrBuf_Utf8StrCut(StrBuf *Buf, int maxlen)
4142 {
4143         char *aptr, *eptr;
4144         int n = 0, m = 0;
4145
4146         aptr = Buf->buf;
4147         eptr = Buf->buf + Buf->BufUsed;
4148         while ((aptr < eptr) && (*aptr != '\0')) {
4149                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
4150                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
4151                         while ((*aptr++ != '\0') && (m-- > 0));
4152                         n ++;
4153                 }
4154                 else {
4155                         n++;
4156                         aptr++;
4157                 }
4158                 if (n > maxlen) {
4159                         *aptr = '\0';
4160                         Buf->BufUsed = aptr - Buf->buf;
4161                         return Buf->BufUsed;
4162                 }
4163         }
4164         return Buf->BufUsed;
4165
4166 }
4167
4168
4169
4170
4171
4172 /*******************************************************************************
4173  *                               wrapping ZLib                                 *
4174  *******************************************************************************/
4175
4176 /**
4177  * @ingroup StrBuf_DeEnCoder
4178  * @brief uses the same calling syntax as compress2(), but it
4179  *   creates a stream compatible with HTTP "Content-encoding: gzip"
4180  * @param dest compressed buffer
4181  * @param destLen length of the compresed data
4182  * @param source source to encode
4183  * @param sourceLen length of source to encode
4184  * @param level compression level
4185  */
4186 #ifdef HAVE_ZLIB
4187 int ZEXPORT compress_gzip(Bytef * dest,
4188                           size_t * destLen,
4189                           const Bytef * source,
4190                           uLong sourceLen,
4191                           int level)
4192 {
4193         /* write gzip header */
4194         snprintf((char *) dest, *destLen,
4195                  "%c%c%c%c%c%c%c%c%c%c",
4196                  gz_magic[0], gz_magic[1], Z_DEFLATED,
4197                  0 /*flags */ , 0, 0, 0, 0 /*time */ , 0 /* xflags */ ,
4198                  OS_CODE);
4199
4200         /* normal deflate */
4201         z_stream stream;
4202         int err;
4203         stream.next_in = (Bytef *) source;
4204         stream.avail_in = (uInt) sourceLen;
4205         stream.next_out = dest + 10L;   // after header
4206         stream.avail_out = (uInt) * destLen;
4207         if ((uLong) stream.avail_out != *destLen)
4208                 return Z_BUF_ERROR;
4209
4210         stream.zalloc = (alloc_func) 0;
4211         stream.zfree = (free_func) 0;
4212         stream.opaque = (voidpf) 0;
4213
4214         err = deflateInit2(&stream, level, Z_DEFLATED, -MAX_WBITS,
4215                            DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
4216         if (err != Z_OK)
4217                 return err;
4218
4219         err = deflate(&stream, Z_FINISH);
4220         if (err != Z_STREAM_END) {
4221                 deflateEnd(&stream);
4222                 return err == Z_OK ? Z_BUF_ERROR : err;
4223         }
4224         *destLen = stream.total_out + 10L;
4225
4226         /* write CRC and Length */
4227         uLong crc = crc32(0L, source, sourceLen);
4228         int n;
4229         for (n = 0; n < 4; ++n, ++*destLen) {
4230                 dest[*destLen] = (int) (crc & 0xff);
4231                 crc >>= 8;
4232         }
4233         uLong len = stream.total_in;
4234         for (n = 0; n < 4; ++n, ++*destLen) {
4235                 dest[*destLen] = (int) (len & 0xff);
4236                 len >>= 8;
4237         }
4238         err = deflateEnd(&stream);
4239         return err;
4240 }
4241 #endif
4242
4243
4244 /**
4245  * @ingroup StrBuf_DeEnCoder
4246  * @brief compress the buffer with gzip
4247  * Attention! If you feed this a Const String, you must maintain the uncompressed buffer yourself!
4248  * @param Buf buffer whose content is to be gzipped
4249  */
4250 int CompressBuffer(StrBuf *Buf)
4251 {
4252 #ifdef HAVE_ZLIB
4253         char *compressed_data = NULL;
4254         size_t compressed_len, bufsize;
4255         int i = 0;
4256
4257         bufsize = compressed_len = Buf->BufUsed +  (Buf->BufUsed / 100) + 100;
4258         compressed_data = malloc(compressed_len);
4259
4260         if (compressed_data == NULL)
4261                 return -1;
4262         /* Flush some space after the used payload so valgrind shuts up... */
4263         while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
4264                 Buf->buf[Buf->BufUsed + i++] = '\0';
4265         if (compress_gzip((Bytef *) compressed_data,
4266                           &compressed_len,
4267                           (Bytef *) Buf->buf,
4268                           (uLongf) Buf->BufUsed, Z_BEST_SPEED) == Z_OK) {
4269                 if (!Buf->ConstBuf)
4270                         free(Buf->buf);
4271                 Buf->buf = compressed_data;
4272                 Buf->BufUsed = compressed_len;
4273                 Buf->BufSize = bufsize;
4274                 /* Flush some space after the used payload so valgrind shuts up... */
4275                 i = 0;
4276                 while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
4277                         Buf->buf[Buf->BufUsed + i++] = '\0';
4278                 return 1;
4279         } else {
4280                 free(compressed_data);
4281         }
4282 #endif  /* HAVE_ZLIB */
4283         return 0;
4284 }
4285
4286 /*******************************************************************************
4287  *           File I/O; Callbacks to libevent                                   *
4288  *******************************************************************************/
4289
4290 long StrBuf_read_one_chunk_callback (int fd, short event, IOBuffer *FB)
4291 {
4292         long bufremain = 0;
4293         int n;
4294
4295         if ((FB == NULL) || (FB->Buf == NULL))
4296                 return -1;
4297
4298         /*
4299          * check whether the read pointer is somewhere in a range
4300          * where a cut left is inexpensive
4301          */
4302
4303         if (FB->ReadWritePointer != NULL)
4304         {
4305                 long already_read;
4306
4307                 already_read = FB->ReadWritePointer - FB->Buf->buf;
4308                 bufremain = FB->Buf->BufSize - FB->Buf->BufUsed - 1;
4309
4310                 if (already_read != 0) {
4311                         long unread;
4312
4313                         unread = FB->Buf->BufUsed - already_read;
4314
4315                         /* else nothing to compact... */
4316                         if (unread == 0) {
4317                                 FB->ReadWritePointer = FB->Buf->buf;
4318                                 bufremain = FB->Buf->BufSize;
4319                         }
4320                         else if ((unread < 64) ||
4321                                  (bufremain < already_read))
4322                         {
4323                                 /*
4324                                  * if its just a tiny bit remaining, or we run out of space...
4325                                  * lets tidy up.
4326                                  */
4327                                 FB->Buf->BufUsed = unread;
4328                                 if (unread < already_read)
4329                                         memcpy(FB->Buf->buf, FB->ReadWritePointer, unread);
4330                                 else
4331                                         memmove(FB->Buf->buf, FB->ReadWritePointer, unread);
4332                                 FB->ReadWritePointer = FB->Buf->buf;
4333                                 bufremain = FB->Buf->BufSize - unread - 1;
4334                         }
4335                         else if (bufremain < (FB->Buf->BufSize / 10))
4336                         {
4337                                 /* get a bigger buffer */
4338
4339                                 IncreaseBuf(FB->Buf, 0, FB->Buf->BufUsed + 1);
4340
4341                                 FB->ReadWritePointer = FB->Buf->buf + unread;
4342
4343                                 bufremain = FB->Buf->BufSize - unread - 1;
4344 /*TODO: special increase function that won't copy the already read! */
4345                         }
4346                 }
4347                 else if (bufremain < 10) {
4348                         IncreaseBuf(FB->Buf, 1, FB->Buf->BufUsed + 10);
4349
4350                         FB->ReadWritePointer = FB->Buf->buf;
4351
4352                         bufremain = FB->Buf->BufSize - FB->Buf->BufUsed - 1;
4353                 }
4354
4355         }
4356         else {
4357                 FB->ReadWritePointer = FB->Buf->buf;
4358                 bufremain = FB->Buf->BufSize - 1;
4359         }
4360
4361         n = read(fd, FB->Buf->buf + FB->Buf->BufUsed, bufremain);
4362
4363         if (n > 0) {
4364                 FB->Buf->BufUsed += n;
4365                 FB->Buf->buf[FB->Buf->BufUsed] = '\0';
4366         }
4367         return n;
4368 }
4369
4370 int StrBuf_write_one_chunk_callback(int fd, short event, IOBuffer *FB)
4371 {
4372         long WriteRemain;
4373         int n;
4374
4375         if ((FB == NULL) || (FB->Buf == NULL))
4376                 return -1;
4377
4378         if (FB->ReadWritePointer != NULL)
4379         {
4380                 WriteRemain = FB->Buf->BufUsed -
4381                         (FB->ReadWritePointer -
4382                          FB->Buf->buf);
4383         }
4384         else {
4385                 FB->ReadWritePointer = FB->Buf->buf;
4386                 WriteRemain = FB->Buf->BufUsed;
4387         }
4388
4389         n = write(fd, FB->ReadWritePointer, WriteRemain);
4390         if (n > 0) {
4391                 FB->ReadWritePointer += n;
4392
4393                 if (FB->ReadWritePointer ==
4394                     FB->Buf->buf + FB->Buf->BufUsed)
4395                 {
4396                         FlushStrBuf(FB->Buf);
4397                         FB->ReadWritePointer = NULL;
4398                         return 0;
4399                 }
4400         // check whether we've got something to write
4401         // get the maximum chunk plus the pointer we can send
4402         // write whats there
4403         // if not all was sent, remember the send pointer for the next time
4404                 return FB->ReadWritePointer - FB->Buf->buf + FB->Buf->BufUsed;
4405         }
4406         return n;
4407 }
4408
4409 /**
4410  * @ingroup StrBuf_IO
4411  * @brief extract a "next line" from Buf; Ptr to persist across several iterations
4412  * @param LineBuf your line will be copied here.
4413  * @param FB BLOB with lines of text...
4414  * @param Ptr moved arround to keep the next-line across several iterations
4415  *        has to be &NULL on start; will be &NotNULL on end of buffer
4416  * @returns size of copied buffer
4417  */
4418 eReadState StrBufChunkSipLine(StrBuf *LineBuf, IOBuffer *FB)
4419 {
4420         const char *aptr, *ptr, *eptr;
4421         char *optr, *xptr;
4422
4423         if ((FB == NULL) || (LineBuf == NULL) || (LineBuf->buf == NULL))
4424                 return eReadFail;
4425
4426
4427         if ((FB->Buf == NULL) || (FB->ReadWritePointer == StrBufNOTNULL)) {
4428                 FB->ReadWritePointer = StrBufNOTNULL;
4429                 return eReadFail;
4430         }
4431
4432         FlushStrBuf(LineBuf);
4433         if (FB->ReadWritePointer == NULL)
4434                 ptr = aptr = FB->Buf->buf;
4435         else
4436                 ptr = aptr = FB->ReadWritePointer;
4437
4438         optr = LineBuf->buf;
4439         eptr = FB->Buf->buf + FB->Buf->BufUsed;
4440         xptr = LineBuf->buf + LineBuf->BufSize - 1;
4441
4442         while ((ptr <= eptr) &&
4443                (*ptr != '\n') &&
4444                (*ptr != '\r') )
4445         {
4446                 *optr = *ptr;
4447                 optr++; ptr++;
4448                 if (optr == xptr) {
4449                         LineBuf->BufUsed = optr - LineBuf->buf;
4450                         IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
4451                         optr = LineBuf->buf + LineBuf->BufUsed;
4452                         xptr = LineBuf->buf + LineBuf->BufSize - 1;
4453                 }
4454         }
4455
4456         if (ptr >= eptr) {
4457                 if (optr > LineBuf->buf)
4458                         optr --;
4459                 if ((*(ptr - 1) != '\r') && (*(ptr - 1) != '\n')) {
4460                         LineBuf->BufUsed = optr - LineBuf->buf;
4461                         *optr = '\0';
4462                         if ((FB->ReadWritePointer != NULL) &&
4463                             (FB->ReadWritePointer != FB->Buf->buf))
4464                         {
4465                                 /* Ok, the client application read all the data
4466                                    it was interested in so far. Since there is more to read,
4467                                    we now shrink the buffer, and move the rest over.
4468                                 */
4469                                 StrBufCutLeft(FB->Buf,
4470                                               FB->ReadWritePointer - FB->Buf->buf);
4471                                 FB->ReadWritePointer = FB->Buf->buf;
4472                         }
4473                         return eMustReadMore;
4474                 }
4475         }
4476         LineBuf->BufUsed = optr - LineBuf->buf;
4477         *optr = '\0';
4478         if ((ptr <= eptr) && (*ptr == '\r'))
4479                 ptr ++;
4480         if ((ptr <= eptr) && (*ptr == '\n'))
4481                 ptr ++;
4482
4483         if (ptr < eptr) {
4484                 FB->ReadWritePointer = ptr;
4485         }
4486         else {
4487                 FlushStrBuf(FB->Buf);
4488                 FB->ReadWritePointer = NULL;
4489         }
4490
4491         return eReadSuccess;
4492 }
4493
4494 /**
4495  * @ingroup StrBuf_CHUNKED_IO
4496  * @brief check whether the chunk-buffer has more data waiting or not.
4497  * @param FB Chunk-Buffer to inspect
4498  */
4499 eReadState StrBufCheckBuffer(IOBuffer *FB)
4500 {
4501         if (FB == NULL)
4502                 return eReadFail;
4503         if (FB->Buf->BufUsed == 0)
4504                 return eReadSuccess;
4505         if (FB->ReadWritePointer == NULL)
4506                 return eBufferNotEmpty;
4507         if (FB->Buf->buf + FB->Buf->BufUsed > FB->ReadWritePointer)
4508                 return eBufferNotEmpty;
4509         return eReadSuccess;
4510 }
4511
4512 long IOBufferStrLength(IOBuffer *FB)
4513 {
4514         if ((FB == NULL) || (FB->Buf == NULL))
4515                 return 0;
4516         if (FB->ReadWritePointer == NULL)
4517                 return StrLength(FB->Buf);
4518
4519         return StrLength(FB->Buf) - (FB->ReadWritePointer - FB->Buf->buf);
4520 }
4521
4522 inline static void FDIOBufferFlush(FDIOBuffer *FDB)
4523 {
4524         memset(FDB, 0, sizeof(FDIOBuffer));
4525         FDB->OtherFD = -1;
4526         FDB->SplicePipe[0] = -1;
4527         FDB->SplicePipe[1] = -1;
4528 }
4529
4530 void FDIOBufferInit(FDIOBuffer *FDB, IOBuffer *IO, int FD, long TotalSendSize)
4531 {
4532         FDIOBufferFlush(FDB);
4533
4534         FDB->TotalSendSize = TotalSendSize;
4535         if (TotalSendSize > 0)
4536                 FDB->ChunkSize = TotalSendSize;
4537         else
4538         {
4539                 TotalSendSize = SIZ * 10;
4540                 FDB->ChunkSize = TotalSendSize;
4541         }
4542         FDB->IOB = IO;
4543
4544 #ifdef LINUX_SPLICE
4545         if (EnableSplice)
4546                 pipe(FDB->SplicePipe);
4547         else
4548 #endif
4549                 FDB->ChunkBuffer = NewStrBufPlain(NULL, TotalSendSize+ 1);
4550
4551         FDB->OtherFD = FD;
4552 }
4553
4554 void FDIOBufferDelete(FDIOBuffer *FDB)
4555 {
4556 #ifdef LINUX_SPLICE
4557         if (EnableSplice)
4558         {
4559                 if (FDB->SplicePipe[0] > 0)
4560                         close(FDB->SplicePipe[0]);
4561                 if (FDB->SplicePipe[1] > 0)
4562                         close(FDB->SplicePipe[1]);
4563         }
4564         else
4565 #endif
4566                 FreeStrBuf(&FDB->ChunkBuffer);
4567
4568         if (FDB->OtherFD > 0)
4569                 close(FDB->OtherFD);
4570         FDIOBufferFlush(FDB);
4571 }
4572
4573 int FileSendChunked(FDIOBuffer *FDB, const char **Err)
4574 {
4575         ssize_t sent, pipesize;
4576
4577         if (FDB->TotalSendSize > 0)
4578         {
4579 #ifdef LINUX_SPLICE
4580                 if (EnableSplice)
4581                 {
4582                         if (FDB->PipeSize == 0)
4583                         {
4584                                 pipesize = splice(FDB->OtherFD,
4585                                                   &FDB->TotalSentAlready,
4586                                                   FDB->SplicePipe[1],
4587                                                   NULL,
4588                                                   FDB->ChunkSendRemain,
4589                                                   SPLICE_F_MOVE);
4590
4591                                 if (pipesize == -1)
4592                                 {
4593                                         *Err = strerror(errno);
4594                                         return pipesize;
4595                                 }
4596                                 FDB->PipeSize = pipesize;
4597                         }
4598                         sent =  splice(FDB->SplicePipe[0],
4599                                        NULL,
4600                                        FDB->IOB->fd,
4601                                        NULL,
4602                                        FDB->PipeSize,
4603                                        SPLICE_F_MORE | SPLICE_F_MOVE | SPLICE_F_NONBLOCK);
4604                         if (sent == -1)
4605                         {
4606                                 *Err = strerror(errno);
4607                                 return sent;
4608                         }
4609                         FDB->PipeSize -= sent;
4610                         FDB->ChunkSendRemain -= sent;
4611                         return sent;
4612                 }
4613                 else
4614 #endif
4615                 {
4616                         char *pRead;
4617                         long nRead = 0;
4618
4619                         pRead = FDB->ChunkBuffer->buf;
4620                         while ((FDB->ChunkBuffer->BufUsed < FDB->TotalSendSize) && (nRead >= 0))
4621                         {
4622                                 nRead = read(FDB->OtherFD, pRead, FDB->TotalSendSize - FDB->ChunkBuffer->BufUsed);
4623                                 if (nRead > 0) {
4624                                         FDB->ChunkBuffer->BufUsed += nRead;
4625                                         FDB->ChunkBuffer->buf[FDB->ChunkBuffer->BufUsed] = '\0';
4626                                 }
4627                                 else if (nRead == 0) {}
4628                                 else return nRead;
4629                         }
4630
4631                         nRead = write(FDB->IOB->fd,
4632                                       FDB->ChunkBuffer->buf     + FDB->TotalSentAlready,
4633                                       FDB->ChunkBuffer->BufUsed - FDB->TotalSentAlready);
4634
4635                         if (nRead >= 0) {
4636                                 FDB->TotalSentAlready += nRead;
4637                                 FDB->ChunkSendRemain -= nRead;
4638                                 return FDB->ChunkSendRemain;
4639                         }
4640                         else {
4641                                 return nRead;
4642                         }
4643                 }
4644         }
4645         else
4646         {
4647 #ifdef LINUX_SPLICE
4648                 if (EnableSplice)
4649                 {
4650                         if (FDB->PipeSize == 0)
4651                         {
4652                                 pipesize = splice(FDB->OtherFD,
4653                                                   &FDB->TotalSentAlready,
4654                                                   FDB->SplicePipe[1],
4655                                                   NULL,
4656                                                   SIZ * 10,
4657                                                   SPLICE_F_MOVE);
4658
4659                                 if (pipesize == -1)
4660                                 {
4661                                         *Err = strerror(errno);
4662                                         return pipesize;
4663                                 }
4664                                 FDB->PipeSize = pipesize;
4665                                 if (pipesize == 0)
4666                                         return -1;
4667                         }
4668                         sent =  splice(FDB->SplicePipe[0],
4669                                        NULL,
4670                                        FDB->IOB->fd,
4671                                        NULL,
4672                                        FDB->PipeSize,
4673                                        SPLICE_F_MORE | SPLICE_F_MOVE | SPLICE_F_NONBLOCK);
4674                         if (sent == -1)
4675                         {
4676                                 *Err = strerror(errno);
4677                                 return sent;
4678                         }
4679                         FDB->PipeSize -= sent;
4680                         FDB->ChunkSendRemain -= sent;
4681                         return sent;
4682                 }
4683                 else
4684 #endif
4685                 {
4686                         char *pRead;
4687                         long nRead = 0;
4688
4689                         pRead = FDB->ChunkBuffer->buf;
4690                         while ((FDB->ChunkSendRemain == 0) &&
4691                                (FDB->ChunkBuffer->BufUsed < FDB->ChunkBuffer->BufSize) &&
4692                                (nRead >= 0))
4693                         {
4694                                 FDB->TotalSentAlready = 0;
4695                                 nRead = read(FDB->OtherFD, pRead, FDB->ChunkBuffer->BufSize - FDB->ChunkBuffer->BufUsed);
4696                                 if (nRead > 0) {
4697                                         FDB->ChunkBuffer->BufUsed += nRead;
4698                                         FDB->ChunkBuffer->buf[FDB->ChunkBuffer->BufUsed] = '\0';
4699                                         FDB->ChunkSendRemain += nRead;
4700                                 }
4701                                 else if (nRead == 0)
4702                                 {
4703                                         return -1;
4704                                 }
4705                                 else
4706                                 {
4707                                         *Err = strerror(errno);
4708                                         return nRead;
4709                                 }
4710                         }
4711
4712                         nRead = write(FDB->IOB->fd,
4713                                       FDB->ChunkBuffer->buf     + FDB->TotalSentAlready,
4714                                       FDB->ChunkBuffer->BufUsed - FDB->TotalSentAlready);
4715
4716                         if (nRead >= 0) {
4717                                 FDB->TotalSentAlready += nRead;
4718                                 FDB->ChunkSendRemain -= nRead;
4719                                 if (FDB->ChunkSendRemain == 0)
4720                                 {
4721                                         FDB->ChunkBuffer->BufUsed = 0;
4722                                         FDB->TotalSentAlready = 0;
4723                                 }
4724                                 return FDB->ChunkSendRemain;
4725                         }
4726                         else {
4727                                 return nRead;
4728                         }
4729                 }
4730         }
4731 }
4732
4733 int FileRecvChunked(FDIOBuffer *FDB, const char **Err)
4734 {
4735         ssize_t sent, pipesize;
4736
4737 #ifdef LINUX_SPLICE
4738         if (EnableSplice)
4739         {
4740                 if (FDB->PipeSize == 0)
4741                 {
4742                         pipesize = splice(FDB->IOB->fd,
4743                                           NULL,
4744                                           FDB->SplicePipe[1],
4745                                           NULL,
4746                                           FDB->ChunkSendRemain,
4747                                           SPLICE_F_MORE | SPLICE_F_MOVE|SPLICE_F_NONBLOCK);
4748
4749                         if (pipesize == -1)
4750                         {
4751                                 *Err = strerror(errno);
4752                                 return pipesize;
4753                         }
4754                         FDB->PipeSize = pipesize;
4755                 }
4756
4757                 sent = splice(FDB->SplicePipe[0],
4758                               NULL,
4759                               FDB->OtherFD,
4760                               &FDB->TotalSentAlready,
4761                               FDB->PipeSize,
4762                               SPLICE_F_MORE | SPLICE_F_MOVE);
4763
4764                 if (sent == -1)
4765                 {
4766                         *Err = strerror(errno);
4767                         return sent;
4768                 }
4769                 FDB->PipeSize -= sent;
4770                 FDB->ChunkSendRemain -= sent;
4771                 return sent;
4772         }
4773         else
4774 #endif
4775         {
4776                 sent = read(FDB->IOB->fd, FDB->ChunkBuffer->buf, FDB->ChunkSendRemain);
4777                 if (sent > 0) {
4778                         int nWritten = 0;
4779                         int rc;
4780
4781                         FDB->ChunkBuffer->BufUsed = sent;
4782
4783                         while (nWritten < FDB->ChunkBuffer->BufUsed) {
4784                                 rc =  write(FDB->OtherFD, FDB->ChunkBuffer->buf + nWritten, FDB->ChunkBuffer->BufUsed - nWritten);
4785                                 if (rc < 0) {
4786                                         *Err = strerror(errno);
4787                                         return rc;
4788                                 }
4789                                 nWritten += rc;
4790
4791                         }
4792                         FDB->ChunkBuffer->BufUsed = 0;
4793                         FDB->TotalSentAlready += sent;
4794                         FDB->ChunkSendRemain -= sent;
4795                         return FDB->ChunkSendRemain;
4796                 }
4797                 else if (sent < 0) {
4798                         *Err = strerror(errno);
4799                         return sent;
4800                 }
4801                 return 0;
4802         }
4803 }
4804
4805 int FileMoveChunked(FDIOBuffer *FDB, const char **Err)
4806 {
4807         ssize_t sent, pipesize;
4808
4809 #ifdef LINUX_SPLICE
4810         if (EnableSplice)
4811         {
4812                 if (FDB->PipeSize == 0)
4813                 {
4814                         pipesize = splice(FDB->IOB->fd,
4815                                           &FDB->TotalReadAlready,
4816                                           FDB->SplicePipe[1],
4817                                           NULL,
4818                                           FDB->ChunkSendRemain,
4819                                           SPLICE_F_MORE | SPLICE_F_MOVE|SPLICE_F_NONBLOCK);
4820
4821                         if (pipesize == -1)
4822                         {
4823                                 *Err = strerror(errno);
4824                                 return pipesize;
4825                         }
4826                         FDB->PipeSize = pipesize;
4827                 }
4828
4829                 sent = splice(FDB->SplicePipe[0],
4830                               NULL,
4831                               FDB->OtherFD,
4832                               &FDB->TotalSentAlready,
4833                               FDB->PipeSize,
4834                               SPLICE_F_MORE | SPLICE_F_MOVE);
4835
4836                 if (sent == -1)
4837                 {
4838                         *Err = strerror(errno);
4839                         return sent;
4840                 }
4841                 FDB->PipeSize -= sent;
4842                 FDB->ChunkSendRemain -= sent;
4843                 return sent;
4844         }
4845         else
4846 #endif
4847         {
4848                 sent = read(FDB->IOB->fd, FDB->ChunkBuffer->buf, FDB->ChunkSendRemain);
4849                 if (sent > 0) {
4850                         int nWritten = 0;
4851                         int rc;
4852
4853                         FDB->ChunkBuffer->BufUsed = sent;
4854
4855                         while (nWritten < FDB->ChunkBuffer->BufUsed) {
4856                                 rc =  write(FDB->OtherFD, FDB->ChunkBuffer->buf + nWritten, FDB->ChunkBuffer->BufUsed - nWritten);
4857                                 if (rc < 0) {
4858                                         *Err = strerror(errno);
4859                                         return rc;
4860                                 }
4861                                 nWritten += rc;
4862
4863                         }
4864                         FDB->ChunkBuffer->BufUsed = 0;
4865                         FDB->TotalSentAlready += sent;
4866                         FDB->ChunkSendRemain -= sent;
4867                         return FDB->ChunkSendRemain;
4868                 }
4869                 else if (sent < 0) {
4870                         *Err = strerror(errno);
4871                         return sent;
4872                 }
4873                 return 0;
4874         }
4875 }
4876
4877 eReadState WriteIOBAlreadyRead(FDIOBuffer *FDB, const char **Error)
4878 {
4879         int IsNonBlock;
4880         int fdflags;
4881         long rlen;
4882         long should_write;
4883         int nSuccessLess = 0;
4884         struct timeval tv;
4885         fd_set rfds;
4886
4887         fdflags = fcntl(FDB->OtherFD, F_GETFL);
4888         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
4889
4890         while ((FDB->IOB->ReadWritePointer - FDB->IOB->Buf->buf < FDB->IOB->Buf->BufUsed) &&
4891                (FDB->ChunkSendRemain > 0))
4892         {
4893                 if (IsNonBlock){
4894                         tv.tv_sec = 1; /* selectresolution; */
4895                         tv.tv_usec = 0;
4896
4897                         FD_ZERO(&rfds);
4898                         FD_SET(FDB->OtherFD, &rfds);
4899                         if (select(FDB->OtherFD + 1, NULL, &rfds, NULL, &tv) == -1) {
4900                                 *Error = strerror(errno);
4901                                 return eReadFail;
4902                         }
4903                 }
4904                 if (IsNonBlock && !  FD_ISSET(FDB->OtherFD, &rfds)) {
4905                         nSuccessLess ++;
4906                         continue;
4907                 }
4908
4909                 should_write = FDB->IOB->Buf->BufUsed -
4910                         (FDB->IOB->ReadWritePointer - FDB->IOB->Buf->buf);
4911                 if (should_write > FDB->ChunkSendRemain)
4912                         should_write = FDB->ChunkSendRemain;
4913
4914                 rlen = write(FDB->OtherFD,
4915                              FDB->IOB->ReadWritePointer,
4916                              should_write);
4917                 if (rlen < 1) {
4918                         *Error = strerror(errno);
4919
4920                         return eReadFail;
4921                 }
4922                 FDB->TotalSentAlready += rlen;
4923                 FDB->IOB->ReadWritePointer += rlen;
4924                 FDB->ChunkSendRemain -= rlen;
4925         }
4926         if (FDB->IOB->ReadWritePointer >= FDB->IOB->Buf->buf + FDB->IOB->Buf->BufUsed)
4927         {
4928                 FlushStrBuf(FDB->IOB->Buf);
4929                 FDB->IOB->ReadWritePointer = NULL;
4930         }
4931
4932         if (FDB->ChunkSendRemain == 0)
4933                 return eReadSuccess;
4934         else
4935                 return eMustReadMore;
4936 }
4937
4938 /*******************************************************************************
4939  *           File I/O; Prefer buffered read since its faster!                  *
4940  *******************************************************************************/
4941
4942 /**
4943  * @ingroup StrBuf_IO
4944  * @brief Read a line from socket
4945  * flushes and closes the FD on error
4946  * @param buf the buffer to get the input to
4947  * @param fd pointer to the filedescriptor to read
4948  * @param append Append to an existing string or replace?
4949  * @param Error strerror() on error
4950  * @returns numbers of chars read
4951  */
4952 int StrBufTCP_read_line(StrBuf *buf, int *fd, int append, const char **Error)
4953 {
4954         int len, rlen, slen;
4955
4956         if ((buf == NULL) || (buf->buf == NULL)) {
4957                 *Error = strerror(EINVAL);
4958                 return -1;
4959         }
4960
4961         if (!append)
4962                 FlushStrBuf(buf);
4963
4964         slen = len = buf->BufUsed;
4965         while (1) {
4966                 rlen = read(*fd, &buf->buf[len], 1);
4967                 if (rlen < 1) {
4968                         *Error = strerror(errno);
4969
4970                         close(*fd);
4971                         *fd = -1;
4972
4973                         return -1;
4974                 }
4975                 if (buf->buf[len] == '\n')
4976                         break;
4977                 if (buf->buf[len] != '\r')
4978                         len ++;
4979                 if (len + 2 >= buf->BufSize) {
4980                         buf->BufUsed = len;
4981                         buf->buf[len+1] = '\0';
4982                         IncreaseBuf(buf, 1, -1);
4983                 }
4984         }
4985         buf->BufUsed = len;
4986         buf->buf[len] = '\0';
4987         return len - slen;
4988 }
4989
4990 /**
4991  * @ingroup StrBuf_BufferedIO
4992  * @brief Read a line from socket
4993  * flushes and closes the FD on error
4994  * @param Line the line to read from the fd / I/O Buffer
4995  * @param buf the buffer to get the input to
4996  * @param fd pointer to the filedescriptor to read
4997  * @param timeout number of successless selects until we bail out
4998  * @param selectresolution how long to wait on each select
4999  * @param Error strerror() on error
5000  * @returns numbers of chars read
5001  */
5002 int StrBufTCP_read_buffered_line(StrBuf *Line,
5003                                  StrBuf *buf,
5004                                  int *fd,
5005                                  int timeout,
5006                                  int selectresolution,
5007                                  const char **Error)
5008 {
5009         int len, rlen;
5010         int nSuccessLess = 0;
5011         fd_set rfds;
5012         char *pch = NULL;
5013         int fdflags;
5014         int IsNonBlock;
5015         struct timeval tv;
5016
5017         if (buf->BufUsed > 0) {
5018                 pch = strchr(buf->buf, '\n');
5019                 if (pch != NULL) {
5020                         rlen = 0;
5021                         len = pch - buf->buf;
5022                         if (len > 0 && (*(pch - 1) == '\r') )
5023                                 rlen ++;
5024                         StrBufSub(Line, buf, 0, len - rlen);
5025                         StrBufCutLeft(buf, len + 1);
5026                         return len - rlen;
5027                 }
5028         }
5029
5030         if (buf->BufSize - buf->BufUsed < 10)
5031                 IncreaseBuf(buf, 1, -1);
5032
5033         fdflags = fcntl(*fd, F_GETFL);
5034         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
5035
5036         while ((nSuccessLess < timeout) && (pch == NULL)) {
5037                 if (IsNonBlock){
5038                         tv.tv_sec = selectresolution;
5039                         tv.tv_usec = 0;
5040
5041                         FD_ZERO(&rfds);
5042                         FD_SET(*fd, &rfds);
5043                         if (select(*fd + 1, NULL, &rfds, NULL, &tv) == -1) {
5044                                 *Error = strerror(errno);
5045                                 close (*fd);
5046                                 *fd = -1;
5047                                 return -1;
5048                         }
5049                 }
5050                 if (IsNonBlock && !  FD_ISSET(*fd, &rfds)) {
5051                         nSuccessLess ++;
5052                         continue;
5053                 }
5054                 rlen = read(*fd,
5055                             &buf->buf[buf->BufUsed],
5056                             buf->BufSize - buf->BufUsed - 1);
5057                 if (rlen < 1) {
5058                         *Error = strerror(errno);
5059                         close(*fd);
5060                         *fd = -1;
5061                         return -1;
5062                 }
5063                 else if (rlen > 0) {
5064                         nSuccessLess = 0;
5065                         buf->BufUsed += rlen;
5066                         buf->buf[buf->BufUsed] = '\0';
5067                         pch = strchr(buf->buf, '\n');
5068                         if ((pch == NULL) &&
5069                             (buf->BufUsed + 10 > buf->BufSize) &&
5070                             (IncreaseBuf(buf, 1, -1) == -1))
5071                                 return -1;
5072                         continue;
5073                 }
5074
5075         }
5076         if (pch != NULL) {
5077                 rlen = 0;
5078                 len = pch - buf->buf;
5079                 if (len > 0 && (*(pch - 1) == '\r') )
5080                         rlen ++;
5081                 StrBufSub(Line, buf, 0, len - rlen);
5082                 StrBufCutLeft(buf, len + 1);
5083                 return len - rlen;
5084         }
5085         return -1;
5086
5087 }
5088
5089 static const char *ErrRBLF_PreConditionFailed="StrBufTCP_read_buffered_line_fast: Wrong arguments or invalid Filedescriptor";
5090 static const char *ErrRBLF_SelectFailed="StrBufTCP_read_buffered_line_fast: Select failed without reason";
5091 static const char *ErrRBLF_NotEnoughSentFromServer="StrBufTCP_read_buffered_line_fast: No complete line was sent from peer";
5092 /**
5093  * @ingroup StrBuf_BufferedIO
5094  * @brief Read a line from socket
5095  * flushes and closes the FD on error
5096  * @param Line where to append our Line read from the fd / I/O Buffer;
5097  * @param IOBuf the buffer to get the input to; lifetime pair to FD
5098  * @param Pos pointer to the current read position, should be NULL initialized on opening the FD it belongs to.!
5099  * @param fd pointer to the filedescriptor to read
5100  * @param timeout number of successless selects until we bail out
5101  * @param selectresolution how long to wait on each select
5102  * @param Error strerror() on error
5103  * @returns numbers of chars read or -1 in case of error. "\n" will become 0
5104  */
5105 int StrBufTCP_read_buffered_line_fast(StrBuf *Line,
5106                                       StrBuf *IOBuf,
5107                                       const char **Pos,
5108                                       int *fd,
5109                                       int timeout,
5110                                       int selectresolution,
5111                                       const char **Error)
5112 {
5113         const char *pche = NULL;
5114         const char *pos = NULL;
5115         const char *pLF;
5116         int len, rlen, retlen;
5117         int nSuccessLess = 0;
5118         fd_set rfds;
5119         const char *pch = NULL;
5120         int fdflags;
5121         int IsNonBlock;
5122         struct timeval tv;
5123
5124         retlen = 0;
5125         if ((Line == NULL) ||
5126             (Pos == NULL) ||
5127             (IOBuf == NULL) ||
5128             (*fd == -1))
5129         {
5130                 if (Pos != NULL)
5131                         *Pos = NULL;
5132                 *Error = ErrRBLF_PreConditionFailed;
5133                 return -1;
5134         }
5135
5136         pos = *Pos;
5137         if ((IOBuf->BufUsed > 0) &&
5138             (pos != NULL) &&
5139             (pos < IOBuf->buf + IOBuf->BufUsed))
5140         {
5141                 char *pcht;
5142
5143                 pche = IOBuf->buf + IOBuf->BufUsed;
5144                 pch = pos;
5145                 pcht = Line->buf;
5146
5147                 while ((pch < pche) && (*pch != '\n'))
5148                 {
5149                         if (Line->BufUsed + 10 > Line->BufSize)
5150                         {
5151                                 long apos;
5152                                 apos = pcht - Line->buf;
5153                                 *pcht = '\0';
5154                                 IncreaseBuf(Line, 1, -1);
5155                                 pcht = Line->buf + apos;
5156                         }
5157                         *pcht++ = *pch++;
5158                         Line->BufUsed++;
5159                         retlen++;
5160                 }
5161
5162                 len = pch - pos;
5163                 if (len > 0 && (*(pch - 1) == '\r') )
5164                 {
5165                         retlen--;
5166                         len --;
5167                         pcht --;
5168                         Line->BufUsed --;
5169                 }
5170                 *pcht = '\0';
5171
5172                 if ((pch >= pche) || (*pch == '\0'))
5173                 {
5174                         FlushStrBuf(IOBuf);
5175                         *Pos = NULL;
5176                         pch = NULL;
5177                         pos = 0;
5178                 }
5179
5180                 if ((pch != NULL) &&
5181                     (pch <= pche))
5182                 {
5183                         if (pch + 1 >= pche) {
5184                                 *Pos = NULL;
5185                                 FlushStrBuf(IOBuf);
5186                         }
5187                         else
5188                                 *Pos = pch + 1;
5189
5190                         return retlen;
5191                 }
5192                 else
5193                         FlushStrBuf(IOBuf);
5194         }
5195
5196         /* If we come here, Pos is Unset since we read everything into Line, and now go for more. */
5197
5198         if (IOBuf->BufSize - IOBuf->BufUsed < 10)
5199                 IncreaseBuf(IOBuf, 1, -1);
5200
5201         fdflags = fcntl(*fd, F_GETFL);
5202         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
5203
5204         pLF = NULL;
5205         while ((nSuccessLess < timeout) &&
5206                (pLF == NULL) &&
5207                (*fd != -1)) {
5208                 if (IsNonBlock)
5209                 {
5210                         tv.tv_sec = 1;
5211                         tv.tv_usec = 0;
5212
5213                         FD_ZERO(&rfds);
5214                         FD_SET(*fd, &rfds);
5215                         if (select((*fd) + 1, &rfds, NULL, NULL, &tv) == -1) {
5216                                 *Error = strerror(errno);
5217                                 close (*fd);
5218                                 *fd = -1;
5219                                 if (*Error == NULL)
5220                                         *Error = ErrRBLF_SelectFailed;
5221                                 return -1;
5222                         }
5223                         if (! FD_ISSET(*fd, &rfds) != 0) {
5224                                 nSuccessLess ++;
5225                                 continue;
5226                         }
5227                 }
5228                 rlen = read(*fd,
5229                             &IOBuf->buf[IOBuf->BufUsed],
5230                             IOBuf->BufSize - IOBuf->BufUsed - 1);
5231                 if (rlen < 1) {
5232                         *Error = strerror(errno);
5233                         close(*fd);
5234                         *fd = -1;
5235                         return -1;
5236                 }
5237                 else if (rlen > 0) {
5238                         nSuccessLess = 0;
5239                         pLF = IOBuf->buf + IOBuf->BufUsed;
5240                         IOBuf->BufUsed += rlen;
5241                         IOBuf->buf[IOBuf->BufUsed] = '\0';
5242
5243                         pche = IOBuf->buf + IOBuf->BufUsed;
5244
5245                         while ((pLF < pche) && (*pLF != '\n'))
5246                                 pLF ++;
5247                         if ((pLF >= pche) || (*pLF == '\0'))
5248                                 pLF = NULL;
5249
5250                         if (IOBuf->BufUsed + 10 > IOBuf->BufSize)
5251                         {
5252                                 long apos = 0;
5253
5254                                 if (pLF != NULL) apos = pLF - IOBuf->buf;
5255                                 IncreaseBuf(IOBuf, 1, -1);
5256                                 if (pLF != NULL) pLF = IOBuf->buf + apos;
5257                         }
5258
5259                         continue;
5260                 }
5261                 else
5262                 {
5263                         nSuccessLess++;
5264                 }
5265         }
5266         *Pos = NULL;
5267         if (pLF != NULL) {
5268                 pos = IOBuf->buf;
5269                 len = pLF - pos;
5270                 if (len > 0 && (*(pLF - 1) == '\r') )
5271                         len --;
5272                 StrBufAppendBufPlain(Line, ChrPtr(IOBuf), len, 0);
5273                 if (pLF + 1 >= IOBuf->buf + IOBuf->BufUsed)
5274                 {
5275                         FlushStrBuf(IOBuf);
5276                 }
5277                 else
5278                         *Pos = pLF + 1;
5279                 return retlen + len;
5280         }
5281         *Error = ErrRBLF_NotEnoughSentFromServer;
5282         return -1;
5283
5284 }
5285
5286 static const char *ErrRBLF_BLOBPreConditionFailed="StrBufReadBLOB: Wrong arguments or invalid Filedescriptor";
5287 /**
5288  * @ingroup StrBuf_IO
5289  * @brief Input binary data from socket
5290  * flushes and closes the FD on error
5291  * @param Buf the buffer to get the input to
5292  * @param fd pointer to the filedescriptor to read
5293  * @param append Append to an existing string or replace?
5294  * @param nBytes the maximal number of bytes to read
5295  * @param Error strerror() on error
5296  * @returns numbers of chars read
5297  */
5298 int StrBufReadBLOB(StrBuf *Buf, int *fd, int append, long nBytes, const char **Error)
5299 {
5300         int fdflags;
5301         int rlen;
5302         int nSuccessLess;
5303         int nRead = 0;
5304         char *ptr;
5305         int IsNonBlock;
5306         struct timeval tv;
5307         fd_set rfds;
5308
5309         if ((Buf == NULL) || (Buf->buf == NULL) || (*fd == -1))
5310         {
5311                 *Error = ErrRBLF_BLOBPreConditionFailed;
5312                 return -1;
5313         }
5314         if (!append)
5315                 FlushStrBuf(Buf);
5316         if (Buf->BufUsed + nBytes >= Buf->BufSize)
5317                 IncreaseBuf(Buf, 1, Buf->BufUsed + nBytes);
5318
5319         ptr = Buf->buf + Buf->BufUsed;
5320
5321         fdflags = fcntl(*fd, F_GETFL);
5322         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
5323         nSuccessLess = 0;
5324         while ((nRead < nBytes) &&
5325                (*fd != -1))
5326         {
5327                 if (IsNonBlock)
5328                 {
5329                         tv.tv_sec = 1;
5330                         tv.tv_usec = 0;
5331
5332                         FD_ZERO(&rfds);
5333                         FD_SET(*fd, &rfds);
5334                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
5335                                 *Error = strerror(errno);
5336                                 close (*fd);
5337                                 *fd = -1;
5338                                 if (*Error == NULL)
5339                                         *Error = ErrRBLF_SelectFailed;
5340                                 return -1;
5341                         }
5342                         if (! FD_ISSET(*fd, &rfds) != 0) {
5343                                 nSuccessLess ++;
5344                                 continue;
5345                         }
5346                 }
5347
5348                 if ((rlen = read(*fd,
5349                                  ptr,
5350                                  nBytes - nRead)) == -1) {
5351                         close(*fd);
5352                         *fd = -1;
5353                         *Error = strerror(errno);
5354                         return rlen;
5355                 }
5356                 nRead += rlen;
5357                 ptr += rlen;
5358                 Buf->BufUsed += rlen;
5359         }
5360         Buf->buf[Buf->BufUsed] = '\0';
5361         return nRead;
5362 }
5363
5364 const char *ErrRBB_BLOBFPreConditionFailed = "StrBufReadBLOBBuffered: to many selects; aborting.";
5365 const char *ErrRBB_too_many_selects        = "StrBufReadBLOBBuffered: to many selects; aborting.";
5366 /**
5367  * @ingroup StrBuf_BufferedIO
5368  * @brief Input binary data from socket
5369  * flushes and closes the FD on error
5370  * @param Blob put binary thing here
5371  * @param IOBuf the buffer to get the input to
5372  * @param Pos offset inside of IOBuf
5373  * @param fd pointer to the filedescriptor to read
5374  * @param append Append to an existing string or replace?
5375  * @param nBytes the maximal number of bytes to read
5376  * @param check whether we should search for '000\n' terminators in case of timeouts
5377  * @param Error strerror() on error
5378  * @returns numbers of chars read
5379  */
5380 int StrBufReadBLOBBuffered(StrBuf *Blob,
5381                            StrBuf *IOBuf,
5382                            const char **Pos,
5383                            int *fd,
5384                            int append,
5385                            long nBytes,
5386                            int check,
5387                            const char **Error)
5388 {
5389         const char *pos;
5390         int fdflags;
5391         int rlen = 0;
5392         int nRead = 0;
5393         int nAlreadyRead = 0;
5394         int IsNonBlock;
5395         char *ptr;
5396         fd_set rfds;
5397         struct timeval tv;
5398         int nSuccessLess = 0;
5399         int MaxTries;
5400
5401         if ((Blob == NULL)  ||
5402             (*fd == -1)     ||
5403             (IOBuf == NULL) ||
5404             (Pos == NULL))
5405         {
5406                 if (Pos != NULL)
5407                         *Pos = NULL;
5408                 *Error = ErrRBB_BLOBFPreConditionFailed;
5409                 return -1;
5410         }
5411
5412         if (!append)
5413                 FlushStrBuf(Blob);
5414         if (Blob->BufUsed + nBytes >= Blob->BufSize)
5415                 IncreaseBuf(Blob, append, Blob->BufUsed + nBytes);
5416
5417         pos = *Pos;
5418
5419         if (pos != NULL)
5420                 rlen = pos - IOBuf->buf;
5421         rlen = IOBuf->BufUsed - rlen;
5422
5423
5424         if ((IOBuf->BufUsed > 0) &&
5425             (pos != NULL) &&
5426             (pos < IOBuf->buf + IOBuf->BufUsed))
5427         {
5428                 if (rlen < nBytes) {
5429                         memcpy(Blob->buf + Blob->BufUsed, pos, rlen);
5430                         Blob->BufUsed += rlen;
5431                         Blob->buf[Blob->BufUsed] = '\0';
5432                         nAlreadyRead = nRead = rlen;
5433                         *Pos = NULL;
5434                 }
5435                 if (rlen >= nBytes) {
5436                         memcpy(Blob->buf + Blob->BufUsed, pos, nBytes);
5437                         Blob->BufUsed += nBytes;
5438                         Blob->buf[Blob->BufUsed] = '\0';
5439                         if (rlen == nBytes) {
5440                                 *Pos = NULL;
5441                                 FlushStrBuf(IOBuf);
5442                         }
5443                         else
5444                                 *Pos += nBytes;
5445                         return nBytes;
5446                 }
5447         }
5448
5449         FlushStrBuf(IOBuf);
5450         *Pos = NULL;
5451         if (IOBuf->BufSize < nBytes - nRead)
5452                 IncreaseBuf(IOBuf, 0, nBytes - nRead);
5453         ptr = IOBuf->buf;
5454
5455         fdflags = fcntl(*fd, F_GETFL);
5456         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
5457         if (IsNonBlock)
5458                 MaxTries =   1000;
5459         else
5460                 MaxTries = 100000;
5461
5462         nBytes -= nRead;
5463         nRead = 0;
5464         while ((nSuccessLess < MaxTries) &&
5465                (nRead < nBytes) &&
5466                (*fd != -1)) {
5467                 if (IsNonBlock)
5468                 {
5469                         tv.tv_sec = 1;
5470                         tv.tv_usec = 0;
5471
5472                         FD_ZERO(&rfds);
5473                         FD_SET(*fd, &rfds);
5474                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
5475                                 *Error = strerror(errno);
5476                                 close (*fd);
5477                                 *fd = -1;
5478                                 if (*Error == NULL)
5479                                         *Error = ErrRBLF_SelectFailed;
5480                                 return -1;
5481                         }
5482                         if (! FD_ISSET(*fd, &rfds) != 0) {
5483                                 nSuccessLess ++;
5484                                 continue;
5485                         }
5486                 }
5487                 rlen = read(*fd,
5488                             ptr,
5489                             IOBuf->BufSize - (ptr - IOBuf->buf));
5490                 if (rlen == -1) {
5491                         close(*fd);
5492                         *fd = -1;
5493                         *Error = strerror(errno);
5494                         return rlen;
5495                 }
5496                 else if (rlen == 0){
5497                         if ((check == NNN_TERM) &&
5498                             (nRead > 5) &&
5499                             (strncmp(IOBuf->buf + IOBuf->BufUsed - 5, "\n000\n", 5) == 0))
5500                         {
5501                                 StrBufPlain(Blob, HKEY("\n000\n"));
5502                                 StrBufCutRight(Blob, 5);
5503                                 return Blob->BufUsed;
5504                         }
5505                         else if (!IsNonBlock)
5506                                 nSuccessLess ++;
5507                         else if (nSuccessLess > MaxTries) {
5508                                 FlushStrBuf(IOBuf);
5509                                 *Error = ErrRBB_too_many_selects;
5510                                 return -1;
5511                         }
5512                 }
5513                 else if (rlen > 0) {
5514                         nSuccessLess = 0;
5515                         nRead += rlen;
5516                         ptr += rlen;
5517                         IOBuf->BufUsed += rlen;
5518                 }
5519         }
5520         if (nSuccessLess >= MaxTries) {
5521                 FlushStrBuf(IOBuf);
5522                 *Error = ErrRBB_too_many_selects;
5523                 return -1;
5524         }
5525
5526         if (nRead > nBytes) {
5527                 *Pos = IOBuf->buf + nBytes;
5528         }
5529         Blob->buf[Blob->BufUsed] = '\0';
5530         StrBufAppendBufPlain(Blob, IOBuf->buf, nBytes, 0);
5531         if (*Pos == NULL) {
5532                 FlushStrBuf(IOBuf);
5533         }
5534         return nRead + nAlreadyRead;
5535 }
5536
5537 /**
5538  * @ingroup StrBuf_IO
5539  * @brief extract a "next line" from Buf; Ptr to persist across several iterations
5540  * @param LineBuf your line will be copied here.
5541  * @param Buf BLOB with lines of text...
5542  * @param Ptr moved arround to keep the next-line across several iterations
5543  *        has to be &NULL on start; will be &NotNULL on end of buffer
5544  * @returns size of remaining buffer
5545  */
5546 int StrBufSipLine(StrBuf *LineBuf, const StrBuf *Buf, const char **Ptr)
5547 {
5548         const char *aptr, *ptr, *eptr;
5549         char *optr, *xptr;
5550
5551         if ((Buf == NULL) ||
5552             (*Ptr == StrBufNOTNULL) ||
5553             (LineBuf == NULL)||
5554             (LineBuf->buf == NULL))
5555         {
5556                 *Ptr = StrBufNOTNULL;
5557                 return 0;
5558         }
5559
5560         FlushStrBuf(LineBuf);
5561         if (*Ptr==NULL)
5562                 ptr = aptr = Buf->buf;
5563         else
5564                 ptr = aptr = *Ptr;
5565
5566         optr = LineBuf->buf;
5567         eptr = Buf->buf + Buf->BufUsed;
5568         xptr = LineBuf->buf + LineBuf->BufSize - 1;
5569
5570         while ((ptr <= eptr) &&
5571                (*ptr != '\n') &&
5572                (*ptr != '\r') )
5573         {
5574                 *optr = *ptr;
5575                 optr++; ptr++;
5576                 if (optr == xptr) {
5577                         LineBuf->BufUsed = optr - LineBuf->buf;
5578                         IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
5579                         optr = LineBuf->buf + LineBuf->BufUsed;
5580                         xptr = LineBuf->buf + LineBuf->BufSize - 1;
5581                 }
5582         }
5583
5584         if ((ptr >= eptr) && (optr > LineBuf->buf))
5585                 optr --;
5586         LineBuf->BufUsed = optr - LineBuf->buf;
5587         *optr = '\0';
5588         if ((ptr <= eptr) && (*ptr == '\r'))
5589                 ptr ++;
5590         if ((ptr <= eptr) && (*ptr == '\n'))
5591                 ptr ++;
5592
5593         if (ptr < eptr) {
5594                 *Ptr = ptr;
5595         }
5596         else {
5597                 *Ptr = StrBufNOTNULL;
5598         }
5599
5600         return Buf->BufUsed - (ptr - Buf->buf);
5601 }
5602
5603
5604 /**
5605  * @ingroup StrBuf_IO
5606  * @brief removes double slashes from pathnames
5607  * @param Dir directory string to filter
5608  * @param RemoveTrailingSlash allows / disallows trailing slashes
5609  */
5610 void StrBufStripSlashes(StrBuf *Dir, int RemoveTrailingSlash)
5611 {
5612         char *a, *b;
5613
5614         a = b = Dir->buf;
5615
5616         while (!IsEmptyStr(a)) {
5617                 if (*a == '/') {
5618                         while (*a == '/')
5619                                 a++;
5620                         *b = '/';
5621                         b++;
5622                 }
5623                 else {
5624                         *b = *a;
5625                         b++; a++;
5626                 }
5627         }
5628         if ((RemoveTrailingSlash) &&
5629             (b > Dir->buf) &&
5630             (*(b - 1) == '/')){
5631                 b--;
5632         }
5633         *b = '\0';
5634         Dir->BufUsed = b - Dir->buf;
5635 }
5636
5637