libcitadel/lib/stringbuf.c

   1 /*
   2  * Copyright (c) 1987-2013 by the citadel.org team
   3  *
   4  * This program is open source software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 3 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #define _GNU_SOURCE
  20 #include "sysdep.h"
  21 #include <ctype.h>
  22 #include <errno.h>
  23 #include <string.h>
  24 #include <unistd.h>
  25 #include <string.h>
  26 #include <stdio.h>
  27 #include <sys/select.h>
  28 #include <fcntl.h>
  29 #include <sys/types.h>
  30 #define SHOW_ME_VAPPEND_PRINTF
  31 #include <stdarg.h>
  32
  33 #include "libcitadel.h"
  34
  35 #include "b64/cencode.h"
  36 #include "b64/cdecode.h"
  37
  38 #ifdef HAVE_ICONV
  39 #include <iconv.h>
  40 #endif
  41
  42 #ifdef HAVE_BACKTRACE
  43 #include <execinfo.h>
  44 #endif
  45
  46 #ifdef UNDEF_MEMCPY
  47 #undef memcpy
  48 #endif
  49
  50 #ifdef HAVE_ZLIB
  51 #include <zlib.h>
  52 int ZEXPORT compress_gzip(Bytef * dest, size_t * destLen,
  53                           const Bytef * source, uLong sourceLen, int level);
  54 #endif
  55 int BaseStrBufSize = 64;
  56 int EnableSplice = 0;
  57
  58 const char *StrBufNOTNULL = ((char*) NULL) - 1;
  59
  60 const char HexList[256][3] = {
  61         "00","01","02","03","04","05","06","07","08","09","0A","0B","0C","0D","0E","0F",
  62         "10","11","12","13","14","15","16","17","18","19","1A","1B","1C","1D","1E","1F",
  63         "20","21","22","23","24","25","26","27","28","29","2A","2B","2C","2D","2E","2F",
  64         "30","31","32","33","34","35","36","37","38","39","3A","3B","3C","3D","3E","3F",
  65         "40","41","42","43","44","45","46","47","48","49","4A","4B","4C","4D","4E","4F",
  66         "50","51","52","53","54","55","56","57","58","59","5A","5B","5C","5D","5E","5F",
  67         "60","61","62","63","64","65","66","67","68","69","6A","6B","6C","6D","6E","6F",
  68         "70","71","72","73","74","75","76","77","78","79","7A","7B","7C","7D","7E","7F",
  69         "80","81","82","83","84","85","86","87","88","89","8A","8B","8C","8D","8E","8F",
  70         "90","91","92","93","94","95","96","97","98","99","9A","9B","9C","9D","9E","9F",
  71         "A0","A1","A2","A3","A4","A5","A6","A7","A8","A9","AA","AB","AC","AD","AE","AF",
  72         "B0","B1","B2","B3","B4","B5","B6","B7","B8","B9","BA","BB","BC","BD","BE","BF",
  73         "C0","C1","C2","C3","C4","C5","C6","C7","C8","C9","CA","CB","CC","CD","CE","CF",
  74         "D0","D1","D2","D3","D4","D5","D6","D7","D8","D9","DA","DB","DC","DD","DE","DF",
  75         "E0","E1","E2","E3","E4","E5","E6","E7","E8","E9","EA","EB","EC","ED","EE","EF",
  76         "F0","F1","F2","F3","F4","F5","F6","F7","F8","F9","FA","FB","FC","FD","FE","FF"};
  77
  78 /**
  79  * @defgroup StrBuf Stringbuffer, A class for manipulating strings with dynamic buffers
  80  * StrBuf is a versatile class, aiding the handling of dynamic strings
  81  *  * reduce de/reallocations
  82  *  * reduce the need to remeasure it
  83  *  * reduce scanning over the string (in @ref StrBuf_NextTokenizer "Tokenizers")
  84  *  * allow asyncroneous IO for line and Blob based operations
  85  *  * reduce the use of memove in those
  86  *  * Quick filling in several operations with append functions
  87  */
  88
  89 /**
  90  * @defgroup StrBuf_DeConstructors Create/Destroy StrBufs
  91  * @ingroup StrBuf
  92  */
  93
  94 /**
  95  * @defgroup StrBuf_Cast Cast operators to interact with char* based code
  96  * @ingroup StrBuf
  97  * use these operators to interfere with code demanding char*;
  98  * if you need to own the content, smash me. Avoid, since we loose the length information.
  99  */
 100
 101 /**
 102  * @defgroup StrBuf_Filler Create/Replace/Append Content into a StrBuf
 103  * @ingroup StrBuf
 104  * operations to get your Strings into a StrBuf, manipulating them, or appending
 105  */
 106 /**
 107  * @defgroup StrBuf_NextTokenizer Fast tokenizer to pull tokens in sequence
 108  * @ingroup StrBuf
 109  * Quick tokenizer; demands of the user to pull its tokens in sequence
 110  */
 111
 112 /**
 113  * @defgroup StrBuf_Tokenizer tokenizer Functions; Slow ones.
 114  * @ingroup StrBuf
 115  * versatile tokenizer; random access to tokens, but slower; Prefer the @ref StrBuf_NextTokenizer "Next Tokenizer"
 116  */
 117
 118 /**
 119  * @defgroup StrBuf_BufferedIO Buffered IO with Asynchroneous reads and no unneeded memmoves (the fast ones)
 120  * @ingroup StrBuf
 121  * File IO to fill StrBufs; Works with work-buffer shared across several calls;
 122  * External Cursor to maintain the current read position inside of the buffer
 123  * the non-fast ones will use memove to keep the start of the buffer the read buffer (which is slower)
 124  */
 125
 126 /**
 127  * @defgroup StrBuf_IO FileIO; Prefer @ref StrBuf_BufferedIO
 128  * @ingroup StrBuf
 129  * Slow I/O; avoid.
 130  */
 131
 132 /**
 133  * @defgroup StrBuf_DeEnCoder functions to translate the contents of a buffer
 134  * @ingroup StrBuf
 135  * these functions translate the content of a buffer into another representation;
 136  * some are combined Fillers and encoders
 137  */
 138
 139 /**
 140  * Private Structure for the Stringbuffer
 141  */
 142 struct StrBuf {
 143         char *buf;         /**< the pointer to the dynamic buffer */
 144         long BufSize;      /**< how many spcae do we optain */
 145         long BufUsed;      /**< StNumber of Chars used excluding the trailing \\0 */
 146         int ConstBuf;      /**< are we just a wrapper arround a static buffer and musn't we be changed? */
 147 #ifdef SIZE_DEBUG
 148         long nIncreases;   /**< for profiling; cound how many times we needed more */
 149         char bt [SIZ];     /**< Stacktrace of last increase */
 150         char bt_lastinc [SIZ]; /**< How much did we increase last time? */
 151 #endif
 152 };
 153
 154
 155 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE);
 156 static inline int Ctdl_IsUtf8SequenceStart(const char Char);
 157
 158 #ifdef SIZE_DEBUG
 159 #ifdef HAVE_BACKTRACE
 160 static void StrBufBacktrace(StrBuf *Buf, int which)
 161 {
 162         int n;
 163         char *pstart, *pch;
 164         void *stack_frames[50];
 165         size_t size, i;
 166         char **strings;
 167
 168         if (which)
 169                 pstart = pch = Buf->bt;
 170         else
 171                 pstart = pch = Buf->bt_lastinc;
 172         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
 173         strings = backtrace_symbols(stack_frames, size);
 174         for (i = 0; i < size; i++) {
 175                 if (strings != NULL)
 176                         n = snprintf(pch, SIZ - (pch - pstart), "%s\\n", strings[i]);
 177                 else
 178                         n = snprintf(pch, SIZ - (pch - pstart), "%p\\n", stack_frames[i]);
 179                 pch += n;
 180         }
 181         free(strings);
 182
 183
 184 }
 185 #endif
 186
 187 void dbg_FreeStrBuf(StrBuf *FreeMe, char *FromWhere)
 188 {
 189         if (hFreeDbglog == -1){
 190                 pid_t pid = getpid();
 191                 char path [SIZ];
 192                 snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
 193                 hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
 194         }
 195         if ((*FreeMe)->nIncreases > 0)
 196         {
 197                 char buf[SIZ * 3];
 198                 long n;
 199                 n = snprintf(buf, SIZ * 3, "%c+|%ld|%ld|%ld|%s|%s|\n",
 200                              FromWhere,
 201                              (*FreeMe)->nIncreases,
 202                              (*FreeMe)->BufUsed,
 203                              (*FreeMe)->BufSize,
 204                              (*FreeMe)->bt,
 205                              (*FreeMe)->bt_lastinc);
 206                 n = write(hFreeDbglog, buf, n);
 207         }
 208         else
 209         {
 210                 char buf[128];
 211                 long n;
 212                 n = snprintf(buf, 128, "%c_|0|%ld%ld|\n",
 213                              FromWhere,
 214                              (*FreeMe)->BufUsed,
 215                              (*FreeMe)->BufSize);
 216                 n = write(hFreeDbglog, buf, n);
 217         }
 218 }
 219
 220 void dbg_IncreaseBuf(StrBuf *IncMe)
 221 {
 222         Buf->nIncreases++;
 223 #ifdef HAVE_BACKTRACE
 224         StrBufBacktrace(Buf, 1);
 225 #endif
 226 }
 227
 228 void dbg_Init(StrBuf *Buf)
 229 {
 230         Buf->nIncreases = 0;
 231         Buf->bt[0] = '\0';
 232         Buf->bt_lastinc[0] = '\0';
 233 #ifdef HAVE_BACKTRACE
 234         StrBufBacktrace(Buf, 0);
 235 #endif
 236 }
 237
 238 #else
 239 /* void it... */
 240 #define dbg_FreeStrBuf(a, b)
 241 #define dbg_IncreaseBuf(a)
 242 #define dbg_Init(a)
 243
 244 #endif
 245
 246 /**
 247  * @ingroup StrBuf
 248  * @brief swaps the contents of two StrBufs
 249  * this is to be used to have cheap switched between a work-buffer and a target buffer
 250  * @param A First one
 251  * @param B second one
 252  */
 253 static inline void SwapBuffers(StrBuf *A, StrBuf *B)
 254 {
 255         StrBuf C;
 256
 257         memcpy(&C, A, sizeof(*A));
 258         memcpy(A, B, sizeof(*B));
 259         memcpy(B, &C, sizeof(C));
 260
 261 }
 262
 263 /**
 264  * @ingroup StrBuf_Cast
 265  * @brief Cast operator to Plain String
 266  * @note if the buffer is altered by StrBuf operations, this pointer may become
 267  *  invalid. So don't lean on it after altering the buffer!
 268  *  Since this operation is considered cheap, rather call it often than risking
 269  *  your pointer to become invalid!
 270  * @param Str the string we want to get the c-string representation for
 271  * @returns the Pointer to the Content. Don't mess with it!
 272  */
 273 inline const char *ChrPtr(const StrBuf *Str)
 274 {
 275         if (Str == NULL)
 276                 return "";
 277         return Str->buf;
 278 }
 279
 280 /**
 281  * @ingroup StrBuf_Cast
 282  * @brief since we know strlen()'s result, provide it here.
 283  * @param Str the string to return the length to
 284  * @returns contentlength of the buffer
 285  */
 286 inline int StrLength(const StrBuf *Str)
 287 {
 288         return (Str != NULL) ? Str->BufUsed : 0;
 289 }
 290
 291 /**
 292  * @ingroup StrBuf_DeConstructors
 293  * @brief local utility function to resize the buffer
 294  * @param Buf the buffer whichs storage we should increase
 295  * @param KeepOriginal should we copy the original buffer or just start over with a new one
 296  * @param DestSize what should fit in after?
 297  */
 298 static int IncreaseBuf(StrBuf *Buf, int KeepOriginal, int DestSize)
 299 {
 300         char *NewBuf;
 301         size_t NewSize = Buf->BufSize * 2;
 302
 303         if (Buf->ConstBuf)
 304                 return -1;
 305
 306         if (DestSize > 0)
 307                 while ((NewSize <= DestSize) && (NewSize != 0))
 308                         NewSize *= 2;
 309
 310         if (NewSize == 0)
 311                 return -1;
 312
 313         NewBuf= (char*) malloc(NewSize);
 314         if (NewBuf == NULL)
 315                 return -1;
 316
 317         if (KeepOriginal && (Buf->BufUsed > 0))
 318         {
 319                 memcpy(NewBuf, Buf->buf, Buf->BufUsed);
 320         }
 321         else
 322         {
 323                 NewBuf[0] = '\0';
 324                 Buf->BufUsed = 0;
 325         }
 326         free (Buf->buf);
 327         Buf->buf = NewBuf;
 328         Buf->BufSize = NewSize;
 329
 330         dbg_IncreaseBuf(Buf);
 331
 332         return Buf->BufSize;
 333 }
 334
 335 /**
 336  * @ingroup StrBuf_DeConstructors
 337  * @brief shrink / increase an _EMPTY_ buffer to NewSize. Buffercontent is thoroughly ignored and flushed.
 338  * @param Buf Buffer to shrink (has to be empty)
 339  * @param ThreshHold if the buffer is bigger then this, its readjusted
 340  * @param NewSize if we Shrink it, how big are we going to be afterwards?
 341  */
 342 void ReAdjustEmptyBuf(StrBuf *Buf, long ThreshHold, long NewSize)
 343 {
 344         if ((Buf != NULL) &&
 345             (Buf->BufUsed == 0) &&
 346             (Buf->BufSize < ThreshHold)) {
 347                 free(Buf->buf);
 348                 Buf->buf = (char*) malloc(NewSize);
 349                 Buf->BufUsed = 0;
 350                 Buf->BufSize = NewSize;
 351         }
 352 }
 353
 354 /**
 355  * @ingroup StrBuf_DeConstructors
 356  * @brief shrink long term buffers to their real size so they don't waste memory
 357  * @param Buf buffer to shrink
 358  * @param Force if not set, will just executed if the buffer is much to big; set for lifetime strings
 359  * @returns physical size of the buffer
 360  */
 361 long StrBufShrinkToFit(StrBuf *Buf, int Force)
 362 {
 363         if (Buf == NULL)
 364                 return -1;
 365         if (Force ||
 366             (Buf->BufUsed + (Buf->BufUsed / 3) > Buf->BufSize))
 367         {
 368                 char *TmpBuf;
 369
 370                 TmpBuf = (char*) malloc(Buf->BufUsed + 1);
 371                 if (TmpBuf == NULL)
 372                         return -1;
 373
 374                 memcpy (TmpBuf, Buf->buf, Buf->BufUsed + 1);
 375                 Buf->BufSize = Buf->BufUsed + 1;
 376                 free(Buf->buf);
 377                 Buf->buf = TmpBuf;
 378         }
 379         return Buf->BufUsed;
 380 }
 381
 382 /**
 383  * @ingroup StrBuf_DeConstructors
 384  * @brief Allocate a new buffer with default buffer size
 385  * @returns the new stringbuffer
 386  */
 387 StrBuf* NewStrBuf(void)
 388 {
 389         StrBuf *NewBuf;
 390
 391         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 392         if (NewBuf == NULL)
 393                 return NULL;
 394
 395         NewBuf->buf = (char*) malloc(BaseStrBufSize);
 396         if (NewBuf->buf == NULL)
 397         {
 398                 free(NewBuf);
 399                 return NULL;
 400         }
 401         NewBuf->buf[0] = '\0';
 402         NewBuf->BufSize = BaseStrBufSize;
 403         NewBuf->BufUsed = 0;
 404         NewBuf->ConstBuf = 0;
 405
 406         dbg_Init (NewBuf);
 407
 408         return NewBuf;
 409 }
 410
 411 /**
 412  * @ingroup StrBuf_DeConstructors
 413  * @brief Copy Constructor; returns a duplicate of CopyMe
 414  * @param CopyMe Buffer to faxmilate
 415  * @returns the new stringbuffer
 416  */
 417 StrBuf* NewStrBufDup(const StrBuf *CopyMe)
 418 {
 419         StrBuf *NewBuf;
 420
 421         if (CopyMe == NULL)
 422                 return NewStrBuf();
 423
 424         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 425         if (NewBuf == NULL)
 426                 return NULL;
 427
 428         NewBuf->buf = (char*) malloc(CopyMe->BufSize);
 429         if (NewBuf->buf == NULL)
 430         {
 431                 free(NewBuf);
 432                 return NULL;
 433         }
 434
 435         memcpy(NewBuf->buf, CopyMe->buf, CopyMe->BufUsed + 1);
 436         NewBuf->BufUsed = CopyMe->BufUsed;
 437         NewBuf->BufSize = CopyMe->BufSize;
 438         NewBuf->ConstBuf = 0;
 439
 440         dbg_Init(NewBuf);
 441
 442         return NewBuf;
 443 }
 444
 445 /**
 446  * @ingroup StrBuf_DeConstructors
 447  * @brief Copy Constructor; CreateRelpaceMe will contain CopyFlushMe afterwards.
 448  * @param NoMe if non-NULL, we will use that buffer as value; KeepOriginal will abused as len.
 449  * @param CopyFlushMe Buffer to faxmilate if KeepOriginal, or to move into CreateRelpaceMe if !KeepOriginal.
 450  * @param CreateRelpaceMe If NULL, will be created, else Flushed and filled CopyFlushMe
 451  * @param KeepOriginal should CopyFlushMe remain intact? or may we Steal its buffer?
 452  * @returns the new stringbuffer
 453  */
 454 void NewStrBufDupAppendFlush(StrBuf **CreateRelpaceMe, StrBuf *CopyFlushMe, const char *NoMe, int KeepOriginal)
 455 {
 456         StrBuf *NewBuf;
 457
 458         if (CreateRelpaceMe == NULL)
 459                 return;
 460
 461         if (NoMe != NULL)
 462         {
 463                 if (*CreateRelpaceMe != NULL)
 464                         StrBufPlain(*CreateRelpaceMe, NoMe, KeepOriginal);
 465                 else
 466                         *CreateRelpaceMe = NewStrBufPlain(NoMe, KeepOriginal);
 467                 return;
 468         }
 469
 470         if (CopyFlushMe == NULL)
 471         {
 472                 if (*CreateRelpaceMe != NULL)
 473                         FlushStrBuf(*CreateRelpaceMe);
 474                 else
 475                         *CreateRelpaceMe = NewStrBuf();
 476                 return;
 477         }
 478
 479         /*
 480          * Randomly Chosen: bigger than 64 chars is cheaper to swap the buffers instead of copying.
 481          * else *CreateRelpaceMe may use more memory than needed in a longer term, CopyFlushMe might
 482          * be a big IO-Buffer...
 483          */
 484         if (KeepOriginal || (StrLength(CopyFlushMe) < 256))
 485         {
 486                 if (*CreateRelpaceMe == NULL)
 487                 {
 488                         *CreateRelpaceMe = NewBuf = NewStrBufPlain(NULL, CopyFlushMe->BufUsed);
 489                         dbg_Init(NewBuf);
 490                 }
 491                 else
 492                 {
 493                         NewBuf = *CreateRelpaceMe;
 494                         FlushStrBuf(NewBuf);
 495                 }
 496                 StrBufAppendBuf(NewBuf, CopyFlushMe, 0);
 497         }
 498         else
 499         {
 500                 if (*CreateRelpaceMe == NULL)
 501                 {
 502                         *CreateRelpaceMe = NewBuf = NewStrBufPlain(NULL, CopyFlushMe->BufUsed);
 503                         dbg_Init(NewBuf);
 504                 }
 505                 else
 506                         NewBuf = *CreateRelpaceMe;
 507                 SwapBuffers (NewBuf, CopyFlushMe);
 508         }
 509         if (!KeepOriginal)
 510                 FlushStrBuf(CopyFlushMe);
 511         return;
 512 }
 513
 514 /**
 515  * @ingroup StrBuf_DeConstructors
 516  * @brief create a new Buffer using an existing c-string
 517  * this function should also be used if you want to pre-suggest
 518  * the buffer size to allocate in conjunction with ptr == NULL
 519  * @param ptr the c-string to copy; may be NULL to create a blank instance
 520  * @param nChars How many chars should we copy; -1 if we should measure the length ourselves
 521  * @returns the new stringbuffer
 522  */
 523 StrBuf* NewStrBufPlain(const char* ptr, int nChars)
 524 {
 525         StrBuf *NewBuf;
 526         size_t Siz = BaseStrBufSize;
 527         size_t CopySize;
 528
 529         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 530         if (NewBuf == NULL)
 531                 return NULL;
 532
 533         if (nChars < 0)
 534                 CopySize = strlen((ptr != NULL)?ptr:"");
 535         else
 536                 CopySize = nChars;
 537
 538         while ((Siz <= CopySize) && (Siz != 0))
 539                 Siz *= 2;
 540
 541         if (Siz == 0)
 542         {
 543                 free(NewBuf);
 544                 return NULL;
 545         }
 546
 547         NewBuf->buf = (char*) malloc(Siz);
 548         if (NewBuf->buf == NULL)
 549         {
 550                 free(NewBuf);
 551                 return NULL;
 552         }
 553         NewBuf->BufSize = Siz;
 554         if (ptr != NULL) {
 555                 memcpy(NewBuf->buf, ptr, CopySize);
 556                 NewBuf->buf[CopySize] = '\0';
 557                 NewBuf->BufUsed = CopySize;
 558         }
 559         else {
 560                 NewBuf->buf[0] = '\0';
 561                 NewBuf->BufUsed = 0;
 562         }
 563         NewBuf->ConstBuf = 0;
 564
 565         dbg_Init(NewBuf);
 566
 567         return NewBuf;
 568 }
 569
 570 /**
 571  * @ingroup StrBuf_DeConstructors
 572  * @brief Set an existing buffer from a c-string
 573  * @param Buf buffer to load
 574  * @param ptr c-string to put into
 575  * @param nChars set to -1 if we should work 0-terminated
 576  * @returns the new length of the string
 577  */
 578 int StrBufPlain(StrBuf *Buf, const char* ptr, int nChars)
 579 {
 580         size_t Siz;
 581         size_t CopySize;
 582
 583         if (Buf == NULL)
 584                 return -1;
 585         if (ptr == NULL) {
 586                 FlushStrBuf(Buf);
 587                 return -1;
 588         }
 589
 590         Siz = Buf->BufSize;
 591
 592         if (nChars < 0)
 593                 CopySize = strlen(ptr);
 594         else
 595                 CopySize = nChars;
 596
 597         while ((Siz <= CopySize) && (Siz != 0))
 598                 Siz *= 2;
 599
 600         if (Siz == 0) {
 601                 FlushStrBuf(Buf);
 602                 return -1;
 603         }
 604
 605         if (Siz != Buf->BufSize)
 606                 IncreaseBuf(Buf, 0, Siz);
 607         memcpy(Buf->buf, ptr, CopySize);
 608         Buf->buf[CopySize] = '\0';
 609         Buf->BufUsed = CopySize;
 610         Buf->ConstBuf = 0;
 611         return CopySize;
 612 }
 613
 614
 615 /**
 616  * @ingroup StrBuf_DeConstructors
 617  * @brief use strbuf as wrapper for a string constant for easy handling
 618  * @param StringConstant a string to wrap
 619  * @param SizeOfStrConstant should be sizeof(StringConstant)-1
 620  */
 621 StrBuf* _NewConstStrBuf(const char* StringConstant, size_t SizeOfStrConstant)
 622 {
 623         StrBuf *NewBuf;
 624
 625         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 626         if (NewBuf == NULL)
 627                 return NULL;
 628         NewBuf->buf = (char*) StringConstant;
 629         NewBuf->BufSize = SizeOfStrConstant;
 630         NewBuf->BufUsed = SizeOfStrConstant;
 631         NewBuf->ConstBuf = 1;
 632
 633         dbg_Init(NewBuf);
 634
 635         return NewBuf;
 636 }
 637
 638
 639 /**
 640  * @ingroup StrBuf_DeConstructors
 641  * @brief flush the content of a Buf; keep its struct
 642  * @param buf Buffer to flush
 643  */
 644 int FlushStrBuf(StrBuf *buf)
 645 {
 646         if ((buf == NULL) || (buf->buf == NULL))
 647                 return -1;
 648         if (buf->ConstBuf)
 649                 return -1;
 650         buf->buf[0] ='\0';
 651         buf->BufUsed = 0;
 652         return 0;
 653 }
 654
 655 /**
 656  * @ingroup StrBuf_DeConstructors
 657  * @brief wipe the content of a Buf thoroughly (overwrite it -> expensive); keep its struct
 658  * @param buf Buffer to wipe
 659  */
 660 int FLUSHStrBuf(StrBuf *buf)
 661 {
 662         if (buf == NULL)
 663                 return -1;
 664         if (buf->ConstBuf)
 665                 return -1;
 666         if (buf->BufUsed > 0) {
 667                 memset(buf->buf, 0, buf->BufUsed);
 668                 buf->BufUsed = 0;
 669         }
 670         return 0;
 671 }
 672
 673 #ifdef SIZE_DEBUG
 674 int hFreeDbglog = -1;
 675 #endif
 676 /**
 677  * @ingroup StrBuf_DeConstructors
 678  * @brief Release a Buffer
 679  * Its a double pointer, so it can NULL your pointer
 680  * so fancy SIG11 appear instead of random results
 681  * @param FreeMe Pointer Pointer to the buffer to free
 682  */
 683 void FreeStrBuf (StrBuf **FreeMe)
 684 {
 685         if (*FreeMe == NULL)
 686                 return;
 687
 688         dbg_FreeStrBuf(FreeMe, 'F');
 689
 690         if (!(*FreeMe)->ConstBuf)
 691                 free((*FreeMe)->buf);
 692         free(*FreeMe);
 693         *FreeMe = NULL;
 694 }
 695
 696 /**
 697  * @ingroup StrBuf_DeConstructors
 698  * @brief flatten a Buffer to the Char * we return
 699  * Its a double pointer, so it can NULL your pointer
 700  * so fancy SIG11 appear instead of random results
 701  * The Callee then owns the buffer and is responsible for freeing it.
 702  * @param SmashMe Pointer Pointer to the buffer to release Buf from and free
 703  * @returns the pointer of the buffer; Callee owns the memory thereafter.
 704  */
 705 char *SmashStrBuf (StrBuf **SmashMe)
 706 {
 707         char *Ret;
 708
 709         if ((SmashMe == NULL) || (*SmashMe == NULL))
 710                 return NULL;
 711
 712         dbg_FreeStrBuf(SmashMe, 'S');
 713
 714         Ret = (*SmashMe)->buf;
 715         free(*SmashMe);
 716         *SmashMe = NULL;
 717         return Ret;
 718 }
 719
 720 /**
 721  * @ingroup StrBuf_DeConstructors
 722  * @brief Release the buffer
 723  * If you want put your StrBuf into a Hash, use this as Destructor.
 724  * @param VFreeMe untyped pointer to a StrBuf. be shure to do the right thing [TM]
 725  */
 726 void HFreeStrBuf (void *VFreeMe)
 727 {
 728         StrBuf *FreeMe = (StrBuf*)VFreeMe;
 729         if (FreeMe == NULL)
 730                 return;
 731
 732         dbg_FreeStrBuf(SmashMe, 'H');
 733
 734         if (!FreeMe->ConstBuf)
 735                 free(FreeMe->buf);
 736         free(FreeMe);
 737 }
 738
 739
 740 /*******************************************************************************
 741  *                      Simple string transformations                          *
 742  *******************************************************************************/
 743
 744 /**
 745  * @ingroup StrBuf
 746  * @brief Wrapper around atol
 747  */
 748 long StrTol(const StrBuf *Buf)
 749 {
 750         if (Buf == NULL)
 751                 return 0;
 752         if(Buf->BufUsed > 0)
 753                 return atol(Buf->buf);
 754         else
 755                 return 0;
 756 }
 757
 758 /**
 759  * @ingroup StrBuf
 760  * @brief Wrapper around atoi
 761  */
 762 int StrToi(const StrBuf *Buf)
 763 {
 764         if (Buf == NULL)
 765                 return 0;
 766         if (Buf->BufUsed > 0)
 767                 return atoi(Buf->buf);
 768         else
 769                 return 0;
 770 }
 771
 772 /**
 773  * @ingroup StrBuf
 774  * @brief Checks to see if the string is a pure number
 775  * @param Buf The buffer to inspect
 776  * @returns 1 if its a pure number, 0, if not.
 777  */
 778 int StrBufIsNumber(const StrBuf *Buf) {
 779         char * pEnd;
 780         if ((Buf == NULL) || (Buf->BufUsed == 0)) {
 781                 return 0;
 782         }
 783         strtoll(Buf->buf, &pEnd, 10);
 784         if (pEnd == Buf->buf)
 785                 return 0;
 786         if ((pEnd != NULL) && (pEnd == Buf->buf + Buf->BufUsed))
 787                 return 1;
 788         if (Buf->buf == pEnd)
 789                 return 0;
 790         return 0;
 791 }
 792
 793 /**
 794  * @ingroup StrBuf_Filler
 795  * @brief modifies a Single char of the Buf
 796  * You can point to it via char* or a zero-based integer
 797  * @param Buf The buffer to manipulate
 798  * @param ptr char* to zero; use NULL if unused
 799  * @param nThChar zero based pointer into the string; use -1 if unused
 800  * @param PeekValue The Character to place into the position
 801  */
 802 long StrBufPeek(StrBuf *Buf, const char* ptr, long nThChar, char PeekValue)
 803 {
 804         if (Buf == NULL)
 805                 return -1;
 806         if (ptr != NULL)
 807                 nThChar = ptr - Buf->buf;
 808         if ((nThChar < 0) || (nThChar > Buf->BufUsed))
 809                 return -1;
 810         Buf->buf[nThChar] = PeekValue;
 811         return nThChar;
 812 }
 813
 814 /**
 815  * @ingroup StrBuf_Filler
 816  * @brief modifies a range of chars of the Buf
 817  * You can point to it via char* or a zero-based integer
 818  * @param Buf The buffer to manipulate
 819  * @param ptr char* to zero; use NULL if unused
 820  * @param nThChar zero based pointer into the string; use -1 if unused
 821  * @param nChars how many chars are to be flushed?
 822  * @param PookValue The Character to place into that area
 823  */
 824 long StrBufPook(StrBuf *Buf, const char* ptr, long nThChar, long nChars, char PookValue)
 825 {
 826         if (Buf == NULL)
 827                 return -1;
 828         if (ptr != NULL)
 829                 nThChar = ptr - Buf->buf;
 830         if ((nThChar < 0) || (nThChar > Buf->BufUsed))
 831                 return -1;
 832         if (nThChar + nChars > Buf->BufUsed)
 833                 nChars =  Buf->BufUsed - nThChar;
 834
 835         memset(Buf->buf + nThChar, PookValue, nChars);
 836         /* just to be shure... */
 837         Buf->buf[Buf->BufUsed] = 0;
 838         return nChars;
 839 }
 840
 841 /**
 842  * @ingroup StrBuf_Filler
 843  * @brief Append a StringBuffer to the buffer
 844  * @param Buf Buffer to modify
 845  * @param AppendBuf Buffer to copy at the end of our buffer
 846  * @param Offset Should we start copying from an offset?
 847  */
 848 void StrBufAppendBuf(StrBuf *Buf, const StrBuf *AppendBuf, unsigned long Offset)
 849 {
 850         if ((AppendBuf == NULL) || (AppendBuf->buf == NULL) ||
 851             (Buf == NULL) || (Buf->buf == NULL))
 852                 return;
 853
 854         if (Buf->BufSize - Offset < AppendBuf->BufUsed + Buf->BufUsed + 1)
 855                 IncreaseBuf(Buf,
 856                             (Buf->BufUsed > 0),
 857                             AppendBuf->BufUsed + Buf->BufUsed);
 858
 859         memcpy(Buf->buf + Buf->BufUsed,
 860                AppendBuf->buf + Offset,
 861                AppendBuf->BufUsed - Offset);
 862         Buf->BufUsed += AppendBuf->BufUsed - Offset;
 863         Buf->buf[Buf->BufUsed] = '\0';
 864 }
 865
 866
 867 /**
 868  * @ingroup StrBuf_Filler
 869  * @brief Append a C-String to the buffer
 870  * @param Buf Buffer to modify
 871  * @param AppendBuf Buffer to copy at the end of our buffer
 872  * @param AppendSize number of bytes to copy; set to -1 if we should count it in advance
 873  * @param Offset Should we start copying from an offset?
 874  */
 875 void StrBufAppendBufPlain(StrBuf *Buf, const char *AppendBuf, long AppendSize, unsigned long Offset)
 876 {
 877         long aps;
 878         long BufSizeRequired;
 879
 880         if ((AppendBuf == NULL) || (Buf == NULL))
 881                 return;
 882
 883         if (AppendSize < 0 )
 884                 aps = strlen(AppendBuf + Offset);
 885         else
 886                 aps = AppendSize - Offset;
 887
 888         BufSizeRequired = Buf->BufUsed + aps + 1;
 889         if (Buf->BufSize <= BufSizeRequired)
 890                 IncreaseBuf(Buf, (Buf->BufUsed > 0), BufSizeRequired);
 891
 892         memcpy(Buf->buf + Buf->BufUsed,
 893                AppendBuf + Offset,
 894                aps);
 895         Buf->BufUsed += aps;
 896         Buf->buf[Buf->BufUsed] = '\0';
 897 }
 898
 899 /**
 900  * @ingroup StrBuf_Filler
 901  * @brief sprintf like function appending the formated string to the buffer
 902  * vsnprintf version to wrap into own calls
 903  * @param Buf Buffer to extend by format and Params
 904  * @param format printf alike format to add
 905  * @param ap va_list containing the items for format
 906  */
 907 void StrBufVAppendPrintf(StrBuf *Buf, const char *format, va_list ap)
 908 {
 909         va_list apl;
 910         size_t BufSize;
 911         size_t nWritten;
 912         size_t Offset;
 913         size_t newused;
 914
 915         if ((Buf == NULL)  || (format == NULL))
 916                 return;
 917
 918         BufSize = Buf->BufSize;
 919         nWritten = Buf->BufSize + 1;
 920         Offset = Buf->BufUsed;
 921         newused = Offset + nWritten;
 922
 923         while (newused >= BufSize) {
 924                 va_copy(apl, ap);
 925                 nWritten = vsnprintf(Buf->buf + Offset,
 926                                      Buf->BufSize - Offset,
 927                                      format, apl);
 928                 va_end(apl);
 929                 newused = Offset + nWritten;
 930                 if (newused >= Buf->BufSize) {
 931                         if (IncreaseBuf(Buf, 1, newused) == -1)
 932                                 return; /* TODO: error handling? */
 933                         newused = Buf->BufSize + 1;
 934                 }
 935                 else {
 936                         Buf->BufUsed = Offset + nWritten;
 937                         BufSize = Buf->BufSize;
 938                 }
 939
 940         }
 941 }
 942
 943 /**
 944  * @ingroup StrBuf_Filler
 945  * @brief sprintf like function appending the formated string to the buffer
 946  * @param Buf Buffer to extend by format and Params
 947  * @param format printf alike format to add
 948  */
 949 void StrBufAppendPrintf(StrBuf *Buf, const char *format, ...)
 950 {
 951         size_t BufSize;
 952         size_t nWritten;
 953         size_t Offset;
 954         size_t newused;
 955         va_list arg_ptr;
 956
 957         if ((Buf == NULL)  || (format == NULL))
 958                 return;
 959
 960         BufSize = Buf->BufSize;
 961         nWritten = Buf->BufSize + 1;
 962         Offset = Buf->BufUsed;
 963         newused = Offset + nWritten;
 964
 965         while (newused >= BufSize) {
 966                 va_start(arg_ptr, format);
 967                 nWritten = vsnprintf(Buf->buf + Buf->BufUsed,
 968                                      Buf->BufSize - Buf->BufUsed,
 969                                      format, arg_ptr);
 970                 va_end(arg_ptr);
 971                 newused = Buf->BufUsed + nWritten;
 972                 if (newused >= Buf->BufSize) {
 973                         if (IncreaseBuf(Buf, 1, newused) == -1)
 974                                 return; /* TODO: error handling? */
 975                         newused = Buf->BufSize + 1;
 976                 }
 977                 else {
 978                         Buf->BufUsed += nWritten;
 979                         BufSize = Buf->BufSize;
 980                 }
 981
 982         }
 983 }
 984
 985 /**
 986  * @ingroup StrBuf_Filler
 987  * @brief sprintf like function putting the formated string into the buffer
 988  * @param Buf Buffer to extend by format and Parameters
 989  * @param format printf alike format to add
 990  */
 991 void StrBufPrintf(StrBuf *Buf, const char *format, ...)
 992 {
 993         size_t nWritten;
 994         va_list arg_ptr;
 995
 996         if ((Buf == NULL)  || (format == NULL))
 997                 return;
 998
 999         nWritten = Buf->BufSize + 1;
1000         while (nWritten >= Buf->BufSize) {
1001                 va_start(arg_ptr, format);
1002                 nWritten = vsnprintf(Buf->buf, Buf->BufSize, format, arg_ptr);
1003                 va_end(arg_ptr);
1004                 if (nWritten >= Buf->BufSize) {
1005                         if (IncreaseBuf(Buf, 0, 0) == -1)
1006                                 return; /* TODO: error handling? */
1007                         nWritten = Buf->BufSize + 1;
1008                         continue;
1009                 }
1010                 Buf->BufUsed = nWritten ;
1011         }
1012 }
1013
1014 /**
1015  * @ingroup StrBuf_Filler
1016  * @brief Callback for cURL to append the webserver reply to a buffer
1017  * @param ptr pre-defined by the cURL API; see man 3 curl for mre info
1018  * @param size pre-defined by the cURL API; see man 3 curl for mre info
1019  * @param nmemb pre-defined by the cURL API; see man 3 curl for mre info
1020  * @param stream pre-defined by the cURL API; see man 3 curl for mre info
1021  */
1022 size_t CurlFillStrBuf_callback(void *ptr, size_t size, size_t nmemb, void *stream)
1023 {
1024
1025         StrBuf *Target;
1026
1027         Target = stream;
1028         if (ptr == NULL)
1029                 return 0;
1030
1031         StrBufAppendBufPlain(Target, ptr, size * nmemb, 0);
1032         return size * nmemb;
1033 }
1034
1035
1036 /**
1037  * @ingroup StrBuf
1038  * @brief extracts a substring from Source into dest
1039  * @param dest buffer to place substring into
1040  * @param Source string to copy substring from
1041  * @param Offset chars to skip from start
1042  * @param nChars number of chars to copy
1043  * @returns the number of chars copied; may be different from nChars due to the size of Source
1044  */
1045 int StrBufSub(StrBuf *dest, const StrBuf *Source, unsigned long Offset, size_t nChars)
1046 {
1047         size_t NCharsRemain;
1048         if (Offset > Source->BufUsed)
1049         {
1050                 if (dest != NULL)
1051                         FlushStrBuf(dest);
1052                 return 0;
1053         }
1054         if (Offset + nChars < Source->BufUsed)
1055         {
1056                 if ((nChars >= dest->BufSize) &&
1057                     (IncreaseBuf(dest, 0, nChars + 1) == -1))
1058                         return 0;
1059                 memcpy(dest->buf, Source->buf + Offset, nChars);
1060                 dest->BufUsed = nChars;
1061                 dest->buf[dest->BufUsed] = '\0';
1062                 return nChars;
1063         }
1064         NCharsRemain = Source->BufUsed - Offset;
1065         if ((NCharsRemain  >= dest->BufSize) &&
1066             (IncreaseBuf(dest, 0, NCharsRemain + 1) == -1))
1067                 return 0;
1068         memcpy(dest->buf, Source->buf + Offset, NCharsRemain);
1069         dest->BufUsed = NCharsRemain;
1070         dest->buf[dest->BufUsed] = '\0';
1071         return NCharsRemain;
1072 }
1073
1074 /**
1075  * @ingroup StrBuf
1076  * @brief Cut nChars from the start of the string
1077  * @param Buf Buffer to modify
1078  * @param nChars how many chars should be skipped?
1079  */
1080 void StrBufCutLeft(StrBuf *Buf, int nChars)
1081 {
1082         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1083         if (nChars >= Buf->BufUsed) {
1084                 FlushStrBuf(Buf);
1085                 return;
1086         }
1087         memmove(Buf->buf, Buf->buf + nChars, Buf->BufUsed - nChars);
1088         Buf->BufUsed -= nChars;
1089         Buf->buf[Buf->BufUsed] = '\0';
1090 }
1091
1092 /**
1093  * @ingroup StrBuf
1094  * @brief Cut the trailing n Chars from the string
1095  * @param Buf Buffer to modify
1096  * @param nChars how many chars should be trunkated?
1097  */
1098 void StrBufCutRight(StrBuf *Buf, int nChars)
1099 {
1100         if ((Buf == NULL) || (Buf->BufUsed == 0) || (Buf->buf == NULL))
1101                 return;
1102
1103         if (nChars >= Buf->BufUsed) {
1104                 FlushStrBuf(Buf);
1105                 return;
1106         }
1107         Buf->BufUsed -= nChars;
1108         Buf->buf[Buf->BufUsed] = '\0';
1109 }
1110
1111 /**
1112  * @ingroup StrBuf
1113  * @brief Cut the string after n Chars
1114  * @param Buf Buffer to modify
1115  * @param AfternChars after how many chars should we trunkate the string?
1116  * @param At if non-null and points inside of our string, cut it there.
1117  */
1118 void StrBufCutAt(StrBuf *Buf, int AfternChars, const char *At)
1119 {
1120         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1121         if (At != NULL){
1122                 AfternChars = At - Buf->buf;
1123         }
1124
1125         if ((AfternChars < 0) || (AfternChars >= Buf->BufUsed))
1126                 return;
1127         Buf->BufUsed = AfternChars;
1128         Buf->buf[Buf->BufUsed] = '\0';
1129 }
1130
1131
1132 /**
1133  * @ingroup StrBuf
1134  * @brief Strip leading and trailing spaces from a string; with premeasured and adjusted length.
1135  * @param Buf the string to modify
1136  */
1137 void StrBufTrim(StrBuf *Buf)
1138 {
1139         int delta = 0;
1140         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1141
1142         while ((Buf->BufUsed > 0) &&
1143                isspace(Buf->buf[Buf->BufUsed - 1]))
1144         {
1145                 Buf->BufUsed --;
1146         }
1147         Buf->buf[Buf->BufUsed] = '\0';
1148
1149         if (Buf->BufUsed == 0) return;
1150
1151         while ((Buf->BufUsed > delta) && (isspace(Buf->buf[delta]))){
1152                 delta ++;
1153         }
1154         if (delta > 0) StrBufCutLeft(Buf, delta);
1155 }
1156 /**
1157  * @ingroup StrBuf
1158  * @brief changes all spaces in the string  (tab, linefeed...) to Blank (0x20)
1159  * @param Buf the string to modify
1160  */
1161 void StrBufSpaceToBlank(StrBuf *Buf)
1162 {
1163         char *pche, *pch;
1164
1165         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1166
1167         pch = Buf->buf;
1168         pche = pch + Buf->BufUsed;
1169         while (pch < pche)
1170         {
1171                 if (isspace(*pch))
1172                         *pch = ' ';
1173                 pch ++;
1174         }
1175 }
1176
1177 void StrBufStripAllBut(StrBuf *Buf, char leftboundary, char rightboundary)
1178 {
1179         const char *pLeft;
1180         const char *pRight;
1181
1182         if ((Buf == NULL) || (Buf->buf == NULL)) {
1183                 return;
1184         }
1185
1186         pRight = strchr(Buf->buf, rightboundary);
1187         if (pRight != NULL) {
1188                 StrBufCutAt(Buf, 0, pRight);
1189         }
1190
1191         pLeft = strrchr(ChrPtr(Buf), leftboundary);
1192         if (pLeft != NULL) {
1193                 StrBufCutLeft(Buf, pLeft - Buf->buf + 1);
1194         }
1195 }
1196
1197
1198 /**
1199  * @ingroup StrBuf_Filler
1200  * @brief uppercase the contents of a buffer
1201  * @param Buf the buffer to translate
1202  */
1203 void StrBufUpCase(StrBuf *Buf)
1204 {
1205         char *pch, *pche;
1206
1207         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1208
1209         pch = Buf->buf;
1210         pche = pch + Buf->BufUsed;
1211         while (pch < pche) {
1212                 *pch = toupper(*pch);
1213                 pch ++;
1214         }
1215 }
1216
1217
1218 /**
1219  * @ingroup StrBuf_Filler
1220  * @brief lowercase the contents of a buffer
1221  * @param Buf the buffer to translate
1222  */
1223 void StrBufLowerCase(StrBuf *Buf)
1224 {
1225         char *pch, *pche;
1226
1227         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1228
1229         pch = Buf->buf;
1230         pche = pch + Buf->BufUsed;
1231         while (pch < pche) {
1232                 *pch = tolower(*pch);
1233                 pch ++;
1234         }
1235 }
1236
1237
1238 /*******************************************************************************
1239  *           a tokenizer that kills, maims, and destroys                       *
1240  *******************************************************************************/
1241
1242 /**
1243  * @ingroup StrBuf_Tokenizer
1244  * @brief Replace a token at a given place with a given length by another token with given length
1245  * @param Buf String where to work on
1246  * @param where where inside of the Buf is the search-token
1247  * @param HowLong How long is the token to be replaced
1248  * @param Repl Token to insert at 'where'
1249  * @param ReplLen Length of repl
1250  * @returns -1 if fail else length of resulting Buf
1251  */
1252 int StrBufReplaceToken(StrBuf *Buf, long where, long HowLong,
1253                        const char *Repl, long ReplLen)
1254 {
1255
1256         if ((Buf == NULL) ||
1257             (where > Buf->BufUsed) ||
1258             (where + HowLong > Buf->BufUsed))
1259                 return -1;
1260
1261         if (where + ReplLen - HowLong > Buf->BufSize)
1262                 if (IncreaseBuf(Buf, 1, Buf->BufUsed + ReplLen) < 0)
1263                         return -1;
1264
1265         memmove(Buf->buf + where + ReplLen,
1266                 Buf->buf + where + HowLong,
1267                 Buf->BufUsed - where - HowLong);
1268
1269         memcpy(Buf->buf + where,
1270                Repl, ReplLen);
1271
1272         Buf->BufUsed += ReplLen - HowLong;
1273
1274         return Buf->BufUsed;
1275 }
1276
1277 /**
1278  * @ingroup StrBuf_Tokenizer
1279  * @brief Counts the numbmer of tokens in a buffer
1280  * @param source String to count tokens in
1281  * @param tok    Tokenizer char to count
1282  * @returns numbers of tokenizer chars found
1283  */
1284 int StrBufNum_tokens(const StrBuf *source, char tok)
1285 {
1286         char *pch, *pche;
1287         long NTokens;
1288         if ((source == NULL) || (source->BufUsed == 0))
1289                 return 0;
1290         if ((source->BufUsed == 1) && (*source->buf == tok))
1291                 return 2;
1292         NTokens = 1;
1293         pch = source->buf;
1294         pche = pch + source->BufUsed;
1295         while (pch < pche)
1296         {
1297                 if (*pch == tok)
1298                         NTokens ++;
1299                 pch ++;
1300         }
1301         return NTokens;
1302 }
1303
1304 /**
1305  * @ingroup StrBuf_Tokenizer
1306  * @brief a string tokenizer
1307  * @param Source StringBuffer to read into
1308  * @param parmnum n'th Parameter to remove
1309  * @param separator tokenizer character
1310  * @returns -1 if not found, else length of token.
1311  */
1312 int StrBufRemove_token(StrBuf *Source, int parmnum, char separator)
1313 {
1314         int ReducedBy;
1315         char *d, *s, *end;              /* dest, source */
1316         int count = 0;
1317
1318         /* Find desired @parameter */
1319         end = Source->buf + Source->BufUsed;
1320         d = Source->buf;
1321         while ((d <= end) &&
1322                (count < parmnum))
1323         {
1324                 /* End of string, bail! */
1325                 if (!*d) {
1326                         d = NULL;
1327                         break;
1328                 }
1329                 if (*d == separator) {
1330                         count++;
1331                 }
1332                 d++;
1333         }
1334         if ((d == NULL) || (d >= end))
1335                 return 0;               /* @Parameter not found */
1336
1337         /* Find next @parameter */
1338         s = d;
1339         while ((s <= end) &&
1340                (*s && *s != separator))
1341         {
1342                 s++;
1343         }
1344         if (*s == separator)
1345                 s++;
1346         ReducedBy = d - s;
1347
1348         /* Hack and slash */
1349         if (s >= end) {
1350                 return 0;
1351         }
1352         else if (*s) {
1353                 memmove(d, s, Source->BufUsed - (s - Source->buf));
1354                 Source->BufUsed += ReducedBy;
1355                 Source->buf[Source->BufUsed] = '\0';
1356         }
1357         else if (d == Source->buf) {
1358                 *d = 0;
1359                 Source->BufUsed = 0;
1360         }
1361         else {
1362                 *--d = '\0';
1363                 Source->BufUsed += ReducedBy;
1364         }
1365         /*
1366         while (*s) {
1367                 *d++ = *s++;
1368         }
1369         *d = 0;
1370         */
1371         return ReducedBy;
1372 }
1373
1374 int StrBufExtract_tokenFromStr(StrBuf *dest, const char *Source, long SourceLen, int parmnum, char separator)
1375 {
1376         const StrBuf Temp = {
1377                 (char*)Source,
1378                 SourceLen,
1379                 SourceLen,
1380                 1
1381 #ifdef SIZE_DEBUG
1382                 ,
1383                 0,
1384                 "",
1385                 ""
1386 #endif
1387         };
1388
1389         return StrBufExtract_token(dest, &Temp, parmnum, separator);
1390 }
1391
1392 /**
1393  * @ingroup StrBuf_Tokenizer
1394  * @brief a string tokenizer
1395  * @param dest Destination StringBuffer
1396  * @param Source StringBuffer to read into
1397  * @param parmnum n'th Parameter to extract
1398  * @param separator tokenizer character
1399  * @returns -1 if not found, else length of token.
1400  */
1401 int StrBufExtract_token(StrBuf *dest, const StrBuf *Source, int parmnum, char separator)
1402 {
1403         const char *s, *e;              //* source * /
1404         int len = 0;                    //* running total length of extracted string * /
1405         int current_token = 0;          //* token currently being processed * /
1406
1407         if (dest != NULL) {
1408                 dest->buf[0] = '\0';
1409                 dest->BufUsed = 0;
1410         }
1411         else
1412                 return(-1);
1413
1414         if ((Source == NULL) || (Source->BufUsed ==0)) {
1415                 return(-1);
1416         }
1417         s = Source->buf;
1418         e = s + Source->BufUsed;
1419
1420         //cit_backtrace();
1421         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1422
1423         while ((s < e) && !IsEmptyStr(s)) {
1424                 if (*s == separator) {
1425                         ++current_token;
1426                 }
1427                 if (len >= dest->BufSize) {
1428                         dest->BufUsed = len;
1429                         if (IncreaseBuf(dest, 1, -1) < 0) {
1430                                 dest->BufUsed --;
1431                                 break;
1432                         }
1433                 }
1434                 if ( (current_token == parmnum) &&
1435                      (*s != separator)) {
1436                         dest->buf[len] = *s;
1437                         ++len;
1438                 }
1439                 else if (current_token > parmnum) {
1440                         break;
1441                 }
1442                 ++s;
1443         }
1444
1445         dest->buf[len] = '\0';
1446         dest->BufUsed = len;
1447
1448         if (current_token < parmnum) {
1449                 //lprintf (CTDL_DEBUG,"test <!: %s\n", dest);
1450                 return(-1);
1451         }
1452         //lprintf (CTDL_DEBUG,"test <: %d; %s\n", len, dest);
1453         return(len);
1454 }
1455
1456
1457
1458
1459
1460 /**
1461  * @ingroup StrBuf_Tokenizer
1462  * @brief a string tokenizer to fetch an integer
1463  * @param Source String containing tokens
1464  * @param parmnum n'th Parameter to extract
1465  * @param separator tokenizer character
1466  * @returns 0 if not found, else integer representation of the token
1467  */
1468 int StrBufExtract_int(const StrBuf* Source, int parmnum, char separator)
1469 {
1470         StrBuf tmp;
1471         char buf[64];
1472
1473         tmp.buf = buf;
1474         buf[0] = '\0';
1475         tmp.BufSize = 64;
1476         tmp.BufUsed = 0;
1477         tmp.ConstBuf = 1;
1478         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1479                 return(atoi(buf));
1480         else
1481                 return 0;
1482 }
1483
1484 /**
1485  * @ingroup StrBuf_Tokenizer
1486  * @brief a string tokenizer to fetch a long integer
1487  * @param Source String containing tokens
1488  * @param parmnum n'th Parameter to extract
1489  * @param separator tokenizer character
1490  * @returns 0 if not found, else long integer representation of the token
1491  */
1492 long StrBufExtract_long(const StrBuf* Source, int parmnum, char separator)
1493 {
1494         StrBuf tmp;
1495         char buf[64];
1496
1497         tmp.buf = buf;
1498         buf[0] = '\0';
1499         tmp.BufSize = 64;
1500         tmp.BufUsed = 0;
1501         tmp.ConstBuf = 1;
1502         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1503                 return(atoi(buf));
1504         else
1505                 return 0;
1506 }
1507
1508
1509 /**
1510  * @ingroup StrBuf_Tokenizer
1511  * @brief a string tokenizer to fetch an unsigned long
1512  * @param Source String containing tokens
1513  * @param parmnum n'th Parameter to extract
1514  * @param separator tokenizer character
1515  * @returns 0 if not found, else unsigned long representation of the token
1516  */
1517 unsigned long StrBufExtract_unsigned_long(const StrBuf* Source, int parmnum, char separator)
1518 {
1519         StrBuf tmp;
1520         char buf[64];
1521         char *pnum;
1522
1523         tmp.buf = buf;
1524         buf[0] = '\0';
1525         tmp.BufSize = 64;
1526         tmp.BufUsed = 0;
1527         tmp.ConstBuf = 1;
1528         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0) {
1529                 pnum = &buf[0];
1530                 if (*pnum == '-')
1531                         pnum ++;
1532                 return (unsigned long) atol(pnum);
1533         }
1534         else
1535                 return 0;
1536 }
1537
1538
1539
1540 /**
1541  * @ingroup StrBuf_NextTokenizer
1542  * @brief a string tokenizer; Bounds checker
1543  *  function to make shure whether StrBufExtract_NextToken and friends have reached the end of the string.
1544  * @param Source our tokenbuffer
1545  * @param pStart the token iterator pointer to inspect
1546  * @returns whether the revolving pointer is inside of the search range
1547  */
1548 int StrBufHaveNextToken(const StrBuf *Source, const char **pStart)
1549 {
1550         if ((Source == NULL) ||
1551             (*pStart == StrBufNOTNULL) ||
1552             (Source->BufUsed == 0))
1553         {
1554                 return 0;
1555         }
1556         if (*pStart == NULL)
1557         {
1558                 return 1;
1559         }
1560         else if (*pStart > Source->buf + Source->BufUsed)
1561         {
1562                 return 0;
1563         }
1564         else if (*pStart <= Source->buf)
1565         {
1566                 return 0;
1567         }
1568
1569         return 1;
1570 }
1571
1572 /**
1573  * @ingroup StrBuf_NextTokenizer
1574  * @brief a string tokenizer
1575  * @param dest Destination StringBuffer
1576  * @param Source StringBuffer to read into
1577  * @param pStart pointer to the end of the last token. Feed with NULL on start.
1578  * @param separator tokenizer
1579  * @returns -1 if not found, else length of token.
1580  */
1581 int StrBufExtract_NextToken(StrBuf *dest, const StrBuf *Source, const char **pStart, char separator)
1582 {
1583         const char *s;          /* source */
1584         const char *EndBuffer;  /* end stop of source buffer */
1585         int current_token = 0;  /* token currently being processed */
1586         int len = 0;            /* running total length of extracted string */
1587
1588         if ((Source          == NULL) ||
1589             (Source->BufUsed == 0)      )
1590         {
1591                 *pStart = StrBufNOTNULL;
1592                 if (dest != NULL)
1593                         FlushStrBuf(dest);
1594                 return -1;
1595         }
1596
1597         EndBuffer = Source->buf + Source->BufUsed;
1598
1599         if (dest != NULL)
1600         {
1601                 dest->buf[0] = '\0';
1602                 dest->BufUsed = 0;
1603         }
1604         else
1605         {
1606                 *pStart = EndBuffer + 1;
1607                 return -1;
1608         }
1609
1610         if (*pStart == NULL)
1611         {
1612                 *pStart = Source->buf; /* we're starting to examine this buffer. */
1613         }
1614         else if ((*pStart < Source->buf) ||
1615                  (*pStart > EndBuffer  )   )
1616         {
1617                 return -1; /* no more tokens to find. */
1618         }
1619
1620         s = *pStart;
1621         /* start to find the next token */
1622         while ((s <= EndBuffer)      &&
1623                (current_token == 0) )
1624         {
1625                 if (*s == separator)
1626                 {
1627                         /* we found the next token */
1628                         ++current_token;
1629                 }
1630
1631                 if (len >= dest->BufSize)
1632                 {
1633                         /* our Dest-buffer isn't big enough, increase it. */
1634                         dest->BufUsed = len;
1635
1636                         if (IncreaseBuf(dest, 1, -1) < 0) {
1637                                 /* WHUT? no more mem? bail out. */
1638                                 s = EndBuffer;
1639                                 dest->BufUsed --;
1640                                 break;
1641                         }
1642                 }
1643
1644                 if ( (current_token == 0 ) &&   /* are we in our target token? */
1645                      (!IsEmptyStr(s)     ) &&
1646                      (separator     != *s)    ) /* don't copy the token itself */
1647                 {
1648                         dest->buf[len] = *s;    /* Copy the payload */
1649                         ++len;                  /* remember the bigger size. */
1650                 }
1651
1652                 ++s;
1653         }
1654
1655         /* did we reach the end? */
1656         if ((s > EndBuffer)) {
1657                 EndBuffer = StrBufNOTNULL;
1658                 *pStart = EndBuffer;
1659         }
1660         else {
1661                 *pStart = s;  /* remember the position for the next run */
1662         }
1663
1664         /* sanitize our extracted token */
1665         dest->buf[len] = '\0';
1666         dest->BufUsed  = len;
1667
1668         return (len);
1669 }
1670
1671
1672 /**
1673  * @ingroup StrBuf_NextTokenizer
1674  * @brief a string tokenizer
1675  * @param Source StringBuffer to read from
1676  * @param pStart pointer to the end of the last token. Feed with NULL.
1677  * @param separator tokenizer character
1678  * @param nTokens number of tokens to fastforward over
1679  * @returns -1 if not found, else length of token.
1680  */
1681 int StrBufSkip_NTokenS(const StrBuf *Source, const char **pStart, char separator, int nTokens)
1682 {
1683         const char *s, *EndBuffer;      //* source * /
1684         int len = 0;                    //* running total length of extracted string * /
1685         int current_token = 0;          //* token currently being processed * /
1686
1687         if ((Source == NULL) ||
1688             (Source->BufUsed ==0)) {
1689                 return(-1);
1690         }
1691         if (nTokens == 0)
1692                 return Source->BufUsed;
1693
1694         if (*pStart == NULL)
1695                 *pStart = Source->buf;
1696
1697         EndBuffer = Source->buf + Source->BufUsed;
1698
1699         if ((*pStart < Source->buf) ||
1700             (*pStart >  EndBuffer)) {
1701                 return (-1);
1702         }
1703
1704
1705         s = *pStart;
1706
1707         //cit_backtrace();
1708         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1709
1710         while ((s < EndBuffer) && !IsEmptyStr(s)) {
1711                 if (*s == separator) {
1712                         ++current_token;
1713                 }
1714                 if (current_token >= nTokens) {
1715                         break;
1716                 }
1717                 ++s;
1718         }
1719         *pStart = s;
1720         (*pStart) ++;
1721
1722         return(len);
1723 }
1724
1725 /**
1726  * @ingroup StrBuf_NextTokenizer
1727  * @brief a string tokenizer to fetch an integer
1728  * @param Source StringBuffer to read from
1729  * @param pStart Cursor on the tokenstring
1730  * @param separator tokenizer character
1731  * @returns 0 if not found, else integer representation of the token
1732  */
1733 int StrBufExtractNext_int(const StrBuf* Source, const char **pStart, char separator)
1734 {
1735         StrBuf tmp;
1736         char buf[64];
1737
1738         tmp.buf = buf;
1739         buf[0] = '\0';
1740         tmp.BufSize = 64;
1741         tmp.BufUsed = 0;
1742         tmp.ConstBuf = 1;
1743         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1744                 return(atoi(buf));
1745         else
1746                 return 0;
1747 }
1748
1749 /**
1750  * @ingroup StrBuf_NextTokenizer
1751  * @brief a string tokenizer to fetch a long integer
1752  * @param Source StringBuffer to read from
1753  * @param pStart Cursor on the tokenstring
1754  * @param separator tokenizer character
1755  * @returns 0 if not found, else long integer representation of the token
1756  */
1757 long StrBufExtractNext_long(const StrBuf* Source, const char **pStart, char separator)
1758 {
1759         StrBuf tmp;
1760         char buf[64];
1761
1762         tmp.buf = buf;
1763         buf[0] = '\0';
1764         tmp.BufSize = 64;
1765         tmp.BufUsed = 0;
1766         tmp.ConstBuf = 1;
1767         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1768                 return(atoi(buf));
1769         else
1770                 return 0;
1771 }
1772
1773
1774 /**
1775  * @ingroup StrBuf_NextTokenizer
1776  * @brief a string tokenizer to fetch an unsigned long
1777  * @param Source StringBuffer to read from
1778  * @param pStart Cursor on the tokenstring
1779  * @param separator tokenizer character
1780  * @returns 0 if not found, else unsigned long representation of the token
1781  */
1782 unsigned long StrBufExtractNext_unsigned_long(const StrBuf* Source, const char **pStart, char separator)
1783 {
1784         StrBuf tmp;
1785         char buf[64];
1786         char *pnum;
1787
1788         tmp.buf = buf;
1789         buf[0] = '\0';
1790         tmp.BufSize = 64;
1791         tmp.BufUsed = 0;
1792         tmp.ConstBuf = 1;
1793         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0) {
1794                 pnum = &buf[0];
1795                 if (*pnum == '-')
1796                         pnum ++;
1797                 return (unsigned long) atol(pnum);
1798         }
1799         else
1800                 return 0;
1801 }
1802
1803
1804
1805
1806
1807 /*******************************************************************************
1808  *                             Escape Appending                                *
1809  *******************************************************************************/
1810
1811 /**
1812  * @ingroup StrBuf_DeEnCoder
1813  * @brief Escape a string for feeding out as a URL while appending it to a Buffer
1814  * @param OutBuf the output buffer
1815  * @param In Buffer to encode
1816  * @param PlainIn way in from plain old c strings
1817  */
1818 void StrBufUrlescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
1819 {
1820         const char *pch, *pche;
1821         char *pt, *pte;
1822         int len;
1823
1824         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1825                 return;
1826         if (PlainIn != NULL) {
1827                 len = strlen(PlainIn);
1828                 pch = PlainIn;
1829                 pche = pch + len;
1830         }
1831         else {
1832                 pch = In->buf;
1833                 pche = pch + In->BufUsed;
1834                 len = In->BufUsed;
1835         }
1836
1837         if (len == 0)
1838                 return;
1839
1840         pt = OutBuf->buf + OutBuf->BufUsed;
1841         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1842
1843         while (pch < pche) {
1844                 if (pt >= pte) {
1845                         IncreaseBuf(OutBuf, 1, -1);
1846                         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1847                         pt = OutBuf->buf + OutBuf->BufUsed;
1848                 }
1849
1850                 if((*pch >= 'a' && *pch <= 'z') ||
1851                    (*pch >= '@' && *pch <= 'Z') || /* @ A-Z */
1852                    (*pch >= '0' && *pch <= ':') || /* 0-9 : */
1853                    (*pch == '!') || (*pch == '_') ||
1854                    (*pch == ',') || (*pch == '.'))
1855                 {
1856                         *(pt++) = *(pch++);
1857                         OutBuf->BufUsed++;
1858                 }
1859                 else {
1860                         *pt = '%';
1861                         *(pt + 1) = HexList[(unsigned char)*pch][0];
1862                         *(pt + 2) = HexList[(unsigned char)*pch][1];
1863                         pt += 3;
1864                         OutBuf->BufUsed += 3;
1865                         pch ++;
1866                 }
1867         }
1868         *pt = '\0';
1869 }
1870
1871 /**
1872  * @ingroup StrBuf_DeEnCoder
1873  * @brief Escape a string for feeding out as a the username/password part of an URL while appending it to a Buffer
1874  * @param OutBuf the output buffer
1875  * @param In Buffer to encode
1876  * @param PlainIn way in from plain old c strings
1877  */
1878 void StrBufUrlescUPAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
1879 {
1880         const char *pch, *pche;
1881         char *pt, *pte;
1882         int len;
1883
1884         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1885                 return;
1886         if (PlainIn != NULL) {
1887                 len = strlen(PlainIn);
1888                 pch = PlainIn;
1889                 pche = pch + len;
1890         }
1891         else {
1892                 pch = In->buf;
1893                 pche = pch + In->BufUsed;
1894                 len = In->BufUsed;
1895         }
1896
1897         if (len == 0)
1898                 return;
1899
1900         pt = OutBuf->buf + OutBuf->BufUsed;
1901         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1902
1903         while (pch < pche) {
1904                 if (pt >= pte) {
1905                         IncreaseBuf(OutBuf, 1, -1);
1906                         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1907                         pt = OutBuf->buf + OutBuf->BufUsed;
1908                 }
1909
1910                 if((*pch >= 'a' && *pch <= 'z') ||
1911                    (*pch >= 'A' && *pch <= 'Z') || /* A-Z */
1912                    (*pch >= '0' && *pch <= ':') || /* 0-9 : */
1913                    (*pch == '!') || (*pch == '_') ||
1914                    (*pch == ',') || (*pch == '.'))
1915                 {
1916                         *(pt++) = *(pch++);
1917                         OutBuf->BufUsed++;
1918                 }
1919                 else {
1920                         *pt = '%';
1921                         *(pt + 1) = HexList[(unsigned char)*pch][0];
1922                         *(pt + 2) = HexList[(unsigned char)*pch][1];
1923                         pt += 3;
1924                         OutBuf->BufUsed += 3;
1925                         pch ++;
1926                 }
1927         }
1928         *pt = '\0';
1929 }
1930
1931 /**
1932  * @ingroup StrBuf_DeEnCoder
1933  * @brief append a string with characters having a special meaning in xml encoded to the buffer
1934  * @param OutBuf the output buffer
1935  * @param In Buffer to encode
1936  * @param PlainIn way in from plain old c strings
1937  * @param PlainInLen way in from plain old c strings; maybe you've got binary data or know the length?
1938  * @param OverrideLowChars should chars < 0x20 be replaced by _ or escaped as xml entity?
1939  */
1940 void StrBufXMLEscAppend(StrBuf *OutBuf,
1941                         const StrBuf *In,
1942                         const char *PlainIn,
1943                         long PlainInLen,
1944                         int OverrideLowChars)
1945 {
1946         const char *pch, *pche;
1947         char *pt, *pte;
1948         int IsUtf8Sequence;
1949         int len;
1950
1951         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1952                 return;
1953         if (PlainIn != NULL) {
1954                 if (PlainInLen < 0)
1955                         len = strlen((const char*)PlainIn);
1956                 else
1957                         len = PlainInLen;
1958                 pch = PlainIn;
1959                 pche = pch + len;
1960         }
1961         else {
1962                 pch = (const char*)In->buf;
1963                 pche = pch + In->BufUsed;
1964                 len = In->BufUsed;
1965         }
1966
1967         if (len == 0)
1968                 return;
1969
1970         pt = OutBuf->buf + OutBuf->BufUsed;
1971         /**< we max append 6 chars at once plus the \0 */
1972         pte = OutBuf->buf + OutBuf->BufSize - 6;
1973
1974         while (pch < pche) {
1975                 if (pt >= pte) {
1976                         OutBuf->BufUsed = pt - OutBuf->buf;
1977                         IncreaseBuf(OutBuf, 1, -1);
1978                         pte = OutBuf->buf + OutBuf->BufSize - 6;
1979                         /**< we max append 3 chars at once plus the \0 */
1980
1981                         pt = OutBuf->buf + OutBuf->BufUsed;
1982                 }
1983
1984                 if (*pch == '<') {
1985                         memcpy(pt, HKEY("&lt;"));
1986                         pt += 4;
1987                         pch ++;
1988                 }
1989                 else if (*pch == '>') {
1990                         memcpy(pt, HKEY("&gt;"));
1991                         pt += 4;
1992                         pch ++;
1993                 }
1994                 else if (*pch == '&') {
1995                         memcpy(pt, HKEY("&amp;"));
1996                         pt += 5;
1997                         pch++;
1998                 }
1999                 else if ((*pch >= 0x20) && (*pch <= 0x7F)) {
2000                         *pt = *pch;
2001                         pt++; pch++;
2002                 }
2003                 else if (*pch < 0x20) {
2004                         /* we probably shouldn't be doing this */
2005                         if (OverrideLowChars)
2006                         {
2007                                 *pt = '_';
2008                                 pt ++;
2009                                 pch ++;
2010                         }
2011                         else
2012                         {
2013                                 *pt = '&';
2014                                 pt++;
2015                                 *pt = HexList[*(unsigned char*)pch][0];
2016                                 pt ++;
2017                                 *pt = HexList[*(unsigned char*)pch][1];
2018                                 pt ++; pch ++;
2019                                 *pt = '&';
2020                                 pt++;
2021                                 pch ++;
2022                         }
2023                 }
2024                 else {
2025                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(pch, pche);
2026                         if (IsUtf8Sequence)
2027                         {
2028                                 while (IsUtf8Sequence > 0){
2029                                         *pt = *pch;
2030                                         pt ++;
2031                                         pch ++;
2032                                         --IsUtf8Sequence;
2033                                 }
2034                         }
2035                         else
2036                         {
2037                                 *pt = '&';
2038                                 pt++;
2039                                 *pt = HexList[*(unsigned char*)pch][0];
2040                                 pt ++;
2041                                 *pt = HexList[*(unsigned char*)pch][1];
2042                                 pt ++; pch ++;
2043                                 *pt = '&';
2044                                 pt++;
2045                                 pch ++;
2046                         }
2047                 }
2048         }
2049         *pt = '\0';
2050         OutBuf->BufUsed = pt - OutBuf->buf;
2051 }
2052
2053
2054 /**
2055  * @ingroup StrBuf_DeEnCoder
2056  * @brief append a string in hex encoding to the buffer
2057  * @param OutBuf the output buffer
2058  * @param In Buffer to encode
2059  * @param PlainIn way in from plain old c strings
2060  * @param PlainInLen way in from plain old c strings; maybe you've got binary data or know the length?
2061  */
2062 void StrBufHexEscAppend(StrBuf *OutBuf, const StrBuf *In, const unsigned char *PlainIn, long PlainInLen)
2063 {
2064         const unsigned char *pch, *pche;
2065         char *pt, *pte;
2066         int len;
2067
2068         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
2069                 return;
2070         if (PlainIn != NULL) {
2071                 if (PlainInLen < 0)
2072                         len = strlen((const char*)PlainIn);
2073                 else
2074                         len = PlainInLen;
2075                 pch = PlainIn;
2076                 pche = pch + len;
2077         }
2078         else {
2079                 pch = (const unsigned char*)In->buf;
2080                 pche = pch + In->BufUsed;
2081                 len = In->BufUsed;
2082         }
2083
2084         if (len == 0)
2085                 return;
2086
2087         pt = OutBuf->buf + OutBuf->BufUsed;
2088         pte = OutBuf->buf + OutBuf->BufSize - 3; /**< we max append 3 chars at once plus the \0 */
2089
2090         while (pch < pche) {
2091                 if (pt >= pte) {
2092                         IncreaseBuf(OutBuf, 1, -1);
2093                         pte = OutBuf->buf + OutBuf->BufSize - 3; /**< we max append 3 chars at once plus the \0 */
2094                         pt = OutBuf->buf + OutBuf->BufUsed;
2095                 }
2096
2097                 *pt = HexList[*pch][0];
2098                 pt ++;
2099                 *pt = HexList[*pch][1];
2100                 pt ++; pch ++; OutBuf->BufUsed += 2;
2101         }
2102         *pt = '\0';
2103 }
2104
2105 void StrBufBase64Append(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn, long PlainInLen, int linebreaks)
2106 {
2107         const char *pch;
2108         char *pt;
2109         int len;
2110         long ExpectLen;
2111
2112         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
2113                 return;
2114         if (PlainIn != NULL) {
2115                 if (PlainInLen < 0)
2116                         len = strlen(PlainIn);
2117                 else
2118                         len = PlainInLen;
2119                 pch = PlainIn;
2120         }
2121         else {
2122                 pch = In->buf;
2123                 len = In->BufUsed;
2124         }
2125
2126         if (len == 0)
2127                 return;
2128
2129         ExpectLen = ((len * 134) / 100) + OutBuf->BufUsed;
2130
2131         if (ExpectLen > OutBuf->BufSize)
2132                 if (IncreaseBuf(OutBuf, 1, ExpectLen) < ExpectLen)
2133                         return;
2134
2135         pt = OutBuf->buf + OutBuf->BufUsed;
2136
2137         len = CtdlEncodeBase64(pt, pch, len, linebreaks);
2138
2139         pt += len;
2140         OutBuf->BufUsed += len;
2141         *pt = '\0';
2142 }
2143
2144 /**
2145  * @ingroup StrBuf_DeEnCoder
2146  * @brief append a string in hex encoding to the buffer
2147  * @param OutBuf the output buffer
2148  * @param In Buffer to encode
2149  * @param PlainIn way in from plain old c strings
2150  */
2151 void StrBufHexescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
2152 {
2153         StrBufHexEscAppend(OutBuf, In, (const unsigned char*) PlainIn, -1);
2154 }
2155
2156 /**
2157  * @ingroup StrBuf_DeEnCoder
2158  * @brief Append a string, escaping characters which have meaning in HTML.
2159  *
2160  * @param Target        target buffer
2161  * @param Source        source buffer; set to NULL if you just have a C-String
2162  * @param PlainIn       Plain-C string to append; set to NULL if unused
2163  * @param nbsp          If nonzero, spaces are converted to non-breaking spaces.
2164  * @param nolinebreaks  if set to 1, linebreaks are removed from the string.
2165  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
2166  */
2167 long StrEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
2168 {
2169         const char *aptr, *eiptr;
2170         char *bptr, *eptr;
2171         long len;
2172
2173         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2174                 return -1;
2175
2176         if (PlainIn != NULL) {
2177                 aptr = PlainIn;
2178                 len = strlen(PlainIn);
2179                 eiptr = aptr + len;
2180         }
2181         else {
2182                 aptr = Source->buf;
2183                 eiptr = aptr + Source->BufUsed;
2184                 len = Source->BufUsed;
2185         }
2186
2187         if (len == 0)
2188                 return -1;
2189
2190         bptr = Target->buf + Target->BufUsed;
2191         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2192
2193         while (aptr < eiptr){
2194                 if(bptr >= eptr) {
2195                         IncreaseBuf(Target, 1, -1);
2196                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2197                         bptr = Target->buf + Target->BufUsed;
2198                 }
2199                 if (*aptr == '<') {
2200                         memcpy(bptr, "&lt;", 4);
2201                         bptr += 4;
2202                         Target->BufUsed += 4;
2203                 }
2204                 else if (*aptr == '>') {
2205                         memcpy(bptr, "&gt;", 4);
2206                         bptr += 4;
2207                         Target->BufUsed += 4;
2208                 }
2209                 else if (*aptr == '&') {
2210                         memcpy(bptr, "&amp;", 5);
2211                         bptr += 5;
2212                         Target->BufUsed += 5;
2213                 }
2214                 else if (*aptr == '"') {
2215                         memcpy(bptr, "&quot;", 6);
2216                         bptr += 6;
2217                         Target->BufUsed += 6;
2218                 }
2219                 else if (*aptr == '\'') {
2220                         memcpy(bptr, "&#39;", 5);
2221                         bptr += 5;
2222                         Target->BufUsed += 5;
2223                 }
2224                 else if (*aptr == LB) {
2225                         *bptr = '<';
2226                         bptr ++;
2227                         Target->BufUsed ++;
2228                 }
2229                 else if (*aptr == RB) {
2230                         *bptr = '>';
2231                         bptr ++;
2232                         Target->BufUsed ++;
2233                 }
2234                 else if (*aptr == QU) {
2235                         *bptr ='"';
2236                         bptr ++;
2237                         Target->BufUsed ++;
2238                 }
2239                 else if ((*aptr == 32) && (nbsp == 1)) {
2240                         memcpy(bptr, "&nbsp;", 6);
2241                         bptr += 6;
2242                         Target->BufUsed += 6;
2243                 }
2244                 else if ((*aptr == '\n') && (nolinebreaks == 1)) {
2245                         *bptr='\0';     /* nothing */
2246                 }
2247                 else if ((*aptr == '\n') && (nolinebreaks == 2)) {
2248                         memcpy(bptr, "&lt;br/&gt;", 11);
2249                         bptr += 11;
2250                         Target->BufUsed += 11;
2251                 }
2252
2253
2254                 else if ((*aptr == '\r') && (nolinebreaks != 0)) {
2255                         *bptr='\0';     /* nothing */
2256                 }
2257                 else{
2258                         *bptr = *aptr;
2259                         bptr++;
2260                         Target->BufUsed ++;
2261                 }
2262                 aptr ++;
2263         }
2264         *bptr = '\0';
2265         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
2266                 return -1;
2267         return Target->BufUsed;
2268 }
2269
2270 /**
2271  * @ingroup StrBuf_DeEnCoder
2272  * @brief Append a string, escaping characters which have meaning in HTML.
2273  * Converts linebreaks into blanks; escapes single quotes
2274  * @param Target        target buffer
2275  * @param Source        source buffer; set to NULL if you just have a C-String
2276  * @param PlainIn       Plain-C string to append; set to NULL if unused
2277  */
2278 void StrMsgEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2279 {
2280         const char *aptr, *eiptr;
2281         char *tptr, *eptr;
2282         long len;
2283
2284         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2285                 return ;
2286
2287         if (PlainIn != NULL) {
2288                 aptr = PlainIn;
2289                 len = strlen(PlainIn);
2290                 eiptr = aptr + len;
2291         }
2292         else {
2293                 aptr = Source->buf;
2294                 eiptr = aptr + Source->BufUsed;
2295                 len = Source->BufUsed;
2296         }
2297
2298         if (len == 0)
2299                 return;
2300
2301         eptr = Target->buf + Target->BufSize - 8;
2302         tptr = Target->buf + Target->BufUsed;
2303
2304         while (aptr < eiptr){
2305                 if(tptr >= eptr) {
2306                         IncreaseBuf(Target, 1, -1);
2307                         eptr = Target->buf + Target->BufSize - 8;
2308                         tptr = Target->buf + Target->BufUsed;
2309                 }
2310
2311                 if (*aptr == '\n') {
2312                         *tptr = ' ';
2313                         Target->BufUsed++;
2314                 }
2315                 else if (*aptr == '\r') {
2316                         *tptr = ' ';
2317                         Target->BufUsed++;
2318                 }
2319                 else if (*aptr == '\'') {
2320                         *(tptr++) = '&';
2321                         *(tptr++) = '#';
2322                         *(tptr++) = '3';
2323                         *(tptr++) = '9';
2324                         *tptr = ';';
2325                         Target->BufUsed += 5;
2326                 } else {
2327                         *tptr = *aptr;
2328                         Target->BufUsed++;
2329                 }
2330                 tptr++; aptr++;
2331         }
2332         *tptr = '\0';
2333 }
2334
2335
2336
2337 /**
2338  * @ingroup StrBuf_DeEnCoder
2339  * @brief Append a string, escaping characters which have meaning in ICAL.
2340  * [\n,]
2341  * @param Target        target buffer
2342  * @param Source        source buffer; set to NULL if you just have a C-String
2343  * @param PlainIn       Plain-C string to append; set to NULL if unused
2344  */
2345 void StrIcalEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2346 {
2347         const char *aptr, *eiptr;
2348         char *tptr, *eptr;
2349         long len;
2350
2351         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2352                 return ;
2353
2354         if (PlainIn != NULL) {
2355                 aptr = PlainIn;
2356                 len = strlen(PlainIn);
2357                 eiptr = aptr + len;
2358         }
2359         else {
2360                 aptr = Source->buf;
2361                 eiptr = aptr + Source->BufUsed;
2362                 len = Source->BufUsed;
2363         }
2364
2365         if (len == 0)
2366                 return;
2367
2368         eptr = Target->buf + Target->BufSize - 8;
2369         tptr = Target->buf + Target->BufUsed;
2370
2371         while (aptr < eiptr){
2372                 if(tptr + 3 >= eptr) {
2373                         IncreaseBuf(Target, 1, -1);
2374                         eptr = Target->buf + Target->BufSize - 8;
2375                         tptr = Target->buf + Target->BufUsed;
2376                 }
2377
2378                 if (*aptr == '\n') {
2379                         *tptr = '\\';
2380                         Target->BufUsed++;
2381                         tptr++;
2382                         *tptr = 'n';
2383                         Target->BufUsed++;
2384                 }
2385                 else if (*aptr == '\r') {
2386                         *tptr = '\\';
2387                         Target->BufUsed++;
2388                         tptr++;
2389                         *tptr = 'r';
2390                         Target->BufUsed++;
2391                 }
2392                 else if (*aptr == ',') {
2393                         *tptr = '\\';
2394                         Target->BufUsed++;
2395                         tptr++;
2396                         *tptr = ',';
2397                         Target->BufUsed++;
2398                 } else {
2399                         *tptr = *aptr;
2400                         Target->BufUsed++;
2401                 }
2402                 tptr++; aptr++;
2403         }
2404         *tptr = '\0';
2405 }
2406
2407 /**
2408  * @ingroup StrBuf_DeEnCoder
2409  * @brief Append a string, escaping characters which have meaning in JavaScript strings .
2410  *
2411  * @param Target        target buffer
2412  * @param Source        source buffer; set to NULL if you just have a C-String
2413  * @param PlainIn       Plain-C string to append; set to NULL if unused
2414  * @returns size of result or -1
2415  */
2416 long StrECMAEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2417 {
2418         const char *aptr, *eiptr;
2419         char *bptr, *eptr;
2420         long len;
2421         int IsUtf8Sequence;
2422
2423         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2424                 return -1;
2425
2426         if (PlainIn != NULL) {
2427                 aptr = PlainIn;
2428                 len = strlen(PlainIn);
2429                 eiptr = aptr + len;
2430         }
2431         else {
2432                 aptr = Source->buf;
2433                 eiptr = aptr + Source->BufUsed;
2434                 len = Source->BufUsed;
2435         }
2436
2437         if (len == 0)
2438                 return -1;
2439
2440         bptr = Target->buf + Target->BufUsed;
2441         eptr = Target->buf + Target->BufSize - 7; /* our biggest unit to put in...  */
2442
2443         while (aptr < eiptr){
2444                 if(bptr >= eptr) {
2445                         IncreaseBuf(Target, 1, -1);
2446                         eptr = Target->buf + Target->BufSize - 7; /* our biggest unit to put in...  */
2447                         bptr = Target->buf + Target->BufUsed;
2448                 }
2449                 switch (*aptr) {
2450                 case '\n':
2451                         memcpy(bptr, HKEY("\\n"));
2452                         bptr += 2;
2453                         Target->BufUsed += 2;
2454                         break;
2455                 case '\r':
2456                         memcpy(bptr, HKEY("\\r"));
2457                         bptr += 2;
2458                         Target->BufUsed += 2;
2459                         break;
2460                 case '"':
2461                         *bptr = '\\';
2462                         bptr ++;
2463                         *bptr = '"';
2464                         bptr ++;
2465                         Target->BufUsed += 2;
2466                         break;
2467                 case '\\':
2468                         if ((*(aptr + 1) == 'u') &&
2469                             isxdigit(*(aptr + 2)) &&
2470                             isxdigit(*(aptr + 3)) &&
2471                             isxdigit(*(aptr + 4)) &&
2472                             isxdigit(*(aptr + 5)))
2473                         { /* oh, a unicode escaper. let it pass through. */
2474                                 memcpy(bptr, aptr, 6);
2475                                 aptr += 5;
2476                                 bptr +=6;
2477                                 Target->BufUsed += 6;
2478                         }
2479                         else
2480                         {
2481                                 *bptr = '\\';
2482                                 bptr ++;
2483                                 *bptr = '\\';
2484                                 bptr ++;
2485                                 Target->BufUsed += 2;
2486                         }
2487                         break;
2488                 case '\b':
2489                         *bptr = '\\';
2490                         bptr ++;
2491                         *bptr = 'b';
2492                         bptr ++;
2493                         Target->BufUsed += 2;
2494                         break;
2495                 case '\f':
2496                         *bptr = '\\';
2497                         bptr ++;
2498                         *bptr = 'f';
2499                         bptr ++;
2500                         Target->BufUsed += 2;
2501                         break;
2502                 case '\t':
2503                         *bptr = '\\';
2504                         bptr ++;
2505                         *bptr = 't';
2506                         bptr ++;
2507                         Target->BufUsed += 2;
2508                         break;
2509                 default:
2510                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(aptr, eiptr);
2511                         while (IsUtf8Sequence > 0){
2512                                 *bptr = *aptr;
2513                                 Target->BufUsed ++;
2514                                 if (--IsUtf8Sequence)
2515                                         aptr++;
2516                                 bptr++;
2517                         }
2518                 }
2519                 aptr ++;
2520         }
2521         *bptr = '\0';
2522         if ((bptr == eptr - 1 ) && !IsEmptyStr(aptr) )
2523                 return -1;
2524         return Target->BufUsed;
2525 }
2526
2527 /**
2528  * @ingroup StrBuf_DeEnCoder
2529  * @brief Append a string, escaping characters which have meaning in HTML + json.
2530  *
2531  * @param Target        target buffer
2532  * @param Source        source buffer; set to NULL if you just have a C-String
2533  * @param PlainIn       Plain-C string to append; set to NULL if unused
2534  * @param nbsp          If nonzero, spaces are converted to non-breaking spaces.
2535  * @param nolinebreaks  if set to 1, linebreaks are removed from the string.
2536  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
2537  */
2538 long StrHtmlEcmaEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
2539 {
2540         const char *aptr, *eiptr;
2541         char *bptr, *eptr;
2542         long len;
2543         int IsUtf8Sequence = 0;
2544
2545         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2546                 return -1;
2547
2548         if (PlainIn != NULL) {
2549                 aptr = PlainIn;
2550                 len = strlen(PlainIn);
2551                 eiptr = aptr + len;
2552         }
2553         else {
2554                 aptr = Source->buf;
2555                 eiptr = aptr + Source->BufUsed;
2556                 len = Source->BufUsed;
2557         }
2558
2559         if (len == 0)
2560                 return -1;
2561
2562         bptr = Target->buf + Target->BufUsed;
2563         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2564
2565         while (aptr < eiptr){
2566                 if(bptr >= eptr) {
2567                         IncreaseBuf(Target, 1, -1);
2568                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2569                         bptr = Target->buf + Target->BufUsed;
2570                 }
2571                 switch (*aptr) {
2572                 case '<':
2573                         memcpy(bptr, HKEY("&lt;"));
2574                         bptr += 4;
2575                         Target->BufUsed += 4;
2576                         break;
2577                 case '>':
2578                         memcpy(bptr, HKEY("&gt;"));
2579                         bptr += 4;
2580                         Target->BufUsed += 4;
2581                         break;
2582                 case '&':
2583                         memcpy(bptr, HKEY("&amp;"));
2584                         bptr += 5;
2585                         Target->BufUsed += 5;
2586                         break;
2587                 case LB:
2588                         *bptr = '<';
2589                         bptr ++;
2590                         Target->BufUsed ++;
2591                         break;
2592                 case RB:
2593                         *bptr = '>';
2594                         bptr ++;
2595                         Target->BufUsed ++;
2596                         break;
2597                 case '\n':
2598                         switch (nolinebreaks) {
2599                         case 1:
2600                                 *bptr='\0';     /* nothing */
2601                                 break;
2602                         case 2:
2603                                 memcpy(bptr, HKEY("&lt;br/&gt;"));
2604                                 bptr += 11;
2605                                 Target->BufUsed += 11;
2606                                 break;
2607                         default:
2608                                 memcpy(bptr, HKEY("\\n"));
2609                                 bptr += 2;
2610                                 Target->BufUsed += 2;
2611                         }
2612                         break;
2613                 case '\r':
2614                         switch (nolinebreaks) {
2615                         case 1:
2616                         case 2:
2617                                 *bptr='\0';     /* nothing */
2618                                 break;
2619                         default:
2620                                 memcpy(bptr, HKEY("\\r"));
2621                                 bptr += 2;
2622                                 Target->BufUsed += 2;
2623                                 break;
2624                         }
2625                         break;
2626                 case '"':
2627                 case QU:
2628                         *bptr = '\\';
2629                         bptr ++;
2630                         *bptr = '"';
2631                         bptr ++;
2632                         Target->BufUsed += 2;
2633                         break;
2634                 case '\\':
2635                         if ((*(aptr + 1) == 'u') &&
2636                             isxdigit(*(aptr + 2)) &&
2637                             isxdigit(*(aptr + 3)) &&
2638                             isxdigit(*(aptr + 4)) &&
2639                             isxdigit(*(aptr + 5)))
2640                         { /* oh, a unicode escaper. let it pass through. */
2641                                 memcpy(bptr, aptr, 6);
2642                                 aptr += 5;
2643                                 bptr +=6;
2644                                 Target->BufUsed += 6;
2645                         }
2646                         else
2647                         {
2648                                 *bptr = '\\';
2649                                 bptr ++;
2650                                 *bptr = '\\';
2651                                 bptr ++;
2652                                 Target->BufUsed += 2;
2653                         }
2654                         break;
2655                 case '\b':
2656                         *bptr = '\\';
2657                         bptr ++;
2658                         *bptr = 'b';
2659                         bptr ++;
2660                         Target->BufUsed += 2;
2661                         break;
2662                 case '\f':
2663                         *bptr = '\\';
2664                         bptr ++;
2665                         *bptr = 'f';
2666                         bptr ++;
2667                         Target->BufUsed += 2;
2668                         break;
2669                 case '\t':
2670                         *bptr = '\\';
2671                         bptr ++;
2672                         *bptr = 't';
2673                         bptr ++;
2674                         Target->BufUsed += 2;
2675                         break;
2676                 case  32:
2677                         if (nbsp == 1) {
2678                                 memcpy(bptr, HKEY("&nbsp;"));
2679                                 bptr += 6;
2680                                 Target->BufUsed += 6;
2681                                 break;
2682                         }
2683                 default:
2684                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(aptr, eiptr);
2685                         while (IsUtf8Sequence > 0){
2686                                 *bptr = *aptr;
2687                                 Target->BufUsed ++;
2688                                 if (--IsUtf8Sequence)
2689                                         aptr++;
2690                                 bptr++;
2691                         }
2692                 }
2693                 aptr ++;
2694         }
2695         *bptr = '\0';
2696         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
2697                 return -1;
2698         return Target->BufUsed;
2699 }
2700
2701
2702 /**
2703  * @ingroup StrBuf_DeEnCoder
2704  * @brief replace all non-Ascii characters by another
2705  * @param Buf buffer to inspect
2706  * @param repl charater to stamp over non ascii chars
2707  */
2708 void StrBufAsciify(StrBuf *Buf, const char repl)
2709 {
2710         long offset;
2711
2712         for (offset = 0; offset < Buf->BufUsed; offset ++)
2713                 if (!isascii(Buf->buf[offset]))
2714                         Buf->buf[offset] = repl;
2715
2716 }
2717
2718 /**
2719  * @ingroup StrBuf_DeEnCoder
2720  * @brief unhide special chars hidden to the HTML escaper
2721  * @param target buffer to put the unescaped string in
2722  * @param source buffer to unescape
2723  */
2724 void StrBufEUid_unescapize(StrBuf *target, const StrBuf *source)
2725 {
2726         int a, b, len;
2727         char hex[3];
2728
2729         if ((source == NULL) || (target == NULL) || (target->buf == NULL))
2730         {
2731                 return;
2732         }
2733
2734         if (target != NULL)
2735                 FlushStrBuf(target);
2736
2737         len = source->BufUsed;
2738         for (a = 0; a < len; ++a) {
2739                 if (target->BufUsed >= target->BufSize)
2740                         IncreaseBuf(target, 1, -1);
2741
2742                 if (source->buf[a] == '=') {
2743                         hex[0] = source->buf[a + 1];
2744                         hex[1] = source->buf[a + 2];
2745                         hex[2] = 0;
2746                         b = 0;
2747                         sscanf(hex, "%02x", &b);
2748                         target->buf[target->BufUsed] = b;
2749                         target->buf[++target->BufUsed] = 0;
2750                         a += 2;
2751                 }
2752                 else {
2753                         target->buf[target->BufUsed] = source->buf[a];
2754                         target->buf[++target->BufUsed] = 0;
2755                 }
2756         }
2757 }
2758
2759
2760 /**
2761  * @ingroup StrBuf_DeEnCoder
2762  * @brief hide special chars from the HTML escapers and friends
2763  * @param target buffer to put the escaped string in
2764  * @param source buffer to escape
2765  */
2766 void StrBufEUid_escapize(StrBuf *target, const StrBuf *source)
2767 {
2768         int i, len;
2769
2770         if (target != NULL)
2771                 FlushStrBuf(target);
2772
2773         if ((source == NULL) || (target == NULL) || (target->buf == NULL))
2774         {
2775                 return;
2776         }
2777
2778         len = source->BufUsed;
2779         for (i=0; i<len; ++i) {
2780                 if (target->BufUsed + 4 >= target->BufSize)
2781                         IncreaseBuf(target, 1, -1);
2782                 if ( (isalnum(source->buf[i])) ||
2783                      (source->buf[i]=='-') ||
2784                      (source->buf[i]=='_') ) {
2785                         target->buf[target->BufUsed++] = source->buf[i];
2786                 }
2787                 else {
2788                         sprintf(&target->buf[target->BufUsed],
2789                                 "=%02X",
2790                                 (0xFF &source->buf[i]));
2791                         target->BufUsed += 3;
2792                 }
2793         }
2794         target->buf[target->BufUsed + 1] = '\0';
2795 }
2796
2797
2798 /*******************************************************************************
2799  *                      Quoted Printable de/encoding                           *
2800  *******************************************************************************/
2801
2802 /**
2803  * @ingroup StrBuf_DeEnCoder
2804  * @brief decode a buffer from base 64 encoding; destroys original
2805  * @param Buf Buffor to transform
2806  */
2807 int StrBufDecodeBase64(StrBuf *Buf)
2808 {
2809         char *xferbuf;
2810         size_t siz;
2811
2812         if (Buf == NULL)
2813                 return -1;
2814
2815         xferbuf = (char*) malloc(Buf->BufSize);
2816         if (xferbuf == NULL)
2817                 return -1;
2818
2819         *xferbuf = '\0';
2820         siz = CtdlDecodeBase64(xferbuf,
2821                                Buf->buf,
2822                                Buf->BufUsed);
2823         free(Buf->buf);
2824         Buf->buf = xferbuf;
2825         Buf->BufUsed = siz;
2826
2827         Buf->buf[Buf->BufUsed] = '\0';
2828         return siz;
2829 }
2830
2831 /**
2832  * @ingroup StrBuf_DeEnCoder
2833  * @brief decode a buffer from base 64 encoding; expects targetbuffer
2834  * @param BufIn Buffor to transform
2835  * @param BufOut Buffer to put result into
2836  */
2837 int StrBufDecodeBase64To(const StrBuf *BufIn, StrBuf *BufOut)
2838 {
2839         if ((BufIn == NULL) || (BufOut == NULL))
2840                 return -1;
2841
2842         if (BufOut->BufSize < BufIn->BufUsed)
2843                 IncreaseBuf(BufOut, BufIn->BufUsed, 0);
2844
2845         BufOut->BufUsed = CtdlDecodeBase64(BufOut->buf,
2846                                            BufIn->buf,
2847                                            BufIn->BufUsed);
2848         return BufOut->BufUsed;
2849 }
2850
2851 void *StrBufNewStreamContext(eStreamType type)
2852 {
2853         base64_decodestate *state;;
2854
2855         switch (type)
2856         {
2857         case eBase64Decode:
2858                 state = (base64_decodestate*) malloc(sizeof(base64_decodestate));
2859                 base64_init_decodestate(state);
2860                 return state;
2861                 break;
2862         }
2863         return NULL;
2864 }
2865
2866 void StrBufDestroyStreamContext(eStreamType type, void **Stream)
2867 {
2868         switch (type)
2869         {
2870         case eBase64Decode:
2871                 free(*Stream);
2872                 *Stream = NULL;
2873                 break;
2874         }
2875 }
2876 void StrBufStreamDecodeTo(StrBuf *Target, const StrBuf *In, const char* pIn, long pInLen, void *Stream)
2877 {
2878         base64_decodestate *state = Stream;
2879         long ExpectLen;
2880
2881         if (In != NULL)
2882         {
2883                 pIn = In->buf;
2884                 pInLen = In->BufUsed;
2885         }
2886         if ((pIn == NULL) || (Stream == NULL))
2887                 return;
2888
2889         ExpectLen = (pInLen / 4) * 3;
2890
2891         if (Target->BufSize - Target->BufUsed < ExpectLen)
2892         {
2893                 IncreaseBuf(Target, 1, Target->BufUsed + ExpectLen + 1);
2894         }
2895
2896         ExpectLen = base64_decode_block(pIn, pInLen, Target->buf + Target->BufUsed, state);
2897         Target->BufUsed += ExpectLen;
2898         Target->buf[Target->BufUsed] = '\0';
2899 }
2900
2901 /**
2902  * @ingroup StrBuf_DeEnCoder
2903  * @brief decode a buffer from base 64 encoding; destroys original
2904  * @param Buf Buffor to transform
2905  */
2906 int StrBufDecodeHex(StrBuf *Buf)
2907 {
2908         unsigned int ch;
2909         char *pch, *pche, *pchi;
2910
2911         if (Buf == NULL) return -1;
2912
2913         pch = pchi = Buf->buf;
2914         pche = pch + Buf->BufUsed;
2915
2916         while (pchi < pche){
2917                 ch = decode_hex(pchi);
2918                 *pch = ch;
2919                 pch ++;
2920                 pchi += 2;
2921         }
2922
2923         *pch = '\0';
2924         Buf->BufUsed = pch - Buf->buf;
2925         return Buf->BufUsed;
2926 }
2927
2928 /**
2929  * @ingroup StrBuf_DeEnCoder
2930  * @brief replace all chars >0x20 && < 0x7F with Mute
2931  * @param Mute char to put over invalid chars
2932  * @param Buf Buffor to transform
2933  */
2934 int StrBufSanitizeAscii(StrBuf *Buf, const char Mute)
2935 {
2936         unsigned char *pch;
2937
2938         if (Buf == NULL) return -1;
2939         pch = (unsigned char *)Buf->buf;
2940         while (pch < (unsigned char *)Buf->buf + Buf->BufUsed) {
2941                 if ((*pch < 0x20) || (*pch > 0x7F))
2942                         *pch = Mute;
2943                 pch ++;
2944         }
2945         return Buf->BufUsed;
2946 }
2947
2948
2949 /**
2950  * @ingroup StrBuf_DeEnCoder
2951  * @brief remove escaped strings from i.e. the url string (like %20 for blanks)
2952  * @param Buf Buffer to translate
2953  * @param StripBlanks Reduce several blanks to one?
2954  */
2955 long StrBufUnescape(StrBuf *Buf, int StripBlanks)
2956 {
2957         int a, b;
2958         char hex[3];
2959         long len;
2960
2961         if (Buf == NULL)
2962                 return -1;
2963
2964         while ((Buf->BufUsed > 0) && (isspace(Buf->buf[Buf->BufUsed - 1]))){
2965                 Buf->buf[Buf->BufUsed - 1] = '\0';
2966                 Buf->BufUsed --;
2967         }
2968
2969         a = 0;
2970         while (a < Buf->BufUsed) {
2971                 if (Buf->buf[a] == '+')
2972                         Buf->buf[a] = ' ';
2973                 else if (Buf->buf[a] == '%') {
2974                         /* don't let % chars through, rather truncate the input. */
2975                         if (a + 2 > Buf->BufUsed) {
2976                                 Buf->buf[a] = '\0';
2977                                 Buf->BufUsed = a;
2978                         }
2979                         else {
2980                                 hex[0] = Buf->buf[a + 1];
2981                                 hex[1] = Buf->buf[a + 2];
2982                                 hex[2] = 0;
2983                                 b = 0;
2984                                 sscanf(hex, "%02x", &b);
2985                                 Buf->buf[a] = (char) b;
2986                                 len = Buf->BufUsed - a - 2;
2987                                 if (len > 0)
2988                                         memmove(&Buf->buf[a + 1], &Buf->buf[a + 3], len);
2989
2990                                 Buf->BufUsed -=2;
2991                         }
2992                 }
2993                 a++;
2994         }
2995         return a;
2996 }
2997
2998
2999 /**
3000  * @ingroup StrBuf_DeEnCoder
3001  * @brief       RFC2047-encode a header field if necessary.
3002  *              If no non-ASCII characters are found, the string
3003  *              will be copied verbatim without encoding.
3004  *
3005  * @param       target          Target buffer.
3006  * @param       source          Source string to be encoded.
3007  * @returns     encoded length; -1 if non success.
3008  */
3009 int StrBufRFC2047encode(StrBuf **target, const StrBuf *source)
3010 {
3011         const char headerStr[] = "=?UTF-8?Q?";
3012         int need_to_encode = 0;
3013         int i = 0;
3014         unsigned char ch;
3015
3016         if ((source == NULL) ||
3017             (target == NULL))
3018             return -1;
3019
3020         while ((i < source->BufUsed) &&
3021                (!IsEmptyStr (&source->buf[i])) &&
3022                (need_to_encode == 0)) {
3023                 if (((unsigned char) source->buf[i] < 32) ||
3024                     ((unsigned char) source->buf[i] > 126)) {
3025                         need_to_encode = 1;
3026                 }
3027                 i++;
3028         }
3029
3030         if (!need_to_encode) {
3031                 if (*target == NULL) {
3032                         *target = NewStrBufPlain(source->buf, source->BufUsed);
3033                 }
3034                 else {
3035                         FlushStrBuf(*target);
3036                         StrBufAppendBuf(*target, source, 0);
3037                 }
3038                 if (*target != 0)
3039                         return (*target)->BufUsed;
3040                 else
3041                         return 0;
3042         }
3043         if (*target == NULL)
3044                 *target = NewStrBufPlain(NULL, sizeof(headerStr) + source->BufUsed * 2);
3045         else if (sizeof(headerStr) + source->BufUsed >= (*target)->BufSize)
3046                 IncreaseBuf(*target, sizeof(headerStr) + source->BufUsed, 0);
3047         memcpy ((*target)->buf, headerStr, sizeof(headerStr) - 1);
3048         (*target)->BufUsed = sizeof(headerStr) - 1;
3049         for (i=0; (i < source->BufUsed); ++i) {
3050                 if ((*target)->BufUsed + 4 >= (*target)->BufSize)
3051                         IncreaseBuf(*target, 1, 0);
3052                 ch = (unsigned char) source->buf[i];
3053                 if ((ch  <  32) ||
3054                     (ch  > 126) ||
3055                     (ch == '=') ||
3056                     (ch == '?') ||
3057                     (ch == '_') ||
3058                     (ch == '[') ||
3059                     (ch == ']')   )
3060                 {
3061                         sprintf(&(*target)->buf[(*target)->BufUsed], "=%02X", ch);
3062                         (*target)->BufUsed += 3;
3063                 }
3064                 else {
3065                         if (ch == ' ')
3066                                 (*target)->buf[(*target)->BufUsed] = '_';
3067                         else
3068                                 (*target)->buf[(*target)->BufUsed] = ch;
3069                         (*target)->BufUsed++;
3070                 }
3071         }
3072
3073         if ((*target)->BufUsed + 4 >= (*target)->BufSize)
3074                 IncreaseBuf(*target, 1, 0);
3075
3076         (*target)->buf[(*target)->BufUsed++] = '?';
3077         (*target)->buf[(*target)->BufUsed++] = '=';
3078         (*target)->buf[(*target)->BufUsed] = '\0';
3079         return (*target)->BufUsed;;
3080 }
3081
3082 /**
3083  * @ingroup StrBuf_DeEnCoder
3084  * @brief       Quoted-Printable encode a message; make it < 80 columns width.
3085  * @param       source          Source string to be encoded.
3086  * @returns     buffer with encoded message.
3087  */
3088 StrBuf *StrBufRFC2047encodeMessage(const StrBuf *EncodeMe)
3089 {
3090         StrBuf *OutBuf;
3091         char *Optr, *OEptr;
3092         const char *ptr, *eptr;
3093         unsigned char ch;
3094         int LinePos;
3095
3096         OutBuf = NewStrBufPlain(NULL, StrLength(EncodeMe) * 4);
3097         Optr = OutBuf->buf;
3098         OEptr = OutBuf->buf + OutBuf->BufSize;
3099         ptr = EncodeMe->buf;
3100         eptr = EncodeMe->buf + EncodeMe->BufUsed;
3101         LinePos = 0;
3102
3103         while (ptr < eptr)
3104         {
3105                 if (Optr + 4 >= OEptr)
3106                 {
3107                         long Offset;
3108                         Offset = Optr - OutBuf->buf;
3109                         OutBuf->BufUsed = Optr - OutBuf->buf;
3110                         IncreaseBuf(OutBuf, 1, 0);
3111                         Optr = OutBuf->buf + Offset;
3112                         OEptr = OutBuf->buf + OutBuf->BufSize;
3113                 }
3114                 if (*ptr == '\r')
3115                 {
3116                         /* ignore carriage returns */
3117                         ptr ++;
3118                 }
3119                 else if (*ptr == '\n') {
3120                         /* hard line break */
3121                         memcpy(Optr, HKEY("=0A"));
3122                         Optr += 3;
3123                         LinePos += 3;
3124                         ptr ++;
3125                 }
3126                 else if (( (*ptr >= 32) && (*ptr <= 60) ) ||
3127                          ( (*ptr >= 62) && (*ptr <= 126) ))
3128                 {
3129                         *Optr = *ptr;
3130                         Optr ++;
3131                         ptr ++;
3132                         LinePos ++;
3133                 }
3134                 else {
3135                         ch = *ptr;
3136                         *Optr = '=';
3137                         Optr ++;
3138                         *Optr = HexList[ch][0];
3139                         Optr ++;
3140                         *Optr = HexList[ch][1];
3141                         Optr ++;
3142                         LinePos += 3;
3143                         ptr ++;
3144                 }
3145
3146                 if (LinePos > 72) {
3147                         /* soft line break */
3148                         if (isspace(*(Optr - 1))) {
3149                                 ch = *(Optr - 1);
3150                                 Optr --;
3151                                 *Optr = '=';
3152                                 Optr ++;
3153                                 *Optr = HexList[ch][0];
3154                                 Optr ++;
3155                                 *Optr = HexList[ch][1];
3156                                 Optr ++;
3157                                 LinePos += 3;
3158                         }
3159                         *Optr = '=';
3160                         Optr ++;
3161                         *Optr = '\n';
3162                         Optr ++;
3163                         LinePos = 0;
3164                 }
3165         }
3166         *Optr = '\0';
3167         OutBuf->BufUsed = Optr - OutBuf->buf;
3168
3169         return OutBuf;
3170 }
3171
3172
3173 static void AddRecipient(StrBuf *Target,
3174                          StrBuf *UserName,
3175                          StrBuf *EmailAddress,
3176                          StrBuf *EncBuf)
3177 {
3178         int QuoteMe = 0;
3179
3180         if (StrLength(Target) > 0) StrBufAppendBufPlain(Target, HKEY(", "), 0);
3181         if (strchr(ChrPtr(UserName), ',') != NULL) QuoteMe = 1;
3182
3183         if (QuoteMe)  StrBufAppendBufPlain(Target, HKEY("\""), 0);
3184         StrBufRFC2047encode(&EncBuf, UserName);
3185         StrBufAppendBuf(Target, EncBuf, 0);
3186         if (QuoteMe)  StrBufAppendBufPlain(Target, HKEY("\" "), 0);
3187         else          StrBufAppendBufPlain(Target, HKEY(" "), 0);
3188
3189         if (StrLength(EmailAddress) > 0){
3190                 StrBufAppendBufPlain(Target, HKEY("<"), 0);
3191                 StrBufAppendBuf(Target, EmailAddress, 0); /* TODO: what about IDN???? */
3192                 StrBufAppendBufPlain(Target, HKEY(">"), 0);
3193         }
3194 }
3195
3196
3197 /**
3198  * \brief QP encode parts of an email TO/CC/BCC vector, and strip/filter invalid parts
3199  * \param Recp Source list of email recipients
3200  * \param UserName Temporary buffer for internal use; Please provide valid buffer.
3201  * \param EmailAddress Temporary buffer for internal use; Please provide valid buffer.
3202  * \param EncBuf Temporary buffer for internal use; Please provide valid buffer.
3203  * \returns encoded & sanitized buffer with the contents of Recp; Caller owns this memory.
3204  */
3205 StrBuf *StrBufSanitizeEmailRecipientVector(const StrBuf *Recp,
3206                                            StrBuf *UserName,
3207                                            StrBuf *EmailAddress,
3208                                            StrBuf *EncBuf)
3209 {
3210         StrBuf *Target;
3211         const char *pch, *pche;
3212         const char *UserStart, *UserEnd, *EmailStart, *EmailEnd, *At;
3213
3214         if ((Recp == NULL) || (StrLength(Recp) == 0))
3215                 return NULL;
3216
3217         pch = ChrPtr(Recp);
3218         pche = pch + StrLength(Recp);
3219
3220         if (!CheckEncode(pch, -1, pche))
3221                 return NewStrBufDup(Recp);
3222
3223         Target = NewStrBufPlain(NULL, StrLength(Recp));
3224
3225         while ((pch != NULL) && (pch < pche))
3226         {
3227                 while (isspace(*pch)) pch++;
3228                 UserEnd = EmailStart = EmailEnd = NULL;
3229
3230                 if ((*pch == '"') || (*pch == '\'')) {
3231                         UserStart = pch + 1;
3232
3233                         UserEnd = strchr(UserStart, *pch);
3234                         if (UserEnd == NULL)
3235                                 break; ///TODO: Userfeedback??
3236                         EmailStart = UserEnd + 1;
3237                         while (isspace(*EmailStart))
3238                                 EmailStart++;
3239                         if (UserEnd == UserStart) {
3240                                 UserStart = UserEnd = NULL;
3241                         }
3242
3243                         if (*EmailStart == '<') {
3244                                 EmailStart++;
3245                                 EmailEnd = strchr(EmailStart, '>');
3246                                 if (EmailEnd == NULL)
3247                                         EmailEnd = strchr(EmailStart, ',');
3248
3249                         }
3250                         else {
3251                                 EmailEnd = strchr(EmailStart, ',');
3252                         }
3253                         if (EmailEnd == NULL)
3254                                 EmailEnd = pche;
3255                         pch = EmailEnd + 1;
3256                 }
3257                 else {
3258                         int gt = 0;
3259                         UserStart = pch;
3260                         EmailEnd = strchr(UserStart, ',');
3261                         if (EmailEnd == NULL) {
3262                                 EmailEnd = strchr(pch, '>');
3263                                 pch = NULL;
3264                                 if (EmailEnd != NULL) {
3265                                         gt = 1;
3266                                 }
3267                                 else {
3268                                         EmailEnd = pche;
3269                                 }
3270                         }
3271                         else {
3272
3273                                 pch = EmailEnd + 1;
3274                                 while ((EmailEnd > UserStart) && !gt &&
3275                                        ((*EmailEnd == ',') ||
3276                                         (*EmailEnd == '>') ||
3277                                         (isspace(*EmailEnd))))
3278                                 {
3279                                         if (*EmailEnd == '>')
3280                                                 gt = 1;
3281                                         else
3282                                                 EmailEnd--;
3283                                 }
3284                                 if (EmailEnd == UserStart)
3285                                         break;
3286                         }
3287                         if (gt) {
3288                                 EmailStart = strchr(UserStart, '<');
3289                                 if ((EmailStart == NULL) || (EmailStart > EmailEnd))
3290                                         break;
3291                                 UserEnd = EmailStart;
3292
3293                                 while ((UserEnd > UserStart) &&
3294                                        isspace (*(UserEnd - 1)))
3295                                         UserEnd --;
3296                                 EmailStart ++;
3297                                 if (UserStart >= UserEnd)
3298                                         UserStart = UserEnd = NULL;
3299                         }
3300                         else { /* this is a local recipient... no domain, just a realname */
3301                                 EmailStart = UserStart;
3302                                 At = strchr(EmailStart, '@');
3303                                 if (At == NULL) {
3304                                         UserEnd = EmailEnd;
3305                                         EmailEnd = NULL;
3306                                 }
3307                                 else {
3308                                         EmailStart = UserStart;
3309                                         UserStart = NULL;
3310                                 }
3311                         }
3312                 }
3313
3314                 if ((UserStart != NULL) && (UserEnd != NULL))
3315                         StrBufPlain(UserName, UserStart, UserEnd - UserStart);
3316                 else if ((UserStart != NULL) && (UserEnd == NULL))
3317                         StrBufPlain(UserName, UserStart, UserEnd - UserStart);
3318                 else
3319                         FlushStrBuf(UserName);
3320
3321                 if ((EmailStart != NULL) && (EmailEnd != NULL))
3322                         StrBufPlain(EmailAddress, EmailStart, EmailEnd - EmailStart);
3323                 else if ((EmailStart != NULL) && (EmailEnd == NULL))
3324                         StrBufPlain(EmailAddress, EmailStart, EmailEnd - pche);
3325                 else
3326                         FlushStrBuf(EmailAddress);
3327
3328                 AddRecipient(Target, UserName, EmailAddress, EncBuf);
3329
3330                 if (pch == NULL)
3331                         break;
3332
3333                 if ((pch != NULL) && (*pch == ','))
3334                         pch ++;
3335                 if (pch != NULL) while (isspace(*pch))
3336                         pch ++;
3337         }
3338         return Target;
3339 }
3340
3341
3342 /**
3343  * @ingroup StrBuf
3344  * @brief replaces all occurances of 'search' by 'replace'
3345  * @param buf Buffer to modify
3346  * @param search character to search
3347  * @param replace character to replace search by
3348  */
3349 void StrBufReplaceChars(StrBuf *buf, char search, char replace)
3350 {
3351         long i;
3352         if (buf == NULL)
3353                 return;
3354         for (i=0; i<buf->BufUsed; i++)
3355                 if (buf->buf[i] == search)
3356                         buf->buf[i] = replace;
3357
3358 }
3359
3360 /**
3361  * @ingroup StrBuf
3362  * @brief removes all \\r s from the string, or replaces them with \n if its not a combination of both.
3363  * @param buf Buffer to modify
3364  */
3365 void StrBufToUnixLF(StrBuf *buf)
3366 {
3367         char *pche, *pchS, *pchT;
3368         if (buf == NULL)
3369                 return;
3370
3371         pche = buf->buf + buf->BufUsed;
3372         pchS = pchT = buf->buf;
3373         while (pchS < pche)
3374         {
3375                 if (*pchS == '\r')
3376                 {
3377                         pchS ++;
3378                         if (*pchS != '\n') {
3379                                 *pchT = '\n';
3380                                 pchT++;
3381                         }
3382                 }
3383                 *pchT = *pchS;
3384                 pchT++; pchS++;
3385         }
3386         *pchT = '\0';
3387         buf->BufUsed = pchT - buf->buf;
3388 }
3389
3390
3391 /*******************************************************************************
3392  *                 Iconv Wrapper; RFC822 de/encoding                           *
3393  *******************************************************************************/
3394
3395 /**
3396  * @ingroup StrBuf_DeEnCoder
3397  * @brief Wrapper around iconv_open()
3398  * Our version adds aliases for non-standard Microsoft charsets
3399  * such as 'MS950', aliasing them to names like 'CP950'
3400  *
3401  * @param tocode        Target encoding
3402  * @param fromcode      Source encoding
3403  * @param pic           anonimized pointer to iconv struct
3404  */
3405 void  ctdl_iconv_open(const char *tocode, const char *fromcode, void *pic)
3406 {
3407 #ifdef HAVE_ICONV
3408         iconv_t ic = (iconv_t)(-1) ;
3409         ic = iconv_open(tocode, fromcode);
3410         if (ic == (iconv_t)(-1) ) {
3411                 char alias_fromcode[64];
3412                 if ( (strlen(fromcode) == 5) && (!strncasecmp(fromcode, "MS", 2)) ) {
3413                         safestrncpy(alias_fromcode, fromcode, sizeof alias_fromcode);
3414                         alias_fromcode[0] = 'C';
3415                         alias_fromcode[1] = 'P';
3416                         ic = iconv_open(tocode, alias_fromcode);
3417                 }
3418         }
3419         *(iconv_t *)pic = ic;
3420 #endif
3421 }
3422
3423
3424 /**
3425  * @ingroup StrBuf_DeEnCoder
3426  * @brief find one chunk of a RFC822 encoded string
3427  * @param Buffer where to search
3428  * @param bptr where to start searching
3429  * @returns found position, NULL if none.
3430  */
3431 static inline const char *FindNextEnd (const StrBuf *Buf, const char *bptr)
3432 {
3433         const char * end;
3434         /* Find the next ?Q? */
3435         if (Buf->BufUsed - (bptr - Buf->buf)  < 6)
3436                 return NULL;
3437
3438         end = strchr(bptr + 2, '?');
3439
3440         if (end == NULL)
3441                 return NULL;
3442
3443         if ((Buf->BufUsed - (end - Buf->buf) > 3) &&
3444             (((*(end + 1) == 'B') || (*(end + 1) == 'Q')) ||
3445              ((*(end + 1) == 'b') || (*(end + 1) == 'q'))) &&
3446             (*(end + 2) == '?')) {
3447                 /* skip on to the end of the cluster, the next ?= */
3448                 end = strstr(end + 3, "?=");
3449         }
3450         else
3451                 /* sort of half valid encoding, try to find an end. */
3452                 end = strstr(bptr, "?=");
3453         return end;
3454 }
3455
3456
3457
3458 /**
3459  * @ingroup StrBuf_DeEnCoder
3460  * @brief convert one buffer according to the preselected iconv pointer PIC
3461  * @param ConvertBuf buffer we need to translate
3462  * @param TmpBuf To share a workbuffer over several iterations. prepare to have it filled with useless stuff afterwards.
3463  * @param pic Pointer to the iconv-session Object
3464  */
3465 void StrBufConvert(StrBuf *ConvertBuf, StrBuf *TmpBuf, void *pic)
3466 {
3467 #ifdef HAVE_ICONV
3468         long trycount = 0;
3469         size_t siz;
3470         iconv_t ic;
3471         char *ibuf;                     /**< Buffer of characters to be converted */
3472         char *obuf;                     /**< Buffer for converted characters */
3473         size_t ibuflen;                 /**< Length of input buffer */
3474         size_t obuflen;                 /**< Length of output buffer */
3475
3476
3477         if ((ConvertBuf == NULL) || (TmpBuf == NULL))
3478                 return;
3479
3480         /* since we're converting to utf-8, one glyph may take up to 6 bytes */
3481         if (ConvertBuf->BufUsed * 6 >= TmpBuf->BufSize)
3482                 IncreaseBuf(TmpBuf, 0, ConvertBuf->BufUsed * 6);
3483 TRYAGAIN:
3484         ic = *(iconv_t*)pic;
3485         ibuf = ConvertBuf->buf;
3486         ibuflen = ConvertBuf->BufUsed;
3487         obuf = TmpBuf->buf;
3488         obuflen = TmpBuf->BufSize;
3489
3490         siz = iconv(ic, &ibuf, &ibuflen, &obuf, &obuflen);
3491
3492         if (siz < 0) {
3493                 if (errno == E2BIG) {
3494                         trycount ++;
3495                         IncreaseBuf(TmpBuf, 0, 0);
3496                         if (trycount < 5)
3497                                 goto TRYAGAIN;
3498
3499                 }
3500                 else if (errno == EILSEQ){
3501                         /* hm, invalid utf8 sequence... what to do now? */
3502                         /* An invalid multibyte sequence has been encountered in the input */
3503                 }
3504                 else if (errno == EINVAL) {
3505                         /* An incomplete multibyte sequence has been encountered in the input. */
3506                 }
3507
3508                 FlushStrBuf(TmpBuf);
3509         }
3510         else {
3511                 TmpBuf->BufUsed = TmpBuf->BufSize - obuflen;
3512                 TmpBuf->buf[TmpBuf->BufUsed] = '\0';
3513
3514                 /* little card game: wheres the red lady? */
3515                 SwapBuffers(ConvertBuf, TmpBuf);
3516                 FlushStrBuf(TmpBuf);
3517         }
3518 #endif
3519 }
3520
3521
3522 /**
3523  * @ingroup StrBuf_DeEnCoder
3524  * @brief catches one RFC822 encoded segment, and decodes it.
3525  * @param Target buffer to fill with result
3526  * @param DecodeMe buffer with stuff to process
3527  * @param SegmentStart points to our current segment in DecodeMe
3528  * @param SegmentEnd Points to the end of our current segment in DecodeMe
3529  * @param ConvertBuf Workbuffer shared between several iterations. Random content; needs to be valid
3530  * @param ConvertBuf2 Workbuffer shared between several iterations. Random content; needs to be valid
3531  * @param FoundCharset Characterset to default decoding to; if we find another we will overwrite it.
3532  */
3533 inline static void DecodeSegment(StrBuf *Target,
3534                                  const StrBuf *DecodeMe,
3535                                  const char *SegmentStart,
3536                                  const char *SegmentEnd,
3537                                  StrBuf *ConvertBuf,
3538                                  StrBuf *ConvertBuf2,
3539                                  StrBuf *FoundCharset)
3540 {
3541         StrBuf StaticBuf;
3542         char charset[128];
3543         char encoding[16];
3544 #ifdef HAVE_ICONV
3545         iconv_t ic = (iconv_t)(-1);
3546 #else
3547         void *ic = NULL;
3548 #endif
3549         /* Now we handle foreign character sets properly encoded
3550          * in RFC2047 format.
3551          */
3552         StaticBuf.buf = (char*) SegmentStart; /*< it will just be read there... */
3553         StaticBuf.BufUsed = SegmentEnd - SegmentStart;
3554         StaticBuf.BufSize = DecodeMe->BufSize - (SegmentStart - DecodeMe->buf);
3555         extract_token(charset, SegmentStart, 1, '?', sizeof charset);
3556         if (FoundCharset != NULL) {
3557                 FlushStrBuf(FoundCharset);
3558                 StrBufAppendBufPlain(FoundCharset, charset, -1, 0);
3559         }
3560         extract_token(encoding, SegmentStart, 2, '?', sizeof encoding);
3561         StrBufExtract_token(ConvertBuf, &StaticBuf, 3, '?');
3562
3563         *encoding = toupper(*encoding);
3564         if (*encoding == 'B') { /**< base64 */
3565                 if (ConvertBuf2->BufSize < ConvertBuf->BufUsed)
3566                         IncreaseBuf(ConvertBuf2, 0, ConvertBuf->BufUsed);
3567                 ConvertBuf2->BufUsed = CtdlDecodeBase64(ConvertBuf2->buf,
3568                                                         ConvertBuf->buf,
3569                                                         ConvertBuf->BufUsed);
3570         }
3571         else if (*encoding == 'Q') {    /**< quoted-printable */
3572                 long pos;
3573
3574                 pos = 0;
3575                 while (pos < ConvertBuf->BufUsed)
3576                 {
3577                         if (ConvertBuf->buf[pos] == '_')
3578                                 ConvertBuf->buf[pos] = ' ';
3579                         pos++;
3580                 }
3581
3582                 if (ConvertBuf2->BufSize < ConvertBuf->BufUsed)
3583                         IncreaseBuf(ConvertBuf2, 0, ConvertBuf->BufUsed);
3584
3585                 ConvertBuf2->BufUsed = CtdlDecodeQuotedPrintable(
3586                         ConvertBuf2->buf,
3587                         ConvertBuf->buf,
3588                         ConvertBuf->BufUsed);
3589         }
3590         else {
3591                 StrBufAppendBuf(ConvertBuf2, ConvertBuf, 0);
3592         }
3593 #ifdef HAVE_ICONV
3594         ctdl_iconv_open("UTF-8", charset, &ic);
3595         if (ic != (iconv_t)(-1) ) {
3596 #endif
3597                 StrBufConvert(ConvertBuf2, ConvertBuf, &ic);
3598                 StrBufAppendBuf(Target, ConvertBuf2, 0);
3599 #ifdef HAVE_ICONV
3600                 iconv_close(ic);
3601         }
3602         else {
3603                 StrBufAppendBufPlain(Target, HKEY("(unreadable)"), 0);
3604         }
3605 #endif
3606 }
3607
3608 /**
3609  * @ingroup StrBuf_DeEnCoder
3610  * @brief Handle subjects with RFC2047 encoding such as: [deprecated old syntax!]
3611  * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
3612  * @param Target where to put the decoded string to
3613  * @param DecodeMe buffer with encoded string
3614  * @param DefaultCharset if we don't find one, which should we use?
3615  * @param FoundCharset overrides DefaultCharset if non-empty; If we find a charset inside of the string,
3616  *        put it here for later use where no string might be known.
3617  */
3618 void StrBuf_RFC822_to_Utf8(StrBuf *Target, const StrBuf *DecodeMe, const StrBuf* DefaultCharset, StrBuf *FoundCharset)
3619 {
3620         StrBuf *ConvertBuf;
3621         StrBuf *ConvertBuf2;
3622         ConvertBuf = NewStrBufPlain(NULL, StrLength(DecodeMe));
3623         ConvertBuf2 = NewStrBufPlain(NULL, StrLength(DecodeMe));
3624
3625         StrBuf_RFC822_2_Utf8(Target,
3626                              DecodeMe,
3627                              DefaultCharset,
3628                              FoundCharset,
3629                              ConvertBuf,
3630                              ConvertBuf2);
3631         FreeStrBuf(&ConvertBuf);
3632         FreeStrBuf(&ConvertBuf2);
3633 }
3634
3635 /**
3636  * @ingroup StrBuf_DeEnCoder
3637  * @brief Handle subjects with RFC2047 encoding such as:
3638  * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
3639  * @param Target where to put the decoded string to
3640  * @param DecodeMe buffer with encoded string
3641  * @param DefaultCharset if we don't find one, which should we use?
3642  * @param FoundCharset overrides DefaultCharset if non-empty; If we find a charset inside of the string,
3643  *        put it here for later use where no string might be known.
3644  * @param ConvertBuf workbuffer. feed in, you shouldn't care about its content.
3645  * @param ConvertBuf2 workbuffer. feed in, you shouldn't care about its content.
3646  */
3647 void StrBuf_RFC822_2_Utf8(StrBuf *Target,
3648                           const StrBuf *DecodeMe,
3649                           const StrBuf* DefaultCharset,
3650                           StrBuf *FoundCharset,
3651                           StrBuf *ConvertBuf,
3652                           StrBuf *ConvertBuf2)
3653 {
3654         StrBuf *DecodedInvalidBuf = NULL;
3655         const StrBuf *DecodeMee = DecodeMe;
3656         const char *start, *end, *next, *nextend, *ptr = NULL;
3657 #ifdef HAVE_ICONV
3658         iconv_t ic = (iconv_t)(-1) ;
3659 #endif
3660         const char *eptr;
3661         int passes = 0;
3662         int i;
3663         int illegal_non_rfc2047_encoding = 0;
3664
3665
3666         if (DecodeMe == NULL)
3667                 return;
3668         /* Sometimes, badly formed messages contain strings which were simply
3669          *  written out directly in some foreign character set instead of
3670          *  using RFC2047 encoding.  This is illegal but we will attempt to
3671          *  handle it anyway by converting from a user-specified default
3672          *  charset to UTF-8 if we see any nonprintable characters.
3673          */
3674
3675         for (i=0; i<DecodeMe->BufUsed; ++i) {
3676                 if ((DecodeMe->buf[i] < 32) || (DecodeMe->buf[i] > 126)) {
3677                         illegal_non_rfc2047_encoding = 1;
3678                         break;
3679                 }
3680         }
3681
3682         if ((illegal_non_rfc2047_encoding) &&
3683             (strcasecmp(ChrPtr(DefaultCharset), "UTF-8")) &&
3684             (strcasecmp(ChrPtr(DefaultCharset), "us-ascii")) )
3685         {
3686 #ifdef HAVE_ICONV
3687                 ctdl_iconv_open("UTF-8", ChrPtr(DefaultCharset), &ic);
3688                 if (ic != (iconv_t)(-1) ) {
3689                         DecodedInvalidBuf = NewStrBufDup(DecodeMe);
3690                         StrBufConvert(DecodedInvalidBuf, ConvertBuf, &ic);///TODO: don't void const?
3691                         DecodeMee = DecodedInvalidBuf;
3692                         iconv_close(ic);
3693                 }
3694 #endif
3695         }
3696
3697         /* pre evaluate the first pair */
3698         end = NULL;
3699         start = strstr(DecodeMee->buf, "=?");
3700         eptr = DecodeMee->buf + DecodeMee->BufUsed;
3701         if (start != NULL)
3702                 end = FindNextEnd (DecodeMee, start + 2);
3703         else {
3704                 StrBufAppendBuf(Target, DecodeMee, 0);
3705                 FreeStrBuf(&DecodedInvalidBuf);
3706                 return;
3707         }
3708
3709
3710         if (start != DecodeMee->buf) {
3711                 long nFront;
3712
3713                 nFront = start - DecodeMee->buf;
3714                 StrBufAppendBufPlain(Target, DecodeMee->buf, nFront, 0);
3715         }
3716         /*
3717          * Since spammers will go to all sorts of absurd lengths to get their
3718          * messages through, there are LOTS of corrupt headers out there.
3719          * So, prevent a really badly formed RFC2047 header from throwing
3720          * this function into an infinite loop.
3721          */
3722         while ((start != NULL) &&
3723                (end != NULL) &&
3724                (start < eptr) &&
3725                (end < eptr) &&
3726                (passes < 20))
3727         {
3728                 passes++;
3729                 DecodeSegment(Target,
3730                               DecodeMee,
3731                               start,
3732                               end,
3733                               ConvertBuf,
3734                               ConvertBuf2,
3735                               FoundCharset);
3736
3737                 next = strstr(end, "=?");
3738                 nextend = NULL;
3739                 if ((next != NULL) &&
3740                     (next < eptr))
3741                         nextend = FindNextEnd(DecodeMee, next);
3742                 if (nextend == NULL)
3743                         next = NULL;
3744
3745                 /* did we find two partitions */
3746                 if ((next != NULL) &&
3747                     ((next - end) > 2))
3748                 {
3749                         ptr = end + 2;
3750                         while ((ptr < next) &&
3751                                (isspace(*ptr) ||
3752                                 (*ptr == '\r') ||
3753                                 (*ptr == '\n') ||
3754                                 (*ptr == '\t')))
3755                                 ptr ++;
3756                         /*
3757                          * did we find a gab just filled with blanks?
3758                          * if not, copy its stuff over.
3759                          */
3760                         if (ptr != next)
3761                         {
3762                                 StrBufAppendBufPlain(Target,
3763                                                      end + 2,
3764                                                      next - end - 2,
3765                                                      0);
3766                         }
3767                 }
3768                 /* our next-pair is our new first pair now. */
3769                 ptr = end + 2;
3770                 start = next;
3771                 end = nextend;
3772         }
3773         end = ptr;
3774         nextend = DecodeMee->buf + DecodeMee->BufUsed;
3775         if ((end != NULL) && (end < nextend)) {
3776                 ptr = end;
3777                 while ( (ptr < nextend) &&
3778                         (isspace(*ptr) ||
3779                          (*ptr == '\r') ||
3780                          (*ptr == '\n') ||
3781                          (*ptr == '\t')))
3782                         ptr ++;
3783                 if (ptr < nextend)
3784                         StrBufAppendBufPlain(Target, end, nextend - end, 0);
3785         }
3786         FreeStrBuf(&DecodedInvalidBuf);
3787 }
3788
3789 /*******************************************************************************
3790  *                   Manipulating UTF-8 Strings                                *
3791  *******************************************************************************/
3792
3793 /**
3794  * @ingroup StrBuf
3795  * @brief evaluate the length of an utf8 special character sequence
3796  * @param Char the character to examine
3797  * @returns width of utf8 chars in bytes; if the sequence is broken 0 is returned; 1 if its simply ASCII.
3798  */
3799 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE)
3800 {
3801         int n = 0;
3802         unsigned char test = (1<<7);
3803
3804         if ((*CharS & 0xC0) != 0xC0)
3805                 return 1;
3806
3807         while ((n < 8) &&
3808                ((test & ((unsigned char)*CharS)) != 0))
3809         {
3810                 test = test >> 1;
3811                 n ++;
3812         }
3813         if ((n > 6) || ((CharE - CharS) < n))
3814                 n = 0;
3815         return n;
3816 }
3817
3818 /**
3819  * @ingroup StrBuf
3820  * @brief detect whether this char starts an utf-8 encoded char
3821  * @param Char character to inspect
3822  * @returns yes or no
3823  */
3824 static inline int Ctdl_IsUtf8SequenceStart(const char Char)
3825 {
3826 /** 11??.???? indicates an UTF8 Sequence. */
3827         return ((Char & 0xC0) == 0xC0);
3828 }
3829
3830 /**
3831  * @ingroup StrBuf
3832  * @brief measure the number of glyphs in an UTF8 string...
3833  * @param Buf string to measure
3834  * @returns the number of glyphs in Buf
3835  */
3836 long StrBuf_Utf8StrLen(StrBuf *Buf)
3837 {
3838         int n = 0;
3839         int m = 0;
3840         char *aptr, *eptr;
3841
3842         if ((Buf == NULL) || (Buf->BufUsed == 0))
3843                 return 0;
3844         aptr = Buf->buf;
3845         eptr = Buf->buf + Buf->BufUsed;
3846         while ((aptr < eptr) && (*aptr != '\0')) {
3847                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
3848                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
3849                         while ((aptr < eptr) && (*aptr++ != '\0')&& (m-- > 0) );
3850                         n ++;
3851                 }
3852                 else {
3853                         n++;
3854                         aptr++;
3855                 }
3856         }
3857         return n;
3858 }
3859
3860 /**
3861  * @ingroup StrBuf
3862  * @brief cuts a string after maxlen glyphs
3863  * @param Buf string to cut to maxlen glyphs
3864  * @param maxlen how long may the string become?
3865  * @returns current length of the string
3866  */
3867 long StrBuf_Utf8StrCut(StrBuf *Buf, int maxlen)
3868 {
3869         char *aptr, *eptr;
3870         int n = 0, m = 0;
3871
3872         aptr = Buf->buf;
3873         eptr = Buf->buf + Buf->BufUsed;
3874         while ((aptr < eptr) && (*aptr != '\0')) {
3875                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
3876                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
3877                         while ((*aptr++ != '\0') && (m-- > 0));
3878                         n ++;
3879                 }
3880                 else {
3881                         n++;
3882                         aptr++;
3883                 }
3884                 if (n > maxlen) {
3885                         *aptr = '\0';
3886                         Buf->BufUsed = aptr - Buf->buf;
3887                         return Buf->BufUsed;
3888                 }
3889         }
3890         return Buf->BufUsed;
3891
3892 }
3893
3894
3895
3896
3897
3898 /*******************************************************************************
3899  *                               wrapping ZLib                                 *
3900  *******************************************************************************/
3901
3902 #ifdef HAVE_ZLIB
3903 #define DEF_MEM_LEVEL 8 /*< memlevel??? */
3904 #define OS_CODE 0x03    /*< unix */
3905
3906 /**
3907  * @ingroup StrBuf_DeEnCoder
3908  * @brief uses the same calling syntax as compress2(), but it
3909  *   creates a stream compatible with HTTP "Content-encoding: gzip"
3910  * @param dest compressed buffer
3911  * @param destLen length of the compresed data
3912  * @param source source to encode
3913  * @param sourceLen length of source to encode
3914  * @param level compression level
3915  */
3916 int ZEXPORT compress_gzip(Bytef * dest,
3917                           size_t * destLen,
3918                           const Bytef * source,
3919                           uLong sourceLen,
3920                           int level)
3921 {
3922         const int gz_magic[2] = { 0x1f, 0x8b }; /* gzip magic header */
3923
3924         /* write gzip header */
3925         snprintf((char *) dest, *destLen,
3926                  "%c%c%c%c%c%c%c%c%c%c",
3927                  gz_magic[0], gz_magic[1], Z_DEFLATED,
3928                  0 /*flags */ , 0, 0, 0, 0 /*time */ , 0 /* xflags */ ,
3929                  OS_CODE);
3930
3931         /* normal deflate */
3932         z_stream stream;
3933         int err;
3934         stream.next_in = (Bytef *) source;
3935         stream.avail_in = (uInt) sourceLen;
3936         stream.next_out = dest + 10L;   // after header
3937         stream.avail_out = (uInt) * destLen;
3938         if ((uLong) stream.avail_out != *destLen)
3939                 return Z_BUF_ERROR;
3940
3941         stream.zalloc = (alloc_func) 0;
3942         stream.zfree = (free_func) 0;
3943         stream.opaque = (voidpf) 0;
3944
3945         err = deflateInit2(&stream, level, Z_DEFLATED, -MAX_WBITS,
3946                            DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
3947         if (err != Z_OK)
3948                 return err;
3949
3950         err = deflate(&stream, Z_FINISH);
3951         if (err != Z_STREAM_END) {
3952                 deflateEnd(&stream);
3953                 return err == Z_OK ? Z_BUF_ERROR : err;
3954         }
3955         *destLen = stream.total_out + 10L;
3956
3957         /* write CRC and Length */
3958         uLong crc = crc32(0L, source, sourceLen);
3959         int n;
3960         for (n = 0; n < 4; ++n, ++*destLen) {
3961                 dest[*destLen] = (int) (crc & 0xff);
3962                 crc >>= 8;
3963         }
3964         uLong len = stream.total_in;
3965         for (n = 0; n < 4; ++n, ++*destLen) {
3966                 dest[*destLen] = (int) (len & 0xff);
3967                 len >>= 8;
3968         }
3969         err = deflateEnd(&stream);
3970         return err;
3971 }
3972 #endif
3973
3974
3975 /**
3976  * @ingroup StrBuf_DeEnCoder
3977  * @brief compress the buffer with gzip
3978  * Attention! If you feed this a Const String, you must maintain the uncompressed buffer yourself!
3979  * @param Buf buffer whose content is to be gzipped
3980  */
3981 int CompressBuffer(StrBuf *Buf)
3982 {
3983 #ifdef HAVE_ZLIB
3984         char *compressed_data = NULL;
3985         size_t compressed_len, bufsize;
3986         int i = 0;
3987
3988         bufsize = compressed_len = Buf->BufUsed +  (Buf->BufUsed / 100) + 100;
3989         compressed_data = malloc(compressed_len);
3990
3991         if (compressed_data == NULL)
3992                 return -1;
3993         /* Flush some space after the used payload so valgrind shuts up... */
3994         while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
3995                 Buf->buf[Buf->BufUsed + i++] = '\0';
3996         if (compress_gzip((Bytef *) compressed_data,
3997                           &compressed_len,
3998                           (Bytef *) Buf->buf,
3999                           (uLongf) Buf->BufUsed, Z_BEST_SPEED) == Z_OK) {
4000                 if (!Buf->ConstBuf)
4001                         free(Buf->buf);
4002                 Buf->buf = compressed_data;
4003                 Buf->BufUsed = compressed_len;
4004                 Buf->BufSize = bufsize;
4005                 /* Flush some space after the used payload so valgrind shuts up... */
4006                 i = 0;
4007                 while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
4008                         Buf->buf[Buf->BufUsed + i++] = '\0';
4009                 return 1;
4010         } else {
4011                 free(compressed_data);
4012         }
4013 #endif  /* HAVE_ZLIB */
4014         return 0;
4015 }
4016
4017 /*******************************************************************************
4018  *           File I/O; Callbacks to libevent                                   *
4019  *******************************************************************************/
4020
4021 long StrBuf_read_one_chunk_callback (int fd, short event, IOBuffer *FB)
4022 {
4023         long bufremain = 0;
4024         int n;
4025
4026         if ((FB == NULL) || (FB->Buf == NULL))
4027                 return -1;
4028
4029         /*
4030          * check whether the read pointer is somewhere in a range
4031          * where a cut left is inexpensive
4032          */
4033
4034         if (FB->ReadWritePointer != NULL)
4035         {
4036                 long already_read;
4037
4038                 already_read = FB->ReadWritePointer - FB->Buf->buf;
4039                 bufremain = FB->Buf->BufSize - FB->Buf->BufUsed - 1;
4040
4041                 if (already_read != 0) {
4042                         long unread;
4043
4044                         unread = FB->Buf->BufUsed - already_read;
4045
4046                         /* else nothing to compact... */
4047                         if (unread == 0) {
4048                                 FB->ReadWritePointer = FB->Buf->buf;
4049                                 bufremain = FB->Buf->BufSize;
4050                         }
4051                         else if ((unread < 64) ||
4052                                  (bufremain < already_read))
4053                         {
4054                                 /*
4055                                  * if its just a tiny bit remaining, or we run out of space...
4056                                  * lets tidy up.
4057                                  */
4058                                 FB->Buf->BufUsed = unread;
4059                                 if (unread < already_read)
4060                                         memcpy(FB->Buf->buf, FB->ReadWritePointer, unread);
4061                                 else
4062                                         memmove(FB->Buf->buf, FB->ReadWritePointer, unread);
4063                                 FB->ReadWritePointer = FB->Buf->buf;
4064                                 bufremain = FB->Buf->BufSize - unread - 1;
4065                         }
4066                         else if (bufremain < (FB->Buf->BufSize / 10))
4067                         {
4068                                 /* get a bigger buffer */
4069
4070                                 IncreaseBuf(FB->Buf, 0, FB->Buf->BufUsed + 1);
4071
4072                                 FB->ReadWritePointer = FB->Buf->buf + unread;
4073
4074                                 bufremain = FB->Buf->BufSize - unread - 1;
4075 /*TODO: special increase function that won't copy the already read! */
4076                         }
4077                 }
4078                 else if (bufremain < 10) {
4079                         IncreaseBuf(FB->Buf, 1, FB->Buf->BufUsed + 10);
4080
4081                         FB->ReadWritePointer = FB->Buf->buf;
4082
4083                         bufremain = FB->Buf->BufSize - FB->Buf->BufUsed - 1;
4084                 }
4085
4086         }
4087         else {
4088                 FB->ReadWritePointer = FB->Buf->buf;
4089                 bufremain = FB->Buf->BufSize - 1;
4090         }
4091
4092         n = read(fd, FB->Buf->buf + FB->Buf->BufUsed, bufremain);
4093
4094         if (n > 0) {
4095                 FB->Buf->BufUsed += n;
4096                 FB->Buf->buf[FB->Buf->BufUsed] = '\0';
4097         }
4098         return n;
4099 }
4100
4101 int StrBuf_write_one_chunk_callback(int fd, short event, IOBuffer *FB)
4102 {
4103         long WriteRemain;
4104         int n;
4105
4106         if ((FB == NULL) || (FB->Buf == NULL))
4107                 return -1;
4108
4109         if (FB->ReadWritePointer != NULL)
4110         {
4111                 WriteRemain = FB->Buf->BufUsed -
4112                         (FB->ReadWritePointer -
4113                          FB->Buf->buf);
4114         }
4115         else {
4116                 FB->ReadWritePointer = FB->Buf->buf;
4117                 WriteRemain = FB->Buf->BufUsed;
4118         }
4119
4120         n = write(fd, FB->ReadWritePointer, WriteRemain);
4121         if (n > 0) {
4122                 FB->ReadWritePointer += n;
4123
4124                 if (FB->ReadWritePointer ==
4125                     FB->Buf->buf + FB->Buf->BufUsed)
4126                 {
4127                         FlushStrBuf(FB->Buf);
4128                         FB->ReadWritePointer = NULL;
4129                         return 0;
4130                 }
4131         // check whether we've got something to write
4132         // get the maximum chunk plus the pointer we can send
4133         // write whats there
4134         // if not all was sent, remember the send pointer for the next time
4135                 return FB->ReadWritePointer - FB->Buf->buf + FB->Buf->BufUsed;
4136         }
4137         return n;
4138 }
4139
4140 /**
4141  * @ingroup StrBuf_IO
4142  * @brief extract a "next line" from Buf; Ptr to persist across several iterations
4143  * @param LineBuf your line will be copied here.
4144  * @param FB BLOB with lines of text...
4145  * @param Ptr moved arround to keep the next-line across several iterations
4146  *        has to be &NULL on start; will be &NotNULL on end of buffer
4147  * @returns size of copied buffer
4148  */
4149 eReadState StrBufChunkSipLine(StrBuf *LineBuf, IOBuffer *FB)
4150 {
4151         const char *aptr, *ptr, *eptr;
4152         char *optr, *xptr;
4153
4154         if ((FB == NULL) || (LineBuf == NULL) || (LineBuf->buf == NULL))
4155                 return eReadFail;
4156
4157
4158         if ((FB->Buf == NULL) || (FB->ReadWritePointer == StrBufNOTNULL)) {
4159                 FB->ReadWritePointer = StrBufNOTNULL;
4160                 return eReadFail;
4161         }
4162
4163         FlushStrBuf(LineBuf);
4164         if (FB->ReadWritePointer == NULL)
4165                 ptr = aptr = FB->Buf->buf;
4166         else
4167                 ptr = aptr = FB->ReadWritePointer;
4168
4169         optr = LineBuf->buf;
4170         eptr = FB->Buf->buf + FB->Buf->BufUsed;
4171         xptr = LineBuf->buf + LineBuf->BufSize - 1;
4172
4173         while ((ptr <= eptr) &&
4174                (*ptr != '\n') &&
4175                (*ptr != '\r') )
4176         {
4177                 *optr = *ptr;
4178                 optr++; ptr++;
4179                 if (optr == xptr) {
4180                         LineBuf->BufUsed = optr - LineBuf->buf;
4181                         IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
4182                         optr = LineBuf->buf + LineBuf->BufUsed;
4183                         xptr = LineBuf->buf + LineBuf->BufSize - 1;
4184                 }
4185         }
4186
4187         if (ptr >= eptr) {
4188                 if (optr > LineBuf->buf)
4189                         optr --;
4190                 if ((*(ptr - 1) != '\r') && (*(ptr - 1) != '\n')) {
4191                         LineBuf->BufUsed = optr - LineBuf->buf;
4192                         *optr = '\0';
4193                         if ((FB->ReadWritePointer != NULL) &&
4194                             (FB->ReadWritePointer != FB->Buf->buf))
4195                         {
4196                                 /* Ok, the client application read all the data
4197                                    it was interested in so far. Since there is more to read,
4198                                    we now shrink the buffer, and move the rest over.
4199                                 */
4200                                 StrBufCutLeft(FB->Buf,
4201                                               FB->ReadWritePointer - FB->Buf->buf);
4202                                 FB->ReadWritePointer = FB->Buf->buf;
4203                         }
4204                         return eMustReadMore;
4205                 }
4206         }
4207         LineBuf->BufUsed = optr - LineBuf->buf;
4208         *optr = '\0';
4209         if ((ptr <= eptr) && (*ptr == '\r'))
4210                 ptr ++;
4211         if ((ptr <= eptr) && (*ptr == '\n'))
4212                 ptr ++;
4213
4214         if (ptr < eptr) {
4215                 FB->ReadWritePointer = ptr;
4216         }
4217         else {
4218                 FlushStrBuf(FB->Buf);
4219                 FB->ReadWritePointer = NULL;
4220         }
4221
4222         return eReadSuccess;
4223 }
4224
4225 /**
4226  * @ingroup StrBuf_CHUNKED_IO
4227  * @brief check whether the chunk-buffer has more data waiting or not.
4228  * @param FB Chunk-Buffer to inspect
4229  */
4230 eReadState StrBufCheckBuffer(IOBuffer *FB)
4231 {
4232         if (FB == NULL)
4233                 return eReadFail;
4234         if (FB->Buf->BufUsed == 0)
4235                 return eReadSuccess;
4236         if (FB->ReadWritePointer == NULL)
4237                 return eBufferNotEmpty;
4238         if (FB->Buf->buf + FB->Buf->BufUsed > FB->ReadWritePointer)
4239                 return eBufferNotEmpty;
4240         return eReadSuccess;
4241 }
4242
4243 long IOBufferStrLength(IOBuffer *FB)
4244 {
4245         if ((FB == NULL) || (FB->Buf == NULL))
4246                 return 0;
4247         if (FB->ReadWritePointer == NULL)
4248                 return StrLength(FB->Buf);
4249
4250         return StrLength(FB->Buf) - (FB->ReadWritePointer - FB->Buf->buf);
4251 }
4252
4253 inline static void FDIOBufferFlush(FDIOBuffer *FDB)
4254 {
4255         memset(FDB, 0, sizeof(FDIOBuffer));
4256         FDB->OtherFD = -1;
4257         FDB->SplicePipe[0] = -1;
4258         FDB->SplicePipe[1] = -1;
4259 }
4260
4261 void FDIOBufferInit(FDIOBuffer *FDB, IOBuffer *IO, int FD, long TotalSendSize)
4262 {
4263         FDIOBufferFlush(FDB);
4264
4265         FDB->TotalSendSize = TotalSendSize;
4266         if (TotalSendSize > 0)
4267                 FDB->ChunkSize = TotalSendSize;
4268         else
4269         {
4270                 TotalSendSize = SIZ * 10;
4271                 FDB->ChunkSize = TotalSendSize;
4272         }
4273         FDB->IOB = IO;
4274
4275 #ifdef LINUX_SPLICE
4276         if (EnableSplice)
4277                 pipe(FDB->SplicePipe);
4278         else
4279 #endif
4280                 FDB->ChunkBuffer = NewStrBufPlain(NULL, TotalSendSize+ 1);
4281
4282         FDB->OtherFD = FD;
4283 }
4284
4285 void FDIOBufferDelete(FDIOBuffer *FDB)
4286 {
4287 #ifdef LINUX_SPLICE
4288         if (EnableSplice)
4289         {
4290                 if (FDB->SplicePipe[0] > 0)
4291                         close(FDB->SplicePipe[0]);
4292                 if (FDB->SplicePipe[1] > 0)
4293                         close(FDB->SplicePipe[1]);
4294         }
4295         else
4296 #endif
4297                 FreeStrBuf(&FDB->ChunkBuffer);
4298
4299         if (FDB->OtherFD > 0)
4300                 close(FDB->OtherFD);
4301         FDIOBufferFlush(FDB);
4302 }
4303
4304 int FileSendChunked(FDIOBuffer *FDB, const char **Err)
4305 {
4306         ssize_t sent, pipesize;
4307
4308         if (FDB->TotalSendSize > 0)
4309         {
4310 #ifdef LINUX_SPLICE
4311                 if (EnableSplice)
4312                 {
4313                         if (FDB->PipeSize == 0)
4314                         {
4315                                 pipesize = splice(FDB->OtherFD,
4316                                                   &FDB->TotalSentAlready,
4317                                                   FDB->SplicePipe[1],
4318                                                   NULL,
4319                                                   FDB->ChunkSendRemain,
4320                                                   SPLICE_F_MOVE);
4321
4322                                 if (pipesize == -1)
4323                                 {
4324                                         *Err = strerror(errno);
4325                                         return pipesize;
4326                                 }
4327                                 FDB->PipeSize = pipesize;
4328                         }
4329                         sent =  splice(FDB->SplicePipe[0],
4330                                        NULL,
4331                                        FDB->IOB->fd,
4332                                        NULL,
4333                                        FDB->PipeSize,
4334                                        SPLICE_F_MORE | SPLICE_F_MOVE | SPLICE_F_NONBLOCK);
4335                         if (sent == -1)
4336                         {
4337                                 *Err = strerror(errno);
4338                                 return sent;
4339                         }
4340                         FDB->PipeSize -= sent;
4341                         FDB->ChunkSendRemain -= sent;
4342                         return sent;
4343                 }
4344                 else
4345 #endif
4346                 {
4347                         char *pRead;
4348                         long nRead = 0;
4349
4350                         pRead = FDB->ChunkBuffer->buf;
4351                         while ((FDB->ChunkBuffer->BufUsed < FDB->TotalSendSize) && (nRead >= 0))
4352                         {
4353                                 nRead = read(FDB->OtherFD, pRead, FDB->TotalSendSize - FDB->ChunkBuffer->BufUsed);
4354                                 if (nRead > 0) {
4355                                         FDB->ChunkBuffer->BufUsed += nRead;
4356                                         FDB->ChunkBuffer->buf[FDB->ChunkBuffer->BufUsed] = '\0';
4357                                 }
4358                                 else if (nRead == 0) {}
4359                                 else return nRead;
4360                         }
4361
4362                         nRead = write(FDB->IOB->fd,
4363                                       FDB->ChunkBuffer->buf     + FDB->TotalSentAlready,
4364                                       FDB->ChunkBuffer->BufUsed - FDB->TotalSentAlready);
4365
4366                         if (nRead >= 0) {
4367                                 FDB->TotalSentAlready += nRead;
4368                                 FDB->ChunkSendRemain -= nRead;
4369                                 return FDB->ChunkSendRemain;
4370                         }
4371                         else {
4372                                 return nRead;
4373                         }
4374                 }
4375         }
4376         else
4377         {
4378 #ifdef LINUX_SPLICE
4379                 if (EnableSplice)
4380                 {
4381                         if (FDB->PipeSize == 0)
4382                         {
4383                                 pipesize = splice(FDB->OtherFD,
4384                                                   &FDB->TotalSentAlready,
4385                                                   FDB->SplicePipe[1],
4386                                                   NULL,
4387                                                   SIZ * 10,
4388                                                   SPLICE_F_MOVE);
4389
4390                                 if (pipesize == -1)
4391                                 {
4392                                         *Err = strerror(errno);
4393                                         return pipesize;
4394                                 }
4395                                 FDB->PipeSize = pipesize;
4396                                 if (pipesize == 0)
4397                                         return -1;
4398                         }
4399                         sent =  splice(FDB->SplicePipe[0],
4400                                        NULL,
4401                                        FDB->IOB->fd,
4402                                        NULL,
4403                                        FDB->PipeSize,
4404                                        SPLICE_F_MORE | SPLICE_F_MOVE | SPLICE_F_NONBLOCK);
4405                         if (sent == -1)
4406                         {
4407                                 *Err = strerror(errno);
4408                                 return sent;
4409                         }
4410                         FDB->PipeSize -= sent;
4411                         FDB->ChunkSendRemain -= sent;
4412                         return sent;
4413                 }
4414                 else
4415 #endif
4416                 {
4417                         char *pRead;
4418                         long nRead = 0;
4419
4420                         pRead = FDB->ChunkBuffer->buf;
4421                         while ((FDB->ChunkSendRemain == 0) &&
4422                                (FDB->ChunkBuffer->BufUsed < FDB->ChunkBuffer->BufSize) &&
4423                                (nRead >= 0))
4424                         {
4425                                 FDB->TotalSentAlready = 0;
4426                                 nRead = read(FDB->OtherFD, pRead, FDB->ChunkBuffer->BufSize - FDB->ChunkBuffer->BufUsed);
4427                                 if (nRead > 0) {
4428                                         FDB->ChunkBuffer->BufUsed += nRead;
4429                                         FDB->ChunkBuffer->buf[FDB->ChunkBuffer->BufUsed] = '\0';
4430                                         FDB->ChunkSendRemain += nRead;
4431                                 }
4432                                 else if (nRead == 0)
4433                                 {
4434                                         return -1;
4435                                 }
4436                                 else
4437                                 {
4438                                         *Err = strerror(errno);
4439                                         return nRead;
4440                                 }
4441                         }
4442
4443                         nRead = write(FDB->IOB->fd,
4444                                       FDB->ChunkBuffer->buf     + FDB->TotalSentAlready,
4445                                       FDB->ChunkBuffer->BufUsed - FDB->TotalSentAlready);
4446
4447                         if (nRead >= 0) {
4448                                 FDB->TotalSentAlready += nRead;
4449                                 FDB->ChunkSendRemain -= nRead;
4450                                 if (FDB->ChunkSendRemain == 0)
4451                                 {
4452                                         FDB->ChunkBuffer->BufUsed = 0;
4453                                         FDB->TotalSentAlready = 0;
4454                                 }
4455                                 return FDB->ChunkSendRemain;
4456                         }
4457                         else {
4458                                 return nRead;
4459                         }
4460                 }
4461         }
4462 }
4463
4464 int FileRecvChunked(FDIOBuffer *FDB, const char **Err)
4465 {
4466         ssize_t sent, pipesize;
4467
4468 #ifdef LINUX_SPLICE
4469         if (EnableSplice)
4470         {
4471                 if (FDB->PipeSize == 0)
4472                 {
4473                         pipesize = splice(FDB->IOB->fd,
4474                                           NULL,
4475                                           FDB->SplicePipe[1],
4476                                           NULL,
4477                                           FDB->ChunkSendRemain,
4478                                           SPLICE_F_MORE | SPLICE_F_MOVE|SPLICE_F_NONBLOCK);
4479
4480                         if (pipesize == -1)
4481                         {
4482                                 *Err = strerror(errno);
4483                                 return pipesize;
4484                         }
4485                         FDB->PipeSize = pipesize;
4486                 }
4487
4488                 sent = splice(FDB->SplicePipe[0],
4489                               NULL,
4490                               FDB->OtherFD,
4491                               &FDB->TotalSentAlready,
4492                               FDB->PipeSize,
4493                               SPLICE_F_MORE | SPLICE_F_MOVE);
4494
4495                 if (sent == -1)
4496                 {
4497                         *Err = strerror(errno);
4498                         return sent;
4499                 }
4500                 FDB->PipeSize -= sent;
4501                 FDB->ChunkSendRemain -= sent;
4502                 return sent;
4503         }
4504         else
4505 #endif
4506         {
4507                 sent = read(FDB->IOB->fd, FDB->ChunkBuffer->buf, FDB->ChunkSendRemain);
4508                 if (sent > 0) {
4509                         int nWritten = 0;
4510                         int rc;
4511
4512                         FDB->ChunkBuffer->BufUsed = sent;
4513
4514                         while (nWritten < FDB->ChunkBuffer->BufUsed) {
4515                                 rc =  write(FDB->OtherFD, FDB->ChunkBuffer->buf + nWritten, FDB->ChunkBuffer->BufUsed - nWritten);
4516                                 if (rc < 0) {
4517                                         *Err = strerror(errno);
4518                                         return rc;
4519                                 }
4520                                 nWritten += rc;
4521
4522                         }
4523                         FDB->ChunkBuffer->BufUsed = 0;
4524                         FDB->TotalSentAlready += sent;
4525                         FDB->ChunkSendRemain -= sent;
4526                         return FDB->ChunkSendRemain;
4527                 }
4528                 else if (sent < 0) {
4529                         *Err = strerror(errno);
4530                         return sent;
4531                 }
4532                 return 0;
4533         }
4534 }
4535
4536 int FileMoveChunked(FDIOBuffer *FDB, const char **Err)
4537 {
4538         ssize_t sent, pipesize;
4539
4540 #ifdef LINUX_SPLICE
4541         if (EnableSplice)
4542         {
4543                 if (FDB->PipeSize == 0)
4544                 {
4545                         pipesize = splice(FDB->IOB->fd,
4546                                           &FDB->TotalReadAlready,
4547                                           FDB->SplicePipe[1],
4548                                           NULL,
4549                                           FDB->ChunkSendRemain,
4550                                           SPLICE_F_MORE | SPLICE_F_MOVE|SPLICE_F_NONBLOCK);
4551
4552                         if (pipesize == -1)
4553                         {
4554                                 *Err = strerror(errno);
4555                                 return pipesize;
4556                         }
4557                         FDB->PipeSize = pipesize;
4558                 }
4559
4560                 sent = splice(FDB->SplicePipe[0],
4561                               NULL,
4562                               FDB->OtherFD,
4563                               &FDB->TotalSentAlready,
4564                               FDB->PipeSize,
4565                               SPLICE_F_MORE | SPLICE_F_MOVE);
4566
4567                 if (sent == -1)
4568                 {
4569                         *Err = strerror(errno);
4570                         return sent;
4571                 }
4572                 FDB->PipeSize -= sent;
4573                 FDB->ChunkSendRemain -= sent;
4574                 return sent;
4575         }
4576         else
4577 #endif
4578         {
4579                 sent = read(FDB->IOB->fd, FDB->ChunkBuffer->buf, FDB->ChunkSendRemain);
4580                 if (sent > 0) {
4581                         int nWritten = 0;
4582                         int rc;
4583
4584                         FDB->ChunkBuffer->BufUsed = sent;
4585
4586                         while (nWritten < FDB->ChunkBuffer->BufUsed) {
4587                                 rc =  write(FDB->OtherFD, FDB->ChunkBuffer->buf + nWritten, FDB->ChunkBuffer->BufUsed - nWritten);
4588                                 if (rc < 0) {
4589                                         *Err = strerror(errno);
4590                                         return rc;
4591                                 }
4592                                 nWritten += rc;
4593
4594                         }
4595                         FDB->ChunkBuffer->BufUsed = 0;
4596                         FDB->TotalSentAlready += sent;
4597                         FDB->ChunkSendRemain -= sent;
4598                         return FDB->ChunkSendRemain;
4599                 }
4600                 else if (sent < 0) {
4601                         *Err = strerror(errno);
4602                         return sent;
4603                 }
4604                 return 0;
4605         }
4606 }
4607
4608 eReadState WriteIOBAlreadyRead(FDIOBuffer *FDB, const char **Error)
4609 {
4610         int IsNonBlock;
4611         int fdflags;
4612         long rlen;
4613         long should_write;
4614         int nSuccessLess = 0;
4615         struct timeval tv;
4616         fd_set rfds;
4617
4618         fdflags = fcntl(FDB->OtherFD, F_GETFL);
4619         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
4620
4621         while ((FDB->IOB->ReadWritePointer - FDB->IOB->Buf->buf < FDB->IOB->Buf->BufUsed) &&
4622                (FDB->ChunkSendRemain > 0))
4623         {
4624                 if (IsNonBlock){
4625                         tv.tv_sec = 1; /* selectresolution; */
4626                         tv.tv_usec = 0;
4627
4628                         FD_ZERO(&rfds);
4629                         FD_SET(FDB->OtherFD, &rfds);
4630                         if (select(FDB->OtherFD + 1, NULL, &rfds, NULL, &tv) == -1) {
4631                                 *Error = strerror(errno);
4632                                 return eReadFail;
4633                         }
4634                 }
4635                 if (IsNonBlock && !  FD_ISSET(FDB->OtherFD, &rfds)) {
4636                         nSuccessLess ++;
4637                         continue;
4638                 }
4639
4640                 should_write = FDB->IOB->Buf->BufUsed -
4641                         (FDB->IOB->ReadWritePointer - FDB->IOB->Buf->buf);
4642                 if (should_write > FDB->ChunkSendRemain)
4643                         should_write = FDB->ChunkSendRemain;
4644
4645                 rlen = write(FDB->OtherFD,
4646                              FDB->IOB->ReadWritePointer,
4647                              should_write);
4648                 if (rlen < 1) {
4649                         *Error = strerror(errno);
4650
4651                         return eReadFail;
4652                 }
4653                 FDB->TotalSentAlready += rlen;
4654                 FDB->IOB->ReadWritePointer += rlen;
4655                 FDB->ChunkSendRemain -= rlen;
4656         }
4657         if (FDB->IOB->ReadWritePointer >= FDB->IOB->Buf->buf + FDB->IOB->Buf->BufUsed)
4658         {
4659                 FlushStrBuf(FDB->IOB->Buf);
4660                 FDB->IOB->ReadWritePointer = NULL;
4661         }
4662
4663         if (FDB->ChunkSendRemain == 0)
4664                 return eReadSuccess;
4665         else
4666                 return eMustReadMore;
4667 }
4668
4669 /*******************************************************************************
4670  *           File I/O; Prefer buffered read since its faster!                  *
4671  *******************************************************************************/
4672
4673 /**
4674  * @ingroup StrBuf_IO
4675  * @brief Read a line from socket
4676  * flushes and closes the FD on error
4677  * @param buf the buffer to get the input to
4678  * @param fd pointer to the filedescriptor to read
4679  * @param append Append to an existing string or replace?
4680  * @param Error strerror() on error
4681  * @returns numbers of chars read
4682  */
4683 int StrBufTCP_read_line(StrBuf *buf, int *fd, int append, const char **Error)
4684 {
4685         int len, rlen, slen;
4686
4687         if ((buf == NULL) || (buf->buf == NULL)) {
4688                 *Error = strerror(EINVAL);
4689                 return -1;
4690         }
4691
4692         if (!append)
4693                 FlushStrBuf(buf);
4694
4695         slen = len = buf->BufUsed;
4696         while (1) {
4697                 rlen = read(*fd, &buf->buf[len], 1);
4698                 if (rlen < 1) {
4699                         *Error = strerror(errno);
4700
4701                         close(*fd);
4702                         *fd = -1;
4703
4704                         return -1;
4705                 }
4706                 if (buf->buf[len] == '\n')
4707                         break;
4708                 if (buf->buf[len] != '\r')
4709                         len ++;
4710                 if (len + 2 >= buf->BufSize) {
4711                         buf->BufUsed = len;
4712                         buf->buf[len+1] = '\0';
4713                         IncreaseBuf(buf, 1, -1);
4714                 }
4715         }
4716         buf->BufUsed = len;
4717         buf->buf[len] = '\0';
4718         return len - slen;
4719 }
4720
4721 /**
4722  * @ingroup StrBuf_BufferedIO
4723  * @brief Read a line from socket
4724  * flushes and closes the FD on error
4725  * @param Line the line to read from the fd / I/O Buffer
4726  * @param buf the buffer to get the input to
4727  * @param fd pointer to the filedescriptor to read
4728  * @param timeout number of successless selects until we bail out
4729  * @param selectresolution how long to wait on each select
4730  * @param Error strerror() on error
4731  * @returns numbers of chars read
4732  */
4733 int StrBufTCP_read_buffered_line(StrBuf *Line,
4734                                  StrBuf *buf,
4735                                  int *fd,
4736                                  int timeout,
4737                                  int selectresolution,
4738                                  const char **Error)
4739 {
4740         int len, rlen;
4741         int nSuccessLess = 0;
4742         fd_set rfds;
4743         char *pch = NULL;
4744         int fdflags;
4745         int IsNonBlock;
4746         struct timeval tv;
4747
4748         if (buf->BufUsed > 0) {
4749                 pch = strchr(buf->buf, '\n');
4750                 if (pch != NULL) {
4751                         rlen = 0;
4752                         len = pch - buf->buf;
4753                         if (len > 0 && (*(pch - 1) == '\r') )
4754                                 rlen ++;
4755                         StrBufSub(Line, buf, 0, len - rlen);
4756                         StrBufCutLeft(buf, len + 1);
4757                         return len - rlen;
4758                 }
4759         }
4760
4761         if (buf->BufSize - buf->BufUsed < 10)
4762                 IncreaseBuf(buf, 1, -1);
4763
4764         fdflags = fcntl(*fd, F_GETFL);
4765         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
4766
4767         while ((nSuccessLess < timeout) && (pch == NULL)) {
4768                 if (IsNonBlock){
4769                         tv.tv_sec = selectresolution;
4770                         tv.tv_usec = 0;
4771
4772                         FD_ZERO(&rfds);
4773                         FD_SET(*fd, &rfds);
4774                         if (select(*fd + 1, NULL, &rfds, NULL, &tv) == -1) {
4775                                 *Error = strerror(errno);
4776                                 close (*fd);
4777                                 *fd = -1;
4778                                 return -1;
4779                         }
4780                 }
4781                 if (IsNonBlock && !  FD_ISSET(*fd, &rfds)) {
4782                         nSuccessLess ++;
4783                         continue;
4784                 }
4785                 rlen = read(*fd,
4786                             &buf->buf[buf->BufUsed],
4787                             buf->BufSize - buf->BufUsed - 1);
4788                 if (rlen < 1) {
4789                         *Error = strerror(errno);
4790                         close(*fd);
4791                         *fd = -1;
4792                         return -1;
4793                 }
4794                 else if (rlen > 0) {
4795                         nSuccessLess = 0;
4796                         buf->BufUsed += rlen;
4797                         buf->buf[buf->BufUsed] = '\0';
4798                         pch = strchr(buf->buf, '\n');
4799                         if ((pch == NULL) &&
4800                             (buf->BufUsed + 10 > buf->BufSize) &&
4801                             (IncreaseBuf(buf, 1, -1) == -1))
4802                                 return -1;
4803                         continue;
4804                 }
4805
4806         }
4807         if (pch != NULL) {
4808                 rlen = 0;
4809                 len = pch - buf->buf;
4810                 if (len > 0 && (*(pch - 1) == '\r') )
4811                         rlen ++;
4812                 StrBufSub(Line, buf, 0, len - rlen);
4813                 StrBufCutLeft(buf, len + 1);
4814                 return len - rlen;
4815         }
4816         return -1;
4817
4818 }
4819
4820 static const char *ErrRBLF_PreConditionFailed="StrBufTCP_read_buffered_line_fast: Wrong arguments or invalid Filedescriptor";
4821 static const char *ErrRBLF_SelectFailed="StrBufTCP_read_buffered_line_fast: Select failed without reason";
4822 static const char *ErrRBLF_NotEnoughSentFromServer="StrBufTCP_read_buffered_line_fast: No complete line was sent from peer";
4823 /**
4824  * @ingroup StrBuf_BufferedIO
4825  * @brief Read a line from socket
4826  * flushes and closes the FD on error
4827  * @param Line where to append our Line read from the fd / I/O Buffer;
4828  * @param IOBuf the buffer to get the input to; lifetime pair to FD
4829  * @param Pos pointer to the current read position, should be NULL initialized on opening the FD it belongs to.!
4830  * @param fd pointer to the filedescriptor to read
4831  * @param timeout number of successless selects until we bail out
4832  * @param selectresolution how long to wait on each select
4833  * @param Error strerror() on error
4834  * @returns numbers of chars read or -1 in case of error. "\n" will become 0
4835  */
4836 int StrBufTCP_read_buffered_line_fast(StrBuf *Line,
4837                                       StrBuf *IOBuf,
4838                                       const char **Pos,
4839                                       int *fd,
4840                                       int timeout,
4841                                       int selectresolution,
4842                                       const char **Error)
4843 {
4844         const char *pche = NULL;
4845         const char *pos = NULL;
4846         const char *pLF;
4847         int len, rlen, retlen;
4848         int nSuccessLess = 0;
4849         fd_set rfds;
4850         const char *pch = NULL;
4851         int fdflags;
4852         int IsNonBlock;
4853         struct timeval tv;
4854
4855         retlen = 0;
4856         if ((Line == NULL) ||
4857             (Pos == NULL) ||
4858             (IOBuf == NULL) ||
4859             (*fd == -1))
4860         {
4861                 if (Pos != NULL)
4862                         *Pos = NULL;
4863                 *Error = ErrRBLF_PreConditionFailed;
4864                 return -1;
4865         }
4866
4867         pos = *Pos;
4868         if ((IOBuf->BufUsed > 0) &&
4869             (pos != NULL) &&
4870             (pos < IOBuf->buf + IOBuf->BufUsed))
4871         {
4872                 char *pcht;
4873
4874                 pche = IOBuf->buf + IOBuf->BufUsed;
4875                 pch = pos;
4876                 pcht = Line->buf;
4877
4878                 while ((pch < pche) && (*pch != '\n'))
4879                 {
4880                         if (Line->BufUsed + 10 > Line->BufSize)
4881                         {
4882                                 long apos;
4883                                 apos = pcht - Line->buf;
4884                                 *pcht = '\0';
4885                                 IncreaseBuf(Line, 1, -1);
4886                                 pcht = Line->buf + apos;
4887                         }
4888                         *pcht++ = *pch++;
4889                         Line->BufUsed++;
4890                         retlen++;
4891                 }
4892
4893                 len = pch - pos;
4894                 if (len > 0 && (*(pch - 1) == '\r') )
4895                 {
4896                         retlen--;
4897                         len --;
4898                         pcht --;
4899                         Line->BufUsed --;
4900                 }
4901                 *pcht = '\0';
4902
4903                 if ((pch >= pche) || (*pch == '\0'))
4904                 {
4905                         FlushStrBuf(IOBuf);
4906                         *Pos = NULL;
4907                         pch = NULL;
4908                         pos = 0;
4909                 }
4910
4911                 if ((pch != NULL) &&
4912                     (pch <= pche))
4913                 {
4914                         if (pch + 1 >= pche) {
4915                                 *Pos = NULL;
4916                                 FlushStrBuf(IOBuf);
4917                         }
4918                         else
4919                                 *Pos = pch + 1;
4920
4921                         return retlen;
4922                 }
4923                 else
4924                         FlushStrBuf(IOBuf);
4925         }
4926
4927         /* If we come here, Pos is Unset since we read everything into Line, and now go for more. */
4928
4929         if (IOBuf->BufSize - IOBuf->BufUsed < 10)
4930                 IncreaseBuf(IOBuf, 1, -1);
4931
4932         fdflags = fcntl(*fd, F_GETFL);
4933         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
4934
4935         pLF = NULL;
4936         while ((nSuccessLess < timeout) &&
4937                (pLF == NULL) &&
4938                (*fd != -1)) {
4939                 if (IsNonBlock)
4940                 {
4941                         tv.tv_sec = 1;
4942                         tv.tv_usec = 0;
4943
4944                         FD_ZERO(&rfds);
4945                         FD_SET(*fd, &rfds);
4946                         if (select((*fd) + 1, &rfds, NULL, NULL, &tv) == -1) {
4947                                 *Error = strerror(errno);
4948                                 close (*fd);
4949                                 *fd = -1;
4950                                 if (*Error == NULL)
4951                                         *Error = ErrRBLF_SelectFailed;
4952                                 return -1;
4953                         }
4954                         if (! FD_ISSET(*fd, &rfds) != 0) {
4955                                 nSuccessLess ++;
4956                                 continue;
4957                         }
4958                 }
4959                 rlen = read(*fd,
4960                             &IOBuf->buf[IOBuf->BufUsed],
4961                             IOBuf->BufSize - IOBuf->BufUsed - 1);
4962                 if (rlen < 1) {
4963                         *Error = strerror(errno);
4964                         close(*fd);
4965                         *fd = -1;
4966                         return -1;
4967                 }
4968                 else if (rlen > 0) {
4969                         nSuccessLess = 0;
4970                         pLF = IOBuf->buf + IOBuf->BufUsed;
4971                         IOBuf->BufUsed += rlen;
4972                         IOBuf->buf[IOBuf->BufUsed] = '\0';
4973
4974                         pche = IOBuf->buf + IOBuf->BufUsed;
4975
4976                         while ((pLF < pche) && (*pLF != '\n'))
4977                                 pLF ++;
4978                         if ((pLF >= pche) || (*pLF == '\0'))
4979                                 pLF = NULL;
4980
4981                         if (IOBuf->BufUsed + 10 > IOBuf->BufSize)
4982                         {
4983                                 long apos = 0;
4984
4985                                 if (pLF != NULL) apos = pLF - IOBuf->buf;
4986                                 IncreaseBuf(IOBuf, 1, -1);
4987                                 if (pLF != NULL) pLF = IOBuf->buf + apos;
4988                         }
4989
4990                         continue;
4991                 }
4992                 else
4993                 {
4994                         nSuccessLess++;
4995                 }
4996         }
4997         *Pos = NULL;
4998         if (pLF != NULL) {
4999                 pos = IOBuf->buf;
5000                 len = pLF - pos;
5001                 if (len > 0 && (*(pLF - 1) == '\r') )
5002                         len --;
5003                 StrBufAppendBufPlain(Line, ChrPtr(IOBuf), len, 0);
5004                 if (pLF + 1 >= IOBuf->buf + IOBuf->BufUsed)
5005                 {
5006                         FlushStrBuf(IOBuf);
5007                 }
5008                 else
5009                         *Pos = pLF + 1;
5010                 return retlen + len;
5011         }
5012         *Error = ErrRBLF_NotEnoughSentFromServer;
5013         return -1;
5014
5015 }
5016
5017 static const char *ErrRBLF_BLOBPreConditionFailed="StrBufReadBLOB: Wrong arguments or invalid Filedescriptor";
5018 /**
5019  * @ingroup StrBuf_IO
5020  * @brief Input binary data from socket
5021  * flushes and closes the FD on error
5022  * @param Buf the buffer to get the input to
5023  * @param fd pointer to the filedescriptor to read
5024  * @param append Append to an existing string or replace?
5025  * @param nBytes the maximal number of bytes to read
5026  * @param Error strerror() on error
5027  * @returns numbers of chars read
5028  */
5029 int StrBufReadBLOB(StrBuf *Buf, int *fd, int append, long nBytes, const char **Error)
5030 {
5031         int fdflags;
5032         int rlen;
5033         int nSuccessLess;
5034         int nRead = 0;
5035         char *ptr;
5036         int IsNonBlock;
5037         struct timeval tv;
5038         fd_set rfds;
5039
5040         if ((Buf == NULL) || (Buf->buf == NULL) || (*fd == -1))
5041         {
5042                 *Error = ErrRBLF_BLOBPreConditionFailed;
5043                 return -1;
5044         }
5045         if (!append)
5046                 FlushStrBuf(Buf);
5047         if (Buf->BufUsed + nBytes >= Buf->BufSize)
5048                 IncreaseBuf(Buf, 1, Buf->BufUsed + nBytes);
5049
5050         ptr = Buf->buf + Buf->BufUsed;
5051
5052         fdflags = fcntl(*fd, F_GETFL);
5053         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
5054         nSuccessLess = 0;
5055         while ((nRead < nBytes) &&
5056                (*fd != -1))
5057         {
5058                 if (IsNonBlock)
5059                 {
5060                         tv.tv_sec = 1;
5061                         tv.tv_usec = 0;
5062
5063                         FD_ZERO(&rfds);
5064                         FD_SET(*fd, &rfds);
5065                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
5066                                 *Error = strerror(errno);
5067                                 close (*fd);
5068                                 *fd = -1;
5069                                 if (*Error == NULL)
5070                                         *Error = ErrRBLF_SelectFailed;
5071                                 return -1;
5072                         }
5073                         if (! FD_ISSET(*fd, &rfds) != 0) {
5074                                 nSuccessLess ++;
5075                                 continue;
5076                         }
5077                 }
5078
5079                 if ((rlen = read(*fd,
5080                                  ptr,
5081                                  nBytes - nRead)) == -1) {
5082                         close(*fd);
5083                         *fd = -1;
5084                         *Error = strerror(errno);
5085                         return rlen;
5086                 }
5087                 nRead += rlen;
5088                 ptr += rlen;
5089                 Buf->BufUsed += rlen;
5090         }
5091         Buf->buf[Buf->BufUsed] = '\0';
5092         return nRead;
5093 }
5094
5095 const char *ErrRBB_BLOBFPreConditionFailed = "StrBufReadBLOBBuffered: to many selects; aborting.";
5096 const char *ErrRBB_too_many_selects        = "StrBufReadBLOBBuffered: to many selects; aborting.";
5097 /**
5098  * @ingroup StrBuf_BufferedIO
5099  * @brief Input binary data from socket
5100  * flushes and closes the FD on error
5101  * @param Blob put binary thing here
5102  * @param IOBuf the buffer to get the input to
5103  * @param Pos offset inside of IOBuf
5104  * @param fd pointer to the filedescriptor to read
5105  * @param append Append to an existing string or replace?
5106  * @param nBytes the maximal number of bytes to read
5107  * @param check whether we should search for '000\n' terminators in case of timeouts
5108  * @param Error strerror() on error
5109  * @returns numbers of chars read
5110  */
5111 int StrBufReadBLOBBuffered(StrBuf *Blob,
5112                            StrBuf *IOBuf,
5113                            const char **Pos,
5114                            int *fd,
5115                            int append,
5116                            long nBytes,
5117                            int check,
5118                            const char **Error)
5119 {
5120         const char *pos;
5121         int fdflags;
5122         int rlen = 0;
5123         int nRead = 0;
5124         int nAlreadyRead = 0;
5125         int IsNonBlock;
5126         char *ptr;
5127         fd_set rfds;
5128         struct timeval tv;
5129         int nSuccessLess = 0;
5130         int MaxTries;
5131
5132         if ((Blob == NULL)  ||
5133             (*fd == -1)     ||
5134             (IOBuf == NULL) ||
5135             (Pos == NULL))
5136         {
5137                 if (Pos != NULL)
5138                         *Pos = NULL;
5139                 *Error = ErrRBB_BLOBFPreConditionFailed;
5140                 return -1;
5141         }
5142
5143         if (!append)
5144                 FlushStrBuf(Blob);
5145         if (Blob->BufUsed + nBytes >= Blob->BufSize)
5146                 IncreaseBuf(Blob, append, Blob->BufUsed + nBytes);
5147
5148         pos = *Pos;
5149
5150         if (pos != NULL)
5151                 rlen = pos - IOBuf->buf;
5152         rlen = IOBuf->BufUsed - rlen;
5153
5154
5155         if ((IOBuf->BufUsed > 0) &&
5156             (pos != NULL) &&
5157             (pos < IOBuf->buf + IOBuf->BufUsed))
5158         {
5159                 if (rlen < nBytes) {
5160                         memcpy(Blob->buf + Blob->BufUsed, pos, rlen);
5161                         Blob->BufUsed += rlen;
5162                         Blob->buf[Blob->BufUsed] = '\0';
5163                         nAlreadyRead = nRead = rlen;
5164                         *Pos = NULL;
5165                 }
5166                 if (rlen >= nBytes) {
5167                         memcpy(Blob->buf + Blob->BufUsed, pos, nBytes);
5168                         Blob->BufUsed += nBytes;
5169                         Blob->buf[Blob->BufUsed] = '\0';
5170                         if (rlen == nBytes) {
5171                                 *Pos = NULL;
5172                                 FlushStrBuf(IOBuf);
5173                         }
5174                         else
5175                                 *Pos += nBytes;
5176                         return nBytes;
5177                 }
5178         }
5179
5180         FlushStrBuf(IOBuf);
5181         *Pos = NULL;
5182         if (IOBuf->BufSize < nBytes - nRead)
5183                 IncreaseBuf(IOBuf, 0, nBytes - nRead);
5184         ptr = IOBuf->buf;
5185
5186         fdflags = fcntl(*fd, F_GETFL);
5187         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
5188         if (IsNonBlock)
5189                 MaxTries =   1000;
5190         else
5191                 MaxTries = 100000;
5192
5193         nBytes -= nRead;
5194         nRead = 0;
5195         while ((nSuccessLess < MaxTries) &&
5196                (nRead < nBytes) &&
5197                (*fd != -1)) {
5198                 if (IsNonBlock)
5199                 {
5200                         tv.tv_sec = 1;
5201                         tv.tv_usec = 0;
5202
5203                         FD_ZERO(&rfds);
5204                         FD_SET(*fd, &rfds);
5205                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
5206                                 *Error = strerror(errno);
5207                                 close (*fd);
5208                                 *fd = -1;
5209                                 if (*Error == NULL)
5210                                         *Error = ErrRBLF_SelectFailed;
5211                                 return -1;
5212                         }
5213                         if (! FD_ISSET(*fd, &rfds) != 0) {
5214                                 nSuccessLess ++;
5215                                 continue;
5216                         }
5217                 }
5218                 rlen = read(*fd,
5219                             ptr,
5220                             IOBuf->BufSize - (ptr - IOBuf->buf));
5221                 if (rlen == -1) {
5222                         close(*fd);
5223                         *fd = -1;
5224                         *Error = strerror(errno);
5225                         return rlen;
5226                 }
5227                 else if (rlen == 0){
5228                         if ((check == NNN_TERM) &&
5229                             (nRead > 5) &&
5230                             (strncmp(IOBuf->buf + IOBuf->BufUsed - 5, "\n000\n", 5) == 0))
5231                         {
5232                                 StrBufPlain(Blob, HKEY("\n000\n"));
5233                                 StrBufCutRight(Blob, 5);
5234                                 return Blob->BufUsed;
5235                         }
5236                         else if (!IsNonBlock)
5237                                 nSuccessLess ++;
5238                         else if (nSuccessLess > MaxTries) {
5239                                 FlushStrBuf(IOBuf);
5240                                 *Error = ErrRBB_too_many_selects;
5241                                 return -1;
5242                         }
5243                 }
5244                 else if (rlen > 0) {
5245                         nSuccessLess = 0;
5246                         nRead += rlen;
5247                         ptr += rlen;
5248                         IOBuf->BufUsed += rlen;
5249                 }
5250         }
5251         if (nSuccessLess >= MaxTries) {
5252                 FlushStrBuf(IOBuf);
5253                 *Error = ErrRBB_too_many_selects;
5254                 return -1;
5255         }
5256
5257         if (nRead > nBytes) {
5258                 *Pos = IOBuf->buf + nBytes;
5259         }
5260         Blob->buf[Blob->BufUsed] = '\0';
5261         StrBufAppendBufPlain(Blob, IOBuf->buf, nBytes, 0);
5262         if (*Pos == NULL) {
5263                 FlushStrBuf(IOBuf);
5264         }
5265         return nRead + nAlreadyRead;
5266 }
5267
5268 /**
5269  * @ingroup StrBuf_IO
5270  * @brief extract a "next line" from Buf; Ptr to persist across several iterations
5271  * @param LineBuf your line will be copied here.
5272  * @param Buf BLOB with lines of text...
5273  * @param Ptr moved arround to keep the next-line across several iterations
5274  *        has to be &NULL on start; will be &NotNULL on end of buffer
5275  * @returns size of remaining buffer
5276  */
5277 int StrBufSipLine(StrBuf *LineBuf, const StrBuf *Buf, const char **Ptr)
5278 {
5279         const char *aptr, *ptr, *eptr;
5280         char *optr, *xptr;
5281
5282         if ((Buf == NULL) ||
5283             (*Ptr == StrBufNOTNULL) ||
5284             (LineBuf == NULL)||
5285             (LineBuf->buf == NULL))
5286         {
5287                 *Ptr = StrBufNOTNULL;
5288                 return 0;
5289         }
5290
5291         FlushStrBuf(LineBuf);
5292         if (*Ptr==NULL)
5293                 ptr = aptr = Buf->buf;
5294         else
5295                 ptr = aptr = *Ptr;
5296
5297         optr = LineBuf->buf;
5298         eptr = Buf->buf + Buf->BufUsed;
5299         xptr = LineBuf->buf + LineBuf->BufSize - 1;
5300
5301         while ((ptr <= eptr) &&
5302                (*ptr != '\n') &&
5303                (*ptr != '\r') )
5304         {
5305                 *optr = *ptr;
5306                 optr++; ptr++;
5307                 if (optr == xptr) {
5308                         LineBuf->BufUsed = optr - LineBuf->buf;
5309                         IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
5310                         optr = LineBuf->buf + LineBuf->BufUsed;
5311                         xptr = LineBuf->buf + LineBuf->BufSize - 1;
5312                 }
5313         }
5314
5315         if ((ptr >= eptr) && (optr > LineBuf->buf))
5316                 optr --;
5317         LineBuf->BufUsed = optr - LineBuf->buf;
5318         *optr = '\0';
5319         if ((ptr <= eptr) && (*ptr == '\r'))
5320                 ptr ++;
5321         if ((ptr <= eptr) && (*ptr == '\n'))
5322                 ptr ++;
5323
5324         if (ptr < eptr) {
5325                 *Ptr = ptr;
5326         }
5327         else {
5328                 *Ptr = StrBufNOTNULL;
5329         }
5330
5331         return Buf->BufUsed - (ptr - Buf->buf);
5332 }
5333
5334
5335 /**
5336  * @ingroup StrBuf_IO
5337  * @brief removes double slashes from pathnames
5338  * @param Dir directory string to filter
5339  * @param RemoveTrailingSlash allows / disallows trailing slashes
5340  */
5341 void StrBufStripSlashes(StrBuf *Dir, int RemoveTrailingSlash)
5342 {
5343         char *a, *b;
5344
5345         a = b = Dir->buf;
5346
5347         while (!IsEmptyStr(a)) {
5348                 if (*a == '/') {
5349                         while (*a == '/')
5350                                 a++;
5351                         *b = '/';
5352                         b++;
5353                 }
5354                 else {
5355                         *b = *a;
5356                         b++; a++;
5357                 }
5358         }
5359         if ((RemoveTrailingSlash) &&
5360             (b > Dir->buf) &&
5361             (*(b - 1) == '/')){
5362                 b--;
5363         }
5364         *b = '\0';
5365         Dir->BufUsed = b - Dir->buf;
5366 }
5367
5368