libcitadel/lib/stringbuf.c

   1 /*
   2  * Copyright (c) 1987-2013 by the citadel.org team
   3  *
   4  * This program is open source software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 3 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #define _GNU_SOURCE
  20 #include "sysdep.h"
  21 #include <ctype.h>
  22 #include <errno.h>
  23 #include <string.h>
  24 #include <unistd.h>
  25 #include <string.h>
  26 #include <stdio.h>
  27 #include <sys/select.h>
  28 #include <fcntl.h>
  29 #include <sys/types.h>
  30 #define SHOW_ME_VAPPEND_PRINTF
  31 #include <stdarg.h>
  32
  33 #include "libcitadel.h"
  34
  35 #include "b64/cencode.h"
  36 #include "b64/cdecode.h"
  37
  38 #ifdef HAVE_ICONV
  39 #include <iconv.h>
  40 #endif
  41
  42 #ifdef HAVE_BACKTRACE
  43 #include <execinfo.h>
  44 #endif
  45
  46 #ifdef UNDEF_MEMCPY
  47 #undef memcpy
  48 #endif
  49
  50 #ifdef HAVE_ZLIB
  51 #include <zlib.h>
  52 int ZEXPORT compress_gzip(Bytef * dest, size_t * destLen,
  53                           const Bytef * source, uLong sourceLen, int level);
  54 #endif
  55 int BaseStrBufSize = 64;
  56 int EnableSplice = 0;
  57
  58 const char *StrBufNOTNULL = ((char*) NULL) - 1;
  59
  60 const char HexList[256][3] = {
  61         "00","01","02","03","04","05","06","07","08","09","0A","0B","0C","0D","0E","0F",
  62         "10","11","12","13","14","15","16","17","18","19","1A","1B","1C","1D","1E","1F",
  63         "20","21","22","23","24","25","26","27","28","29","2A","2B","2C","2D","2E","2F",
  64         "30","31","32","33","34","35","36","37","38","39","3A","3B","3C","3D","3E","3F",
  65         "40","41","42","43","44","45","46","47","48","49","4A","4B","4C","4D","4E","4F",
  66         "50","51","52","53","54","55","56","57","58","59","5A","5B","5C","5D","5E","5F",
  67         "60","61","62","63","64","65","66","67","68","69","6A","6B","6C","6D","6E","6F",
  68         "70","71","72","73","74","75","76","77","78","79","7A","7B","7C","7D","7E","7F",
  69         "80","81","82","83","84","85","86","87","88","89","8A","8B","8C","8D","8E","8F",
  70         "90","91","92","93","94","95","96","97","98","99","9A","9B","9C","9D","9E","9F",
  71         "A0","A1","A2","A3","A4","A5","A6","A7","A8","A9","AA","AB","AC","AD","AE","AF",
  72         "B0","B1","B2","B3","B4","B5","B6","B7","B8","B9","BA","BB","BC","BD","BE","BF",
  73         "C0","C1","C2","C3","C4","C5","C6","C7","C8","C9","CA","CB","CC","CD","CE","CF",
  74         "D0","D1","D2","D3","D4","D5","D6","D7","D8","D9","DA","DB","DC","DD","DE","DF",
  75         "E0","E1","E2","E3","E4","E5","E6","E7","E8","E9","EA","EB","EC","ED","EE","EF",
  76         "F0","F1","F2","F3","F4","F5","F6","F7","F8","F9","FA","FB","FC","FD","FE","FF"};
  77
  78 /**
  79  * @defgroup StrBuf Stringbuffer, A class for manipulating strings with dynamic buffers
  80  * StrBuf is a versatile class, aiding the handling of dynamic strings
  81  *  * reduce de/reallocations
  82  *  * reduce the need to remeasure it
  83  *  * reduce scanning over the string (in @ref StrBuf_NextTokenizer "Tokenizers")
  84  *  * allow asyncroneous IO for line and Blob based operations
  85  *  * reduce the use of memove in those
  86  *  * Quick filling in several operations with append functions
  87  */
  88
  89 /**
  90  * @defgroup StrBuf_DeConstructors Create/Destroy StrBufs
  91  * @ingroup StrBuf
  92  */
  93
  94 /**
  95  * @defgroup StrBuf_Cast Cast operators to interact with char* based code
  96  * @ingroup StrBuf
  97  * use these operators to interfere with code demanding char*;
  98  * if you need to own the content, smash me. Avoid, since we loose the length information.
  99  */
 100
 101 /**
 102  * @defgroup StrBuf_Filler Create/Replace/Append Content into a StrBuf
 103  * @ingroup StrBuf
 104  * operations to get your Strings into a StrBuf, manipulating them, or appending
 105  */
 106 /**
 107  * @defgroup StrBuf_NextTokenizer Fast tokenizer to pull tokens in sequence
 108  * @ingroup StrBuf
 109  * Quick tokenizer; demands of the user to pull its tokens in sequence
 110  */
 111
 112 /**
 113  * @defgroup StrBuf_Tokenizer tokenizer Functions; Slow ones.
 114  * @ingroup StrBuf
 115  * versatile tokenizer; random access to tokens, but slower; Prefer the @ref StrBuf_NextTokenizer "Next Tokenizer"
 116  */
 117
 118 /**
 119  * @defgroup StrBuf_BufferedIO Buffered IO with Asynchroneous reads and no unneeded memmoves (the fast ones)
 120  * @ingroup StrBuf
 121  * File IO to fill StrBufs; Works with work-buffer shared across several calls;
 122  * External Cursor to maintain the current read position inside of the buffer
 123  * the non-fast ones will use memove to keep the start of the buffer the read buffer (which is slower)
 124  */
 125
 126 /**
 127  * @defgroup StrBuf_IO FileIO; Prefer @ref StrBuf_BufferedIO
 128  * @ingroup StrBuf
 129  * Slow I/O; avoid.
 130  */
 131
 132 /**
 133  * @defgroup StrBuf_DeEnCoder functions to translate the contents of a buffer
 134  * @ingroup StrBuf
 135  * these functions translate the content of a buffer into another representation;
 136  * some are combined Fillers and encoders
 137  */
 138
 139 /**
 140  * Private Structure for the Stringbuffer
 141  */
 142 struct StrBuf {
 143         char *buf;         /**< the pointer to the dynamic buffer */
 144         long BufSize;      /**< how many spcae do we optain */
 145         long BufUsed;      /**< StNumber of Chars used excluding the trailing \\0 */
 146         int ConstBuf;      /**< are we just a wrapper arround a static buffer and musn't we be changed? */
 147 #ifdef SIZE_DEBUG
 148         long nIncreases;   /**< for profiling; cound how many times we needed more */
 149         char bt [SIZ];     /**< Stacktrace of last increase */
 150         char bt_lastinc [SIZ]; /**< How much did we increase last time? */
 151 #endif
 152 };
 153
 154
 155 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE);
 156 static inline int Ctdl_IsUtf8SequenceStart(const char Char);
 157
 158 #ifdef SIZE_DEBUG
 159 #ifdef HAVE_BACKTRACE
 160 static void StrBufBacktrace(StrBuf *Buf, int which)
 161 {
 162         int n;
 163         char *pstart, *pch;
 164         void *stack_frames[50];
 165         size_t size, i;
 166         char **strings;
 167
 168         if (which)
 169                 pstart = pch = Buf->bt;
 170         else
 171                 pstart = pch = Buf->bt_lastinc;
 172         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
 173         strings = backtrace_symbols(stack_frames, size);
 174         for (i = 0; i < size; i++) {
 175                 if (strings != NULL)
 176                         n = snprintf(pch, SIZ - (pch - pstart), "%s\\n", strings[i]);
 177                 else
 178                         n = snprintf(pch, SIZ - (pch - pstart), "%p\\n", stack_frames[i]);
 179                 pch += n;
 180         }
 181         free(strings);
 182
 183
 184 }
 185 #endif
 186
 187 void dbg_FreeStrBuf(StrBuf *FreeMe, char *FromWhere)
 188 {
 189         if (hFreeDbglog == -1){
 190                 pid_t pid = getpid();
 191                 char path [SIZ];
 192                 snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
 193                 hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
 194         }
 195         if ((*FreeMe)->nIncreases > 0)
 196         {
 197                 char buf[SIZ * 3];
 198                 long n;
 199                 n = snprintf(buf, SIZ * 3, "%c+|%ld|%ld|%ld|%s|%s|\n",
 200                              FromWhere,
 201                              (*FreeMe)->nIncreases,
 202                              (*FreeMe)->BufUsed,
 203                              (*FreeMe)->BufSize,
 204                              (*FreeMe)->bt,
 205                              (*FreeMe)->bt_lastinc);
 206                 n = write(hFreeDbglog, buf, n);
 207         }
 208         else
 209         {
 210                 char buf[128];
 211                 long n;
 212                 n = snprintf(buf, 128, "%c_|0|%ld%ld|\n",
 213                              FromWhere,
 214                              (*FreeMe)->BufUsed,
 215                              (*FreeMe)->BufSize);
 216                 n = write(hFreeDbglog, buf, n);
 217         }
 218 }
 219
 220 void dbg_IncreaseBuf(StrBuf *IncMe)
 221 {
 222         Buf->nIncreases++;
 223 #ifdef HAVE_BACKTRACE
 224         StrBufBacktrace(Buf, 1);
 225 #endif
 226 }
 227
 228 void dbg_Init(StrBuf *Buf)
 229 {
 230         Buf->nIncreases = 0;
 231         Buf->bt[0] = '\0';
 232         Buf->bt_lastinc[0] = '\0';
 233 #ifdef HAVE_BACKTRACE
 234         StrBufBacktrace(Buf, 0);
 235 #endif
 236 }
 237
 238 #else
 239 /* void it... */
 240 #define dbg_FreeStrBuf(a, b)
 241 #define dbg_IncreaseBuf(a)
 242 #define dbg_Init(a)
 243
 244 #endif
 245
 246 /**
 247  * @ingroup StrBuf
 248  * @brief swaps the contents of two StrBufs
 249  * this is to be used to have cheap switched between a work-buffer and a target buffer
 250  * @param A First one
 251  * @param B second one
 252  */
 253 static inline void SwapBuffers(StrBuf *A, StrBuf *B)
 254 {
 255         StrBuf C;
 256
 257         memcpy(&C, A, sizeof(*A));
 258         memcpy(A, B, sizeof(*B));
 259         memcpy(B, &C, sizeof(C));
 260
 261 }
 262
 263 /**
 264  * @ingroup StrBuf_Cast
 265  * @brief Cast operator to Plain String
 266  * @note if the buffer is altered by StrBuf operations, this pointer may become
 267  *  invalid. So don't lean on it after altering the buffer!
 268  *  Since this operation is considered cheap, rather call it often than risking
 269  *  your pointer to become invalid!
 270  * @param Str the string we want to get the c-string representation for
 271  * @returns the Pointer to the Content. Don't mess with it!
 272  */
 273 inline const char *ChrPtr(const StrBuf *Str)
 274 {
 275         if (Str == NULL)
 276                 return "";
 277         return Str->buf;
 278 }
 279
 280 /**
 281  * @ingroup StrBuf_Cast
 282  * @brief since we know strlen()'s result, provide it here.
 283  * @param Str the string to return the length to
 284  * @returns contentlength of the buffer
 285  */
 286 inline int StrLength(const StrBuf *Str)
 287 {
 288         return (Str != NULL) ? Str->BufUsed : 0;
 289 }
 290
 291 /**
 292  * @ingroup StrBuf_DeConstructors
 293  * @brief local utility function to resize the buffer
 294  * @param Buf the buffer whichs storage we should increase
 295  * @param KeepOriginal should we copy the original buffer or just start over with a new one
 296  * @param DestSize what should fit in after?
 297  */
 298 static int IncreaseBuf(StrBuf *Buf, int KeepOriginal, int DestSize)
 299 {
 300         char *NewBuf;
 301         size_t NewSize = Buf->BufSize * 2;
 302
 303         if (Buf->ConstBuf)
 304                 return -1;
 305
 306         if (DestSize > 0)
 307                 while ((NewSize <= DestSize) && (NewSize != 0))
 308                         NewSize *= 2;
 309
 310         if (NewSize == 0)
 311                 return -1;
 312
 313         NewBuf= (char*) malloc(NewSize);
 314         if (NewBuf == NULL)
 315                 return -1;
 316
 317         if (KeepOriginal && (Buf->BufUsed > 0))
 318         {
 319                 memcpy(NewBuf, Buf->buf, Buf->BufUsed);
 320         }
 321         else
 322         {
 323                 NewBuf[0] = '\0';
 324                 Buf->BufUsed = 0;
 325         }
 326         free (Buf->buf);
 327         Buf->buf = NewBuf;
 328         Buf->BufSize = NewSize;
 329
 330         dbg_IncreaseBuf(Buf);
 331
 332         return Buf->BufSize;
 333 }
 334
 335 /**
 336  * @ingroup StrBuf_DeConstructors
 337  * @brief shrink / increase an _EMPTY_ buffer to NewSize. Buffercontent is thoroughly ignored and flushed.
 338  * @param Buf Buffer to shrink (has to be empty)
 339  * @param ThreshHold if the buffer is bigger then this, its readjusted
 340  * @param NewSize if we Shrink it, how big are we going to be afterwards?
 341  */
 342 void ReAdjustEmptyBuf(StrBuf *Buf, long ThreshHold, long NewSize)
 343 {
 344         if ((Buf != NULL) &&
 345             (Buf->BufUsed == 0) &&
 346             (Buf->BufSize < ThreshHold)) {
 347                 free(Buf->buf);
 348                 Buf->buf = (char*) malloc(NewSize);
 349                 Buf->BufUsed = 0;
 350                 Buf->BufSize = NewSize;
 351         }
 352 }
 353
 354 /**
 355  * @ingroup StrBuf_DeConstructors
 356  * @brief shrink long term buffers to their real size so they don't waste memory
 357  * @param Buf buffer to shrink
 358  * @param Force if not set, will just executed if the buffer is much to big; set for lifetime strings
 359  * @returns physical size of the buffer
 360  */
 361 long StrBufShrinkToFit(StrBuf *Buf, int Force)
 362 {
 363         if (Buf == NULL)
 364                 return -1;
 365         if (Force ||
 366             (Buf->BufUsed + (Buf->BufUsed / 3) > Buf->BufSize))
 367         {
 368                 char *TmpBuf;
 369
 370                 TmpBuf = (char*) malloc(Buf->BufUsed + 1);
 371                 if (TmpBuf == NULL)
 372                         return -1;
 373
 374                 memcpy (TmpBuf, Buf->buf, Buf->BufUsed + 1);
 375                 Buf->BufSize = Buf->BufUsed + 1;
 376                 free(Buf->buf);
 377                 Buf->buf = TmpBuf;
 378         }
 379         return Buf->BufUsed;
 380 }
 381
 382 /**
 383  * @ingroup StrBuf_DeConstructors
 384  * @brief Allocate a new buffer with default buffer size
 385  * @returns the new stringbuffer
 386  */
 387 StrBuf* NewStrBuf(void)
 388 {
 389         StrBuf *NewBuf;
 390
 391         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 392         if (NewBuf == NULL)
 393                 return NULL;
 394
 395         NewBuf->buf = (char*) malloc(BaseStrBufSize);
 396         if (NewBuf->buf == NULL)
 397         {
 398                 free(NewBuf);
 399                 return NULL;
 400         }
 401         NewBuf->buf[0] = '\0';
 402         NewBuf->BufSize = BaseStrBufSize;
 403         NewBuf->BufUsed = 0;
 404         NewBuf->ConstBuf = 0;
 405
 406         dbg_Init (NewBuf);
 407
 408         return NewBuf;
 409 }
 410
 411 /**
 412  * @ingroup StrBuf_DeConstructors
 413  * @brief Copy Constructor; returns a duplicate of CopyMe
 414  * @param CopyMe Buffer to faxmilate
 415  * @returns the new stringbuffer
 416  */
 417 StrBuf* NewStrBufDup(const StrBuf *CopyMe)
 418 {
 419         StrBuf *NewBuf;
 420
 421         if (CopyMe == NULL)
 422                 return NewStrBuf();
 423
 424         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 425         if (NewBuf == NULL)
 426                 return NULL;
 427
 428         NewBuf->buf = (char*) malloc(CopyMe->BufSize);
 429         if (NewBuf->buf == NULL)
 430         {
 431                 free(NewBuf);
 432                 return NULL;
 433         }
 434
 435         memcpy(NewBuf->buf, CopyMe->buf, CopyMe->BufUsed + 1);
 436         NewBuf->BufUsed = CopyMe->BufUsed;
 437         NewBuf->BufSize = CopyMe->BufSize;
 438         NewBuf->ConstBuf = 0;
 439
 440         dbg_Init(NewBuf);
 441
 442         return NewBuf;
 443 }
 444
 445 /**
 446  * @ingroup StrBuf_DeConstructors
 447  * @brief Copy Constructor; CreateRelpaceMe will contain CopyFlushMe afterwards.
 448  * @param NoMe if non-NULL, we will use that buffer as value; KeepOriginal will abused as len.
 449  * @param CopyFlushMe Buffer to faxmilate if KeepOriginal, or to move into CreateRelpaceMe if !KeepOriginal.
 450  * @param CreateRelpaceMe If NULL, will be created, else Flushed and filled CopyFlushMe
 451  * @param KeepOriginal should CopyFlushMe remain intact? or may we Steal its buffer?
 452  * @returns the new stringbuffer
 453  */
 454 void NewStrBufDupAppendFlush(StrBuf **CreateRelpaceMe, StrBuf *CopyFlushMe, const char *NoMe, int KeepOriginal)
 455 {
 456         StrBuf *NewBuf;
 457
 458         if (CreateRelpaceMe == NULL)
 459                 return;
 460
 461         if (NoMe != NULL)
 462         {
 463                 if (*CreateRelpaceMe != NULL)
 464                         StrBufPlain(*CreateRelpaceMe, NoMe, KeepOriginal);
 465                 else
 466                         *CreateRelpaceMe = NewStrBufPlain(NoMe, KeepOriginal);
 467                 return;
 468         }
 469
 470         if (CopyFlushMe == NULL)
 471         {
 472                 if (*CreateRelpaceMe != NULL)
 473                         FlushStrBuf(*CreateRelpaceMe);
 474                 else
 475                         *CreateRelpaceMe = NewStrBuf();
 476                 return;
 477         }
 478
 479         /*
 480          * Randomly Chosen: bigger than 64 chars is cheaper to swap the buffers instead of copying.
 481          * else *CreateRelpaceMe may use more memory than needed in a longer term, CopyFlushMe might
 482          * be a big IO-Buffer...
 483          */
 484         if (KeepOriginal || (StrLength(CopyFlushMe) < 256))
 485         {
 486                 if (*CreateRelpaceMe == NULL)
 487                 {
 488                         *CreateRelpaceMe = NewBuf = NewStrBufPlain(NULL, CopyFlushMe->BufUsed);
 489                         dbg_Init(NewBuf);
 490                 }
 491                 else
 492                 {
 493                         NewBuf = *CreateRelpaceMe;
 494                         FlushStrBuf(NewBuf);
 495                 }
 496                 StrBufAppendBuf(NewBuf, CopyFlushMe, 0);
 497         }
 498         else
 499         {
 500                 if (*CreateRelpaceMe == NULL)
 501                 {
 502                         *CreateRelpaceMe = NewBuf = NewStrBufPlain(NULL, CopyFlushMe->BufUsed);
 503                         dbg_Init(NewBuf);
 504                 }
 505                 else
 506                         NewBuf = *CreateRelpaceMe;
 507                 SwapBuffers (NewBuf, CopyFlushMe);
 508         }
 509         if (!KeepOriginal)
 510                 FlushStrBuf(CopyFlushMe);
 511         return;
 512 }
 513
 514 /**
 515  * @ingroup StrBuf_DeConstructors
 516  * @brief create a new Buffer using an existing c-string
 517  * this function should also be used if you want to pre-suggest
 518  * the buffer size to allocate in conjunction with ptr == NULL
 519  * @param ptr the c-string to copy; may be NULL to create a blank instance
 520  * @param nChars How many chars should we copy; -1 if we should measure the length ourselves
 521  * @returns the new stringbuffer
 522  */
 523 StrBuf* NewStrBufPlain(const char* ptr, int nChars)
 524 {
 525         StrBuf *NewBuf;
 526         size_t Siz = BaseStrBufSize;
 527         size_t CopySize;
 528
 529         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 530         if (NewBuf == NULL)
 531                 return NULL;
 532
 533         if (nChars < 0)
 534                 CopySize = strlen((ptr != NULL)?ptr:"");
 535         else
 536                 CopySize = nChars;
 537
 538         while ((Siz <= CopySize) && (Siz != 0))
 539                 Siz *= 2;
 540
 541         if (Siz == 0)
 542         {
 543                 free(NewBuf);
 544                 return NULL;
 545         }
 546
 547         NewBuf->buf = (char*) malloc(Siz);
 548         if (NewBuf->buf == NULL)
 549         {
 550                 free(NewBuf);
 551                 return NULL;
 552         }
 553         NewBuf->BufSize = Siz;
 554         if (ptr != NULL) {
 555                 memcpy(NewBuf->buf, ptr, CopySize);
 556                 NewBuf->buf[CopySize] = '\0';
 557                 NewBuf->BufUsed = CopySize;
 558         }
 559         else {
 560                 NewBuf->buf[0] = '\0';
 561                 NewBuf->BufUsed = 0;
 562         }
 563         NewBuf->ConstBuf = 0;
 564
 565         dbg_Init(NewBuf);
 566
 567         return NewBuf;
 568 }
 569
 570 /**
 571  * @ingroup StrBuf_DeConstructors
 572  * @brief Set an existing buffer from a c-string
 573  * @param Buf buffer to load
 574  * @param ptr c-string to put into
 575  * @param nChars set to -1 if we should work 0-terminated
 576  * @returns the new length of the string
 577  */
 578 int StrBufPlain(StrBuf *Buf, const char* ptr, int nChars)
 579 {
 580         size_t Siz;
 581         size_t CopySize;
 582
 583         if (Buf == NULL)
 584                 return -1;
 585         if (ptr == NULL) {
 586                 FlushStrBuf(Buf);
 587                 return -1;
 588         }
 589
 590         Siz = Buf->BufSize;
 591
 592         if (nChars < 0)
 593                 CopySize = strlen(ptr);
 594         else
 595                 CopySize = nChars;
 596
 597         while ((Siz <= CopySize) && (Siz != 0))
 598                 Siz *= 2;
 599
 600         if (Siz == 0) {
 601                 FlushStrBuf(Buf);
 602                 return -1;
 603         }
 604
 605         if (Siz != Buf->BufSize)
 606                 IncreaseBuf(Buf, 0, Siz);
 607         memcpy(Buf->buf, ptr, CopySize);
 608         Buf->buf[CopySize] = '\0';
 609         Buf->BufUsed = CopySize;
 610         Buf->ConstBuf = 0;
 611         return CopySize;
 612 }
 613
 614
 615 /**
 616  * @ingroup StrBuf_DeConstructors
 617  * @brief use strbuf as wrapper for a string constant for easy handling
 618  * @param StringConstant a string to wrap
 619  * @param SizeOfStrConstant should be sizeof(StringConstant)-1
 620  */
 621 StrBuf* _NewConstStrBuf(const char* StringConstant, size_t SizeOfStrConstant)
 622 {
 623         StrBuf *NewBuf;
 624
 625         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 626         if (NewBuf == NULL)
 627                 return NULL;
 628         NewBuf->buf = (char*) StringConstant;
 629         NewBuf->BufSize = SizeOfStrConstant;
 630         NewBuf->BufUsed = SizeOfStrConstant;
 631         NewBuf->ConstBuf = 1;
 632
 633         dbg_Init(NewBuf);
 634
 635         return NewBuf;
 636 }
 637
 638
 639 /**
 640  * @ingroup StrBuf_DeConstructors
 641  * @brief flush the content of a Buf; keep its struct
 642  * @param buf Buffer to flush
 643  */
 644 int FlushStrBuf(StrBuf *buf)
 645 {
 646         if ((buf == NULL) || (buf->buf == NULL))
 647                 return -1;
 648         if (buf->ConstBuf)
 649                 return -1;
 650         buf->buf[0] ='\0';
 651         buf->BufUsed = 0;
 652         return 0;
 653 }
 654
 655 /**
 656  * @ingroup StrBuf_DeConstructors
 657  * @brief wipe the content of a Buf thoroughly (overwrite it -> expensive); keep its struct
 658  * @param buf Buffer to wipe
 659  */
 660 int FLUSHStrBuf(StrBuf *buf)
 661 {
 662         if (buf == NULL)
 663                 return -1;
 664         if (buf->ConstBuf)
 665                 return -1;
 666         if (buf->BufUsed > 0) {
 667                 memset(buf->buf, 0, buf->BufUsed);
 668                 buf->BufUsed = 0;
 669         }
 670         return 0;
 671 }
 672
 673 #ifdef SIZE_DEBUG
 674 int hFreeDbglog = -1;
 675 #endif
 676 /**
 677  * @ingroup StrBuf_DeConstructors
 678  * @brief Release a Buffer
 679  * Its a double pointer, so it can NULL your pointer
 680  * so fancy SIG11 appear instead of random results
 681  * @param FreeMe Pointer Pointer to the buffer to free
 682  */
 683 void FreeStrBuf (StrBuf **FreeMe)
 684 {
 685         if (*FreeMe == NULL)
 686                 return;
 687
 688         dbg_FreeStrBuf(FreeMe, 'F');
 689
 690         if (!(*FreeMe)->ConstBuf)
 691                 free((*FreeMe)->buf);
 692         free(*FreeMe);
 693         *FreeMe = NULL;
 694 }
 695
 696 /**
 697  * @ingroup StrBuf_DeConstructors
 698  * @brief flatten a Buffer to the Char * we return
 699  * Its a double pointer, so it can NULL your pointer
 700  * so fancy SIG11 appear instead of random results
 701  * The Callee then owns the buffer and is responsible for freeing it.
 702  * @param SmashMe Pointer Pointer to the buffer to release Buf from and free
 703  * @returns the pointer of the buffer; Callee owns the memory thereafter.
 704  */
 705 char *SmashStrBuf (StrBuf **SmashMe)
 706 {
 707         char *Ret;
 708
 709         if ((SmashMe == NULL) || (*SmashMe == NULL))
 710                 return NULL;
 711
 712         dbg_FreeStrBuf(SmashMe, 'S');
 713
 714         Ret = (*SmashMe)->buf;
 715         free(*SmashMe);
 716         *SmashMe = NULL;
 717         return Ret;
 718 }
 719
 720 /**
 721  * @ingroup StrBuf_DeConstructors
 722  * @brief Release the buffer
 723  * If you want put your StrBuf into a Hash, use this as Destructor.
 724  * @param VFreeMe untyped pointer to a StrBuf. be shure to do the right thing [TM]
 725  */
 726 void HFreeStrBuf (void *VFreeMe)
 727 {
 728         StrBuf *FreeMe = (StrBuf*)VFreeMe;
 729         if (FreeMe == NULL)
 730                 return;
 731
 732         dbg_FreeStrBuf(SmashMe, 'H');
 733
 734         if (!FreeMe->ConstBuf)
 735                 free(FreeMe->buf);
 736         free(FreeMe);
 737 }
 738
 739
 740 /*******************************************************************************
 741  *                      Simple string transformations                          *
 742  *******************************************************************************/
 743
 744 /**
 745  * @ingroup StrBuf
 746  * @brief Wrapper around atol
 747  */
 748 long StrTol(const StrBuf *Buf)
 749 {
 750         if (Buf == NULL)
 751                 return 0;
 752         if(Buf->BufUsed > 0)
 753                 return atol(Buf->buf);
 754         else
 755                 return 0;
 756 }
 757
 758 /**
 759  * @ingroup StrBuf
 760  * @brief Wrapper around atoi
 761  */
 762 int StrToi(const StrBuf *Buf)
 763 {
 764         if (Buf == NULL)
 765                 return 0;
 766         if (Buf->BufUsed > 0)
 767                 return atoi(Buf->buf);
 768         else
 769                 return 0;
 770 }
 771
 772 /**
 773  * @ingroup StrBuf
 774  * @brief Checks to see if the string is a pure number
 775  * @param Buf The buffer to inspect
 776  * @returns 1 if its a pure number, 0, if not.
 777  */
 778 int StrBufIsNumber(const StrBuf *Buf) {
 779         char * pEnd;
 780         if ((Buf == NULL) || (Buf->BufUsed == 0)) {
 781                 return 0;
 782         }
 783         strtoll(Buf->buf, &pEnd, 10);
 784         if (pEnd == Buf->buf)
 785                 return 0;
 786         if ((pEnd != NULL) && (pEnd == Buf->buf + Buf->BufUsed))
 787                 return 1;
 788         if (Buf->buf == pEnd)
 789                 return 0;
 790         return 0;
 791 }
 792
 793 /**
 794  * @ingroup StrBuf_Filler
 795  * @brief modifies a Single char of the Buf
 796  * You can point to it via char* or a zero-based integer
 797  * @param Buf The buffer to manipulate
 798  * @param ptr char* to zero; use NULL if unused
 799  * @param nThChar zero based pointer into the string; use -1 if unused
 800  * @param PeekValue The Character to place into the position
 801  */
 802 long StrBufPeek(StrBuf *Buf, const char* ptr, long nThChar, char PeekValue)
 803 {
 804         if (Buf == NULL)
 805                 return -1;
 806         if (ptr != NULL)
 807                 nThChar = ptr - Buf->buf;
 808         if ((nThChar < 0) || (nThChar > Buf->BufUsed))
 809                 return -1;
 810         Buf->buf[nThChar] = PeekValue;
 811         return nThChar;
 812 }
 813
 814 /**
 815  * @ingroup StrBuf_Filler
 816  * @brief modifies a range of chars of the Buf
 817  * You can point to it via char* or a zero-based integer
 818  * @param Buf The buffer to manipulate
 819  * @param ptr char* to zero; use NULL if unused
 820  * @param nThChar zero based pointer into the string; use -1 if unused
 821  * @param nChars how many chars are to be flushed?
 822  * @param PookValue The Character to place into that area
 823  */
 824 long StrBufPook(StrBuf *Buf, const char* ptr, long nThChar, long nChars, char PookValue)
 825 {
 826         if (Buf == NULL)
 827                 return -1;
 828         if (ptr != NULL)
 829                 nThChar = ptr - Buf->buf;
 830         if ((nThChar < 0) || (nThChar > Buf->BufUsed))
 831                 return -1;
 832         if (nThChar + nChars > Buf->BufUsed)
 833                 nChars =  Buf->BufUsed - nThChar;
 834
 835         memset(Buf->buf + nThChar, PookValue, nChars);
 836         /* just to be shure... */
 837         Buf->buf[Buf->BufUsed] = 0;
 838         return nChars;
 839 }
 840
 841 /**
 842  * @ingroup StrBuf_Filler
 843  * @brief Append a StringBuffer to the buffer
 844  * @param Buf Buffer to modify
 845  * @param AppendBuf Buffer to copy at the end of our buffer
 846  * @param Offset Should we start copying from an offset?
 847  */
 848 void StrBufAppendBuf(StrBuf *Buf, const StrBuf *AppendBuf, unsigned long Offset)
 849 {
 850         if ((AppendBuf == NULL) || (AppendBuf->buf == NULL) ||
 851             (Buf == NULL) || (Buf->buf == NULL))
 852                 return;
 853
 854         if (Buf->BufSize - Offset < AppendBuf->BufUsed + Buf->BufUsed + 1)
 855                 IncreaseBuf(Buf,
 856                             (Buf->BufUsed > 0),
 857                             AppendBuf->BufUsed + Buf->BufUsed);
 858
 859         memcpy(Buf->buf + Buf->BufUsed,
 860                AppendBuf->buf + Offset,
 861                AppendBuf->BufUsed - Offset);
 862         Buf->BufUsed += AppendBuf->BufUsed - Offset;
 863         Buf->buf[Buf->BufUsed] = '\0';
 864 }
 865
 866
 867 /**
 868  * @ingroup StrBuf_Filler
 869  * @brief Append a C-String to the buffer
 870  * @param Buf Buffer to modify
 871  * @param AppendBuf Buffer to copy at the end of our buffer
 872  * @param AppendSize number of bytes to copy; set to -1 if we should count it in advance
 873  * @param Offset Should we start copying from an offset?
 874  */
 875 void StrBufAppendBufPlain(StrBuf *Buf, const char *AppendBuf, long AppendSize, unsigned long Offset)
 876 {
 877         long aps;
 878         long BufSizeRequired;
 879
 880         if ((AppendBuf == NULL) || (Buf == NULL))
 881                 return;
 882
 883         if (AppendSize < 0 )
 884                 aps = strlen(AppendBuf + Offset);
 885         else
 886                 aps = AppendSize - Offset;
 887
 888         BufSizeRequired = Buf->BufUsed + aps + 1;
 889         if (Buf->BufSize <= BufSizeRequired)
 890                 IncreaseBuf(Buf, (Buf->BufUsed > 0), BufSizeRequired);
 891
 892         memcpy(Buf->buf + Buf->BufUsed,
 893                AppendBuf + Offset,
 894                aps);
 895         Buf->BufUsed += aps;
 896         Buf->buf[Buf->BufUsed] = '\0';
 897 }
 898
 899 /**
 900  * @ingroup StrBuf_Filler
 901  * @brief sprintf like function appending the formated string to the buffer
 902  * vsnprintf version to wrap into own calls
 903  * @param Buf Buffer to extend by format and Params
 904  * @param format printf alike format to add
 905  * @param ap va_list containing the items for format
 906  */
 907 void StrBufVAppendPrintf(StrBuf *Buf, const char *format, va_list ap)
 908 {
 909         va_list apl;
 910         size_t BufSize;
 911         size_t nWritten;
 912         size_t Offset;
 913         size_t newused;
 914
 915         if ((Buf == NULL)  || (format == NULL))
 916                 return;
 917
 918         BufSize = Buf->BufSize;
 919         nWritten = Buf->BufSize + 1;
 920         Offset = Buf->BufUsed;
 921         newused = Offset + nWritten;
 922
 923         while (newused >= BufSize) {
 924                 va_copy(apl, ap);
 925                 nWritten = vsnprintf(Buf->buf + Offset,
 926                                      Buf->BufSize - Offset,
 927                                      format, apl);
 928                 va_end(apl);
 929                 newused = Offset + nWritten;
 930                 if (newused >= Buf->BufSize) {
 931                         if (IncreaseBuf(Buf, 1, newused) == -1)
 932                                 return; /* TODO: error handling? */
 933                         newused = Buf->BufSize + 1;
 934                 }
 935                 else {
 936                         Buf->BufUsed = Offset + nWritten;
 937                         BufSize = Buf->BufSize;
 938                 }
 939
 940         }
 941 }
 942
 943 /**
 944  * @ingroup StrBuf_Filler
 945  * @brief sprintf like function appending the formated string to the buffer
 946  * @param Buf Buffer to extend by format and Params
 947  * @param format printf alike format to add
 948  */
 949 void StrBufAppendPrintf(StrBuf *Buf, const char *format, ...)
 950 {
 951         size_t BufSize;
 952         size_t nWritten;
 953         size_t Offset;
 954         size_t newused;
 955         va_list arg_ptr;
 956
 957         if ((Buf == NULL)  || (format == NULL))
 958                 return;
 959
 960         BufSize = Buf->BufSize;
 961         nWritten = Buf->BufSize + 1;
 962         Offset = Buf->BufUsed;
 963         newused = Offset + nWritten;
 964
 965         while (newused >= BufSize) {
 966                 va_start(arg_ptr, format);
 967                 nWritten = vsnprintf(Buf->buf + Buf->BufUsed,
 968                                      Buf->BufSize - Buf->BufUsed,
 969                                      format, arg_ptr);
 970                 va_end(arg_ptr);
 971                 newused = Buf->BufUsed + nWritten;
 972                 if (newused >= Buf->BufSize) {
 973                         if (IncreaseBuf(Buf, 1, newused) == -1)
 974                                 return; /* TODO: error handling? */
 975                         newused = Buf->BufSize + 1;
 976                 }
 977                 else {
 978                         Buf->BufUsed += nWritten;
 979                         BufSize = Buf->BufSize;
 980                 }
 981
 982         }
 983 }
 984
 985 /**
 986  * @ingroup StrBuf_Filler
 987  * @brief sprintf like function putting the formated string into the buffer
 988  * @param Buf Buffer to extend by format and Parameters
 989  * @param format printf alike format to add
 990  */
 991 void StrBufPrintf(StrBuf *Buf, const char *format, ...)
 992 {
 993         size_t nWritten;
 994         va_list arg_ptr;
 995
 996         if ((Buf == NULL)  || (format == NULL))
 997                 return;
 998
 999         nWritten = Buf->BufSize + 1;
1000         while (nWritten >= Buf->BufSize) {
1001                 va_start(arg_ptr, format);
1002                 nWritten = vsnprintf(Buf->buf, Buf->BufSize, format, arg_ptr);
1003                 va_end(arg_ptr);
1004                 if (nWritten >= Buf->BufSize) {
1005                         if (IncreaseBuf(Buf, 0, 0) == -1)
1006                                 return; /* TODO: error handling? */
1007                         nWritten = Buf->BufSize + 1;
1008                         continue;
1009                 }
1010                 Buf->BufUsed = nWritten ;
1011         }
1012 }
1013
1014 /**
1015  * @ingroup StrBuf_Filler
1016  * @brief Callback for cURL to append the webserver reply to a buffer
1017  * @param ptr pre-defined by the cURL API; see man 3 curl for mre info
1018  * @param size pre-defined by the cURL API; see man 3 curl for mre info
1019  * @param nmemb pre-defined by the cURL API; see man 3 curl for mre info
1020  * @param stream pre-defined by the cURL API; see man 3 curl for mre info
1021  */
1022 size_t CurlFillStrBuf_callback(void *ptr, size_t size, size_t nmemb, void *stream)
1023 {
1024
1025         StrBuf *Target;
1026
1027         Target = stream;
1028         if (ptr == NULL)
1029                 return 0;
1030
1031         StrBufAppendBufPlain(Target, ptr, size * nmemb, 0);
1032         return size * nmemb;
1033 }
1034
1035
1036 /**
1037  * @ingroup StrBuf
1038  * @brief extracts a substring from Source into dest
1039  * @param dest buffer to place substring into
1040  * @param Source string to copy substring from
1041  * @param Offset chars to skip from start
1042  * @param nChars number of chars to copy
1043  * @returns the number of chars copied; may be different from nChars due to the size of Source
1044  */
1045 int StrBufSub(StrBuf *dest, const StrBuf *Source, unsigned long Offset, size_t nChars)
1046 {
1047         size_t NCharsRemain;
1048         if (Offset > Source->BufUsed)
1049         {
1050                 if (dest != NULL)
1051                         FlushStrBuf(dest);
1052                 return 0;
1053         }
1054         if (Offset + nChars < Source->BufUsed)
1055         {
1056                 if ((nChars >= dest->BufSize) &&
1057                     (IncreaseBuf(dest, 0, nChars + 1) == -1))
1058                         return 0;
1059                 memcpy(dest->buf, Source->buf + Offset, nChars);
1060                 dest->BufUsed = nChars;
1061                 dest->buf[dest->BufUsed] = '\0';
1062                 return nChars;
1063         }
1064         NCharsRemain = Source->BufUsed - Offset;
1065         if ((NCharsRemain  >= dest->BufSize) &&
1066             (IncreaseBuf(dest, 0, NCharsRemain + 1) == -1))
1067                 return 0;
1068         memcpy(dest->buf, Source->buf + Offset, NCharsRemain);
1069         dest->BufUsed = NCharsRemain;
1070         dest->buf[dest->BufUsed] = '\0';
1071         return NCharsRemain;
1072 }
1073
1074 /**
1075  * @ingroup StrBuf
1076  * @brief Cut nChars from the start of the string
1077  * @param Buf Buffer to modify
1078  * @param nChars how many chars should be skipped?
1079  */
1080 void StrBufCutLeft(StrBuf *Buf, int nChars)
1081 {
1082         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1083         if (nChars >= Buf->BufUsed) {
1084                 FlushStrBuf(Buf);
1085                 return;
1086         }
1087         memmove(Buf->buf, Buf->buf + nChars, Buf->BufUsed - nChars);
1088         Buf->BufUsed -= nChars;
1089         Buf->buf[Buf->BufUsed] = '\0';
1090 }
1091
1092 /**
1093  * @ingroup StrBuf
1094  * @brief Cut the trailing n Chars from the string
1095  * @param Buf Buffer to modify
1096  * @param nChars how many chars should be trunkated?
1097  */
1098 void StrBufCutRight(StrBuf *Buf, int nChars)
1099 {
1100         if ((Buf == NULL) || (Buf->BufUsed == 0) || (Buf->buf == NULL))
1101                 return;
1102
1103         if (nChars >= Buf->BufUsed) {
1104                 FlushStrBuf(Buf);
1105                 return;
1106         }
1107         Buf->BufUsed -= nChars;
1108         Buf->buf[Buf->BufUsed] = '\0';
1109 }
1110
1111 /**
1112  * @ingroup StrBuf
1113  * @brief Cut the string after n Chars
1114  * @param Buf Buffer to modify
1115  * @param AfternChars after how many chars should we trunkate the string?
1116  * @param At if non-null and points inside of our string, cut it there.
1117  */
1118 void StrBufCutAt(StrBuf *Buf, int AfternChars, const char *At)
1119 {
1120         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1121         if (At != NULL){
1122                 AfternChars = At - Buf->buf;
1123         }
1124
1125         if ((AfternChars < 0) || (AfternChars >= Buf->BufUsed))
1126                 return;
1127         Buf->BufUsed = AfternChars;
1128         Buf->buf[Buf->BufUsed] = '\0';
1129 }
1130
1131
1132 /**
1133  * @ingroup StrBuf
1134  * @brief Strip leading and trailing spaces from a string; with premeasured and adjusted length.
1135  * @param Buf the string to modify
1136  */
1137 void StrBufTrim(StrBuf *Buf)
1138 {
1139         int delta = 0;
1140         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1141
1142         while ((Buf->BufUsed > 0) &&
1143                isspace(Buf->buf[Buf->BufUsed - 1]))
1144         {
1145                 Buf->BufUsed --;
1146         }
1147         Buf->buf[Buf->BufUsed] = '\0';
1148
1149         if (Buf->BufUsed == 0) return;
1150
1151         while ((Buf->BufUsed > delta) && (isspace(Buf->buf[delta]))){
1152                 delta ++;
1153         }
1154         if (delta > 0) StrBufCutLeft(Buf, delta);
1155 }
1156 /**
1157  * @ingroup StrBuf
1158  * @brief changes all spaces in the string  (tab, linefeed...) to Blank (0x20)
1159  * @param Buf the string to modify
1160  */
1161 void StrBufSpaceToBlank(StrBuf *Buf)
1162 {
1163         char *pche, *pch;
1164
1165         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1166
1167         pch = Buf->buf;
1168         pche = pch + Buf->BufUsed;
1169         while (pch < pche)
1170         {
1171                 if (isspace(*pch))
1172                         *pch = ' ';
1173                 pch ++;
1174         }
1175 }
1176
1177 void StrBufStripAllBut(StrBuf *Buf, char leftboundary, char rightboundary)
1178 {
1179         const char *pLeft;
1180         const char *pRight;
1181
1182         if ((Buf == NULL) || (Buf->buf == NULL)) {
1183                 return;
1184         }
1185
1186         pRight = strchr(Buf->buf, rightboundary);
1187         if (pRight != NULL) {
1188                 StrBufCutAt(Buf, 0, pRight);
1189         }
1190
1191         pLeft = strrchr(ChrPtr(Buf), leftboundary);
1192         if (pLeft != NULL) {
1193                 StrBufCutLeft(Buf, pLeft - Buf->buf + 1);
1194         }
1195 }
1196
1197
1198 /**
1199  * @ingroup StrBuf_Filler
1200  * @brief uppercase the contents of a buffer
1201  * @param Buf the buffer to translate
1202  */
1203 void StrBufUpCase(StrBuf *Buf)
1204 {
1205         char *pch, *pche;
1206
1207         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1208
1209         pch = Buf->buf;
1210         pche = pch + Buf->BufUsed;
1211         while (pch < pche) {
1212                 *pch = toupper(*pch);
1213                 pch ++;
1214         }
1215 }
1216
1217
1218 /**
1219  * @ingroup StrBuf_Filler
1220  * @brief lowercase the contents of a buffer
1221  * @param Buf the buffer to translate
1222  */
1223 void StrBufLowerCase(StrBuf *Buf)
1224 {
1225         char *pch, *pche;
1226
1227         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1228
1229         pch = Buf->buf;
1230         pche = pch + Buf->BufUsed;
1231         while (pch < pche) {
1232                 *pch = tolower(*pch);
1233                 pch ++;
1234         }
1235 }
1236
1237
1238 /*******************************************************************************
1239  *           a tokenizer that kills, maims, and destroys                       *
1240  *******************************************************************************/
1241
1242 /**
1243  * @ingroup StrBuf_Tokenizer
1244  * @brief Replace a token at a given place with a given length by another token with given length
1245  * @param Buf String where to work on
1246  * @param where where inside of the Buf is the search-token
1247  * @param HowLong How long is the token to be replaced
1248  * @param Repl Token to insert at 'where'
1249  * @param ReplLen Length of repl
1250  * @returns -1 if fail else length of resulting Buf
1251  */
1252 int StrBufReplaceToken(StrBuf *Buf, long where, long HowLong,
1253                        const char *Repl, long ReplLen)
1254 {
1255
1256         if ((Buf == NULL) ||
1257             (where > Buf->BufUsed) ||
1258             (where + HowLong > Buf->BufUsed))
1259                 return -1;
1260
1261         if (where + ReplLen - HowLong > Buf->BufSize)
1262                 if (IncreaseBuf(Buf, 1, Buf->BufUsed + ReplLen) < 0)
1263                         return -1;
1264
1265         memmove(Buf->buf + where + ReplLen,
1266                 Buf->buf + where + HowLong,
1267                 Buf->BufUsed - where - HowLong);
1268
1269         memcpy(Buf->buf + where,
1270                Repl, ReplLen);
1271
1272         Buf->BufUsed += ReplLen - HowLong;
1273
1274         return Buf->BufUsed;
1275 }
1276
1277 /**
1278  * @ingroup StrBuf_Tokenizer
1279  * @brief Counts the numbmer of tokens in a buffer
1280  * @param source String to count tokens in
1281  * @param tok    Tokenizer char to count
1282  * @returns numbers of tokenizer chars found
1283  */
1284 int StrBufNum_tokens(const StrBuf *source, char tok)
1285 {
1286         char *pch, *pche;
1287         long NTokens;
1288         if ((source == NULL) || (source->BufUsed == 0))
1289                 return 0;
1290         if ((source->BufUsed == 1) && (*source->buf == tok))
1291                 return 2;
1292         NTokens = 1;
1293         pch = source->buf;
1294         pche = pch + source->BufUsed;
1295         while (pch < pche)
1296         {
1297                 if (*pch == tok)
1298                         NTokens ++;
1299                 pch ++;
1300         }
1301         return NTokens;
1302 }
1303
1304 /**
1305  * @ingroup StrBuf_Tokenizer
1306  * @brief a string tokenizer
1307  * @param Source StringBuffer to read into
1308  * @param parmnum n'th Parameter to remove
1309  * @param separator tokenizer character
1310  * @returns -1 if not found, else length of token.
1311  */
1312 int StrBufRemove_token(StrBuf *Source, int parmnum, char separator)
1313 {
1314         int ReducedBy;
1315         char *d, *s, *end;              /* dest, source */
1316         int count = 0;
1317
1318         /* Find desired @parameter */
1319         end = Source->buf + Source->BufUsed;
1320         d = Source->buf;
1321         while ((d <= end) &&
1322                (count < parmnum))
1323         {
1324                 /* End of string, bail! */
1325                 if (!*d) {
1326                         d = NULL;
1327                         break;
1328                 }
1329                 if (*d == separator) {
1330                         count++;
1331                 }
1332                 d++;
1333         }
1334         if ((d == NULL) || (d >= end))
1335                 return 0;               /* @Parameter not found */
1336
1337         /* Find next @parameter */
1338         s = d;
1339         while ((s <= end) &&
1340                (*s && *s != separator))
1341         {
1342                 s++;
1343         }
1344         if (*s == separator)
1345                 s++;
1346         ReducedBy = d - s;
1347
1348         /* Hack and slash */
1349         if (s >= end) {
1350                 return 0;
1351         }
1352         else if (*s) {
1353                 memmove(d, s, Source->BufUsed - (s - Source->buf));
1354                 Source->BufUsed += ReducedBy;
1355                 Source->buf[Source->BufUsed] = '\0';
1356         }
1357         else if (d == Source->buf) {
1358                 *d = 0;
1359                 Source->BufUsed = 0;
1360         }
1361         else {
1362                 *--d = '\0';
1363                 Source->BufUsed += ReducedBy;
1364         }
1365         /*
1366         while (*s) {
1367                 *d++ = *s++;
1368         }
1369         *d = 0;
1370         */
1371         return ReducedBy;
1372 }
1373
1374 int StrBufExtract_tokenFromStr(StrBuf *dest, const char *Source, long SourceLen, int parmnum, char separator)
1375 {
1376         const StrBuf Temp = {
1377                 (char*)Source,
1378                 SourceLen,
1379                 SourceLen,
1380                 1
1381 #ifdef SIZE_DEBUG
1382                 ,
1383                 0,
1384                 "",
1385                 ""
1386 #endif
1387         };
1388
1389         return StrBufExtract_token(dest, &Temp, parmnum, separator);
1390 }
1391
1392 /**
1393  * @ingroup StrBuf_Tokenizer
1394  * @brief a string tokenizer
1395  * @param dest Destination StringBuffer
1396  * @param Source StringBuffer to read into
1397  * @param parmnum n'th Parameter to extract
1398  * @param separator tokenizer character
1399  * @returns -1 if not found, else length of token.
1400  */
1401 int StrBufExtract_token(StrBuf *dest, const StrBuf *Source, int parmnum, char separator)
1402 {
1403         const char *s, *e;              //* source * /
1404         int len = 0;                    //* running total length of extracted string * /
1405         int current_token = 0;          //* token currently being processed * /
1406
1407         if (dest != NULL) {
1408                 dest->buf[0] = '\0';
1409                 dest->BufUsed = 0;
1410         }
1411         else
1412                 return(-1);
1413
1414         if ((Source == NULL) || (Source->BufUsed ==0)) {
1415                 return(-1);
1416         }
1417         s = Source->buf;
1418         e = s + Source->BufUsed;
1419
1420         //cit_backtrace();
1421         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1422
1423         while ((s < e) && !IsEmptyStr(s)) {
1424                 if (*s == separator) {
1425                         ++current_token;
1426                 }
1427                 if (len >= dest->BufSize) {
1428                         dest->BufUsed = len;
1429                         if (IncreaseBuf(dest, 1, -1) < 0) {
1430                                 dest->BufUsed --;
1431                                 break;
1432                         }
1433                 }
1434                 if ( (current_token == parmnum) &&
1435                      (*s != separator)) {
1436                         dest->buf[len] = *s;
1437                         ++len;
1438                 }
1439                 else if (current_token > parmnum) {
1440                         break;
1441                 }
1442                 ++s;
1443         }
1444
1445         dest->buf[len] = '\0';
1446         dest->BufUsed = len;
1447
1448         if (current_token < parmnum) {
1449                 //lprintf (CTDL_DEBUG,"test <!: %s\n", dest);
1450                 return(-1);
1451         }
1452         //lprintf (CTDL_DEBUG,"test <: %d; %s\n", len, dest);
1453         return(len);
1454 }
1455
1456
1457
1458
1459
1460 /**
1461  * @ingroup StrBuf_Tokenizer
1462  * @brief a string tokenizer to fetch an integer
1463  * @param Source String containing tokens
1464  * @param parmnum n'th Parameter to extract
1465  * @param separator tokenizer character
1466  * @returns 0 if not found, else integer representation of the token
1467  */
1468 int StrBufExtract_int(const StrBuf* Source, int parmnum, char separator)
1469 {
1470         StrBuf tmp;
1471         char buf[64];
1472
1473         tmp.buf = buf;
1474         buf[0] = '\0';
1475         tmp.BufSize = 64;
1476         tmp.BufUsed = 0;
1477         tmp.ConstBuf = 1;
1478         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1479                 return(atoi(buf));
1480         else
1481                 return 0;
1482 }
1483
1484 /**
1485  * @ingroup StrBuf_Tokenizer
1486  * @brief a string tokenizer to fetch a long integer
1487  * @param Source String containing tokens
1488  * @param parmnum n'th Parameter to extract
1489  * @param separator tokenizer character
1490  * @returns 0 if not found, else long integer representation of the token
1491  */
1492 long StrBufExtract_long(const StrBuf* Source, int parmnum, char separator)
1493 {
1494         StrBuf tmp;
1495         char buf[64];
1496
1497         tmp.buf = buf;
1498         buf[0] = '\0';
1499         tmp.BufSize = 64;
1500         tmp.BufUsed = 0;
1501         tmp.ConstBuf = 1;
1502         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1503                 return(atoi(buf));
1504         else
1505                 return 0;
1506 }
1507
1508
1509 /**
1510  * @ingroup StrBuf_Tokenizer
1511  * @brief a string tokenizer to fetch an unsigned long
1512  * @param Source String containing tokens
1513  * @param parmnum n'th Parameter to extract
1514  * @param separator tokenizer character
1515  * @returns 0 if not found, else unsigned long representation of the token
1516  */
1517 unsigned long StrBufExtract_unsigned_long(const StrBuf* Source, int parmnum, char separator)
1518 {
1519         StrBuf tmp;
1520         char buf[64];
1521         char *pnum;
1522
1523         tmp.buf = buf;
1524         buf[0] = '\0';
1525         tmp.BufSize = 64;
1526         tmp.BufUsed = 0;
1527         tmp.ConstBuf = 1;
1528         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0) {
1529                 pnum = &buf[0];
1530                 if (*pnum == '-')
1531                         pnum ++;
1532                 return (unsigned long) atol(pnum);
1533         }
1534         else
1535                 return 0;
1536 }
1537
1538
1539
1540 /**
1541  * @ingroup StrBuf_NextTokenizer
1542  * @brief a string tokenizer; Bounds checker
1543  *  function to make shure whether StrBufExtract_NextToken and friends have reached the end of the string.
1544  * @param Source our tokenbuffer
1545  * @param pStart the token iterator pointer to inspect
1546  * @returns whether the revolving pointer is inside of the search range
1547  */
1548 int StrBufHaveNextToken(const StrBuf *Source, const char **pStart)
1549 {
1550         if ((Source == NULL) ||
1551             (*pStart == StrBufNOTNULL) ||
1552             (Source->BufUsed == 0))
1553         {
1554                 return 0;
1555         }
1556         if (*pStart == NULL)
1557         {
1558                 return 1;
1559         }
1560         else if (*pStart > Source->buf + Source->BufUsed)
1561         {
1562                 return 0;
1563         }
1564         else if (*pStart <= Source->buf)
1565         {
1566                 return 0;
1567         }
1568
1569         return 1;
1570 }
1571
1572 /**
1573  * @ingroup StrBuf_NextTokenizer
1574  * @brief a string tokenizer
1575  * @param dest Destination StringBuffer
1576  * @param Source StringBuffer to read into
1577  * @param pStart pointer to the end of the last token. Feed with NULL on start.
1578  * @param separator tokenizer
1579  * @returns -1 if not found, else length of token.
1580  */
1581 int StrBufExtract_NextToken(StrBuf *dest, const StrBuf *Source, const char **pStart, char separator)
1582 {
1583         const char *s;          /* source */
1584         const char *EndBuffer;  /* end stop of source buffer */
1585         int current_token = 0;  /* token currently being processed */
1586         int len = 0;            /* running total length of extracted string */
1587
1588         if ((Source          == NULL) ||
1589             (Source->BufUsed == 0)      )
1590         {
1591                 *pStart = StrBufNOTNULL;
1592                 if (dest != NULL)
1593                         FlushStrBuf(dest);
1594                 return -1;
1595         }
1596
1597         EndBuffer = Source->buf + Source->BufUsed;
1598
1599         if (dest != NULL)
1600         {
1601                 dest->buf[0] = '\0';
1602                 dest->BufUsed = 0;
1603         }
1604         else
1605         {
1606                 *pStart = EndBuffer + 1;
1607                 return -1;
1608         }
1609
1610         if (*pStart == NULL)
1611         {
1612                 *pStart = Source->buf; /* we're starting to examine this buffer. */
1613         }
1614         else if ((*pStart < Source->buf) ||
1615                  (*pStart > EndBuffer  )   )
1616         {
1617                 return -1; /* no more tokens to find. */
1618         }
1619
1620         s = *pStart;
1621         /* start to find the next token */
1622         while ((s <= EndBuffer)      &&
1623                (current_token == 0) )
1624         {
1625                 if (*s == separator)
1626                 {
1627                         /* we found the next token */
1628                         ++current_token;
1629                 }
1630
1631                 if (len >= dest->BufSize)
1632                 {
1633                         /* our Dest-buffer isn't big enough, increase it. */
1634                         dest->BufUsed = len;
1635
1636                         if (IncreaseBuf(dest, 1, -1) < 0) {
1637                                 /* WHUT? no more mem? bail out. */
1638                                 s = EndBuffer;
1639                                 dest->BufUsed --;
1640                                 break;
1641                         }
1642                 }
1643
1644                 if ( (current_token == 0 ) &&   /* are we in our target token? */
1645                      (!IsEmptyStr(s)     ) &&
1646                      (separator     != *s)    ) /* don't copy the token itself */
1647                 {
1648                         dest->buf[len] = *s;    /* Copy the payload */
1649                         ++len;                  /* remember the bigger size. */
1650                 }
1651
1652                 ++s;
1653         }
1654
1655         /* did we reach the end? */
1656         if ((s > EndBuffer)) {
1657                 EndBuffer = StrBufNOTNULL;
1658                 *pStart = EndBuffer;
1659         }
1660         else {
1661                 *pStart = s;  /* remember the position for the next run */
1662         }
1663
1664         /* sanitize our extracted token */
1665         dest->buf[len] = '\0';
1666         dest->BufUsed  = len;
1667
1668         return (len);
1669 }
1670
1671
1672 /**
1673  * @ingroup StrBuf_NextTokenizer
1674  * @brief a string tokenizer
1675  * @param Source StringBuffer to read from
1676  * @param pStart pointer to the end of the last token. Feed with NULL.
1677  * @param separator tokenizer character
1678  * @param nTokens number of tokens to fastforward over
1679  * @returns -1 if not found, else length of token.
1680  */
1681 int StrBufSkip_NTokenS(const StrBuf *Source, const char **pStart, char separator, int nTokens)
1682 {
1683         const char *s, *EndBuffer;      //* source * /
1684         int len = 0;                    //* running total length of extracted string * /
1685         int current_token = 0;          //* token currently being processed * /
1686
1687         if ((Source == NULL) ||
1688             (Source->BufUsed ==0)) {
1689                 return(-1);
1690         }
1691         if (nTokens == 0)
1692                 return Source->BufUsed;
1693
1694         if (*pStart == NULL)
1695                 *pStart = Source->buf;
1696
1697         EndBuffer = Source->buf + Source->BufUsed;
1698
1699         if ((*pStart < Source->buf) ||
1700             (*pStart >  EndBuffer)) {
1701                 return (-1);
1702         }
1703
1704
1705         s = *pStart;
1706
1707         //cit_backtrace();
1708         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1709
1710         while ((s < EndBuffer) && !IsEmptyStr(s)) {
1711                 if (*s == separator) {
1712                         ++current_token;
1713                 }
1714                 if (current_token >= nTokens) {
1715                         break;
1716                 }
1717                 ++s;
1718         }
1719         *pStart = s;
1720         (*pStart) ++;
1721
1722         return(len);
1723 }
1724
1725 /**
1726  * @ingroup StrBuf_NextTokenizer
1727  * @brief a string tokenizer to fetch an integer
1728  * @param Source StringBuffer to read from
1729  * @param pStart Cursor on the tokenstring
1730  * @param separator tokenizer character
1731  * @returns 0 if not found, else integer representation of the token
1732  */
1733 int StrBufExtractNext_int(const StrBuf* Source, const char **pStart, char separator)
1734 {
1735         StrBuf tmp;
1736         char buf[64];
1737
1738         tmp.buf = buf;
1739         buf[0] = '\0';
1740         tmp.BufSize = 64;
1741         tmp.BufUsed = 0;
1742         tmp.ConstBuf = 1;
1743         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1744                 return(atoi(buf));
1745         else
1746                 return 0;
1747 }
1748
1749 /**
1750  * @ingroup StrBuf_NextTokenizer
1751  * @brief a string tokenizer to fetch a long integer
1752  * @param Source StringBuffer to read from
1753  * @param pStart Cursor on the tokenstring
1754  * @param separator tokenizer character
1755  * @returns 0 if not found, else long integer representation of the token
1756  */
1757 long StrBufExtractNext_long(const StrBuf* Source, const char **pStart, char separator)
1758 {
1759         StrBuf tmp;
1760         char buf[64];
1761
1762         tmp.buf = buf;
1763         buf[0] = '\0';
1764         tmp.BufSize = 64;
1765         tmp.BufUsed = 0;
1766         tmp.ConstBuf = 1;
1767         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1768                 return(atoi(buf));
1769         else
1770                 return 0;
1771 }
1772
1773
1774 /**
1775  * @ingroup StrBuf_NextTokenizer
1776  * @brief a string tokenizer to fetch an unsigned long
1777  * @param Source StringBuffer to read from
1778  * @param pStart Cursor on the tokenstring
1779  * @param separator tokenizer character
1780  * @returns 0 if not found, else unsigned long representation of the token
1781  */
1782 unsigned long StrBufExtractNext_unsigned_long(const StrBuf* Source, const char **pStart, char separator)
1783 {
1784         StrBuf tmp;
1785         char buf[64];
1786         char *pnum;
1787
1788         tmp.buf = buf;
1789         buf[0] = '\0';
1790         tmp.BufSize = 64;
1791         tmp.BufUsed = 0;
1792         tmp.ConstBuf = 1;
1793         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0) {
1794                 pnum = &buf[0];
1795                 if (*pnum == '-')
1796                         pnum ++;
1797                 return (unsigned long) atol(pnum);
1798         }
1799         else
1800                 return 0;
1801 }
1802
1803
1804
1805
1806
1807 /*******************************************************************************
1808  *                             Escape Appending                                *
1809  *******************************************************************************/
1810
1811 /**
1812  * @ingroup StrBuf_DeEnCoder
1813  * @brief Escape a string for feeding out as a URL while appending it to a Buffer
1814  * @param OutBuf the output buffer
1815  * @param In Buffer to encode
1816  * @param PlainIn way in from plain old c strings
1817  */
1818 void StrBufUrlescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
1819 {
1820         const char *pch, *pche;
1821         char *pt, *pte;
1822         int len;
1823
1824         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1825                 return;
1826         if (PlainIn != NULL) {
1827                 len = strlen(PlainIn);
1828                 pch = PlainIn;
1829                 pche = pch + len;
1830         }
1831         else {
1832                 pch = In->buf;
1833                 pche = pch + In->BufUsed;
1834                 len = In->BufUsed;
1835         }
1836
1837         if (len == 0)
1838                 return;
1839
1840         pt = OutBuf->buf + OutBuf->BufUsed;
1841         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1842
1843         while (pch < pche) {
1844                 if (pt >= pte) {
1845                         IncreaseBuf(OutBuf, 1, -1);
1846                         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1847                         pt = OutBuf->buf + OutBuf->BufUsed;
1848                 }
1849
1850                 if((*pch >= 'a' && *pch <= 'z') ||
1851                    (*pch >= '@' && *pch <= 'Z') || /* @ A-Z */
1852                    (*pch >= '0' && *pch <= ':') || /* 0-9 : */
1853                    (*pch == '!') || (*pch == '_') ||
1854                    (*pch == ',') || (*pch == '.'))
1855                 {
1856                         *(pt++) = *(pch++);
1857                         OutBuf->BufUsed++;
1858                 }
1859                 else {
1860                         *pt = '%';
1861                         *(pt + 1) = HexList[(unsigned char)*pch][0];
1862                         *(pt + 2) = HexList[(unsigned char)*pch][1];
1863                         pt += 3;
1864                         OutBuf->BufUsed += 3;
1865                         pch ++;
1866                 }
1867         }
1868         *pt = '\0';
1869 }
1870
1871 /**
1872  * @ingroup StrBuf_DeEnCoder
1873  * @brief Escape a string for feeding out as a the username/password part of an URL while appending it to a Buffer
1874  * @param OutBuf the output buffer
1875  * @param In Buffer to encode
1876  * @param PlainIn way in from plain old c strings
1877  */
1878 void StrBufUrlescUPAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
1879 {
1880         const char *pch, *pche;
1881         char *pt, *pte;
1882         int len;
1883
1884         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1885                 return;
1886         if (PlainIn != NULL) {
1887                 len = strlen(PlainIn);
1888                 pch = PlainIn;
1889                 pche = pch + len;
1890         }
1891         else {
1892                 pch = In->buf;
1893                 pche = pch + In->BufUsed;
1894                 len = In->BufUsed;
1895         }
1896
1897         if (len == 0)
1898                 return;
1899
1900         pt = OutBuf->buf + OutBuf->BufUsed;
1901         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1902
1903         while (pch < pche) {
1904                 if (pt >= pte) {
1905                         IncreaseBuf(OutBuf, 1, -1);
1906                         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1907                         pt = OutBuf->buf + OutBuf->BufUsed;
1908                 }
1909
1910                 if((*pch >= 'a' && *pch <= 'z') ||
1911                    (*pch >= 'A' && *pch <= 'Z') || /* A-Z */
1912                    (*pch >= '0' && *pch <= ':') || /* 0-9 : */
1913                    (*pch == '!') || (*pch == '_') ||
1914                    (*pch == ',') || (*pch == '.'))
1915                 {
1916                         *(pt++) = *(pch++);
1917                         OutBuf->BufUsed++;
1918                 }
1919                 else {
1920                         *pt = '%';
1921                         *(pt + 1) = HexList[(unsigned char)*pch][0];
1922                         *(pt + 2) = HexList[(unsigned char)*pch][1];
1923                         pt += 3;
1924                         OutBuf->BufUsed += 3;
1925                         pch ++;
1926                 }
1927         }
1928         *pt = '\0';
1929 }
1930
1931 /**
1932  * @ingroup StrBuf_DeEnCoder
1933  * @brief append a string with characters having a special meaning in xml encoded to the buffer
1934  * @param OutBuf the output buffer
1935  * @param In Buffer to encode
1936  * @param PlainIn way in from plain old c strings
1937  * @param PlainInLen way in from plain old c strings; maybe you've got binary data or know the length?
1938  * @param OverrideLowChars should chars < 0x20 be replaced by _ or escaped as xml entity?
1939  */
1940 void StrBufXMLEscAppend(StrBuf *OutBuf,
1941                         const StrBuf *In,
1942                         const char *PlainIn,
1943                         long PlainInLen,
1944                         int OverrideLowChars)
1945 {
1946         const char *pch, *pche;
1947         char *pt, *pte;
1948         int IsUtf8Sequence;
1949         int len;
1950
1951         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1952                 return;
1953         if (PlainIn != NULL) {
1954                 if (PlainInLen < 0)
1955                         len = strlen((const char*)PlainIn);
1956                 else
1957                         len = PlainInLen;
1958                 pch = PlainIn;
1959                 pche = pch + len;
1960         }
1961         else {
1962                 pch = (const char*)In->buf;
1963                 pche = pch + In->BufUsed;
1964                 len = In->BufUsed;
1965         }
1966
1967         if (len == 0)
1968                 return;
1969
1970         pt = OutBuf->buf + OutBuf->BufUsed;
1971         /**< we max append 6 chars at once plus the \0 */
1972         pte = OutBuf->buf + OutBuf->BufSize - 6;
1973
1974         while (pch < pche) {
1975                 if (pt >= pte) {
1976                         OutBuf->BufUsed = pt - OutBuf->buf;
1977                         IncreaseBuf(OutBuf, 1, -1);
1978                         pte = OutBuf->buf + OutBuf->BufSize - 6;
1979                         /**< we max append 3 chars at once plus the \0 */
1980
1981                         pt = OutBuf->buf + OutBuf->BufUsed;
1982                 }
1983
1984                 if (*pch == '<') {
1985                         memcpy(pt, HKEY("&lt;"));
1986                         pt += 4;
1987                         pch ++;
1988                 }
1989                 else if (*pch == '>') {
1990                         memcpy(pt, HKEY("&gt;"));
1991                         pt += 4;
1992                         pch ++;
1993                 }
1994                 else if (*pch == '&') {
1995                         memcpy(pt, HKEY("&amp;"));
1996                         pt += 5;
1997                         pch++;
1998                 }
1999                 else if ((*pch >= 0x20) && (*pch <= 0x7F)) {
2000                         *pt = *pch;
2001                         pt++; pch++;
2002                 }
2003                 else if (*pch < 0x20) {
2004                         /* we probably shouldn't be doing this */
2005                         if (OverrideLowChars)
2006                         {
2007                                 *pt = '_';
2008                                 pt ++;
2009                                 pch ++;
2010                         }
2011                         else
2012                         {
2013                                 *pt = '&';
2014                                 pt++;
2015                                 *pt = HexList[*(unsigned char*)pch][0];
2016                                 pt ++;
2017                                 *pt = HexList[*(unsigned char*)pch][1];
2018                                 pt ++; pch ++;
2019                                 *pt = '&';
2020                                 pt++;
2021                                 pch ++;
2022                         }
2023                 }
2024                 else {
2025                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(pch, pche);
2026                         if (IsUtf8Sequence)
2027                         {
2028                                 while (IsUtf8Sequence > 0){
2029                                         *pt = *pch;
2030                                         pt ++;
2031                                         pch ++;
2032                                         --IsUtf8Sequence;
2033                                 }
2034                         }
2035                         else
2036                         {
2037                                 *pt = '&';
2038                                 pt++;
2039                                 *pt = HexList[*(unsigned char*)pch][0];
2040                                 pt ++;
2041                                 *pt = HexList[*(unsigned char*)pch][1];
2042                                 pt ++; pch ++;
2043                                 *pt = '&';
2044                                 pt++;
2045                                 pch ++;
2046                         }
2047                 }
2048         }
2049         *pt = '\0';
2050         OutBuf->BufUsed = pt - OutBuf->buf;
2051 }
2052
2053
2054 /**
2055  * @ingroup StrBuf_DeEnCoder
2056  * @brief append a string in hex encoding to the buffer
2057  * @param OutBuf the output buffer
2058  * @param In Buffer to encode
2059  * @param PlainIn way in from plain old c strings
2060  * @param PlainInLen way in from plain old c strings; maybe you've got binary data or know the length?
2061  */
2062 void StrBufHexEscAppend(StrBuf *OutBuf, const StrBuf *In, const unsigned char *PlainIn, long PlainInLen)
2063 {
2064         const unsigned char *pch, *pche;
2065         char *pt, *pte;
2066         int len;
2067
2068         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
2069                 return;
2070         if (PlainIn != NULL) {
2071                 if (PlainInLen < 0)
2072                         len = strlen((const char*)PlainIn);
2073                 else
2074                         len = PlainInLen;
2075                 pch = PlainIn;
2076                 pche = pch + len;
2077         }
2078         else {
2079                 pch = (const unsigned char*)In->buf;
2080                 pche = pch + In->BufUsed;
2081                 len = In->BufUsed;
2082         }
2083
2084         if (len == 0)
2085                 return;
2086
2087         pt = OutBuf->buf + OutBuf->BufUsed;
2088         pte = OutBuf->buf + OutBuf->BufSize - 3; /**< we max append 3 chars at once plus the \0 */
2089
2090         while (pch < pche) {
2091                 if (pt >= pte) {
2092                         IncreaseBuf(OutBuf, 1, -1);
2093                         pte = OutBuf->buf + OutBuf->BufSize - 3; /**< we max append 3 chars at once plus the \0 */
2094                         pt = OutBuf->buf + OutBuf->BufUsed;
2095                 }
2096
2097                 *pt = HexList[*pch][0];
2098                 pt ++;
2099                 *pt = HexList[*pch][1];
2100                 pt ++; pch ++; OutBuf->BufUsed += 2;
2101         }
2102         *pt = '\0';
2103 }
2104
2105 void StrBufBase64Append(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn, long PlainInLen, int linebreaks)
2106 {
2107         const char *pch;
2108         char *pt;
2109         int len;
2110         long ExpectLen;
2111
2112         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
2113                 return;
2114         if (PlainIn != NULL) {
2115                 if (PlainInLen < 0)
2116                         len = strlen(PlainIn);
2117                 else
2118                         len = PlainInLen;
2119                 pch = PlainIn;
2120         }
2121         else {
2122                 pch = In->buf;
2123                 len = In->BufUsed;
2124         }
2125
2126         if (len == 0)
2127                 return;
2128
2129         ExpectLen = ((len * 134) / 100) + OutBuf->BufUsed;
2130
2131         if (ExpectLen > OutBuf->BufSize)
2132                 if (IncreaseBuf(OutBuf, 1, ExpectLen) < ExpectLen)
2133                         return;
2134
2135         pt = OutBuf->buf + OutBuf->BufUsed;
2136
2137         len = CtdlEncodeBase64(pt, pch, len, linebreaks);
2138
2139         pt += len;
2140         OutBuf->BufUsed += len;
2141         *pt = '\0';
2142 }
2143
2144 /**
2145  * @ingroup StrBuf_DeEnCoder
2146  * @brief append a string in hex encoding to the buffer
2147  * @param OutBuf the output buffer
2148  * @param In Buffer to encode
2149  * @param PlainIn way in from plain old c strings
2150  */
2151 void StrBufHexescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
2152 {
2153         StrBufHexEscAppend(OutBuf, In, (const unsigned char*) PlainIn, -1);
2154 }
2155
2156 /**
2157  * @ingroup StrBuf_DeEnCoder
2158  * @brief Append a string, escaping characters which have meaning in HTML.
2159  *
2160  * @param Target        target buffer
2161  * @param Source        source buffer; set to NULL if you just have a C-String
2162  * @param PlainIn       Plain-C string to append; set to NULL if unused
2163  * @param nbsp          If nonzero, spaces are converted to non-breaking spaces.
2164  * @param nolinebreaks  if set to 1, linebreaks are removed from the string.
2165  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
2166  */
2167 long StrEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
2168 {
2169         const char *aptr, *eiptr;
2170         char *bptr, *eptr;
2171         long len;
2172
2173         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2174                 return -1;
2175
2176         if (PlainIn != NULL) {
2177                 aptr = PlainIn;
2178                 len = strlen(PlainIn);
2179                 eiptr = aptr + len;
2180         }
2181         else {
2182                 aptr = Source->buf;
2183                 eiptr = aptr + Source->BufUsed;
2184                 len = Source->BufUsed;
2185         }
2186
2187         if (len == 0)
2188                 return -1;
2189
2190         bptr = Target->buf + Target->BufUsed;
2191         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2192
2193         while (aptr < eiptr){
2194                 if(bptr >= eptr) {
2195                         IncreaseBuf(Target, 1, -1);
2196                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2197                         bptr = Target->buf + Target->BufUsed;
2198                 }
2199                 if (*aptr == '<') {
2200                         memcpy(bptr, "&lt;", 4);
2201                         bptr += 4;
2202                         Target->BufUsed += 4;
2203                 }
2204                 else if (*aptr == '>') {
2205                         memcpy(bptr, "&gt;", 4);
2206                         bptr += 4;
2207                         Target->BufUsed += 4;
2208                 }
2209                 else if (*aptr == '&') {
2210                         memcpy(bptr, "&amp;", 5);
2211                         bptr += 5;
2212                         Target->BufUsed += 5;
2213                 }
2214                 else if (*aptr == '"') {
2215                         memcpy(bptr, "&quot;", 6);
2216                         bptr += 6;
2217                         Target->BufUsed += 6;
2218                 }
2219                 else if (*aptr == '\'') {
2220                         memcpy(bptr, "&#39;", 5);
2221                         bptr += 5;
2222                         Target->BufUsed += 5;
2223                 }
2224                 else if (*aptr == LB) {
2225                         *bptr = '<';
2226                         bptr ++;
2227                         Target->BufUsed ++;
2228                 }
2229                 else if (*aptr == RB) {
2230                         *bptr = '>';
2231                         bptr ++;
2232                         Target->BufUsed ++;
2233                 }
2234                 else if (*aptr == QU) {
2235                         *bptr ='"';
2236                         bptr ++;
2237                         Target->BufUsed ++;
2238                 }
2239                 else if ((*aptr == 32) && (nbsp == 1)) {
2240                         memcpy(bptr, "&nbsp;", 6);
2241                         bptr += 6;
2242                         Target->BufUsed += 6;
2243                 }
2244                 else if ((*aptr == '\n') && (nolinebreaks == 1)) {
2245                         *bptr='\0';     /* nothing */
2246                 }
2247                 else if ((*aptr == '\n') && (nolinebreaks == 2)) {
2248                         memcpy(bptr, "&lt;br/&gt;", 11);
2249                         bptr += 11;
2250                         Target->BufUsed += 11;
2251                 }
2252
2253
2254                 else if ((*aptr == '\r') && (nolinebreaks != 0)) {
2255                         *bptr='\0';     /* nothing */
2256                 }
2257                 else{
2258                         *bptr = *aptr;
2259                         bptr++;
2260                         Target->BufUsed ++;
2261                 }
2262                 aptr ++;
2263         }
2264         *bptr = '\0';
2265         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
2266                 return -1;
2267         return Target->BufUsed;
2268 }
2269
2270 /**
2271  * @ingroup StrBuf_DeEnCoder
2272  * @brief Append a string, escaping characters which have meaning in HTML.
2273  * Converts linebreaks into blanks; escapes single quotes
2274  * @param Target        target buffer
2275  * @param Source        source buffer; set to NULL if you just have a C-String
2276  * @param PlainIn       Plain-C string to append; set to NULL if unused
2277  */
2278 void StrMsgEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2279 {
2280         const char *aptr, *eiptr;
2281         char *tptr, *eptr;
2282         long len;
2283
2284         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2285                 return ;
2286
2287         if (PlainIn != NULL) {
2288                 aptr = PlainIn;
2289                 len = strlen(PlainIn);
2290                 eiptr = aptr + len;
2291         }
2292         else {
2293                 aptr = Source->buf;
2294                 eiptr = aptr + Source->BufUsed;
2295                 len = Source->BufUsed;
2296         }
2297
2298         if (len == 0)
2299                 return;
2300
2301         eptr = Target->buf + Target->BufSize - 8;
2302         tptr = Target->buf + Target->BufUsed;
2303
2304         while (aptr < eiptr){
2305                 if(tptr >= eptr) {
2306                         IncreaseBuf(Target, 1, -1);
2307                         eptr = Target->buf + Target->BufSize - 8;
2308                         tptr = Target->buf + Target->BufUsed;
2309                 }
2310
2311                 if (*aptr == '\n') {
2312                         *tptr = ' ';
2313                         Target->BufUsed++;
2314                 }
2315                 else if (*aptr == '\r') {
2316                         *tptr = ' ';
2317                         Target->BufUsed++;
2318                 }
2319                 else if (*aptr == '\'') {
2320                         *(tptr++) = '&';
2321                         *(tptr++) = '#';
2322                         *(tptr++) = '3';
2323                         *(tptr++) = '9';
2324                         *tptr = ';';
2325                         Target->BufUsed += 5;
2326                 } else {
2327                         *tptr = *aptr;
2328                         Target->BufUsed++;
2329                 }
2330                 tptr++; aptr++;
2331         }
2332         *tptr = '\0';
2333 }
2334
2335
2336
2337 /**
2338  * @ingroup StrBuf_DeEnCoder
2339  * @brief Append a string, escaping characters which have meaning in ICAL.
2340  * [\n,]
2341  * @param Target        target buffer
2342  * @param Source        source buffer; set to NULL if you just have a C-String
2343  * @param PlainIn       Plain-C string to append; set to NULL if unused
2344  */
2345 void StrIcalEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2346 {
2347         const char *aptr, *eiptr;
2348         char *tptr, *eptr;
2349         long len;
2350
2351         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2352                 return ;
2353
2354         if (PlainIn != NULL) {
2355                 aptr = PlainIn;
2356                 len = strlen(PlainIn);
2357                 eiptr = aptr + len;
2358         }
2359         else {
2360                 aptr = Source->buf;
2361                 eiptr = aptr + Source->BufUsed;
2362                 len = Source->BufUsed;
2363         }
2364
2365         if (len == 0)
2366                 return;
2367
2368         eptr = Target->buf + Target->BufSize - 8;
2369         tptr = Target->buf + Target->BufUsed;
2370
2371         while (aptr < eiptr){
2372                 if(tptr + 3 >= eptr) {
2373                         IncreaseBuf(Target, 1, -1);
2374                         eptr = Target->buf + Target->BufSize - 8;
2375                         tptr = Target->buf + Target->BufUsed;
2376                 }
2377
2378                 if (*aptr == '\n') {
2379                         *tptr = '\\';
2380                         Target->BufUsed++;
2381                         tptr++;
2382                         *tptr = 'n';
2383                         Target->BufUsed++;
2384                 }
2385                 else if (*aptr == '\r') {
2386                         *tptr = '\\';
2387                         Target->BufUsed++;
2388                         tptr++;
2389                         *tptr = 'r';
2390                         Target->BufUsed++;
2391                 }
2392                 else if (*aptr == ',') {
2393                         *tptr = '\\';
2394                         Target->BufUsed++;
2395                         tptr++;
2396                         *tptr = ',';
2397                         Target->BufUsed++;
2398                 } else {
2399                         *tptr = *aptr;
2400                         Target->BufUsed++;
2401                 }
2402                 tptr++; aptr++;
2403         }
2404         *tptr = '\0';
2405 }
2406
2407 /**
2408  * @ingroup StrBuf_DeEnCoder
2409  * @brief Append a string, escaping characters which have meaning in JavaScript strings .
2410  *
2411  * @param Target        target buffer
2412  * @param Source        source buffer; set to NULL if you just have a C-String
2413  * @param PlainIn       Plain-C string to append; set to NULL if unused
2414  * @returns size of result or -1
2415  */
2416 long StrECMAEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2417 {
2418         const char *aptr, *eiptr;
2419         char *bptr, *eptr;
2420         long len;
2421         int IsUtf8Sequence;
2422
2423         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2424                 return -1;
2425
2426         if (PlainIn != NULL) {
2427                 aptr = PlainIn;
2428                 len = strlen(PlainIn);
2429                 eiptr = aptr + len;
2430         }
2431         else {
2432                 aptr = Source->buf;
2433                 eiptr = aptr + Source->BufUsed;
2434                 len = Source->BufUsed;
2435         }
2436
2437         if (len == 0)
2438                 return -1;
2439
2440         bptr = Target->buf + Target->BufUsed;
2441         eptr = Target->buf + Target->BufSize - 7; /* our biggest unit to put in...  */
2442
2443         while (aptr < eiptr){
2444                 if(bptr >= eptr) {
2445                         IncreaseBuf(Target, 1, -1);
2446                         eptr = Target->buf + Target->BufSize - 7; /* our biggest unit to put in...  */
2447                         bptr = Target->buf + Target->BufUsed;
2448                 }
2449                 switch (*aptr) {
2450                 case '\n':
2451                         memcpy(bptr, HKEY("\\n"));
2452                         bptr += 2;
2453                         Target->BufUsed += 2;
2454                         break;
2455                 case '\r':
2456                         memcpy(bptr, HKEY("\\r"));
2457                         bptr += 2;
2458                         Target->BufUsed += 2;
2459                         break;
2460                 case '"':
2461                         *bptr = '\\';
2462                         bptr ++;
2463                         *bptr = '"';
2464                         bptr ++;
2465                         Target->BufUsed += 2;
2466                         break;
2467                 case '\\':
2468                         if ((*(aptr + 1) == 'u') &&
2469                             isxdigit(*(aptr + 2)) &&
2470                             isxdigit(*(aptr + 3)) &&
2471                             isxdigit(*(aptr + 4)) &&
2472                             isxdigit(*(aptr + 5)))
2473                         { /* oh, a unicode escaper. let it pass through. */
2474                                 memcpy(bptr, aptr, 6);
2475                                 aptr += 5;
2476                                 bptr +=6;
2477                                 Target->BufUsed += 6;
2478                         }
2479                         else
2480                         {
2481                                 *bptr = '\\';
2482                                 bptr ++;
2483                                 *bptr = '\\';
2484                                 bptr ++;
2485                                 Target->BufUsed += 2;
2486                         }
2487                         break;
2488                 case '\b':
2489                         *bptr = '\\';
2490                         bptr ++;
2491                         *bptr = 'b';
2492                         bptr ++;
2493                         Target->BufUsed += 2;
2494                         break;
2495                 case '\f':
2496                         *bptr = '\\';
2497                         bptr ++;
2498                         *bptr = 'f';
2499                         bptr ++;
2500                         Target->BufUsed += 2;
2501                         break;
2502                 case '\t':
2503                         *bptr = '\\';
2504                         bptr ++;
2505                         *bptr = 't';
2506                         bptr ++;
2507                         Target->BufUsed += 2;
2508                         break;
2509                 default:
2510                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(aptr, eiptr);
2511                         while (IsUtf8Sequence > 0){
2512                                 *bptr = *aptr;
2513                                 Target->BufUsed ++;
2514                                 if (--IsUtf8Sequence)
2515                                         aptr++;
2516                                 bptr++;
2517                         }
2518                 }
2519                 aptr ++;
2520         }
2521         *bptr = '\0';
2522         if ((bptr == eptr - 1 ) && !IsEmptyStr(aptr) )
2523                 return -1;
2524         return Target->BufUsed;
2525 }
2526
2527 /**
2528  * @ingroup StrBuf_DeEnCoder
2529  * @brief Append a string, escaping characters which have meaning in HTML + json.
2530  *
2531  * @param Target        target buffer
2532  * @param Source        source buffer; set to NULL if you just have a C-String
2533  * @param PlainIn       Plain-C string to append; set to NULL if unused
2534  * @param nbsp          If nonzero, spaces are converted to non-breaking spaces.
2535  * @param nolinebreaks  if set to 1, linebreaks are removed from the string.
2536  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
2537  */
2538 long StrHtmlEcmaEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
2539 {
2540         const char *aptr, *eiptr;
2541         char *bptr, *eptr;
2542         long len;
2543         int IsUtf8Sequence = 0;
2544
2545         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2546                 return -1;
2547
2548         if (PlainIn != NULL) {
2549                 aptr = PlainIn;
2550                 len = strlen(PlainIn);
2551                 eiptr = aptr + len;
2552         }
2553         else {
2554                 aptr = Source->buf;
2555                 eiptr = aptr + Source->BufUsed;
2556                 len = Source->BufUsed;
2557         }
2558
2559         if (len == 0)
2560                 return -1;
2561
2562         bptr = Target->buf + Target->BufUsed;
2563         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2564
2565         while (aptr < eiptr){
2566                 if(bptr >= eptr) {
2567                         IncreaseBuf(Target, 1, -1);
2568                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2569                         bptr = Target->buf + Target->BufUsed;
2570                 }
2571                 switch (*aptr) {
2572                 case '<':
2573                         memcpy(bptr, HKEY("&lt;"));
2574                         bptr += 4;
2575                         Target->BufUsed += 4;
2576                         break;
2577                 case '>':
2578                         memcpy(bptr, HKEY("&gt;"));
2579                         bptr += 4;
2580                         Target->BufUsed += 4;
2581                         break;
2582                 case '&':
2583                         memcpy(bptr, HKEY("&amp;"));
2584                         bptr += 5;
2585                         Target->BufUsed += 5;
2586                         break;
2587                 case LB:
2588                         *bptr = '<';
2589                         bptr ++;
2590                         Target->BufUsed ++;
2591                         break;
2592                 case RB:
2593                         *bptr = '>';
2594                         bptr ++;
2595                         Target->BufUsed ++;
2596                         break;
2597                 case '\n':
2598                         switch (nolinebreaks) {
2599                         case 1:
2600                                 *bptr='\0';     /* nothing */
2601                                 break;
2602                         case 2:
2603                                 memcpy(bptr, HKEY("&lt;br/&gt;"));
2604                                 bptr += 11;
2605                                 Target->BufUsed += 11;
2606                                 break;
2607                         default:
2608                                 memcpy(bptr, HKEY("\\n"));
2609                                 bptr += 2;
2610                                 Target->BufUsed += 2;
2611                         }
2612                         break;
2613                 case '\r':
2614                         switch (nolinebreaks) {
2615                         case 1:
2616                         case 2:
2617                                 *bptr='\0';     /* nothing */
2618                                 break;
2619                         default:
2620                                 memcpy(bptr, HKEY("\\r"));
2621                                 bptr += 2;
2622                                 Target->BufUsed += 2;
2623                                 break;
2624                         }
2625                         break;
2626                 case '"':
2627                 case QU:
2628                         *bptr = '\\';
2629                         bptr ++;
2630                         *bptr = '"';
2631                         bptr ++;
2632                         Target->BufUsed += 2;
2633                         break;
2634                 case '\\':
2635                         if ((*(aptr + 1) == 'u') &&
2636                             isxdigit(*(aptr + 2)) &&
2637                             isxdigit(*(aptr + 3)) &&
2638                             isxdigit(*(aptr + 4)) &&
2639                             isxdigit(*(aptr + 5)))
2640                         { /* oh, a unicode escaper. let it pass through. */
2641                                 memcpy(bptr, aptr, 6);
2642                                 aptr += 5;
2643                                 bptr +=6;
2644                                 Target->BufUsed += 6;
2645                         }
2646                         else
2647                         {
2648                                 *bptr = '\\';
2649                                 bptr ++;
2650                                 *bptr = '\\';
2651                                 bptr ++;
2652                                 Target->BufUsed += 2;
2653                         }
2654                         break;
2655                 case '\b':
2656                         *bptr = '\\';
2657                         bptr ++;
2658                         *bptr = 'b';
2659                         bptr ++;
2660                         Target->BufUsed += 2;
2661                         break;
2662                 case '\f':
2663                         *bptr = '\\';
2664                         bptr ++;
2665                         *bptr = 'f';
2666                         bptr ++;
2667                         Target->BufUsed += 2;
2668                         break;
2669                 case '\t':
2670                         *bptr = '\\';
2671                         bptr ++;
2672                         *bptr = 't';
2673                         bptr ++;
2674                         Target->BufUsed += 2;
2675                         break;
2676                 case  32:
2677                         if (nbsp == 1) {
2678                                 memcpy(bptr, HKEY("&nbsp;"));
2679                                 bptr += 6;
2680                                 Target->BufUsed += 6;
2681                                 break;
2682                         }
2683                 default:
2684                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(aptr, eiptr);
2685                         while (IsUtf8Sequence > 0){
2686                                 *bptr = *aptr;
2687                                 Target->BufUsed ++;
2688                                 if (--IsUtf8Sequence)
2689                                         aptr++;
2690                                 bptr++;
2691                         }
2692                 }
2693                 aptr ++;
2694         }
2695         *bptr = '\0';
2696         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
2697                 return -1;
2698         return Target->BufUsed;
2699 }
2700
2701
2702 /**
2703  * @ingroup StrBuf_DeEnCoder
2704  * @brief replace all non-Ascii characters by another
2705  * @param Buf buffer to inspect
2706  * @param repl charater to stamp over non ascii chars
2707  */
2708 void StrBufAsciify(StrBuf *Buf, const char repl)
2709 {
2710         long offset;
2711
2712         for (offset = 0; offset < Buf->BufUsed; offset ++)
2713                 if (!isascii(Buf->buf[offset]))
2714                         Buf->buf[offset] = repl;
2715
2716 }
2717
2718 /**
2719  * @ingroup StrBuf_DeEnCoder
2720  * @brief unhide special chars hidden to the HTML escaper
2721  * @param target buffer to put the unescaped string in
2722  * @param source buffer to unescape
2723  */
2724 void StrBufEUid_unescapize(StrBuf *target, const StrBuf *source)
2725 {
2726         int a, b, len;
2727         char hex[3];
2728
2729         if ((source == NULL) || (target == NULL) || (target->buf == NULL))
2730         {
2731                 return;
2732         }
2733
2734         if (target != NULL)
2735                 FlushStrBuf(target);
2736
2737         len = source->BufUsed;
2738         for (a = 0; a < len; ++a) {
2739                 if (target->BufUsed >= target->BufSize)
2740                         IncreaseBuf(target, 1, -1);
2741
2742                 if (source->buf[a] == '=') {
2743                         hex[0] = source->buf[a + 1];
2744                         hex[1] = source->buf[a + 2];
2745                         hex[2] = 0;
2746                         b = 0;
2747                         sscanf(hex, "%02x", &b);
2748                         target->buf[target->BufUsed] = b;
2749                         target->buf[++target->BufUsed] = 0;
2750                         a += 2;
2751                 }
2752                 else {
2753                         target->buf[target->BufUsed] = source->buf[a];
2754                         target->buf[++target->BufUsed] = 0;
2755                 }
2756         }
2757 }
2758
2759
2760 /**
2761  * @ingroup StrBuf_DeEnCoder
2762  * @brief hide special chars from the HTML escapers and friends
2763  * @param target buffer to put the escaped string in
2764  * @param source buffer to escape
2765  */
2766 void StrBufEUid_escapize(StrBuf *target, const StrBuf *source)
2767 {
2768         int i, len;
2769
2770         if (target != NULL)
2771                 FlushStrBuf(target);
2772
2773         if ((source == NULL) || (target == NULL) || (target->buf == NULL))
2774         {
2775                 return;
2776         }
2777
2778         len = source->BufUsed;
2779         for (i=0; i<len; ++i) {
2780                 if (target->BufUsed + 4 >= target->BufSize)
2781                         IncreaseBuf(target, 1, -1);
2782                 if ( (isalnum(source->buf[i])) ||
2783                      (source->buf[i]=='-') ||
2784                      (source->buf[i]=='_') ) {
2785                         target->buf[target->BufUsed++] = source->buf[i];
2786                 }
2787                 else {
2788                         sprintf(&target->buf[target->BufUsed],
2789                                 "=%02X",
2790                                 (0xFF &source->buf[i]));
2791                         target->BufUsed += 3;
2792                 }
2793         }
2794         target->buf[target->BufUsed + 1] = '\0';
2795 }
2796
2797
2798 /*******************************************************************************
2799  *                      Quoted Printable de/encoding                           *
2800  *******************************************************************************/
2801
2802 /**
2803  * @ingroup StrBuf_DeEnCoder
2804  * @brief decode a buffer from base 64 encoding; destroys original
2805  * @param Buf Buffor to transform
2806  */
2807 int StrBufDecodeBase64(StrBuf *Buf)
2808 {
2809         char *xferbuf;
2810         size_t siz;
2811
2812         if (Buf == NULL)
2813                 return -1;
2814
2815         xferbuf = (char*) malloc(Buf->BufSize);
2816         if (xferbuf == NULL)
2817                 return -1;
2818
2819         *xferbuf = '\0';
2820         siz = CtdlDecodeBase64(xferbuf,
2821                                Buf->buf,
2822                                Buf->BufUsed);
2823         free(Buf->buf);
2824         Buf->buf = xferbuf;
2825         Buf->BufUsed = siz;
2826         return siz;
2827 }
2828
2829 /**
2830  * @ingroup StrBuf_DeEnCoder
2831  * @brief decode a buffer from base 64 encoding; expects targetbuffer
2832  * @param BufIn Buffor to transform
2833  * @param BufOut Buffer to put result into
2834  */
2835 int StrBufDecodeBase64To(const StrBuf *BufIn, StrBuf *BufOut)
2836 {
2837         if ((BufIn == NULL) || (BufOut == NULL))
2838                 return -1;
2839
2840         if (BufOut->BufSize < BufIn->BufUsed)
2841                 IncreaseBuf(BufOut, BufIn->BufUsed, 0);
2842
2843         BufOut->BufUsed = CtdlDecodeBase64(BufOut->buf,
2844                                            BufIn->buf,
2845                                            BufIn->BufUsed);
2846         return BufOut->BufUsed;
2847 }
2848
2849 void *StrBufNewStreamContext(eStreamType type)
2850 {
2851         base64_decodestate *state;;
2852
2853         switch (type)
2854         {
2855         case eBase64Decode:
2856                 state = (base64_decodestate*) malloc(sizeof(base64_decodestate));
2857                 base64_init_decodestate(state);
2858                 return state;
2859                 break;
2860         }
2861         return NULL;
2862 }
2863
2864 void StrBufDestroyStreamContext(eStreamType type, void **Stream)
2865 {
2866         switch (type)
2867         {
2868         case eBase64Decode:
2869                 free(*Stream);
2870                 *Stream = NULL;
2871                 break;
2872         }
2873 }
2874 void StrBufStreamDecodeTo(StrBuf *Target, const StrBuf *In, const char* pIn, long pInLen, void *Stream)
2875 {
2876         base64_decodestate *state = Stream;
2877         long ExpectLen;
2878
2879         if (In != NULL)
2880         {
2881                 pIn = In->buf;
2882                 pInLen = In->BufUsed;
2883         }
2884         if ((pIn == NULL) || (Stream == NULL))
2885                 return;
2886
2887         ExpectLen = (pInLen / 4) * 3;
2888
2889         if (Target->BufSize - Target->BufUsed < ExpectLen)
2890         {
2891                 IncreaseBuf(Target, 1, Target->BufUsed + ExpectLen + 1);
2892         }
2893
2894         ExpectLen = base64_decode_block(pIn, pInLen, Target->buf + Target->BufUsed, state);
2895         Target->BufUsed += ExpectLen;
2896         Target->buf[Target->BufUsed] = '\0';
2897 }
2898
2899 /**
2900  * @ingroup StrBuf_DeEnCoder
2901  * @brief decode a buffer from base 64 encoding; destroys original
2902  * @param Buf Buffor to transform
2903  */
2904 int StrBufDecodeHex(StrBuf *Buf)
2905 {
2906         unsigned int ch;
2907         char *pch, *pche, *pchi;
2908
2909         if (Buf == NULL) return -1;
2910
2911         pch = pchi = Buf->buf;
2912         pche = pch + Buf->BufUsed;
2913
2914         while (pchi < pche){
2915                 ch = decode_hex(pchi);
2916                 *pch = ch;
2917                 pch ++;
2918                 pchi += 2;
2919         }
2920
2921         *pch = '\0';
2922         Buf->BufUsed = pch - Buf->buf;
2923         return Buf->BufUsed;
2924 }
2925
2926 /**
2927  * @ingroup StrBuf_DeEnCoder
2928  * @brief replace all chars >0x20 && < 0x7F with Mute
2929  * @param Mute char to put over invalid chars
2930  * @param Buf Buffor to transform
2931  */
2932 int StrBufSanitizeAscii(StrBuf *Buf, const char Mute)
2933 {
2934         unsigned char *pch;
2935
2936         if (Buf == NULL) return -1;
2937         pch = (unsigned char *)Buf->buf;
2938         while (pch < (unsigned char *)Buf->buf + Buf->BufUsed) {
2939                 if ((*pch < 0x20) || (*pch > 0x7F))
2940                         *pch = Mute;
2941                 pch ++;
2942         }
2943         return Buf->BufUsed;
2944 }
2945
2946
2947 /**
2948  * @ingroup StrBuf_DeEnCoder
2949  * @brief remove escaped strings from i.e. the url string (like %20 for blanks)
2950  * @param Buf Buffer to translate
2951  * @param StripBlanks Reduce several blanks to one?
2952  */
2953 long StrBufUnescape(StrBuf *Buf, int StripBlanks)
2954 {
2955         int a, b;
2956         char hex[3];
2957         long len;
2958
2959         if (Buf == NULL)
2960                 return -1;
2961
2962         while ((Buf->BufUsed > 0) && (isspace(Buf->buf[Buf->BufUsed - 1]))){
2963                 Buf->buf[Buf->BufUsed - 1] = '\0';
2964                 Buf->BufUsed --;
2965         }
2966
2967         a = 0;
2968         while (a < Buf->BufUsed) {
2969                 if (Buf->buf[a] == '+')
2970                         Buf->buf[a] = ' ';
2971                 else if (Buf->buf[a] == '%') {
2972                         /* don't let % chars through, rather truncate the input. */
2973                         if (a + 2 > Buf->BufUsed) {
2974                                 Buf->buf[a] = '\0';
2975                                 Buf->BufUsed = a;
2976                         }
2977                         else {
2978                                 hex[0] = Buf->buf[a + 1];
2979                                 hex[1] = Buf->buf[a + 2];
2980                                 hex[2] = 0;
2981                                 b = 0;
2982                                 sscanf(hex, "%02x", &b);
2983                                 Buf->buf[a] = (char) b;
2984                                 len = Buf->BufUsed - a - 2;
2985                                 if (len > 0)
2986                                         memmove(&Buf->buf[a + 1], &Buf->buf[a + 3], len);
2987
2988                                 Buf->BufUsed -=2;
2989                         }
2990                 }
2991                 a++;
2992         }
2993         return a;
2994 }
2995
2996
2997 /**
2998  * @ingroup StrBuf_DeEnCoder
2999  * @brief       RFC2047-encode a header field if necessary.
3000  *              If no non-ASCII characters are found, the string
3001  *              will be copied verbatim without encoding.
3002  *
3003  * @param       target          Target buffer.
3004  * @param       source          Source string to be encoded.
3005  * @returns     encoded length; -1 if non success.
3006  */
3007 int StrBufRFC2047encode(StrBuf **target, const StrBuf *source)
3008 {
3009         const char headerStr[] = "=?UTF-8?Q?";
3010         int need_to_encode = 0;
3011         int i = 0;
3012         unsigned char ch;
3013
3014         if ((source == NULL) ||
3015             (target == NULL))
3016             return -1;
3017
3018         while ((i < source->BufUsed) &&
3019                (!IsEmptyStr (&source->buf[i])) &&
3020                (need_to_encode == 0)) {
3021                 if (((unsigned char) source->buf[i] < 32) ||
3022                     ((unsigned char) source->buf[i] > 126)) {
3023                         need_to_encode = 1;
3024                 }
3025                 i++;
3026         }
3027
3028         if (!need_to_encode) {
3029                 if (*target == NULL) {
3030                         *target = NewStrBufPlain(source->buf, source->BufUsed);
3031                 }
3032                 else {
3033                         FlushStrBuf(*target);
3034                         StrBufAppendBuf(*target, source, 0);
3035                 }
3036                 if (*target != 0)
3037                         return (*target)->BufUsed;
3038                 else
3039                         return 0;
3040         }
3041         if (*target == NULL)
3042                 *target = NewStrBufPlain(NULL, sizeof(headerStr) + source->BufUsed * 2);
3043         else if (sizeof(headerStr) + source->BufUsed >= (*target)->BufSize)
3044                 IncreaseBuf(*target, sizeof(headerStr) + source->BufUsed, 0);
3045         memcpy ((*target)->buf, headerStr, sizeof(headerStr) - 1);
3046         (*target)->BufUsed = sizeof(headerStr) - 1;
3047         for (i=0; (i < source->BufUsed); ++i) {
3048                 if ((*target)->BufUsed + 4 >= (*target)->BufSize)
3049                         IncreaseBuf(*target, 1, 0);
3050                 ch = (unsigned char) source->buf[i];
3051                 if ((ch  <  32) ||
3052                     (ch  > 126) ||
3053                     (ch == '=') ||
3054                     (ch == '?') ||
3055                     (ch == '_') ||
3056                     (ch == '[') ||
3057                     (ch == ']')   )
3058                 {
3059                         sprintf(&(*target)->buf[(*target)->BufUsed], "=%02X", ch);
3060                         (*target)->BufUsed += 3;
3061                 }
3062                 else {
3063                         if (ch == ' ')
3064                                 (*target)->buf[(*target)->BufUsed] = '_';
3065                         else
3066                                 (*target)->buf[(*target)->BufUsed] = ch;
3067                         (*target)->BufUsed++;
3068                 }
3069         }
3070
3071         if ((*target)->BufUsed + 4 >= (*target)->BufSize)
3072                 IncreaseBuf(*target, 1, 0);
3073
3074         (*target)->buf[(*target)->BufUsed++] = '?';
3075         (*target)->buf[(*target)->BufUsed++] = '=';
3076         (*target)->buf[(*target)->BufUsed] = '\0';
3077         return (*target)->BufUsed;;
3078 }
3079
3080 /**
3081  * @ingroup StrBuf_DeEnCoder
3082  * @brief       Quoted-Printable encode a message; make it < 80 columns width.
3083  * @param       source          Source string to be encoded.
3084  * @returns     buffer with encoded message.
3085  */
3086 StrBuf *StrBufRFC2047encodeMessage(const StrBuf *EncodeMe)
3087 {
3088         StrBuf *OutBuf;
3089         char *Optr, *OEptr;
3090         const char *ptr, *eptr;
3091         unsigned char ch;
3092         int LinePos;
3093
3094         OutBuf = NewStrBufPlain(NULL, StrLength(EncodeMe) * 4);
3095         Optr = OutBuf->buf;
3096         OEptr = OutBuf->buf + OutBuf->BufSize;
3097         ptr = EncodeMe->buf;
3098         eptr = EncodeMe->buf + EncodeMe->BufUsed;
3099         LinePos = 0;
3100
3101         while (ptr < eptr)
3102         {
3103                 if (Optr + 4 >= OEptr)
3104                 {
3105                         long Offset;
3106                         Offset = Optr - OutBuf->buf;
3107                         OutBuf->BufUsed = Optr - OutBuf->buf;
3108                         IncreaseBuf(OutBuf, 1, 0);
3109                         Optr = OutBuf->buf + Offset;
3110                         OEptr = OutBuf->buf + OutBuf->BufSize;
3111                 }
3112                 if (*ptr == '\r')
3113                 {
3114                         /* ignore carriage returns */
3115                         ptr ++;
3116                 }
3117                 else if (*ptr == '\n') {
3118                         /* hard line break */
3119                         memcpy(Optr, HKEY("=0A"));
3120                         Optr += 3;
3121                         LinePos += 3;
3122                         ptr ++;
3123                 }
3124                 else if (( (*ptr >= 32) && (*ptr <= 60) ) ||
3125                          ( (*ptr >= 62) && (*ptr <= 126) ))
3126                 {
3127                         *Optr = *ptr;
3128                         Optr ++;
3129                         ptr ++;
3130                         LinePos ++;
3131                 }
3132                 else {
3133                         ch = *ptr;
3134                         *Optr = '=';
3135                         Optr ++;
3136                         *Optr = HexList[ch][0];
3137                         Optr ++;
3138                         *Optr = HexList[ch][1];
3139                         Optr ++;
3140                         LinePos += 3;
3141                         ptr ++;
3142                 }
3143
3144                 if (LinePos > 72) {
3145                         /* soft line break */
3146                         if (isspace(*(Optr - 1))) {
3147                                 ch = *(Optr - 1);
3148                                 Optr --;
3149                                 *Optr = '=';
3150                                 Optr ++;
3151                                 *Optr = HexList[ch][0];
3152                                 Optr ++;
3153                                 *Optr = HexList[ch][1];
3154                                 Optr ++;
3155                                 LinePos += 3;
3156                         }
3157                         *Optr = '=';
3158                         Optr ++;
3159                         *Optr = '\n';
3160                         Optr ++;
3161                         LinePos = 0;
3162                 }
3163         }
3164         *Optr = '\0';
3165         OutBuf->BufUsed = Optr - OutBuf->buf;
3166
3167         return OutBuf;
3168 }
3169
3170
3171 static void AddRecipient(StrBuf *Target,
3172                          StrBuf *UserName,
3173                          StrBuf *EmailAddress,
3174                          StrBuf *EncBuf)
3175 {
3176         int QuoteMe = 0;
3177
3178         if (StrLength(Target) > 0) StrBufAppendBufPlain(Target, HKEY(", "), 0);
3179         if (strchr(ChrPtr(UserName), ',') != NULL) QuoteMe = 1;
3180
3181         if (QuoteMe)  StrBufAppendBufPlain(Target, HKEY("\""), 0);
3182         StrBufRFC2047encode(&EncBuf, UserName);
3183         StrBufAppendBuf(Target, EncBuf, 0);
3184         if (QuoteMe)  StrBufAppendBufPlain(Target, HKEY("\" "), 0);
3185         else          StrBufAppendBufPlain(Target, HKEY(" "), 0);
3186
3187         if (StrLength(EmailAddress) > 0){
3188                 StrBufAppendBufPlain(Target, HKEY("<"), 0);
3189                 StrBufAppendBuf(Target, EmailAddress, 0); /* TODO: what about IDN???? */
3190                 StrBufAppendBufPlain(Target, HKEY(">"), 0);
3191         }
3192 }
3193
3194
3195 /**
3196  * \brief QP encode parts of an email TO/CC/BCC vector, and strip/filter invalid parts
3197  * \param Recp Source list of email recipients
3198  * \param UserName Temporary buffer for internal use; Please provide valid buffer.
3199  * \param EmailAddress Temporary buffer for internal use; Please provide valid buffer.
3200  * \param EncBuf Temporary buffer for internal use; Please provide valid buffer.
3201  * \returns encoded & sanitized buffer with the contents of Recp; Caller owns this memory.
3202  */
3203 StrBuf *StrBufSanitizeEmailRecipientVector(const StrBuf *Recp,
3204                                            StrBuf *UserName,
3205                                            StrBuf *EmailAddress,
3206                                            StrBuf *EncBuf)
3207 {
3208         StrBuf *Target;
3209         const char *pch, *pche;
3210         const char *UserStart, *UserEnd, *EmailStart, *EmailEnd, *At;
3211
3212         if ((Recp == NULL) || (StrLength(Recp) == 0))
3213                 return NULL;
3214
3215         pch = ChrPtr(Recp);
3216         pche = pch + StrLength(Recp);
3217
3218         if (!CheckEncode(pch, -1, pche))
3219                 return NewStrBufDup(Recp);
3220
3221         Target = NewStrBufPlain(NULL, StrLength(Recp));
3222
3223         while ((pch != NULL) && (pch < pche))
3224         {
3225                 while (isspace(*pch)) pch++;
3226                 UserEnd = EmailStart = EmailEnd = NULL;
3227
3228                 if ((*pch == '"') || (*pch == '\'')) {
3229                         UserStart = pch + 1;
3230
3231                         UserEnd = strchr(UserStart, *pch);
3232                         if (UserEnd == NULL)
3233                                 break; ///TODO: Userfeedback??
3234                         EmailStart = UserEnd + 1;
3235                         while (isspace(*EmailStart))
3236                                 EmailStart++;
3237                         if (UserEnd == UserStart) {
3238                                 UserStart = UserEnd = NULL;
3239                         }
3240
3241                         if (*EmailStart == '<') {
3242                                 EmailStart++;
3243                                 EmailEnd = strchr(EmailStart, '>');
3244                                 if (EmailEnd == NULL)
3245                                         EmailEnd = strchr(EmailStart, ',');
3246
3247                         }
3248                         else {
3249                                 EmailEnd = strchr(EmailStart, ',');
3250                         }
3251                         if (EmailEnd == NULL)
3252                                 EmailEnd = pche;
3253                         pch = EmailEnd + 1;
3254                 }
3255                 else {
3256                         int gt = 0;
3257                         UserStart = pch;
3258                         EmailEnd = strchr(UserStart, ',');
3259                         if (EmailEnd == NULL) {
3260                                 EmailEnd = strchr(pch, '>');
3261                                 pch = NULL;
3262                                 if (EmailEnd != NULL) {
3263                                         gt = 1;
3264                                 }
3265                                 else {
3266                                         EmailEnd = pche;
3267                                 }
3268                         }
3269                         else {
3270
3271                                 pch = EmailEnd + 1;
3272                                 while ((EmailEnd > UserStart) && !gt &&
3273                                        ((*EmailEnd == ',') ||
3274                                         (*EmailEnd == '>') ||
3275                                         (isspace(*EmailEnd))))
3276                                 {
3277                                         if (*EmailEnd == '>')
3278                                                 gt = 1;
3279                                         else
3280                                                 EmailEnd--;
3281                                 }
3282                                 if (EmailEnd == UserStart)
3283                                         break;
3284                         }
3285                         if (gt) {
3286                                 EmailStart = strchr(UserStart, '<');
3287                                 if ((EmailStart == NULL) || (EmailStart > EmailEnd))
3288                                         break;
3289                                 UserEnd = EmailStart;
3290
3291                                 while ((UserEnd > UserStart) &&
3292                                        isspace (*(UserEnd - 1)))
3293                                         UserEnd --;
3294                                 EmailStart ++;
3295                                 if (UserStart >= UserEnd)
3296                                         UserStart = UserEnd = NULL;
3297                         }
3298                         else { /* this is a local recipient... no domain, just a realname */
3299                                 EmailStart = UserStart;
3300                                 At = strchr(EmailStart, '@');
3301                                 if (At == NULL) {
3302                                         UserEnd = EmailEnd;
3303                                         EmailEnd = NULL;
3304                                 }
3305                                 else {
3306                                         EmailStart = UserStart;
3307                                         UserStart = NULL;
3308                                 }
3309                         }
3310                 }
3311
3312                 if ((UserStart != NULL) && (UserEnd != NULL))
3313                         StrBufPlain(UserName, UserStart, UserEnd - UserStart);
3314                 else if ((UserStart != NULL) && (UserEnd == NULL))
3315                         StrBufPlain(UserName, UserStart, UserEnd - UserStart);
3316                 else
3317                         FlushStrBuf(UserName);
3318
3319                 if ((EmailStart != NULL) && (EmailEnd != NULL))
3320                         StrBufPlain(EmailAddress, EmailStart, EmailEnd - EmailStart);
3321                 else if ((EmailStart != NULL) && (EmailEnd == NULL))
3322                         StrBufPlain(EmailAddress, EmailStart, EmailEnd - pche);
3323                 else
3324                         FlushStrBuf(EmailAddress);
3325
3326                 AddRecipient(Target, UserName, EmailAddress, EncBuf);
3327
3328                 if (pch == NULL)
3329                         break;
3330
3331                 if ((pch != NULL) && (*pch == ','))
3332                         pch ++;
3333                 if (pch != NULL) while (isspace(*pch))
3334                         pch ++;
3335         }
3336         return Target;
3337 }
3338
3339
3340 /**
3341  * @ingroup StrBuf
3342  * @brief replaces all occurances of 'search' by 'replace'
3343  * @param buf Buffer to modify
3344  * @param search character to search
3345  * @param replace character to replace search by
3346  */
3347 void StrBufReplaceChars(StrBuf *buf, char search, char replace)
3348 {
3349         long i;
3350         if (buf == NULL)
3351                 return;
3352         for (i=0; i<buf->BufUsed; i++)
3353                 if (buf->buf[i] == search)
3354                         buf->buf[i] = replace;
3355
3356 }
3357
3358 /**
3359  * @ingroup StrBuf
3360  * @brief removes all \\r s from the string, or replaces them with \n if its not a combination of both.
3361  * @param buf Buffer to modify
3362  */
3363 void StrBufToUnixLF(StrBuf *buf)
3364 {
3365         char *pche, *pchS, *pchT;
3366         if (buf == NULL)
3367                 return;
3368
3369         pche = buf->buf + buf->BufUsed;
3370         pchS = pchT = buf->buf;
3371         while (pchS < pche)
3372         {
3373                 if (*pchS == '\r')
3374                 {
3375                         pchS ++;
3376                         if (*pchS != '\n') {
3377                                 *pchT = '\n';
3378                                 pchT++;
3379                         }
3380                 }
3381                 *pchT = *pchS;
3382                 pchT++; pchS++;
3383         }
3384         *pchT = '\0';
3385         buf->BufUsed = pchT - buf->buf;
3386 }
3387
3388
3389 /*******************************************************************************
3390  *                 Iconv Wrapper; RFC822 de/encoding                           *
3391  *******************************************************************************/
3392
3393 /**
3394  * @ingroup StrBuf_DeEnCoder
3395  * @brief Wrapper around iconv_open()
3396  * Our version adds aliases for non-standard Microsoft charsets
3397  * such as 'MS950', aliasing them to names like 'CP950'
3398  *
3399  * @param tocode        Target encoding
3400  * @param fromcode      Source encoding
3401  * @param pic           anonimized pointer to iconv struct
3402  */
3403 void  ctdl_iconv_open(const char *tocode, const char *fromcode, void *pic)
3404 {
3405 #ifdef HAVE_ICONV
3406         iconv_t ic = (iconv_t)(-1) ;
3407         ic = iconv_open(tocode, fromcode);
3408         if (ic == (iconv_t)(-1) ) {
3409                 char alias_fromcode[64];
3410                 if ( (strlen(fromcode) == 5) && (!strncasecmp(fromcode, "MS", 2)) ) {
3411                         safestrncpy(alias_fromcode, fromcode, sizeof alias_fromcode);
3412                         alias_fromcode[0] = 'C';
3413                         alias_fromcode[1] = 'P';
3414                         ic = iconv_open(tocode, alias_fromcode);
3415                 }
3416         }
3417         *(iconv_t *)pic = ic;
3418 #endif
3419 }
3420
3421
3422 /**
3423  * @ingroup StrBuf_DeEnCoder
3424  * @brief find one chunk of a RFC822 encoded string
3425  * @param Buffer where to search
3426  * @param bptr where to start searching
3427  * @returns found position, NULL if none.
3428  */
3429 static inline const char *FindNextEnd (const StrBuf *Buf, const char *bptr)
3430 {
3431         const char * end;
3432         /* Find the next ?Q? */
3433         if (Buf->BufUsed - (bptr - Buf->buf)  < 6)
3434                 return NULL;
3435
3436         end = strchr(bptr + 2, '?');
3437
3438         if (end == NULL)
3439                 return NULL;
3440
3441         if ((Buf->BufUsed - (end - Buf->buf) > 3) &&
3442             (((*(end + 1) == 'B') || (*(end + 1) == 'Q')) ||
3443              ((*(end + 1) == 'b') || (*(end + 1) == 'q'))) &&
3444             (*(end + 2) == '?')) {
3445                 /* skip on to the end of the cluster, the next ?= */
3446                 end = strstr(end + 3, "?=");
3447         }
3448         else
3449                 /* sort of half valid encoding, try to find an end. */
3450                 end = strstr(bptr, "?=");
3451         return end;
3452 }
3453
3454
3455
3456 /**
3457  * @ingroup StrBuf_DeEnCoder
3458  * @brief convert one buffer according to the preselected iconv pointer PIC
3459  * @param ConvertBuf buffer we need to translate
3460  * @param TmpBuf To share a workbuffer over several iterations. prepare to have it filled with useless stuff afterwards.
3461  * @param pic Pointer to the iconv-session Object
3462  */
3463 void StrBufConvert(StrBuf *ConvertBuf, StrBuf *TmpBuf, void *pic)
3464 {
3465 #ifdef HAVE_ICONV
3466         long trycount = 0;
3467         size_t siz;
3468         iconv_t ic;
3469         char *ibuf;                     /**< Buffer of characters to be converted */
3470         char *obuf;                     /**< Buffer for converted characters */
3471         size_t ibuflen;                 /**< Length of input buffer */
3472         size_t obuflen;                 /**< Length of output buffer */
3473
3474
3475         if ((ConvertBuf == NULL) || (TmpBuf == NULL))
3476                 return;
3477
3478         /* since we're converting to utf-8, one glyph may take up to 6 bytes */
3479         if (ConvertBuf->BufUsed * 6 >= TmpBuf->BufSize)
3480                 IncreaseBuf(TmpBuf, 0, ConvertBuf->BufUsed * 6);
3481 TRYAGAIN:
3482         ic = *(iconv_t*)pic;
3483         ibuf = ConvertBuf->buf;
3484         ibuflen = ConvertBuf->BufUsed;
3485         obuf = TmpBuf->buf;
3486         obuflen = TmpBuf->BufSize;
3487
3488         siz = iconv(ic, &ibuf, &ibuflen, &obuf, &obuflen);
3489
3490         if (siz < 0) {
3491                 if (errno == E2BIG) {
3492                         trycount ++;
3493                         IncreaseBuf(TmpBuf, 0, 0);
3494                         if (trycount < 5)
3495                                 goto TRYAGAIN;
3496
3497                 }
3498                 else if (errno == EILSEQ){
3499                         /* hm, invalid utf8 sequence... what to do now? */
3500                         /* An invalid multibyte sequence has been encountered in the input */
3501                 }
3502                 else if (errno == EINVAL) {
3503                         /* An incomplete multibyte sequence has been encountered in the input. */
3504                 }
3505
3506                 FlushStrBuf(TmpBuf);
3507         }
3508         else {
3509                 TmpBuf->BufUsed = TmpBuf->BufSize - obuflen;
3510                 TmpBuf->buf[TmpBuf->BufUsed] = '\0';
3511
3512                 /* little card game: wheres the red lady? */
3513                 SwapBuffers(ConvertBuf, TmpBuf);
3514                 FlushStrBuf(TmpBuf);
3515         }
3516 #endif
3517 }
3518
3519
3520 /**
3521  * @ingroup StrBuf_DeEnCoder
3522  * @brief catches one RFC822 encoded segment, and decodes it.
3523  * @param Target buffer to fill with result
3524  * @param DecodeMe buffer with stuff to process
3525  * @param SegmentStart points to our current segment in DecodeMe
3526  * @param SegmentEnd Points to the end of our current segment in DecodeMe
3527  * @param ConvertBuf Workbuffer shared between several iterations. Random content; needs to be valid
3528  * @param ConvertBuf2 Workbuffer shared between several iterations. Random content; needs to be valid
3529  * @param FoundCharset Characterset to default decoding to; if we find another we will overwrite it.
3530  */
3531 inline static void DecodeSegment(StrBuf *Target,
3532                                  const StrBuf *DecodeMe,
3533                                  const char *SegmentStart,
3534                                  const char *SegmentEnd,
3535                                  StrBuf *ConvertBuf,
3536                                  StrBuf *ConvertBuf2,
3537                                  StrBuf *FoundCharset)
3538 {
3539         StrBuf StaticBuf;
3540         char charset[128];
3541         char encoding[16];
3542 #ifdef HAVE_ICONV
3543         iconv_t ic = (iconv_t)(-1);
3544 #else
3545         void *ic = NULL;
3546 #endif
3547         /* Now we handle foreign character sets properly encoded
3548          * in RFC2047 format.
3549          */
3550         StaticBuf.buf = (char*) SegmentStart; /*< it will just be read there... */
3551         StaticBuf.BufUsed = SegmentEnd - SegmentStart;
3552         StaticBuf.BufSize = DecodeMe->BufSize - (SegmentStart - DecodeMe->buf);
3553         extract_token(charset, SegmentStart, 1, '?', sizeof charset);
3554         if (FoundCharset != NULL) {
3555                 FlushStrBuf(FoundCharset);
3556                 StrBufAppendBufPlain(FoundCharset, charset, -1, 0);
3557         }
3558         extract_token(encoding, SegmentStart, 2, '?', sizeof encoding);
3559         StrBufExtract_token(ConvertBuf, &StaticBuf, 3, '?');
3560
3561         *encoding = toupper(*encoding);
3562         if (*encoding == 'B') { /**< base64 */
3563                 if (ConvertBuf2->BufSize < ConvertBuf->BufUsed)
3564                         IncreaseBuf(ConvertBuf2, 0, ConvertBuf->BufUsed);
3565                 ConvertBuf2->BufUsed = CtdlDecodeBase64(ConvertBuf2->buf,
3566                                                         ConvertBuf->buf,
3567                                                         ConvertBuf->BufUsed);
3568         }
3569         else if (*encoding == 'Q') {    /**< quoted-printable */
3570                 long pos;
3571
3572                 pos = 0;
3573                 while (pos < ConvertBuf->BufUsed)
3574                 {
3575                         if (ConvertBuf->buf[pos] == '_')
3576                                 ConvertBuf->buf[pos] = ' ';
3577                         pos++;
3578                 }
3579
3580                 if (ConvertBuf2->BufSize < ConvertBuf->BufUsed)
3581                         IncreaseBuf(ConvertBuf2, 0, ConvertBuf->BufUsed);
3582
3583                 ConvertBuf2->BufUsed = CtdlDecodeQuotedPrintable(
3584                         ConvertBuf2->buf,
3585                         ConvertBuf->buf,
3586                         ConvertBuf->BufUsed);
3587         }
3588         else {
3589                 StrBufAppendBuf(ConvertBuf2, ConvertBuf, 0);
3590         }
3591 #ifdef HAVE_ICONV
3592         ctdl_iconv_open("UTF-8", charset, &ic);
3593         if (ic != (iconv_t)(-1) ) {
3594 #endif
3595                 StrBufConvert(ConvertBuf2, ConvertBuf, &ic);
3596                 StrBufAppendBuf(Target, ConvertBuf2, 0);
3597 #ifdef HAVE_ICONV
3598                 iconv_close(ic);
3599         }
3600         else {
3601                 StrBufAppendBufPlain(Target, HKEY("(unreadable)"), 0);
3602         }
3603 #endif
3604 }
3605
3606 /**
3607  * @ingroup StrBuf_DeEnCoder
3608  * @brief Handle subjects with RFC2047 encoding such as: [deprecated old syntax!]
3609  * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
3610  * @param Target where to put the decoded string to
3611  * @param DecodeMe buffer with encoded string
3612  * @param DefaultCharset if we don't find one, which should we use?
3613  * @param FoundCharset overrides DefaultCharset if non-empty; If we find a charset inside of the string,
3614  *        put it here for later use where no string might be known.
3615  */
3616 void StrBuf_RFC822_to_Utf8(StrBuf *Target, const StrBuf *DecodeMe, const StrBuf* DefaultCharset, StrBuf *FoundCharset)
3617 {
3618         StrBuf *ConvertBuf;
3619         StrBuf *ConvertBuf2;
3620         ConvertBuf = NewStrBufPlain(NULL, StrLength(DecodeMe));
3621         ConvertBuf2 = NewStrBufPlain(NULL, StrLength(DecodeMe));
3622
3623         StrBuf_RFC822_2_Utf8(Target,
3624                              DecodeMe,
3625                              DefaultCharset,
3626                              FoundCharset,
3627                              ConvertBuf,
3628                              ConvertBuf2);
3629         FreeStrBuf(&ConvertBuf);
3630         FreeStrBuf(&ConvertBuf2);
3631 }
3632
3633 /**
3634  * @ingroup StrBuf_DeEnCoder
3635  * @brief Handle subjects with RFC2047 encoding such as:
3636  * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
3637  * @param Target where to put the decoded string to
3638  * @param DecodeMe buffer with encoded string
3639  * @param DefaultCharset if we don't find one, which should we use?
3640  * @param FoundCharset overrides DefaultCharset if non-empty; If we find a charset inside of the string,
3641  *        put it here for later use where no string might be known.
3642  * @param ConvertBuf workbuffer. feed in, you shouldn't care about its content.
3643  * @param ConvertBuf2 workbuffer. feed in, you shouldn't care about its content.
3644  */
3645 void StrBuf_RFC822_2_Utf8(StrBuf *Target,
3646                           const StrBuf *DecodeMe,
3647                           const StrBuf* DefaultCharset,
3648                           StrBuf *FoundCharset,
3649                           StrBuf *ConvertBuf,
3650                           StrBuf *ConvertBuf2)
3651 {
3652         StrBuf *DecodedInvalidBuf = NULL;
3653         const StrBuf *DecodeMee = DecodeMe;
3654         const char *start, *end, *next, *nextend, *ptr = NULL;
3655 #ifdef HAVE_ICONV
3656         iconv_t ic = (iconv_t)(-1) ;
3657 #endif
3658         const char *eptr;
3659         int passes = 0;
3660         int i;
3661         int illegal_non_rfc2047_encoding = 0;
3662
3663
3664         if (DecodeMe == NULL)
3665                 return;
3666         /* Sometimes, badly formed messages contain strings which were simply
3667          *  written out directly in some foreign character set instead of
3668          *  using RFC2047 encoding.  This is illegal but we will attempt to
3669          *  handle it anyway by converting from a user-specified default
3670          *  charset to UTF-8 if we see any nonprintable characters.
3671          */
3672
3673         for (i=0; i<DecodeMe->BufUsed; ++i) {
3674                 if ((DecodeMe->buf[i] < 32) || (DecodeMe->buf[i] > 126)) {
3675                         illegal_non_rfc2047_encoding = 1;
3676                         break;
3677                 }
3678         }
3679
3680         if ((illegal_non_rfc2047_encoding) &&
3681             (strcasecmp(ChrPtr(DefaultCharset), "UTF-8")) &&
3682             (strcasecmp(ChrPtr(DefaultCharset), "us-ascii")) )
3683         {
3684 #ifdef HAVE_ICONV
3685                 ctdl_iconv_open("UTF-8", ChrPtr(DefaultCharset), &ic);
3686                 if (ic != (iconv_t)(-1) ) {
3687                         DecodedInvalidBuf = NewStrBufDup(DecodeMe);
3688                         StrBufConvert(DecodedInvalidBuf, ConvertBuf, &ic);///TODO: don't void const?
3689                         DecodeMee = DecodedInvalidBuf;
3690                         iconv_close(ic);
3691                 }
3692 #endif
3693         }
3694
3695         /* pre evaluate the first pair */
3696         end = NULL;
3697         start = strstr(DecodeMee->buf, "=?");
3698         eptr = DecodeMee->buf + DecodeMee->BufUsed;
3699         if (start != NULL)
3700                 end = FindNextEnd (DecodeMee, start + 2);
3701         else {
3702                 StrBufAppendBuf(Target, DecodeMee, 0);
3703                 FreeStrBuf(&DecodedInvalidBuf);
3704                 return;
3705         }
3706
3707
3708         if (start != DecodeMee->buf) {
3709                 long nFront;
3710
3711                 nFront = start - DecodeMee->buf;
3712                 StrBufAppendBufPlain(Target, DecodeMee->buf, nFront, 0);
3713         }
3714         /*
3715          * Since spammers will go to all sorts of absurd lengths to get their
3716          * messages through, there are LOTS of corrupt headers out there.
3717          * So, prevent a really badly formed RFC2047 header from throwing
3718          * this function into an infinite loop.
3719          */
3720         while ((start != NULL) &&
3721                (end != NULL) &&
3722                (start < eptr) &&
3723                (end < eptr) &&
3724                (passes < 20))
3725         {
3726                 passes++;
3727                 DecodeSegment(Target,
3728                               DecodeMee,
3729                               start,
3730                               end,
3731                               ConvertBuf,
3732                               ConvertBuf2,
3733                               FoundCharset);
3734
3735                 next = strstr(end, "=?");
3736                 nextend = NULL;
3737                 if ((next != NULL) &&
3738                     (next < eptr))
3739                         nextend = FindNextEnd(DecodeMee, next);
3740                 if (nextend == NULL)
3741                         next = NULL;
3742
3743                 /* did we find two partitions */
3744                 if ((next != NULL) &&
3745                     ((next - end) > 2))
3746                 {
3747                         ptr = end + 2;
3748                         while ((ptr < next) &&
3749                                (isspace(*ptr) ||
3750                                 (*ptr == '\r') ||
3751                                 (*ptr == '\n') ||
3752                                 (*ptr == '\t')))
3753                                 ptr ++;
3754                         /*
3755                          * did we find a gab just filled with blanks?
3756                          * if not, copy its stuff over.
3757                          */
3758                         if (ptr != next)
3759                         {
3760                                 StrBufAppendBufPlain(Target,
3761                                                      end + 2,
3762                                                      next - end - 2,
3763                                                      0);
3764                         }
3765                 }
3766                 /* our next-pair is our new first pair now. */
3767                 ptr = end + 2;
3768                 start = next;
3769                 end = nextend;
3770         }
3771         end = ptr;
3772         nextend = DecodeMee->buf + DecodeMee->BufUsed;
3773         if ((end != NULL) && (end < nextend)) {
3774                 ptr = end;
3775                 while ( (ptr < nextend) &&
3776                         (isspace(*ptr) ||
3777                          (*ptr == '\r') ||
3778                          (*ptr == '\n') ||
3779                          (*ptr == '\t')))
3780                         ptr ++;
3781                 if (ptr < nextend)
3782                         StrBufAppendBufPlain(Target, end, nextend - end, 0);
3783         }
3784         FreeStrBuf(&DecodedInvalidBuf);
3785 }
3786
3787 /*******************************************************************************
3788  *                   Manipulating UTF-8 Strings                                *
3789  *******************************************************************************/
3790
3791 /**
3792  * @ingroup StrBuf
3793  * @brief evaluate the length of an utf8 special character sequence
3794  * @param Char the character to examine
3795  * @returns width of utf8 chars in bytes; if the sequence is broken 0 is returned; 1 if its simply ASCII.
3796  */
3797 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE)
3798 {
3799         int n = 0;
3800         unsigned char test = (1<<7);
3801
3802         if ((*CharS & 0xC0) != 0xC0)
3803                 return 1;
3804
3805         while ((n < 8) &&
3806                ((test & ((unsigned char)*CharS)) != 0))
3807         {
3808                 test = test >> 1;
3809                 n ++;
3810         }
3811         if ((n > 6) || ((CharE - CharS) < n))
3812                 n = 0;
3813         return n;
3814 }
3815
3816 /**
3817  * @ingroup StrBuf
3818  * @brief detect whether this char starts an utf-8 encoded char
3819  * @param Char character to inspect
3820  * @returns yes or no
3821  */
3822 static inline int Ctdl_IsUtf8SequenceStart(const char Char)
3823 {
3824 /** 11??.???? indicates an UTF8 Sequence. */
3825         return ((Char & 0xC0) == 0xC0);
3826 }
3827
3828 /**
3829  * @ingroup StrBuf
3830  * @brief measure the number of glyphs in an UTF8 string...
3831  * @param Buf string to measure
3832  * @returns the number of glyphs in Buf
3833  */
3834 long StrBuf_Utf8StrLen(StrBuf *Buf)
3835 {
3836         int n = 0;
3837         int m = 0;
3838         char *aptr, *eptr;
3839
3840         if ((Buf == NULL) || (Buf->BufUsed == 0))
3841                 return 0;
3842         aptr = Buf->buf;
3843         eptr = Buf->buf + Buf->BufUsed;
3844         while ((aptr < eptr) && (*aptr != '\0')) {
3845                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
3846                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
3847                         while ((aptr < eptr) && (*aptr++ != '\0')&& (m-- > 0) );
3848                         n ++;
3849                 }
3850                 else {
3851                         n++;
3852                         aptr++;
3853                 }
3854         }
3855         return n;
3856 }
3857
3858 /**
3859  * @ingroup StrBuf
3860  * @brief cuts a string after maxlen glyphs
3861  * @param Buf string to cut to maxlen glyphs
3862  * @param maxlen how long may the string become?
3863  * @returns current length of the string
3864  */
3865 long StrBuf_Utf8StrCut(StrBuf *Buf, int maxlen)
3866 {
3867         char *aptr, *eptr;
3868         int n = 0, m = 0;
3869
3870         aptr = Buf->buf;
3871         eptr = Buf->buf + Buf->BufUsed;
3872         while ((aptr < eptr) && (*aptr != '\0')) {
3873                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
3874                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
3875                         while ((*aptr++ != '\0') && (m-- > 0));
3876                         n ++;
3877                 }
3878                 else {
3879                         n++;
3880                         aptr++;
3881                 }
3882                 if (n > maxlen) {
3883                         *aptr = '\0';
3884                         Buf->BufUsed = aptr - Buf->buf;
3885                         return Buf->BufUsed;
3886                 }
3887         }
3888         return Buf->BufUsed;
3889
3890 }
3891
3892
3893
3894
3895
3896 /*******************************************************************************
3897  *                               wrapping ZLib                                 *
3898  *******************************************************************************/
3899
3900 #ifdef HAVE_ZLIB
3901 #define DEF_MEM_LEVEL 8 /*< memlevel??? */
3902 #define OS_CODE 0x03    /*< unix */
3903
3904 /**
3905  * @ingroup StrBuf_DeEnCoder
3906  * @brief uses the same calling syntax as compress2(), but it
3907  *   creates a stream compatible with HTTP "Content-encoding: gzip"
3908  * @param dest compressed buffer
3909  * @param destLen length of the compresed data
3910  * @param source source to encode
3911  * @param sourceLen length of source to encode
3912  * @param level compression level
3913  */
3914 int ZEXPORT compress_gzip(Bytef * dest,
3915                           size_t * destLen,
3916                           const Bytef * source,
3917                           uLong sourceLen,
3918                           int level)
3919 {
3920         const int gz_magic[2] = { 0x1f, 0x8b }; /* gzip magic header */
3921
3922         /* write gzip header */
3923         snprintf((char *) dest, *destLen,
3924                  "%c%c%c%c%c%c%c%c%c%c",
3925                  gz_magic[0], gz_magic[1], Z_DEFLATED,
3926                  0 /*flags */ , 0, 0, 0, 0 /*time */ , 0 /* xflags */ ,
3927                  OS_CODE);
3928
3929         /* normal deflate */
3930         z_stream stream;
3931         int err;
3932         stream.next_in = (Bytef *) source;
3933         stream.avail_in = (uInt) sourceLen;
3934         stream.next_out = dest + 10L;   // after header
3935         stream.avail_out = (uInt) * destLen;
3936         if ((uLong) stream.avail_out != *destLen)
3937                 return Z_BUF_ERROR;
3938
3939         stream.zalloc = (alloc_func) 0;
3940         stream.zfree = (free_func) 0;
3941         stream.opaque = (voidpf) 0;
3942
3943         err = deflateInit2(&stream, level, Z_DEFLATED, -MAX_WBITS,
3944                            DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
3945         if (err != Z_OK)
3946                 return err;
3947
3948         err = deflate(&stream, Z_FINISH);
3949         if (err != Z_STREAM_END) {
3950                 deflateEnd(&stream);
3951                 return err == Z_OK ? Z_BUF_ERROR : err;
3952         }
3953         *destLen = stream.total_out + 10L;
3954
3955         /* write CRC and Length */
3956         uLong crc = crc32(0L, source, sourceLen);
3957         int n;
3958         for (n = 0; n < 4; ++n, ++*destLen) {
3959                 dest[*destLen] = (int) (crc & 0xff);
3960                 crc >>= 8;
3961         }
3962         uLong len = stream.total_in;
3963         for (n = 0; n < 4; ++n, ++*destLen) {
3964                 dest[*destLen] = (int) (len & 0xff);
3965                 len >>= 8;
3966         }
3967         err = deflateEnd(&stream);
3968         return err;
3969 }
3970 #endif
3971
3972
3973 /**
3974  * @ingroup StrBuf_DeEnCoder
3975  * @brief compress the buffer with gzip
3976  * Attention! If you feed this a Const String, you must maintain the uncompressed buffer yourself!
3977  * @param Buf buffer whose content is to be gzipped
3978  */
3979 int CompressBuffer(StrBuf *Buf)
3980 {
3981 #ifdef HAVE_ZLIB
3982         char *compressed_data = NULL;
3983         size_t compressed_len, bufsize;
3984         int i = 0;
3985
3986         bufsize = compressed_len = Buf->BufUsed +  (Buf->BufUsed / 100) + 100;
3987         compressed_data = malloc(compressed_len);
3988
3989         if (compressed_data == NULL)
3990                 return -1;
3991         /* Flush some space after the used payload so valgrind shuts up... */
3992         while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
3993                 Buf->buf[Buf->BufUsed + i++] = '\0';
3994         if (compress_gzip((Bytef *) compressed_data,
3995                           &compressed_len,
3996                           (Bytef *) Buf->buf,
3997                           (uLongf) Buf->BufUsed, Z_BEST_SPEED) == Z_OK) {
3998                 if (!Buf->ConstBuf)
3999                         free(Buf->buf);
4000                 Buf->buf = compressed_data;
4001                 Buf->BufUsed = compressed_len;
4002                 Buf->BufSize = bufsize;
4003                 /* Flush some space after the used payload so valgrind shuts up... */
4004                 i = 0;
4005                 while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
4006                         Buf->buf[Buf->BufUsed + i++] = '\0';
4007                 return 1;
4008         } else {
4009                 free(compressed_data);
4010         }
4011 #endif  /* HAVE_ZLIB */
4012         return 0;
4013 }
4014
4015 /*******************************************************************************
4016  *           File I/O; Callbacks to libevent                                   *
4017  *******************************************************************************/
4018
4019 long StrBuf_read_one_chunk_callback (int fd, short event, IOBuffer *FB)
4020 {
4021         long bufremain = 0;
4022         int n;
4023
4024         if ((FB == NULL) || (FB->Buf == NULL))
4025                 return -1;
4026
4027         /*
4028          * check whether the read pointer is somewhere in a range
4029          * where a cut left is inexpensive
4030          */
4031
4032         if (FB->ReadWritePointer != NULL)
4033         {
4034                 long already_read;
4035
4036                 already_read = FB->ReadWritePointer - FB->Buf->buf;
4037                 bufremain = FB->Buf->BufSize - FB->Buf->BufUsed - 1;
4038
4039                 if (already_read != 0) {
4040                         long unread;
4041
4042                         unread = FB->Buf->BufUsed - already_read;
4043
4044                         /* else nothing to compact... */
4045                         if (unread == 0) {
4046                                 FB->ReadWritePointer = FB->Buf->buf;
4047                                 bufremain = FB->Buf->BufSize;
4048                         }
4049                         else if ((unread < 64) ||
4050                                  (bufremain < already_read))
4051                         {
4052                                 /*
4053                                  * if its just a tiny bit remaining, or we run out of space...
4054                                  * lets tidy up.
4055                                  */
4056                                 FB->Buf->BufUsed = unread;
4057                                 if (unread < already_read)
4058                                         memcpy(FB->Buf->buf, FB->ReadWritePointer, unread);
4059                                 else
4060                                         memmove(FB->Buf->buf, FB->ReadWritePointer, unread);
4061                                 FB->ReadWritePointer = FB->Buf->buf;
4062                                 bufremain = FB->Buf->BufSize - unread - 1;
4063                         }
4064                         else if (bufremain < (FB->Buf->BufSize / 10))
4065                         {
4066                                 /* get a bigger buffer */
4067
4068                                 IncreaseBuf(FB->Buf, 0, FB->Buf->BufUsed + 1);
4069
4070                                 FB->ReadWritePointer = FB->Buf->buf + unread;
4071
4072                                 bufremain = FB->Buf->BufSize - unread - 1;
4073 /*TODO: special increase function that won't copy the already read! */
4074                         }
4075                 }
4076                 else if (bufremain < 10) {
4077                         IncreaseBuf(FB->Buf, 1, FB->Buf->BufUsed + 10);
4078
4079                         FB->ReadWritePointer = FB->Buf->buf;
4080
4081                         bufremain = FB->Buf->BufSize - FB->Buf->BufUsed - 1;
4082                 }
4083
4084         }
4085         else {
4086                 FB->ReadWritePointer = FB->Buf->buf;
4087                 bufremain = FB->Buf->BufSize - 1;
4088         }
4089
4090         n = read(fd, FB->Buf->buf + FB->Buf->BufUsed, bufremain);
4091
4092         if (n > 0) {
4093                 FB->Buf->BufUsed += n;
4094                 FB->Buf->buf[FB->Buf->BufUsed] = '\0';
4095         }
4096         return n;
4097 }
4098
4099 int StrBuf_write_one_chunk_callback(int fd, short event, IOBuffer *FB)
4100 {
4101         long WriteRemain;
4102         int n;
4103
4104         if ((FB == NULL) || (FB->Buf == NULL))
4105                 return -1;
4106
4107         if (FB->ReadWritePointer != NULL)
4108         {
4109                 WriteRemain = FB->Buf->BufUsed -
4110                         (FB->ReadWritePointer -
4111                          FB->Buf->buf);
4112         }
4113         else {
4114                 FB->ReadWritePointer = FB->Buf->buf;
4115                 WriteRemain = FB->Buf->BufUsed;
4116         }
4117
4118         n = write(fd, FB->ReadWritePointer, WriteRemain);
4119         if (n > 0) {
4120                 FB->ReadWritePointer += n;
4121
4122                 if (FB->ReadWritePointer ==
4123                     FB->Buf->buf + FB->Buf->BufUsed)
4124                 {
4125                         FlushStrBuf(FB->Buf);
4126                         FB->ReadWritePointer = NULL;
4127                         return 0;
4128                 }
4129         // check whether we've got something to write
4130         // get the maximum chunk plus the pointer we can send
4131         // write whats there
4132         // if not all was sent, remember the send pointer for the next time
4133                 return FB->ReadWritePointer - FB->Buf->buf + FB->Buf->BufUsed;
4134         }
4135         return n;
4136 }
4137
4138 /**
4139  * @ingroup StrBuf_IO
4140  * @brief extract a "next line" from Buf; Ptr to persist across several iterations
4141  * @param LineBuf your line will be copied here.
4142  * @param FB BLOB with lines of text...
4143  * @param Ptr moved arround to keep the next-line across several iterations
4144  *        has to be &NULL on start; will be &NotNULL on end of buffer
4145  * @returns size of copied buffer
4146  */
4147 eReadState StrBufChunkSipLine(StrBuf *LineBuf, IOBuffer *FB)
4148 {
4149         const char *aptr, *ptr, *eptr;
4150         char *optr, *xptr;
4151
4152         if ((FB == NULL) || (LineBuf == NULL) || (LineBuf->buf == NULL))
4153                 return eReadFail;
4154
4155
4156         if ((FB->Buf == NULL) || (FB->ReadWritePointer == StrBufNOTNULL)) {
4157                 FB->ReadWritePointer = StrBufNOTNULL;
4158                 return eReadFail;
4159         }
4160
4161         FlushStrBuf(LineBuf);
4162         if (FB->ReadWritePointer == NULL)
4163                 ptr = aptr = FB->Buf->buf;
4164         else
4165                 ptr = aptr = FB->ReadWritePointer;
4166
4167         optr = LineBuf->buf;
4168         eptr = FB->Buf->buf + FB->Buf->BufUsed;
4169         xptr = LineBuf->buf + LineBuf->BufSize - 1;
4170
4171         while ((ptr <= eptr) &&
4172                (*ptr != '\n') &&
4173                (*ptr != '\r') )
4174         {
4175                 *optr = *ptr;
4176                 optr++; ptr++;
4177                 if (optr == xptr) {
4178                         LineBuf->BufUsed = optr - LineBuf->buf;
4179                         IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
4180                         optr = LineBuf->buf + LineBuf->BufUsed;
4181                         xptr = LineBuf->buf + LineBuf->BufSize - 1;
4182                 }
4183         }
4184
4185         if (ptr >= eptr) {
4186                 if (optr > LineBuf->buf)
4187                         optr --;
4188                 if ((*(ptr - 1) != '\r') && (*(ptr - 1) != '\n')) {
4189                         LineBuf->BufUsed = optr - LineBuf->buf;
4190                         *optr = '\0';
4191                         if ((FB->ReadWritePointer != NULL) &&
4192                             (FB->ReadWritePointer != FB->Buf->buf))
4193                         {
4194                                 /* Ok, the client application read all the data
4195                                    it was interested in so far. Since there is more to read,
4196                                    we now shrink the buffer, and move the rest over.
4197                                 */
4198                                 StrBufCutLeft(FB->Buf,
4199                                               FB->ReadWritePointer - FB->Buf->buf);
4200                                 FB->ReadWritePointer = FB->Buf->buf;
4201                         }
4202                         return eMustReadMore;
4203                 }
4204         }
4205         LineBuf->BufUsed = optr - LineBuf->buf;
4206         *optr = '\0';
4207         if ((ptr <= eptr) && (*ptr == '\r'))
4208                 ptr ++;
4209         if ((ptr <= eptr) && (*ptr == '\n'))
4210                 ptr ++;
4211
4212         if (ptr < eptr) {
4213                 FB->ReadWritePointer = ptr;
4214         }
4215         else {
4216                 FlushStrBuf(FB->Buf);
4217                 FB->ReadWritePointer = NULL;
4218         }
4219
4220         return eReadSuccess;
4221 }
4222
4223 /**
4224  * @ingroup StrBuf_CHUNKED_IO
4225  * @brief check whether the chunk-buffer has more data waiting or not.
4226  * @param FB Chunk-Buffer to inspect
4227  */
4228 eReadState StrBufCheckBuffer(IOBuffer *FB)
4229 {
4230         if (FB == NULL)
4231                 return eReadFail;
4232         if (FB->Buf->BufUsed == 0)
4233                 return eReadSuccess;
4234         if (FB->ReadWritePointer == NULL)
4235                 return eBufferNotEmpty;
4236         if (FB->Buf->buf + FB->Buf->BufUsed > FB->ReadWritePointer)
4237                 return eBufferNotEmpty;
4238         return eReadSuccess;
4239 }
4240
4241 long IOBufferStrLength(IOBuffer *FB)
4242 {
4243         if ((FB == NULL) || (FB->Buf == NULL))
4244                 return 0;
4245         if (FB->ReadWritePointer == NULL)
4246                 return StrLength(FB->Buf);
4247
4248         return StrLength(FB->Buf) - (FB->ReadWritePointer - FB->Buf->buf);
4249 }
4250
4251 inline static void FDIOBufferFlush(FDIOBuffer *FDB)
4252 {
4253         memset(FDB, 0, sizeof(FDIOBuffer));
4254         FDB->OtherFD = -1;
4255         FDB->SplicePipe[0] = -1;
4256         FDB->SplicePipe[1] = -1;
4257 }
4258
4259 void FDIOBufferInit(FDIOBuffer *FDB, IOBuffer *IO, int FD, long TotalSendSize)
4260 {
4261         FDIOBufferFlush(FDB);
4262
4263         FDB->TotalSendSize = TotalSendSize;
4264         if (TotalSendSize > 0)
4265                 FDB->ChunkSize = TotalSendSize;
4266         else
4267         {
4268                 TotalSendSize = SIZ * 10;
4269                 FDB->ChunkSize = TotalSendSize;
4270         }
4271         FDB->IOB = IO;
4272
4273 #ifdef LINUX_SPLICE
4274         if (EnableSplice)
4275                 pipe(FDB->SplicePipe);
4276         else
4277 #endif
4278                 FDB->ChunkBuffer = NewStrBufPlain(NULL, TotalSendSize+ 1);
4279
4280         FDB->OtherFD = FD;
4281 }
4282
4283 void FDIOBufferDelete(FDIOBuffer *FDB)
4284 {
4285 #ifdef LINUX_SPLICE
4286         if (EnableSplice)
4287         {
4288                 if (FDB->SplicePipe[0] > 0)
4289                         close(FDB->SplicePipe[0]);
4290                 if (FDB->SplicePipe[1] > 0)
4291                         close(FDB->SplicePipe[1]);
4292         }
4293         else
4294 #endif
4295                 FreeStrBuf(&FDB->ChunkBuffer);
4296
4297         if (FDB->OtherFD > 0)
4298                 close(FDB->OtherFD);
4299         FDIOBufferFlush(FDB);
4300 }
4301
4302 int FileSendChunked(FDIOBuffer *FDB, const char **Err)
4303 {
4304         ssize_t sent, pipesize;
4305
4306         if (FDB->TotalSendSize > 0)
4307         {
4308 #ifdef LINUX_SPLICE
4309                 if (EnableSplice)
4310                 {
4311                         if (FDB->PipeSize == 0)
4312                         {
4313                                 pipesize = splice(FDB->OtherFD,
4314                                                   &FDB->TotalSentAlready,
4315                                                   FDB->SplicePipe[1],
4316                                                   NULL,
4317                                                   FDB->ChunkSendRemain,
4318                                                   SPLICE_F_MOVE);
4319
4320                                 if (pipesize == -1)
4321                                 {
4322                                         *Err = strerror(errno);
4323                                         return pipesize;
4324                                 }
4325                                 FDB->PipeSize = pipesize;
4326                         }
4327                         sent =  splice(FDB->SplicePipe[0],
4328                                        NULL,
4329                                        FDB->IOB->fd,
4330                                        NULL,
4331                                        FDB->PipeSize,
4332                                        SPLICE_F_MORE | SPLICE_F_MOVE | SPLICE_F_NONBLOCK);
4333                         if (sent == -1)
4334                         {
4335                                 *Err = strerror(errno);
4336                                 return sent;
4337                         }
4338                         FDB->PipeSize -= sent;
4339                         FDB->ChunkSendRemain -= sent;
4340                         return sent;
4341                 }
4342                 else
4343 #endif
4344                 {
4345                         char *pRead;
4346                         long nRead = 0;
4347
4348                         pRead = FDB->ChunkBuffer->buf;
4349                         while ((FDB->ChunkBuffer->BufUsed < FDB->TotalSendSize) && (nRead >= 0))
4350                         {
4351                                 nRead = read(FDB->OtherFD, pRead, FDB->TotalSendSize - FDB->ChunkBuffer->BufUsed);
4352                                 if (nRead > 0) {
4353                                         FDB->ChunkBuffer->BufUsed += nRead;
4354                                         FDB->ChunkBuffer->buf[FDB->ChunkBuffer->BufUsed] = '\0';
4355                                 }
4356                                 else if (nRead == 0) {}
4357                                 else return nRead;
4358                         }
4359
4360                         nRead = write(FDB->IOB->fd,
4361                                       FDB->ChunkBuffer->buf     + FDB->TotalSentAlready,
4362                                       FDB->ChunkBuffer->BufUsed - FDB->TotalSentAlready);
4363
4364                         if (nRead >= 0) {
4365                                 FDB->TotalSentAlready += nRead;
4366                                 FDB->ChunkSendRemain -= nRead;
4367                                 return FDB->ChunkSendRemain;
4368                         }
4369                         else {
4370                                 return nRead;
4371                         }
4372                 }
4373         }
4374         else
4375         {
4376 #ifdef LINUX_SPLICE
4377                 if (EnableSplice)
4378                 {
4379                         if (FDB->PipeSize == 0)
4380                         {
4381                                 pipesize = splice(FDB->OtherFD,
4382                                                   &FDB->TotalSentAlready,
4383                                                   FDB->SplicePipe[1],
4384                                                   NULL,
4385                                                   SIZ * 10,
4386                                                   SPLICE_F_MOVE);
4387
4388                                 if (pipesize == -1)
4389                                 {
4390                                         *Err = strerror(errno);
4391                                         return pipesize;
4392                                 }
4393                                 FDB->PipeSize = pipesize;
4394                                 if (pipesize == 0)
4395                                         return -1;
4396                         }
4397                         sent =  splice(FDB->SplicePipe[0],
4398                                        NULL,
4399                                        FDB->IOB->fd,
4400                                        NULL,
4401                                        FDB->PipeSize,
4402                                        SPLICE_F_MORE | SPLICE_F_MOVE | SPLICE_F_NONBLOCK);
4403                         if (sent == -1)
4404                         {
4405                                 *Err = strerror(errno);
4406                                 return sent;
4407                         }
4408                         FDB->PipeSize -= sent;
4409                         FDB->ChunkSendRemain -= sent;
4410                         return sent;
4411                 }
4412                 else
4413 #endif
4414                 {
4415                         char *pRead;
4416                         long nRead = 0;
4417
4418                         pRead = FDB->ChunkBuffer->buf;
4419                         while ((FDB->ChunkSendRemain == 0) &&
4420                                (FDB->ChunkBuffer->BufUsed < FDB->ChunkBuffer->BufSize) &&
4421                                (nRead >= 0))
4422                         {
4423                                 FDB->TotalSentAlready = 0;
4424                                 nRead = read(FDB->OtherFD, pRead, FDB->ChunkBuffer->BufSize - FDB->ChunkBuffer->BufUsed);
4425                                 if (nRead > 0) {
4426                                         FDB->ChunkBuffer->BufUsed += nRead;
4427                                         FDB->ChunkBuffer->buf[FDB->ChunkBuffer->BufUsed] = '\0';
4428                                         FDB->ChunkSendRemain += nRead;
4429                                 }
4430                                 else if (nRead == 0)
4431                                 {
4432                                         return -1;
4433                                 }
4434                                 else
4435                                 {
4436                                         *Err = strerror(errno);
4437                                         return nRead;
4438                                 }
4439                         }
4440
4441                         nRead = write(FDB->IOB->fd,
4442                                       FDB->ChunkBuffer->buf     + FDB->TotalSentAlready,
4443                                       FDB->ChunkBuffer->BufUsed - FDB->TotalSentAlready);
4444
4445                         if (nRead >= 0) {
4446                                 FDB->TotalSentAlready += nRead;
4447                                 FDB->ChunkSendRemain -= nRead;
4448                                 if (FDB->ChunkSendRemain == 0)
4449                                 {
4450                                         FDB->ChunkBuffer->BufUsed = 0;
4451                                         FDB->TotalSentAlready = 0;
4452                                 }
4453                                 return FDB->ChunkSendRemain;
4454                         }
4455                         else {
4456                                 return nRead;
4457                         }
4458                 }
4459         }
4460 }
4461
4462 int FileRecvChunked(FDIOBuffer *FDB, const char **Err)
4463 {
4464         ssize_t sent, pipesize;
4465
4466 #ifdef LINUX_SPLICE
4467         if (EnableSplice)
4468         {
4469                 if (FDB->PipeSize == 0)
4470                 {
4471                         pipesize = splice(FDB->IOB->fd,
4472                                           NULL,
4473                                           FDB->SplicePipe[1],
4474                                           NULL,
4475                                           FDB->ChunkSendRemain,
4476                                           SPLICE_F_MORE | SPLICE_F_MOVE|SPLICE_F_NONBLOCK);
4477
4478                         if (pipesize == -1)
4479                         {
4480                                 *Err = strerror(errno);
4481                                 return pipesize;
4482                         }
4483                         FDB->PipeSize = pipesize;
4484                 }
4485
4486                 sent = splice(FDB->SplicePipe[0],
4487                               NULL,
4488                               FDB->OtherFD,
4489                               &FDB->TotalSentAlready,
4490                               FDB->PipeSize,
4491                               SPLICE_F_MORE | SPLICE_F_MOVE);
4492
4493                 if (sent == -1)
4494                 {
4495                         *Err = strerror(errno);
4496                         return sent;
4497                 }
4498                 FDB->PipeSize -= sent;
4499                 FDB->ChunkSendRemain -= sent;
4500                 return sent;
4501         }
4502         else
4503 #endif
4504         {
4505                 sent = read(FDB->IOB->fd, FDB->ChunkBuffer->buf, FDB->ChunkSendRemain);
4506                 if (sent > 0) {
4507                         int nWritten = 0;
4508                         int rc;
4509
4510                         FDB->ChunkBuffer->BufUsed = sent;
4511
4512                         while (nWritten < FDB->ChunkBuffer->BufUsed) {
4513                                 rc =  write(FDB->OtherFD, FDB->ChunkBuffer->buf + nWritten, FDB->ChunkBuffer->BufUsed - nWritten);
4514                                 if (rc < 0) {
4515                                         *Err = strerror(errno);
4516                                         return rc;
4517                                 }
4518                                 nWritten += rc;
4519
4520                         }
4521                         FDB->ChunkBuffer->BufUsed = 0;
4522                         FDB->TotalSentAlready += sent;
4523                         FDB->ChunkSendRemain -= sent;
4524                         return FDB->ChunkSendRemain;
4525                 }
4526                 else if (sent < 0) {
4527                         *Err = strerror(errno);
4528                         return sent;
4529                 }
4530                 return 0;
4531         }
4532 }
4533
4534 int FileMoveChunked(FDIOBuffer *FDB, const char **Err)
4535 {
4536         ssize_t sent, pipesize;
4537
4538 #ifdef LINUX_SPLICE
4539         if (EnableSplice)
4540         {
4541                 if (FDB->PipeSize == 0)
4542                 {
4543                         pipesize = splice(FDB->IOB->fd,
4544                                           &FDB->TotalReadAlready,
4545                                           FDB->SplicePipe[1],
4546                                           NULL,
4547                                           FDB->ChunkSendRemain,
4548                                           SPLICE_F_MORE | SPLICE_F_MOVE|SPLICE_F_NONBLOCK);
4549
4550                         if (pipesize == -1)
4551                         {
4552                                 *Err = strerror(errno);
4553                                 return pipesize;
4554                         }
4555                         FDB->PipeSize = pipesize;
4556                 }
4557
4558                 sent = splice(FDB->SplicePipe[0],
4559                               NULL,
4560                               FDB->OtherFD,
4561                               &FDB->TotalSentAlready,
4562                               FDB->PipeSize,
4563                               SPLICE_F_MORE | SPLICE_F_MOVE);
4564
4565                 if (sent == -1)
4566                 {
4567                         *Err = strerror(errno);
4568                         return sent;
4569                 }
4570                 FDB->PipeSize -= sent;
4571                 FDB->ChunkSendRemain -= sent;
4572                 return sent;
4573         }
4574         else
4575 #endif
4576         {
4577                 sent = read(FDB->IOB->fd, FDB->ChunkBuffer->buf, FDB->ChunkSendRemain);
4578                 if (sent > 0) {
4579                         int nWritten = 0;
4580                         int rc;
4581
4582                         FDB->ChunkBuffer->BufUsed = sent;
4583
4584                         while (nWritten < FDB->ChunkBuffer->BufUsed) {
4585                                 rc =  write(FDB->OtherFD, FDB->ChunkBuffer->buf + nWritten, FDB->ChunkBuffer->BufUsed - nWritten);
4586                                 if (rc < 0) {
4587                                         *Err = strerror(errno);
4588                                         return rc;
4589                                 }
4590                                 nWritten += rc;
4591
4592                         }
4593                         FDB->ChunkBuffer->BufUsed = 0;
4594                         FDB->TotalSentAlready += sent;
4595                         FDB->ChunkSendRemain -= sent;
4596                         return FDB->ChunkSendRemain;
4597                 }
4598                 else if (sent < 0) {
4599                         *Err = strerror(errno);
4600                         return sent;
4601                 }
4602                 return 0;
4603         }
4604 }
4605
4606 eReadState WriteIOBAlreadyRead(FDIOBuffer *FDB, const char **Error)
4607 {
4608         int IsNonBlock;
4609         int fdflags;
4610         long rlen;
4611         long should_write;
4612         int nSuccessLess = 0;
4613         struct timeval tv;
4614         fd_set rfds;
4615
4616         fdflags = fcntl(FDB->OtherFD, F_GETFL);
4617         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
4618
4619         while ((FDB->IOB->ReadWritePointer - FDB->IOB->Buf->buf < FDB->IOB->Buf->BufUsed) &&
4620                (FDB->ChunkSendRemain > 0))
4621         {
4622                 if (IsNonBlock){
4623                         tv.tv_sec = 1; /* selectresolution; */
4624                         tv.tv_usec = 0;
4625
4626                         FD_ZERO(&rfds);
4627                         FD_SET(FDB->OtherFD, &rfds);
4628                         if (select(FDB->OtherFD + 1, NULL, &rfds, NULL, &tv) == -1) {
4629                                 *Error = strerror(errno);
4630                                 return eReadFail;
4631                         }
4632                 }
4633                 if (IsNonBlock && !  FD_ISSET(FDB->OtherFD, &rfds)) {
4634                         nSuccessLess ++;
4635                         continue;
4636                 }
4637
4638                 should_write = FDB->IOB->Buf->BufUsed -
4639                         (FDB->IOB->ReadWritePointer - FDB->IOB->Buf->buf);
4640                 if (should_write > FDB->ChunkSendRemain)
4641                         should_write = FDB->ChunkSendRemain;
4642
4643                 rlen = write(FDB->OtherFD,
4644                              FDB->IOB->ReadWritePointer,
4645                              should_write);
4646                 if (rlen < 1) {
4647                         *Error = strerror(errno);
4648
4649                         return eReadFail;
4650                 }
4651                 FDB->TotalSentAlready += rlen;
4652                 FDB->IOB->ReadWritePointer += rlen;
4653                 FDB->ChunkSendRemain -= rlen;
4654         }
4655         if (FDB->IOB->ReadWritePointer >= FDB->IOB->Buf->buf + FDB->IOB->Buf->BufUsed)
4656         {
4657                 FlushStrBuf(FDB->IOB->Buf);
4658                 FDB->IOB->ReadWritePointer = NULL;
4659         }
4660
4661         if (FDB->ChunkSendRemain == 0)
4662                 return eReadSuccess;
4663         else
4664                 return eMustReadMore;
4665 }
4666
4667 /*******************************************************************************
4668  *           File I/O; Prefer buffered read since its faster!                  *
4669  *******************************************************************************/
4670
4671 /**
4672  * @ingroup StrBuf_IO
4673  * @brief Read a line from socket
4674  * flushes and closes the FD on error
4675  * @param buf the buffer to get the input to
4676  * @param fd pointer to the filedescriptor to read
4677  * @param append Append to an existing string or replace?
4678  * @param Error strerror() on error
4679  * @returns numbers of chars read
4680  */
4681 int StrBufTCP_read_line(StrBuf *buf, int *fd, int append, const char **Error)
4682 {
4683         int len, rlen, slen;
4684
4685         if ((buf == NULL) || (buf->buf == NULL)) {
4686                 *Error = strerror(EINVAL);
4687                 return -1;
4688         }
4689
4690         if (!append)
4691                 FlushStrBuf(buf);
4692
4693         slen = len = buf->BufUsed;
4694         while (1) {
4695                 rlen = read(*fd, &buf->buf[len], 1);
4696                 if (rlen < 1) {
4697                         *Error = strerror(errno);
4698
4699                         close(*fd);
4700                         *fd = -1;
4701
4702                         return -1;
4703                 }
4704                 if (buf->buf[len] == '\n')
4705                         break;
4706                 if (buf->buf[len] != '\r')
4707                         len ++;
4708                 if (len + 2 >= buf->BufSize) {
4709                         buf->BufUsed = len;
4710                         buf->buf[len+1] = '\0';
4711                         IncreaseBuf(buf, 1, -1);
4712                 }
4713         }
4714         buf->BufUsed = len;
4715         buf->buf[len] = '\0';
4716         return len - slen;
4717 }
4718
4719 /**
4720  * @ingroup StrBuf_BufferedIO
4721  * @brief Read a line from socket
4722  * flushes and closes the FD on error
4723  * @param Line the line to read from the fd / I/O Buffer
4724  * @param buf the buffer to get the input to
4725  * @param fd pointer to the filedescriptor to read
4726  * @param timeout number of successless selects until we bail out
4727  * @param selectresolution how long to wait on each select
4728  * @param Error strerror() on error
4729  * @returns numbers of chars read
4730  */
4731 int StrBufTCP_read_buffered_line(StrBuf *Line,
4732                                  StrBuf *buf,
4733                                  int *fd,
4734                                  int timeout,
4735                                  int selectresolution,
4736                                  const char **Error)
4737 {
4738         int len, rlen;
4739         int nSuccessLess = 0;
4740         fd_set rfds;
4741         char *pch = NULL;
4742         int fdflags;
4743         int IsNonBlock;
4744         struct timeval tv;
4745
4746         if (buf->BufUsed > 0) {
4747                 pch = strchr(buf->buf, '\n');
4748                 if (pch != NULL) {
4749                         rlen = 0;
4750                         len = pch - buf->buf;
4751                         if (len > 0 && (*(pch - 1) == '\r') )
4752                                 rlen ++;
4753                         StrBufSub(Line, buf, 0, len - rlen);
4754                         StrBufCutLeft(buf, len + 1);
4755                         return len - rlen;
4756                 }
4757         }
4758
4759         if (buf->BufSize - buf->BufUsed < 10)
4760                 IncreaseBuf(buf, 1, -1);
4761
4762         fdflags = fcntl(*fd, F_GETFL);
4763         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
4764
4765         while ((nSuccessLess < timeout) && (pch == NULL)) {
4766                 if (IsNonBlock){
4767                         tv.tv_sec = selectresolution;
4768                         tv.tv_usec = 0;
4769
4770                         FD_ZERO(&rfds);
4771                         FD_SET(*fd, &rfds);
4772                         if (select(*fd + 1, NULL, &rfds, NULL, &tv) == -1) {
4773                                 *Error = strerror(errno);
4774                                 close (*fd);
4775                                 *fd = -1;
4776                                 return -1;
4777                         }
4778                 }
4779                 if (IsNonBlock && !  FD_ISSET(*fd, &rfds)) {
4780                         nSuccessLess ++;
4781                         continue;
4782                 }
4783                 rlen = read(*fd,
4784                             &buf->buf[buf->BufUsed],
4785                             buf->BufSize - buf->BufUsed - 1);
4786                 if (rlen < 1) {
4787                         *Error = strerror(errno);
4788                         close(*fd);
4789                         *fd = -1;
4790                         return -1;
4791                 }
4792                 else if (rlen > 0) {
4793                         nSuccessLess = 0;
4794                         buf->BufUsed += rlen;
4795                         buf->buf[buf->BufUsed] = '\0';
4796                         pch = strchr(buf->buf, '\n');
4797                         if ((pch == NULL) &&
4798                             (buf->BufUsed + 10 > buf->BufSize) &&
4799                             (IncreaseBuf(buf, 1, -1) == -1))
4800                                 return -1;
4801                         continue;
4802                 }
4803
4804         }
4805         if (pch != NULL) {
4806                 rlen = 0;
4807                 len = pch - buf->buf;
4808                 if (len > 0 && (*(pch - 1) == '\r') )
4809                         rlen ++;
4810                 StrBufSub(Line, buf, 0, len - rlen);
4811                 StrBufCutLeft(buf, len + 1);
4812                 return len - rlen;
4813         }
4814         return -1;
4815
4816 }
4817
4818 static const char *ErrRBLF_PreConditionFailed="StrBufTCP_read_buffered_line_fast: Wrong arguments or invalid Filedescriptor";
4819 static const char *ErrRBLF_SelectFailed="StrBufTCP_read_buffered_line_fast: Select failed without reason";
4820 static const char *ErrRBLF_NotEnoughSentFromServer="StrBufTCP_read_buffered_line_fast: No complete line was sent from peer";
4821 /**
4822  * @ingroup StrBuf_BufferedIO
4823  * @brief Read a line from socket
4824  * flushes and closes the FD on error
4825  * @param Line where to append our Line read from the fd / I/O Buffer;
4826  * @param IOBuf the buffer to get the input to; lifetime pair to FD
4827  * @param Pos pointer to the current read position, should be NULL initialized on opening the FD it belongs to.!
4828  * @param fd pointer to the filedescriptor to read
4829  * @param timeout number of successless selects until we bail out
4830  * @param selectresolution how long to wait on each select
4831  * @param Error strerror() on error
4832  * @returns numbers of chars read or -1 in case of error. "\n" will become 0
4833  */
4834 int StrBufTCP_read_buffered_line_fast(StrBuf *Line,
4835                                       StrBuf *IOBuf,
4836                                       const char **Pos,
4837                                       int *fd,
4838                                       int timeout,
4839                                       int selectresolution,
4840                                       const char **Error)
4841 {
4842         const char *pche = NULL;
4843         const char *pos = NULL;
4844         const char *pLF;
4845         int len, rlen, retlen;
4846         int nSuccessLess = 0;
4847         fd_set rfds;
4848         const char *pch = NULL;
4849         int fdflags;
4850         int IsNonBlock;
4851         struct timeval tv;
4852
4853         retlen = 0;
4854         if ((Line == NULL) ||
4855             (Pos == NULL) ||
4856             (IOBuf == NULL) ||
4857             (*fd == -1))
4858         {
4859                 if (Pos != NULL)
4860                         *Pos = NULL;
4861                 *Error = ErrRBLF_PreConditionFailed;
4862                 return -1;
4863         }
4864
4865         pos = *Pos;
4866         if ((IOBuf->BufUsed > 0) &&
4867             (pos != NULL) &&
4868             (pos < IOBuf->buf + IOBuf->BufUsed))
4869         {
4870                 char *pcht;
4871
4872                 pche = IOBuf->buf + IOBuf->BufUsed;
4873                 pch = pos;
4874                 pcht = Line->buf;
4875
4876                 while ((pch < pche) && (*pch != '\n'))
4877                 {
4878                         if (Line->BufUsed + 10 > Line->BufSize)
4879                         {
4880                                 long apos;
4881                                 apos = pcht - Line->buf;
4882                                 *pcht = '\0';
4883                                 IncreaseBuf(Line, 1, -1);
4884                                 pcht = Line->buf + apos;
4885                         }
4886                         *pcht++ = *pch++;
4887                         Line->BufUsed++;
4888                         retlen++;
4889                 }
4890
4891                 len = pch - pos;
4892                 if (len > 0 && (*(pch - 1) == '\r') )
4893                 {
4894                         retlen--;
4895                         len --;
4896                         pcht --;
4897                         Line->BufUsed --;
4898                 }
4899                 *pcht = '\0';
4900
4901                 if ((pch >= pche) || (*pch == '\0'))
4902                 {
4903                         FlushStrBuf(IOBuf);
4904                         *Pos = NULL;
4905                         pch = NULL;
4906                         pos = 0;
4907                 }
4908
4909                 if ((pch != NULL) &&
4910                     (pch <= pche))
4911                 {
4912                         if (pch + 1 >= pche) {
4913                                 *Pos = NULL;
4914                                 FlushStrBuf(IOBuf);
4915                         }
4916                         else
4917                                 *Pos = pch + 1;
4918
4919                         return retlen;
4920                 }
4921                 else
4922                         FlushStrBuf(IOBuf);
4923         }
4924
4925         /* If we come here, Pos is Unset since we read everything into Line, and now go for more. */
4926
4927         if (IOBuf->BufSize - IOBuf->BufUsed < 10)
4928                 IncreaseBuf(IOBuf, 1, -1);
4929
4930         fdflags = fcntl(*fd, F_GETFL);
4931         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
4932
4933         pLF = NULL;
4934         while ((nSuccessLess < timeout) &&
4935                (pLF == NULL) &&
4936                (*fd != -1)) {
4937                 if (IsNonBlock)
4938                 {
4939                         tv.tv_sec = 1;
4940                         tv.tv_usec = 0;
4941
4942                         FD_ZERO(&rfds);
4943                         FD_SET(*fd, &rfds);
4944                         if (select((*fd) + 1, &rfds, NULL, NULL, &tv) == -1) {
4945                                 *Error = strerror(errno);
4946                                 close (*fd);
4947                                 *fd = -1;
4948                                 if (*Error == NULL)
4949                                         *Error = ErrRBLF_SelectFailed;
4950                                 return -1;
4951                         }
4952                         if (! FD_ISSET(*fd, &rfds) != 0) {
4953                                 nSuccessLess ++;
4954                                 continue;
4955                         }
4956                 }
4957                 rlen = read(*fd,
4958                             &IOBuf->buf[IOBuf->BufUsed],
4959                             IOBuf->BufSize - IOBuf->BufUsed - 1);
4960                 if (rlen < 1) {
4961                         *Error = strerror(errno);
4962                         close(*fd);
4963                         *fd = -1;
4964                         return -1;
4965                 }
4966                 else if (rlen > 0) {
4967                         nSuccessLess = 0;
4968                         pLF = IOBuf->buf + IOBuf->BufUsed;
4969                         IOBuf->BufUsed += rlen;
4970                         IOBuf->buf[IOBuf->BufUsed] = '\0';
4971
4972                         pche = IOBuf->buf + IOBuf->BufUsed;
4973
4974                         while ((pLF < pche) && (*pLF != '\n'))
4975                                 pLF ++;
4976                         if ((pLF >= pche) || (*pLF == '\0'))
4977                                 pLF = NULL;
4978
4979                         if (IOBuf->BufUsed + 10 > IOBuf->BufSize)
4980                         {
4981                                 long apos = 0;
4982
4983                                 if (pLF != NULL) apos = pLF - IOBuf->buf;
4984                                 IncreaseBuf(IOBuf, 1, -1);
4985                                 if (pLF != NULL) pLF = IOBuf->buf + apos;
4986                         }
4987
4988                         continue;
4989                 }
4990                 else
4991                 {
4992                         nSuccessLess++;
4993                 }
4994         }
4995         *Pos = NULL;
4996         if (pLF != NULL) {
4997                 pos = IOBuf->buf;
4998                 len = pLF - pos;
4999                 if (len > 0 && (*(pLF - 1) == '\r') )
5000                         len --;
5001                 StrBufAppendBufPlain(Line, ChrPtr(IOBuf), len, 0);
5002                 if (pLF + 1 >= IOBuf->buf + IOBuf->BufUsed)
5003                 {
5004                         FlushStrBuf(IOBuf);
5005                 }
5006                 else
5007                         *Pos = pLF + 1;
5008                 return retlen + len;
5009         }
5010         *Error = ErrRBLF_NotEnoughSentFromServer;
5011         return -1;
5012
5013 }
5014
5015 static const char *ErrRBLF_BLOBPreConditionFailed="StrBufReadBLOB: Wrong arguments or invalid Filedescriptor";
5016 /**
5017  * @ingroup StrBuf_IO
5018  * @brief Input binary data from socket
5019  * flushes and closes the FD on error
5020  * @param Buf the buffer to get the input to
5021  * @param fd pointer to the filedescriptor to read
5022  * @param append Append to an existing string or replace?
5023  * @param nBytes the maximal number of bytes to read
5024  * @param Error strerror() on error
5025  * @returns numbers of chars read
5026  */
5027 int StrBufReadBLOB(StrBuf *Buf, int *fd, int append, long nBytes, const char **Error)
5028 {
5029         int fdflags;
5030         int rlen;
5031         int nSuccessLess;
5032         int nRead = 0;
5033         char *ptr;
5034         int IsNonBlock;
5035         struct timeval tv;
5036         fd_set rfds;
5037
5038         if ((Buf == NULL) || (Buf->buf == NULL) || (*fd == -1))
5039         {
5040                 *Error = ErrRBLF_BLOBPreConditionFailed;
5041                 return -1;
5042         }
5043         if (!append)
5044                 FlushStrBuf(Buf);
5045         if (Buf->BufUsed + nBytes >= Buf->BufSize)
5046                 IncreaseBuf(Buf, 1, Buf->BufUsed + nBytes);
5047
5048         ptr = Buf->buf + Buf->BufUsed;
5049
5050         fdflags = fcntl(*fd, F_GETFL);
5051         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
5052         nSuccessLess = 0;
5053         while ((nRead < nBytes) &&
5054                (*fd != -1))
5055         {
5056                 if (IsNonBlock)
5057                 {
5058                         tv.tv_sec = 1;
5059                         tv.tv_usec = 0;
5060
5061                         FD_ZERO(&rfds);
5062                         FD_SET(*fd, &rfds);
5063                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
5064                                 *Error = strerror(errno);
5065                                 close (*fd);
5066                                 *fd = -1;
5067                                 if (*Error == NULL)
5068                                         *Error = ErrRBLF_SelectFailed;
5069                                 return -1;
5070                         }
5071                         if (! FD_ISSET(*fd, &rfds) != 0) {
5072                                 nSuccessLess ++;
5073                                 continue;
5074                         }
5075                 }
5076
5077                 if ((rlen = read(*fd,
5078                                  ptr,
5079                                  nBytes - nRead)) == -1) {
5080                         close(*fd);
5081                         *fd = -1;
5082                         *Error = strerror(errno);
5083                         return rlen;
5084                 }
5085                 nRead += rlen;
5086                 ptr += rlen;
5087                 Buf->BufUsed += rlen;
5088         }
5089         Buf->buf[Buf->BufUsed] = '\0';
5090         return nRead;
5091 }
5092
5093 const char *ErrRBB_BLOBFPreConditionFailed = "StrBufReadBLOBBuffered: to many selects; aborting.";
5094 const char *ErrRBB_too_many_selects        = "StrBufReadBLOBBuffered: to many selects; aborting.";
5095 /**
5096  * @ingroup StrBuf_BufferedIO
5097  * @brief Input binary data from socket
5098  * flushes and closes the FD on error
5099  * @param Blob put binary thing here
5100  * @param IOBuf the buffer to get the input to
5101  * @param Pos offset inside of IOBuf
5102  * @param fd pointer to the filedescriptor to read
5103  * @param append Append to an existing string or replace?
5104  * @param nBytes the maximal number of bytes to read
5105  * @param check whether we should search for '000\n' terminators in case of timeouts
5106  * @param Error strerror() on error
5107  * @returns numbers of chars read
5108  */
5109 int StrBufReadBLOBBuffered(StrBuf *Blob,
5110                            StrBuf *IOBuf,
5111                            const char **Pos,
5112                            int *fd,
5113                            int append,
5114                            long nBytes,
5115                            int check,
5116                            const char **Error)
5117 {
5118         const char *pos;
5119         int fdflags;
5120         int rlen = 0;
5121         int nRead = 0;
5122         int nAlreadyRead = 0;
5123         int IsNonBlock;
5124         char *ptr;
5125         fd_set rfds;
5126         struct timeval tv;
5127         int nSuccessLess = 0;
5128         int MaxTries;
5129
5130         if ((Blob == NULL)  ||
5131             (*fd == -1)     ||
5132             (IOBuf == NULL) ||
5133             (Pos == NULL))
5134         {
5135                 if (Pos != NULL)
5136                         *Pos = NULL;
5137                 *Error = ErrRBB_BLOBFPreConditionFailed;
5138                 return -1;
5139         }
5140
5141         if (!append)
5142                 FlushStrBuf(Blob);
5143         if (Blob->BufUsed + nBytes >= Blob->BufSize)
5144                 IncreaseBuf(Blob, append, Blob->BufUsed + nBytes);
5145
5146         pos = *Pos;
5147
5148         if (pos != NULL)
5149                 rlen = pos - IOBuf->buf;
5150         rlen = IOBuf->BufUsed - rlen;
5151
5152
5153         if ((IOBuf->BufUsed > 0) &&
5154             (pos != NULL) &&
5155             (pos < IOBuf->buf + IOBuf->BufUsed))
5156         {
5157                 if (rlen < nBytes) {
5158                         memcpy(Blob->buf + Blob->BufUsed, pos, rlen);
5159                         Blob->BufUsed += rlen;
5160                         Blob->buf[Blob->BufUsed] = '\0';
5161                         nAlreadyRead = nRead = rlen;
5162                         *Pos = NULL;
5163                 }
5164                 if (rlen >= nBytes) {
5165                         memcpy(Blob->buf + Blob->BufUsed, pos, nBytes);
5166                         Blob->BufUsed += nBytes;
5167                         Blob->buf[Blob->BufUsed] = '\0';
5168                         if (rlen == nBytes) {
5169                                 *Pos = NULL;
5170                                 FlushStrBuf(IOBuf);
5171                         }
5172                         else
5173                                 *Pos += nBytes;
5174                         return nBytes;
5175                 }
5176         }
5177
5178         FlushStrBuf(IOBuf);
5179         *Pos = NULL;
5180         if (IOBuf->BufSize < nBytes - nRead)
5181                 IncreaseBuf(IOBuf, 0, nBytes - nRead);
5182         ptr = IOBuf->buf;
5183
5184         fdflags = fcntl(*fd, F_GETFL);
5185         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
5186         if (IsNonBlock)
5187                 MaxTries =   1000;
5188         else
5189                 MaxTries = 100000;
5190
5191         nBytes -= nRead;
5192         nRead = 0;
5193         while ((nSuccessLess < MaxTries) &&
5194                (nRead < nBytes) &&
5195                (*fd != -1)) {
5196                 if (IsNonBlock)
5197                 {
5198                         tv.tv_sec = 1;
5199                         tv.tv_usec = 0;
5200
5201                         FD_ZERO(&rfds);
5202                         FD_SET(*fd, &rfds);
5203                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
5204                                 *Error = strerror(errno);
5205                                 close (*fd);
5206                                 *fd = -1;
5207                                 if (*Error == NULL)
5208                                         *Error = ErrRBLF_SelectFailed;
5209                                 return -1;
5210                         }
5211                         if (! FD_ISSET(*fd, &rfds) != 0) {
5212                                 nSuccessLess ++;
5213                                 continue;
5214                         }
5215                 }
5216                 rlen = read(*fd,
5217                             ptr,
5218                             IOBuf->BufSize - (ptr - IOBuf->buf));
5219                 if (rlen == -1) {
5220                         close(*fd);
5221                         *fd = -1;
5222                         *Error = strerror(errno);
5223                         return rlen;
5224                 }
5225                 else if (rlen == 0){
5226                         if ((check == NNN_TERM) &&
5227                             (nRead > 5) &&
5228                             (strncmp(IOBuf->buf + IOBuf->BufUsed - 5, "\n000\n", 5) == 0))
5229                         {
5230                                 StrBufPlain(Blob, HKEY("\n000\n"));
5231                                 StrBufCutRight(Blob, 5);
5232                                 return Blob->BufUsed;
5233                         }
5234                         else if (!IsNonBlock)
5235                                 nSuccessLess ++;
5236                         else if (nSuccessLess > MaxTries) {
5237                                 FlushStrBuf(IOBuf);
5238                                 *Error = ErrRBB_too_many_selects;
5239                                 return -1;
5240                         }
5241                 }
5242                 else if (rlen > 0) {
5243                         nSuccessLess = 0;
5244                         nRead += rlen;
5245                         ptr += rlen;
5246                         IOBuf->BufUsed += rlen;
5247                 }
5248         }
5249         if (nSuccessLess >= MaxTries) {
5250                 FlushStrBuf(IOBuf);
5251                 *Error = ErrRBB_too_many_selects;
5252                 return -1;
5253         }
5254
5255         if (nRead > nBytes) {
5256                 *Pos = IOBuf->buf + nBytes;
5257         }
5258         Blob->buf[Blob->BufUsed] = '\0';
5259         StrBufAppendBufPlain(Blob, IOBuf->buf, nBytes, 0);
5260         if (*Pos == NULL) {
5261                 FlushStrBuf(IOBuf);
5262         }
5263         return nRead + nAlreadyRead;
5264 }
5265
5266 /**
5267  * @ingroup StrBuf_IO
5268  * @brief extract a "next line" from Buf; Ptr to persist across several iterations
5269  * @param LineBuf your line will be copied here.
5270  * @param Buf BLOB with lines of text...
5271  * @param Ptr moved arround to keep the next-line across several iterations
5272  *        has to be &NULL on start; will be &NotNULL on end of buffer
5273  * @returns size of remaining buffer
5274  */
5275 int StrBufSipLine(StrBuf *LineBuf, const StrBuf *Buf, const char **Ptr)
5276 {
5277         const char *aptr, *ptr, *eptr;
5278         char *optr, *xptr;
5279
5280         if ((Buf == NULL) ||
5281             (*Ptr == StrBufNOTNULL) ||
5282             (LineBuf == NULL)||
5283             (LineBuf->buf == NULL))
5284         {
5285                 *Ptr = StrBufNOTNULL;
5286                 return 0;
5287         }
5288
5289         FlushStrBuf(LineBuf);
5290         if (*Ptr==NULL)
5291                 ptr = aptr = Buf->buf;
5292         else
5293                 ptr = aptr = *Ptr;
5294
5295         optr = LineBuf->buf;
5296         eptr = Buf->buf + Buf->BufUsed;
5297         xptr = LineBuf->buf + LineBuf->BufSize - 1;
5298
5299         while ((ptr <= eptr) &&
5300                (*ptr != '\n') &&
5301                (*ptr != '\r') )
5302         {
5303                 *optr = *ptr;
5304                 optr++; ptr++;
5305                 if (optr == xptr) {
5306                         LineBuf->BufUsed = optr - LineBuf->buf;
5307                         IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
5308                         optr = LineBuf->buf + LineBuf->BufUsed;
5309                         xptr = LineBuf->buf + LineBuf->BufSize - 1;
5310                 }
5311         }
5312
5313         if ((ptr >= eptr) && (optr > LineBuf->buf))
5314                 optr --;
5315         LineBuf->BufUsed = optr - LineBuf->buf;
5316         *optr = '\0';
5317         if ((ptr <= eptr) && (*ptr == '\r'))
5318                 ptr ++;
5319         if ((ptr <= eptr) && (*ptr == '\n'))
5320                 ptr ++;
5321
5322         if (ptr < eptr) {
5323                 *Ptr = ptr;
5324         }
5325         else {
5326                 *Ptr = StrBufNOTNULL;
5327         }
5328
5329         return Buf->BufUsed - (ptr - Buf->buf);
5330 }
5331
5332
5333 /**
5334  * @ingroup StrBuf_IO
5335  * @brief removes double slashes from pathnames
5336  * @param Dir directory string to filter
5337  * @param RemoveTrailingSlash allows / disallows trailing slashes
5338  */
5339 void StrBufStripSlashes(StrBuf *Dir, int RemoveTrailingSlash)
5340 {
5341         char *a, *b;
5342
5343         a = b = Dir->buf;
5344
5345         while (!IsEmptyStr(a)) {
5346                 if (*a == '/') {
5347                         while (*a == '/')
5348                                 a++;
5349                         *b = '/';
5350                         b++;
5351                 }
5352                 else {
5353                         *b = *a;
5354                         b++; a++;
5355                 }
5356         }
5357         if ((RemoveTrailingSlash) &&
5358             (b > Dir->buf) &&
5359             (*(b - 1) == '/')){
5360                 b--;
5361         }
5362         *b = '\0';
5363         Dir->BufUsed = b - Dir->buf;
5364 }
5365
5366