libcitadel/lib/stringbuf.c

   1 // Copyright (c) 1987-2022 by the citadel.org team
   2 //
   3 // This program is open source software; you can redistribute it and/or modify
   4 // it under the terms of the GNU General Public License as published by
   5 // the Free Software Foundation; either version 3 of the License, or
   6 // (at your option) any later version.
   7 //
   8 // This program is distributed in the hope that it will be useful,
   9 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 // GNU General Public License for more details.
  12 //
  13 // You should have received a copy of the GNU General Public License
  14 // along with this program; if not, write to the Free Software
  15 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  16
  17 #define _GNU_SOURCE
  18 #include "sysdep.h"
  19 #include <ctype.h>
  20 #include <errno.h>
  21 #include <string.h>
  22 #include <unistd.h>
  23 #include <string.h>
  24 #include <stdio.h>
  25 #include <sys/select.h>
  26 #include <fcntl.h>
  27 #include <sys/types.h>
  28 #define SHOW_ME_VAPPEND_PRINTF
  29 #include <stdarg.h>
  30
  31 #include "libcitadel.h"
  32
  33 #ifdef HAVE_ICONV
  34 #include <iconv.h>
  35 #endif
  36
  37 #ifdef HAVE_BACKTRACE
  38 #include <execinfo.h>
  39 #endif
  40
  41 #ifdef UNDEF_MEMCPY
  42 #undef memcpy
  43 #endif
  44
  45 #ifdef HAVE_ZLIB
  46 #include <zlib.h>
  47 int ZEXPORT compress_gzip(Bytef * dest, size_t * destLen,
  48                           const Bytef * source, uLong sourceLen, int level);
  49 #endif
  50 int BaseStrBufSize = 64;
  51 int EnableSplice = 0;
  52 int ZLibCompressionRatio = -1; /* defaults to 6 */
  53 #ifdef HAVE_ZLIB
  54 #define DEF_MEM_LEVEL 8 /*< memlevel??? */
  55 #define OS_CODE 0x03    /*< unix */
  56 const int gz_magic[2] = { 0x1f, 0x8b }; /* gzip magic header */
  57 #endif
  58
  59 const char *StrBufNOTNULL = ((char*) NULL) - 1;
  60
  61 const char HexList[256][3] = {
  62         "00","01","02","03","04","05","06","07","08","09","0A","0B","0C","0D","0E","0F",
  63         "10","11","12","13","14","15","16","17","18","19","1A","1B","1C","1D","1E","1F",
  64         "20","21","22","23","24","25","26","27","28","29","2A","2B","2C","2D","2E","2F",
  65         "30","31","32","33","34","35","36","37","38","39","3A","3B","3C","3D","3E","3F",
  66         "40","41","42","43","44","45","46","47","48","49","4A","4B","4C","4D","4E","4F",
  67         "50","51","52","53","54","55","56","57","58","59","5A","5B","5C","5D","5E","5F",
  68         "60","61","62","63","64","65","66","67","68","69","6A","6B","6C","6D","6E","6F",
  69         "70","71","72","73","74","75","76","77","78","79","7A","7B","7C","7D","7E","7F",
  70         "80","81","82","83","84","85","86","87","88","89","8A","8B","8C","8D","8E","8F",
  71         "90","91","92","93","94","95","96","97","98","99","9A","9B","9C","9D","9E","9F",
  72         "A0","A1","A2","A3","A4","A5","A6","A7","A8","A9","AA","AB","AC","AD","AE","AF",
  73         "B0","B1","B2","B3","B4","B5","B6","B7","B8","B9","BA","BB","BC","BD","BE","BF",
  74         "C0","C1","C2","C3","C4","C5","C6","C7","C8","C9","CA","CB","CC","CD","CE","CF",
  75         "D0","D1","D2","D3","D4","D5","D6","D7","D8","D9","DA","DB","DC","DD","DE","DF",
  76         "E0","E1","E2","E3","E4","E5","E6","E7","E8","E9","EA","EB","EC","ED","EE","EF",
  77         "F0","F1","F2","F3","F4","F5","F6","F7","F8","F9","FA","FB","FC","FD","FE","FF"};
  78
  79
  80 /*
  81  * Private Structure for the Stringbuffer
  82  */
  83 struct StrBuf {
  84         char *buf;         /**< the pointer to the dynamic buffer */
  85         long BufSize;      /**< how many spcae do we optain */
  86         long BufUsed;      /**< StNumber of Chars used excluding the trailing \\0 */
  87         int ConstBuf;      /**< are we just a wrapper arround a static buffer and musn't we be changed? */
  88 #ifdef SIZE_DEBUG
  89         long nIncreases;   /**< for profiling; cound how many times we needed more */
  90         char bt [SIZ];     /**< Stacktrace of last increase */
  91         char bt_lastinc [SIZ]; /**< How much did we increase last time? */
  92 #endif
  93 };
  94
  95
  96 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE);
  97 static inline int Ctdl_IsUtf8SequenceStart(const char Char);
  98
  99 #ifdef SIZE_DEBUG
 100 #ifdef HAVE_BACKTRACE
 101 static void StrBufBacktrace(StrBuf *Buf, int which) {
 102         int n;
 103         char *pstart, *pch;
 104         void *stack_frames[50];
 105         size_t size, i;
 106         char **strings;
 107
 108         if (which)
 109                 pstart = pch = Buf->bt;
 110         else
 111                 pstart = pch = Buf->bt_lastinc;
 112         size = backtrace(stack_frames, sizeof(stack_frames) / sizeof(void*));
 113         strings = backtrace_symbols(stack_frames, size);
 114         for (i = 0; i < size; i++) {
 115                 if (strings != NULL)
 116                         n = snprintf(pch, SIZ - (pch - pstart), "%s\\n", strings[i]);
 117                 else
 118                         n = snprintf(pch, SIZ - (pch - pstart), "%p\\n", stack_frames[i]);
 119                 pch += n;
 120         }
 121         free(strings);
 122
 123
 124 }
 125 #endif
 126
 127
 128 void dbg_FreeStrBuf(StrBuf *FreeMe, char *FromWhere) {
 129         if (hFreeDbglog == -1) {
 130                 pid_t pid = getpid();
 131                 char path [SIZ];
 132                 snprintf(path, SIZ, "/tmp/libcitadel_strbuf_realloc.log.%d", pid);
 133                 hFreeDbglog = open(path, O_APPEND|O_CREAT|O_WRONLY);
 134         }
 135         if ((*FreeMe)->nIncreases > 0) {
 136                 char buf[SIZ * 3];
 137                 long n;
 138                 n = snprintf(buf, SIZ * 3, "%c+|%ld|%ld|%ld|%s|%s|\n",
 139                              FromWhere,
 140                              (*FreeMe)->nIncreases,
 141                              (*FreeMe)->BufUsed,
 142                              (*FreeMe)->BufSize,
 143                              (*FreeMe)->bt,
 144                              (*FreeMe)->bt_lastinc);
 145                 n = write(hFreeDbglog, buf, n);
 146         }
 147         else {
 148                 char buf[128];
 149                 long n;
 150                 n = snprintf(buf, 128, "%c_|0|%ld%ld|\n",
 151                              FromWhere,
 152                              (*FreeMe)->BufUsed,
 153                              (*FreeMe)->BufSize);
 154                 n = write(hFreeDbglog, buf, n);
 155         }
 156 }
 157
 158
 159 void dbg_IncreaseBuf(StrBuf *IncMe) {
 160         Buf->nIncreases++;
 161 #ifdef HAVE_BACKTRACE
 162         StrBufBacktrace(Buf, 1);
 163 #endif
 164 }
 165
 166
 167 void dbg_Init(StrBuf *Buf) {
 168         Buf->nIncreases = 0;
 169         Buf->bt[0] = '\0';
 170         Buf->bt_lastinc[0] = '\0';
 171 #ifdef HAVE_BACKTRACE
 172         StrBufBacktrace(Buf, 0);
 173 #endif
 174 }
 175
 176 #else
 177 /* void it... */
 178 #define dbg_FreeStrBuf(a, b)
 179 #define dbg_IncreaseBuf(a)
 180 #define dbg_Init(a)
 181
 182 #endif
 183
 184 /*
 185  *  swaps the contents of two StrBufs
 186  * this is to be used to have cheap switched between a work-buffer and a target buffer
 187  *  A First one
 188  *  B second one
 189  */
 190 static inline void iSwapBuffers(StrBuf *A, StrBuf *B) {
 191         StrBuf C;
 192
 193         memcpy(&C, A, sizeof(*A));
 194         memcpy(A, B, sizeof(*B));
 195         memcpy(B, &C, sizeof(C));
 196
 197 }
 198
 199
 200 void SwapBuffers(StrBuf *A, StrBuf *B) {
 201         iSwapBuffers(A, B);
 202 }
 203
 204
 205 /*
 206  *  Cast operator to Plain String
 207  * @note if the buffer is altered by StrBuf operations, this pointer may become
 208  *  invalid. So don't lean on it after altering the buffer!
 209  *  Since this operation is considered cheap, rather call it often than risking
 210  *  your pointer to become invalid!
 211  *  Str the string we want to get the c-string representation for
 212  * @returns the Pointer to the Content. Don't mess with it!
 213  */
 214 inline const char *ChrPtr(const StrBuf *Str) {
 215         if (Str == NULL)
 216                 return "";
 217         return Str->buf;
 218 }
 219
 220
 221 /*
 222  *  since we know strlen()'s result, provide it here.
 223  *  Str the string to return the length to
 224  * @returns contentlength of the buffer
 225  */
 226 inline int StrLength(const StrBuf *Str) {
 227         return (Str != NULL) ? Str->BufUsed : 0;
 228 }
 229
 230
 231 // local utility function to resize the buffer
 232 // Buf          the buffer whichs storage we should increase
 233 // KeepOriginal should we copy the original buffer or just start over with a new one
 234 // DestSize     what should fit in after?
 235 static int IncreaseBuf(StrBuf *Buf, int KeepOriginal, int DestSize) {
 236         char *NewBuf;
 237         size_t NewSize = Buf->BufSize * 2;
 238
 239         if (Buf->ConstBuf) {
 240                 return -1;
 241         }
 242
 243         if (DestSize > 0) {
 244                 while ((NewSize <= DestSize) && (NewSize != 0)) {
 245                         NewSize *= 2;
 246                 }
 247         }
 248
 249         if (NewSize == 0) {
 250                 return -1;
 251         }
 252
 253         NewBuf = (char*) malloc(NewSize);
 254         if (NewBuf == NULL) {
 255                 return -1;
 256         }
 257
 258         if (KeepOriginal && (Buf->BufUsed > 0)) {
 259                 memcpy(NewBuf, Buf->buf, Buf->BufUsed);
 260         }
 261         else {
 262                 NewBuf[0] = '\0';
 263                 Buf->BufUsed = 0;
 264         }
 265         free (Buf->buf);
 266         Buf->buf = NewBuf;
 267         Buf->BufSize = NewSize;
 268
 269         dbg_IncreaseBuf(Buf);
 270
 271         return Buf->BufSize;
 272 }
 273
 274
 275 // shrink / increase an _EMPTY_ buffer to NewSize. Buffercontent is thoroughly ignored and flushed.
 276 // Buf          Buffer to shrink (has to be empty)
 277 // ThreshHold   if the buffer is bigger then this, its readjusted
 278 // NewSize      if we Shrink it, how big are we going to be afterwards?
 279 void ReAdjustEmptyBuf(StrBuf *Buf, long ThreshHold, long NewSize) {
 280         if ((Buf != NULL) && (Buf->BufUsed == 0) && (Buf->BufSize < ThreshHold)) {
 281                 free(Buf->buf);
 282                 Buf->buf = (char*) malloc(NewSize);
 283                 Buf->BufUsed = 0;
 284                 Buf->BufSize = NewSize;
 285         }
 286 }
 287
 288
 289 /*
 290  *  shrink long term buffers to their real size so they don't waste memory
 291  *  Buf buffer to shrink
 292  *  Force if not set, will just executed if the buffer is much to big; set for lifetime strings
 293  * @returns physical size of the buffer
 294  */
 295 long StrBufShrinkToFit(StrBuf *Buf, int Force) {
 296         if (Buf == NULL)
 297                 return -1;
 298         if (Force || (Buf->BufUsed + (Buf->BufUsed / 3) > Buf->BufSize)) {
 299                 char *TmpBuf;
 300
 301                 TmpBuf = (char*) malloc(Buf->BufUsed + 1);
 302                 if (TmpBuf == NULL)
 303                         return -1;
 304
 305                 memcpy (TmpBuf, Buf->buf, Buf->BufUsed + 1);
 306                 Buf->BufSize = Buf->BufUsed + 1;
 307                 free(Buf->buf);
 308                 Buf->buf = TmpBuf;
 309         }
 310         return Buf->BufUsed;
 311 }
 312
 313
 314 /*
 315  *  Allocate a new buffer with default buffer size
 316  * @returns the new stringbuffer
 317  */
 318 StrBuf *NewStrBuf(void) {
 319         StrBuf *NewBuf;
 320
 321         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 322         if (NewBuf == NULL)
 323                 return NULL;
 324
 325         NewBuf->buf = (char*) malloc(BaseStrBufSize);
 326         if (NewBuf->buf == NULL) {
 327                 free(NewBuf);
 328                 return NULL;
 329         }
 330         NewBuf->buf[0] = '\0';
 331         NewBuf->BufSize = BaseStrBufSize;
 332         NewBuf->BufUsed = 0;
 333         NewBuf->ConstBuf = 0;
 334
 335         dbg_Init (NewBuf);
 336         return NewBuf;
 337 }
 338
 339
 340 /*
 341  *  Copy Constructor; returns a duplicate of CopyMe
 342  *  CopyMe Buffer to faxmilate
 343  * @returns the new stringbuffer
 344  */
 345 StrBuf *NewStrBufDup(const StrBuf *CopyMe) {
 346         StrBuf *NewBuf;
 347
 348         if (CopyMe == NULL)
 349                 return NewStrBuf();
 350
 351         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 352         if (NewBuf == NULL)
 353                 return NULL;
 354
 355         NewBuf->buf = (char*) malloc(CopyMe->BufSize);
 356         if (NewBuf->buf == NULL) {
 357                 free(NewBuf);
 358                 return NULL;
 359         }
 360
 361         memcpy(NewBuf->buf, CopyMe->buf, CopyMe->BufUsed + 1);
 362         NewBuf->BufUsed = CopyMe->BufUsed;
 363         NewBuf->BufSize = CopyMe->BufSize;
 364         NewBuf->ConstBuf = 0;
 365
 366         dbg_Init(NewBuf);
 367
 368         return NewBuf;
 369 }
 370
 371
 372 /*
 373  *  Copy Constructor; CreateRelpaceMe will contain CopyFlushMe afterwards.
 374  *  NoMe if non-NULL, we will use that buffer as value; KeepOriginal will abused as len.
 375  *  CopyFlushMe Buffer to faxmilate if KeepOriginal, or to move into CreateRelpaceMe if !KeepOriginal.
 376  *  CreateRelpaceMe If NULL, will be created, else Flushed and filled CopyFlushMe
 377  *  KeepOriginal should CopyFlushMe remain intact? or may we Steal its buffer?
 378  * @returns the new stringbuffer
 379  */
 380 void NewStrBufDupAppendFlush(StrBuf **CreateRelpaceMe, StrBuf *CopyFlushMe, const char *NoMe, int KeepOriginal) {
 381         StrBuf *NewBuf;
 382
 383         if (CreateRelpaceMe == NULL)
 384                 return;
 385
 386         if (NoMe != NULL) {
 387                 if (*CreateRelpaceMe != NULL)
 388                         StrBufPlain(*CreateRelpaceMe, NoMe, KeepOriginal);
 389                 else
 390                         *CreateRelpaceMe = NewStrBufPlain(NoMe, KeepOriginal);
 391                 return;
 392         }
 393
 394         if (CopyFlushMe == NULL) {
 395                 if (*CreateRelpaceMe != NULL)
 396                         FlushStrBuf(*CreateRelpaceMe);
 397                 else
 398                         *CreateRelpaceMe = NewStrBuf();
 399                 return;
 400         }
 401
 402         /*
 403          * Randomly Chosen: bigger than 64 chars is cheaper to swap the buffers instead of copying.
 404          * else *CreateRelpaceMe may use more memory than needed in a longer term, CopyFlushMe might
 405          * be a big IO-Buffer...
 406          */
 407         if (KeepOriginal || (StrLength(CopyFlushMe) < 256)) {
 408                 if (*CreateRelpaceMe == NULL) {
 409                         *CreateRelpaceMe = NewBuf = NewStrBufPlain(NULL, CopyFlushMe->BufUsed);
 410                         dbg_Init(NewBuf);
 411                 }
 412                 else {
 413                         NewBuf = *CreateRelpaceMe;
 414                         FlushStrBuf(NewBuf);
 415                 }
 416                 StrBufAppendBuf(NewBuf, CopyFlushMe, 0);
 417         }
 418         else {
 419                 if (*CreateRelpaceMe == NULL) {
 420                         *CreateRelpaceMe = NewBuf = NewStrBufPlain(NULL, CopyFlushMe->BufUsed);
 421                         dbg_Init(NewBuf);
 422                 }
 423                 else  {
 424                         NewBuf = *CreateRelpaceMe;
 425                 }
 426                 iSwapBuffers (NewBuf, CopyFlushMe);
 427         }
 428         if (!KeepOriginal) {
 429                 FlushStrBuf(CopyFlushMe);
 430         }
 431         return;
 432 }
 433
 434
 435 /*
 436  *  create a new Buffer using an existing c-string
 437  * this function should also be used if you want to pre-suggest
 438  * the buffer size to allocate in conjunction with ptr == NULL
 439  *  ptr the c-string to copy; may be NULL to create a blank instance
 440  *  nChars How many chars should we copy; -1 if we should measure the length ourselves
 441  * @returns the new stringbuffer
 442  */
 443 StrBuf *NewStrBufPlain(const char* ptr, int nChars) {
 444         StrBuf *NewBuf;
 445         size_t Siz = BaseStrBufSize;
 446         size_t CopySize;
 447
 448         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 449         if (NewBuf == NULL)
 450                 return NULL;
 451
 452         if (nChars < 0)
 453                 CopySize = strlen((ptr != NULL)?ptr:"");
 454         else
 455                 CopySize = nChars;
 456
 457         while ((Siz <= CopySize) && (Siz != 0))
 458                 Siz *= 2;
 459
 460         if (Siz == 0) {
 461                 free(NewBuf);
 462                 return NULL;
 463         }
 464
 465         NewBuf->buf = (char*) malloc(Siz);
 466         if (NewBuf->buf == NULL) {
 467                 free(NewBuf);
 468                 return NULL;
 469         }
 470         NewBuf->BufSize = Siz;
 471         if (ptr != NULL) {
 472                 memcpy(NewBuf->buf, ptr, CopySize);
 473                 NewBuf->buf[CopySize] = '\0';
 474                 NewBuf->BufUsed = CopySize;
 475         }
 476         else {
 477                 NewBuf->buf[0] = '\0';
 478                 NewBuf->BufUsed = 0;
 479         }
 480         NewBuf->ConstBuf = 0;
 481
 482         dbg_Init(NewBuf);
 483
 484         return NewBuf;
 485 }
 486
 487
 488 /*
 489  *  Set an existing buffer from a c-string
 490  *  Buf buffer to load
 491  *  ptr c-string to put into
 492  *  nChars set to -1 if we should work 0-terminated
 493  * @returns the new length of the string
 494  */
 495 int StrBufPlain(StrBuf *Buf, const char* ptr, int nChars) {
 496         size_t Siz;
 497         size_t CopySize;
 498
 499         if (Buf == NULL)
 500                 return -1;
 501         if (ptr == NULL) {
 502                 FlushStrBuf(Buf);
 503                 return -1;
 504         }
 505
 506         Siz = Buf->BufSize;
 507
 508         if (nChars < 0)
 509                 CopySize = strlen(ptr);
 510         else
 511                 CopySize = nChars;
 512
 513         while ((Siz <= CopySize) && (Siz != 0))
 514                 Siz *= 2;
 515
 516         if (Siz == 0) {
 517                 FlushStrBuf(Buf);
 518                 return -1;
 519         }
 520
 521         if (Siz != Buf->BufSize)
 522                 IncreaseBuf(Buf, 0, Siz);
 523         memcpy(Buf->buf, ptr, CopySize);
 524         Buf->buf[CopySize] = '\0';
 525         Buf->BufUsed = CopySize;
 526         Buf->ConstBuf = 0;
 527         return CopySize;
 528 }
 529
 530
 531 /**
 532  *  use strbuf as wrapper for a string constant for easy handling
 533  *  StringConstant a string to wrap
 534  *  SizeOfStrConstant should be sizeof(StringConstant)-1
 535  */
 536 StrBuf* _NewConstStrBuf(const char* StringConstant, size_t SizeOfStrConstant)
 537 {
 538         StrBuf *NewBuf;
 539
 540         NewBuf = (StrBuf*) malloc(sizeof(StrBuf));
 541         if (NewBuf == NULL)
 542                 return NULL;
 543         NewBuf->buf = (char*) StringConstant;
 544         NewBuf->BufSize = SizeOfStrConstant;
 545         NewBuf->BufUsed = SizeOfStrConstant;
 546         NewBuf->ConstBuf = 1;
 547
 548         dbg_Init(NewBuf);
 549
 550         return NewBuf;
 551 }
 552
 553
 554 /**
 555  *  flush the content of a Buf; keep its struct
 556  *  buf Buffer to flush
 557  */
 558 int FlushStrBuf(StrBuf *buf)
 559 {
 560         if ((buf == NULL) || (buf->buf == NULL))
 561                 return -1;
 562         if (buf->ConstBuf)
 563                 return -1;
 564         buf->buf[0] ='\0';
 565         buf->BufUsed = 0;
 566         return 0;
 567 }
 568
 569 /**
 570  *  wipe the content of a Buf thoroughly (overwrite it -> expensive); keep its struct
 571  *  buf Buffer to wipe
 572  */
 573 int FLUSHStrBuf(StrBuf *buf)
 574 {
 575         if (buf == NULL)
 576                 return -1;
 577         if (buf->ConstBuf)
 578                 return -1;
 579         if (buf->BufUsed > 0) {
 580                 memset(buf->buf, 0, buf->BufUsed);
 581                 buf->BufUsed = 0;
 582         }
 583         return 0;
 584 }
 585
 586 #ifdef SIZE_DEBUG
 587 int hFreeDbglog = -1;
 588 #endif
 589 /**
 590  *  Release a Buffer
 591  * Its a double pointer, so it can NULL your pointer
 592  * so fancy SIG11 appear instead of random results
 593  *  FreeMe Pointer Pointer to the buffer to free
 594  */
 595 void FreeStrBuf (StrBuf **FreeMe)
 596 {
 597         if (*FreeMe == NULL)
 598                 return;
 599
 600         dbg_FreeStrBuf(FreeMe, 'F');
 601
 602         if (!(*FreeMe)->ConstBuf)
 603                 free((*FreeMe)->buf);
 604         free(*FreeMe);
 605         *FreeMe = NULL;
 606 }
 607
 608 /*
 609  *  flatten a Buffer to the Char * we return
 610  * Its a double pointer, so it can NULL your pointer
 611  * so fancy SIG11 appear instead of random results
 612  * The Callee then owns the buffer and is responsible for freeing it.
 613  *  SmashMe Pointer Pointer to the buffer to release Buf from and free
 614  * @returns the pointer of the buffer; Callee owns the memory thereafter.
 615  */
 616 char *SmashStrBuf (StrBuf **SmashMe) {
 617         char *Ret;
 618
 619         if ((SmashMe == NULL) || (*SmashMe == NULL))
 620                 return NULL;
 621
 622         dbg_FreeStrBuf(SmashMe, 'S');
 623
 624         Ret = (*SmashMe)->buf;
 625         free(*SmashMe);
 626         *SmashMe = NULL;
 627         return Ret;
 628 }
 629
 630
 631 /*
 632  *  Release the buffer
 633  * If you want put your StrBuf into a Hash, use this as Destructor.
 634  *  VFreeMe untyped pointer to a StrBuf. be shure to do the right thing [TM]
 635  */
 636 void HFreeStrBuf (void *VFreeMe) {
 637         StrBuf *FreeMe = (StrBuf*)VFreeMe;
 638         if (FreeMe == NULL)
 639                 return;
 640
 641         dbg_FreeStrBuf(SmashMe, 'H');
 642
 643         if (!FreeMe->ConstBuf)
 644                 free(FreeMe->buf);
 645         free(FreeMe);
 646 }
 647
 648
 649 /*******************************************************************************
 650  *                      Simple string transformations                          *
 651  *******************************************************************************/
 652
 653 /**
 654  *  Wrapper around atol
 655  */
 656 long StrTol(const StrBuf *Buf)
 657 {
 658         if (Buf == NULL)
 659                 return 0;
 660         if(Buf->BufUsed > 0)
 661                 return atol(Buf->buf);
 662         else
 663                 return 0;
 664 }
 665
 666 /**
 667  *  Wrapper around atoi
 668  */
 669 int StrToi(const StrBuf *Buf)
 670 {
 671         if (Buf == NULL)
 672                 return 0;
 673         if (Buf->BufUsed > 0)
 674                 return atoi(Buf->buf);
 675         else
 676                 return 0;
 677 }
 678
 679 /**
 680  *  Checks to see if the string is a pure number
 681  *  Buf The buffer to inspect
 682  * @returns 1 if its a pure number, 0, if not.
 683  */
 684 int StrBufIsNumber(const StrBuf *Buf) {
 685         char * pEnd;
 686         if ((Buf == NULL) || (Buf->BufUsed == 0)) {
 687                 return 0;
 688         }
 689         strtoll(Buf->buf, &pEnd, 10);
 690         if (pEnd == Buf->buf)
 691                 return 0;
 692         if ((pEnd != NULL) && (pEnd == Buf->buf + Buf->BufUsed))
 693                 return 1;
 694         if (Buf->buf == pEnd)
 695                 return 0;
 696         return 0;
 697 }
 698
 699 /**
 700  *  modifies a Single char of the Buf
 701  * You can point to it via char* or a zero-based integer
 702  *  Buf The buffer to manipulate
 703  *  ptr char* to zero; use NULL if unused
 704  *  nThChar zero based pointer into the string; use -1 if unused
 705  *  PeekValue The Character to place into the position
 706  */
 707 long StrBufPeek(StrBuf *Buf, const char* ptr, long nThChar, char PeekValue)
 708 {
 709         if (Buf == NULL)
 710                 return -1;
 711         if (ptr != NULL)
 712                 nThChar = ptr - Buf->buf;
 713         if ((nThChar < 0) || (nThChar > Buf->BufUsed))
 714                 return -1;
 715         Buf->buf[nThChar] = PeekValue;
 716         return nThChar;
 717 }
 718
 719 /**
 720  *  modifies a range of chars of the Buf
 721  * You can point to it via char* or a zero-based integer
 722  *  Buf The buffer to manipulate
 723  *  ptr char* to zero; use NULL if unused
 724  *  nThChar zero based pointer into the string; use -1 if unused
 725  *  nChars how many chars are to be flushed?
 726  *  PookValue The Character to place into that area
 727  */
 728 long StrBufPook(StrBuf *Buf, const char* ptr, long nThChar, long nChars, char PookValue)
 729 {
 730         if (Buf == NULL)
 731                 return -1;
 732         if (ptr != NULL)
 733                 nThChar = ptr - Buf->buf;
 734         if ((nThChar < 0) || (nThChar > Buf->BufUsed))
 735                 return -1;
 736         if (nThChar + nChars > Buf->BufUsed)
 737                 nChars =  Buf->BufUsed - nThChar;
 738
 739         memset(Buf->buf + nThChar, PookValue, nChars);
 740         /* just to be shure... */
 741         Buf->buf[Buf->BufUsed] = 0;
 742         return nChars;
 743 }
 744
 745 /**
 746  *  Append a StringBuffer to the buffer
 747  *  Buf Buffer to modify
 748  *  AppendBuf Buffer to copy at the end of our buffer
 749  *  Offset Should we start copying from an offset?
 750  */
 751 void StrBufAppendBuf(StrBuf *Buf, const StrBuf *AppendBuf, unsigned long Offset)
 752 {
 753         if ((AppendBuf == NULL) || (AppendBuf->buf == NULL) ||
 754             (Buf == NULL) || (Buf->buf == NULL))
 755                 return;
 756
 757         if (Buf->BufSize - Offset < AppendBuf->BufUsed + Buf->BufUsed + 1)
 758                 IncreaseBuf(Buf, (Buf->BufUsed > 0), AppendBuf->BufUsed + Buf->BufUsed);
 759
 760         memcpy(Buf->buf + Buf->BufUsed, AppendBuf->buf + Offset, AppendBuf->BufUsed - Offset);
 761         Buf->BufUsed += AppendBuf->BufUsed - Offset;
 762         Buf->buf[Buf->BufUsed] = '\0';
 763 }
 764
 765
 766 // Append a C-String to the buffer
 767 // Buf          Buffer to modify
 768 // AppendBuf    Buffer to copy at the end of our buffer
 769 // AppendSize   number of bytes to copy; set to -1 if we should count it in advance
 770 // Offset       Should we start copying from an offset?
 771 void StrBufAppendBufPlain(StrBuf *Buf, const char *AppendBuf, long AppendSize, unsigned long Offset) {
 772         long aps;
 773         long BufSizeRequired;
 774
 775         if ((AppendBuf == NULL) || (Buf == NULL))
 776                 return;
 777
 778         if (AppendSize < 0) {
 779                 aps = strlen(AppendBuf + Offset);
 780         }
 781         else {
 782                 aps = AppendSize - Offset;
 783         }
 784
 785         BufSizeRequired = Buf->BufUsed + aps + 1;
 786         if (Buf->BufSize <= BufSizeRequired) {
 787                 IncreaseBuf(Buf, (Buf->BufUsed > 0), BufSizeRequired);
 788         }
 789
 790         memcpy(Buf->buf + Buf->BufUsed,
 791                AppendBuf + Offset,
 792                aps);
 793         Buf->BufUsed += aps;
 794         Buf->buf[Buf->BufUsed] = '\0';
 795 }
 796
 797 /*
 798  *  sprintf like function appending the formated string to the buffer
 799  * vsnprintf version to wrap into own calls
 800  *  Buf Buffer to extend by format and Params
 801  *  format printf alike format to add
 802  *  ap va_list containing the items for format
 803  */
 804 void StrBufVAppendPrintf(StrBuf *Buf, const char *format, va_list ap)
 805 {
 806         va_list apl;
 807         size_t BufSize;
 808         size_t nWritten;
 809         size_t Offset;
 810         size_t newused;
 811
 812         if ((Buf == NULL)  || (format == NULL))
 813                 return;
 814
 815         BufSize = Buf->BufSize;
 816         nWritten = Buf->BufSize + 1;
 817         Offset = Buf->BufUsed;
 818         newused = Offset + nWritten;
 819
 820         while (newused >= BufSize) {
 821                 va_copy(apl, ap);
 822                 nWritten = vsnprintf(Buf->buf + Offset,
 823                                      Buf->BufSize - Offset,
 824                                      format, apl);
 825                 va_end(apl);
 826                 newused = Offset + nWritten;
 827                 if (newused >= Buf->BufSize) {
 828                         if (IncreaseBuf(Buf, 1, newused) == -1)
 829                                 return; /* TODO: error handling? */
 830                         newused = Buf->BufSize + 1;
 831                 }
 832                 else {
 833                         Buf->BufUsed = Offset + nWritten;
 834                         BufSize = Buf->BufSize;
 835                 }
 836
 837         }
 838 }
 839
 840 /**
 841  *  sprintf like function appending the formated string to the buffer
 842  *  Buf Buffer to extend by format and Params
 843  *  format printf alike format to add
 844  */
 845 void StrBufAppendPrintf(StrBuf *Buf, const char *format, ...)
 846 {
 847         size_t BufSize;
 848         size_t nWritten;
 849         size_t Offset;
 850         size_t newused;
 851         va_list arg_ptr;
 852
 853         if ((Buf == NULL)  || (format == NULL))
 854                 return;
 855
 856         BufSize = Buf->BufSize;
 857         nWritten = Buf->BufSize + 1;
 858         Offset = Buf->BufUsed;
 859         newused = Offset + nWritten;
 860
 861         while (newused >= BufSize) {
 862                 va_start(arg_ptr, format);
 863                 nWritten = vsnprintf(Buf->buf + Buf->BufUsed,
 864                                      Buf->BufSize - Buf->BufUsed,
 865                                      format, arg_ptr);
 866                 va_end(arg_ptr);
 867                 newused = Buf->BufUsed + nWritten;
 868                 if (newused >= Buf->BufSize) {
 869                         if (IncreaseBuf(Buf, 1, newused) == -1)
 870                                 return; /* TODO: error handling? */
 871                         newused = Buf->BufSize + 1;
 872                 }
 873                 else {
 874                         Buf->BufUsed += nWritten;
 875                         BufSize = Buf->BufSize;
 876                 }
 877
 878         }
 879 }
 880
 881 /**
 882  *  sprintf like function putting the formated string into the buffer
 883  *  Buf Buffer to extend by format and Parameters
 884  *  format printf alike format to add
 885  */
 886 void StrBufPrintf(StrBuf *Buf, const char *format, ...)
 887 {
 888         size_t nWritten;
 889         va_list arg_ptr;
 890
 891         if ((Buf == NULL)  || (format == NULL))
 892                 return;
 893
 894         nWritten = Buf->BufSize + 1;
 895         while (nWritten >= Buf->BufSize) {
 896                 va_start(arg_ptr, format);
 897                 nWritten = vsnprintf(Buf->buf, Buf->BufSize, format, arg_ptr);
 898                 va_end(arg_ptr);
 899                 if (nWritten >= Buf->BufSize) {
 900                         if (IncreaseBuf(Buf, 0, 0) == -1)
 901                                 return; /* TODO: error handling? */
 902                         nWritten = Buf->BufSize + 1;
 903                         continue;
 904                 }
 905                 Buf->BufUsed = nWritten ;
 906         }
 907 }
 908
 909 /**
 910  *  Callback for cURL to append the webserver reply to a buffer
 911  *  ptr pre-defined by the cURL API; see man 3 curl for mre info
 912  *  size pre-defined by the cURL API; see man 3 curl for mre info
 913  *  nmemb pre-defined by the cURL API; see man 3 curl for mre info
 914  *  stream pre-defined by the cURL API; see man 3 curl for mre info
 915  */
 916 size_t CurlFillStrBuf_callback(void *ptr, size_t size, size_t nmemb, void *stream)
 917 {
 918
 919         StrBuf *Target;
 920
 921         Target = stream;
 922         if (ptr == NULL)
 923                 return 0;
 924
 925         StrBufAppendBufPlain(Target, ptr, size * nmemb, 0);
 926         return size * nmemb;
 927 }
 928
 929
 930 /**
 931  *  extracts a substring from Source into dest
 932  *  dest buffer to place substring into
 933  *  Source string to copy substring from
 934  *  Offset chars to skip from start
 935  *  nChars number of chars to copy
 936  * @returns the number of chars copied; may be different from nChars due to the size of Source
 937  */
 938 int StrBufSub(StrBuf *dest, const StrBuf *Source, unsigned long Offset, size_t nChars)
 939 {
 940         size_t NCharsRemain;
 941         if (Offset > Source->BufUsed)
 942         {
 943                 if (dest != NULL)
 944                         FlushStrBuf(dest);
 945                 return 0;
 946         }
 947         if (Offset + nChars < Source->BufUsed)
 948         {
 949                 if ((nChars >= dest->BufSize) &&
 950                     (IncreaseBuf(dest, 0, nChars + 1) == -1))
 951                         return 0;
 952                 memcpy(dest->buf, Source->buf + Offset, nChars);
 953                 dest->BufUsed = nChars;
 954                 dest->buf[dest->BufUsed] = '\0';
 955                 return nChars;
 956         }
 957         NCharsRemain = Source->BufUsed - Offset;
 958         if ((NCharsRemain  >= dest->BufSize) &&
 959             (IncreaseBuf(dest, 0, NCharsRemain + 1) == -1))
 960                 return 0;
 961         memcpy(dest->buf, Source->buf + Offset, NCharsRemain);
 962         dest->BufUsed = NCharsRemain;
 963         dest->buf[dest->BufUsed] = '\0';
 964         return NCharsRemain;
 965 }
 966
 967 /**
 968  *  Cut nChars from the start of the string
 969  *  Buf Buffer to modify
 970  *  nChars how many chars should be skipped?
 971  */
 972 void StrBufCutLeft(StrBuf *Buf, int nChars)
 973 {
 974         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
 975         if (nChars >= Buf->BufUsed) {
 976                 FlushStrBuf(Buf);
 977                 return;
 978         }
 979         memmove(Buf->buf, Buf->buf + nChars, Buf->BufUsed - nChars);
 980         Buf->BufUsed -= nChars;
 981         Buf->buf[Buf->BufUsed] = '\0';
 982 }
 983
 984 /**
 985  *  Cut the trailing n Chars from the string
 986  *  Buf Buffer to modify
 987  *  nChars how many chars should be trunkated?
 988  */
 989 void StrBufCutRight(StrBuf *Buf, int nChars)
 990 {
 991         if ((Buf == NULL) || (Buf->BufUsed == 0) || (Buf->buf == NULL))
 992                 return;
 993
 994         if (nChars >= Buf->BufUsed) {
 995                 FlushStrBuf(Buf);
 996                 return;
 997         }
 998         Buf->BufUsed -= nChars;
 999         Buf->buf[Buf->BufUsed] = '\0';
1000 }
1001
1002 /**
1003  *  Cut the string after n Chars
1004  *  Buf Buffer to modify
1005  *  AfternChars after how many chars should we trunkate the string?
1006  *  At if non-null and points inside of our string, cut it there.
1007  */
1008 void StrBufCutAt(StrBuf *Buf, int AfternChars, const char *At)
1009 {
1010         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1011         if (At != NULL){
1012                 AfternChars = At - Buf->buf;
1013         }
1014
1015         if ((AfternChars < 0) || (AfternChars >= Buf->BufUsed))
1016                 return;
1017         Buf->BufUsed = AfternChars;
1018         Buf->buf[Buf->BufUsed] = '\0';
1019 }
1020
1021
1022 /**
1023  *  Strip leading and trailing spaces from a string; with premeasured and adjusted length.
1024  *  Buf the string to modify
1025  */
1026 void StrBufTrim(StrBuf *Buf)
1027 {
1028         int delta = 0;
1029         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1030
1031         while ((Buf->BufUsed > 0) &&
1032                isspace(Buf->buf[Buf->BufUsed - 1]))
1033         {
1034                 Buf->BufUsed --;
1035         }
1036         Buf->buf[Buf->BufUsed] = '\0';
1037
1038         if (Buf->BufUsed == 0) return;
1039
1040         while ((Buf->BufUsed > delta) && (isspace(Buf->buf[delta]))){
1041                 delta ++;
1042         }
1043         if (delta > 0) StrBufCutLeft(Buf, delta);
1044 }
1045 /**
1046  *  changes all spaces in the string  (tab, linefeed...) to Blank (0x20)
1047  *  Buf the string to modify
1048  */
1049 void StrBufSpaceToBlank(StrBuf *Buf)
1050 {
1051         char *pche, *pch;
1052
1053         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1054
1055         pch = Buf->buf;
1056         pche = pch + Buf->BufUsed;
1057         while (pch < pche)
1058         {
1059                 if (isspace(*pch))
1060                         *pch = ' ';
1061                 pch ++;
1062         }
1063 }
1064
1065 void StrBufStripAllBut(StrBuf *Buf, char leftboundary, char rightboundary)
1066 {
1067         const char *pLeft;
1068         const char *pRight;
1069
1070         if ((Buf == NULL) || (Buf->buf == NULL)) {
1071                 return;
1072         }
1073
1074         pRight = strchr(Buf->buf, rightboundary);
1075         if (pRight != NULL) {
1076                 StrBufCutAt(Buf, 0, pRight);
1077         }
1078
1079         pLeft = strrchr(ChrPtr(Buf), leftboundary);
1080         if (pLeft != NULL) {
1081                 StrBufCutLeft(Buf, pLeft - Buf->buf + 1);
1082         }
1083 }
1084
1085
1086 /**
1087  *  uppercase the contents of a buffer
1088  *  Buf the buffer to translate
1089  */
1090 void StrBufUpCase(StrBuf *Buf)
1091 {
1092         char *pch, *pche;
1093
1094         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1095
1096         pch = Buf->buf;
1097         pche = pch + Buf->BufUsed;
1098         while (pch < pche) {
1099                 *pch = toupper(*pch);
1100                 pch ++;
1101         }
1102 }
1103
1104
1105 /**
1106  *  lowercase the contents of a buffer
1107  *  Buf the buffer to translate
1108  */
1109 void StrBufLowerCase(StrBuf *Buf)
1110 {
1111         char *pch, *pche;
1112
1113         if ((Buf == NULL) || (Buf->BufUsed == 0)) return;
1114
1115         pch = Buf->buf;
1116         pche = pch + Buf->BufUsed;
1117         while (pch < pche) {
1118                 *pch = tolower(*pch);
1119                 pch ++;
1120         }
1121 }
1122
1123
1124 /*******************************************************************************
1125  *           a tokenizer that kills, maims, and destroys                       *
1126  *******************************************************************************/
1127
1128 /**
1129  *  Replace a token at a given place with a given length by another token with given length
1130  *  Buf String where to work on
1131  *  where where inside of the Buf is the search-token
1132  *  HowLong How long is the token to be replaced
1133  *  Repl Token to insert at 'where'
1134  *  ReplLen Length of repl
1135  * @returns -1 if fail else length of resulting Buf
1136  */
1137 int StrBufReplaceToken(StrBuf *Buf, long where, long HowLong,
1138                        const char *Repl, long ReplLen)
1139 {
1140
1141         if ((Buf == NULL) ||
1142             (where > Buf->BufUsed) ||
1143             (where + HowLong > Buf->BufUsed))
1144                 return -1;
1145
1146         if (where + ReplLen - HowLong > Buf->BufSize)
1147                 if (IncreaseBuf(Buf, 1, Buf->BufUsed + ReplLen) < 0)
1148                         return -1;
1149
1150         memmove(Buf->buf + where + ReplLen,
1151                 Buf->buf + where + HowLong,
1152                 Buf->BufUsed - where - HowLong);
1153
1154         memcpy(Buf->buf + where,
1155                Repl, ReplLen);
1156
1157         Buf->BufUsed += ReplLen - HowLong;
1158
1159         return Buf->BufUsed;
1160 }
1161
1162 /**
1163  *  Counts the numbmer of tokens in a buffer
1164  *  source String to count tokens in
1165  *  tok    Tokenizer char to count
1166  * @returns numbers of tokenizer chars found
1167  */
1168 int StrBufNum_tokens(const StrBuf *source, char tok)
1169 {
1170         char *pch, *pche;
1171         long NTokens;
1172         if ((source == NULL) || (source->BufUsed == 0))
1173                 return 0;
1174         if ((source->BufUsed == 1) && (*source->buf == tok))
1175                 return 2;
1176         NTokens = 1;
1177         pch = source->buf;
1178         pche = pch + source->BufUsed;
1179         while (pch < pche)
1180         {
1181                 if (*pch == tok)
1182                         NTokens ++;
1183                 pch ++;
1184         }
1185         return NTokens;
1186 }
1187
1188 /**
1189  *  a string tokenizer
1190  *  Source StringBuffer to read into
1191  *  parmnum n'th Parameter to remove
1192  *  separator tokenizer character
1193  * @returns -1 if not found, else length of token.
1194  */
1195 int StrBufRemove_token(StrBuf *Source, int parmnum, char separator)
1196 {
1197         int ReducedBy;
1198         char *d, *s, *end;              /* dest, source */
1199         int count = 0;
1200
1201         /* Find desired eter */
1202         end = Source->buf + Source->BufUsed;
1203         d = Source->buf;
1204         while ((d <= end) &&
1205                (count < parmnum))
1206         {
1207                 /* End of string, bail! */
1208                 if (!*d) {
1209                         d = NULL;
1210                         break;
1211                 }
1212                 if (*d == separator) {
1213                         count++;
1214                 }
1215                 d++;
1216         }
1217         if ((d == NULL) || (d >= end))
1218                 return 0;               /* @Parameter not found */
1219
1220         /* Find next eter */
1221         s = d;
1222         while ((s <= end) &&
1223                (*s && *s != separator))
1224         {
1225                 s++;
1226         }
1227         if (*s == separator)
1228                 s++;
1229         ReducedBy = d - s;
1230
1231         /* Hack and slash */
1232         if (s >= end) {
1233                 return 0;
1234         }
1235         else if (*s) {
1236                 memmove(d, s, Source->BufUsed - (s - Source->buf));
1237                 Source->BufUsed += ReducedBy;
1238                 Source->buf[Source->BufUsed] = '\0';
1239         }
1240         else if (d == Source->buf) {
1241                 *d = 0;
1242                 Source->BufUsed = 0;
1243         }
1244         else {
1245                 *--d = '\0';
1246                 Source->BufUsed += ReducedBy;
1247         }
1248         /*
1249         while (*s) {
1250                 *d++ = *s++;
1251         }
1252         *d = 0;
1253         */
1254         return ReducedBy;
1255 }
1256
1257 int StrBufExtract_tokenFromStr(StrBuf *dest, const char *Source, long SourceLen, int parmnum, char separator)
1258 {
1259         const StrBuf Temp = {
1260                 (char*)Source,
1261                 SourceLen,
1262                 SourceLen,
1263                 1
1264 #ifdef SIZE_DEBUG
1265                 ,
1266                 0,
1267                 "",
1268                 ""
1269 #endif
1270         };
1271
1272         return StrBufExtract_token(dest, &Temp, parmnum, separator);
1273 }
1274
1275 /**
1276  *  a string tokenizer
1277  *  dest Destination StringBuffer
1278  *  Source StringBuffer to read into
1279  *  parmnum n'th Parameter to extract
1280  *  separator tokenizer character
1281  * @returns -1 if not found, else length of token.
1282  */
1283 int StrBufExtract_token(StrBuf *dest, const StrBuf *Source, int parmnum, char separator)
1284 {
1285         const char *s, *e;              //* source * /
1286         int len = 0;                    //* running total length of extracted string * /
1287         int current_token = 0;          //* token currently being processed * /
1288
1289         if (dest != NULL) {
1290                 dest->buf[0] = '\0';
1291                 dest->BufUsed = 0;
1292         }
1293         else
1294                 return(-1);
1295
1296         if ((Source == NULL) || (Source->BufUsed ==0)) {
1297                 return(-1);
1298         }
1299         s = Source->buf;
1300         e = s + Source->BufUsed;
1301
1302         //cit_backtrace();
1303         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1304
1305         while ((s < e) && !IsEmptyStr(s)) {
1306                 if (*s == separator) {
1307                         ++current_token;
1308                 }
1309                 if (len >= dest->BufSize) {
1310                         dest->BufUsed = len;
1311                         if (IncreaseBuf(dest, 1, -1) < 0) {
1312                                 dest->BufUsed --;
1313                                 break;
1314                         }
1315                 }
1316                 if ( (current_token == parmnum) &&
1317                      (*s != separator)) {
1318                         dest->buf[len] = *s;
1319                         ++len;
1320                 }
1321                 else if (current_token > parmnum) {
1322                         break;
1323                 }
1324                 ++s;
1325         }
1326
1327         dest->buf[len] = '\0';
1328         dest->BufUsed = len;
1329
1330         if (current_token < parmnum) {
1331                 //lprintf (CTDL_DEBUG,"test <!: %s\n", dest);
1332                 return(-1);
1333         }
1334         //lprintf (CTDL_DEBUG,"test <: %d; %s\n", len, dest);
1335         return(len);
1336 }
1337
1338
1339
1340
1341
1342 /**
1343  *  a string tokenizer to fetch an integer
1344  *  Source String containing tokens
1345  *  parmnum n'th Parameter to extract
1346  *  separator tokenizer character
1347  * @returns 0 if not found, else integer representation of the token
1348  */
1349 int StrBufExtract_int(const StrBuf* Source, int parmnum, char separator)
1350 {
1351         StrBuf tmp;
1352         char buf[64];
1353
1354         tmp.buf = buf;
1355         buf[0] = '\0';
1356         tmp.BufSize = 64;
1357         tmp.BufUsed = 0;
1358         tmp.ConstBuf = 1;
1359         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1360                 return(atoi(buf));
1361         else
1362                 return 0;
1363 }
1364
1365 /**
1366  *  a string tokenizer to fetch a long integer
1367  *  Source String containing tokens
1368  *  parmnum n'th Parameter to extract
1369  *  separator tokenizer character
1370  * @returns 0 if not found, else long integer representation of the token
1371  */
1372 long StrBufExtract_long(const StrBuf* Source, int parmnum, char separator)
1373 {
1374         StrBuf tmp;
1375         char buf[64];
1376
1377         tmp.buf = buf;
1378         buf[0] = '\0';
1379         tmp.BufSize = 64;
1380         tmp.BufUsed = 0;
1381         tmp.ConstBuf = 1;
1382         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0)
1383                 return(atoi(buf));
1384         else
1385                 return 0;
1386 }
1387
1388
1389 /**
1390  *  a string tokenizer to fetch an unsigned long
1391  *  Source String containing tokens
1392  *  parmnum n'th Parameter to extract
1393  *  separator tokenizer character
1394  * @returns 0 if not found, else unsigned long representation of the token
1395  */
1396 unsigned long StrBufExtract_unsigned_long(const StrBuf* Source, int parmnum, char separator)
1397 {
1398         StrBuf tmp;
1399         char buf[64];
1400         char *pnum;
1401
1402         tmp.buf = buf;
1403         buf[0] = '\0';
1404         tmp.BufSize = 64;
1405         tmp.BufUsed = 0;
1406         tmp.ConstBuf = 1;
1407         if (StrBufExtract_token(&tmp, Source, parmnum, separator) > 0) {
1408                 pnum = &buf[0];
1409                 if (*pnum == '-')
1410                         pnum ++;
1411                 return (unsigned long) atol(pnum);
1412         }
1413         else
1414                 return 0;
1415 }
1416
1417
1418
1419 /**
1420  *  a string tokenizer; Bounds checker
1421  *  function to make shure whether StrBufExtract_NextToken and friends have reached the end of the string.
1422  *  Source our tokenbuffer
1423  *  pStart the token iterator pointer to inspect
1424  * @returns whether the revolving pointer is inside of the search range
1425  */
1426 int StrBufHaveNextToken(const StrBuf *Source, const char **pStart)
1427 {
1428         if ((Source == NULL) ||
1429             (*pStart == StrBufNOTNULL) ||
1430             (Source->BufUsed == 0))
1431         {
1432                 return 0;
1433         }
1434         if (*pStart == NULL)
1435         {
1436                 return 1;
1437         }
1438         else if (*pStart > Source->buf + Source->BufUsed)
1439         {
1440                 return 0;
1441         }
1442         else if (*pStart <= Source->buf)
1443         {
1444                 return 0;
1445         }
1446
1447         return 1;
1448 }
1449
1450 /**
1451  *  a string tokenizer
1452  *  dest Destination StringBuffer
1453  *  Source StringBuffer to read into
1454  *  pStart pointer to the end of the last token. Feed with NULL on start.
1455  *  separator tokenizer
1456  * @returns -1 if not found, else length of token.
1457  */
1458 int StrBufExtract_NextToken(StrBuf *dest, const StrBuf *Source, const char **pStart, char separator)
1459 {
1460         const char *s;          /* source */
1461         const char *EndBuffer;  /* end stop of source buffer */
1462         int current_token = 0;  /* token currently being processed */
1463         int len = 0;            /* running total length of extracted string */
1464
1465         if ((Source          == NULL) ||
1466             (Source->BufUsed == 0)      )
1467         {
1468                 *pStart = StrBufNOTNULL;
1469                 if (dest != NULL)
1470                         FlushStrBuf(dest);
1471                 return -1;
1472         }
1473
1474         EndBuffer = Source->buf + Source->BufUsed;
1475
1476         if (dest != NULL)
1477         {
1478                 dest->buf[0] = '\0';
1479                 dest->BufUsed = 0;
1480         }
1481         else
1482         {
1483                 *pStart = EndBuffer + 1;
1484                 return -1;
1485         }
1486
1487         if (*pStart == NULL)
1488         {
1489                 *pStart = Source->buf; /* we're starting to examine this buffer. */
1490         }
1491         else if ((*pStart < Source->buf) ||
1492                  (*pStart > EndBuffer  )   )
1493         {
1494                 return -1; /* no more tokens to find. */
1495         }
1496
1497         s = *pStart;
1498         /* start to find the next token */
1499         while ((s <= EndBuffer)      &&
1500                (current_token == 0) )
1501         {
1502                 if (*s == separator)
1503                 {
1504                         /* we found the next token */
1505                         ++current_token;
1506                 }
1507
1508                 if (len >= dest->BufSize)
1509                 {
1510                         /* our Dest-buffer isn't big enough, increase it. */
1511                         dest->BufUsed = len;
1512
1513                         if (IncreaseBuf(dest, 1, -1) < 0) {
1514                                 /* WHUT? no more mem? bail out. */
1515                                 s = EndBuffer;
1516                                 dest->BufUsed --;
1517                                 break;
1518                         }
1519                 }
1520
1521                 if ( (current_token == 0 ) &&   /* are we in our target token? */
1522                      (!IsEmptyStr(s)     ) &&
1523                      (separator     != *s)    ) /* don't copy the token itself */
1524                 {
1525                         dest->buf[len] = *s;    /* Copy the payload */
1526                         ++len;                  /* remember the bigger size. */
1527                 }
1528
1529                 ++s;
1530         }
1531
1532         /* did we reach the end? */
1533         if ((s > EndBuffer)) {
1534                 EndBuffer = StrBufNOTNULL;
1535                 *pStart = EndBuffer;
1536         }
1537         else {
1538                 *pStart = s;  /* remember the position for the next run */
1539         }
1540
1541         /* sanitize our extracted token */
1542         dest->buf[len] = '\0';
1543         dest->BufUsed  = len;
1544
1545         return (len);
1546 }
1547
1548
1549 /**
1550  *  a string tokenizer
1551  *  Source StringBuffer to read from
1552  *  pStart pointer to the end of the last token. Feed with NULL.
1553  *  separator tokenizer character
1554  *  nTokens number of tokens to fastforward over
1555  * @returns -1 if not found, else length of token.
1556  */
1557 int StrBufSkip_NTokenS(const StrBuf *Source, const char **pStart, char separator, int nTokens)
1558 {
1559         const char *s, *EndBuffer;      //* source * /
1560         int len = 0;                    //* running total length of extracted string * /
1561         int current_token = 0;          //* token currently being processed * /
1562
1563         if ((Source == NULL) ||
1564             (Source->BufUsed ==0)) {
1565                 return(-1);
1566         }
1567         if (nTokens == 0)
1568                 return Source->BufUsed;
1569
1570         if (*pStart == NULL)
1571                 *pStart = Source->buf;
1572
1573         EndBuffer = Source->buf + Source->BufUsed;
1574
1575         if ((*pStart < Source->buf) ||
1576             (*pStart >  EndBuffer)) {
1577                 return (-1);
1578         }
1579
1580
1581         s = *pStart;
1582
1583         //cit_backtrace();
1584         //lprintf (CTDL_DEBUG, "test >: n: %d sep: %c source: %s \n willi \n", parmnum, separator, source);
1585
1586         while ((s < EndBuffer) && !IsEmptyStr(s)) {
1587                 if (*s == separator) {
1588                         ++current_token;
1589                 }
1590                 if (current_token >= nTokens) {
1591                         break;
1592                 }
1593                 ++s;
1594         }
1595         *pStart = s;
1596         (*pStart) ++;
1597
1598         return(len);
1599 }
1600
1601 /**
1602  *  a string tokenizer to fetch an integer
1603  *  Source StringBuffer to read from
1604  *  pStart Cursor on the tokenstring
1605  *  separator tokenizer character
1606  * @returns 0 if not found, else integer representation of the token
1607  */
1608 int StrBufExtractNext_int(const StrBuf* Source, const char **pStart, char separator)
1609 {
1610         StrBuf tmp;
1611         char buf[64];
1612
1613         tmp.buf = buf;
1614         buf[0] = '\0';
1615         tmp.BufSize = 64;
1616         tmp.BufUsed = 0;
1617         tmp.ConstBuf = 1;
1618         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1619                 return(atoi(buf));
1620         else
1621                 return 0;
1622 }
1623
1624 /**
1625  *  a string tokenizer to fetch a long integer
1626  *  Source StringBuffer to read from
1627  *  pStart Cursor on the tokenstring
1628  *  separator tokenizer character
1629  * @returns 0 if not found, else long integer representation of the token
1630  */
1631 long StrBufExtractNext_long(const StrBuf* Source, const char **pStart, char separator)
1632 {
1633         StrBuf tmp;
1634         char buf[64];
1635
1636         tmp.buf = buf;
1637         buf[0] = '\0';
1638         tmp.BufSize = 64;
1639         tmp.BufUsed = 0;
1640         tmp.ConstBuf = 1;
1641         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0)
1642                 return(atoi(buf));
1643         else
1644                 return 0;
1645 }
1646
1647
1648 /**
1649  *  a string tokenizer to fetch an unsigned long
1650  *  Source StringBuffer to read from
1651  *  pStart Cursor on the tokenstring
1652  *  separator tokenizer character
1653  * @returns 0 if not found, else unsigned long representation of the token
1654  */
1655 unsigned long StrBufExtractNext_unsigned_long(const StrBuf* Source, const char **pStart, char separator)
1656 {
1657         StrBuf tmp;
1658         char buf[64];
1659         char *pnum;
1660
1661         tmp.buf = buf;
1662         buf[0] = '\0';
1663         tmp.BufSize = 64;
1664         tmp.BufUsed = 0;
1665         tmp.ConstBuf = 1;
1666         if (StrBufExtract_NextToken(&tmp, Source, pStart, separator) > 0) {
1667                 pnum = &buf[0];
1668                 if (*pnum == '-')
1669                         pnum ++;
1670                 return (unsigned long) atol(pnum);
1671         }
1672         else
1673                 return 0;
1674 }
1675
1676
1677
1678
1679
1680 /*******************************************************************************
1681  *                             Escape Appending                                *
1682  *******************************************************************************/
1683
1684 /**
1685  *  Escape a string for feeding out as a URL while appending it to a Buffer
1686  *  OutBuf the output buffer
1687  *  In Buffer to encode
1688  *  PlainIn way in from plain old c strings
1689  */
1690 void StrBufUrlescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn)
1691 {
1692         const char *pch, *pche;
1693         char *pt, *pte;
1694         int len;
1695
1696         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1697                 return;
1698         if (PlainIn != NULL) {
1699                 len = strlen(PlainIn);
1700                 pch = PlainIn;
1701                 pche = pch + len;
1702         }
1703         else {
1704                 pch = In->buf;
1705                 pche = pch + In->BufUsed;
1706                 len = In->BufUsed;
1707         }
1708
1709         if (len == 0)
1710                 return;
1711
1712         pt = OutBuf->buf + OutBuf->BufUsed;
1713         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1714
1715         while (pch < pche) {
1716                 if (pt >= pte) {
1717                         IncreaseBuf(OutBuf, 1, -1);
1718                         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1719                         pt = OutBuf->buf + OutBuf->BufUsed;
1720                 }
1721
1722                 if((*pch >= 'a' && *pch <= 'z') ||
1723                    (*pch >= '@' && *pch <= 'Z') || /* @ A-Z */
1724                    (*pch >= '0' && *pch <= ':') || /* 0-9 : */
1725                    (*pch == '!') || (*pch == '_') ||
1726                    (*pch == ',') || (*pch == '.'))
1727                 {
1728                         *(pt++) = *(pch++);
1729                         OutBuf->BufUsed++;
1730                 }
1731                 else {
1732                         *pt = '%';
1733                         *(pt + 1) = HexList[(unsigned char)*pch][0];
1734                         *(pt + 2) = HexList[(unsigned char)*pch][1];
1735                         pt += 3;
1736                         OutBuf->BufUsed += 3;
1737                         pch ++;
1738                 }
1739         }
1740         *pt = '\0';
1741 }
1742
1743
1744 /*
1745  *  Escape a string for feeding out as a the username/password part of an URL while appending it to a Buffer
1746  *  OutBuf the output buffer
1747  *  In Buffer to encode
1748  *  PlainIn way in from plain old c strings
1749  */
1750 void StrBufUrlescUPAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn) {
1751         const char *pch, *pche;
1752         char *pt, *pte;
1753         int len;
1754
1755         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1756                 return;
1757         if (PlainIn != NULL) {
1758                 len = strlen(PlainIn);
1759                 pch = PlainIn;
1760                 pche = pch + len;
1761         }
1762         else {
1763                 pch = In->buf;
1764                 pche = pch + In->BufUsed;
1765                 len = In->BufUsed;
1766         }
1767
1768         if (len == 0)
1769                 return;
1770
1771         pt = OutBuf->buf + OutBuf->BufUsed;
1772         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1773
1774         while (pch < pche) {
1775                 if (pt >= pte) {
1776                         IncreaseBuf(OutBuf, 1, -1);
1777                         pte = OutBuf->buf + OutBuf->BufSize - 4; /**< we max append 3 chars at once plus the \0 */
1778                         pt = OutBuf->buf + OutBuf->BufUsed;
1779                 }
1780
1781                 if((*pch >= 'a' && *pch <= 'z') ||
1782                    (*pch >= 'A' && *pch <= 'Z') || /* A-Z */
1783                    (*pch >= '0' && *pch <= ':') || /* 0-9 : */
1784                    (*pch == '!') || (*pch == '_') ||
1785                    (*pch == ',') || (*pch == '.'))
1786                 {
1787                         *(pt++) = *(pch++);
1788                         OutBuf->BufUsed++;
1789                 }
1790                 else {
1791                         *pt = '%';
1792                         *(pt + 1) = HexList[(unsigned char)*pch][0];
1793                         *(pt + 2) = HexList[(unsigned char)*pch][1];
1794                         pt += 3;
1795                         OutBuf->BufUsed += 3;
1796                         pch ++;
1797                 }
1798         }
1799         *pt = '\0';
1800 }
1801
1802 /*
1803  *  append a string with characters having a special meaning in xml encoded to the buffer
1804  *  OutBuf the output buffer
1805  *  In Buffer to encode
1806  *  PlainIn way in from plain old c strings
1807  *  PlainInLen way in from plain old c strings; maybe you've got binary data or know the length?
1808  *  OverrideLowChars should chars < 0x20 be replaced by _ or escaped as xml entity?
1809  */
1810 void StrBufXMLEscAppend(StrBuf *OutBuf,
1811                         const StrBuf *In,
1812                         const char *PlainIn,
1813                         long PlainInLen,
1814                         int OverrideLowChars)
1815 {
1816         const char *pch, *pche;
1817         char *pt, *pte;
1818         int IsUtf8Sequence;
1819         int len;
1820
1821         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1822                 return;
1823         if (PlainIn != NULL) {
1824                 if (PlainInLen < 0)
1825                         len = strlen((const char*)PlainIn);
1826                 else
1827                         len = PlainInLen;
1828                 pch = PlainIn;
1829                 pche = pch + len;
1830         }
1831         else {
1832                 pch = (const char*)In->buf;
1833                 pche = pch + In->BufUsed;
1834                 len = In->BufUsed;
1835         }
1836
1837         if (len == 0)
1838                 return;
1839
1840         pt = OutBuf->buf + OutBuf->BufUsed;
1841         /**< we max append 6 chars at once plus the \0 */
1842         pte = OutBuf->buf + OutBuf->BufSize - 6;
1843
1844         while (pch < pche) {
1845                 if (pt >= pte) {
1846                         OutBuf->BufUsed = pt - OutBuf->buf;
1847                         IncreaseBuf(OutBuf, 1, -1);
1848                         pte = OutBuf->buf + OutBuf->BufSize - 6;
1849                         /**< we max append 3 chars at once plus the \0 */
1850
1851                         pt = OutBuf->buf + OutBuf->BufUsed;
1852                 }
1853
1854                 if (*pch == '<') {
1855                         memcpy(pt, HKEY("&lt;"));
1856                         pt += 4;
1857                         pch ++;
1858                 }
1859                 else if (*pch == '>') {
1860                         memcpy(pt, HKEY("&gt;"));
1861                         pt += 4;
1862                         pch ++;
1863                 }
1864                 else if (*pch == '&') {
1865                         memcpy(pt, HKEY("&amp;"));
1866                         pt += 5;
1867                         pch++;
1868                 }
1869                 else if ((*pch >= 0x20) && (*pch <= 0x7F)) {
1870                         *pt = *pch;
1871                         pt++; pch++;
1872                 }
1873                 else {
1874                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(pch, pche);
1875                         if (IsUtf8Sequence)
1876                         {
1877                                 while ((IsUtf8Sequence > 0) &&
1878                                        (pch < pche))
1879                                 {
1880                                         *pt = *pch;
1881                                         pt ++;
1882                                         pch ++;
1883                                         --IsUtf8Sequence;
1884                                 }
1885                         }
1886                         else
1887                         {
1888                                 *pt = '&';
1889                                 pt++;
1890                                 *pt = HexList[*(unsigned char*)pch][0];
1891                                 pt ++;
1892                                 *pt = HexList[*(unsigned char*)pch][1];
1893                                 pt ++; pch ++;
1894                                 *pt = '&';
1895                                 pt++;
1896                                 pch ++;
1897                         }
1898                 }
1899         }
1900         *pt = '\0';
1901         OutBuf->BufUsed = pt - OutBuf->buf;
1902 }
1903
1904
1905 /**
1906  *  append a string in hex encoding to the buffer
1907  *  OutBuf the output buffer
1908  *  In Buffer to encode
1909  *  PlainIn way in from plain old c strings
1910  *  PlainInLen way in from plain old c strings; maybe you've got binary data or know the length?
1911  */
1912 void StrBufHexEscAppend(StrBuf *OutBuf, const StrBuf *In, const unsigned char *PlainIn, long PlainInLen)
1913 {
1914         const unsigned char *pch, *pche;
1915         char *pt, *pte;
1916         int len;
1917
1918         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1919                 return;
1920         if (PlainIn != NULL) {
1921                 if (PlainInLen < 0)
1922                         len = strlen((const char*)PlainIn);
1923                 else
1924                         len = PlainInLen;
1925                 pch = PlainIn;
1926                 pche = pch + len;
1927         }
1928         else {
1929                 pch = (const unsigned char*)In->buf;
1930                 pche = pch + In->BufUsed;
1931                 len = In->BufUsed;
1932         }
1933
1934         if (len == 0)
1935                 return;
1936
1937         pt = OutBuf->buf + OutBuf->BufUsed;
1938         pte = OutBuf->buf + OutBuf->BufSize - 3; /**< we max append 3 chars at once plus the \0 */
1939
1940         while (pch < pche) {
1941                 if (pt >= pte) {
1942                         IncreaseBuf(OutBuf, 1, -1);
1943                         pte = OutBuf->buf + OutBuf->BufSize - 3; /**< we max append 3 chars at once plus the \0 */
1944                         pt = OutBuf->buf + OutBuf->BufUsed;
1945                 }
1946
1947                 *pt = HexList[*pch][0];
1948                 pt ++;
1949                 *pt = HexList[*pch][1];
1950                 pt ++; pch ++; OutBuf->BufUsed += 2;
1951         }
1952         *pt = '\0';
1953 }
1954
1955
1956 void StrBufBase64Append(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn, long PlainInLen, int linebreaks) {
1957         const char *pch;
1958         char *pt;
1959         int len;
1960         long ExpectLen;
1961
1962         if (((In == NULL) && (PlainIn == NULL)) || (OutBuf == NULL) )
1963                 return;
1964         if (PlainIn != NULL) {
1965                 if (PlainInLen < 0)
1966                         len = strlen(PlainIn);
1967                 else
1968                         len = PlainInLen;
1969                 pch = PlainIn;
1970         }
1971         else {
1972                 pch = In->buf;
1973                 len = In->BufUsed;
1974         }
1975
1976         if (len == 0)
1977                 return;
1978
1979         ExpectLen = ((len * 134) / 100) + OutBuf->BufUsed;
1980
1981         if (ExpectLen > OutBuf->BufSize)
1982                 if (IncreaseBuf(OutBuf, 1, ExpectLen) < ExpectLen)
1983                         return;
1984
1985         pt = OutBuf->buf + OutBuf->BufUsed;
1986
1987         len = CtdlEncodeBase64(pt, pch, len, linebreaks);
1988
1989         pt += len;
1990         OutBuf->BufUsed += len;
1991         *pt = '\0';
1992 }
1993
1994
1995 // append a string in hex encoding to the buffer
1996 // OutBuf       the output buffer
1997 // In           Buffer to encode
1998 // PlainIn      way in from plain old c strings
1999 void StrBufHexescAppend(StrBuf *OutBuf, const StrBuf *In, const char *PlainIn) {
2000         StrBufHexEscAppend(OutBuf, In, (const unsigned char*) PlainIn, -1);
2001 }
2002
2003 /*
2004  *  Append a string, escaping characters which have meaning in HTML.
2005  *
2006  *  Target      target buffer
2007  *  Source      source buffer; set to NULL if you just have a C-String
2008  *  PlainIn       Plain-C string to append; set to NULL if unused
2009  *  nbsp                If nonzero, spaces are converted to non-breaking spaces.
2010  *  nolinebreaks        if set to 1, linebreaks are removed from the string.
2011  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
2012  */
2013 long StrEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
2014 {
2015         const char *aptr, *eiptr;
2016         char *bptr, *eptr;
2017         long len;
2018
2019         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2020                 return -1;
2021
2022         if (PlainIn != NULL) {
2023                 aptr = PlainIn;
2024                 len = strlen(PlainIn);
2025                 eiptr = aptr + len;
2026         }
2027         else {
2028                 aptr = Source->buf;
2029                 eiptr = aptr + Source->BufUsed;
2030                 len = Source->BufUsed;
2031         }
2032
2033         if (len == 0)
2034                 return -1;
2035
2036         bptr = Target->buf + Target->BufUsed;
2037         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2038
2039         while (aptr < eiptr){
2040                 if(bptr >= eptr) {
2041                         IncreaseBuf(Target, 1, -1);
2042                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2043                         bptr = Target->buf + Target->BufUsed;
2044                 }
2045                 if (*aptr == '<') {
2046                         memcpy(bptr, "&lt;", 4);
2047                         bptr += 4;
2048                         Target->BufUsed += 4;
2049                 }
2050                 else if (*aptr == '>') {
2051                         memcpy(bptr, "&gt;", 4);
2052                         bptr += 4;
2053                         Target->BufUsed += 4;
2054                 }
2055                 else if (*aptr == '&') {
2056                         memcpy(bptr, "&amp;", 5);
2057                         bptr += 5;
2058                         Target->BufUsed += 5;
2059                 }
2060                 else if (*aptr == '"') {
2061                         memcpy(bptr, "&quot;", 6);
2062                         bptr += 6;
2063                         Target->BufUsed += 6;
2064                 }
2065                 else if (*aptr == '\'') {
2066                         memcpy(bptr, "&#39;", 5);
2067                         bptr += 5;
2068                         Target->BufUsed += 5;
2069                 }
2070                 else if (*aptr == LB) {
2071                         *bptr = '<';
2072                         bptr ++;
2073                         Target->BufUsed ++;
2074                 }
2075                 else if (*aptr == RB) {
2076                         *bptr = '>';
2077                         bptr ++;
2078                         Target->BufUsed ++;
2079                 }
2080                 else if (*aptr == QU) {
2081                         *bptr ='"';
2082                         bptr ++;
2083                         Target->BufUsed ++;
2084                 }
2085                 else if ((*aptr == 32) && (nbsp == 1)) {
2086                         memcpy(bptr, "&nbsp;", 6);
2087                         bptr += 6;
2088                         Target->BufUsed += 6;
2089                 }
2090                 else if ((*aptr == '\n') && (nolinebreaks == 1)) {
2091                         *bptr='\0';     /* nothing */
2092                 }
2093                 else if ((*aptr == '\n') && (nolinebreaks == 2)) {
2094                         memcpy(bptr, "&lt;br/&gt;", 11);
2095                         bptr += 11;
2096                         Target->BufUsed += 11;
2097                 }
2098
2099
2100                 else if ((*aptr == '\r') && (nolinebreaks != 0)) {
2101                         *bptr='\0';     /* nothing */
2102                 }
2103                 else{
2104                         *bptr = *aptr;
2105                         bptr++;
2106                         Target->BufUsed ++;
2107                 }
2108                 aptr ++;
2109         }
2110         *bptr = '\0';
2111         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
2112                 return -1;
2113         return Target->BufUsed;
2114 }
2115
2116 /**
2117  *  Append a string, escaping characters which have meaning in HTML.
2118  * Converts linebreaks into blanks; escapes single quotes
2119  *  Target      target buffer
2120  *  Source      source buffer; set to NULL if you just have a C-String
2121  *  PlainIn       Plain-C string to append; set to NULL if unused
2122  */
2123 void StrMsgEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2124 {
2125         const char *aptr, *eiptr;
2126         char *tptr, *eptr;
2127         long len;
2128
2129         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2130                 return ;
2131
2132         if (PlainIn != NULL) {
2133                 aptr = PlainIn;
2134                 len = strlen(PlainIn);
2135                 eiptr = aptr + len;
2136         }
2137         else {
2138                 aptr = Source->buf;
2139                 eiptr = aptr + Source->BufUsed;
2140                 len = Source->BufUsed;
2141         }
2142
2143         if (len == 0)
2144                 return;
2145
2146         eptr = Target->buf + Target->BufSize - 8;
2147         tptr = Target->buf + Target->BufUsed;
2148
2149         while (aptr < eiptr){
2150                 if(tptr >= eptr) {
2151                         IncreaseBuf(Target, 1, -1);
2152                         eptr = Target->buf + Target->BufSize - 8;
2153                         tptr = Target->buf + Target->BufUsed;
2154                 }
2155
2156                 if (*aptr == '\n') {
2157                         *tptr = ' ';
2158                         Target->BufUsed++;
2159                 }
2160                 else if (*aptr == '\r') {
2161                         *tptr = ' ';
2162                         Target->BufUsed++;
2163                 }
2164                 else if (*aptr == '\'') {
2165                         *(tptr++) = '&';
2166                         *(tptr++) = '#';
2167                         *(tptr++) = '3';
2168                         *(tptr++) = '9';
2169                         *tptr = ';';
2170                         Target->BufUsed += 5;
2171                 } else {
2172                         *tptr = *aptr;
2173                         Target->BufUsed++;
2174                 }
2175                 tptr++; aptr++;
2176         }
2177         *tptr = '\0';
2178 }
2179
2180
2181
2182 /**
2183  *  Append a string, escaping characters which have meaning in ICAL.
2184  * [\n,]
2185  *  Target      target buffer
2186  *  Source      source buffer; set to NULL if you just have a C-String
2187  *  PlainIn       Plain-C string to append; set to NULL if unused
2188  */
2189 void StrIcalEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2190 {
2191         const char *aptr, *eiptr;
2192         char *tptr, *eptr;
2193         long len;
2194
2195         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2196                 return ;
2197
2198         if (PlainIn != NULL) {
2199                 aptr = PlainIn;
2200                 len = strlen(PlainIn);
2201                 eiptr = aptr + len;
2202         }
2203         else {
2204                 aptr = Source->buf;
2205                 eiptr = aptr + Source->BufUsed;
2206                 len = Source->BufUsed;
2207         }
2208
2209         if (len == 0)
2210                 return;
2211
2212         eptr = Target->buf + Target->BufSize - 8;
2213         tptr = Target->buf + Target->BufUsed;
2214
2215         while (aptr < eiptr){
2216                 if(tptr + 3 >= eptr) {
2217                         IncreaseBuf(Target, 1, -1);
2218                         eptr = Target->buf + Target->BufSize - 8;
2219                         tptr = Target->buf + Target->BufUsed;
2220                 }
2221
2222                 if (*aptr == '\n') {
2223                         *tptr = '\\';
2224                         Target->BufUsed++;
2225                         tptr++;
2226                         *tptr = 'n';
2227                         Target->BufUsed++;
2228                 }
2229                 else if (*aptr == '\r') {
2230                         *tptr = '\\';
2231                         Target->BufUsed++;
2232                         tptr++;
2233                         *tptr = 'r';
2234                         Target->BufUsed++;
2235                 }
2236                 else if (*aptr == ',') {
2237                         *tptr = '\\';
2238                         Target->BufUsed++;
2239                         tptr++;
2240                         *tptr = ',';
2241                         Target->BufUsed++;
2242                 } else {
2243                         *tptr = *aptr;
2244                         Target->BufUsed++;
2245                 }
2246                 tptr++; aptr++;
2247         }
2248         *tptr = '\0';
2249 }
2250
2251 /**
2252  *  Append a string, escaping characters which have meaning in JavaScript strings .
2253  *
2254  *  Target      target buffer
2255  *  Source      source buffer; set to NULL if you just have a C-String
2256  *  PlainIn       Plain-C string to append; set to NULL if unused
2257  * @returns size of result or -1
2258  */
2259 long StrECMAEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn)
2260 {
2261         const char *aptr, *eiptr;
2262         char *bptr, *eptr;
2263         long len;
2264         int IsUtf8Sequence;
2265
2266         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2267                 return -1;
2268
2269         if (PlainIn != NULL) {
2270                 aptr = PlainIn;
2271                 len = strlen(PlainIn);
2272                 eiptr = aptr + len;
2273         }
2274         else {
2275                 aptr = Source->buf;
2276                 eiptr = aptr + Source->BufUsed;
2277                 len = Source->BufUsed;
2278         }
2279
2280         if (len == 0)
2281                 return -1;
2282
2283         bptr = Target->buf + Target->BufUsed;
2284         eptr = Target->buf + Target->BufSize - 7; /* our biggest unit to put in...  */
2285
2286         while (aptr < eiptr){
2287                 if(bptr >= eptr) {
2288                         IncreaseBuf(Target, 1, -1);
2289                         eptr = Target->buf + Target->BufSize - 7; /* our biggest unit to put in...  */
2290                         bptr = Target->buf + Target->BufUsed;
2291                 }
2292                 switch (*aptr) {
2293                 case '\n':
2294                         memcpy(bptr, HKEY("\\n"));
2295                         bptr += 2;
2296                         Target->BufUsed += 2;
2297                         break;
2298                 case '\r':
2299                         memcpy(bptr, HKEY("\\r"));
2300                         bptr += 2;
2301                         Target->BufUsed += 2;
2302                         break;
2303                 case '"':
2304                         *bptr = '\\';
2305                         bptr ++;
2306                         *bptr = '"';
2307                         bptr ++;
2308                         Target->BufUsed += 2;
2309                         break;
2310                 case '\\':
2311                         if ((*(aptr + 1) == 'u') &&
2312                             isxdigit(*(aptr + 2)) &&
2313                             isxdigit(*(aptr + 3)) &&
2314                             isxdigit(*(aptr + 4)) &&
2315                             isxdigit(*(aptr + 5)))
2316                         { /* oh, a unicode escaper. let it pass through. */
2317                                 memcpy(bptr, aptr, 6);
2318                                 aptr += 5;
2319                                 bptr +=6;
2320                                 Target->BufUsed += 6;
2321                         }
2322                         else
2323                         {
2324                                 *bptr = '\\';
2325                                 bptr ++;
2326                                 *bptr = '\\';
2327                                 bptr ++;
2328                                 Target->BufUsed += 2;
2329                         }
2330                         break;
2331                 case '\b':
2332                         *bptr = '\\';
2333                         bptr ++;
2334                         *bptr = 'b';
2335                         bptr ++;
2336                         Target->BufUsed += 2;
2337                         break;
2338                 case '\f':
2339                         *bptr = '\\';
2340                         bptr ++;
2341                         *bptr = 'f';
2342                         bptr ++;
2343                         Target->BufUsed += 2;
2344                         break;
2345                 case '\t':
2346                         *bptr = '\\';
2347                         bptr ++;
2348                         *bptr = 't';
2349                         bptr ++;
2350                         Target->BufUsed += 2;
2351                         break;
2352                 default:
2353                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(aptr, eiptr);
2354                         while (IsUtf8Sequence > 0){
2355                                 *bptr = *aptr;
2356                                 Target->BufUsed ++;
2357                                 if (--IsUtf8Sequence)
2358                                         aptr++;
2359                                 bptr++;
2360                         }
2361                 }
2362                 aptr ++;
2363         }
2364         *bptr = '\0';
2365         if ((bptr == eptr - 1 ) && !IsEmptyStr(aptr) )
2366                 return -1;
2367         return Target->BufUsed;
2368 }
2369
2370 /**
2371  *  Append a string, escaping characters which have meaning in HTML + json.
2372  *
2373  *  Target      target buffer
2374  *  Source      source buffer; set to NULL if you just have a C-String
2375  *  PlainIn       Plain-C string to append; set to NULL if unused
2376  *  nbsp                If nonzero, spaces are converted to non-breaking spaces.
2377  *  nolinebreaks        if set to 1, linebreaks are removed from the string.
2378  *                      if set to 2, linebreaks are replaced by &ltbr/&gt
2379  */
2380 long StrHtmlEcmaEscAppend(StrBuf *Target, const StrBuf *Source, const char *PlainIn, int nbsp, int nolinebreaks)
2381 {
2382         const char *aptr, *eiptr;
2383         char *bptr, *eptr;
2384         long len;
2385         int IsUtf8Sequence = 0;
2386
2387         if (((Source == NULL) && (PlainIn == NULL)) || (Target == NULL) )
2388                 return -1;
2389
2390         if (PlainIn != NULL) {
2391                 aptr = PlainIn;
2392                 len = strlen(PlainIn);
2393                 eiptr = aptr + len;
2394         }
2395         else {
2396                 aptr = Source->buf;
2397                 eiptr = aptr + Source->BufUsed;
2398                 len = Source->BufUsed;
2399         }
2400
2401         if (len == 0)
2402                 return -1;
2403
2404         bptr = Target->buf + Target->BufUsed;
2405         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2406
2407         while (aptr < eiptr){
2408                 if(bptr >= eptr) {
2409                         IncreaseBuf(Target, 1, -1);
2410                         eptr = Target->buf + Target->BufSize - 11; /* our biggest unit to put in...  */
2411                         bptr = Target->buf + Target->BufUsed;
2412                 }
2413                 switch (*aptr) {
2414                 case '<':
2415                         memcpy(bptr, HKEY("&lt;"));
2416                         bptr += 4;
2417                         Target->BufUsed += 4;
2418                         break;
2419                 case '>':
2420                         memcpy(bptr, HKEY("&gt;"));
2421                         bptr += 4;
2422                         Target->BufUsed += 4;
2423                         break;
2424                 case '&':
2425                         memcpy(bptr, HKEY("&amp;"));
2426                         bptr += 5;
2427                         Target->BufUsed += 5;
2428                         break;
2429                 case LB:
2430                         *bptr = '<';
2431                         bptr ++;
2432                         Target->BufUsed ++;
2433                         break;
2434                 case RB:
2435                         *bptr = '>';
2436                         bptr ++;
2437                         Target->BufUsed ++;
2438                         break;
2439                 case '\n':
2440                         switch (nolinebreaks) {
2441                         case 1:
2442                                 *bptr='\0';     /* nothing */
2443                                 break;
2444                         case 2:
2445                                 memcpy(bptr, HKEY("&lt;br/&gt;"));
2446                                 bptr += 11;
2447                                 Target->BufUsed += 11;
2448                                 break;
2449                         default:
2450                                 memcpy(bptr, HKEY("\\n"));
2451                                 bptr += 2;
2452                                 Target->BufUsed += 2;
2453                         }
2454                         break;
2455                 case '\r':
2456                         switch (nolinebreaks) {
2457                         case 1:
2458                         case 2:
2459                                 *bptr='\0';     /* nothing */
2460                                 break;
2461                         default:
2462                                 memcpy(bptr, HKEY("\\r"));
2463                                 bptr += 2;
2464                                 Target->BufUsed += 2;
2465                                 break;
2466                         }
2467                         break;
2468                 case '"':
2469                 case QU:
2470                         *bptr = '\\';
2471                         bptr ++;
2472                         *bptr = '"';
2473                         bptr ++;
2474                         Target->BufUsed += 2;
2475                         break;
2476                 case '\\':
2477                         if ((*(aptr + 1) == 'u') &&
2478                             isxdigit(*(aptr + 2)) &&
2479                             isxdigit(*(aptr + 3)) &&
2480                             isxdigit(*(aptr + 4)) &&
2481                             isxdigit(*(aptr + 5)))
2482                         { /* oh, a unicode escaper. let it pass through. */
2483                                 memcpy(bptr, aptr, 6);
2484                                 aptr += 5;
2485                                 bptr +=6;
2486                                 Target->BufUsed += 6;
2487                         }
2488                         else
2489                         {
2490                                 *bptr = '\\';
2491                                 bptr ++;
2492                                 *bptr = '\\';
2493                                 bptr ++;
2494                                 Target->BufUsed += 2;
2495                         }
2496                         break;
2497                 case '\b':
2498                         *bptr = '\\';
2499                         bptr ++;
2500                         *bptr = 'b';
2501                         bptr ++;
2502                         Target->BufUsed += 2;
2503                         break;
2504                 case '\f':
2505                         *bptr = '\\';
2506                         bptr ++;
2507                         *bptr = 'f';
2508                         bptr ++;
2509                         Target->BufUsed += 2;
2510                         break;
2511                 case '\t':
2512                         *bptr = '\\';
2513                         bptr ++;
2514                         *bptr = 't';
2515                         bptr ++;
2516                         Target->BufUsed += 2;
2517                         break;
2518                 case  32:
2519                         if (nbsp == 1) {
2520                                 memcpy(bptr, HKEY("&nbsp;"));
2521                                 bptr += 6;
2522                                 Target->BufUsed += 6;
2523                                 break;
2524                         }
2525                 default:
2526                         IsUtf8Sequence =  Ctdl_GetUtf8SequenceLength(aptr, eiptr);
2527                         while (IsUtf8Sequence > 0){
2528                                 *bptr = *aptr;
2529                                 Target->BufUsed ++;
2530                                 if (--IsUtf8Sequence)
2531                                         aptr++;
2532                                 bptr++;
2533                         }
2534                 }
2535                 aptr ++;
2536         }
2537         *bptr = '\0';
2538         if ((bptr = eptr - 1 ) && !IsEmptyStr(aptr) )
2539                 return -1;
2540         return Target->BufUsed;
2541 }
2542
2543
2544 /*
2545  *  replace all non-Ascii characters by another
2546  *  Buf buffer to inspect
2547  *  repl charater to stamp over non ascii chars
2548  */
2549 void StrBufAsciify(StrBuf *Buf, const char repl) {
2550         long offset;
2551
2552         for (offset = 0; offset < Buf->BufUsed; offset ++)
2553                 if (!isascii(Buf->buf[offset]))
2554                         Buf->buf[offset] = repl;
2555
2556 }
2557
2558 /*
2559  *  unhide special chars hidden to the HTML escaper
2560  *  target buffer to put the unescaped string in
2561  *  source buffer to unescape
2562  */
2563 void StrBufEUid_unescapize(StrBuf *target, const StrBuf *source) {
2564         int a, b, len;
2565         char hex[3];
2566
2567         if ((source == NULL) || (target == NULL) || (target->buf == NULL)) {
2568                 return;
2569         }
2570
2571         if (target != NULL)
2572                 FlushStrBuf(target);
2573
2574         len = source->BufUsed;
2575         for (a = 0; a < len; ++a) {
2576                 if (target->BufUsed >= target->BufSize)
2577                         IncreaseBuf(target, 1, -1);
2578
2579                 if (source->buf[a] == '=') {
2580                         hex[0] = source->buf[a + 1];
2581                         hex[1] = source->buf[a + 2];
2582                         hex[2] = 0;
2583                         b = 0;
2584                         sscanf(hex, "%02x", &b);
2585                         target->buf[target->BufUsed] = b;
2586                         target->buf[++target->BufUsed] = 0;
2587                         a += 2;
2588                 }
2589                 else {
2590                         target->buf[target->BufUsed] = source->buf[a];
2591                         target->buf[++target->BufUsed] = 0;
2592                 }
2593         }
2594 }
2595
2596
2597 /*
2598  *  hide special chars from the HTML escapers and friends
2599  *  target buffer to put the escaped string in
2600  *  source buffer to escape
2601  */
2602 void StrBufEUid_escapize(StrBuf *target, const StrBuf *source) {
2603         int i, len;
2604
2605         if (target != NULL)
2606                 FlushStrBuf(target);
2607
2608         if ((source == NULL) || (target == NULL) || (target->buf == NULL))
2609         {
2610                 return;
2611         }
2612
2613         len = source->BufUsed;
2614         for (i=0; i<len; ++i) {
2615                 if (target->BufUsed + 4 >= target->BufSize)
2616                         IncreaseBuf(target, 1, -1);
2617                 if ( (isalnum(source->buf[i])) ||
2618                      (source->buf[i]=='-') ||
2619                      (source->buf[i]=='_') ) {
2620                         target->buf[target->BufUsed++] = source->buf[i];
2621                 }
2622                 else {
2623                         sprintf(&target->buf[target->BufUsed],
2624                                 "=%02X",
2625                                 (0xFF &source->buf[i]));
2626                         target->BufUsed += 3;
2627                 }
2628         }
2629         target->buf[target->BufUsed + 1] = '\0';
2630 }
2631
2632
2633 /*******************************************************************************
2634  *                      Quoted Printable de/encoding                           *
2635  *******************************************************************************/
2636
2637 /*
2638  *  decode a buffer from base 64 encoding; destroys original
2639  *  Buf Buffor to transform
2640  */
2641 int StrBufDecodeBase64(StrBuf *Buf) {
2642         char *xferbuf;
2643         size_t siz;
2644
2645         if (Buf == NULL) {
2646                 return -1;
2647         }
2648
2649         xferbuf = (char*) malloc(Buf->BufSize);
2650         if (xferbuf == NULL)
2651                 return -1;
2652
2653         *xferbuf = '\0';
2654         siz = CtdlDecodeBase64(xferbuf, Buf->buf, Buf->BufUsed);
2655         free(Buf->buf);
2656         Buf->buf = xferbuf;
2657         Buf->BufUsed = siz;
2658
2659         Buf->buf[Buf->BufUsed] = '\0';
2660         return siz;
2661 }
2662
2663
2664 /*
2665  *  decode a buffer from base 64 encoding; expects targetbuffer
2666  *  BufIn Buffor to transform
2667  *  BufOut Buffer to put result into
2668  */
2669 int StrBufDecodeBase64To(const StrBuf *BufIn, StrBuf *BufOut) {
2670         if ((BufIn == NULL) || (BufOut == NULL))
2671                 return -1;
2672
2673         if (BufOut->BufSize < BufIn->BufUsed) {
2674                 IncreaseBuf(BufOut, 0, BufIn->BufUsed);
2675         }
2676
2677         BufOut->BufUsed = CtdlDecodeBase64(BufOut->buf, BufIn->buf, BufIn->BufUsed);
2678         return BufOut->BufUsed;
2679 }
2680
2681 typedef struct __z_enc_stream {
2682         StrBuf OutBuf;
2683         z_stream zstream;
2684 } z_enc_stream;
2685
2686
2687 vStreamT *StrBufNewStreamContext(eStreamType type, const char **Err) {
2688         //base64_decodestate *state;;
2689         *Err = NULL;
2690
2691         switch (type) {
2692
2693                 //case eBase64Decode:
2694                 //case eBase64Encode:
2695                         //state = (base64_decodestate*) malloc(sizeof(base64_decodestate));
2696                         //base64_init_decodestate(state);
2697                         //return (vStreamT*) state;
2698                         //break;
2699
2700                 case eZLibDecode: {
2701
2702                         z_enc_stream *stream;
2703                         int err;
2704
2705                         stream = (z_enc_stream *) malloc(sizeof(z_enc_stream));
2706                         memset(stream, 0, sizeof(z_enc_stream));
2707                         stream->OutBuf.BufSize = 4*SIZ; /// TODO 64
2708                         stream->OutBuf.buf = (char*)malloc(stream->OutBuf.BufSize);
2709
2710                         err = inflateInit(&stream->zstream);
2711
2712                         if (err != Z_OK) {
2713                                 StrBufDestroyStreamContext(type, (vStreamT**) &stream, Err);
2714                                 *Err = zError(err);
2715                                 return NULL;
2716                         }
2717                         return (vStreamT*) stream;
2718
2719                 }
2720
2721                 case eZLibEncode: {
2722                         z_enc_stream *stream;
2723                         int err;
2724
2725                         stream = (z_enc_stream *) malloc(sizeof(z_enc_stream));
2726                         memset(stream, 0, sizeof(z_enc_stream));
2727                         stream->OutBuf.BufSize = 4*SIZ; /// todo 64
2728                         stream->OutBuf.buf = (char*)malloc(stream->OutBuf.BufSize);
2729                         /* write gzip header */
2730                         stream->OutBuf.BufUsed = snprintf
2731                                 (stream->OutBuf.buf,
2732                                 stream->OutBuf.BufSize,
2733                                 "%c%c%c%c%c%c%c%c%c%c",
2734                                 gz_magic[0], gz_magic[1], Z_DEFLATED,
2735                                 0 /*flags */ , 0, 0, 0, 0 /*time */ , 0 /* xflags */ ,
2736                                 OS_CODE);
2737
2738                         err = deflateInit2(&stream->zstream,
2739                                         ZLibCompressionRatio,
2740                                         Z_DEFLATED,
2741                                         -MAX_WBITS,
2742                                         DEF_MEM_LEVEL,
2743                                         Z_DEFAULT_STRATEGY);
2744                         if (err != Z_OK) {
2745                                 StrBufDestroyStreamContext(type, (vStreamT**) &stream, Err);
2746                                 *Err = zError(err);
2747                                 return NULL;
2748                         }
2749                         return (vStreamT*) stream;
2750                 }
2751
2752                 case eEmtyCodec:
2753                         /// TODO
2754                         break;
2755                 }
2756         return NULL;
2757 }
2758
2759
2760 int StrBufDestroyStreamContext(eStreamType type, vStreamT **vStream, const char **Err) {
2761         int err;
2762         int rc = 0;
2763         *Err = NULL;
2764
2765         if ((vStream == NULL) || (*vStream==NULL)) {
2766                 *Err = strerror(EINVAL);
2767                 return EINVAL;
2768         }
2769         switch (type)
2770         {
2771         //case eBase64Encode:
2772         //case eBase64Decode:
2773                 //free(*vStream);
2774                 //*vStream = NULL;
2775                 //break;
2776         case eZLibDecode:
2777         {
2778                 z_enc_stream *stream = (z_enc_stream *)*vStream;
2779                 (void)inflateEnd(&stream->zstream);
2780                 free(stream->OutBuf.buf);
2781                 free(stream);
2782         }
2783         case eZLibEncode:
2784         {
2785                 z_enc_stream *stream = (z_enc_stream *)*vStream;
2786                 err = deflateEnd(&stream->zstream);
2787                 if (err != Z_OK) {
2788                         *Err = zError(err);
2789                         rc = -1;
2790                 }
2791                 free(stream->OutBuf.buf);
2792                 free(stream);
2793                 *vStream = NULL;
2794                 break;
2795         }
2796         case eEmtyCodec:
2797                 break; /// TODO
2798         }
2799         return rc;
2800 }
2801
2802 int StrBufStreamTranscode(eStreamType type, IOBuffer *Target, IOBuffer *In, const char* pIn, long pInLen, vStreamT *vStream, int LastChunk, const char **Err) {
2803         int rc = 0;
2804         switch (type)
2805         {
2806         //case eBase64Encode:
2807         //{
2808                 // ???
2809         //}
2810         //break;
2811         //case eBase64Decode:
2812         //{
2813                 //// ???
2814         //}
2815         //break;
2816         case eZLibEncode:
2817         {
2818                 z_enc_stream *stream = (z_enc_stream *)vStream;
2819                 int org_outbuf_len = stream->OutBuf.BufUsed;
2820                 int err;
2821                 unsigned int chunkavail;
2822
2823                 if (In->ReadWritePointer != NULL)
2824                 {
2825                         stream->zstream.next_in = (Bytef *) In->ReadWritePointer;
2826                         stream->zstream.avail_in = (uInt) In->Buf->BufUsed -
2827                                 (In->ReadWritePointer - In->Buf->buf);
2828                 }
2829                 else
2830                 {
2831                         stream->zstream.next_in = (Bytef *) In->Buf->buf;
2832                         stream->zstream.avail_in = (uInt) In->Buf->BufUsed;
2833                 }
2834
2835                 stream->zstream.next_out = (unsigned char*)stream->OutBuf.buf + stream->OutBuf.BufUsed;
2836                 stream->zstream.avail_out = chunkavail = (uInt) stream->OutBuf.BufSize - stream->OutBuf.BufUsed;
2837
2838                 err = deflate(&stream->zstream,  (LastChunk) ? Z_FINISH : Z_NO_FLUSH);
2839
2840                 stream->OutBuf.BufUsed += (chunkavail - stream->zstream.avail_out);
2841
2842                 if (Target && (LastChunk || (stream->OutBuf.BufUsed != org_outbuf_len))) {
2843                         iSwapBuffers(Target->Buf, &stream->OutBuf);
2844                 }
2845
2846                 if (stream->zstream.avail_in == 0) {
2847                         FlushStrBuf(In->Buf);
2848                         In->ReadWritePointer = NULL;
2849                 }
2850                 else {
2851                         if (stream->zstream.avail_in < 64) {
2852                                 memmove(In->Buf->buf,
2853                                         In->Buf->buf + In->Buf->BufUsed - stream->zstream.avail_in,
2854                                         stream->zstream.avail_in);
2855
2856                                 In->Buf->BufUsed = stream->zstream.avail_in;
2857                                 In->Buf->buf[In->Buf->BufUsed] = '\0';
2858                         }
2859                         else {
2860                                 In->ReadWritePointer = In->Buf->buf + (In->Buf->BufUsed - stream->zstream.avail_in);
2861                         }
2862                 }
2863                 rc = (LastChunk && (err != Z_FINISH));
2864                 if (!rc && (err != Z_OK)) {
2865                         *Err = zError(err);
2866                 }
2867
2868         }
2869         break;
2870         case eZLibDecode: {
2871                 z_enc_stream *stream = (z_enc_stream *)vStream;
2872                 int org_outbuf_len = stream->zstream.total_out;
2873                 int err;
2874
2875                 if ((stream->zstream.avail_out != 0) && (stream->zstream.next_in != NULL)) {
2876                         if (In->ReadWritePointer != NULL) {
2877                                 stream->zstream.next_in = (Bytef *) In->ReadWritePointer;
2878                                 stream->zstream.avail_in = (uInt) In->Buf->BufUsed - (In->ReadWritePointer - In->Buf->buf);
2879                         }
2880                         else {
2881                                 stream->zstream.next_in = (Bytef *) In->Buf->buf;
2882                                 stream->zstream.avail_in = (uInt) In->Buf->BufUsed;
2883                         }
2884                 }
2885
2886                 stream->zstream.next_out = (unsigned char*)stream->OutBuf.buf + stream->OutBuf.BufUsed;
2887                 stream->zstream.avail_out = (uInt) stream->OutBuf.BufSize - stream->OutBuf.BufUsed;
2888
2889                 err = inflate(&stream->zstream, Z_NO_FLUSH);
2890
2891                 ///assert(ret != Z_STREAM_ERROR);  /* state not clobbered * /
2892                 switch (err) {
2893                 case Z_NEED_DICT:
2894                         err = Z_DATA_ERROR;     /* and fall through */
2895
2896                 case Z_DATA_ERROR:
2897                         *Err = zError(err);
2898                 case Z_MEM_ERROR:
2899                         (void)inflateEnd(&stream->zstream);
2900                         return err;
2901                 }
2902
2903                 stream->OutBuf.BufUsed += stream->zstream.total_out + org_outbuf_len;
2904
2905                 if (Target) iSwapBuffers(Target->Buf, &stream->OutBuf);
2906
2907                 if (stream->zstream.avail_in == 0) {
2908                         FlushStrBuf(In->Buf);
2909                         In->ReadWritePointer = NULL;
2910                 }
2911                 else {
2912                         if (stream->zstream.avail_in < 64) {
2913                                 memmove(In->Buf->buf,
2914                                         In->Buf->buf + In->Buf->BufUsed - stream->zstream.avail_in,
2915                                         stream->zstream.avail_in);
2916
2917                                 In->Buf->BufUsed = stream->zstream.avail_in;
2918                                 In->Buf->buf[In->Buf->BufUsed] = '\0';
2919                         }
2920                         else {
2921
2922                                 In->ReadWritePointer = In->Buf->buf + (In->Buf->BufUsed - stream->zstream.avail_in);
2923                         }
2924                 }
2925         }
2926                 break;
2927         case eEmtyCodec: {
2928
2929         }
2930                 break; /// TODO
2931         }
2932         return rc;
2933 }
2934
2935 /**
2936  *  decode a buffer from base 64 encoding; destroys original
2937  *  Buf Buffor to transform
2938  */
2939 int StrBufDecodeHex(StrBuf *Buf) {
2940         unsigned int ch;
2941         char *pch, *pche, *pchi;
2942
2943         if (Buf == NULL) return -1;
2944
2945         pch = pchi = Buf->buf;
2946         pche = pch + Buf->BufUsed;
2947
2948         while (pchi < pche){
2949                 ch = decode_hex(pchi);
2950                 *pch = ch;
2951                 pch ++;
2952                 pchi += 2;
2953         }
2954
2955         *pch = '\0';
2956         Buf->BufUsed = pch - Buf->buf;
2957         return Buf->BufUsed;
2958 }
2959
2960 /**
2961  *  replace all chars >0x20 && < 0x7F with Mute
2962  *  Mute char to put over invalid chars
2963  *  Buf Buffor to transform
2964  */
2965 int StrBufSanitizeAscii(StrBuf *Buf, const char Mute)
2966 {
2967         unsigned char *pch;
2968
2969         if (Buf == NULL) return -1;
2970         pch = (unsigned char *)Buf->buf;
2971         while (pch < (unsigned char *)Buf->buf + Buf->BufUsed) {
2972                 if ((*pch < 0x20) || (*pch > 0x7F))
2973                         *pch = Mute;
2974                 pch ++;
2975         }
2976         return Buf->BufUsed;
2977 }
2978
2979
2980 /**
2981  *  remove escaped strings from i.e. the url string (like %20 for blanks)
2982  *  Buf Buffer to translate
2983  *  StripBlanks Reduce several blanks to one?
2984  */
2985 long StrBufUnescape(StrBuf *Buf, int StripBlanks)
2986 {
2987         int a, b;
2988         char hex[3];
2989         long len;
2990
2991         if (Buf == NULL)
2992                 return -1;
2993
2994         while ((Buf->BufUsed > 0) && (isspace(Buf->buf[Buf->BufUsed - 1]))){
2995                 Buf->buf[Buf->BufUsed - 1] = '\0';
2996                 Buf->BufUsed --;
2997         }
2998
2999         a = 0;
3000         while (a < Buf->BufUsed) {
3001                 if (Buf->buf[a] == '+')
3002                         Buf->buf[a] = ' ';
3003                 else if (Buf->buf[a] == '%') {
3004                         /* don't let % chars through, rather truncate the input. */
3005                         if (a + 2 > Buf->BufUsed) {
3006                                 Buf->buf[a] = '\0';
3007                                 Buf->BufUsed = a;
3008                         }
3009                         else {
3010                                 hex[0] = Buf->buf[a + 1];
3011                                 hex[1] = Buf->buf[a + 2];
3012                                 hex[2] = 0;
3013                                 b = 0;
3014                                 sscanf(hex, "%02x", &b);
3015                                 Buf->buf[a] = (char) b;
3016                                 len = Buf->BufUsed - a - 2;
3017                                 if (len > 0)
3018                                         memmove(&Buf->buf[a + 1], &Buf->buf[a + 3], len);
3019
3020                                 Buf->BufUsed -=2;
3021                         }
3022                 }
3023                 a++;
3024         }
3025         return a;
3026 }
3027
3028
3029 /**
3030  *      RFC2047-encode a header field if necessary.
3031  *              If no non-ASCII characters are found, the string
3032  *              will be copied verbatim without encoding.
3033  *
3034  *      target          Target buffer.
3035  *      source          Source string to be encoded.
3036  * @returns     encoded length; -1 if non success.
3037  */
3038 int StrBufRFC2047encode(StrBuf **target, const StrBuf *source)
3039 {
3040         const char headerStr[] = "=?UTF-8?Q?";
3041         int need_to_encode = 0;
3042         int i = 0;
3043         unsigned char ch;
3044
3045         if ((source == NULL) ||
3046             (target == NULL))
3047             return -1;
3048
3049         while ((i < source->BufUsed) &&
3050                (!IsEmptyStr (&source->buf[i])) &&
3051                (need_to_encode == 0)) {
3052                 if (((unsigned char) source->buf[i] < 32) ||
3053                     ((unsigned char) source->buf[i] > 126)) {
3054                         need_to_encode = 1;
3055                 }
3056                 i++;
3057         }
3058
3059         if (!need_to_encode) {
3060                 if (*target == NULL) {
3061                         *target = NewStrBufPlain(source->buf, source->BufUsed);
3062                 }
3063                 else {
3064                         FlushStrBuf(*target);
3065                         StrBufAppendBuf(*target, source, 0);
3066                 }
3067                 if (*target != 0)
3068                         return (*target)->BufUsed;
3069                 else
3070                         return 0;
3071         }
3072         if (*target == NULL)
3073                 *target = NewStrBufPlain(NULL, sizeof(headerStr) + source->BufUsed * 2);
3074         else if (sizeof(headerStr) + source->BufUsed >= (*target)->BufSize)
3075                 IncreaseBuf(*target, sizeof(headerStr) + source->BufUsed, 0);
3076         memcpy ((*target)->buf, headerStr, sizeof(headerStr) - 1);
3077         (*target)->BufUsed = sizeof(headerStr) - 1;
3078         for (i=0; (i < source->BufUsed); ++i) {
3079                 if ((*target)->BufUsed + 4 >= (*target)->BufSize)
3080                         IncreaseBuf(*target, 1, 0);
3081                 ch = (unsigned char) source->buf[i];
3082                 if ((ch  <  32) ||
3083                     (ch  > 126) ||
3084                     (ch == '=') ||
3085                     (ch == '?') ||
3086                     (ch == '_') ||
3087                     (ch == '[') ||
3088                     (ch == ']')   )
3089                 {
3090                         sprintf(&(*target)->buf[(*target)->BufUsed], "=%02X", ch);
3091                         (*target)->BufUsed += 3;
3092                 }
3093                 else {
3094                         if (ch == ' ')
3095                                 (*target)->buf[(*target)->BufUsed] = '_';
3096                         else
3097                                 (*target)->buf[(*target)->BufUsed] = ch;
3098                         (*target)->BufUsed++;
3099                 }
3100         }
3101
3102         if ((*target)->BufUsed + 4 >= (*target)->BufSize)
3103                 IncreaseBuf(*target, 1, 0);
3104
3105         (*target)->buf[(*target)->BufUsed++] = '?';
3106         (*target)->buf[(*target)->BufUsed++] = '=';
3107         (*target)->buf[(*target)->BufUsed] = '\0';
3108         return (*target)->BufUsed;;
3109 }
3110
3111 /**
3112  *      Quoted-Printable encode a message; make it < 80 columns width.
3113  *      source          Source string to be encoded.
3114  * @returns     buffer with encoded message.
3115  */
3116 StrBuf *StrBufRFC2047encodeMessage(const StrBuf *EncodeMe)
3117 {
3118         StrBuf *OutBuf;
3119         char *Optr, *OEptr;
3120         const char *ptr, *eptr;
3121         unsigned char ch;
3122         int LinePos;
3123
3124         OutBuf = NewStrBufPlain(NULL, StrLength(EncodeMe) * 4);
3125         Optr = OutBuf->buf;
3126         OEptr = OutBuf->buf + OutBuf->BufSize;
3127         ptr = EncodeMe->buf;
3128         eptr = EncodeMe->buf + EncodeMe->BufUsed;
3129         LinePos = 0;
3130
3131         while (ptr < eptr)
3132         {
3133                 if (Optr + 4 >= OEptr)
3134                 {
3135                         long Offset;
3136                         Offset = Optr - OutBuf->buf;
3137                         OutBuf->BufUsed = Optr - OutBuf->buf;
3138                         IncreaseBuf(OutBuf, 1, 0);
3139                         Optr = OutBuf->buf + Offset;
3140                         OEptr = OutBuf->buf + OutBuf->BufSize;
3141                 }
3142                 if (*ptr == '\r')
3143                 {
3144                         /* ignore carriage returns */
3145                         ptr ++;
3146                 }
3147                 else if (*ptr == '\n') {
3148                         /* hard line break */
3149                         memcpy(Optr, HKEY("=0A"));
3150                         Optr += 3;
3151                         LinePos += 3;
3152                         ptr ++;
3153                 }
3154                 else if (( (*ptr >= 32) && (*ptr <= 60) ) ||
3155                          ( (*ptr >= 62) && (*ptr <= 126) ))
3156                 {
3157                         *Optr = *ptr;
3158                         Optr ++;
3159                         ptr ++;
3160                         LinePos ++;
3161                 }
3162                 else {
3163                         ch = *ptr;
3164                         *Optr = '=';
3165                         Optr ++;
3166                         *Optr = HexList[ch][0];
3167                         Optr ++;
3168                         *Optr = HexList[ch][1];
3169                         Optr ++;
3170                         LinePos += 3;
3171                         ptr ++;
3172                 }
3173
3174                 if (LinePos > 72) {
3175                         /* soft line break */
3176                         if (isspace(*(Optr - 1))) {
3177                                 ch = *(Optr - 1);
3178                                 Optr --;
3179                                 *Optr = '=';
3180                                 Optr ++;
3181                                 *Optr = HexList[ch][0];
3182                                 Optr ++;
3183                                 *Optr = HexList[ch][1];
3184                                 Optr ++;
3185                                 LinePos += 3;
3186                         }
3187                         *Optr = '=';
3188                         Optr ++;
3189                         *Optr = '\n';
3190                         Optr ++;
3191                         LinePos = 0;
3192                 }
3193         }
3194         *Optr = '\0';
3195         OutBuf->BufUsed = Optr - OutBuf->buf;
3196
3197         return OutBuf;
3198 }
3199
3200
3201 static void AddRecipient(StrBuf *Target,
3202                          StrBuf *UserName,
3203                          StrBuf *EmailAddress,
3204                          StrBuf *EncBuf)
3205 {
3206         int QuoteMe = 0;
3207
3208         if (StrLength(Target) > 0) StrBufAppendBufPlain(Target, HKEY(", "), 0);
3209         if (strchr(ChrPtr(UserName), ',') != NULL) QuoteMe = 1;
3210
3211         if (QuoteMe)  StrBufAppendBufPlain(Target, HKEY("\""), 0);
3212         StrBufRFC2047encode(&EncBuf, UserName);
3213         StrBufAppendBuf(Target, EncBuf, 0);
3214         if (QuoteMe)  StrBufAppendBufPlain(Target, HKEY("\" "), 0);
3215         else          StrBufAppendBufPlain(Target, HKEY(" "), 0);
3216
3217         if (StrLength(EmailAddress) > 0){
3218                 StrBufAppendBufPlain(Target, HKEY("<"), 0);
3219                 StrBufAppendBuf(Target, EmailAddress, 0); /* TODO: what about IDN???? */
3220                 StrBufAppendBufPlain(Target, HKEY(">"), 0);
3221         }
3222 }
3223
3224
3225 /**
3226  * \brief QP encode parts of an email TO/CC/BCC vector, and strip/filter invalid parts
3227  * \param Recp Source list of email recipients
3228  * \param UserName Temporary buffer for internal use; Please provide valid buffer.
3229  * \param EmailAddress Temporary buffer for internal use; Please provide valid buffer.
3230  * \param EncBuf Temporary buffer for internal use; Please provide valid buffer.
3231  * \returns encoded & sanitized buffer with the contents of Recp; Caller owns this memory.
3232  */
3233 StrBuf *StrBufSanitizeEmailRecipientVector(const StrBuf *Recp,
3234                                            StrBuf *UserName,
3235                                            StrBuf *EmailAddress,
3236                                            StrBuf *EncBuf)
3237 {
3238         StrBuf *Target;
3239         const char *pch, *pche;
3240         const char *UserStart, *UserEnd, *EmailStart, *EmailEnd, *At;
3241
3242         if ((Recp == NULL) || (StrLength(Recp) == 0))
3243                 return NULL;
3244
3245         pch = ChrPtr(Recp);
3246         pche = pch + StrLength(Recp);
3247
3248         if (!CheckEncode(pch, -1, pche))
3249                 return NewStrBufDup(Recp);
3250
3251         Target = NewStrBufPlain(NULL, StrLength(Recp));
3252
3253         while ((pch != NULL) && (pch < pche))
3254         {
3255                 while (isspace(*pch)) pch++;
3256                 UserEnd = EmailStart = EmailEnd = NULL;
3257
3258                 if ((*pch == '"') || (*pch == '\'')) {
3259                         UserStart = pch + 1;
3260
3261                         UserEnd = strchr(UserStart, *pch);
3262                         if (UserEnd == NULL)
3263                                 break; ///TODO: Userfeedback??
3264                         EmailStart = UserEnd + 1;
3265                         while (isspace(*EmailStart))
3266                                 EmailStart++;
3267                         if (UserEnd == UserStart) {
3268                                 UserStart = UserEnd = NULL;
3269                         }
3270
3271                         if (*EmailStart == '<') {
3272                                 EmailStart++;
3273                                 EmailEnd = strchr(EmailStart, '>');
3274                                 if (EmailEnd == NULL)
3275                                         EmailEnd = strchr(EmailStart, ',');
3276
3277                         }
3278                         else {
3279                                 EmailEnd = strchr(EmailStart, ',');
3280                         }
3281                         if (EmailEnd == NULL)
3282                                 EmailEnd = pche;
3283                         pch = EmailEnd + 1;
3284                 }
3285                 else {
3286                         int gt = 0;
3287                         UserStart = pch;
3288                         EmailEnd = strchr(UserStart, ',');
3289                         if (EmailEnd == NULL) {
3290                                 EmailEnd = strchr(pch, '>');
3291                                 pch = NULL;
3292                                 if (EmailEnd != NULL) {
3293                                         gt = 1;
3294                                 }
3295                                 else {
3296                                         EmailEnd = pche;
3297                                 }
3298                         }
3299                         else {
3300
3301                                 pch = EmailEnd + 1;
3302                                 while ((EmailEnd > UserStart) && !gt &&
3303                                        ((*EmailEnd == ',') ||
3304                                         (*EmailEnd == '>') ||
3305                                         (isspace(*EmailEnd))))
3306                                 {
3307                                         if (*EmailEnd == '>')
3308                                                 gt = 1;
3309                                         else
3310                                                 EmailEnd--;
3311                                 }
3312                                 if (EmailEnd == UserStart)
3313                                         break;
3314                         }
3315                         if (gt) {
3316                                 EmailStart = strchr(UserStart, '<');
3317                                 if ((EmailStart == NULL) || (EmailStart > EmailEnd))
3318                                         break;
3319                                 UserEnd = EmailStart;
3320
3321                                 while ((UserEnd > UserStart) &&
3322                                        isspace (*(UserEnd - 1)))
3323                                         UserEnd --;
3324                                 EmailStart ++;
3325                                 if (UserStart >= UserEnd)
3326                                         UserStart = UserEnd = NULL;
3327                         }
3328                         else { /* this is a local recipient... no domain, just a realname */
3329                                 EmailStart = UserStart;
3330                                 At = strchr(EmailStart, '@');
3331                                 if (At == NULL) {
3332                                         UserEnd = EmailEnd;
3333                                         EmailEnd = NULL;
3334                                 }
3335                                 else {
3336                                         EmailStart = UserStart;
3337                                         UserStart = NULL;
3338                                 }
3339                         }
3340                 }
3341
3342                 if ((UserStart != NULL) && (UserEnd != NULL))
3343                         StrBufPlain(UserName, UserStart, UserEnd - UserStart);
3344                 else if ((UserStart != NULL) && (UserEnd == NULL))
3345                         StrBufPlain(UserName, UserStart, UserEnd - UserStart);
3346                 else
3347                         FlushStrBuf(UserName);
3348
3349                 if ((EmailStart != NULL) && (EmailEnd != NULL))
3350                         StrBufPlain(EmailAddress, EmailStart, EmailEnd - EmailStart);
3351                 else if ((EmailStart != NULL) && (EmailEnd == NULL))
3352                         StrBufPlain(EmailAddress, EmailStart, EmailEnd - pche);
3353                 else
3354                         FlushStrBuf(EmailAddress);
3355
3356                 AddRecipient(Target, UserName, EmailAddress, EncBuf);
3357
3358                 if (pch == NULL)
3359                         break;
3360
3361                 if ((pch != NULL) && (*pch == ','))
3362                         pch ++;
3363                 if (pch != NULL) while (isspace(*pch))
3364                         pch ++;
3365         }
3366         return Target;
3367 }
3368
3369
3370 /**
3371  *  replaces all occurances of 'search' by 'replace'
3372  *  buf Buffer to modify
3373  *  search character to search
3374  *  replace character to replace search by
3375  */
3376 void StrBufReplaceChars(StrBuf *buf, char search, char replace)
3377 {
3378         long i;
3379         if (buf == NULL)
3380                 return;
3381         for (i=0; i<buf->BufUsed; i++)
3382                 if (buf->buf[i] == search)
3383                         buf->buf[i] = replace;
3384
3385 }
3386
3387 /**
3388  *  removes all \\r s from the string, or replaces them with \n if its not a combination of both.
3389  *  buf Buffer to modify
3390  */
3391 void StrBufToUnixLF(StrBuf *buf)
3392 {
3393         char *pche, *pchS, *pchT;
3394         if (buf == NULL)
3395                 return;
3396
3397         pche = buf->buf + buf->BufUsed;
3398         pchS = pchT = buf->buf;
3399         while (pchS < pche)
3400         {
3401                 if (*pchS == '\r')
3402                 {
3403                         pchS ++;
3404                         if (*pchS != '\n') {
3405                                 *pchT = '\n';
3406                                 pchT++;
3407                         }
3408                 }
3409                 *pchT = *pchS;
3410                 pchT++; pchS++;
3411         }
3412         *pchT = '\0';
3413         buf->BufUsed = pchT - buf->buf;
3414 }
3415
3416
3417 /*******************************************************************************
3418  *                 Iconv Wrapper; RFC822 de/encoding                           *
3419  *******************************************************************************/
3420
3421 /**
3422  *  Wrapper around iconv_open()
3423  * Our version adds aliases for non-standard Microsoft charsets
3424  * such as 'MS950', aliasing them to names like 'CP950'
3425  *
3426  *  tocode      Target encoding
3427  *  fromcode    Source encoding
3428  *  pic           anonimized pointer to iconv struct
3429  */
3430 void  ctdl_iconv_open(const char *tocode, const char *fromcode, void *pic)
3431 {
3432 #ifdef HAVE_ICONV
3433         iconv_t ic = (iconv_t)(-1) ;
3434         ic = iconv_open(tocode, fromcode);
3435         if (ic == (iconv_t)(-1) ) {
3436                 char alias_fromcode[64];
3437                 if ( (strlen(fromcode) == 5) && (!strncasecmp(fromcode, "MS", 2)) ) {
3438                         safestrncpy(alias_fromcode, fromcode, sizeof alias_fromcode);
3439                         alias_fromcode[0] = 'C';
3440                         alias_fromcode[1] = 'P';
3441                         ic = iconv_open(tocode, alias_fromcode);
3442                 }
3443         }
3444         *(iconv_t *)pic = ic;
3445 #endif
3446 }
3447
3448
3449 /**
3450  *  find one chunk of a RFC822 encoded string
3451  *  Buffer where to search
3452  *  bptr where to start searching
3453  * @returns found position, NULL if none.
3454  */
3455 static inline const char *FindNextEnd (const StrBuf *Buf, const char *bptr)
3456 {
3457         const char * end;
3458         /* Find the next ?Q? */
3459         if (Buf->BufUsed - (bptr - Buf->buf)  < 6)
3460                 return NULL;
3461
3462         end = strchr(bptr + 2, '?');
3463
3464         if (end == NULL)
3465                 return NULL;
3466
3467         if ((Buf->BufUsed - (end - Buf->buf) > 3) &&
3468             (((*(end + 1) == 'B') || (*(end + 1) == 'Q')) ||
3469              ((*(end + 1) == 'b') || (*(end + 1) == 'q'))) &&
3470             (*(end + 2) == '?')) {
3471                 /* skip on to the end of the cluster, the next ?= */
3472                 end = strstr(end + 3, "?=");
3473         }
3474         else
3475                 /* sort of half valid encoding, try to find an end. */
3476                 end = strstr(bptr, "?=");
3477         return end;
3478 }
3479
3480
3481
3482 /**
3483  *  convert one buffer according to the preselected iconv pointer PIC
3484  *  ConvertBuf buffer we need to translate
3485  *  TmpBuf To share a workbuffer over several iterations. prepare to have it filled with useless stuff afterwards.
3486  *  pic Pointer to the iconv-session Object
3487  */
3488 void StrBufConvert(StrBuf *ConvertBuf, StrBuf *TmpBuf, void *pic)
3489 {
3490 #ifdef HAVE_ICONV
3491         long trycount = 0;
3492         size_t siz;
3493         iconv_t ic;
3494         char *ibuf;                     /**< Buffer of characters to be converted */
3495         char *obuf;                     /**< Buffer for converted characters */
3496         size_t ibuflen;                 /**< Length of input buffer */
3497         size_t obuflen;                 /**< Length of output buffer */
3498
3499
3500         if ((ConvertBuf == NULL) || (TmpBuf == NULL))
3501                 return;
3502
3503         /* since we're converting to utf-8, one glyph may take up to 6 bytes */
3504         if (ConvertBuf->BufUsed * 6 >= TmpBuf->BufSize)
3505                 IncreaseBuf(TmpBuf, 0, ConvertBuf->BufUsed * 6);
3506 TRYAGAIN:
3507         ic = *(iconv_t*)pic;
3508         ibuf = ConvertBuf->buf;
3509         ibuflen = ConvertBuf->BufUsed;
3510         obuf = TmpBuf->buf;
3511         obuflen = TmpBuf->BufSize;
3512
3513         siz = iconv(ic, &ibuf, &ibuflen, &obuf, &obuflen);
3514
3515         if (siz < 0) {
3516                 if (errno == E2BIG) {
3517                         trycount ++;
3518                         IncreaseBuf(TmpBuf, 0, 0);
3519                         if (trycount < 5)
3520                                 goto TRYAGAIN;
3521
3522                 }
3523                 else if (errno == EILSEQ){
3524                         /* hm, invalid utf8 sequence... what to do now? */
3525                         /* An invalid multibyte sequence has been encountered in the input */
3526                 }
3527                 else if (errno == EINVAL) {
3528                         /* An incomplete multibyte sequence has been encountered in the input. */
3529                 }
3530
3531                 FlushStrBuf(TmpBuf);
3532         }
3533         else {
3534                 TmpBuf->BufUsed = TmpBuf->BufSize - obuflen;
3535                 TmpBuf->buf[TmpBuf->BufUsed] = '\0';
3536
3537                 /* little card game: wheres the red lady? */
3538                 iSwapBuffers(ConvertBuf, TmpBuf);
3539                 FlushStrBuf(TmpBuf);
3540         }
3541 #endif
3542 }
3543
3544
3545 /**
3546  *  catches one RFC822 encoded segment, and decodes it.
3547  *  Target buffer to fill with result
3548  *  DecodeMe buffer with stuff to process
3549  *  SegmentStart points to our current segment in DecodeMe
3550  *  SegmentEnd Points to the end of our current segment in DecodeMe
3551  *  ConvertBuf Workbuffer shared between several iterations. Random content; needs to be valid
3552  *  ConvertBuf2 Workbuffer shared between several iterations. Random content; needs to be valid
3553  *  FoundCharset Characterset to default decoding to; if we find another we will overwrite it.
3554  */
3555 inline static void DecodeSegment(StrBuf *Target,
3556                                  const StrBuf *DecodeMe,
3557                                  const char *SegmentStart,
3558                                  const char *SegmentEnd,
3559                                  StrBuf *ConvertBuf,
3560                                  StrBuf *ConvertBuf2,
3561                                  StrBuf *FoundCharset)
3562 {
3563         StrBuf StaticBuf;
3564         char charset[128];
3565         char encoding[16];
3566 #ifdef HAVE_ICONV
3567         iconv_t ic = (iconv_t)(-1);
3568 #else
3569         void *ic = NULL;
3570 #endif
3571         /* Now we handle foreign character sets properly encoded
3572          * in RFC2047 format.
3573          */
3574         StaticBuf.buf = (char*) SegmentStart; /*< it will just be read there... */
3575         StaticBuf.BufUsed = SegmentEnd - SegmentStart;
3576         StaticBuf.BufSize = DecodeMe->BufSize - (SegmentStart - DecodeMe->buf);
3577         extract_token(charset, SegmentStart, 1, '?', sizeof charset);
3578         if (FoundCharset != NULL) {
3579                 FlushStrBuf(FoundCharset);
3580                 StrBufAppendBufPlain(FoundCharset, charset, -1, 0);
3581         }
3582         extract_token(encoding, SegmentStart, 2, '?', sizeof encoding);
3583         StrBufExtract_token(ConvertBuf, &StaticBuf, 3, '?');
3584
3585         *encoding = toupper(*encoding);
3586         if (*encoding == 'B') { /**< base64 */
3587                 if (ConvertBuf2->BufSize < ConvertBuf->BufUsed)
3588                         IncreaseBuf(ConvertBuf2, 0, ConvertBuf->BufUsed);
3589                 ConvertBuf2->BufUsed = CtdlDecodeBase64(ConvertBuf2->buf,
3590                                                         ConvertBuf->buf,
3591                                                         ConvertBuf->BufUsed);
3592         }
3593         else if (*encoding == 'Q') {    /**< quoted-printable */
3594                 long pos;
3595
3596                 pos = 0;
3597                 while (pos < ConvertBuf->BufUsed)
3598                 {
3599                         if (ConvertBuf->buf[pos] == '_')
3600                                 ConvertBuf->buf[pos] = ' ';
3601                         pos++;
3602                 }
3603
3604                 if (ConvertBuf2->BufSize < ConvertBuf->BufUsed)
3605                         IncreaseBuf(ConvertBuf2, 0, ConvertBuf->BufUsed);
3606
3607                 ConvertBuf2->BufUsed = CtdlDecodeQuotedPrintable(
3608                         ConvertBuf2->buf,
3609                         ConvertBuf->buf,
3610                         ConvertBuf->BufUsed);
3611         }
3612         else {
3613                 StrBufAppendBuf(ConvertBuf2, ConvertBuf, 0);
3614         }
3615 #ifdef HAVE_ICONV
3616         ctdl_iconv_open("UTF-8", charset, &ic);
3617         if (ic != (iconv_t)(-1) ) {
3618 #endif
3619                 StrBufConvert(ConvertBuf2, ConvertBuf, &ic);
3620                 StrBufAppendBuf(Target, ConvertBuf2, 0);
3621 #ifdef HAVE_ICONV
3622                 iconv_close(ic);
3623         }
3624         else {
3625                 StrBufAppendBufPlain(Target, HKEY("(unreadable)"), 0);
3626         }
3627 #endif
3628 }
3629
3630 /**
3631  *  Handle subjects with RFC2047 encoding such as: [deprecated old syntax!]
3632  * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
3633  *  Target where to put the decoded string to
3634  *  DecodeMe buffer with encoded string
3635  *  DefaultCharset if we don't find one, which should we use?
3636  *  FoundCharset overrides DefaultCharset if non-empty; If we find a charset inside of the string,
3637  *        put it here for later use where no string might be known.
3638  */
3639 void StrBuf_RFC822_to_Utf8(StrBuf *Target, const StrBuf *DecodeMe, const StrBuf* DefaultCharset, StrBuf *FoundCharset)
3640 {
3641         StrBuf *ConvertBuf;
3642         StrBuf *ConvertBuf2;
3643         ConvertBuf = NewStrBufPlain(NULL, StrLength(DecodeMe));
3644         ConvertBuf2 = NewStrBufPlain(NULL, StrLength(DecodeMe));
3645
3646         StrBuf_RFC822_2_Utf8(Target,
3647                              DecodeMe,
3648                              DefaultCharset,
3649                              FoundCharset,
3650                              ConvertBuf,
3651                              ConvertBuf2);
3652         FreeStrBuf(&ConvertBuf);
3653         FreeStrBuf(&ConvertBuf2);
3654 }
3655
3656 /**
3657  *  Handle subjects with RFC2047 encoding such as:
3658  * =?koi8-r?B?78bP0s3Mxc7JxSDXz9rE1dvO2c3JINvB0sHNySDP?=
3659  *  Target where to put the decoded string to
3660  *  DecodeMe buffer with encoded string
3661  *  DefaultCharset if we don't find one, which should we use?
3662  *  FoundCharset overrides DefaultCharset if non-empty; If we find a charset inside of the string,
3663  *        put it here for later use where no string might be known.
3664  *  ConvertBuf workbuffer. feed in, you shouldn't care about its content.
3665  *  ConvertBuf2 workbuffer. feed in, you shouldn't care about its content.
3666  */
3667 void StrBuf_RFC822_2_Utf8(StrBuf *Target,
3668                           const StrBuf *DecodeMe,
3669                           const StrBuf* DefaultCharset,
3670                           StrBuf *FoundCharset,
3671                           StrBuf *ConvertBuf,
3672                           StrBuf *ConvertBuf2)
3673 {
3674         StrBuf *DecodedInvalidBuf = NULL;
3675         const StrBuf *DecodeMee = DecodeMe;
3676         const char *start, *end, *next, *nextend, *ptr = NULL;
3677 #ifdef HAVE_ICONV
3678         iconv_t ic = (iconv_t)(-1) ;
3679 #endif
3680         const char *eptr;
3681         int passes = 0;
3682         int i;
3683         int illegal_non_rfc2047_encoding = 0;
3684
3685
3686         if (DecodeMe == NULL)
3687                 return;
3688         /* Sometimes, badly formed messages contain strings which were simply
3689          *  written out directly in some foreign character set instead of
3690          *  using RFC2047 encoding.  This is illegal but we will attempt to
3691          *  handle it anyway by converting from a user-specified default
3692          *  charset to UTF-8 if we see any nonprintable characters.
3693          */
3694
3695         for (i=0; i<DecodeMe->BufUsed; ++i) {
3696                 if ((DecodeMe->buf[i] < 32) || (DecodeMe->buf[i] > 126)) {
3697                         illegal_non_rfc2047_encoding = 1;
3698                         break;
3699                 }
3700         }
3701
3702         if ((illegal_non_rfc2047_encoding) &&
3703             (strcasecmp(ChrPtr(DefaultCharset), "UTF-8")) &&
3704             (strcasecmp(ChrPtr(DefaultCharset), "us-ascii")) )
3705         {
3706 #ifdef HAVE_ICONV
3707                 ctdl_iconv_open("UTF-8", ChrPtr(DefaultCharset), &ic);
3708                 if (ic != (iconv_t)(-1) ) {
3709                         DecodedInvalidBuf = NewStrBufDup(DecodeMe);
3710                         StrBufConvert(DecodedInvalidBuf, ConvertBuf, &ic);///TODO: don't void const?
3711                         DecodeMee = DecodedInvalidBuf;
3712                         iconv_close(ic);
3713                 }
3714 #endif
3715         }
3716
3717         /* pre evaluate the first pair */
3718         end = NULL;
3719         start = strstr(DecodeMee->buf, "=?");
3720         eptr = DecodeMee->buf + DecodeMee->BufUsed;
3721         if (start != NULL)
3722                 end = FindNextEnd (DecodeMee, start + 2);
3723         else {
3724                 StrBufAppendBuf(Target, DecodeMee, 0);
3725                 FreeStrBuf(&DecodedInvalidBuf);
3726                 return;
3727         }
3728
3729
3730         if (start != DecodeMee->buf) {
3731                 long nFront;
3732
3733                 nFront = start - DecodeMee->buf;
3734                 StrBufAppendBufPlain(Target, DecodeMee->buf, nFront, 0);
3735         }
3736         /*
3737          * Since spammers will go to all sorts of absurd lengths to get their
3738          * messages through, there are LOTS of corrupt headers out there.
3739          * So, prevent a really badly formed RFC2047 header from throwing
3740          * this function into an infinite loop.
3741          */
3742         while ((start != NULL) &&
3743                (end != NULL) &&
3744                (start < eptr) &&
3745                (end < eptr) &&
3746                (passes < 20))
3747         {
3748                 passes++;
3749                 DecodeSegment(Target,
3750                               DecodeMee,
3751                               start,
3752                               end,
3753                               ConvertBuf,
3754                               ConvertBuf2,
3755                               FoundCharset);
3756
3757                 next = strstr(end, "=?");
3758                 nextend = NULL;
3759                 if ((next != NULL) &&
3760                     (next < eptr))
3761                         nextend = FindNextEnd(DecodeMee, next);
3762                 if (nextend == NULL)
3763                         next = NULL;
3764
3765                 /* did we find two partitions */
3766                 if ((next != NULL) &&
3767                     ((next - end) > 2))
3768                 {
3769                         ptr = end + 2;
3770                         while ((ptr < next) &&
3771                                (isspace(*ptr) ||
3772                                 (*ptr == '\r') ||
3773                                 (*ptr == '\n') ||
3774                                 (*ptr == '\t')))
3775                                 ptr ++;
3776                         /*
3777                          * did we find a gab just filled with blanks?
3778                          * if not, copy its stuff over.
3779                          */
3780                         if (ptr != next)
3781                         {
3782                                 StrBufAppendBufPlain(Target,
3783                                                      end + 2,
3784                                                      next - end - 2,
3785                                                      0);
3786                         }
3787                 }
3788                 /* our next-pair is our new first pair now. */
3789                 ptr = end + 2;
3790                 start = next;
3791                 end = nextend;
3792         }
3793         end = ptr;
3794         nextend = DecodeMee->buf + DecodeMee->BufUsed;
3795         if ((end != NULL) && (end < nextend)) {
3796                 ptr = end;
3797                 while ( (ptr < nextend) &&
3798                         (isspace(*ptr) ||
3799                          (*ptr == '\r') ||
3800                          (*ptr == '\n') ||
3801                          (*ptr == '\t')))
3802                         ptr ++;
3803                 if (ptr < nextend)
3804                         StrBufAppendBufPlain(Target, end, nextend - end, 0);
3805         }
3806         FreeStrBuf(&DecodedInvalidBuf);
3807 }
3808
3809 /*******************************************************************************
3810  *                   Manipulating UTF-8 Strings                                *
3811  *******************************************************************************/
3812
3813 /**
3814  *  evaluate the length of an utf8 special character sequence
3815  *  Char the character to examine
3816  * @returns width of utf8 chars in bytes; if the sequence is broken 0 is returned; 1 if its simply ASCII.
3817  */
3818 static inline int Ctdl_GetUtf8SequenceLength(const char *CharS, const char *CharE)
3819 {
3820         int n = 0;
3821         unsigned char test = (1<<7);
3822
3823         if ((*CharS & 0xC0) != 0xC0)
3824                 return 1;
3825
3826         while ((n < 8) &&
3827                ((test & ((unsigned char)*CharS)) != 0))
3828         {
3829                 test = test >> 1;
3830                 n ++;
3831         }
3832         if ((n > 6) || ((CharE - CharS) < n))
3833                 n = 0;
3834         return n;
3835 }
3836
3837 /**
3838  *  detect whether this char starts an utf-8 encoded char
3839  *  Char character to inspect
3840  * @returns yes or no
3841  */
3842 static inline int Ctdl_IsUtf8SequenceStart(const char Char)
3843 {
3844 /** 11??.???? indicates an UTF8 Sequence. */
3845         return ((Char & 0xC0) == 0xC0);
3846 }
3847
3848 /**
3849  *  measure the number of glyphs in an UTF8 string...
3850  *  Buf string to measure
3851  * @returns the number of glyphs in Buf
3852  */
3853 long StrBuf_Utf8StrLen(StrBuf *Buf)
3854 {
3855         int n = 0;
3856         int m = 0;
3857         char *aptr, *eptr;
3858
3859         if ((Buf == NULL) || (Buf->BufUsed == 0))
3860                 return 0;
3861         aptr = Buf->buf;
3862         eptr = Buf->buf + Buf->BufUsed;
3863         while ((aptr < eptr) && (*aptr != '\0')) {
3864                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
3865                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
3866                         while ((aptr < eptr) && (*aptr++ != '\0')&& (m-- > 0) );
3867                         n ++;
3868                 }
3869                 else {
3870                         n++;
3871                         aptr++;
3872                 }
3873         }
3874         return n;
3875 }
3876
3877 /**
3878  *  cuts a string after maxlen glyphs
3879  *  Buf string to cut to maxlen glyphs
3880  *  maxlen how long may the string become?
3881  * @returns current length of the string
3882  */
3883 long StrBuf_Utf8StrCut(StrBuf *Buf, int maxlen)
3884 {
3885         char *aptr, *eptr;
3886         int n = 0, m = 0;
3887
3888         aptr = Buf->buf;
3889         eptr = Buf->buf + Buf->BufUsed;
3890         while ((aptr < eptr) && (*aptr != '\0')) {
3891                 if (Ctdl_IsUtf8SequenceStart(*aptr)){
3892                         m = Ctdl_GetUtf8SequenceLength(aptr, eptr);
3893                         while ((*aptr++ != '\0') && (m-- > 0));
3894                         n ++;
3895                 }
3896                 else {
3897                         n++;
3898                         aptr++;
3899                 }
3900                 if (n >= maxlen) {
3901                         *aptr = '\0';
3902                         Buf->BufUsed = aptr - Buf->buf;
3903                         return Buf->BufUsed;
3904                 }
3905         }
3906         return Buf->BufUsed;
3907
3908 }
3909
3910
3911
3912
3913
3914 /*******************************************************************************
3915  *                               wrapping ZLib                                 *
3916  *******************************************************************************/
3917
3918 /**
3919  *  uses the same calling syntax as compress2(), but it
3920  *   creates a stream compatible with HTTP "Content-encoding: gzip"
3921  *  dest compressed buffer
3922  *  destLen length of the compresed data
3923  *  source source to encode
3924  *  sourceLen length of source to encode
3925  *  level compression level
3926  */
3927 #ifdef HAVE_ZLIB
3928 int ZEXPORT compress_gzip(Bytef * dest,
3929                           size_t * destLen,
3930                           const Bytef * source,
3931                           uLong sourceLen,
3932                           int level)
3933 {
3934         /* write gzip header */
3935         snprintf((char *) dest, *destLen,
3936                  "%c%c%c%c%c%c%c%c%c%c",
3937                  gz_magic[0], gz_magic[1], Z_DEFLATED,
3938                  0 /*flags */ , 0, 0, 0, 0 /*time */ , 0 /* xflags */ ,
3939                  OS_CODE);
3940
3941         /* normal deflate */
3942         z_stream stream;
3943         int err;
3944         stream.next_in = (Bytef *) source;
3945         stream.avail_in = (uInt) sourceLen;
3946         stream.next_out = dest + 10L;   // after header
3947         stream.avail_out = (uInt) * destLen;
3948         if ((uLong) stream.avail_out != *destLen)
3949                 return Z_BUF_ERROR;
3950
3951         stream.zalloc = (alloc_func) 0;
3952         stream.zfree = (free_func) 0;
3953         stream.opaque = (voidpf) 0;
3954
3955         err = deflateInit2(&stream, level, Z_DEFLATED, -MAX_WBITS,
3956                            DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY);
3957         if (err != Z_OK)
3958                 return err;
3959
3960         err = deflate(&stream, Z_FINISH);
3961         if (err != Z_STREAM_END) {
3962                 deflateEnd(&stream);
3963                 return err == Z_OK ? Z_BUF_ERROR : err;
3964         }
3965         *destLen = stream.total_out + 10L;
3966
3967         /* write CRC and Length */
3968         uLong crc = crc32(0L, source, sourceLen);
3969         int n;
3970         for (n = 0; n < 4; ++n, ++*destLen) {
3971                 dest[*destLen] = (int) (crc & 0xff);
3972                 crc >>= 8;
3973         }
3974         uLong len = stream.total_in;
3975         for (n = 0; n < 4; ++n, ++*destLen) {
3976                 dest[*destLen] = (int) (len & 0xff);
3977                 len >>= 8;
3978         }
3979         err = deflateEnd(&stream);
3980         return err;
3981 }
3982 #endif
3983
3984
3985 /**
3986  *  compress the buffer with gzip
3987  * Attention! If you feed this a Const String, you must maintain the uncompressed buffer yourself!
3988  *  Buf buffer whose content is to be gzipped
3989  */
3990 int CompressBuffer(StrBuf *Buf)
3991 {
3992 #ifdef HAVE_ZLIB
3993         char *compressed_data = NULL;
3994         size_t compressed_len, bufsize;
3995         int i = 0;
3996
3997         bufsize = compressed_len = Buf->BufUsed +  (Buf->BufUsed / 100) + 100;
3998         compressed_data = malloc(compressed_len);
3999
4000         if (compressed_data == NULL)
4001                 return -1;
4002         /* Flush some space after the used payload so valgrind shuts up... */
4003         while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
4004                 Buf->buf[Buf->BufUsed + i++] = '\0';
4005         if (compress_gzip((Bytef *) compressed_data,
4006                           &compressed_len,
4007                           (Bytef *) Buf->buf,
4008                           (uLongf) Buf->BufUsed, Z_BEST_SPEED) == Z_OK) {
4009                 if (!Buf->ConstBuf)
4010                         free(Buf->buf);
4011                 Buf->buf = compressed_data;
4012                 Buf->BufUsed = compressed_len;
4013                 Buf->BufSize = bufsize;
4014                 /* Flush some space after the used payload so valgrind shuts up... */
4015                 i = 0;
4016                 while ((i < 10) && (Buf->BufUsed + i < Buf->BufSize))
4017                         Buf->buf[Buf->BufUsed + i++] = '\0';
4018                 return 1;
4019         } else {
4020                 free(compressed_data);
4021         }
4022 #endif  /* HAVE_ZLIB */
4023         return 0;
4024 }
4025
4026 /*******************************************************************************
4027  *           File I/O; Callbacks to libevent                                   *
4028  *******************************************************************************/
4029
4030 long StrBuf_read_one_chunk_callback (int fd, short event, IOBuffer *FB)
4031 {
4032         long bufremain = 0;
4033         int n;
4034
4035         if ((FB == NULL) || (FB->Buf == NULL))
4036                 return -1;
4037
4038         /*
4039          * check whether the read pointer is somewhere in a range
4040          * where a cut left is inexpensive
4041          */
4042
4043         if (FB->ReadWritePointer != NULL)
4044         {
4045                 long already_read;
4046
4047                 already_read = FB->ReadWritePointer - FB->Buf->buf;
4048                 bufremain = FB->Buf->BufSize - FB->Buf->BufUsed - 1;
4049
4050                 if (already_read != 0) {
4051                         long unread;
4052
4053                         unread = FB->Buf->BufUsed - already_read;
4054
4055                         /* else nothing to compact... */
4056                         if (unread == 0) {
4057                                 FB->ReadWritePointer = FB->Buf->buf;
4058                                 bufremain = FB->Buf->BufSize;
4059                         }
4060                         else if ((unread < 64) ||
4061                                  (bufremain < already_read))
4062                         {
4063                                 /*
4064                                  * if its just a tiny bit remaining, or we run out of space...
4065                                  * lets tidy up.
4066                                  */
4067                                 FB->Buf->BufUsed = unread;
4068                                 if (unread < already_read)
4069                                         memcpy(FB->Buf->buf, FB->ReadWritePointer, unread);
4070                                 else
4071                                         memmove(FB->Buf->buf, FB->ReadWritePointer, unread);
4072                                 FB->ReadWritePointer = FB->Buf->buf;
4073                                 bufremain = FB->Buf->BufSize - unread - 1;
4074                         }
4075                         else if (bufremain < (FB->Buf->BufSize / 10))
4076                         {
4077                                 /* get a bigger buffer */
4078
4079                                 IncreaseBuf(FB->Buf, 0, FB->Buf->BufUsed + 1);
4080
4081                                 FB->ReadWritePointer = FB->Buf->buf + unread;
4082
4083                                 bufremain = FB->Buf->BufSize - unread - 1;
4084 /*TODO: special increase function that won't copy the already read! */
4085                         }
4086                 }
4087                 else if (bufremain < 10) {
4088                         IncreaseBuf(FB->Buf, 1, FB->Buf->BufUsed + 10);
4089
4090                         FB->ReadWritePointer = FB->Buf->buf;
4091
4092                         bufremain = FB->Buf->BufSize - FB->Buf->BufUsed - 1;
4093                 }
4094
4095         }
4096         else {
4097                 FB->ReadWritePointer = FB->Buf->buf;
4098                 bufremain = FB->Buf->BufSize - 1;
4099         }
4100
4101         n = read(fd, FB->Buf->buf + FB->Buf->BufUsed, bufremain);
4102
4103         if (n > 0) {
4104                 FB->Buf->BufUsed += n;
4105                 FB->Buf->buf[FB->Buf->BufUsed] = '\0';
4106         }
4107         return n;
4108 }
4109
4110 int StrBuf_write_one_chunk_callback(int fd, short event, IOBuffer *FB)
4111 {
4112         long WriteRemain;
4113         int n;
4114
4115         if ((FB == NULL) || (FB->Buf == NULL))
4116                 return -1;
4117
4118         if (FB->ReadWritePointer != NULL)
4119         {
4120                 WriteRemain = FB->Buf->BufUsed -
4121                         (FB->ReadWritePointer -
4122                          FB->Buf->buf);
4123         }
4124         else {
4125                 FB->ReadWritePointer = FB->Buf->buf;
4126                 WriteRemain = FB->Buf->BufUsed;
4127         }
4128
4129         n = write(fd, FB->ReadWritePointer, WriteRemain);
4130         if (n > 0) {
4131                 FB->ReadWritePointer += n;
4132
4133                 if (FB->ReadWritePointer ==
4134                     FB->Buf->buf + FB->Buf->BufUsed)
4135                 {
4136                         FlushStrBuf(FB->Buf);
4137                         FB->ReadWritePointer = NULL;
4138                         return 0;
4139                 }
4140         // check whether we've got something to write
4141         // get the maximum chunk plus the pointer we can send
4142         // write whats there
4143         // if not all was sent, remember the send pointer for the next time
4144                 return FB->ReadWritePointer - FB->Buf->buf + FB->Buf->BufUsed;
4145         }
4146         return n;
4147 }
4148
4149 /**
4150  *  extract a "next line" from Buf; Ptr to persist across several iterations
4151  *  LineBuf your line will be copied here.
4152  *  FB BLOB with lines of text...
4153  *  Ptr moved arround to keep the next-line across several iterations
4154  *        has to be &NULL on start; will be &NotNULL on end of buffer
4155  * @returns size of copied buffer
4156  */
4157 eReadState StrBufChunkSipLine(StrBuf *LineBuf, IOBuffer *FB)
4158 {
4159         const char *aptr, *ptr, *eptr;
4160         char *optr, *xptr;
4161
4162         if ((FB == NULL) || (LineBuf == NULL) || (LineBuf->buf == NULL))
4163                 return eReadFail;
4164
4165
4166         if ((FB->Buf == NULL) || (FB->ReadWritePointer == StrBufNOTNULL)) {
4167                 FB->ReadWritePointer = StrBufNOTNULL;
4168                 return eReadFail;
4169         }
4170
4171         FlushStrBuf(LineBuf);
4172         if (FB->ReadWritePointer == NULL)
4173                 ptr = aptr = FB->Buf->buf;
4174         else
4175                 ptr = aptr = FB->ReadWritePointer;
4176
4177         optr = LineBuf->buf;
4178         eptr = FB->Buf->buf + FB->Buf->BufUsed;
4179         xptr = LineBuf->buf + LineBuf->BufSize - 1;
4180
4181         while ((ptr <= eptr) &&
4182                (*ptr != '\n') &&
4183                (*ptr != '\r') )
4184         {
4185                 *optr = *ptr;
4186                 optr++; ptr++;
4187                 if (optr == xptr) {
4188                         LineBuf->BufUsed = optr - LineBuf->buf;
4189                         IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
4190                         optr = LineBuf->buf + LineBuf->BufUsed;
4191                         xptr = LineBuf->buf + LineBuf->BufSize - 1;
4192                 }
4193         }
4194
4195         if (ptr >= eptr) {
4196                 if (optr > LineBuf->buf)
4197                         optr --;
4198                 if ((*(ptr - 1) != '\r') && (*(ptr - 1) != '\n')) {
4199                         LineBuf->BufUsed = optr - LineBuf->buf;
4200                         *optr = '\0';
4201                         if ((FB->ReadWritePointer != NULL) &&
4202                             (FB->ReadWritePointer != FB->Buf->buf))
4203                         {
4204                                 /* Ok, the client application read all the data
4205                                    it was interested in so far. Since there is more to read,
4206                                    we now shrink the buffer, and move the rest over.
4207                                 */
4208                                 StrBufCutLeft(FB->Buf,
4209                                               FB->ReadWritePointer - FB->Buf->buf);
4210                                 FB->ReadWritePointer = FB->Buf->buf;
4211                         }
4212                         return eMustReadMore;
4213                 }
4214         }
4215         LineBuf->BufUsed = optr - LineBuf->buf;
4216         *optr = '\0';
4217         if ((ptr <= eptr) && (*ptr == '\r'))
4218                 ptr ++;
4219         if ((ptr <= eptr) && (*ptr == '\n'))
4220                 ptr ++;
4221
4222         if (ptr < eptr) {
4223                 FB->ReadWritePointer = ptr;
4224         }
4225         else {
4226                 FlushStrBuf(FB->Buf);
4227                 FB->ReadWritePointer = NULL;
4228         }
4229
4230         return eReadSuccess;
4231 }
4232
4233 /**
4234  *  check whether the chunk-buffer has more data waiting or not.
4235  *  FB Chunk-Buffer to inspect
4236  */
4237 eReadState StrBufCheckBuffer(IOBuffer *FB)
4238 {
4239         if (FB == NULL)
4240                 return eReadFail;
4241         if (FB->Buf->BufUsed == 0)
4242                 return eReadSuccess;
4243         if (FB->ReadWritePointer == NULL)
4244                 return eBufferNotEmpty;
4245         if (FB->Buf->buf + FB->Buf->BufUsed > FB->ReadWritePointer)
4246                 return eBufferNotEmpty;
4247         return eReadSuccess;
4248 }
4249
4250 long IOBufferStrLength(IOBuffer *FB)
4251 {
4252         if ((FB == NULL) || (FB->Buf == NULL))
4253                 return 0;
4254         if (FB->ReadWritePointer == NULL)
4255                 return StrLength(FB->Buf);
4256
4257         return StrLength(FB->Buf) - (FB->ReadWritePointer - FB->Buf->buf);
4258 }
4259
4260
4261 /*******************************************************************************
4262  *           File I/O; Prefer buffered read since its faster!                  *
4263  *******************************************************************************/
4264
4265 /**
4266  *  Read a line from socket
4267  * flushes and closes the FD on error
4268  *  buf the buffer to get the input to
4269  *  fd pointer to the filedescriptor to read
4270  *  append Append to an existing string or replace?
4271  *  Error strerror() on error
4272  * @returns numbers of chars read
4273  */
4274 int StrBufTCP_read_line(StrBuf *buf, int *fd, int append, const char **Error)
4275 {
4276         int len, rlen, slen;
4277
4278         if ((buf == NULL) || (buf->buf == NULL)) {
4279                 *Error = strerror(EINVAL);
4280                 return -1;
4281         }
4282
4283         if (!append)
4284                 FlushStrBuf(buf);
4285
4286         slen = len = buf->BufUsed;
4287         while (1) {
4288                 rlen = read(*fd, &buf->buf[len], 1);
4289                 if (rlen < 1) {
4290                         *Error = strerror(errno);
4291
4292                         close(*fd);
4293                         *fd = -1;
4294
4295                         return -1;
4296                 }
4297                 if (buf->buf[len] == '\n')
4298                         break;
4299                 if (buf->buf[len] != '\r')
4300                         len ++;
4301                 if (len + 2 >= buf->BufSize) {
4302                         buf->BufUsed = len;
4303                         buf->buf[len+1] = '\0';
4304                         IncreaseBuf(buf, 1, -1);
4305                 }
4306         }
4307         buf->BufUsed = len;
4308         buf->buf[len] = '\0';
4309         return len - slen;
4310 }
4311
4312
4313 /**
4314  *  Read a line from socket
4315  * flushes and closes the FD on error
4316  *  Line the line to read from the fd / I/O Buffer
4317  *  buf the buffer to get the input to
4318  *  fd pointer to the filedescriptor to read
4319  *  timeout number of successless selects until we bail out
4320  *  selectresolution how long to wait on each select
4321  *  Error strerror() on error
4322  * @returns numbers of chars read
4323  */
4324 int StrBufTCP_read_buffered_line(StrBuf *Line,
4325                                  StrBuf *buf,
4326                                  int *fd,
4327                                  int timeout,
4328                                  int selectresolution,
4329                                  const char **Error)
4330 {
4331         int len, rlen;
4332         int nSuccessLess = 0;
4333         fd_set rfds;
4334         char *pch = NULL;
4335         int fdflags;
4336         int IsNonBlock;
4337         struct timeval tv;
4338
4339         if (buf->BufUsed > 0) {
4340                 pch = strchr(buf->buf, '\n');
4341                 if (pch != NULL) {
4342                         rlen = 0;
4343                         len = pch - buf->buf;
4344                         if (len > 0 && (*(pch - 1) == '\r') )
4345                                 rlen ++;
4346                         StrBufSub(Line, buf, 0, len - rlen);
4347                         StrBufCutLeft(buf, len + 1);
4348                         return len - rlen;
4349                 }
4350         }
4351
4352         if (buf->BufSize - buf->BufUsed < 10)
4353                 IncreaseBuf(buf, 1, -1);
4354
4355         fdflags = fcntl(*fd, F_GETFL);
4356         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
4357
4358         while ((nSuccessLess < timeout) && (pch == NULL)) {
4359                 if (IsNonBlock){
4360                         tv.tv_sec = selectresolution;
4361                         tv.tv_usec = 0;
4362
4363                         FD_ZERO(&rfds);
4364                         FD_SET(*fd, &rfds);
4365                         if (select(*fd + 1, NULL, &rfds, NULL, &tv) == -1) {
4366                                 *Error = strerror(errno);
4367                                 close (*fd);
4368                                 *fd = -1;
4369                                 return -1;
4370                         }
4371                 }
4372                 if (IsNonBlock && !  FD_ISSET(*fd, &rfds)) {
4373                         nSuccessLess ++;
4374                         continue;
4375                 }
4376                 rlen = read(*fd,
4377                             &buf->buf[buf->BufUsed],
4378                             buf->BufSize - buf->BufUsed - 1);
4379                 if (rlen < 1) {
4380                         *Error = strerror(errno);
4381                         close(*fd);
4382                         *fd = -1;
4383                         return -1;
4384                 }
4385                 else if (rlen > 0) {
4386                         nSuccessLess = 0;
4387                         buf->BufUsed += rlen;
4388                         buf->buf[buf->BufUsed] = '\0';
4389                         pch = strchr(buf->buf, '\n');
4390                         if ((pch == NULL) &&
4391                             (buf->BufUsed + 10 > buf->BufSize) &&
4392                             (IncreaseBuf(buf, 1, -1) == -1))
4393                                 return -1;
4394                         continue;
4395                 }
4396
4397         }
4398         if (pch != NULL) {
4399                 rlen = 0;
4400                 len = pch - buf->buf;
4401                 if (len > 0 && (*(pch - 1) == '\r') )
4402                         rlen ++;
4403                 StrBufSub(Line, buf, 0, len - rlen);
4404                 StrBufCutLeft(buf, len + 1);
4405                 return len - rlen;
4406         }
4407         return -1;
4408
4409 }
4410
4411 static const char *ErrRBLF_PreConditionFailed="StrBufTCP_read_buffered_line_fast: Wrong arguments or invalid Filedescriptor";
4412 static const char *ErrRBLF_SelectFailed="StrBufTCP_read_buffered_line_fast: Select failed without reason";
4413 static const char *ErrRBLF_NotEnoughSentFromServer="StrBufTCP_read_buffered_line_fast: No complete line was sent from peer";
4414 /**
4415  *  Read a line from socket
4416  * flushes and closes the FD on error
4417  *  Line where to append our Line read from the fd / I/O Buffer;
4418  *  IOBuf the buffer to get the input to; lifetime pair to FD
4419  *  Pos pointer to the current read position, should be NULL initialized on opening the FD it belongs to.!
4420  *  fd pointer to the filedescriptor to read
4421  *  timeout number of successless selects until we bail out
4422  *  selectresolution how long to wait on each select
4423  *  Error strerror() on error
4424  * @returns numbers of chars read or -1 in case of error. "\n" will become 0
4425  */
4426 int StrBufTCP_read_buffered_line_fast(StrBuf *Line,
4427                                       StrBuf *IOBuf,
4428                                       const char **Pos,
4429                                       int *fd,
4430                                       int timeout,
4431                                       int selectresolution,
4432                                       const char **Error)
4433 {
4434         const char *pche = NULL;
4435         const char *pos = NULL;
4436         const char *pLF;
4437         int len, rlen, retlen;
4438         int nSuccessLess = 0;
4439         fd_set rfds;
4440         const char *pch = NULL;
4441         int fdflags;
4442         int IsNonBlock;
4443         struct timeval tv;
4444
4445         retlen = 0;
4446         if ((Line == NULL) ||
4447             (Pos == NULL) ||
4448             (IOBuf == NULL) ||
4449             (*fd == -1))
4450         {
4451                 if (Pos != NULL)
4452                         *Pos = NULL;
4453                 *Error = ErrRBLF_PreConditionFailed;
4454                 return -1;
4455         }
4456
4457         pos = *Pos;
4458         if ((IOBuf->BufUsed > 0) &&
4459             (pos != NULL) &&
4460             (pos < IOBuf->buf + IOBuf->BufUsed))
4461         {
4462                 char *pcht;
4463
4464                 pche = IOBuf->buf + IOBuf->BufUsed;
4465                 pch = pos;
4466                 pcht = Line->buf;
4467
4468                 while ((pch < pche) && (*pch != '\n'))
4469                 {
4470                         if (Line->BufUsed + 10 > Line->BufSize)
4471                         {
4472                                 long apos;
4473                                 apos = pcht - Line->buf;
4474                                 *pcht = '\0';
4475                                 IncreaseBuf(Line, 1, -1);
4476                                 pcht = Line->buf + apos;
4477                         }
4478                         *pcht++ = *pch++;
4479                         Line->BufUsed++;
4480                         retlen++;
4481                 }
4482
4483                 len = pch - pos;
4484                 if (len > 0 && (*(pch - 1) == '\r') )
4485                 {
4486                         retlen--;
4487                         len --;
4488                         pcht --;
4489                         Line->BufUsed --;
4490                 }
4491                 *pcht = '\0';
4492
4493                 if ((pch >= pche) || (*pch == '\0'))
4494                 {
4495                         FlushStrBuf(IOBuf);
4496                         *Pos = NULL;
4497                         pch = NULL;
4498                         pos = 0;
4499                 }
4500
4501                 if ((pch != NULL) &&
4502                     (pch <= pche))
4503                 {
4504                         if (pch + 1 >= pche) {
4505                                 *Pos = NULL;
4506                                 FlushStrBuf(IOBuf);
4507                         }
4508                         else
4509                                 *Pos = pch + 1;
4510
4511                         return retlen;
4512                 }
4513                 else
4514                         FlushStrBuf(IOBuf);
4515         }
4516
4517         /* If we come here, Pos is Unset since we read everything into Line, and now go for more. */
4518
4519         if (IOBuf->BufSize - IOBuf->BufUsed < 10)
4520                 IncreaseBuf(IOBuf, 1, -1);
4521
4522         fdflags = fcntl(*fd, F_GETFL);
4523         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
4524
4525         pLF = NULL;
4526         while ((nSuccessLess < timeout) &&
4527                (pLF == NULL) &&
4528                (*fd != -1)) {
4529                 if (IsNonBlock)
4530                 {
4531                         tv.tv_sec = 1;
4532                         tv.tv_usec = 0;
4533
4534                         FD_ZERO(&rfds);
4535                         FD_SET(*fd, &rfds);
4536                         if (select((*fd) + 1, &rfds, NULL, NULL, &tv) == -1) {
4537                                 *Error = strerror(errno);
4538                                 close (*fd);
4539                                 *fd = -1;
4540                                 if (*Error == NULL)
4541                                         *Error = ErrRBLF_SelectFailed;
4542                                 return -1;
4543                         }
4544                         if (! FD_ISSET(*fd, &rfds) != 0) {
4545                                 nSuccessLess ++;
4546                                 continue;
4547                         }
4548                 }
4549                 rlen = read(*fd,
4550                             &IOBuf->buf[IOBuf->BufUsed],
4551                             IOBuf->BufSize - IOBuf->BufUsed - 1);
4552                 if (rlen < 1) {
4553                         *Error = strerror(errno);
4554                         close(*fd);
4555                         *fd = -1;
4556                         return -1;
4557                 }
4558                 else if (rlen > 0) {
4559                         nSuccessLess = 0;
4560                         pLF = IOBuf->buf + IOBuf->BufUsed;
4561                         IOBuf->BufUsed += rlen;
4562                         IOBuf->buf[IOBuf->BufUsed] = '\0';
4563
4564                         pche = IOBuf->buf + IOBuf->BufUsed;
4565
4566                         while ((pLF < pche) && (*pLF != '\n'))
4567                                 pLF ++;
4568                         if ((pLF >= pche) || (*pLF == '\0'))
4569                                 pLF = NULL;
4570
4571                         if (IOBuf->BufUsed + 10 > IOBuf->BufSize)
4572                         {
4573                                 long apos = 0;
4574
4575                                 if (pLF != NULL) apos = pLF - IOBuf->buf;
4576                                 IncreaseBuf(IOBuf, 1, -1);
4577                                 if (pLF != NULL) pLF = IOBuf->buf + apos;
4578                         }
4579
4580                         continue;
4581                 }
4582                 else
4583                 {
4584                         nSuccessLess++;
4585                 }
4586         }
4587         *Pos = NULL;
4588         if (pLF != NULL) {
4589                 pos = IOBuf->buf;
4590                 len = pLF - pos;
4591                 if (len > 0 && (*(pLF - 1) == '\r') )
4592                         len --;
4593                 StrBufAppendBufPlain(Line, ChrPtr(IOBuf), len, 0);
4594                 if (pLF + 1 >= IOBuf->buf + IOBuf->BufUsed)
4595                 {
4596                         FlushStrBuf(IOBuf);
4597                 }
4598                 else
4599                         *Pos = pLF + 1;
4600                 return retlen + len;
4601         }
4602         *Error = ErrRBLF_NotEnoughSentFromServer;
4603         return -1;
4604
4605 }
4606
4607 static const char *ErrRBLF_BLOBPreConditionFailed="StrBufReadBLOB: Wrong arguments or invalid Filedescriptor";
4608 /**
4609  *  Input binary data from socket
4610  * flushes and closes the FD on error
4611  *  Buf the buffer to get the input to
4612  *  fd pointer to the filedescriptor to read
4613  *  append Append to an existing string or replace?
4614  *  nBytes the maximal number of bytes to read
4615  *  Error strerror() on error
4616  * @returns numbers of chars read
4617  */
4618 int StrBufReadBLOB(StrBuf *Buf, int *fd, int append, long nBytes, const char **Error)
4619 {
4620         int fdflags;
4621         int rlen;
4622         int nSuccessLess;
4623         int nRead = 0;
4624         char *ptr;
4625         int IsNonBlock;
4626         struct timeval tv;
4627         fd_set rfds;
4628
4629         if ((Buf == NULL) || (Buf->buf == NULL) || (*fd == -1))
4630         {
4631                 *Error = ErrRBLF_BLOBPreConditionFailed;
4632                 return -1;
4633         }
4634         if (!append)
4635                 FlushStrBuf(Buf);
4636         if (Buf->BufUsed + nBytes >= Buf->BufSize)
4637                 IncreaseBuf(Buf, 1, Buf->BufUsed + nBytes);
4638
4639         ptr = Buf->buf + Buf->BufUsed;
4640
4641         fdflags = fcntl(*fd, F_GETFL);
4642         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
4643         nSuccessLess = 0;
4644         while ((nRead < nBytes) &&
4645                (*fd != -1))
4646         {
4647                 if (IsNonBlock)
4648                 {
4649                         tv.tv_sec = 1;
4650                         tv.tv_usec = 0;
4651
4652                         FD_ZERO(&rfds);
4653                         FD_SET(*fd, &rfds);
4654                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
4655                                 *Error = strerror(errno);
4656                                 close (*fd);
4657                                 *fd = -1;
4658                                 if (*Error == NULL)
4659                                         *Error = ErrRBLF_SelectFailed;
4660                                 return -1;
4661                         }
4662                         if (! FD_ISSET(*fd, &rfds) != 0) {
4663                                 nSuccessLess ++;
4664                                 continue;
4665                         }
4666                 }
4667
4668                 if ((rlen = read(*fd,
4669                                  ptr,
4670                                  nBytes - nRead)) == -1) {
4671                         close(*fd);
4672                         *fd = -1;
4673                         *Error = strerror(errno);
4674                         return rlen;
4675                 }
4676                 nRead += rlen;
4677                 ptr += rlen;
4678                 Buf->BufUsed += rlen;
4679         }
4680         Buf->buf[Buf->BufUsed] = '\0';
4681         return nRead;
4682 }
4683
4684 const char *ErrRBB_BLOBFPreConditionFailed = "StrBufReadBLOBBuffered: to many selects; aborting.";
4685 const char *ErrRBB_too_many_selects        = "StrBufReadBLOBBuffered: to many selects; aborting.";
4686 /**
4687  *  Input binary data from socket
4688  * flushes and closes the FD on error
4689  *  Blob put binary thing here
4690  *  IOBuf the buffer to get the input to
4691  *  Pos offset inside of IOBuf
4692  *  fd pointer to the filedescriptor to read
4693  *  append Append to an existing string or replace?
4694  *  nBytes the maximal number of bytes to read
4695  *  check whether we should search for '000\n' terminators in case of timeouts
4696  *  Error strerror() on error
4697  * @returns numbers of chars read
4698  */
4699 int StrBufReadBLOBBuffered(StrBuf *Blob,
4700                            StrBuf *IOBuf,
4701                            const char **Pos,
4702                            int *fd,
4703                            int append,
4704                            long nBytes,
4705                            int check,
4706                            const char **Error)
4707 {
4708         const char *pos;
4709         int fdflags;
4710         int rlen = 0;
4711         int nRead = 0;
4712         int nAlreadyRead = 0;
4713         int IsNonBlock;
4714         char *ptr;
4715         fd_set rfds;
4716         struct timeval tv;
4717         int nSuccessLess = 0;
4718         int MaxTries;
4719
4720         if ((Blob == NULL)  ||
4721             (*fd == -1)     ||
4722             (IOBuf == NULL) ||
4723             (Pos == NULL))
4724         {
4725                 if (Pos != NULL)
4726                         *Pos = NULL;
4727                 *Error = ErrRBB_BLOBFPreConditionFailed;
4728                 return -1;
4729         }
4730
4731         if (!append)
4732                 FlushStrBuf(Blob);
4733         if (Blob->BufUsed + nBytes >= Blob->BufSize)
4734                 IncreaseBuf(Blob, append, Blob->BufUsed + nBytes);
4735
4736         pos = *Pos;
4737
4738         if (pos != NULL) {
4739                 rlen = pos - IOBuf->buf;
4740         }
4741         rlen = IOBuf->BufUsed - rlen;
4742
4743
4744         if ((IOBuf->BufUsed > 0) && (pos != NULL) && (pos < IOBuf->buf + IOBuf->BufUsed))
4745         {
4746                 if (rlen < nBytes) {
4747                         memcpy(Blob->buf + Blob->BufUsed, pos, rlen);
4748                         Blob->BufUsed += rlen;
4749                         Blob->buf[Blob->BufUsed] = '\0';
4750                         nAlreadyRead = nRead = rlen;
4751                         *Pos = NULL;
4752                 }
4753                 if (rlen >= nBytes) {
4754                         memcpy(Blob->buf + Blob->BufUsed, pos, nBytes);
4755                         Blob->BufUsed += nBytes;
4756                         Blob->buf[Blob->BufUsed] = '\0';
4757                         if (rlen == nBytes) {
4758                                 *Pos = NULL;
4759                                 FlushStrBuf(IOBuf);
4760                         }
4761                         else
4762                                 *Pos += nBytes;
4763                         return nBytes;
4764                 }
4765         }
4766
4767         FlushStrBuf(IOBuf);
4768         *Pos = NULL;
4769         if (IOBuf->BufSize < nBytes - nRead) {
4770                 IncreaseBuf(IOBuf, 0, nBytes - nRead);
4771         }
4772         ptr = IOBuf->buf;
4773
4774         fdflags = fcntl(*fd, F_GETFL);
4775         IsNonBlock = (fdflags & O_NONBLOCK) == O_NONBLOCK;
4776         if (IsNonBlock)
4777                 MaxTries =   1000;
4778         else
4779                 MaxTries = 100000;
4780
4781         nBytes -= nRead;
4782         nRead = 0;
4783         while ((nSuccessLess < MaxTries) && (nRead < nBytes) && (*fd != -1)) {
4784                 if (IsNonBlock) {
4785                         tv.tv_sec = 1;
4786                         tv.tv_usec = 0;
4787
4788                         FD_ZERO(&rfds);
4789                         FD_SET(*fd, &rfds);
4790                         if (select(*fd + 1, &rfds, NULL, NULL, &tv) == -1) {
4791                                 *Error = strerror(errno);
4792                                 close (*fd);
4793                                 *fd = -1;
4794                                 if (*Error == NULL) {
4795                                         *Error = ErrRBLF_SelectFailed;
4796                                 }
4797                                 return -1;
4798                         }
4799                         if (! FD_ISSET(*fd, &rfds) != 0) {
4800                                 nSuccessLess ++;
4801                                 continue;
4802                         }
4803                 }
4804                 rlen = read(*fd, ptr, IOBuf->BufSize - (ptr - IOBuf->buf));
4805                 // if (rlen == -1) {            2021feb27 ajc changed this, apparently we will always get at least 1 byte unless the connection is broken
4806                 if (rlen < 1) {
4807                         close(*fd);
4808                         *fd = -1;
4809                         *Error = strerror(errno);
4810                         return rlen;
4811                 }
4812                 else if (rlen == 0){
4813                         if ((check == NNN_TERM) && (nRead > 5) && (strncmp(IOBuf->buf + IOBuf->BufUsed - 5, "\n000\n", 5) == 0)) {
4814                                 StrBufPlain(Blob, HKEY("\n000\n"));
4815                                 StrBufCutRight(Blob, 5);
4816                                 return Blob->BufUsed;
4817                         }
4818                         else if (!IsNonBlock)
4819                                 nSuccessLess ++;
4820                         else if (nSuccessLess > MaxTries) {
4821                                 FlushStrBuf(IOBuf);
4822                                 *Error = ErrRBB_too_many_selects;
4823                                 return -1;
4824                         }
4825                 }
4826                 else if (rlen > 0) {
4827                         nSuccessLess = 0;
4828                         nRead += rlen;
4829                         ptr += rlen;
4830                         IOBuf->BufUsed += rlen;
4831                 }
4832         }
4833         if (nSuccessLess >= MaxTries) {
4834                 FlushStrBuf(IOBuf);
4835                 *Error = ErrRBB_too_many_selects;
4836                 return -1;
4837         }
4838
4839         if (nRead > nBytes) {
4840                 *Pos = IOBuf->buf + nBytes;
4841         }
4842         Blob->buf[Blob->BufUsed] = '\0';
4843         StrBufAppendBufPlain(Blob, IOBuf->buf, nBytes, 0);
4844         if (*Pos == NULL) {
4845                 FlushStrBuf(IOBuf);
4846         }
4847         return nRead + nAlreadyRead;
4848 }
4849
4850 /**
4851  *  extract a "next line" from Buf; Ptr to persist across several iterations
4852  *  LineBuf your line will be copied here.
4853  *  Buf BLOB with lines of text...
4854  *  Ptr moved arround to keep the next-line across several iterations
4855  *        has to be &NULL on start; will be &NotNULL on end of buffer
4856  * @returns size of remaining buffer
4857  */
4858 int StrBufSipLine(StrBuf *LineBuf, const StrBuf *Buf, const char **Ptr)
4859 {
4860         const char *aptr, *ptr, *eptr;
4861         char *optr, *xptr;
4862
4863         if ((Buf == NULL) ||
4864             (*Ptr == StrBufNOTNULL) ||
4865             (LineBuf == NULL)||
4866             (LineBuf->buf == NULL))
4867         {
4868                 *Ptr = StrBufNOTNULL;
4869                 return 0;
4870         }
4871
4872         FlushStrBuf(LineBuf);
4873         if (*Ptr==NULL)
4874                 ptr = aptr = Buf->buf;
4875         else
4876                 ptr = aptr = *Ptr;
4877
4878         optr = LineBuf->buf;
4879         eptr = Buf->buf + Buf->BufUsed;
4880         xptr = LineBuf->buf + LineBuf->BufSize - 1;
4881
4882         while ((ptr <= eptr) &&
4883                (*ptr != '\n') &&
4884                (*ptr != '\r') )
4885         {
4886                 *optr = *ptr;
4887                 optr++; ptr++;
4888                 if (optr == xptr) {
4889                         LineBuf->BufUsed = optr - LineBuf->buf;
4890                         IncreaseBuf(LineBuf,  1, LineBuf->BufUsed + 1);
4891                         optr = LineBuf->buf + LineBuf->BufUsed;
4892                         xptr = LineBuf->buf + LineBuf->BufSize - 1;
4893                 }
4894         }
4895
4896         if ((ptr >= eptr) && (optr > LineBuf->buf))
4897                 optr --;
4898         LineBuf->BufUsed = optr - LineBuf->buf;
4899         *optr = '\0';
4900         if ((ptr <= eptr) && (*ptr == '\r'))
4901                 ptr ++;
4902         if ((ptr <= eptr) && (*ptr == '\n'))
4903                 ptr ++;
4904
4905         if (ptr < eptr) {
4906                 *Ptr = ptr;
4907         }
4908         else {
4909                 *Ptr = StrBufNOTNULL;
4910         }
4911
4912         return Buf->BufUsed - (ptr - Buf->buf);
4913 }
4914
4915
4916 /**
4917  *  removes double slashes from pathnames
4918  *  Dir directory string to filter
4919  *  RemoveTrailingSlash allows / disallows trailing slashes
4920  */
4921 void StrBufStripSlashes(StrBuf *Dir, int RemoveTrailingSlash)
4922 {
4923         char *a, *b;
4924
4925         a = b = Dir->buf;
4926
4927         while (!IsEmptyStr(a)) {
4928                 if (*a == '/') {
4929                         while (*a == '/')
4930                                 a++;
4931                         *b = '/';
4932                         b++;
4933                 }
4934                 else {
4935                         *b = *a;
4936                         b++; a++;
4937                 }
4938         }
4939         if ((RemoveTrailingSlash) &&
4940             (b > Dir->buf) &&
4941             (*(b - 1) == '/')){
4942                 b--;
4943         }
4944         *b = '\0';
4945         Dir->BufUsed = b - Dir->buf;
4946 }
4947
4948
4949 /*
4950  * Decode a quoted-printable encoded StrBuf buffer "in place"
4951  * This is possible because the decoded will always be shorter than the encoded
4952  * so we don't have to worry about the buffer being to small.
4953  */
4954 void StrBufDecodeQP(StrBuf *Buf)
4955 {
4956         if (!Buf) {                             // sanity check #1
4957                 return;
4958         }
4959
4960         int source_len = StrLength(Buf);
4961         if (source_len < 1) {                   // sanity check #2
4962                 return;
4963         }
4964
4965         int spos = 0;                           // source position
4966         int tpos = 0;                           // target position
4967
4968         while (spos < source_len) {
4969                 if (!strncmp(&Buf->buf[spos], "=\r\n", 3)) {
4970                         spos += 3;
4971                 }
4972                 else if (!strncmp(&Buf->buf[spos], "=\n", 2)) {
4973                         spos += 2;
4974                 }
4975                 else if (Buf->buf[spos] == '=') {
4976                         ++spos;
4977                         int ch;
4978                         sscanf(&Buf->buf[spos], "%02x", &ch);
4979                         Buf->buf[tpos++] = ch;
4980                         spos +=2;
4981                 }
4982                 else {
4983                         Buf->buf[tpos++] = Buf->buf[spos++];
4984                 }
4985         }
4986
4987         Buf->buf[tpos] = 0;
4988         Buf->BufUsed = tpos;
4989 }