libcitadel/lib/html_to_ascii.c

   1 // Functions which handle translation between HTML and plain text
   2 // Copyright (c) 2000-2023 by the citadel.org team
   3 //
   4 // This program is open source software.  Use, duplication, or disclosure
   5 // is subject to the terms of the GNU General Public License, version 3.
   6
   7 #include <stdlib.h>
   8 #include <unistd.h>
   9 #include <stdio.h>
  10 #include <signal.h>
  11 #include <sys/types.h>
  12 #include <ctype.h>
  13 #include <string.h>
  14 #include <sys/stat.h>
  15 #include <errno.h>
  16 #include <limits.h>
  17 #include <time.h>
  18 #include "libcitadel.h"
  19
  20
  21 // Convert HTML to plain text.
  22 //
  23 // inputmsg     = pointer to raw HTML message
  24 // msglen       = stop reading after this many bytes
  25 // screenwidth  = desired output screenwidth
  26 // ansi         = if nonzero, assume output is to a terminal that supports ANSI escape codes
  27 //
  28 char *html_to_ascii(const char *inputmsg, int msglen, int screenwidth, int ansi) {
  29         char inbuf[SIZ];
  30         int inbuf_len = 0;
  31         char outbuf[SIZ];
  32         char tag[1024];
  33         int done_reading = 0;
  34         const char *inptr;
  35         char *outptr;
  36         size_t outptr_buffer_size;
  37         size_t output_len = 0;
  38         int i, j, ch, did_out, rb, scanch;
  39         int nest = 0;                           // Bracket nesting level
  40         int blockquote = 0;                     // BLOCKQUOTE nesting level
  41         int styletag = 0;                       // STYLE tag nesting level
  42         int styletag_start = 0;
  43         int bytes_processed = 0;
  44         char nl[128];
  45
  46         tag[0] = '\0';
  47         strcpy(nl, "\n");
  48         inptr = inputmsg;
  49         strcpy(inbuf, "");
  50         strcpy(outbuf, "");
  51         if (msglen == 0) msglen = strlen(inputmsg);
  52
  53         outptr_buffer_size = strlen(inptr) + SIZ;
  54         outptr = malloc(outptr_buffer_size);
  55         if (outptr == NULL) return NULL;
  56         strcpy(outptr, "");
  57         output_len = 0;
  58
  59         do {
  60                 // Fill the input buffer
  61                 inbuf_len = strlen(inbuf);
  62                 if ( (done_reading == 0) && (inbuf_len < (SIZ-128)) ) {
  63
  64                         ch = *inptr++;
  65                         if (ch != 0) {
  66                                 inbuf[inbuf_len++] = ch;
  67                                 inbuf[inbuf_len] = 0;
  68                         }
  69                         else {
  70                                 done_reading = 1;
  71                         }
  72
  73                         ++bytes_processed;
  74                         if (bytes_processed > msglen) {
  75                                 done_reading = 1;
  76                         }
  77
  78                 }
  79
  80                 // Do some parsing
  81                 if (!IsEmptyStr(inbuf)) {
  82
  83                     // Fold in all the spacing
  84                     for (i=0; !IsEmptyStr(&inbuf[i]); ++i) {
  85                         if (inbuf[i]==10) inbuf[i]=32;
  86                         if (inbuf[i]==13) inbuf[i]=32;
  87                         if (inbuf[i]==9) inbuf[i]=32;
  88                     }
  89                     for (i=0; !IsEmptyStr(&inbuf[i]); ++i) {
  90                         while ((inbuf[i]==32)&&(inbuf[i+1]==32)) {
  91                                 strcpy(&inbuf[i], &inbuf[i+1]);
  92                         }
  93                     }
  94
  95                     for (i=0; !IsEmptyStr(&inbuf[i]); ++i) {
  96
  97                         ch = inbuf[i];
  98
  99                         if (ch == '<') {
 100                                 ++nest;
 101                                 strcpy(tag, "");
 102                         }
 103
 104                         else if (ch == '>') {   // We have a tag.
 105                                 if (nest > 0) --nest;
 106
 107                                 // Unqualify the tag (truncate at first space)
 108                                 if (strchr(tag, ' ') != NULL) {
 109                                         strcpy(strchr(tag, ' '), "");
 110                                 }
 111
 112                                 if (!strcasecmp(tag, "P")) {
 113                                         strcat(outbuf, nl);
 114                                         strcat(outbuf, nl);
 115                                 }
 116
 117                                 if (!strcasecmp(tag, "/DIV")) {
 118                                         strcat(outbuf, nl);
 119                                         strcat(outbuf, nl);
 120                                 }
 121
 122                                 if (!strcasecmp(tag, "LI")) {
 123                                         strcat(outbuf, nl);
 124                                         strcat(outbuf, " * ");
 125                                 }
 126
 127                                 else if (!strcasecmp(tag, "/UL")) {
 128                                         strcat(outbuf, nl);
 129                                         strcat(outbuf, nl);
 130                                 }
 131
 132                                 else if (!strcasecmp(tag, "H1")) {
 133                                         strcat(outbuf, nl);
 134                                         strcat(outbuf, nl);
 135                                 }
 136
 137                                 else if (!strcasecmp(tag, "H2")) {
 138                                         strcat(outbuf, nl);
 139                                         strcat(outbuf, nl);
 140                                 }
 141
 142                                 else if (!strcasecmp(tag, "H3")) {
 143                                         strcat(outbuf, nl);
 144                                         strcat(outbuf, nl);
 145                                 }
 146
 147                                 else if (!strcasecmp(tag, "H4")) {
 148                                         strcat(outbuf, nl);
 149                                         strcat(outbuf, nl);
 150                                 }
 151
 152                                 else if (!strcasecmp(tag, "/H1")) {
 153                                         strcat(outbuf, nl);
 154                                 }
 155
 156                                 else if (!strcasecmp(tag, "/H2")) {
 157                                         strcat(outbuf, nl);
 158                                 }
 159
 160                                 else if (!strcasecmp(tag, "/H3")) {
 161                                         strcat(outbuf, nl);
 162                                 }
 163
 164                                 else if (!strcasecmp(tag, "/H4")) {
 165                                         strcat(outbuf, nl);
 166                                 }
 167
 168                                 else if (!strcasecmp(tag, "HR")) {
 169                                         strcat(outbuf, nl);
 170                                         strcat(outbuf, " ");
 171                                         for (j=0; j<screenwidth-2; ++j)
 172                                                 strcat(outbuf, "-");
 173                                         strcat(outbuf, nl);
 174                                 }
 175
 176                                 else if (
 177                                         (!strcasecmp(tag, "B"))
 178                                         || (!strcasecmp(tag, "STRONG"))
 179                                 ) {
 180                                         if (ansi) {
 181                                                 strcat(outbuf, "\033[1m");
 182                                         }
 183                                 }
 184                                 else if (
 185                                         (!strcasecmp(tag, "/B"))
 186                                         || (!strcasecmp(tag, "/STRONG"))
 187                                 ) {
 188                                         if (ansi) {
 189                                                 strcat(outbuf, "\033[22m");
 190                                         }
 191                                 }
 192
 193                                 else if (
 194                                         (!strcasecmp(tag, "I"))
 195                                         || (!strcasecmp(tag, "EM"))
 196                                 ) {
 197                                         if (ansi) {
 198                                                 strcat(outbuf, "\033[3m");
 199                                         }
 200                                 }
 201
 202                                 else if (
 203                                         (!strcasecmp(tag, "/I"))
 204                                         || (!strcasecmp(tag, "/EM"))
 205                                 ) {
 206                                         if (ansi) {
 207                                                 strcat(outbuf, "\033[23m");
 208                                         }
 209                                 }
 210
 211                                 else if (!strcasecmp(tag, "U")) {
 212                                         if (ansi) {
 213                                                 strcat(outbuf, "\033[4m");
 214                                         }
 215                                 }
 216
 217                                 else if (!strcasecmp(tag, "/U")) {
 218                                         if (ansi) {
 219                                                 strcat(outbuf, "\033[24m");
 220                                         }
 221                                 }
 222
 223                                 else if (!strcasecmp(tag, "BR")) {
 224                                         strcat(outbuf, nl);
 225                                 }
 226
 227                                 else if (!strcasecmp(tag, "TR")) {
 228                                         strcat(outbuf, nl);
 229                                 }
 230
 231                                 else if (!strcasecmp(tag, "/TABLE")) {
 232                                         strcat(outbuf, nl);
 233                                 }
 234
 235                                 else if (!strcasecmp(tag, "BLOCKQUOTE")) {
 236                                         ++blockquote;
 237                                         strcpy(nl, "\n");
 238                                         if ( (blockquote == 1) && (ansi) ) {
 239                                                 strcat(nl, "\033[2m\033[2m");
 240                                         }
 241                                         for (j=0; j<blockquote; ++j) strcat(nl, ">");
 242                                         strcat(outbuf, nl);
 243                                 }
 244
 245                                 else if (!strcasecmp(tag, "/BLOCKQUOTE")) {
 246                                         strcat(outbuf, "\n");
 247                                         --blockquote;
 248                                         if ( (blockquote == 0) && (ansi) ) {
 249                                                 strcat(outbuf, "\033[22m\033[22m");
 250                                         }
 251                                         strcpy(nl, "\n");
 252                                         for (j=0; j<blockquote; ++j) strcat(nl, ">");
 253                                         strcat(outbuf, nl);
 254                                 }
 255
 256                                 else if (!strcasecmp(tag, "STYLE")) {
 257                                         ++styletag;
 258                                         if (styletag == 1) {
 259                                                 styletag_start = strlen(outbuf);
 260                                         }
 261                                 }
 262
 263                                 else if (!strcasecmp(tag, "/STYLE")) {
 264                                         --styletag;
 265                                         if (styletag == 0) {
 266                                                 outbuf[styletag_start] = 0;
 267                                         }
 268                                 }
 269
 270                         }
 271
 272                         else if ((nest > 0) && (strlen(tag)<(sizeof(tag)-1))) {
 273                                 tag[strlen(tag)+1] = 0;
 274                                 tag[strlen(tag)] = ch;
 275                         }
 276
 277                         else if ((!nest) && (styletag == 0)) {
 278                                 outbuf[strlen(outbuf)+1] = 0;
 279                                 outbuf[strlen(outbuf)] = ch;
 280                         }
 281                     }
 282                     strcpy(inbuf, &inbuf[i]);
 283                 }
 284
 285                 // Convert &; tags to the forbidden characters
 286                 if (!IsEmptyStr(outbuf)) for (i=0; !IsEmptyStr(&outbuf[i]); ++i) {
 287
 288                         // Character entity references
 289                         if (!strncasecmp(&outbuf[i], "&nbsp;", 6)) {
 290                                 outbuf[i] = ' ';
 291                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 292                         }
 293
 294                         if (!strncasecmp(&outbuf[i], "&ensp;", 6)) {
 295                                 outbuf[i] = ' ';
 296                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 297                         }
 298
 299                         if (!strncasecmp(&outbuf[i], "&emsp;", 6)) {
 300                                 outbuf[i] = ' ';
 301                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 302                         }
 303
 304                         if (!strncasecmp(&outbuf[i], "&thinsp;", 8)) {
 305                                 outbuf[i] = ' ';
 306                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 307                         }
 308
 309                         else if (!strncasecmp(&outbuf[i], "&lt;", 4)) {
 310                                 outbuf[i] = '<';
 311                                 strcpy(&outbuf[i+1], &outbuf[i+4]);
 312                         }
 313
 314                         else if (!strncasecmp(&outbuf[i], "&gt;", 4)) {
 315                                 outbuf[i] = '>';
 316                                 strcpy(&outbuf[i+1], &outbuf[i+4]);
 317                         }
 318
 319                         else if (!strncasecmp(&outbuf[i], "&amp;", 5)) {
 320                                 strcpy(&outbuf[i+1], &outbuf[i+5]);
 321                         }
 322
 323                         else if (!strncasecmp(&outbuf[i], "&quot;", 6)) {
 324                                 outbuf[i] = '\"';
 325                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 326                         }
 327
 328                         else if (!strncasecmp(&outbuf[i], "&lsquo;", 7)) {
 329                                 outbuf[i] = '`';
 330                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 331                         }
 332
 333                         else if (!strncasecmp(&outbuf[i], "&rsquo;", 7)) {
 334                                 outbuf[i] = '\'';
 335                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 336                         }
 337
 338                         else if (!strncasecmp(&outbuf[i], "&copy;", 6)) {
 339                                 outbuf[i] = '(';
 340                                 outbuf[i+1] = 'c';
 341                                 outbuf[i+2] = ')';
 342                                 strcpy(&outbuf[i+3], &outbuf[i+6]);
 343                         }
 344
 345                         else if (!strncasecmp(&outbuf[i], "&bull;", 6)) {
 346                                 outbuf[i] = ' ';
 347                                 outbuf[i+1] = '*';
 348                                 outbuf[i+2] = ' ';
 349                                 strcpy(&outbuf[i+3], &outbuf[i+6]);
 350                         }
 351
 352                         else if (!strncasecmp(&outbuf[i], "&hellip;", 8)) {
 353                                 outbuf[i] = '.';
 354                                 outbuf[i+1] = '.';
 355                                 outbuf[i+2] = '.';
 356                                 strcpy(&outbuf[i+3], &outbuf[i+8]);
 357                         }
 358
 359                         else if (!strncasecmp(&outbuf[i], "&trade;", 7)) {
 360                                 outbuf[i] = '(';
 361                                 outbuf[i+1] = 't';
 362                                 outbuf[i+2] = 'm';
 363                                 outbuf[i+3] = ')';
 364                                 strcpy(&outbuf[i+4], &outbuf[i+7]);
 365                         }
 366
 367                         else if (!strncasecmp(&outbuf[i], "&reg;", 5)) {
 368                                 outbuf[i] = '(';
 369                                 outbuf[i+1] = 'r';
 370                                 outbuf[i+2] = ')';
 371                                 strcpy(&outbuf[i+3], &outbuf[i+5]);
 372                         }
 373
 374                         else if (!strncasecmp(&outbuf[i], "&frac14;", 8)) {
 375                                 outbuf[i] = '1';
 376                                 outbuf[i+1] = '/';
 377                                 outbuf[i+2] = '4';
 378                                 strcpy(&outbuf[i+3], &outbuf[i+8]);
 379                         }
 380
 381                         else if (!strncasecmp(&outbuf[i], "&frac12;", 8)) {
 382                                 outbuf[i] = '1';
 383                                 outbuf[i+1] = '/';
 384                                 outbuf[i+2] = '2';
 385                                 strcpy(&outbuf[i+3], &outbuf[i+8]);
 386                         }
 387
 388                         else if (!strncasecmp(&outbuf[i], "&frac34;", 8)) {
 389                                 outbuf[i] = '3';
 390                                 outbuf[i+1] = '/';
 391                                 outbuf[i+2] = '4';
 392                                 strcpy(&outbuf[i+3], &outbuf[i+8]);
 393                         }
 394
 395                         else if (!strncasecmp(&outbuf[i], "&ndash;", 7)) {
 396                                 outbuf[i] = '-';
 397                                 outbuf[i+1] = '-';
 398                                 strcpy(&outbuf[i+2], &outbuf[i+7]);
 399                         }
 400
 401                         else if (!strncasecmp(&outbuf[i], "&mdash;", 7)) {
 402                                 outbuf[i] = '-';
 403                                 outbuf[i+1] = '-';
 404                                 outbuf[i+2] = '-';
 405                                 strcpy(&outbuf[i+3], &outbuf[i+7]);
 406                         }
 407
 408                         else if (!strncmp(&outbuf[i], "&Ccedil;", 8)) {
 409                                 outbuf[i] = 'C';
 410                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 411                         }
 412
 413                         else if (!strncasecmp(&outbuf[i], "&ccedil;", 8)) {
 414                                 outbuf[i] = 'c';
 415                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 416                         }
 417
 418                         else if (!strncmp(&outbuf[i], "&Egrave;", 8)) {
 419                                 outbuf[i] = 'E';
 420                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 421                         }
 422
 423                         else if (!strncasecmp(&outbuf[i], "&egrave;", 8)) {
 424                                 outbuf[i] = 'e';
 425                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 426                         }
 427
 428                         else if (!strncmp(&outbuf[i], "&Ecirc;", 7)) {
 429                                 outbuf[i] = 'E';
 430                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 431                         }
 432
 433                         else if (!strncasecmp(&outbuf[i], "&ecirc;", 7)) {
 434                                 outbuf[i] = 'e';
 435                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 436                         }
 437
 438                         else if (!strncmp(&outbuf[i], "&Eacute;", 8)) {
 439                                 outbuf[i] = 'E';
 440                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 441                         }
 442
 443                         else if (!strncasecmp(&outbuf[i], "&eacute;", 8)) {
 444                                 outbuf[i] = 'e';
 445                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 446                         }
 447
 448                         else if (!strncmp(&outbuf[i], "&Agrave;", 8)) {
 449                                 outbuf[i] = 'A';
 450                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 451                         }
 452
 453                         else if (!strncasecmp(&outbuf[i], "&agrave;", 8)) {
 454                                 outbuf[i] = 'a';
 455                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 456                         }
 457
 458                         else if (!strncasecmp(&outbuf[i], "&ldquo;", 7)) {
 459                                 outbuf[i] = '\"';
 460                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 461                         }
 462
 463                         else if (!strncasecmp(&outbuf[i], "&rdquo;", 7)) {
 464                                 outbuf[i] = '\"';
 465                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 466                         }
 467
 468                         else if (!strncasecmp(&outbuf[i], "&acute;", 7)) {
 469                                 outbuf[i] = '\'';
 470                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 471                         }
 472
 473                         else if (!strncasecmp(&outbuf[i], "&#8217;", 7)) {
 474                                 outbuf[i] = '\'';
 475                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 476                         }
 477
 478                         else if (!strncasecmp(&outbuf[i], "&#8211;", 7)) {
 479                                 outbuf[i] = '-';
 480                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 481                         }
 482
 483                         // two-digit decimal equivalents
 484                         else if (outbuf[i] == '&'       &&
 485                                  outbuf[i + 1] == '#'   &&
 486                                  isdigit(outbuf[i + 2]) &&
 487                                  isdigit(outbuf[i + 3]) &&
 488                                  (outbuf[i+4] == ';') )
 489                         {
 490                                 scanch = 0;
 491                                 sscanf(&outbuf[i+2], "%02d", &scanch);
 492                                 outbuf[i] = scanch;
 493                                 strcpy(&outbuf[i+1], &outbuf[i+5]);
 494                         }
 495
 496                         // three-digit decimal equivalents
 497                         else if (outbuf[i] == '&'       &&
 498                                  outbuf[i + 1] == '#'   &&
 499                                  isdigit(outbuf[i + 2]) &&
 500                                  isdigit(outbuf[i + 3]) &&
 501                                  isdigit(outbuf[i + 4]) &&
 502                                  (outbuf[i + 5] == ';') )
 503                         {
 504                                 scanch = 0;
 505                                 sscanf(&outbuf[i+2], "%03d", &scanch);
 506                                 outbuf[i] = scanch;
 507                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 508                         }
 509
 510                         // four-digit decimal equivalents
 511                         else if (outbuf[i] == '&'       &&
 512                                  outbuf[i + 1] == '#'   &&
 513                                  isdigit(outbuf[i + 2]) &&
 514                                  isdigit(outbuf[i + 3]) &&
 515                                  isdigit(outbuf[i + 4]) &&
 516                                  isdigit(outbuf[i + 5]) &&
 517                                  (outbuf[i + 6] == ';') )
 518                         {
 519                                 scanch = 0;
 520                                 sscanf(&outbuf[i+2], "%04d", &scanch);
 521                                 outbuf[i] = scanch;
 522                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 523                         }
 524
 525                 }
 526
 527                 // Make sure the output buffer is big enough
 528                 if ((output_len + strlen(outbuf) + SIZ) > outptr_buffer_size) {
 529                         outptr_buffer_size += SIZ;
 530                         outptr = realloc(outptr, outptr_buffer_size);
 531                         if (outptr == NULL) {
 532                                 abort();
 533                         }
 534                 }
 535
 536                 // Output any lines terminated with hard line breaks
 537                 do {
 538                         did_out = 0;
 539                         if (strlen(outbuf) > 0) {
 540                             for (i = 0; i<strlen(outbuf); ++i) {
 541                                 if ( (i<(screenwidth-2)) && (outbuf[i]=='\n')) {
 542
 543                                         strncpy(&outptr[output_len], outbuf, i+1);
 544                                         output_len += (i+1);
 545
 546                                         strcpy(outbuf, &outbuf[i+1]);
 547                                         i = 0;
 548                                         did_out = 1;
 549                                 }
 550                         }
 551                     }
 552                 } while (did_out);
 553
 554                 // Add soft line breaks
 555                 if (strlen(outbuf) > (screenwidth - 2 )) {
 556                         rb = (-1);
 557                         for (i=0; i<(screenwidth-2); ++i) {
 558                                 if (outbuf[i]==32) rb = i;
 559                         }
 560                         if (rb>=0) {
 561                                 strncpy(&outptr[output_len], outbuf, rb);
 562                                 output_len += rb;
 563                                 strcpy(&outptr[output_len], nl);
 564                                 output_len += strlen(nl);
 565                                 strcpy(outbuf, &outbuf[rb+1]);
 566                         }
 567                         else {
 568                                 strncpy(&outptr[output_len], outbuf, screenwidth-2);
 569                                 output_len += (screenwidth-2);
 570                                 strcpy(&outptr[output_len], nl);
 571                                 output_len += strlen(nl);
 572                                 strcpy(outbuf, &outbuf[screenwidth-2]);
 573                         }
 574                 }
 575
 576         } while (done_reading == 0);
 577
 578         strcpy(&outptr[output_len], outbuf);
 579         output_len += strlen(outbuf);
 580
 581         // Strip leading/trailing whitespace.
 582         while ((output_len > 0) && (isspace(outptr[0]))) {
 583                 strcpy(outptr, &outptr[1]);
 584                 --output_len;
 585         }
 586         while ((output_len > 0) && (isspace(outptr[output_len-1]))) {
 587                 outptr[output_len-1] = 0;
 588                 --output_len;
 589         }
 590
 591         // Make sure the final line ends with a newline character.
 592         if ((output_len > 0) && (outptr[output_len-1] != '\n')) {
 593                 strcat(outptr, "\n");
 594                 ++output_len;
 595         }
 596
 597         return outptr;
 598
 599 }