libcitadel/lib/html_to_ascii.c

   1 /*
   2  * $Id: html.c 6014 2008-02-04 18:38:35Z ajc $
   3  *
   4  * Functions which handle translation between HTML and plain text
   5  * Copyright (c) 2000-2010 by the citadel.org team
   6  *
   7  *  This program is free software; you can redistribute it and/or modify
   8  *  it under the terms of the GNU General Public License as published by
   9  *  the Free Software Foundation; either version 3 of the License, or
  10  *  (at your option) any later version.
  11  *
  12  *  This program is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *  GNU General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU General Public License
  18  *  along with this program; if not, write to the Free Software
  19  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20  */
  21
  22 #include <stdlib.h>
  23 #include <unistd.h>
  24 #include <stdio.h>
  25 #include <signal.h>
  26 #include <sys/types.h>
  27 #include <ctype.h>
  28 #include <string.h>
  29 #include <sys/stat.h>
  30 #include <errno.h>
  31 #include <limits.h>
  32
  33 #if TIME_WITH_SYS_TIME
  34 # include <sys/time.h>
  35 # include <time.h>
  36 #else
  37 # if HAVE_SYS_TIME_H
  38 #  include <sys/time.h>
  39 # else
  40 #  include <time.h>
  41 # endif
  42 #endif
  43
  44 #include "libcitadel.h"
  45
  46
  47 /*
  48  * Convert HTML to plain text.
  49  *
  50  * inputmsg      = pointer to raw HTML message
  51  * screenwidth   = desired output screenwidth
  52  * do_citaformat = set to 1 to indent newlines with spaces
  53  */
  54 char *html_to_ascii(char *inputmsg, int msglen, int screenwidth, int do_citaformat) {
  55         char inbuf[SIZ];
  56         int inbuf_len = 0;
  57         char outbuf[SIZ];
  58         char tag[1024];
  59         int done_reading = 0;
  60         char *inptr;
  61         char *outptr;
  62         size_t outptr_buffer_size;
  63         size_t output_len = 0;
  64         int i, j, ch, did_out, rb, scanch;
  65         int nest = 0;           /* Bracket nesting level */
  66         int blockquote = 0;     /* BLOCKQUOTE nesting level */
  67         int styletag = 0;       /* STYLE tag nesting level */
  68         int styletag_start = 0;
  69         int bytes_processed = 0;
  70         char nl[128];
  71
  72         strcpy(nl, "\n");
  73         inptr = inputmsg;
  74         strcpy(inbuf, "");
  75         strcpy(outbuf, "");
  76         if (msglen == 0) msglen = strlen(inputmsg);
  77
  78         outptr_buffer_size = strlen(inptr) + SIZ;
  79         outptr = malloc(outptr_buffer_size);
  80         if (outptr == NULL) return NULL;
  81         strcpy(outptr, "");
  82         output_len = 0;
  83
  84         do {
  85                 /* Fill the input buffer */
  86                 inbuf_len = strlen(inbuf);
  87                 if ( (done_reading == 0) && (inbuf_len < (SIZ-128)) ) {
  88
  89                         ch = *inptr++;
  90                         if (ch != 0) {
  91                                 inbuf[inbuf_len++] = ch;
  92                                 inbuf[inbuf_len] = 0;
  93                         }
  94                         else {
  95                                 done_reading = 1;
  96                         }
  97
  98                         ++bytes_processed;
  99                         if (bytes_processed > msglen) {
 100                                 done_reading = 1;
 101                         }
 102
 103                 }
 104
 105                 /* Do some parsing */
 106                 if (!IsEmptyStr(inbuf)) {
 107
 108
 109                     /* Fold in all the spacing */
 110                     for (i=0; !IsEmptyStr(&inbuf[i]); ++i) {
 111                         if (inbuf[i]==10) inbuf[i]=32;
 112                         if (inbuf[i]==13) inbuf[i]=32;
 113                         if (inbuf[i]==9) inbuf[i]=32;
 114                     }
 115                     for (i=0; !IsEmptyStr(&inbuf[i]); ++i) {
 116                         while ((inbuf[i]==32)&&(inbuf[i+1]==32)) {
 117                                 strcpy(&inbuf[i], &inbuf[i+1]);
 118                         }
 119                     }
 120
 121                     for (i=0; !IsEmptyStr(&inbuf[i]); ++i) {
 122
 123                         ch = inbuf[i];
 124
 125                         if (ch == '<') {
 126                                 ++nest;
 127                                 strcpy(tag, "");
 128                         }
 129
 130                         else if (ch == '>') {   /* We have a tag. */
 131                                 if (nest > 0) --nest;
 132
 133                                 /* Unqualify the tag (truncate at first space) */
 134                                 if (strchr(tag, ' ') != NULL) {
 135                                         strcpy(strchr(tag, ' '), "");
 136                                 }
 137
 138                                 if (!strcasecmp(tag, "P")) {
 139                                         strcat(outbuf, nl);
 140                                         strcat(outbuf, nl);
 141                                 }
 142
 143                                 if (!strcasecmp(tag, "/DIV")) {
 144                                         strcat(outbuf, nl);
 145                                         strcat(outbuf, nl);
 146                                 }
 147
 148                                 if (!strcasecmp(tag, "LI")) {
 149                                         strcat(outbuf, nl);
 150                                         strcat(outbuf, " * ");
 151                                 }
 152
 153                                 else if (!strcasecmp(tag, "/UL")) {
 154                                         strcat(outbuf, nl);
 155                                         strcat(outbuf, nl);
 156                                 }
 157
 158                                 else if (!strcasecmp(tag, "H1")) {
 159                                         strcat(outbuf, nl);
 160                                         strcat(outbuf, nl);
 161                                 }
 162
 163                                 else if (!strcasecmp(tag, "H2")) {
 164                                         strcat(outbuf, nl);
 165                                         strcat(outbuf, nl);
 166                                 }
 167
 168                                 else if (!strcasecmp(tag, "H3")) {
 169                                         strcat(outbuf, nl);
 170                                         strcat(outbuf, nl);
 171                                 }
 172
 173                                 else if (!strcasecmp(tag, "H4")) {
 174                                         strcat(outbuf, nl);
 175                                         strcat(outbuf, nl);
 176                                 }
 177
 178                                 else if (!strcasecmp(tag, "/H1")) {
 179                                         strcat(outbuf, nl);
 180                                 }
 181
 182                                 else if (!strcasecmp(tag, "/H2")) {
 183                                         strcat(outbuf, nl);
 184                                 }
 185
 186                                 else if (!strcasecmp(tag, "/H3")) {
 187                                         strcat(outbuf, nl);
 188                                 }
 189
 190                                 else if (!strcasecmp(tag, "/H4")) {
 191                                         strcat(outbuf, nl);
 192                                 }
 193
 194                                 else if (!strcasecmp(tag, "HR")) {
 195                                         strcat(outbuf, nl);
 196                                         strcat(outbuf, " ");
 197                                         for (j=0; j<screenwidth-2; ++j)
 198                                                 strcat(outbuf, "-");
 199                                         strcat(outbuf, nl);
 200                                 }
 201
 202                                 else if (
 203                                         (!strcasecmp(tag, "B"))
 204                                         || (!strcasecmp(tag, "/B"))
 205                                         || (!strcasecmp(tag, "STRONG"))
 206                                         || (!strcasecmp(tag, "/STRONG"))
 207                                 ) {
 208                                         strcat(outbuf, "*");
 209
 210                                 }
 211
 212                                 else if (
 213                                         (!strcasecmp(tag, "I"))
 214                                         || (!strcasecmp(tag, "/I"))
 215                                         || (!strcasecmp(tag, "EM"))
 216                                         || (!strcasecmp(tag, "/EM"))
 217                                 ) {
 218                                         strcat(outbuf, "/");
 219
 220                                 }
 221
 222                                 else if (
 223                                         (!strcasecmp(tag, "U"))
 224                                         || (!strcasecmp(tag, "/U"))
 225                                 ) {
 226                                         strcat(outbuf, "_");
 227
 228                                 }
 229
 230                                 else if (!strcasecmp(tag, "BR")) {
 231                                         strcat(outbuf, nl);
 232                                 }
 233
 234                                 else if (!strcasecmp(tag, "TR")) {
 235                                         strcat(outbuf, nl);
 236                                 }
 237
 238                                 else if (!strcasecmp(tag, "/TABLE")) {
 239                                         strcat(outbuf, nl);
 240                                 }
 241
 242                                 else if (!strcasecmp(tag, "BLOCKQUOTE")) {
 243                                         ++blockquote;
 244                                         strcpy(nl, "\n");
 245                                         for (j=0; j<blockquote; ++j) strcat(nl, ">");
 246                                         strcat(outbuf, nl);
 247                                 }
 248
 249                                 else if (!strcasecmp(tag, "/BLOCKQUOTE")) {
 250                                         strcat(outbuf, "\n");
 251                                         --blockquote;
 252                                         strcpy(nl, "\n");
 253                                         for (j=0; j<blockquote; ++j) strcat(nl, ">");
 254                                         strcat(outbuf, nl);
 255                                 }
 256
 257                                 else if (!strcasecmp(tag, "STYLE")) {
 258                                         ++styletag;
 259                                         if (styletag == 1) {
 260                                                 styletag_start = strlen(outbuf);
 261                                         }
 262                                 }
 263
 264                                 else if (!strcasecmp(tag, "/STYLE")) {
 265                                         --styletag;
 266                                         if (styletag == 0) {
 267                                                 outbuf[styletag_start] = 0;
 268                                         }
 269                                 }
 270
 271                         }
 272
 273                         else if ((nest > 0) && (strlen(tag)<(sizeof(tag)-1))) {
 274                                 tag[strlen(tag)+1] = 0;
 275                                 tag[strlen(tag)] = ch;
 276                         }
 277
 278                         else if (!nest) {
 279                                 outbuf[strlen(outbuf)+1] = 0;
 280                                 outbuf[strlen(outbuf)] = ch;
 281                         }
 282                     }
 283                     strcpy(inbuf, &inbuf[i]);
 284                 }
 285
 286                 /* Convert &; tags to the forbidden characters */
 287                 if (!IsEmptyStr(outbuf)) for (i=0; !IsEmptyStr(&outbuf[i]); ++i) {
 288
 289                         /* Character entity references */
 290                         if (!strncasecmp(&outbuf[i], "&nbsp;", 6)) {
 291                                 outbuf[i] = ' ';
 292                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 293                         }
 294
 295                         if (!strncasecmp(&outbuf[i], "&ensp;", 6)) {
 296                                 outbuf[i] = ' ';
 297                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 298                         }
 299
 300                         if (!strncasecmp(&outbuf[i], "&emsp;", 6)) {
 301                                 outbuf[i] = ' ';
 302                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 303                         }
 304
 305                         if (!strncasecmp(&outbuf[i], "&thinsp;", 8)) {
 306                                 outbuf[i] = ' ';
 307                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 308                         }
 309
 310                         else if (!strncasecmp(&outbuf[i], "&lt;", 4)) {
 311                                 outbuf[i] = '<';
 312                                 strcpy(&outbuf[i+1], &outbuf[i+4]);
 313                         }
 314
 315                         else if (!strncasecmp(&outbuf[i], "&gt;", 4)) {
 316                                 outbuf[i] = '>';
 317                                 strcpy(&outbuf[i+1], &outbuf[i+4]);
 318                         }
 319
 320                         else if (!strncasecmp(&outbuf[i], "&amp;", 5)) {
 321                                 strcpy(&outbuf[i+1], &outbuf[i+5]);
 322                         }
 323
 324                         else if (!strncasecmp(&outbuf[i], "&quot;", 6)) {
 325                                 outbuf[i] = '\"';
 326                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 327                         }
 328
 329                         else if (!strncasecmp(&outbuf[i], "&lsquo;", 7)) {
 330                                 outbuf[i] = '`';
 331                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 332                         }
 333
 334                         else if (!strncasecmp(&outbuf[i], "&rsquo;", 7)) {
 335                                 outbuf[i] = '\'';
 336                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 337                         }
 338
 339                         else if (!strncasecmp(&outbuf[i], "&copy;", 6)) {
 340                                 outbuf[i] = '(';
 341                                 outbuf[i+1] = 'c';
 342                                 outbuf[i+2] = ')';
 343                                 strcpy(&outbuf[i+3], &outbuf[i+6]);
 344                         }
 345
 346                         else if (!strncasecmp(&outbuf[i], "&bull;", 6)) {
 347                                 outbuf[i] = ' ';
 348                                 outbuf[i+1] = '*';
 349                                 outbuf[i+2] = ' ';
 350                                 strcpy(&outbuf[i+3], &outbuf[i+6]);
 351                         }
 352
 353                         else if (!strncasecmp(&outbuf[i], "&hellip;", 8)) {
 354                                 outbuf[i] = '.';
 355                                 outbuf[i+1] = '.';
 356                                 outbuf[i+2] = '.';
 357                                 strcpy(&outbuf[i+3], &outbuf[i+8]);
 358                         }
 359
 360                         else if (!strncasecmp(&outbuf[i], "&trade;", 7)) {
 361                                 outbuf[i] = '(';
 362                                 outbuf[i+1] = 't';
 363                                 outbuf[i+2] = 'm';
 364                                 outbuf[i+3] = ')';
 365                                 strcpy(&outbuf[i+4], &outbuf[i+7]);
 366                         }
 367
 368                         else if (!strncasecmp(&outbuf[i], "&reg;", 5)) {
 369                                 outbuf[i] = '(';
 370                                 outbuf[i+1] = 'r';
 371                                 outbuf[i+2] = ')';
 372                                 strcpy(&outbuf[i+3], &outbuf[i+5]);
 373                         }
 374
 375                         else if (!strncasecmp(&outbuf[i], "&frac14;", 8)) {
 376                                 outbuf[i] = '1';
 377                                 outbuf[i+1] = '/';
 378                                 outbuf[i+2] = '4';
 379                                 strcpy(&outbuf[i+3], &outbuf[i+8]);
 380                         }
 381
 382                         else if (!strncasecmp(&outbuf[i], "&frac12;", 8)) {
 383                                 outbuf[i] = '1';
 384                                 outbuf[i+1] = '/';
 385                                 outbuf[i+2] = '2';
 386                                 strcpy(&outbuf[i+3], &outbuf[i+8]);
 387                         }
 388
 389                         else if (!strncasecmp(&outbuf[i], "&frac34;", 8)) {
 390                                 outbuf[i] = '3';
 391                                 outbuf[i+1] = '/';
 392                                 outbuf[i+2] = '4';
 393                                 strcpy(&outbuf[i+3], &outbuf[i+8]);
 394                         }
 395
 396                         else if (!strncasecmp(&outbuf[i], "&ndash;", 7)) {
 397                                 outbuf[i] = '-';
 398                                 outbuf[i+1] = '-';
 399                                 strcpy(&outbuf[i+2], &outbuf[i+7]);
 400                         }
 401
 402                         else if (!strncasecmp(&outbuf[i], "&mdash;", 7)) {
 403                                 outbuf[i] = '-';
 404                                 outbuf[i+1] = '-';
 405                                 outbuf[i+2] = '-';
 406                                 strcpy(&outbuf[i+3], &outbuf[i+7]);
 407                         }
 408
 409                         else if (!strncmp(&outbuf[i], "&Ccedil;", 8)) {
 410                                 outbuf[i] = 'C';
 411                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 412                         }
 413
 414                         else if (!strncasecmp(&outbuf[i], "&ccedil;", 8)) {
 415                                 outbuf[i] = 'c';
 416                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 417                         }
 418
 419                         else if (!strncmp(&outbuf[i], "&Egrave;", 8)) {
 420                                 outbuf[i] = 'E';
 421                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 422                         }
 423
 424                         else if (!strncasecmp(&outbuf[i], "&egrave;", 8)) {
 425                                 outbuf[i] = 'e';
 426                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 427                         }
 428
 429                         else if (!strncmp(&outbuf[i], "&Ecirc;", 7)) {
 430                                 outbuf[i] = 'E';
 431                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 432                         }
 433
 434                         else if (!strncasecmp(&outbuf[i], "&ecirc;", 7)) {
 435                                 outbuf[i] = 'e';
 436                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 437                         }
 438
 439                         else if (!strncmp(&outbuf[i], "&Eacute;", 8)) {
 440                                 outbuf[i] = 'E';
 441                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 442                         }
 443
 444                         else if (!strncasecmp(&outbuf[i], "&eacute;", 8)) {
 445                                 outbuf[i] = 'e';
 446                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 447                         }
 448
 449                         else if (!strncmp(&outbuf[i], "&Agrave;", 8)) {
 450                                 outbuf[i] = 'A';
 451                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 452                         }
 453
 454                         else if (!strncasecmp(&outbuf[i], "&agrave;", 8)) {
 455                                 outbuf[i] = 'a';
 456                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 457                         }
 458
 459                         else if (!strncasecmp(&outbuf[i], "&ldquo;", 7)) {
 460                                 outbuf[i] = '\"';
 461                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 462                         }
 463
 464                         else if (!strncasecmp(&outbuf[i], "&rdquo;", 7)) {
 465                                 outbuf[i] = '\"';
 466                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 467                         }
 468
 469                         else if (!strncasecmp(&outbuf[i], "&acute;", 7)) {
 470                                 outbuf[i] = '\'';
 471                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 472                         }
 473
 474                         /* two-digit decimal equivalents */
 475                         else if ((!strncmp(&outbuf[i], "&#", 2))
 476                               && (outbuf[i+4] == ';') ) {
 477                                 scanch = 0;
 478                                 sscanf(&outbuf[i+2], "%02d", &scanch);
 479                                 outbuf[i] = scanch;
 480                                 strcpy(&outbuf[i+1], &outbuf[i+5]);
 481                         }
 482
 483                         /* three-digit decimal equivalents */
 484                         else if ((!strncmp(&outbuf[i], "&#", 2))
 485                               && (outbuf[i+5] == ';') ) {
 486                                 scanch = 0;
 487                                 sscanf(&outbuf[i+2], "%03d", &scanch);
 488                                 outbuf[i] = scanch;
 489                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 490                         }
 491
 492                 }
 493
 494                 /* Make sure the output buffer is big enough */
 495                 if ((output_len + strlen(outbuf) + SIZ) > outptr_buffer_size) {
 496                         outptr_buffer_size += SIZ;
 497                         outptr = realloc(outptr, outptr_buffer_size);
 498                         if (outptr == NULL) {
 499                                 abort();
 500                         }
 501                 }
 502
 503                 /* Output any lines terminated with hard line breaks */
 504                 do {
 505                         did_out = 0;
 506                         if (strlen(outbuf) > 0) {
 507                             for (i = 0; i<strlen(outbuf); ++i) {
 508                                 if ( (i<(screenwidth-2)) && (outbuf[i]=='\n')) {
 509
 510                                         strncpy(&outptr[output_len], outbuf, i+1);
 511                                         output_len += (i+1);
 512
 513                                         if (do_citaformat) {
 514                                                 strcpy(&outptr[output_len], " ");
 515                                                 ++output_len;
 516                                         }
 517
 518                                         strcpy(outbuf, &outbuf[i+1]);
 519                                         i = 0;
 520                                         did_out = 1;
 521                                 }
 522                         }
 523                     }
 524                 } while (did_out);
 525
 526                 /* Add soft line breaks */
 527                 if (strlen(outbuf) > (screenwidth - 2 )) {
 528                         rb = (-1);
 529                         for (i=0; i<(screenwidth-2); ++i) {
 530                                 if (outbuf[i]==32) rb = i;
 531                         }
 532                         if (rb>=0) {
 533                                 strncpy(&outptr[output_len], outbuf, rb);
 534                                 output_len += rb;
 535                                 strcpy(&outptr[output_len], nl);
 536                                 output_len += strlen(nl);
 537                                 if (do_citaformat) {
 538                                         strcpy(&outptr[output_len], " ");
 539                                         ++output_len;
 540                                 }
 541                                 strcpy(outbuf, &outbuf[rb+1]);
 542                         } else {
 543                                 strncpy(&outptr[output_len], outbuf,
 544                                         screenwidth-2);
 545                                 output_len += (screenwidth-2);
 546                                 strcpy(&outptr[output_len], nl);
 547                                 output_len += strlen(nl);
 548                                 if (do_citaformat) {
 549                                         strcpy(&outptr[output_len], " ");
 550                                         ++output_len;
 551                                 }
 552                                 strcpy(outbuf, &outbuf[screenwidth-2]);
 553                         }
 554                 }
 555
 556         } while (done_reading == 0);
 557
 558         strcpy(&outptr[output_len], outbuf);
 559         output_len += strlen(outbuf);
 560
 561         /* Strip leading/trailing whitespace.  We can't do this with
 562          * striplt() because it uses too many strlen()'s
 563          */
 564         while ((output_len > 0) && (isspace(outptr[0]))) {
 565                 strcpy(outptr, &outptr[1]);
 566                 --output_len;
 567         }
 568         while ((output_len > 0) && (isspace(outptr[output_len-1]))) {
 569                 outptr[output_len-1] = 0;
 570                 --output_len;
 571         }
 572
 573         if ((output_len > 0) && (outptr[output_len-1] != '\n')) {
 574                 strcat(outptr, "\n");
 575                 ++output_len;
 576         }
 577
 578         return outptr;
 579
 580 }