citadel/html.c

   1 /*
   2  * $Id$
   3  *
   4  * Functions which handle translation between HTML and plain text
   5  * Copyright (c) 2000-2005 by Art Cancro and others.   This program is
   6  * released under the terms of the GNU General Public License.
   7  */
   8
   9 #include "sysdep.h"
  10 #include <stdlib.h>
  11 #include <unistd.h>
  12 #include <stdio.h>
  13 #include <fcntl.h>
  14 #include <signal.h>
  15
  16 #if TIME_WITH_SYS_TIME
  17 # include <sys/time.h>
  18 # include <time.h>
  19 #else
  20 # if HAVE_SYS_TIME_H
  21 #  include <sys/time.h>
  22 # else
  23 #  include <time.h>
  24 # endif
  25 #endif
  26
  27 #include <ctype.h>
  28 #include <string.h>
  29 #include <errno.h>
  30 #include <limits.h>
  31 #include "citadel.h"
  32 #include "server.h"
  33 #include "serv_extensions.h"
  34 #include "control.h"
  35 #include "sysdep_decls.h"
  36 #include "support.h"
  37 #include "config.h"
  38 #include "msgbase.h"
  39 #include "tools.h"
  40 #include "room_ops.h"
  41 #include "html.h"
  42
  43
  44 /*
  45  * Convert HTML to plain text.
  46  *
  47  * inputmsg      = pointer to raw HTML message
  48  * screenwidth   = desired output screenwidth
  49  * do_citaformat = set to 1 to indent newlines with spaces
  50  */
  51 char *html_to_ascii(char *inputmsg, int msglen, int screenwidth, int do_citaformat) {
  52         char inbuf[SIZ];
  53         char outbuf[SIZ];
  54         char tag[1024];
  55         int done_reading = 0;
  56         char *inptr;
  57         char *outptr;
  58         size_t outptr_buffer_size;
  59         size_t output_len = 0;
  60         int i, j, ch, did_out, rb, scanch;
  61         int nest = 0;           /* Bracket nesting level */
  62         int blockquote = 0;     /* BLOCKQUOTE nesting level */
  63         int styletag = 0;       /* STYLE tag nesting level */
  64         int styletag_start = 0;
  65         int bytes_processed = 0;
  66         char nl[128];
  67
  68         strcpy(nl, "\n");
  69         inptr = inputmsg;
  70         strcpy(inbuf, "");
  71         strcpy(outbuf, "");
  72         if (msglen == 0) msglen = strlen(inputmsg);
  73
  74         outptr_buffer_size = strlen(inptr) + SIZ;
  75         outptr = malloc(outptr_buffer_size);
  76         if (outptr == NULL) return NULL;
  77         strcpy(outptr, "");
  78         output_len = 0;
  79
  80         do {
  81                 /* Fill the input buffer */
  82                 if ( (done_reading == 0) && (strlen(inbuf) < (SIZ-128)) ) {
  83
  84                         ch = *inptr++;
  85                         if (ch != 0) {
  86                                 inbuf[strlen(inbuf)+1] = 0;
  87                                 inbuf[strlen(inbuf)] = ch;
  88                         }
  89                         else {
  90                                 done_reading = 1;
  91                         }
  92
  93                         ++bytes_processed;
  94                         if (bytes_processed > msglen) {
  95                                 done_reading = 1;
  96                         }
  97
  98                 }
  99
 100                 /* Do some parsing */
 101                 if (strlen(inbuf)>0) {
 102
 103                     /* Fold in all the spacing */
 104                     for (i=0; i<strlen(inbuf); ++i) {
 105                         if (inbuf[i]==10) inbuf[i]=32;
 106                         if (inbuf[i]==13) inbuf[i]=32;
 107                         if (inbuf[i]==9) inbuf[i]=32;
 108                         /*** we like foreign characters now.
 109                         if ((inbuf[i]<32) || (inbuf[i]>126)) {
 110                                 inbuf[i] = '?';
 111                         } */
 112                     }
 113                     for (i=0; i<strlen(inbuf); ++i) {
 114                         while ((inbuf[i]==32)&&(inbuf[i+1]==32))
 115                                 strcpy(&inbuf[i], &inbuf[i+1]);
 116                     }
 117
 118                     for (i=0; i<strlen(inbuf); ++i) {
 119
 120                         ch = inbuf[i];
 121
 122                         if (ch == '<') {
 123                                 ++nest;
 124                                 strcpy(tag, "");
 125                         }
 126
 127                         else if (ch == '>') {   /* We have a tag. */
 128                                 if (nest > 0) --nest;
 129
 130                                 /* Unqualify the tag (truncate at first space) */
 131                                 if (strchr(tag, ' ') != NULL) {
 132                                         strcpy(strchr(tag, ' '), "");
 133                                 }
 134
 135                                 if (!strcasecmp(tag, "P")) {
 136                                         strcat(outbuf, nl);
 137                                         strcat(outbuf, nl);
 138                                 }
 139
 140                                 if (!strcasecmp(tag, "/DIV")) {
 141                                         strcat(outbuf, nl);
 142                                         strcat(outbuf, nl);
 143                                 }
 144
 145                                 if (!strcasecmp(tag, "LI")) {
 146                                         strcat(outbuf, nl);
 147                                         strcat(outbuf, " * ");
 148                                 }
 149
 150                                 else if (!strcasecmp(tag, "/UL")) {
 151                                         strcat(outbuf, nl);
 152                                         strcat(outbuf, nl);
 153                                 }
 154
 155                                 else if (!strcasecmp(tag, "H1")) {
 156                                         strcat(outbuf, nl);
 157                                         strcat(outbuf, nl);
 158                                 }
 159
 160                                 else if (!strcasecmp(tag, "H2")) {
 161                                         strcat(outbuf, nl);
 162                                         strcat(outbuf, nl);
 163                                 }
 164
 165                                 else if (!strcasecmp(tag, "H3")) {
 166                                         strcat(outbuf, nl);
 167                                         strcat(outbuf, nl);
 168                                 }
 169
 170                                 else if (!strcasecmp(tag, "H4")) {
 171                                         strcat(outbuf, nl);
 172                                         strcat(outbuf, nl);
 173                                 }
 174
 175                                 else if (!strcasecmp(tag, "/H1")) {
 176                                         strcat(outbuf, nl);
 177                                 }
 178
 179                                 else if (!strcasecmp(tag, "/H2")) {
 180                                         strcat(outbuf, nl);
 181                                 }
 182
 183                                 else if (!strcasecmp(tag, "/H3")) {
 184                                         strcat(outbuf, nl);
 185                                 }
 186
 187                                 else if (!strcasecmp(tag, "/H4")) {
 188                                         strcat(outbuf, nl);
 189                                 }
 190
 191                                 else if (!strcasecmp(tag, "HR")) {
 192                                         strcat(outbuf, nl);
 193                                         strcat(outbuf, " ");
 194                                         for (j=0; j<screenwidth-2; ++j)
 195                                                 strcat(outbuf, "-");
 196                                         strcat(outbuf, nl);
 197                                 }
 198
 199                                 else if (!strcasecmp(tag, "BR")) {
 200                                         strcat(outbuf, nl);
 201                                 }
 202
 203                                 else if (!strcasecmp(tag, "TR")) {
 204                                         strcat(outbuf, nl);
 205                                 }
 206
 207                                 else if (!strcasecmp(tag, "/TABLE")) {
 208                                         strcat(outbuf, nl);
 209                                 }
 210
 211                                 else if (!strcasecmp(tag, "BLOCKQUOTE")) {
 212                                         ++blockquote;
 213                                         strcpy(nl, "\n");
 214                                         for (j=0; j<blockquote; ++j) strcat(nl, ">");
 215                                         strcat(outbuf, nl);
 216                                 }
 217
 218                                 else if (!strcasecmp(tag, "/BLOCKQUOTE")) {
 219                                         strcat(outbuf, "\n");
 220                                         --blockquote;
 221                                         strcpy(nl, "\n");
 222                                         for (j=0; j<blockquote; ++j) strcat(nl, ">");
 223                                         strcat(outbuf, nl);
 224                                 }
 225
 226                                 else if (!strcasecmp(tag, "STYLE")) {
 227                                         ++styletag;
 228                                         if (styletag == 1) {
 229                                                 styletag_start = strlen(outbuf);
 230                                         }
 231                                 }
 232
 233                                 else if (!strcasecmp(tag, "/STYLE")) {
 234                                         --styletag;
 235                                         if (styletag == 0) {
 236                                                 outbuf[styletag_start] = 0;
 237                                         }
 238                                 }
 239
 240                         }
 241
 242                         else if ((nest > 0) && (strlen(tag)<(sizeof(tag)-1))) {
 243                                 tag[strlen(tag)+1] = 0;
 244                                 tag[strlen(tag)] = ch;
 245                         }
 246
 247                         else if (!nest) {
 248                                 outbuf[strlen(outbuf)+1] = 0;
 249                                 outbuf[strlen(outbuf)] = ch;
 250                         }
 251                     }
 252                     strcpy(inbuf, &inbuf[i]);
 253                 }
 254
 255                 /* Convert &; tags to the forbidden characters */
 256                 if (strlen(outbuf)>0) for (i=0; i<strlen(outbuf); ++i) {
 257
 258                         /* Character entity references */
 259                         if (!strncasecmp(&outbuf[i], "&nbsp;", 6)) {
 260                                 outbuf[i] = ' ';
 261                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 262                         }
 263
 264                         if (!strncasecmp(&outbuf[i], "&ensp;", 6)) {
 265                                 outbuf[i] = ' ';
 266                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 267                         }
 268
 269                         if (!strncasecmp(&outbuf[i], "&emsp;", 6)) {
 270                                 outbuf[i] = ' ';
 271                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 272                         }
 273
 274                         if (!strncasecmp(&outbuf[i], "&thinsp;", 8)) {
 275                                 outbuf[i] = ' ';
 276                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 277                         }
 278
 279                         else if (!strncasecmp(&outbuf[i], "&lt;", 4)) {
 280                                 outbuf[i] = '<';
 281                                 strcpy(&outbuf[i+1], &outbuf[i+4]);
 282                         }
 283
 284                         else if (!strncasecmp(&outbuf[i], "&gt;", 4)) {
 285                                 outbuf[i] = '>';
 286                                 strcpy(&outbuf[i+1], &outbuf[i+4]);
 287                         }
 288
 289                         else if (!strncasecmp(&outbuf[i], "&amp;", 5)) {
 290                                 strcpy(&outbuf[i+1], &outbuf[i+5]);
 291                         }
 292
 293                         else if (!strncasecmp(&outbuf[i], "&quot;", 6)) {
 294                                 outbuf[i] = '\"';
 295                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 296                         }
 297
 298                         else if (!strncasecmp(&outbuf[i], "&lsquo;", 7)) {
 299                                 outbuf[i] = '`';
 300                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 301                         }
 302
 303                         else if (!strncasecmp(&outbuf[i], "&rsquo;", 7)) {
 304                                 outbuf[i] = '\'';
 305                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 306                         }
 307
 308                         else if (!strncasecmp(&outbuf[i], "&copy;", 6)) {
 309                                 outbuf[i] = '(';
 310                                 outbuf[i+1] = 'c';
 311                                 outbuf[i+2] = ')';
 312                                 strcpy(&outbuf[i+3], &outbuf[i+6]);
 313                         }
 314
 315                         else if (!strncasecmp(&outbuf[i], "&hellip;", 8)) {
 316                                 outbuf[i] = '.';
 317                                 outbuf[i+1] = '.';
 318                                 outbuf[i+2] = '.';
 319                                 strcpy(&outbuf[i+3], &outbuf[i+8]);
 320                         }
 321
 322                         else if (!strncasecmp(&outbuf[i], "&trade;", 7)) {
 323                                 outbuf[i] = '(';
 324                                 outbuf[i+1] = 't';
 325                                 outbuf[i+2] = 'm';
 326                                 outbuf[i+3] = ')';
 327                                 strcpy(&outbuf[i+4], &outbuf[i+7]);
 328                         }
 329
 330                         else if (!strncasecmp(&outbuf[i], "&reg;", 5)) {
 331                                 outbuf[i] = '(';
 332                                 outbuf[i+1] = 'r';
 333                                 outbuf[i+2] = ')';
 334                                 strcpy(&outbuf[i+3], &outbuf[i+5]);
 335                         }
 336
 337                         else if (!strncasecmp(&outbuf[i], "&frac14;", 8)) {
 338                                 outbuf[i] = '1';
 339                                 outbuf[i+1] = '/';
 340                                 outbuf[i+2] = '4';
 341                                 strcpy(&outbuf[i+3], &outbuf[i+8]);
 342                         }
 343
 344                         else if (!strncasecmp(&outbuf[i], "&frac12;", 8)) {
 345                                 outbuf[i] = '1';
 346                                 outbuf[i+1] = '/';
 347                                 outbuf[i+2] = '2';
 348                                 strcpy(&outbuf[i+3], &outbuf[i+8]);
 349                         }
 350
 351                         else if (!strncasecmp(&outbuf[i], "&frac34;", 8)) {
 352                                 outbuf[i] = '3';
 353                                 outbuf[i+1] = '/';
 354                                 outbuf[i+2] = '4';
 355                                 strcpy(&outbuf[i+3], &outbuf[i+8]);
 356                         }
 357
 358                         else if (!strncasecmp(&outbuf[i], "&ndash;", 7)) {
 359                                 outbuf[i] = '-';
 360                                 outbuf[i+1] = '-';
 361                                 strcpy(&outbuf[i+2], &outbuf[i+7]);
 362                         }
 363
 364                         else if (!strncasecmp(&outbuf[i], "&mdash;", 7)) {
 365                                 outbuf[i] = '-';
 366                                 outbuf[i+1] = '-';
 367                                 outbuf[i+2] = '-';
 368                                 strcpy(&outbuf[i+3], &outbuf[i+7]);
 369                         }
 370
 371                         else if (!strncmp(&outbuf[i], "&Ccedil;", 8)) {
 372                                 outbuf[i] = 'C';
 373                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 374                         }
 375
 376                         else if (!strncasecmp(&outbuf[i], "&ccedil;", 8)) {
 377                                 outbuf[i] = 'c';
 378                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 379                         }
 380
 381                         else if (!strncmp(&outbuf[i], "&Egrave;", 8)) {
 382                                 outbuf[i] = 'E';
 383                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 384                         }
 385
 386                         else if (!strncasecmp(&outbuf[i], "&egrave;", 8)) {
 387                                 outbuf[i] = 'e';
 388                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 389                         }
 390
 391                         else if (!strncmp(&outbuf[i], "&Ecirc;", 7)) {
 392                                 outbuf[i] = 'E';
 393                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 394                         }
 395
 396                         else if (!strncasecmp(&outbuf[i], "&ecirc;", 7)) {
 397                                 outbuf[i] = 'e';
 398                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 399                         }
 400
 401                         else if (!strncmp(&outbuf[i], "&Eacute;", 8)) {
 402                                 outbuf[i] = 'E';
 403                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 404                         }
 405
 406                         else if (!strncasecmp(&outbuf[i], "&eacute;", 8)) {
 407                                 outbuf[i] = 'e';
 408                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 409                         }
 410
 411                         else if (!strncmp(&outbuf[i], "&Agrave;", 8)) {
 412                                 outbuf[i] = 'A';
 413                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 414                         }
 415
 416                         else if (!strncasecmp(&outbuf[i], "&agrave;", 8)) {
 417                                 outbuf[i] = 'a';
 418                                 strcpy(&outbuf[i+1], &outbuf[i+8]);
 419                         }
 420
 421                         else if (!strncasecmp(&outbuf[i], "&ldquo;", 7)) {
 422                                 outbuf[i] = '\"';
 423                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 424                         }
 425
 426                         else if (!strncasecmp(&outbuf[i], "&rdquo;", 7)) {
 427                                 outbuf[i] = '\"';
 428                                 strcpy(&outbuf[i+1], &outbuf[i+7]);
 429                         }
 430
 431                         /* two-digit decimal equivalents */
 432                         else if ((!strncmp(&outbuf[i], "&#", 2))
 433                               && (outbuf[i+4] == ';') ) {
 434                                 scanch = 0;
 435                                 sscanf(&outbuf[i+2], "%02d", &scanch);
 436                                 outbuf[i] = scanch;
 437                                 strcpy(&outbuf[i+1], &outbuf[i+5]);
 438                         }
 439
 440                         /* three-digit decimal equivalents */
 441                         else if ((!strncmp(&outbuf[i], "&#", 2))
 442                               && (outbuf[i+5] == ';') ) {
 443                                 scanch = 0;
 444                                 sscanf(&outbuf[i+2], "%03d", &scanch);
 445                                 outbuf[i] = scanch;
 446                                 strcpy(&outbuf[i+1], &outbuf[i+6]);
 447                         }
 448
 449                 }
 450
 451                 /* Make sure the output buffer is big enough */
 452                 if ((output_len + strlen(outbuf) + SIZ)
 453                    > outptr_buffer_size) {
 454                         outptr_buffer_size += SIZ;
 455                         outptr = realloc(outptr, outptr_buffer_size);
 456                 }
 457
 458                 /* Output any lines terminated with hard line breaks */
 459                 do {
 460                         did_out = 0;
 461                         if (strlen(outbuf)>0) {
 462                             for (i = 0; i<strlen(outbuf); ++i) {
 463                                 if ( (i<(screenwidth-2)) && (outbuf[i]=='\n')) {
 464
 465                                         strncpy(&outptr[output_len],
 466                                                 outbuf, i+1);
 467                                         output_len += (i+1);
 468
 469                                         if (do_citaformat) {
 470                                                 strcpy(&outptr[output_len],
 471                                                         " ");
 472                                                 ++output_len;
 473                                         }
 474
 475                                         strcpy(outbuf, &outbuf[i+1]);
 476                                         i = 0;
 477                                         did_out = 1;
 478                                 }
 479                         }
 480                     }
 481                 } while (did_out);
 482
 483                 /* Add soft line breaks */
 484                 if (strlen(outbuf) > (screenwidth - 2 )) {
 485                         rb = (-1);
 486                         for (i=0; i<(screenwidth-2); ++i) {
 487                                 if (outbuf[i]==32) rb = i;
 488                         }
 489                         if (rb>=0) {
 490                                 strncpy(&outptr[output_len], outbuf, rb);
 491                                 output_len += rb;
 492                                 strcpy(&outptr[output_len], nl);
 493                                 output_len += strlen(nl);
 494                                 if (do_citaformat) {
 495                                         strcpy(&outptr[output_len], " ");
 496                                         ++output_len;
 497                                 }
 498                                 strcpy(outbuf, &outbuf[rb+1]);
 499                         } else {
 500                                 strncpy(&outptr[output_len], outbuf,
 501                                         screenwidth-2);
 502                                 output_len += (screenwidth-2);
 503                                 strcpy(&outptr[output_len], nl);
 504                                 output_len += strlen(nl);
 505                                 if (do_citaformat) {
 506                                         strcpy(&outptr[output_len], " ");
 507                                         ++output_len;
 508                                 }
 509                                 strcpy(outbuf, &outbuf[screenwidth-2]);
 510                         }
 511                 }
 512
 513         } while (done_reading == 0);
 514
 515         strcpy(&outptr[output_len], outbuf);
 516         output_len += strlen(outbuf);
 517
 518         /* Strip leading/trailing whitespace.  We can't do this with
 519          * striplt() because it uses too many strlen()'s
 520          */
 521         while ((output_len > 0) && (isspace(outptr[0]))) {
 522                 strcpy(outptr, &outptr[1]);
 523                 --output_len;
 524         }
 525         while ((output_len > 0) && (isspace(outptr[output_len-1]))) {
 526                 outptr[output_len-1] = 0;
 527                 --output_len;
 528         }
 529
 530         if (outptr[output_len-1] != '\n') {
 531                 strcat(outptr, "\n");
 532                 ++output_len;
 533         }
 534
 535         return outptr;
 536
 537 }