citadel/modules/rssclient/serv_rssclient.c

   1 /*
   2  * Bring external RSS feeds into rooms.
   3  *
   4  * Copyright (c) 2007-2010 by the citadel.org team
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <expat.h>
  42 #include <curl/curl.h>
  43 #include <libcitadel.h>
  44 #include "citadel.h"
  45 #include "server.h"
  46 #include "citserver.h"
  47 #include "support.h"
  48 #include "config.h"
  49 #include "threads.h"
  50 #include "ctdl_module.h"
  51 #include "clientsocket.h"
  52 #include "msgbase.h"
  53 #include "parsedate.h"
  54 #include "database.h"
  55 #include "citadel_dirs.h"
  56 #include "md5.h"
  57 #include "context.h"
  58
  59
  60
  61 const char *LinkShortenerServices[] = {
  62 "http://bit.ly/",
  63 "http://krz.ch/",
  64 "http://flic.kr/",
  65 "http://sns.ly/",
  66 "http://wp.me/",
  67 "http://ow.ly/",
  68 "http://tinyurl.com/",
  69 NULL
  70 };
  71
  72 typedef struct rssnetcfg rssnetcfg;
  73 struct rssnetcfg {
  74         rssnetcfg *next;
  75         char url[256];
  76         char *rooms;
  77         time_t last_error_when;
  78         int ItemType;
  79 };
  80
  81 #define RSS_UNSET       (1<<0)
  82 #define RSS_RSS         (1<<1)
  83 #define RSS_ATOM        (1<<2)
  84 #define RSS_REQUIRE_BUF (1<<3)
  85
  86 typedef struct _rss_item {
  87         char *roomlist;
  88         int done_parsing;
  89         StrBuf *guid;
  90         StrBuf *title;
  91         StrBuf *link;
  92         StrBuf *linkTitle;
  93         StrBuf *reLink;
  94         StrBuf *reLinkTitle;
  95         StrBuf *description;
  96         time_t pubdate;
  97         StrBuf *channel_title;
  98         int item_tag_nesting;
  99         StrBuf *author_or_creator;
 100         StrBuf *author_url;
 101         StrBuf *author_email;
 102 }rss_item;
 103
 104
 105 typedef void (*rss_handler_func)(StrBuf *CData,
 106                                  rss_item *ri,
 107                                  rssnetcfg *Cfg,
 108                                  const char** Attr);
 109
 110 typedef struct __rss_xml_handler {
 111         int Flags;
 112         rss_handler_func Handler;
 113 }rss_xml_handler;
 114
 115
 116 typedef struct _rsscollection {
 117         StrBuf *CData;
 118         StrBuf *Key;
 119
 120         rss_item *Item;
 121         rssnetcfg *Cfg;
 122
 123         rss_xml_handler *Current;
 124 } rsscollection;
 125
 126 struct rssnetcfg *rnclist = NULL;
 127 HashList *StartHandlers;
 128 HashList *EndHandlers;
 129 HashList *KnownNameSpaces;
 130 void AddRSSStartHandler(rss_handler_func Handler, int Flags, const char *key, long len)
 131 {
 132         rss_xml_handler *h;
 133         h = (rss_xml_handler*) malloc(sizeof (rss_xml_handler));
 134         h->Flags = Flags;
 135         h->Handler = Handler;
 136         Put(StartHandlers, key, len, h, NULL);
 137 }
 138 void AddRSSEndHandler(rss_handler_func Handler, int Flags, const char *key, long len)
 139 {
 140         rss_xml_handler *h;
 141         h = (rss_xml_handler*) malloc(sizeof (rss_xml_handler));
 142         h->Flags = Flags;
 143         h->Handler = Handler;
 144         Put(EndHandlers, key, len, h, NULL);
 145 }
 146
 147 #if 0
 148 //#ifdef HAVE_ICONV
 149 #include <iconv.h>
 150
 151
 152 /*
 153  * dug this out of the trashcan of the midgard project, lets see if it works for us.
 154  * original code by Alexander Bokovoy <bokovoy@avilink.ne> distributed under GPL V2 or later
 155  */
 156
 157 /* Returns:
 158  >= 0 - successfull, 0 means conversion doesn't use multibyte sequences
 159    -1 - error during iconv_open call
 160    -2 - error during iconv_close call
 161    ----------------------------------
 162    This function expects that multibyte encoding in 'charset' wouldn't have
 163    characters with more than 3 bytes. It is not intended to convert UTF-8 because
 164    we'll never receive UTF-8 in our handler (it is handled by Exat itself).
 165 */
 166 static int
 167 fill_encoding_info (const char *charset, XML_Encoding * info)
 168 {
 169   iconv_t cd = (iconv_t)(-1);
 170   int flag;
 171         CtdlLogPrintf(0, "RSS: fill encoding info ...\n");
 172
 173 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
 174   cd = iconv_open ("UCS-2LE", charset);
 175 #else
 176   cd = iconv_open ("UCS-2BE", charset);
 177 #endif
 178
 179   if (cd == (iconv_t) (-1))
 180     {
 181       return -1;
 182     }
 183
 184   {
 185     unsigned short out = 0;
 186     unsigned char buf[4];
 187     unsigned int i0, i1, i2;
 188     int result = 0;
 189     flag = 0;
 190     for (i0 = 0; i0 < 0x100; i0++)
 191       {
 192         buf[0] = i0;
 193         info->map[i0] = 0;
 194         //result = try (cd, buf, 1, &out);
 195         if (result < 0)
 196           {
 197           }
 198         else if (result > 0)
 199           {
 200             info->map[i0] = out;
 201           }
 202         else
 203           {
 204             for (i1 = 0; i1 < 0x100; i1++)
 205               {
 206                 buf[1] = i1;
 207                 ///result = try (cd, buf, 2, &out);
 208                 if (result < 0)
 209                   {
 210                   }
 211                 else if (result > 0)
 212                   {
 213                     flag++;
 214                     info->map[i0] = -2;
 215                   }
 216                 else
 217                   {
 218                     for (i2 = 0; i2 < 0x100; i2++)
 219                       {
 220                         buf[2] = i2;
 221                         ////result = try (cd, buf, 3, &out);
 222                         if (result < 0)
 223                           {
 224                           }
 225                         else if (result > 0)
 226                           {
 227                             flag++;
 228                             info->map[i0] = -3;
 229                           }
 230                       }
 231                   }
 232               }
 233           }
 234       }
 235   }
 236
 237   if (iconv_close (cd) < 0)
 238     {
 239       return -2;
 240     }
 241   return flag;
 242 }
 243
 244 static int
 245 iconv_convertor (void *data, const char *s)
 246 {
 247   XML_Encoding *info = data;
 248   int res;
 249         CtdlLogPrintf(0, "RSS: Converting ...\n");
 250
 251   if (s == NULL)
 252     return -1;
 253 /*
 254   GByteArray *result;
 255   result = g_byte_array_new ();
 256   if (process_block (info->data, (char *) s, strlen (s), result) == 0)
 257     {
 258       res = *(result->data);
 259       g_byte_array_free (result, TRUE);
 260       return res;
 261     }
 262   g_byte_array_free (result, TRUE);
 263 */
 264   return -1;
 265 }
 266
 267 static void
 268 my_release (void *data)
 269 {
 270   iconv_t cd = (iconv_t) data;
 271   if (iconv_close (cd) != 0)
 272     {
 273 /// TODO: uh no.      exit (1);
 274     }
 275 }
 276 int
 277 handle_unknown_xml_encoding (void *encodingHandleData,
 278                              const XML_Char * name,
 279                              XML_Encoding * info)
 280 {
 281   int result;
 282   CtdlLogPrintf(0, "RSS: unknown encoding ...\n");
 283   result = fill_encoding_info (name, info);
 284   if (result >= 0)
 285     {
 286       /*
 287         Special case: client asked for reverse conversion, we'll provide him with
 288         iconv descriptor which handles it. Client should release it by himself.
 289       */
 290       if(encodingHandleData != NULL)
 291             *((iconv_t *)encodingHandleData) = iconv_open(name, "UTF-8");
 292       /*
 293          Optimization: we do not need conversion function if encoding is one-to-one,
 294          info->map table will be enough
 295        */
 296       if (result == 0)
 297         {
 298           info->data = NULL;
 299           info->convert = NULL;
 300           info->release = NULL;
 301           return 1;
 302         }
 303       /*
 304          We do need conversion function because this encoding uses multibyte sequences
 305        */
 306       info->data = (void *) iconv_open ("UTF-8", name);
 307       if ((int)info->data == -1)
 308         return -1;
 309       info->convert = iconv_convertor;
 310       info->release = my_release;
 311       return 1;
 312     }
 313   if(encodingHandleData != NULL)
 314     *(iconv_t *)encodingHandleData = NULL;
 315   return 0;
 316 }
 317
 318 ///#endif
 319 #endif
 320 size_t GetLocationString( void *ptr, size_t size, size_t nmemb, void *userdata)
 321 {
 322 #define LOCATION "location"
 323         if (strncasecmp((char*)ptr, LOCATION, sizeof(LOCATION) - 1) == 0)
 324         {
 325                 StrBuf *pURL = (StrBuf*) userdata;
 326                 char *pch = (char*) ptr;
 327                 char *pche;
 328
 329                 pche = pch + (size * nmemb);
 330                 pch += sizeof(LOCATION);
 331
 332                 while (isspace(*pch) || (*pch == ':'))
 333                         pch ++;
 334
 335                 while (isspace(*pche) || (*pche == '\0'))
 336                         pche--;
 337
 338                 FlushStrBuf(pURL);
 339                 StrBufPlain(pURL, pch, pche - pch + 1);
 340         }
 341         return size * nmemb;
 342 }
 343
 344 int LookupUrl(StrBuf *ShorterUrlStr)
 345 {
 346         CURL *curl;
 347         char errmsg[1024] = "";
 348         StrBuf *Answer;
 349         int rc = 0;
 350
 351         curl = curl_easy_init();
 352         if (!curl) {
 353                 CtdlLogPrintf(CTDL_ALERT, "Unable to initialize libcurl.\n");
 354                 return 0;
 355         }
 356         Answer = NewStrBufPlain(NULL, SIZ);
 357
 358         curl_easy_setopt(curl, CURLOPT_URL, ChrPtr(ShorterUrlStr));
 359         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
 360         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
 361         curl_easy_setopt(curl, CURLOPT_WRITEDATA, Answer);
 362 //      curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, rss_libcurl_callback);
 363         curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlFillStrBuf_callback);
 364         curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errmsg);
 365         curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
 366 #ifdef CURLOPT_HTTP_CONTENT_DECODING
 367         curl_easy_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, 1);
 368         curl_easy_setopt(curl, CURLOPT_ENCODING, "");
 369 #endif
 370         curl_easy_setopt(curl, CURLOPT_USERAGENT, CITADEL);
 371         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 180);           /* die after 180 seconds */
 372         curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0);
 373
 374         curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION , GetLocationString);
 375         curl_easy_setopt(curl, CURLOPT_WRITEHEADER, ShorterUrlStr);
 376
 377
 378         if (
 379                 (!IsEmptyStr(config.c_ip_addr))
 380                 && (strcmp(config.c_ip_addr, "*"))
 381                 && (strcmp(config.c_ip_addr, "::"))
 382                 && (strcmp(config.c_ip_addr, "0.0.0.0"))
 383         ) {
 384                 curl_easy_setopt(curl, CURLOPT_INTERFACE, config.c_ip_addr);
 385         }
 386
 387         if (CtdlThreadCheckStop())
 388                 goto shutdown ;
 389
 390         rc = curl_easy_perform(curl);
 391         if (rc) {
 392                 CtdlLogPrintf(CTDL_ALERT, "libcurl error %d: %s\n", rc, errmsg);
 393                 rc = 0;
 394         }
 395         else
 396                 rc = 1;
 397
 398 shutdown:
 399         curl_easy_cleanup(curl);
 400
 401         return rc;
 402
 403 }
 404
 405
 406
 407 void CrawlMessageForShorterUrls(HashList *pUrls, StrBuf *Message)
 408 {
 409         int nHits = 0;
 410         const char *pShortenerService;
 411         int nShorter = 0;
 412         const char *pch;
 413         const char *pUrl;
 414         ConstStr *pCUrl;
 415
 416         pShortenerService = LinkShortenerServices[nShorter++];
 417         while (pShortenerService != NULL)
 418         {
 419                 pch = ChrPtr(Message);
 420                 pUrl = strstr(pch, pShortenerService);
 421                 while ((pUrl != NULL) && (nHits < 99))
 422                 {
 423                         pCUrl = malloc(sizeof(ConstStr));
 424
 425                         pCUrl->Key = pUrl;
 426                         pch = pUrl + strlen(pShortenerService);
 427                         while (isalnum(*pch)||(*pch == '-')||(*pch == '/'))
 428                                 pch++;
 429                         pCUrl->len = pch - pCUrl->Key;
 430
 431                         Put(pUrls, IKEY(nHits), pCUrl, NULL);
 432                         nHits ++;
 433                         pUrl = strstr(pch, pShortenerService);
 434                 }
 435                 pShortenerService = LinkShortenerServices[nShorter++];
 436         }
 437 }
 438
 439 int SortConstStrByPosition(const void *Item1, const void *Item2)
 440 {
 441         const ConstStr *p1, *p2;
 442         p1 = (const ConstStr*) Item1;
 443         p2 = (const ConstStr*) Item2;
 444         if (p1->Key == p2->Key)
 445                 return 0;
 446         if (p1->Key > p2->Key)
 447                 return 1;
 448         return -1;
 449 }
 450
 451 void ExpandShortUrls(StrBuf *Message)
 452 {
 453         StrBuf *Shadow;
 454         HashList *pUrls;
 455         ConstStr *pCUrl;
 456         const char *pch;
 457         const char *pche;
 458
 459         /* we just suspect URL shorteners to be inside of feeds from twitter
 460          * or other short content messages, so don't crawl through real blogs.
 461          */
 462         if (StrLength(Message) > 500)
 463                 return;
 464
 465         pUrls = NewHash(1, Flathash);
 466         CrawlMessageForShorterUrls(pUrls, Message);
 467
 468         if (GetCount(pUrls) > 0)
 469         {
 470                 StrBuf *ShorterUrlStr;
 471                 HashPos *Pos;
 472                 const char *Key;
 473                 void *pv;
 474                 long len;
 475
 476                 Shadow = NewStrBufPlain(NULL, StrLength(Message));
 477                 SortByPayload (pUrls, SortConstStrByPosition);
 478
 479                 ShorterUrlStr = NewStrBufPlain(NULL, StrLength(Message));
 480
 481                 pch = ChrPtr(Message);
 482                 pche = pch + StrLength(Message);
 483                 Pos = GetNewHashPos(pUrls, 1);
 484                 while (GetNextHashPos(pUrls, Pos, &len, &Key, &pv))
 485                 {
 486                         pCUrl = (ConstStr*) pv;
 487
 488                         if (pch != pCUrl->Key)
 489                                 StrBufAppendBufPlain(Shadow, pch, pCUrl->Key - pch, 0);
 490
 491                         StrBufPlain(ShorterUrlStr, CKEY(*pCUrl));
 492                         if (LookupUrl(ShorterUrlStr))
 493                         {
 494                                 StrBufAppendBufPlain(Shadow, HKEY("<a href=\""), 0);
 495                                 StrBufAppendBuf(Shadow, ShorterUrlStr, 0);
 496                                 StrBufAppendBufPlain(Shadow, HKEY("\">"), 0);
 497                                 StrBufAppendBuf(Shadow, ShorterUrlStr, 0);
 498                                 StrBufAppendBufPlain(Shadow, HKEY("["), 0);
 499                                 StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
 500                                 StrBufAppendBufPlain(Shadow, HKEY("]</a>"), 0);
 501                         }
 502                         else
 503                         {
 504                                 StrBufAppendBufPlain(Shadow, HKEY("<a href=\""), 0);
 505                                 StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
 506                                 StrBufAppendBufPlain(Shadow, HKEY("\">"), 0);
 507                                 StrBufAppendBufPlain(Shadow, pCUrl->Key, pCUrl->len, 0);
 508                                 StrBufAppendBufPlain(Shadow, HKEY("</a>"), 0);
 509                         }
 510                         pch = pCUrl->Key + pCUrl->len + 1;
 511
 512                 }
 513                 if (pch < pche)
 514                         StrBufAppendBufPlain(Shadow, pch, pche - pch, 0);
 515                 FlushStrBuf(Message);
 516                 StrBufAppendBuf(Message, Shadow, 0);
 517
 518                 FreeStrBuf(&ShorterUrlStr);
 519                 FreeStrBuf(&Shadow);
 520                 DeleteHashPos(&Pos);
 521         }
 522
 523         DeleteHash(&pUrls);
 524 }
 525
 526
 527 void AppendLink(StrBuf *Message, StrBuf *link, StrBuf *LinkTitle, const char *Title)
 528 {
 529         if (StrLength(link) > 0)
 530         {
 531                 StrBufAppendBufPlain(Message, HKEY("<a href=\""), 0);
 532                 StrBufAppendBuf(Message, link, 0);
 533                 StrBufAppendBufPlain(Message, HKEY("\">"), 0);
 534                 if (StrLength(LinkTitle) > 0)
 535                         StrBufAppendBuf(Message, LinkTitle, 0);
 536                 else if ((Title != NULL) && !IsEmptyStr(Title))
 537                         StrBufAppendBufPlain(Message, Title, -1, 0);
 538                 else
 539                         StrBufAppendBuf(Message, link, 0);
 540                 StrBufAppendBufPlain(Message, HKEY("</a><br>\n"), 0);
 541         }
 542 }
 543 /*
 544  * Commit a fetched and parsed RSS item to disk
 545  */
 546 void rss_save_item(rss_item *ri)
 547 {
 548
 549         struct MD5Context md5context;
 550         u_char rawdigest[MD5_DIGEST_LEN];
 551         int i;
 552         char utmsgid[SIZ];
 553         struct cdbdata *cdbut;
 554         struct UseTable ut;
 555         struct CtdlMessage *msg;
 556         struct recptypes *recp = NULL;
 557         int msglen = 0;
 558         StrBuf *Message;
 559
 560         recp = (struct recptypes *) malloc(sizeof(struct recptypes));
 561         if (recp == NULL) return;
 562         memset(recp, 0, sizeof(struct recptypes));
 563         memset(&ut, 0, sizeof(struct UseTable));
 564         recp->recp_room = strdup(ri->roomlist);
 565         recp->num_room = num_tokens(ri->roomlist, '|');
 566         recp->recptypes_magic = RECPTYPES_MAGIC;
 567
 568         /* Construct a GUID to use in the S_USETABLE table.
 569          * If one is not present in the item itself, make one up.
 570          */
 571         if (ri->guid != NULL) {
 572                 StrBufSpaceToBlank(ri->guid);
 573                 StrBufTrim(ri->guid);
 574                 snprintf(utmsgid, sizeof utmsgid, "rss/%s", ChrPtr(ri->guid));
 575         }
 576         else {
 577                 MD5Init(&md5context);
 578                 if (ri->title != NULL) {
 579                         MD5Update(&md5context, (const unsigned char*)ChrPtr(ri->title), StrLength(ri->title));
 580                 }
 581                 if (ri->link != NULL) {
 582                         MD5Update(&md5context, (const unsigned char*)ChrPtr(ri->link), StrLength(ri->link));
 583                 }
 584                 MD5Final(rawdigest, &md5context);
 585                 for (i=0; i<MD5_DIGEST_LEN; i++) {
 586                         sprintf(&utmsgid[i*2], "%02X", (unsigned char) (rawdigest[i] & 0xff));
 587                         utmsgid[i*2] = tolower(utmsgid[i*2]);
 588                         utmsgid[(i*2)+1] = tolower(utmsgid[(i*2)+1]);
 589                 }
 590                 strcat(utmsgid, "_rss2ctdl");
 591         }
 592
 593         /* Find out if we've already seen this item */
 594
 595         cdbut = cdb_fetch(CDB_USETABLE, utmsgid, strlen(utmsgid));
 596 #ifndef DEBUG_RSS
 597         if (cdbut != NULL) {
 598                 /* Item has already been seen */
 599                 CtdlLogPrintf(CTDL_DEBUG, "%s has already been seen\n", utmsgid);
 600                 cdb_free(cdbut);
 601
 602                 /* rewrite the record anyway, to update the timestamp */
 603                 strcpy(ut.ut_msgid, utmsgid);
 604                 ut.ut_timestamp = time(NULL);
 605                 cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) );
 606         }
 607         else
 608 #endif
 609 {
 610                 /* Item has not been seen, so save it. */
 611                 CtdlLogPrintf(CTDL_DEBUG, "RSS: saving item...\n");
 612                 if (ri->description == NULL) ri->description = NewStrBufPlain(HKEY(""));
 613                 StrBufSpaceToBlank(ri->description);
 614                 msg = malloc(sizeof(struct CtdlMessage));
 615                 memset(msg, 0, sizeof(struct CtdlMessage));
 616                 msg->cm_magic = CTDLMESSAGE_MAGIC;
 617                 msg->cm_anon_type = MES_NORMAL;
 618                 msg->cm_format_type = FMT_RFC822;
 619
 620                 if (ri->guid != NULL) {
 621                         msg->cm_fields['E'] = strdup(ChrPtr(ri->guid));
 622                 }
 623
 624                 if (ri->author_or_creator != NULL) {
 625                         char *From;
 626                         StrBuf *Encoded, *QPEncoded;
 627                         StrBuf *UserName;
 628                         StrBuf *EmailAddress;
 629                         StrBuf *EncBuf;
 630                         int FromAt;
 631                         int FromLen;
 632
 633                         UserName = NewStrBuf();
 634                         EmailAddress = NewStrBuf();
 635                         EncBuf = NewStrBuf();
 636 ////TODO!
 637                         StrBufTrim(ri->author_or_creator);
 638                         From = html_to_ascii(ChrPtr(ri->author_or_creator),
 639                                              StrLength(ri->author_or_creator),
 640                                              512, 0);
 641                         FromLen = strlen(From);
 642                         if (From[FromLen - 1] == '\n')
 643                         {
 644                                 From[FromLen - 1] = '\0';
 645                         }
 646                         FromAt = strchr(From, '@') != NULL;
 647                         if (!FromAt && StrLength (ri->author_email) > 0)
 648                         {
 649                                 Encoded = NewStrBuf();
 650                                 if (!IsEmptyStr(From))
 651                                 {
 652                                         StrBufPrintf(Encoded,
 653                                                      "%s<%s>",
 654                                                      From,
 655                                                      ChrPtr(ri->author_email));
 656                                 }
 657                                 else
 658                                 {
 659                                         StrBufPrintf(Encoded,
 660                                                      "<%s>",
 661                                                      ChrPtr(ri->author_email));
 662                                 }
 663                         }
 664                         else
 665                         {
 666                                 if (FromAt)
 667                                         Encoded = NewStrBufPlain(From, -1);
 668                                 else
 669                                 {
 670                                         Encoded = NewStrBuf();
 671                                         StrBufPrintf(Encoded,
 672                                                      "%s<%s>",
 673                                                      From,
 674                                                      "rss@localhost"); /// TODO: get hostname?
 675                                 }
 676                         }
 677                         free(From);
 678                         StrBufTrim(Encoded);
 679                         QPEncoded = StrBufSanitizeEmailRecipientVector(Encoded, UserName, EmailAddress, EncBuf);
 680                         msg->cm_fields['A'] = SmashStrBuf(&QPEncoded);
 681
 682                         FreeStrBuf(&Encoded);
 683                         FreeStrBuf(&UserName);
 684                         FreeStrBuf(&EmailAddress);
 685                         FreeStrBuf(&EncBuf);
 686
 687                 }
 688                 else {
 689                         msg->cm_fields['A'] = strdup("rss");
 690                 }
 691
 692                 msg->cm_fields['N'] = strdup(NODENAME);
 693                 if (ri->title != NULL) {
 694                         long len;
 695                         char *Sbj;
 696                         StrBuf *Encoded, *QPEncoded;
 697
 698                         QPEncoded = NULL;
 699                         StrBufSpaceToBlank(ri->title);
 700                         len = StrLength(ri->title);
 701                         Sbj = html_to_ascii(ChrPtr(ri->title), len, 512, 0);
 702                         len = strlen(Sbj);
 703                         if (Sbj[len - 1] == '\n')
 704                         {
 705                                 len --;
 706                                 Sbj[len] = '\0';
 707                         }
 708                         Encoded = NewStrBufPlain(Sbj, len);
 709                         free(Sbj);
 710
 711                         StrBufTrim(Encoded);
 712                         StrBufRFC2047encode(&QPEncoded, Encoded);
 713
 714                         msg->cm_fields['U'] = SmashStrBuf(&QPEncoded);
 715                         FreeStrBuf(&Encoded);
 716                 }
 717                 msg->cm_fields['T'] = malloc(64);
 718                 snprintf(msg->cm_fields['T'], 64, "%ld", ri->pubdate);
 719                 if (ri->channel_title != NULL) {
 720                         if (StrLength(ri->channel_title) > 0) {
 721                                 msg->cm_fields['O'] = strdup(ChrPtr(ri->channel_title));
 722                         }
 723                 }
 724                 if (ri->link == NULL)
 725                         ri->link = NewStrBufPlain(HKEY(""));
 726                 ExpandShortUrls(ri->description);
 727                 msglen += 1024 + StrLength(ri->link) + StrLength(ri->description) ;
 728
 729                 Message = NewStrBufPlain(NULL, StrLength(ri->description));
 730
 731                 StrBufPlain(Message, HKEY(
 732                          "Content-type: text/html; charset=\"UTF-8\"\r\n\r\n"
 733                          "<html><body>\n"));
 734
 735                 StrBufAppendBuf(Message, ri->description, 0);
 736                 StrBufAppendBufPlain(Message, HKEY("<br><br>\n"), 0);
 737
 738                 AppendLink(Message, ri->link, ri->linkTitle, NULL);
 739                 AppendLink(Message, ri->reLink, ri->reLinkTitle, "Reply to this");
 740                 StrBufAppendBufPlain(Message, HKEY("</body></html>\n"), 0);
 741
 742                 msg->cm_fields['M'] = SmashStrBuf(&Message);
 743
 744                 CtdlSubmitMsg(msg, recp, NULL, 0);
 745                 CtdlFreeMessage(msg);
 746
 747                 /* write the uidl to the use table so we don't store this item again */
 748                 strcpy(ut.ut_msgid, utmsgid);
 749                 ut.ut_timestamp = time(NULL);
 750                 cdb_store(CDB_USETABLE, utmsgid, strlen(utmsgid), &ut, sizeof(struct UseTable) );
 751         }
 752         free_recipients(recp);
 753 }
 754
 755
 756
 757 /*
 758  * Convert an RDF/RSS datestamp into a time_t
 759  */
 760 time_t rdf_parsedate(const char *p)
 761 {
 762         struct tm tm;
 763         time_t t = 0;
 764
 765         if (!p) return 0L;
 766         if (strlen(p) < 10) return 0L;
 767
 768         memset(&tm, 0, sizeof tm);
 769
 770         /*
 771          * If the timestamp appears to be in W3C datetime format, try to
 772          * parse it.  See also: http://www.w3.org/TR/NOTE-datetime
 773          *
 774          * This code, along with parsedate.c, is a potential candidate for
 775          * moving into libcitadel.
 776          */
 777         if ( (p[4] == '-') && (p[7] == '-') ) {
 778                 tm.tm_year = atoi(&p[0]) - 1900;
 779                 tm.tm_mon = atoi(&p[5]) - 1;
 780                 tm.tm_mday = atoi(&p[8]);
 781                 if ( (p[10] == 'T') && (p[13] == ':') ) {
 782                         tm.tm_hour = atoi(&p[11]);
 783                         tm.tm_min = atoi(&p[14]);
 784                 }
 785                 return mktime(&tm);
 786         }
 787
 788         /* hmm... try RFC822 date stamp format */
 789
 790         t = parsedate(p);
 791         if (t > 0) return(t);
 792
 793         /* yeesh.  ok, just return the current date and time. */
 794         return(time(NULL));
 795 }
 796
 797 void flush_rss_item(rss_item *ri)
 798 {
 799         /* Initialize the feed item data structure */
 800         FreeStrBuf(&ri->guid);
 801         FreeStrBuf(&ri->title);
 802         FreeStrBuf(&ri->link);
 803         FreeStrBuf(&ri->author_or_creator);
 804         FreeStrBuf(&ri->author_email);
 805         FreeStrBuf(&ri->author_url);
 806         FreeStrBuf(&ri->description);
 807 }
 808
 809 void rss_xml_start(void *data, const char *supplied_el, const char **attr)
 810 {
 811         rss_xml_handler *h;
 812         rsscollection   *rssc = (rsscollection*) data;
 813         rssnetcfg       *Cfg = rssc->Cfg;
 814         rss_item        *ri = rssc->Item;
 815         void            *pv;
 816         const char      *pel;
 817         char            *sep = NULL;
 818
 819         /* Axe the namespace, we don't care about it */
 820 ///     CtdlLogPrintf(0, "RSS: supplied el %d: %s...\n", rssc->Cfg->ItemType, supplied_el);
 821         pel = supplied_el;
 822         while (sep = strchr(pel, ':'), sep) {
 823                 pel = sep + 1;
 824         }
 825
 826         if (pel != supplied_el)
 827         {
 828                 void *v;
 829
 830                 if (!GetHash(KnownNameSpaces,
 831                              supplied_el,
 832                              pel - supplied_el - 1,
 833                              &v))
 834                 {
 835 #ifdef DEBUG_RSS
 836                         CtdlLogPrintf(0, "RSS: START ignoring because of wrong namespace [%s] = [%s]\n",
 837                                       supplied_el);
 838 #endif
 839                         return;
 840                 }
 841         }
 842
 843         StrBufPlain(rssc->Key, pel, -1);
 844         StrBufLowerCase(rssc->Key);
 845         if (GetHash(StartHandlers, SKEY(rssc->Key), &pv))
 846         {
 847                 rssc->Current = h = (rss_xml_handler*) pv;
 848
 849                 if (((h->Flags & RSS_UNSET) != 0) &&
 850                     (Cfg->ItemType == RSS_UNSET))
 851                 {
 852                         h->Handler(rssc->CData, ri, Cfg, attr);
 853                 }
 854                 else if (((h->Flags & RSS_RSS) != 0) &&
 855                     (Cfg->ItemType == RSS_RSS))
 856                 {
 857                         h->Handler(rssc->CData, ri, Cfg, attr);
 858                 }
 859                 else if (((h->Flags & RSS_ATOM) != 0) &&
 860                          (Cfg->ItemType == RSS_ATOM))
 861                 {
 862                         h->Handler(rssc->CData, ri, Cfg, attr);
 863                 }
 864 #ifdef DEBUG_RSS
 865                 else
 866                         CtdlLogPrintf(0, "RSS: START unhandled: [%s] [%s]...\n", pel, supplied_el);
 867 #endif
 868         }
 869 #ifdef DEBUG_RSS
 870         else
 871                 CtdlLogPrintf(0, "RSS: START unhandled: [%s] [%s]...\n", pel,  supplied_el);
 872 #endif
 873 }
 874
 875 void rss_xml_end(void *data, const char *supplied_el)
 876 {
 877         rss_xml_handler *h;
 878         rsscollection   *rssc = (rsscollection*) data;
 879         rssnetcfg       *Cfg = rssc->Cfg;
 880         rss_item        *ri = rssc->Item;
 881         const char      *pel;
 882         char            *sep = NULL;
 883         void            *pv;
 884
 885         /* Axe the namespace, we don't care about it */
 886         pel = supplied_el;
 887         while (sep = strchr(pel, ':'), sep) {
 888                 pel = sep + 1;
 889         }
 890 //      CtdlLogPrintf(0, "RSS: END %s...\n", el);
 891         if (pel != supplied_el)
 892         {
 893                 void *v;
 894
 895                 if (!GetHash(KnownNameSpaces,
 896                              supplied_el,
 897                              pel - supplied_el - 1,
 898                              &v))
 899                 {
 900 #ifdef DEBUG_RSS
 901                         CtdlLogPrintf(0, "RSS: END ignoring because of wrong namespace [%s] = [%s]\n",
 902                                       supplied_el, ChrPtr(rssc->CData));
 903 #endif
 904                         FlushStrBuf(rssc->CData);
 905                         return;
 906                 }
 907         }
 908
 909         StrBufPlain(rssc->Key, pel, -1);
 910         StrBufLowerCase(rssc->Key);
 911         if (GetHash(EndHandlers, SKEY(rssc->Key), &pv))
 912         {
 913                 h = (rss_xml_handler*) pv;
 914
 915                 if (((h->Flags & RSS_UNSET) != 0) &&
 916                     (Cfg->ItemType == RSS_UNSET))
 917                 {
 918                         h->Handler(rssc->CData, ri, Cfg, NULL);
 919                 }
 920                 else if (((h->Flags & RSS_RSS) != 0) &&
 921                     (Cfg->ItemType == RSS_RSS))
 922                 {
 923                         h->Handler(rssc->CData, ri, Cfg, NULL);
 924                 }
 925                 else if (((h->Flags & RSS_ATOM) != 0) &&
 926                          (Cfg->ItemType == RSS_ATOM))
 927                 {
 928                         h->Handler(rssc->CData, ri, Cfg, NULL);
 929                 }
 930 #ifdef DEBUG_RSS
 931                 else
 932                         CtdlLogPrintf(0, "RSS: END   unhandled: [%s]  [%s] = [%s]...\n", pel, supplied_el, ChrPtr(rssc->CData));
 933 #endif
 934         }
 935 #ifdef DEBUG_RSS
 936         else
 937                 CtdlLogPrintf(0, "RSS: END   unhandled: [%s]  [%s] = [%s]...\n", pel, supplied_el, ChrPtr(rssc->CData));
 938 #endif
 939         FlushStrBuf(rssc->CData);
 940         rssc->Current = NULL;
 941 }
 942
 943
 944
 945
 946
 947 void RSS_item_rss_start (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
 948 {
 949         CtdlLogPrintf(CTDL_DEBUG, "RSS: This is an RSS feed.\n");
 950         Cfg->ItemType = RSS_RSS;
 951 }
 952
 953 void RSS_item_rdf_start(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
 954 {
 955         CtdlLogPrintf(CTDL_DEBUG, "RSS: This is an RDF feed.\n");
 956         Cfg->ItemType = RSS_RSS;
 957 }
 958
 959 void ATOM_item_feed_start(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
 960 {
 961         CtdlLogPrintf(CTDL_DEBUG, "RSS: This is an ATOM feed.\n");
 962         Cfg->ItemType = RSS_ATOM;
 963 }
 964
 965
 966 void RSS_item_item_start(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
 967 {
 968         ri->item_tag_nesting ++;
 969         flush_rss_item(ri);
 970 }
 971
 972 void ATOM_item_entry_start(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
 973 {
 974 /* Atom feed... */
 975         ri->item_tag_nesting ++;
 976         flush_rss_item(ri);
 977 }
 978
 979 void ATOM_item_link_start (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
 980 {
 981         int i;
 982         const char *pHref = NULL;
 983         const char *pType = NULL;
 984         const char *pRel = NULL;
 985         const char *pTitle = NULL;
 986
 987         for (i = 0; Attr[i] != NULL; i+=2)
 988         {
 989                 if (!strcmp(Attr[i], "href"))
 990                 {
 991                         pHref = Attr[i+1];
 992                 }
 993                 else if (!strcmp(Attr[i], "rel"))
 994                 {
 995                         pRel = Attr[i+1];
 996                 }
 997                 else if (!strcmp(Attr[i], "type"))
 998                 {
 999                         pType = Attr[i+1];
1000                 }
1001                 else if (!strcmp(Attr[i], "title"))
1002                 {
1003                         pTitle = Attr[i+1];
1004                 }
1005         }
1006         if (pHref == NULL)
1007                 return; /* WHUT? Pointing... where? */
1008         if ((pType != NULL) && !strcasecmp(pType, "application/atom+xml"))
1009                 return; /* these just point to other rss resources, we're not interested in them. */
1010         if (pRel != NULL)
1011         {
1012                 if (!strcasecmp (pRel, "replies"))
1013                 {
1014                         NewStrBufDupAppendFlush(&ri->reLink, NULL, pHref, -1);
1015                         StrBufTrim(ri->link);
1016                         NewStrBufDupAppendFlush(&ri->reLinkTitle, NULL, pTitle, -1);
1017                 }
1018                 else if (!strcasecmp(pRel, "alternate")) /* Alternative representation of this Item... */
1019                 {
1020                         NewStrBufDupAppendFlush(&ri->link, NULL, pHref, -1);
1021                         StrBufTrim(ri->link);
1022                         NewStrBufDupAppendFlush(&ri->linkTitle, NULL, pTitle, -1);
1023
1024                 }
1025 #if 0 /* these are also defined, but dunno what to do with them.. */
1026                 else if (!strcasecmp(pRel, "related"))
1027                 {
1028                 }
1029                 else if (!strcasecmp(pRel, "self"))
1030                 {
1031                 }
1032                 else if (!strcasecmp(pRel, "enclosure"))
1033                 {/* this reference can get big, and is probably the full article... */
1034                 }
1035                 else if (!strcasecmp(pRel, "via"))
1036                 {/* this article was provided via... */
1037                 }
1038 #endif
1039         }
1040         else if (StrLength(ri->link) == 0)
1041         {
1042                 NewStrBufDupAppendFlush(&ri->link, NULL, pHref, -1);
1043                 StrBufTrim(ri->link);
1044                 NewStrBufDupAppendFlush(&ri->linkTitle, NULL, pTitle, -1);
1045         }
1046 }
1047
1048
1049
1050
1051 void ATOMRSS_item_title_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1052 {
1053         if ((ri->item_tag_nesting == 0) && (StrLength(CData) > 0)) {
1054                 NewStrBufDupAppendFlush(&ri->channel_title, CData, NULL, 0);
1055                 StrBufTrim(ri->channel_title);
1056         }
1057 }
1058
1059 void RSS_item_guid_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1060 {
1061         if (StrLength(CData) > 0) {
1062                 NewStrBufDupAppendFlush(&ri->guid, CData, NULL, 0);
1063         }
1064 }
1065
1066 void ATOM_item_id_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1067 {
1068         if (StrLength(CData) > 0) {
1069                 NewStrBufDupAppendFlush(&ri->guid, CData, NULL, 0);
1070         }
1071 }
1072
1073
1074 void RSS_item_link_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1075 {
1076         if (StrLength(CData) > 0) {
1077                 NewStrBufDupAppendFlush(&ri->link, CData, NULL, 0);
1078                 StrBufTrim(ri->link);
1079         }
1080 }
1081 void RSS_item_relink_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1082 {
1083         if (StrLength(CData) > 0) {
1084                 NewStrBufDupAppendFlush(&ri->reLink, CData, NULL, 0);
1085                 StrBufTrim(ri->reLink);
1086         }
1087 }
1088
1089 void RSSATOM_item_title_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1090 {
1091         if (StrLength(CData) > 0) {
1092                 NewStrBufDupAppendFlush(&ri->title, CData, NULL, 0);
1093                 StrBufTrim(ri->title);
1094         }
1095 }
1096
1097 void ATOM_item_content_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1098 {
1099         long olen = StrLength (ri->description);
1100         long clen = StrLength (CData);
1101         if (clen > 0)
1102         {
1103                 if (olen == 0) {
1104                         NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0);
1105                         StrBufTrim(ri->description);
1106                 }
1107                 else if (olen < clen) {
1108                         FlushStrBuf(ri->description);
1109                         NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0);
1110                         StrBufTrim(ri->description);
1111                 }
1112         }
1113 }
1114 void ATOM_item_summary_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1115 {
1116         /* this can contain an abstract of the article. but we don't want to verwrite a full document if we already have it. */
1117         if ((StrLength(CData) > 0) && (StrLength(ri->description) == 0))
1118         {
1119                 NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0);
1120                 StrBufTrim(ri->description);
1121         }
1122 }
1123
1124 void RSS_item_description_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1125 {
1126         long olen = StrLength (ri->description);
1127         long clen = StrLength (CData);
1128         if (clen > 0)
1129         {
1130                 if (olen == 0) {
1131                         NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0);
1132                         StrBufTrim(ri->description);
1133                 }
1134                 else if (olen < clen) {
1135                         FlushStrBuf(ri->description);
1136                         NewStrBufDupAppendFlush(&ri->description, CData, NULL, 0);
1137                         StrBufTrim(ri->description);
1138                 }
1139         }
1140 }
1141
1142 void ATOM_item_published_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1143 {
1144         if (StrLength(CData) > 0) {
1145                 StrBufTrim(CData);
1146                 ri->pubdate = rdf_parsedate(ChrPtr(CData));
1147         }
1148 }
1149
1150 void ATOM_item_updated_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1151 {
1152         if (StrLength(CData) > 0) {
1153                 StrBufTrim(CData);
1154                 ri->pubdate = rdf_parsedate(ChrPtr(CData));
1155         }
1156 }
1157
1158 void RSS_item_pubdate_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1159 {
1160         if (StrLength(CData) > 0) {
1161                 StrBufTrim(CData);
1162                 ri->pubdate = rdf_parsedate(ChrPtr(CData));
1163         }
1164 }
1165
1166
1167 void RSS_item_date_end (StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1168 {
1169         if (StrLength(CData) > 0) {
1170                 StrBufTrim(CData);
1171                 ri->pubdate = rdf_parsedate(ChrPtr(CData));
1172         }
1173 }
1174
1175
1176
1177 void RSS_item_author_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1178 {
1179         if (StrLength(CData) > 0) {
1180                 NewStrBufDupAppendFlush(&ri->author_or_creator, CData, NULL, 0);
1181                 StrBufTrim(ri->author_or_creator);
1182         }
1183 }
1184
1185
1186 void ATOM_item_name_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1187 {
1188         if (StrLength(CData) > 0) {
1189                 NewStrBufDupAppendFlush(&ri->author_or_creator, CData, NULL, 0);
1190                 StrBufTrim(ri->author_or_creator);
1191         }
1192 }
1193
1194 void ATOM_item_email_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1195 {
1196         if (StrLength(CData) > 0) {
1197                 NewStrBufDupAppendFlush(&ri->author_email, CData, NULL, 0);
1198                 StrBufTrim(ri->author_email);
1199         }
1200 }
1201
1202 void RSS_item_creator_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1203 {
1204         if ((StrLength(CData) > 0) &&
1205             (StrLength(ri->author_or_creator) == 0))
1206         {
1207                 NewStrBufDupAppendFlush(&ri->author_or_creator, CData, NULL, 0);
1208                 StrBufTrim(ri->author_or_creator);
1209         }
1210 }
1211
1212
1213 void ATOM_item_uri_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1214 {
1215         if (StrLength(CData) > 0) {
1216                 NewStrBufDupAppendFlush(&ri->author_url, CData, NULL, 0);
1217                 StrBufTrim(ri->author_url);
1218         }
1219 }
1220
1221 void RSS_item_item_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1222 {
1223         --ri->item_tag_nesting;
1224         rss_save_item(ri);
1225 }
1226
1227
1228 void ATOM_item_entry_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1229 {
1230         --ri->item_tag_nesting;
1231         rss_save_item(ri);
1232 }
1233
1234 void RSS_item_rss_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1235 {
1236 //              CtdlLogPrintf(CTDL_DEBUG, "End of feed detected.  Closing parser.\n");
1237         ri->done_parsing = 1;
1238
1239 }
1240 void RSS_item_rdf_end(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1241 {
1242 //              CtdlLogPrintf(CTDL_DEBUG, "End of feed detected.  Closing parser.\n");
1243         ri->done_parsing = 1;
1244 }
1245
1246
1247 void RSSATOM_item_ignore(StrBuf *CData, rss_item *ri, rssnetcfg *Cfg, const char** Attr)
1248 {
1249 }
1250
1251
1252
1253 /*
1254  * This callback stores up the data which appears in between tags.
1255  */
1256 void rss_xml_cdata_start(void *data)
1257 {
1258         rsscollection *rssc = (rsscollection*) data;
1259
1260         FlushStrBuf(rssc->CData);
1261 }
1262
1263 void rss_xml_cdata_end(void *data)
1264 {
1265 }
1266 void rss_xml_chardata(void *data, const XML_Char *s, int len)
1267 {
1268         rsscollection *rssc = (rsscollection*) data;
1269
1270         StrBufAppendBufPlain (rssc->CData, s, len, 0);
1271 }
1272
1273 /*
1274  * Callback function for passing libcurl's output to expat for parsing
1275  */
1276 size_t rss_libcurl_callback(void *ptr, size_t size, size_t nmemb, void *stream)
1277 {
1278         XML_Parse((XML_Parser)stream, ptr, (size * nmemb), 0);
1279         return (size*nmemb);
1280 }
1281
1282
1283
1284 /*
1285  * Begin a feed parse
1286  */
1287 void rss_do_fetching(rssnetcfg *Cfg) {
1288         rsscollection rssc;
1289         rss_item ri;
1290         XML_Parser xp = NULL;
1291         StrBuf *Answer;
1292
1293         CURL *curl;
1294         CURLcode res;
1295         char errmsg[1024] = "";
1296         char *ptr;
1297         const char *at;
1298         long len;
1299
1300         memset(&ri, 0, sizeof(rss_item));
1301         rssc.Item = &ri;
1302         rssc.Cfg = Cfg;
1303
1304         CtdlLogPrintf(CTDL_DEBUG, "Fetching RSS feed <%s>\n", Cfg->url);
1305
1306         curl = curl_easy_init();
1307         if (!curl) {
1308                 CtdlLogPrintf(CTDL_ALERT, "Unable to initialize libcurl.\n");
1309                 return;
1310         }
1311         Answer = NewStrBufPlain(NULL, SIZ);
1312
1313         curl_easy_setopt(curl, CURLOPT_URL, Cfg->url);
1314         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
1315         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0);
1316         curl_easy_setopt(curl, CURLOPT_WRITEDATA, Answer);
1317 //      curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, rss_libcurl_callback);
1318         curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlFillStrBuf_callback);
1319         curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errmsg);
1320         curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
1321 #ifdef CURLOPT_HTTP_CONTENT_DECODING
1322         curl_easy_setopt(curl, CURLOPT_HTTP_CONTENT_DECODING, 1);
1323         curl_easy_setopt(curl, CURLOPT_ENCODING, "");
1324 #endif
1325         curl_easy_setopt(curl, CURLOPT_USERAGENT, CITADEL);
1326         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 180);           /* die after 180 seconds */
1327         if (
1328                 (!IsEmptyStr(config.c_ip_addr))
1329                 && (strcmp(config.c_ip_addr, "*"))
1330                 && (strcmp(config.c_ip_addr, "::"))
1331                 && (strcmp(config.c_ip_addr, "0.0.0.0"))
1332         ) {
1333                 curl_easy_setopt(curl, CURLOPT_INTERFACE, config.c_ip_addr);
1334         }
1335
1336         if (CtdlThreadCheckStop())
1337         {
1338                 curl_easy_cleanup(curl);
1339                 return;
1340         }
1341
1342         if (CtdlThreadCheckStop())
1343                 goto shutdown ;
1344
1345         res = curl_easy_perform(curl);
1346         if (res) {
1347                 CtdlLogPrintf(CTDL_ALERT, "libcurl error %d: %s\n", res, errmsg);
1348         }
1349
1350         if (CtdlThreadCheckStop())
1351                 goto shutdown ;
1352
1353
1354
1355
1356         memset(&ri, 0, sizeof(rss_item));
1357         ri.roomlist = Cfg->rooms;
1358         rssc.CData = NewStrBufPlain(NULL, SIZ);
1359         rssc.Key = NewStrBuf();
1360         at = NULL;
1361         StrBufSipLine(rssc.Key, Answer, &at);
1362         ptr = NULL;
1363
1364 #define encoding "encoding=\""
1365         ptr = strstr(ChrPtr(rssc.Key), encoding);
1366         if (ptr != NULL)
1367         {
1368                 char *pche;
1369
1370                 ptr += sizeof (encoding) - 1;
1371                 pche = strchr(ptr, '"');
1372                 if (pche != NULL)
1373                         StrBufCutAt(rssc.Key, -1, pche);
1374                 else
1375                         ptr = "UTF-8";
1376         }
1377         else
1378                 ptr = "UTF-8";
1379
1380
1381         xp = XML_ParserCreateNS(ptr, ':');
1382         if (!xp) {
1383                 CtdlLogPrintf(CTDL_ALERT, "Cannot create XML parser!\n");
1384                 goto shutdown;
1385         }
1386         FlushStrBuf(rssc.Key);
1387 //#ifdef HAVE_ICONV
1388 #if 0
1389         XML_SetUnknownEncodingHandler(xp,
1390                                       handle_unknown_xml_encoding,
1391                                       &rssc);
1392 #endif
1393 //#endif
1394         XML_SetElementHandler(xp, rss_xml_start, rss_xml_end);
1395         XML_SetCharacterDataHandler(xp, rss_xml_chardata);
1396         XML_SetUserData(xp, &rssc);
1397         XML_SetCdataSectionHandler(xp,
1398                                    rss_xml_cdata_start,
1399                                    rss_xml_cdata_end);
1400
1401
1402         len = StrLength(Answer);
1403         ptr = SmashStrBuf(&Answer);
1404         XML_Parse(xp, ptr, len, 0);
1405         free (ptr);
1406         if (ri.done_parsing == 0)
1407                 XML_Parse(xp, "", 0, 1);
1408
1409
1410         CtdlLogPrintf(CTDL_ALERT, "RSS: XML Status [%s] \n",
1411                       XML_ErrorString(
1412                               XML_GetErrorCode(xp)));
1413
1414 shutdown:
1415         curl_easy_cleanup(curl);
1416         XML_ParserFree(xp);
1417
1418         flush_rss_item(&ri);
1419         FreeStrBuf(&rssc.CData);
1420         FreeStrBuf(&rssc.Key);
1421 }
1422
1423
1424 /*
1425  * Scan a room's netconfig to determine whether it is requesting any RSS feeds
1426  */
1427 void rssclient_scan_room(struct ctdlroom *qrbuf, void *data)
1428 {
1429         char filename[PATH_MAX];
1430         char buf[1024];
1431         char instr[32];
1432         FILE *fp;
1433         char feedurl[256];
1434         rssnetcfg *rncptr = NULL;
1435         rssnetcfg *use_this_rncptr = NULL;
1436         int len = 0;
1437         char *ptr = NULL;
1438
1439         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
1440
1441         if (CtdlThreadCheckStop())
1442                 return;
1443
1444         /* Only do net processing for rooms that have netconfigs */
1445         fp = fopen(filename, "r");
1446         if (fp == NULL) {
1447                 return;
1448         }
1449
1450         while (fgets(buf, sizeof buf, fp) != NULL && !CtdlThreadCheckStop()) {
1451                 buf[strlen(buf)-1] = 0;
1452
1453                 extract_token(instr, buf, 0, '|', sizeof instr);
1454                 if (!strcasecmp(instr, "rssclient")) {
1455
1456                         use_this_rncptr = NULL;
1457
1458                         extract_token(feedurl, buf, 1, '|', sizeof feedurl);
1459
1460                         /* If any other rooms have requested the same feed, then we will just add this
1461                          * room to the target list for that client request.
1462                          */
1463                         for (rncptr=rnclist; rncptr!=NULL; rncptr=rncptr->next) {
1464                                 if (!strcmp(rncptr->url, feedurl)) {
1465                                         use_this_rncptr = rncptr;
1466                                 }
1467                         }
1468
1469                         /* Otherwise create a new client request */
1470                         if (use_this_rncptr == NULL) {
1471                                 rncptr = (rssnetcfg *) malloc(sizeof(rssnetcfg));
1472                                 rncptr->ItemType = RSS_UNSET;
1473                                 if (rncptr != NULL) {
1474                                         rncptr->next = rnclist;
1475                                         safestrncpy(rncptr->url, feedurl, sizeof rncptr->url);
1476                                         rncptr->rooms = NULL;
1477                                         rnclist = rncptr;
1478                                         use_this_rncptr = rncptr;
1479                                 }
1480                         }
1481
1482                         /* Add the room name to the request */
1483                         if (use_this_rncptr != NULL) {
1484                                 if (use_this_rncptr->rooms == NULL) {
1485                                         rncptr->rooms = strdup(qrbuf->QRname);
1486                                 }
1487                                 else {
1488                                         len = strlen(use_this_rncptr->rooms) + strlen(qrbuf->QRname) + 5;
1489                                         ptr = realloc(use_this_rncptr->rooms, len);
1490                                         if (ptr != NULL) {
1491                                                 strcat(ptr, "|");
1492                                                 strcat(ptr, qrbuf->QRname);
1493                                                 use_this_rncptr->rooms = ptr;
1494                                         }
1495                                 }
1496                         }
1497                 }
1498
1499         }
1500
1501         fclose(fp);
1502
1503 }
1504
1505 /*
1506  * Scan for rooms that have RSS client requests configured
1507  */
1508 void rssclient_scan(void) {
1509         static time_t last_run = 0L;
1510         static int doing_rssclient = 0;
1511         rssnetcfg *rptr = NULL;
1512
1513         /*
1514          * This is a simple concurrency check to make sure only one rssclient run
1515          * is done at a time.  We could do this with a mutex, but since we
1516          * don't really require extremely fine granularity here, we'll do it
1517          * with a static variable instead.
1518          */
1519         if (doing_rssclient) return;
1520         doing_rssclient = 1;
1521
1522         CtdlLogPrintf(CTDL_DEBUG, "rssclient started\n");
1523         CtdlForEachRoom(rssclient_scan_room, NULL);
1524
1525         while (rnclist != NULL && !CtdlThreadCheckStop()) {
1526                 rss_do_fetching(rnclist);
1527                 rptr = rnclist;
1528                 rnclist = rnclist->next;
1529                 if (rptr->rooms != NULL) free(rptr->rooms);
1530                 free(rptr);
1531         }
1532
1533         CtdlLogPrintf(CTDL_DEBUG, "rssclient ended\n");
1534         last_run = time(NULL);
1535         doing_rssclient = 0;
1536         return;
1537 }
1538
1539
1540 CTDL_MODULE_INIT(rssclient)
1541 {
1542         if (threading)
1543         {
1544                 CtdlLogPrintf(CTDL_INFO, "%s\n", curl_version());
1545                 CtdlRegisterSessionHook(rssclient_scan, EVT_TIMER);
1546         }
1547
1548         StartHandlers = NewHash(1, NULL);
1549         EndHandlers = NewHash(1, NULL);
1550
1551         AddRSSStartHandler(RSS_item_rss_start,     RSS_UNSET, HKEY("rss"));
1552         AddRSSStartHandler(RSS_item_rdf_start,     RSS_UNSET, HKEY("rdf"));
1553         AddRSSStartHandler(ATOM_item_feed_start,    RSS_UNSET, HKEY("feed"));
1554         AddRSSStartHandler(RSS_item_item_start,    RSS_RSS, HKEY("item"));
1555         AddRSSStartHandler(ATOM_item_entry_start,  RSS_ATOM, HKEY("entry"));
1556         AddRSSStartHandler(ATOM_item_link_start,   RSS_ATOM, HKEY("link"));
1557
1558         AddRSSEndHandler(ATOMRSS_item_title_end,   RSS_ATOM|RSS_RSS|RSS_REQUIRE_BUF, HKEY("title"));
1559         AddRSSEndHandler(RSS_item_guid_end,        RSS_RSS|RSS_REQUIRE_BUF, HKEY("guid"));
1560         AddRSSEndHandler(ATOM_item_id_end,         RSS_ATOM|RSS_REQUIRE_BUF, HKEY("id"));
1561         AddRSSEndHandler(RSS_item_link_end,        RSS_RSS|RSS_REQUIRE_BUF, HKEY("link"));
1562 #if 0
1563 // hm, rss to the comments of that blog, might be interesting in future, but...
1564         AddRSSEndHandler(RSS_item_relink_end,      RSS_RSS|RSS_REQUIRE_BUF, HKEY("commentrss"));
1565 // comment count...
1566         AddRSSEndHandler(RSS_item_relink_end,      RSS_RSS|RSS_REQUIRE_BUF, HKEY("comments"));
1567 #endif
1568         AddRSSEndHandler(RSSATOM_item_title_end,   RSS_ATOM|RSS_RSS|RSS_REQUIRE_BUF, HKEY("title"));
1569         AddRSSEndHandler(ATOM_item_content_end,    RSS_ATOM|RSS_REQUIRE_BUF, HKEY("content"));
1570         AddRSSEndHandler(RSS_item_description_end, RSS_RSS|RSS_ATOM|RSS_REQUIRE_BUF, HKEY("encoded"));
1571         AddRSSEndHandler(ATOM_item_summary_end,    RSS_ATOM|RSS_REQUIRE_BUF, HKEY("summary"));
1572         AddRSSEndHandler(RSS_item_description_end, RSS_RSS|RSS_REQUIRE_BUF, HKEY("description"));
1573         AddRSSEndHandler(ATOM_item_published_end,  RSS_ATOM|RSS_REQUIRE_BUF, HKEY("published"));
1574         AddRSSEndHandler(ATOM_item_updated_end,    RSS_ATOM|RSS_REQUIRE_BUF, HKEY("updated"));
1575         AddRSSEndHandler(RSS_item_pubdate_end,     RSS_RSS|RSS_REQUIRE_BUF, HKEY("pubdate"));
1576         AddRSSEndHandler(RSS_item_date_end,        RSS_RSS|RSS_REQUIRE_BUF, HKEY("date"));
1577         AddRSSEndHandler(RSS_item_author_end,      RSS_RSS|RSS_REQUIRE_BUF, HKEY("author"));
1578         AddRSSEndHandler(RSS_item_creator_end,     RSS_RSS|RSS_REQUIRE_BUF, HKEY("creator"));
1579 /* <author> */
1580         AddRSSEndHandler(ATOM_item_email_end,      RSS_ATOM|RSS_REQUIRE_BUF, HKEY("email"));
1581         AddRSSEndHandler(ATOM_item_name_end,       RSS_ATOM|RSS_REQUIRE_BUF, HKEY("name"));
1582         AddRSSEndHandler(ATOM_item_uri_end,        RSS_ATOM|RSS_REQUIRE_BUF, HKEY("uri"));
1583 /* </author> */
1584         AddRSSEndHandler(RSS_item_item_end,        RSS_RSS, HKEY("item"));
1585         AddRSSEndHandler(RSS_item_rss_end,         RSS_RSS, HKEY("rss"));
1586         AddRSSEndHandler(RSS_item_rdf_end,         RSS_RSS, HKEY("rdf"));
1587         AddRSSEndHandler(ATOM_item_entry_end,      RSS_ATOM, HKEY("entry"));
1588
1589
1590 /* at the start of atoms: <seq> <li>link to resource</li></seq> ignore them. */
1591         AddRSSStartHandler(RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("seq"));
1592         AddRSSEndHandler  (RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("seq"));
1593         AddRSSStartHandler(RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("li"));
1594         AddRSSEndHandler  (RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("li"));
1595
1596 /* links to other feed generators... */
1597         AddRSSStartHandler(RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("feedflare"));
1598         AddRSSEndHandler  (RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("feedflare"));
1599         AddRSSStartHandler(RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("browserfriendly"));
1600         AddRSSEndHandler  (RSSATOM_item_ignore,      RSS_RSS|RSS_ATOM, HKEY("browserfriendly"));
1601
1602         KnownNameSpaces = NewHash(1, NULL);
1603         Put(KnownNameSpaces, HKEY("http://a9.com/-/spec/opensearch/1.1/"), NULL, reference_free_handler);
1604         Put(KnownNameSpaces, HKEY("http://a9.com/-/spec/opensearchrss/1.0/"), NULL, reference_free_handler);
1605         Put(KnownNameSpaces, HKEY("http://backend.userland.com/creativeCommonsRssModule"), NULL, reference_free_handler);
1606         Put(KnownNameSpaces, HKEY("http://purl.org/atom/ns#"), NULL, reference_free_handler);
1607         Put(KnownNameSpaces, HKEY("http://purl.org/dc/elements/1.1/"), NULL, reference_free_handler);
1608         Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/"), NULL, reference_free_handler);
1609         Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/modules/content/"), NULL, reference_free_handler);
1610         Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/modules/slash/"), NULL, reference_free_handler);
1611         Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/modules/syndication/"), NULL, reference_free_handler);
1612         Put(KnownNameSpaces, HKEY("http://purl.org/rss/1.0/"), NULL, reference_free_handler);
1613         Put(KnownNameSpaces, HKEY("http://purl.org/syndication/thread/1.0"), NULL, reference_free_handler);
1614         Put(KnownNameSpaces, HKEY("http://rssnamespace.org/feedburner/ext/1.0"), NULL, reference_free_handler);
1615         Put(KnownNameSpaces, HKEY("http://schemas.google.com/g/2005"), NULL, reference_free_handler);
1616         Put(KnownNameSpaces, HKEY("http://webns.net/mvcb/"), NULL, reference_free_handler);
1617         Put(KnownNameSpaces, HKEY("http://web.resource.org/cc/"), NULL, reference_free_handler);
1618         Put(KnownNameSpaces, HKEY("http://wellformedweb.org/CommentAPI/"), NULL, reference_free_handler);
1619         Put(KnownNameSpaces, HKEY("http://www.georss.org/georss"), NULL, reference_free_handler);
1620         Put(KnownNameSpaces, HKEY("http://www.w3.org/1999/xhtml"), NULL, reference_free_handler);
1621         Put(KnownNameSpaces, HKEY("http://www.w3.org/1999/02/22-rdf-syntax-ns#"), NULL, reference_free_handler);
1622         Put(KnownNameSpaces, HKEY("http://www.w3.org/1999/02/22-rdf-syntax-ns#"), NULL, reference_free_handler);
1623         Put(KnownNameSpaces, HKEY("http://www.w3.org/2003/01/geo/wgs84_pos#"), NULL, reference_free_handler);
1624         Put(KnownNameSpaces, HKEY("http://www.w3.org/2005/Atom"), NULL, reference_free_handler);
1625         Put(KnownNameSpaces, HKEY("urn:flickr:"), NULL, reference_free_handler);
1626 #if 0
1627         /* we don't like these namespaces because of they shadow our usefull parameters. */
1628         Put(KnownNameSpaces, HKEY("http://search.yahoo.com/mrss/"), NULL, reference_free_handler);
1629 #endif
1630         return "rssclient";
1631 }