citadel/server/modules/rssclient/serv_rssclient.c

   1 // Bring external RSS and/or Atom feeds into rooms.  This module implements a
   2 // very loose parser that scrapes both kinds of feeds and is not picky about
   3 // the standards compliance of the source data.
   4 //
   5 // Copyright (c) 2007-2023 by the citadel.org team
   6 //
   7 // This program is open source software.  Use, duplication, or disclosure
   8 // is subject to the terms of the GNU General Public License, version 3.
   9
  10 #include <stdlib.h>
  11 #include <unistd.h>
  12 #include <stdio.h>
  13 #include <time.h>
  14 #include <ctype.h>
  15 #include <string.h>
  16 #include <errno.h>
  17 #include <sys/types.h>
  18 #include <sys/stat.h>
  19 #include <expat.h>
  20 #include <curl/curl.h>
  21 #include <libcitadel.h>
  22 #include "../../citadel_defs.h"
  23 #include "../../server.h"
  24 #include "../../citserver.h"
  25 #include "../../support.h"
  26 #include "../../config.h"
  27 #include "../../threads.h"
  28 #include "../../ctdl_module.h"
  29 #include "../../msgbase.h"
  30 #include "../../parsedate.h"
  31 #include "../../database.h"
  32 #include "../../citadel_dirs.h"
  33 #include "../../context.h"
  34 #include "../../internet_addressing.h"
  35
  36 struct rssfeed {
  37         char url[SIZ];                  // string containing the URL of an RSS or Atom feed
  38         char room[ROOMNAMELEN];         // the name of a room which is pulling this feed
  39 };
  40
  41 struct rssparser {
  42         char url[SIZ];
  43         StrBuf *CData;
  44         struct CtdlMessage *msg;
  45         char *link;
  46         char *description;
  47         char *item_id;
  48 };
  49
  50 Array *feeds = NULL;
  51 time_t last_run = 0L;
  52
  53
  54 // This handler is called whenever an XML tag opens.
  55 void rss_start_element(void *data, const char *el, const char **attribute) {
  56         struct rssparser *r = (struct rssparser *)data;
  57         int i;
  58
  59         if (server_shutting_down) return;                       // shunt the whole operation if we're exiting
  60
  61         if (
  62                 (!strcasecmp(el, "entry"))
  63                 || (!strcasecmp(el, "item"))
  64         ) {
  65                 // this is the start of a new item(rss) or entry(atom)
  66                 if (r->msg != NULL) {
  67                         CM_Free(r->msg);
  68                         r->msg = NULL;
  69                 }
  70                 r->msg = malloc(sizeof(struct CtdlMessage));
  71                 memset(r->msg, 0, sizeof(struct CtdlMessage));
  72                 r->msg->cm_magic = CTDLMESSAGE_MAGIC;
  73                 r->msg->cm_anon_type = MES_NORMAL;
  74                 r->msg->cm_format_type = FMT_RFC822;
  75         }
  76
  77         else if (!strcasecmp(el, "link")) {                     // atom feeds have the link as an attribute
  78                 for(i = 0; attribute[i]; i += 2) {
  79                         if (!strcasecmp(attribute[i], "href")) {
  80                                 if (r->link != NULL) {
  81                                         free(r->link);
  82                                         r->link = NULL;
  83                                 }
  84                                 r->link = strdup(attribute[i+1]);
  85                                 string_trim(r->link);
  86                         }
  87                 }
  88         }
  89 }
  90
  91
  92 // This handler is called whenever an XML tag closes.
  93 void rss_end_element(void *data, const char *el) {
  94         struct rssparser *r = (struct rssparser *)data;
  95         StrBuf *encoded_field;
  96         long msgnum;
  97
  98         if (server_shutting_down) return;                       // shunt the whole operation if we're exiting
  99
 100         if (StrLength(r->CData) > 0) {                          // strip leading/trailing whitespace from field
 101                 StrBufTrim(r->CData);
 102         }
 103
 104         if (                                                    // end of a new item(rss) or entry(atom)
 105                 (!strcasecmp(el, "entry"))
 106                 || (!strcasecmp(el, "item"))
 107         ) {
 108                 if (r->msg != NULL) {                           // Save the message to the rooms
 109
 110                         // use the link as an item id if nothing else is available
 111                         if ((r->item_id == NULL) && (r->link != NULL)) {
 112                                 r->item_id = strdup(r->link);
 113                         }
 114
 115                         // check the use table
 116                         StrBuf *u = NewStrBuf();
 117                         StrBufAppendPrintf(u, "rss/%s", r->item_id);
 118                         int already_seen = CheckIfAlreadySeen(u);
 119                         FreeStrBuf(&u);
 120
 121                         if (already_seen == 0) {
 122
 123                                 // Compose the message text
 124
 125                                 StrBuf *TheMessage = NewStrBuf();
 126                                 StrBufAppendPrintf(TheMessage, "<html><head></head><body>");
 127
 128                                 if (r->description != NULL) {
 129                                         StrBufAppendPrintf(TheMessage, "%s<br><br>\r\n", r->description);
 130                                         free(r->description);
 131                                         r->description = NULL;
 132                                 }
 133
 134                                 if (r->link != NULL) {
 135                                         StrBufAppendPrintf(TheMessage, "<a href=\"%s\">%s</a>\r\n", r->link, r->link);
 136                                         free(r->link);
 137                                         r->link = NULL;
 138                                 }
 139
 140                                 StrBufAppendPrintf(TheMessage, "</body></html>\r\n");
 141
 142                                 // Quoted-Printable encode the HTML message, because RSS and Atom make no guarantee of line length limits.
 143                                 StrBuf *TheMessage_Encoded = StrBufQuotedPrintableEncode(TheMessage);
 144
 145                                 // Now we reuse TheMessage -- this time it will contain the MIME headers concatenated with the encoded message.
 146                                 FlushStrBuf(TheMessage);
 147                                 StrBufAppendBufPlain(TheMessage, HKEY(
 148                                         "Content-type: text/html; charset=UTF-8\r\n"
 149                                         "Content-Transfer-Encoding: quoted-printable\r\n"
 150                                         "\r\n"
 151                                         ), 0
 152                                 );
 153                                 StrBufAppendBuf(TheMessage, TheMessage_Encoded, 0);
 154                                 FreeStrBuf(&TheMessage_Encoded);
 155
 156                                 CM_SetField(r->msg, eMesageText, ChrPtr(TheMessage), StrLength(TheMessage));
 157                                 FreeStrBuf(&TheMessage);
 158
 159                                 if (CM_IsEmpty(r->msg, eAuthor)) {
 160                                         CM_SetField(r->msg, eAuthor, HKEY("rss"));
 161                                 }
 162
 163                                 if (CM_IsEmpty(r->msg, eTimestamp)) {
 164                                         CM_SetFieldLONG(r->msg, eTimestamp, time(NULL));
 165                                 }
 166
 167                                 msgnum = -1 ;
 168                                 for (int i=0; i<array_len(feeds); ++i) {
 169                                         struct rssfeed *rf = (struct rssfeed *) array_get_element_at(feeds, i);
 170                                         if (!strcmp(rf->url, r->url)) {
 171                                                 if (msgnum < 0) {
 172                                                         // Save the item in a room
 173                                                         msgnum = CtdlSubmitMsg(r->msg, NULL, rf->room);
 174                                                 }
 175                                                 else {
 176                                                         // If the same item goes to multiple rooms, just save a pointer
 177                                                         CtdlSaveMsgPointerInRoom(rf->room, msgnum, 0, NULL);
 178                                                 }
 179                                         }
 180                                 }
 181                         }
 182                         else {
 183                                 syslog(LOG_DEBUG, "rssclient: already seen %s", r->item_id);
 184                         }
 185
 186                         CM_Free(r->msg);
 187                         r->msg = NULL;
 188                 }
 189
 190                 if (r->item_id != NULL) {
 191                         free(r->item_id);
 192                         r->item_id = NULL;
 193                 }
 194         }
 195
 196         else if (!strcasecmp(el, "title")) {                    // item subject (rss and atom)
 197                 if ((r->msg != NULL) && (CM_IsEmpty(r->msg, eMsgSubject))) {
 198                         encoded_field = NewStrBuf();
 199                         StrBufRFC2047encode(&encoded_field, r->CData);
 200                         CM_SetAsFieldSB(r->msg, eMsgSubject, &encoded_field);
 201                 }
 202         }
 203
 204         else if (!strcasecmp(el, "creator")) {                  // <creator> can be used if <author> is not present
 205                 if ((r->msg != NULL) && (CM_IsEmpty(r->msg, eAuthor))) {
 206                         encoded_field = NewStrBuf();
 207                         StrBufRFC2047encode(&encoded_field, r->CData);
 208                         CM_SetAsFieldSB(r->msg, eAuthor, &encoded_field);
 209                 }
 210         }
 211
 212         else if (!strcasecmp(el, "author")) {                   // <author> supercedes <creator> if both are present
 213                 if (r->msg != NULL) {
 214                         encoded_field = NewStrBuf();
 215                         StrBufRFC2047encode(&encoded_field, r->CData);
 216                         CM_SetAsFieldSB(r->msg, eAuthor, &encoded_field);
 217                 }
 218         }
 219
 220         else if (!strcasecmp(el, "pubdate")) {                  // date/time stamp (rss) Sat, 25 Feb 2017 14:28:01 EST
 221                 if ((r->msg)&&(r->msg->cm_fields[eTimestamp]==NULL)) {
 222                         CM_SetFieldLONG(r->msg, eTimestamp, parsedate(ChrPtr(r->CData)));
 223                 }
 224         }
 225
 226         else if (!strcasecmp(el, "updated")) {                  // date/time stamp (atom) 2003-12-13T18:30:02Z
 227                 if ((r->msg)&&(r->msg->cm_fields[eTimestamp]==NULL)) {
 228                         struct tm t;
 229                         char zulu;
 230                         memset(&t, 0, sizeof t);
 231                         sscanf(ChrPtr(r->CData), "%d-%d-%dT%d:%d:%d%c", &t.tm_year, &t.tm_mon, &t.tm_mday, &t.tm_hour, &t.tm_min, &t.tm_sec, &zulu);
 232                         t.tm_year -= 1900;
 233                         t.tm_mon -= 1;
 234                         CM_SetFieldLONG(r->msg, eTimestamp, mktime(&t));
 235                 }
 236         }
 237
 238         else if (!strcasecmp(el, "link")) {                     // link to story (rss)
 239                 if (r->link != NULL) {
 240                         free(r->link);
 241                         r->link = NULL;
 242                 }
 243                 r->link = strdup(ChrPtr(r->CData));
 244         }
 245
 246         else if (
 247                 (!strcasecmp(el, "guid"))                       // unique item id (rss)
 248                 || (!strcasecmp(el, "id"))                      // unique item id (atom)
 249         ) {
 250                 if (r->item_id != NULL) {
 251                         free(r->item_id);
 252                         r->item_id = NULL;
 253                 }
 254                 r->item_id = strdup(ChrPtr(r->CData));
 255         }
 256
 257         else if (
 258                 (!strcasecmp(el, "description"))                // message text (rss)
 259                 || (!strcasecmp(el, "summary"))                 // message text (atom)
 260                 || (!strcasecmp(el, "content"))                 // message text (atom)
 261         ) {
 262                 if (r->description != NULL) {
 263                         free(r->description);
 264                         r->description = NULL;
 265                 }
 266                 r->description = strdup(ChrPtr(r->CData));
 267         }
 268
 269         if (r->CData != NULL) {
 270                 FreeStrBuf(&r->CData);
 271                 r->CData = NULL;
 272         }
 273 }
 274
 275
 276 // This handler is called whenever data appears between opening and closing tags.
 277 void rss_handle_data(void *data, const char *content, int length) {
 278         struct rssparser *r = (struct rssparser *)data;
 279
 280         if (r->CData == NULL) {
 281                 r->CData = NewStrBuf();
 282         }
 283
 284         StrBufAppendBufPlain(r->CData, content, length, 0);
 285 }
 286
 287
 288 // Feed has been downloaded, now parse it.
 289 // `Feed` is the actual RSS downloaded from the site.
 290 // `url` is a string containing the feed URL
 291 void rss_parse_feed(StrBuf *Feed, char *url) {
 292         struct rssparser r;
 293
 294         memset(&r, 0, sizeof r);
 295         strcpy(r.url, url);
 296         XML_Parser p = XML_ParserCreate("UTF-8");
 297         XML_SetElementHandler(p, rss_start_element, rss_end_element);
 298         XML_SetCharacterDataHandler(p, rss_handle_data);
 299         XML_SetUserData(p, (void *)&r);
 300         XML_Parse(p, ChrPtr(Feed), StrLength(Feed), XML_TRUE);
 301         XML_ParserFree(p);
 302 }
 303
 304
 305 // pull the first feed in the list
 306 void rss_pull_one_feed(char *url) {
 307         CURL *curl;
 308         CURLcode res;
 309         StrBuf *Downloaded = NULL;
 310
 311         curl = curl_easy_init();
 312         if (!curl) {
 313                 return;
 314         }
 315
 316         Downloaded = NewStrBuf();
 317
 318         syslog(LOG_DEBUG, "rssclient: fetching %s", url);
 319         curl_easy_setopt(curl, CURLOPT_URL, url);
 320         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
 321         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
 322         curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);                     // Follow redirects
 323         curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlFillStrBuf_callback); // What to do with downloaded data
 324         curl_easy_setopt(curl, CURLOPT_WRITEDATA, Downloaded);                  // Give it our StrBuf to work with
 325         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L);                           // Time out after 20 seconds
 326         res = curl_easy_perform(curl);                                          // Perform the request
 327         if (res != CURLE_OK) {
 328                 syslog(LOG_WARNING, "rssclient: failed to load feed: %s", curl_easy_strerror(res));
 329         }
 330         curl_easy_cleanup(curl);
 331
 332         rss_parse_feed(Downloaded, url);
 333         FreeStrBuf(&Downloaded);                                                // free the downloaded feed data
 334 }
 335
 336
 337 // We have a list, now download the feeds
 338 void rss_pull_feeds(void) {
 339         char url[SIZ];
 340
 341         while (array_len(feeds) > 0) {
 342                 struct rssfeed *r = (struct rssfeed *) array_get_element_at(feeds, 0);
 343                 strcpy(url, r->url);
 344                 rss_pull_one_feed(url);
 345                 while (r = array_get_element_at(feeds, 0), (r && !strcmp(r->url, url))) {
 346                         array_delete_element_at(feeds, 0);
 347                 }
 348         }
 349
 350 }
 351
 352
 353 // Scan a room's netconfig looking for RSS feed parsing requests
 354 void rssclient_scan_room(struct ctdlroom *qrbuf, void *data) {
 355         char *serialized_config = NULL;
 356         int num_configs = 0;
 357         char cfgline[SIZ];
 358         struct rssfeed one_feed;
 359         int i = 0;
 360
 361         if (server_shutting_down) return;
 362
 363         serialized_config = LoadRoomNetConfigFile(qrbuf->QRnumber);
 364         if (!serialized_config) {
 365                 return;
 366         }
 367
 368         num_configs = num_tokens(serialized_config, '\n');
 369         for (i=0; i<num_configs; ++i) {
 370                 extract_token(cfgline, serialized_config, i, '\n', sizeof cfgline);
 371                 if (!strncasecmp(cfgline, HKEY("rssclient|"))) {
 372                         strcpy(cfgline, &cfgline[10]);
 373                         char *vbar = strchr(cfgline, '|');
 374                         if (vbar != NULL) {
 375                                 *vbar = 0;
 376                         }
 377                         safestrncpy(one_feed.url, cfgline, SIZ);
 378                         safestrncpy(one_feed.room, qrbuf->QRname, ROOMNAMELEN);
 379                         array_append(feeds, &one_feed);
 380                 }
 381         }
 382
 383         free(serialized_config);
 384 }
 385
 386
 387 // Scan for rooms that have RSS client requests configured
 388 void rssclient_scan(void) {
 389         time_t now = time(NULL);
 390
 391         // Run no more than once every 15 minutes.
 392         if ((now - last_run) < 900) {
 393                 syslog(LOG_DEBUG,
 394                         "rssclient: polling interval not yet reached; last run was %ldm%lds ago",
 395                         ((now - last_run) / 60),
 396                         ((now - last_run) % 60)
 397                 );
 398                 return;
 399         }
 400
 401         syslog(LOG_DEBUG, "rssclient: started");
 402         feeds = array_new(sizeof(struct rssfeed));
 403         if (feeds == NULL) {
 404                 syslog(LOG_DEBUG, "rssclient: cannot allocate memory for feed list");
 405                 return;
 406         }
 407         CtdlForEachRoom(rssclient_scan_room, NULL);
 408         array_sort(feeds, (int (*)(const void *, const void *))strcmp);
 409         rss_pull_feeds();
 410         array_free(feeds);
 411         syslog(LOG_DEBUG, "rssclient: ended");
 412         last_run = time(NULL);
 413         return;
 414 }
 415
 416
 417 // Initialization function, called from modules_init.c
 418 char *ctdl_module_init_rssclient(void) {
 419         if (!threading) {
 420                 syslog(LOG_INFO, "rssclient: using %s", curl_version());
 421                 CtdlRegisterSessionHook(rssclient_scan, EVT_TIMER, PRIO_AGGR + 300);
 422         }
 423         return "rssclient";
 424 }