citadel/server/modules/rssclient/serv_rssclient.c

   1 // Bring external RSS and/or Atom feeds into rooms.  This module implements a
   2 // very loose parser that scrapes both kinds of feeds and is not picky about
   3 // the standards compliance of the source data.
   4 //
   5 // Copyright (c) 2007-2023 by the citadel.org team
   6 //
   7 // This program is open source software.  Use, duplication, or disclosure
   8 // is subject to the terms of the GNU General Public License, version 3.
   9
  10 #include <stdlib.h>
  11 #include <unistd.h>
  12 #include <stdio.h>
  13 #include <time.h>
  14 #include <ctype.h>
  15 #include <string.h>
  16 #include <errno.h>
  17 #include <sys/types.h>
  18 #include <sys/stat.h>
  19 #include <expat.h>
  20 #include <curl/curl.h>
  21 #include <libcitadel.h>
  22 #include "../../citadel_defs.h"
  23 #include "../../server.h"
  24 #include "../../citserver.h"
  25 #include "../../support.h"
  26 #include "../../config.h"
  27 #include "../../threads.h"
  28 #include "../../ctdl_module.h"
  29 #include "../../msgbase.h"
  30 #include "../../parsedate.h"
  31 #include "../../database.h"
  32 #include "../../citadel_dirs.h"
  33 #include "../../context.h"
  34 #include "../../internet_addressing.h"
  35
  36 struct rssroom {
  37         struct rssroom *next;
  38         char *room;
  39 };
  40
  41 struct rssurl {
  42         struct rssurl *next;
  43         char *url;
  44         struct rssroom *rooms;
  45 };
  46
  47 struct rssparser {
  48         StrBuf *CData;
  49         struct CtdlMessage *msg;
  50         char *link;
  51         char *description;
  52         char *item_id;
  53         struct rssroom *rooms;
  54 };
  55
  56 time_t last_run = 0L;
  57 struct rssurl *rsstodo = NULL;
  58
  59
  60 // This handler is called whenever an XML tag opens.
  61 void rss_start_element(void *data, const char *el, const char **attribute) {
  62         struct rssparser *r = (struct rssparser *)data;
  63         int i;
  64
  65         if (server_shutting_down) return;                       // shunt the whole operation if we're exiting
  66
  67         if (
  68                 (!strcasecmp(el, "entry"))
  69                 || (!strcasecmp(el, "item"))
  70         ) {
  71                 // this is the start of a new item(rss) or entry(atom)
  72                 if (r->msg != NULL) {
  73                         CM_Free(r->msg);
  74                         r->msg = NULL;
  75                 }
  76                 r->msg = malloc(sizeof(struct CtdlMessage));
  77                 memset(r->msg, 0, sizeof(struct CtdlMessage));
  78                 r->msg->cm_magic = CTDLMESSAGE_MAGIC;
  79                 r->msg->cm_anon_type = MES_NORMAL;
  80                 r->msg->cm_format_type = FMT_RFC822;
  81         }
  82
  83         else if (!strcasecmp(el, "link")) {                     // atom feeds have the link as an attribute
  84                 for(i = 0; attribute[i]; i += 2) {
  85                         if (!strcasecmp(attribute[i], "href")) {
  86                                 if (r->link != NULL) {
  87                                         free(r->link);
  88                                         r->link = NULL;
  89                                 }
  90                                 r->link = strdup(attribute[i+1]);
  91                                 string_trim(r->link);
  92                         }
  93                 }
  94         }
  95 }
  96
  97
  98 // This handler is called whenever an XML tag closes.
  99 void rss_end_element(void *data, const char *el) {
 100         struct rssparser *r = (struct rssparser *)data;
 101         StrBuf *encoded_field;
 102
 103         if (server_shutting_down) return;                       // shunt the whole operation if we're exiting
 104
 105         if (StrLength(r->CData) > 0) {                          // strip leading/trailing whitespace from field
 106                 StrBufTrim(r->CData);
 107         }
 108
 109         if (                                                    // end of a new item(rss) or entry(atom)
 110                 (!strcasecmp(el, "entry"))
 111                 || (!strcasecmp(el, "item"))
 112         ) {
 113                 if (r->msg != NULL) {                           // Save the message to the rooms
 114
 115                         // use the link as an item id if nothing else is available
 116                         if ((r->item_id == NULL) && (r->link != NULL)) {
 117                                 r->item_id = strdup(r->link);
 118                         }
 119
 120                         // check the use table
 121                         StrBuf *u = NewStrBuf();
 122                         StrBufAppendPrintf(u, "rss/%s", r->item_id);
 123                         int already_seen = CheckIfAlreadySeen(u);
 124                         FreeStrBuf(&u);
 125
 126                         if (already_seen == 0) {
 127
 128                                 // Compose the message text
 129
 130                                 StrBuf *TheMessage = NewStrBuf();
 131                                 StrBufAppendPrintf(TheMessage, "<html><head></head><body>");
 132
 133                                 if (r->description != NULL) {
 134                                         StrBufAppendPrintf(TheMessage, "%s<br><br>\r\n", r->description);
 135                                         free(r->description);
 136                                         r->description = NULL;
 137                                 }
 138
 139                                 if (r->link != NULL) {
 140                                         StrBufAppendPrintf(TheMessage, "<a href=\"%s\">%s</a>\r\n", r->link, r->link);
 141                                         free(r->link);
 142                                         r->link = NULL;
 143                                 }
 144
 145                                 StrBufAppendPrintf(TheMessage, "</body></html>\r\n");
 146
 147                                 // Quoted-Printable encode the HTML message, because RSS and Atom make no guarantee of line length limits.
 148                                 StrBuf *TheMessage_Encoded = StrBufQuotedPrintableEncode(TheMessage);
 149
 150                                 // Now we reuse TheMessage -- this time it will contain the MIME headers concatenated with the encoded message.
 151                                 FlushStrBuf(TheMessage);
 152                                 StrBufAppendBufPlain(TheMessage, HKEY(
 153                                         "Content-type: text/html; charset=UTF-8\r\n"
 154                                         "Content-Transfer-Encoding: quoted-printable\r\n"
 155                                         "\r\n"
 156                                         ), 0
 157                                 );
 158                                 StrBufAppendBuf(TheMessage, TheMessage_Encoded, 0);
 159                                 FreeStrBuf(&TheMessage_Encoded);
 160
 161                                 CM_SetField(r->msg, eMesageText, ChrPtr(TheMessage), StrLength(TheMessage));
 162                                 FreeStrBuf(&TheMessage);
 163
 164                                 if (CM_IsEmpty(r->msg, eAuthor)) {
 165                                         CM_SetField(r->msg, eAuthor, HKEY("rss"));
 166                                 }
 167
 168                                 if (CM_IsEmpty(r->msg, eTimestamp)) {
 169                                         CM_SetFieldLONG(r->msg, eTimestamp, time(NULL));
 170                                 }
 171
 172                                 // Save it to the room(s)
 173                                 struct rssroom *rr = NULL;
 174                                 long msgnum = (-1);
 175                                 for (rr=r->rooms; rr!=NULL; rr=rr->next) {
 176                                         if (rr == r->rooms) {
 177                                                 msgnum = CtdlSubmitMsg(r->msg, NULL, rr->room);         // in first room, save msg
 178                                         }
 179                                         else {
 180                                                 CtdlSaveMsgPointerInRoom(rr->room, msgnum, 0, NULL);    // elsewhere, save a pointer
 181                                         }
 182                                         syslog(LOG_DEBUG, "rssclient: saved message %ld to %s", msgnum, rr->room);
 183                                 }
 184                         }
 185                         else {
 186                                 syslog(LOG_DEBUG, "rssclient: already seen %s", r->item_id);
 187                         }
 188
 189                         CM_Free(r->msg);
 190                         r->msg = NULL;
 191                 }
 192
 193                 if (r->item_id != NULL) {
 194                         free(r->item_id);
 195                         r->item_id = NULL;
 196                 }
 197         }
 198
 199         else if (!strcasecmp(el, "title")) {                    // item subject (rss and atom)
 200                 if ((r->msg != NULL) && (CM_IsEmpty(r->msg, eMsgSubject))) {
 201                         encoded_field = NewStrBuf();
 202                         StrBufRFC2047encode(&encoded_field, r->CData);
 203                         CM_SetAsFieldSB(r->msg, eMsgSubject, &encoded_field);
 204                 }
 205         }
 206
 207         else if (!strcasecmp(el, "creator")) {                  // <creator> can be used if <author> is not present
 208                 if ((r->msg != NULL) && (CM_IsEmpty(r->msg, eAuthor))) {
 209                         encoded_field = NewStrBuf();
 210                         StrBufRFC2047encode(&encoded_field, r->CData);
 211                         CM_SetAsFieldSB(r->msg, eAuthor, &encoded_field);
 212                 }
 213         }
 214
 215         else if (!strcasecmp(el, "author")) {                   // <author> supercedes <creator> if both are present
 216                 if (r->msg != NULL) {
 217                         encoded_field = NewStrBuf();
 218                         StrBufRFC2047encode(&encoded_field, r->CData);
 219                         CM_SetAsFieldSB(r->msg, eAuthor, &encoded_field);
 220                 }
 221         }
 222
 223         else if (!strcasecmp(el, "pubdate")) {                  // date/time stamp (rss) Sat, 25 Feb 2017 14:28:01 EST
 224                 if ((r->msg)&&(r->msg->cm_fields[eTimestamp]==NULL)) {
 225                         CM_SetFieldLONG(r->msg, eTimestamp, parsedate(ChrPtr(r->CData)));
 226                 }
 227         }
 228
 229         else if (!strcasecmp(el, "updated")) {                  // date/time stamp (atom) 2003-12-13T18:30:02Z
 230                 if ((r->msg)&&(r->msg->cm_fields[eTimestamp]==NULL)) {
 231                         struct tm t;
 232                         char zulu;
 233                         memset(&t, 0, sizeof t);
 234                         sscanf(ChrPtr(r->CData), "%d-%d-%dT%d:%d:%d%c", &t.tm_year, &t.tm_mon, &t.tm_mday, &t.tm_hour, &t.tm_min, &t.tm_sec, &zulu);
 235                         t.tm_year -= 1900;
 236                         t.tm_mon -= 1;
 237                         CM_SetFieldLONG(r->msg, eTimestamp, mktime(&t));
 238                 }
 239         }
 240
 241         else if (!strcasecmp(el, "link")) {                     // link to story (rss)
 242                 if (r->link != NULL) {
 243                         free(r->link);
 244                         r->link = NULL;
 245                 }
 246                 r->link = strdup(ChrPtr(r->CData));
 247         }
 248
 249         else if (
 250                 (!strcasecmp(el, "guid"))                       // unique item id (rss)
 251                 || (!strcasecmp(el, "id"))                      // unique item id (atom)
 252         ) {
 253                 if (r->item_id != NULL) {
 254                         free(r->item_id);
 255                         r->item_id = NULL;
 256                 }
 257                 r->item_id = strdup(ChrPtr(r->CData));
 258         }
 259
 260         else if (
 261                 (!strcasecmp(el, "description"))                // message text (rss)
 262                 || (!strcasecmp(el, "summary"))                 // message text (atom)
 263                 || (!strcasecmp(el, "content"))                 // message text (atom)
 264         ) {
 265                 if (r->description != NULL) {
 266                         free(r->description);
 267                         r->description = NULL;
 268                 }
 269                 r->description = strdup(ChrPtr(r->CData));
 270         }
 271
 272         if (r->CData != NULL) {
 273                 FreeStrBuf(&r->CData);
 274                 r->CData = NULL;
 275         }
 276 }
 277
 278
 279 // This handler is called whenever data appears between opening and closing tags.
 280 void rss_handle_data(void *data, const char *content, int length) {
 281         struct rssparser *r = (struct rssparser *)data;
 282
 283         if (r->CData == NULL) {
 284                 r->CData = NewStrBuf();
 285         }
 286
 287         StrBufAppendBufPlain(r->CData, content, length, 0);
 288 }
 289
 290
 291 // Feed has been downloaded, now parse it.
 292 void rss_parse_feed(StrBuf *Feed, struct rssroom *rooms) {
 293         struct rssparser r;
 294
 295         memset(&r, 0, sizeof r);
 296         r.rooms = rooms;
 297         XML_Parser p = XML_ParserCreate("UTF-8");
 298         XML_SetElementHandler(p, rss_start_element, rss_end_element);
 299         XML_SetCharacterDataHandler(p, rss_handle_data);
 300         XML_SetUserData(p, (void *)&r);
 301         XML_Parse(p, ChrPtr(Feed), StrLength(Feed), XML_TRUE);
 302         XML_ParserFree(p);
 303 }
 304
 305
 306 // Add a feed/room pair into the todo list
 307 void rssclient_push_todo(char *rssurl, char *roomname) {
 308         struct rssurl *r = NULL;
 309         struct rssurl *thisone = NULL;
 310         struct rssroom *newroom = NULL;
 311
 312         syslog(LOG_DEBUG, "rssclient: will fetch %s to %s", rssurl, roomname);
 313
 314         for (r=rsstodo; r!=NULL; r=r->next) {
 315                 if (!strcasecmp(r->url, rssurl)) {
 316                         thisone = r;
 317                 }
 318         }
 319
 320         if (thisone == NULL) {
 321                 thisone = malloc(sizeof(struct rssurl));
 322                 thisone->url = strdup(rssurl);
 323                 thisone->rooms = NULL;
 324                 thisone->next = rsstodo;
 325                 rsstodo = thisone;
 326         }
 327
 328         newroom = malloc(sizeof(struct rssroom));
 329         newroom->room = strdup(roomname);
 330         newroom->next = thisone->rooms;
 331         thisone->rooms = newroom;
 332 }
 333
 334
 335 // pull one feed (possibly multiple rooms)
 336 void rss_pull_one_feed(struct rssurl *url) {
 337         CURL *curl;
 338         CURLcode res;
 339         StrBuf *Downloaded = NULL;
 340
 341         syslog(LOG_DEBUG, "rssclient: fetching %s", url->url);
 342
 343         curl = curl_easy_init();
 344         if (!curl) {
 345                 return;
 346         }
 347
 348         Downloaded = NewStrBuf();
 349
 350         curl_easy_setopt(curl, CURLOPT_URL, url->url);
 351         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
 352         curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
 353         curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);                     // Follow redirects
 354         curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlFillStrBuf_callback); // What to do with downloaded data
 355         curl_easy_setopt(curl, CURLOPT_WRITEDATA, Downloaded);                  // Give it our StrBuf to work with
 356         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L);                           // Time out after 20 seconds
 357         res = curl_easy_perform(curl);                                          // Perform the request
 358         if (res != CURLE_OK) {
 359                 syslog(LOG_WARNING, "rssclient: failed to load feed: %s", curl_easy_strerror(res));
 360         }
 361         curl_easy_cleanup(curl);
 362
 363         rss_parse_feed(Downloaded, url->rooms);                                 // parse the feed
 364         FreeStrBuf(&Downloaded);                                                // free the downloaded feed data
 365 }
 366
 367
 368 // We have a list, now download the feeds
 369 void rss_pull_feeds(void) {
 370         struct rssurl *r;
 371         struct rssroom *rr;
 372
 373         while ((rsstodo != NULL) && (!server_shutting_down)) {
 374                 rss_pull_one_feed(rsstodo);
 375                 r = rsstodo;
 376                 rsstodo = rsstodo->next;
 377                 while (r->rooms != NULL) {
 378                         rr = r->rooms;
 379                         r->rooms = r->rooms->next;
 380                         free(rr->room);
 381                         free(rr);
 382                 }
 383                 free(r->url);
 384                 free(r);
 385         }
 386 }
 387
 388
 389 // Scan a room's netconfig looking for RSS feed parsing requests
 390 void rssclient_scan_room(struct ctdlroom *qrbuf, void *data) {
 391         char *serialized_config = NULL;
 392         int num_configs = 0;
 393         char cfgline[SIZ];
 394         int i = 0;
 395
 396         if (server_shutting_down) return;
 397
 398         serialized_config = LoadRoomNetConfigFile(qrbuf->QRnumber);
 399         if (!serialized_config) {
 400                 return;
 401         }
 402
 403         num_configs = num_tokens(serialized_config, '\n');
 404         for (i=0; i<num_configs; ++i) {
 405                 extract_token(cfgline, serialized_config, i, '\n', sizeof cfgline);
 406                 if (!strncasecmp(cfgline, HKEY("rssclient|"))) {
 407                         strcpy(cfgline, &cfgline[10]);
 408                         char *vbar = strchr(cfgline, '|');
 409                         if (vbar != NULL) {
 410                                 *vbar = 0;
 411                         }
 412                         rssclient_push_todo(cfgline, qrbuf->QRname);
 413                 }
 414         }
 415
 416         free(serialized_config);
 417 }
 418
 419
 420 // Scan for rooms that have RSS client requests configured
 421 void rssclient_scan(void) {
 422         time_t now = time(NULL);
 423
 424         // Run no more than once every 15 minutes.
 425         if ((now - last_run) < 900) {
 426                 syslog(LOG_DEBUG,
 427                         "rssclient: polling interval not yet reached; last run was %ldm%lds ago",
 428                         ((now - last_run) / 60),
 429                         ((now - last_run) % 60)
 430                 );
 431                 return;
 432         }
 433
 434         syslog(LOG_DEBUG, "rssclient: started");
 435         CtdlForEachRoom(rssclient_scan_room, NULL);
 436         rss_pull_feeds();
 437         syslog(LOG_DEBUG, "rssclient: ended");
 438         last_run = time(NULL);
 439         return;
 440 }
 441
 442
 443 // Initialization function, called from modules_init.c
 444 char *ctdl_module_init_rssclient(void) {
 445         if (!threading) {
 446                 syslog(LOG_INFO, "rssclient: using %s", curl_version());
 447                 CtdlRegisterSessionHook(rssclient_scan, EVT_TIMER, PRIO_AGGR + 300);
 448         }
 449         return "rssclient";
 450 }