citadel/modules/rssclient/serv_rssclient.c

   1 /*
   2  * Bring external RSS feeds into rooms.
   3  *
   4  * Copyright (c) 2007-2017 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 3.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
  12  * GNU General Public License for more details.
  13  */
  14
  15 #include <stdlib.h>
  16 #include <unistd.h>
  17 #include <stdio.h>
  18
  19 #if TIME_WITH_SYS_TIME
  20 # include <sys/time.h>
  21 # include <time.h>
  22 #else
  23 # if HAVE_SYS_TIME_H
  24 #include <sys/time.h>
  25 # else
  26 #include <time.h>
  27 # endif
  28 #endif
  29
  30 #include <ctype.h>
  31 #include <string.h>
  32 #include <errno.h>
  33 #include <sys/types.h>
  34 #include <sys/stat.h>
  35 #include <expat.h>
  36 #include <curl/curl.h>
  37 #include <libcitadel.h>
  38 #include "citadel.h"
  39 #include "server.h"
  40 #include "citserver.h"
  41 #include "support.h"
  42 #include "config.h"
  43 #include "threads.h"
  44 #include "ctdl_module.h"
  45 #include "msgbase.h"
  46 #include "parsedate.h"
  47 #include "database.h"
  48 #include "citadel_dirs.h"
  49 #include "md5.h"
  50 #include "context.h"
  51 #include "internet_addressing.h"
  52
  53 struct rssroom {
  54         struct rssroom *next;
  55         char *room;
  56 };
  57
  58 struct rssurl {
  59         struct rssurl *next;
  60         char *url;
  61         struct rssroom *rooms;
  62 };
  63
  64 struct rssparser {
  65         StrBuf *CData;
  66         struct CtdlMessage *msg;
  67         char *link;
  68         char *description;
  69         char *item_id;
  70         struct rssroom *rooms;
  71 };
  72
  73 time_t last_run = 0L;
  74 struct CitContext rss_CC;
  75 struct rssurl *rsstodo = NULL;
  76
  77
  78 // This handler is called whenever an XML tag opens.
  79 //
  80 void rss_start_element(void *data, const char *el, const char **attribute)
  81 {
  82         struct rssparser *r = (struct rssparser *)data;
  83         int i;
  84
  85         if (
  86                 (!strcasecmp(el, "entry"))
  87                 || (!strcasecmp(el, "item"))
  88         ) {
  89                 // this is the start of a new item(rss) or entry(atom)
  90                 if (r->msg != NULL) {
  91                         CM_Free(r->msg);
  92                         r->msg = NULL;
  93                 }
  94                 r->msg = malloc(sizeof(struct CtdlMessage));
  95                 memset(r->msg, 0, sizeof(struct CtdlMessage));
  96                 r->msg->cm_magic = CTDLMESSAGE_MAGIC;
  97                 r->msg->cm_anon_type = MES_NORMAL;
  98                 r->msg->cm_format_type = FMT_RFC822;
  99         }
 100
 101         else if (!strcasecmp(el, "link")) {                     // atom feeds have the link as an attribute
 102                 for(i = 0; attribute[i]; i += 2) {
 103                         if (!strcasecmp(attribute[i], "href")) {
 104                                 if (r->link != NULL) {
 105                                         free(r->link);
 106                                         r->link = NULL;
 107                                 }
 108                                 r->link = strdup(attribute[i+1]);
 109                                 striplt(r->link);
 110                         }
 111                 }
 112         }
 113 }
 114
 115
 116 // This handler is called whenever an XML tag closes.
 117 //
 118 void rss_end_element(void *data, const char *el)
 119 {
 120         struct rssparser *r = (struct rssparser *)data;
 121
 122         syslog(LOG_DEBUG, "end: %s", el);
 123         if (                                                    // end of a new item(rss) or entry(atom)
 124                 (!strcasecmp(el, "entry"))
 125                 || (!strcasecmp(el, "item"))
 126         ) {
 127
 128                 if (r->msg != NULL) {                           // Save the message to the rooms
 129
 130                         // use the link as an item id if nothing else is available
 131                         if ((r->item_id == NULL) && (r->link != NULL)) {
 132                                 r->item_id = strdup(r->link);
 133                         }
 134
 135                         // check the use table
 136                         StrBuf *u = NewStrBuf();
 137                         StrBufAppendPrintf(u, "rss/%s", r->item_id);
 138                         time_t already_seen = CheckIfAlreadySeen("rss", u, time(NULL), 604800, eUpdate, 0, 0);
 139                         FreeStrBuf(&u);
 140
 141                         if (already_seen == 0) {
 142
 143                                 // Compose the message text
 144                                 StrBuf *TheMessage = NewStrBuf();
 145                                 StrBufAppendPrintf(TheMessage,
 146                                         "Content-type: text/html\n\n"
 147                                         "\n\n"
 148                                         "<html><head></head><body>"
 149                                 );
 150
 151                                 if (r->description != NULL) {
 152                                         StrBufAppendPrintf(TheMessage, "%s<br><br>\r\n", r->description);
 153                                         free(r->description);
 154                                         r->description = NULL;
 155                                 }
 156
 157                                 if (r->link != NULL) {
 158                                         StrBufAppendPrintf(TheMessage, "<a href=\"%s\">%s</a>\r\n", r->link, r->link);
 159                                         free(r->link);
 160                                         r->link = NULL;
 161                                 }
 162
 163                                 StrBufAppendPrintf(TheMessage, "</body></html>\r\n");
 164                                 CM_SetField(r->msg, eMesageText, ChrPtr(TheMessage), StrLength(TheMessage));
 165                                 FreeStrBuf(&TheMessage);
 166
 167                                 if (CM_IsEmpty(r->msg, eAuthor)) {
 168                                         CM_SetField(r->msg, eAuthor, HKEY("rss"));
 169                                 }
 170
 171                                 if (CM_IsEmpty(r->msg, eTimestamp)) {
 172                                         CM_SetFieldLONG(r->msg, eTimestamp, time(NULL));
 173                                 }
 174
 175                                 // Save it to the room(s)
 176                                 struct rssroom *rr = NULL;
 177                                 long msgnum = (-1);
 178                                 for (rr=r->rooms; rr!=NULL; rr=rr->next) {
 179                                         if (rr == r->rooms) {
 180                                                 msgnum = CtdlSubmitMsg(r->msg, NULL, rr->room, 0);
 181                                         }
 182                                         else {
 183                                                 CtdlSaveMsgPointerInRoom(rr->room, msgnum, 0, NULL);
 184                                         }
 185                                         syslog(LOG_DEBUG, "Saved message %ld to %s", msgnum, rr->room);
 186                                 }
 187                         }
 188                         else {
 189                                 syslog(LOG_DEBUG, "%s was already seen %ld seconds ago", r->item_id, already_seen);
 190                         }
 191
 192                         CM_Free(r->msg);
 193                         r->msg = NULL;
 194                 }
 195
 196                 if (r->item_id != NULL) {
 197                         free(r->item_id);
 198                         r->item_id = NULL;
 199                 }
 200         }
 201
 202         else if (!strcasecmp(el, "title")) {                    // item subject (rss and atom)
 203                 if ((r->msg != NULL) && (CM_IsEmpty(r->msg, eMsgSubject))) {
 204                         CM_SetField(r->msg, eMsgSubject, ChrPtr(r->CData), StrLength(r->CData));
 205                         striplt(r->msg->cm_fields[eMsgSubject]);
 206                 }
 207         }
 208
 209         else if (!strcasecmp(el, "author")) {                   // author of item (rss and maybe atom)
 210                 if ((r->msg != NULL) && (CM_IsEmpty(r->msg, eAuthor))) {
 211                         CM_SetField(r->msg, eAuthor, ChrPtr(r->CData), StrLength(r->CData));
 212                         striplt(r->msg->cm_fields[eAuthor]);
 213                 }
 214         }
 215
 216         else if (!strcasecmp(el, "pubdate")) {                  // date/time stamp (rss) Sat, 25 Feb 2017 14:28:01 EST
 217                 if ((r->msg)&&(r->msg->cm_fields[eTimestamp]==NULL)) {
 218                         CM_SetFieldLONG(r->msg, eTimestamp, parsedate(ChrPtr(r->CData)));
 219                 }
 220         }
 221
 222         else if (!strcasecmp(el, "updated")) {                  // date/time stamp (atom) 2003-12-13T18:30:02Z
 223                 // FIXME parse it
 224         }
 225
 226         else if (!strcasecmp(el, "link")) {                     // link to story (rss)
 227                 if (r->link != NULL) {
 228                         free(r->link);
 229                         r->link = NULL;
 230                 }
 231                 r->link = strdup(ChrPtr(r->CData));
 232                 striplt(r->link);
 233         }
 234
 235         else if (
 236                 (!strcasecmp(el, "guid"))                       // unique item id (rss)
 237                 || (!strcasecmp(el, "id"))                      // unique item id (atom)
 238         ) {
 239                 if (r->item_id != NULL) {
 240                         free(r->item_id);
 241                         r->item_id = NULL;
 242                 }
 243                 r->item_id = strdup(ChrPtr(r->CData));
 244                 striplt(r->item_id);
 245         }
 246
 247         else if (
 248                 (!strcasecmp(el, "description"))                // message text (rss)
 249                 || (!strcasecmp(el, "summary"))                 // message text (atom)
 250         ) {
 251                 if (r->description != NULL) {
 252                         free(r->description);
 253                         r->description = NULL;
 254                 }
 255                 r->description = strdup(ChrPtr(r->CData));
 256                 striplt(r->description);
 257         }
 258
 259         if (r->CData != NULL) {
 260                 FreeStrBuf(&r->CData);
 261                 r->CData = NULL;
 262         }
 263 }
 264
 265
 266 // This handler is called whenever data appears between opening and closing tags.
 267 //
 268 void rss_handle_data(void *data, const char *content, int length)
 269 {
 270         struct rssparser *r = (struct rssparser *)data;
 271
 272         if (r->CData == NULL) {
 273                 r->CData = NewStrBuf();
 274         }
 275
 276         StrBufAppendBufPlain(r->CData, content, length, 0);
 277 }
 278
 279
 280 // Feed has been downloaded, now parse it.
 281 //
 282 void rss_parse_feed(StrBuf *Feed, struct rssroom *rooms)
 283 {
 284         struct rssparser r;
 285
 286         memset(&r, 0, sizeof r);
 287         r.rooms = rooms;
 288         XML_Parser p = XML_ParserCreate("UTF-8");
 289         XML_SetElementHandler(p, rss_start_element, rss_end_element);
 290         XML_SetCharacterDataHandler(p, rss_handle_data);
 291         XML_SetUserData(p, (void *)&r);
 292         XML_Parse(p, ChrPtr(Feed), StrLength(Feed), XML_TRUE);
 293         XML_ParserFree(p);
 294 }
 295
 296
 297 // Add a feed/room pair into the todo list
 298 //
 299 void rssclient_push_todo(char *rssurl, char *roomname)
 300 {
 301         struct rssurl *r = NULL;
 302         struct rssurl *thisone = NULL;
 303         struct rssroom *newroom = NULL;
 304
 305         syslog(LOG_DEBUG, "rssclient_push_todo(%s, %s)", rssurl, roomname);
 306
 307         for (r=rsstodo; r!=NULL; r=r->next) {
 308                 if (!strcasecmp(r->url, rssurl)) {
 309                         thisone = r;
 310                 }
 311         }
 312         if (thisone == NULL) {
 313                 thisone = malloc(sizeof(struct rssurl));
 314                 thisone->url = strdup(rssurl);
 315                 thisone->rooms = NULL;
 316                 thisone->next = rsstodo;
 317                 rsstodo = thisone;
 318         }
 319
 320         newroom = malloc(sizeof(struct rssroom));
 321         newroom->room = strdup(roomname);
 322         newroom->next = thisone->rooms;
 323         thisone->rooms = newroom;
 324 }
 325
 326
 327 // Callback function for curl
 328 //
 329 size_t rss_pof_write_data(void *buffer, size_t size, size_t nmemb, void *userp)
 330 {
 331         StrBuf *Downloaded = (StrBuf *)userp;
 332         size_t bytes = size * nmemb;
 333         StrBufAppendBufPlain(Downloaded, buffer, bytes, 0);
 334         return(bytes);
 335 }
 336
 337
 338 // pull one feed (possibly multiple rooms)
 339 //
 340 void rss_pull_one_feed(struct rssurl *url)
 341 {
 342         CURL *curl;
 343         CURLcode res;
 344         StrBuf *Downloaded = NULL;
 345
 346         syslog(LOG_DEBUG, "rss_pull_one_feed(%s)", url->url);
 347
 348         curl = curl_easy_init();
 349         if (!curl) {
 350                 return;
 351         }
 352
 353         Downloaded = NewStrBuf();
 354
 355         curl_easy_setopt(curl, CURLOPT_URL, url->url);
 356         curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);                     // Follow redirects
 357         curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, rss_pof_write_data);      // What to do with downloaded data
 358         curl_easy_setopt(curl, CURLOPT_WRITEDATA, Downloaded);                  // Give it our StrBuf to work with
 359         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L);                           // Time out after 20 seconds
 360         res = curl_easy_perform(curl);                                          // Perform the request
 361         if (res != CURLE_OK) {
 362                 syslog(LOG_WARNING, "Failed to load feed: %s", curl_easy_strerror(res));
 363         }
 364         curl_easy_cleanup(curl);
 365
 366         rss_parse_feed(Downloaded, url->rooms);                                 // parse the feed
 367         FreeStrBuf(&Downloaded);                                                // free the downloaded feed data
 368 }
 369
 370
 371 // We have a list, now download the feeds
 372 //
 373 void rss_pull_feeds(void)
 374 {
 375         struct rssurl *r;
 376         struct rssroom *rr;
 377
 378         while (rsstodo != NULL) {
 379                 rss_pull_one_feed(rsstodo);
 380                 r = rsstodo;
 381                 rsstodo = rsstodo->next;
 382                 while (r->rooms != NULL) {
 383                         rr = r->rooms;
 384                         r->rooms = r->rooms->next;
 385                         free(rr->room);
 386                         free(rr);
 387                 }
 388                 free(r->url);
 389                 free(r);
 390         }
 391 }
 392
 393
 394 // Scan a room's netconfig looking for RSS feed parsing requests
 395 //
 396 void rssclient_scan_room(struct ctdlroom *qrbuf, void *data)
 397 {
 398         char *serialized_config = NULL;
 399         int num_configs = 0;
 400         char cfgline[SIZ];
 401         int i = 0;
 402
 403         serialized_config = LoadRoomNetConfigFile(qrbuf->QRnumber);
 404         if (!serialized_config) {
 405                 return;
 406         }
 407
 408         num_configs = num_tokens(serialized_config, '\n');
 409         for (i=0; i<num_configs; ++i) {
 410                 extract_token(cfgline, serialized_config, i, '\n', sizeof cfgline);
 411                 if (!strncasecmp(cfgline, HKEY("rssclient|"))) {
 412                         strcpy(cfgline, &cfgline[10]);
 413                         char *vbar = strchr(cfgline, '|');
 414                         if (vbar != NULL) {
 415                                 *vbar = 0;
 416                         }
 417                         rssclient_push_todo(cfgline, qrbuf->QRname);
 418                 }
 419         }
 420
 421         free(serialized_config);
 422 }
 423
 424
 425 /*
 426  * Scan for rooms that have RSS client requests configured
 427  */
 428 void rssclient_scan(void) {
 429         time_t now = time(NULL);
 430
 431         /* Run no more than once every 15 minutes. */
 432         if ((now - last_run) < 900) {
 433                 syslog(LOG_DEBUG,
 434                         "Client: polling interval not yet reached; last run was %ldm%lds ago",
 435                         ((now - last_run) / 60),
 436                         ((now - last_run) % 60)
 437                 );
 438                 return;
 439         }
 440
 441         become_session(&rss_CC);
 442         syslog(LOG_DEBUG, "rssclient started");
 443         CtdlForEachRoom(rssclient_scan_room, NULL);
 444         rss_pull_feeds();
 445         syslog(LOG_DEBUG, "rssclient ended");
 446         last_run = time(NULL);
 447         return;
 448 }
 449
 450
 451 CTDL_MODULE_INIT(rssclient)
 452 {
 453         if (!threading)
 454         {
 455                 syslog(LOG_INFO, "%s", curl_version());
 456                 CtdlRegisterSessionHook(rssclient_scan, EVT_TIMER, PRIO_AGGR + 300);
 457         }
 458         else
 459         {
 460                 CtdlFillSystemContext(&rss_CC, "rssclient");
 461         }
 462         return "rssclient";
 463 }
 464