citadel/modules/rssclient/serv_rssclient.c

   1 /*
   2  * Bring external RSS feeds into rooms.
   3  *
   4  * Copyright (c) 2007-2017 by the citadel.org team
   5  *
   6  * This program is open source software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 3.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
  12  * GNU General Public License for more details.
  13  */
  14
  15 #include <stdlib.h>
  16 #include <unistd.h>
  17 #include <stdio.h>
  18
  19 #if TIME_WITH_SYS_TIME
  20 # include <sys/time.h>
  21 # include <time.h>
  22 #else
  23 # if HAVE_SYS_TIME_H
  24 #include <sys/time.h>
  25 # else
  26 #include <time.h>
  27 # endif
  28 #endif
  29
  30 #include <ctype.h>
  31 #include <string.h>
  32 #include <errno.h>
  33 #include <sys/types.h>
  34 #include <sys/stat.h>
  35 #include <expat.h>
  36 #include <curl/curl.h>
  37 #include <libcitadel.h>
  38 #include "citadel.h"
  39 #include "server.h"
  40 #include "citserver.h"
  41 #include "support.h"
  42 #include "config.h"
  43 #include "threads.h"
  44 #include "ctdl_module.h"
  45 #include "msgbase.h"
  46 #include "parsedate.h"
  47 #include "database.h"
  48 #include "citadel_dirs.h"
  49 #include "md5.h"
  50 #include "context.h"
  51
  52 struct rssroom {
  53         struct rssroom *next;
  54         char *room;
  55 };
  56
  57 struct rssurl {
  58         struct rssurl *next;
  59         char *url;
  60         struct rssroom *rooms;
  61 };
  62
  63 time_t last_run = 0L;
  64 struct CitContext rss_CC;
  65 struct rssurl *rsstodo = NULL;
  66
  67
  68 // Add a feed/room pair into the todo list
  69 //
  70 void rssclient_push_todo(char *rssurl, char *roomname)
  71 {
  72         struct rssurl *r = NULL;
  73         struct rssurl *thisone = NULL;
  74         struct rssroom *newroom = NULL;
  75
  76         syslog(LOG_DEBUG, "rssclient_push_todo(%s, %s)", rssurl, roomname);
  77
  78         for (r=rsstodo; r!=NULL; r=r->next) {
  79                 if (!strcasecmp(r->url, rssurl)) {
  80                         thisone = r;
  81                 }
  82         }
  83         if (thisone == NULL) {
  84                 thisone = malloc(sizeof(struct rssurl));
  85                 thisone->url = strdup(rssurl);
  86                 thisone->rooms = NULL;
  87                 thisone->next = rsstodo;
  88                 rsstodo = thisone;
  89         }
  90
  91         newroom = malloc(sizeof(struct rssroom));
  92         newroom->room = strdup(roomname);
  93         newroom->next = thisone->rooms;
  94         thisone->rooms = newroom;
  95 }
  96
  97
  98 // Callback function for curl
  99 //
 100 size_t rss_pof_write_data(void *buffer, size_t size, size_t nmemb, void *userp)
 101 {
 102         StrBuf *Downloaded = (StrBuf *)userp;
 103         size_t bytes = size * nmemb;
 104         StrBufAppendBufPlain(Downloaded, buffer, bytes, 0);
 105         return(bytes);
 106 }
 107
 108
 109 // pull one feed (possibly multiple rooms)
 110 //
 111 void rss_pull_one_feed(struct rssurl *url)
 112 {
 113         struct rssroom *r;
 114         CURL *curl;
 115         CURLcode res;
 116         StrBuf *Downloaded = NULL;
 117
 118         syslog(LOG_DEBUG, "rss_pull_one_feed(%s)", url->url);
 119
 120         curl = curl_easy_init();
 121         if (!curl) {
 122                 return;
 123         }
 124
 125         Downloaded = NewStrBuf();
 126
 127         curl_easy_setopt(curl, CURLOPT_URL, url->url);
 128         curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);                     // Follow redirects
 129         curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, rss_pof_write_data);      // What to do with downloaded data
 130         curl_easy_setopt(curl, CURLOPT_WRITEDATA, Downloaded);                  // Give it our StrBuf to work with
 131         curl_easy_setopt(curl, CURLOPT_TIMEOUT, 20L);                           // Time out after 20 seconds
 132         res = curl_easy_perform(curl);                                          // Perform the request
 133         if (res != CURLE_OK) {
 134                 syslog(LOG_WARNING, "Failed to load feed: %s", curl_easy_strerror(res));
 135         }
 136         curl_easy_cleanup(curl);
 137
 138         // FIXME parse the feed, dummeh ... it's in ChrPtr(Downloaded)
 139
 140         for (r=url->rooms; r!=NULL; r=r->next) {
 141                 syslog(LOG_DEBUG, "Saving item to %s", r->room);
 142                 // FIXME save to rooms
 143         }
 144
 145         FreeStrBuf(&Downloaded);
 146 }
 147
 148
 149 // We have a list, now download the feeds
 150 //
 151 void rss_pull_feeds(void)
 152 {
 153         struct rssurl *r;
 154         struct rssroom *rr;
 155
 156         while (rsstodo != NULL) {
 157                 rss_pull_one_feed(rsstodo);
 158                 r = rsstodo;
 159                 rsstodo = rsstodo->next;
 160                 while (r->rooms != NULL) {
 161                         rr = r->rooms;
 162                         r->rooms = r->rooms->next;
 163                         free(rr->room);
 164                         free(rr);
 165                 }
 166                 free(r->url);
 167                 free(r);
 168         }
 169 }
 170
 171
 172 // Scan a room's netconfig looking for RSS feed parsing requests
 173 //
 174 void rssclient_scan_room(struct ctdlroom *qrbuf, void *data)
 175 {
 176         char *serialized_config = NULL;
 177         int num_configs = 0;
 178         char cfgline[SIZ];
 179         int i = 0;
 180
 181         serialized_config = LoadRoomNetConfigFile(qrbuf->QRnumber);
 182         if (!serialized_config) {
 183                 return;
 184         }
 185
 186         num_configs = num_tokens(serialized_config, '\n');
 187         for (i=0; i<num_configs; ++i) {
 188                 extract_token(cfgline, serialized_config, i, '\n', sizeof cfgline);
 189                 if (!strncasecmp(cfgline, HKEY("rssclient|"))) {
 190                         strcpy(cfgline, &cfgline[10]);
 191                         char *vbar = strchr(cfgline, '|');
 192                         if (vbar != NULL) {
 193                                 *vbar = 0;
 194                         }
 195                         rssclient_push_todo(cfgline, qrbuf->QRname);
 196                 }
 197         }
 198
 199         free(serialized_config);
 200 }
 201
 202
 203 /*
 204  * Scan for rooms that have RSS client requests configured
 205  */
 206 void rssclient_scan(void) {
 207         time_t now = time(NULL);
 208
 209         /* Run no more than once every 15 minutes. */
 210         if ((now - last_run) < 900) {
 211                 syslog(LOG_DEBUG,
 212                               "Client: polling interval not yet reached; last run was %ldm%lds ago",
 213                               ((now - last_run) / 60),
 214                               ((now - last_run) % 60)
 215                 );
 216                 return;
 217         }
 218
 219         become_session(&rss_CC);
 220         syslog(LOG_DEBUG, "rssclient started");
 221         CtdlForEachRoom(rssclient_scan_room, NULL);
 222         rss_pull_feeds();
 223         syslog(LOG_DEBUG, "rssclient ended");
 224         last_run = time(NULL);
 225         return;
 226 }
 227
 228
 229 CTDL_MODULE_INIT(rssclient)
 230 {
 231         if (!threading)
 232         {
 233                 syslog(LOG_INFO, "%s", curl_version());
 234                 CtdlRegisterSessionHook(rssclient_scan, EVT_TIMER, PRIO_AGGR + 300);
 235         }
 236         else
 237         {
 238                 CtdlFillSystemContext(&rss_CC, "rssclient");
 239         }
 240         return "rssclient";
 241 }
 242