citadel/modules/rssclient/serv_rssclient.c

   1 /*
   2  * Bring external RSS feeds into rooms.
   3  *
   4  * Copyright (c) 2007-2010 by the citadel.org team
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 3 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  19  */
  20
  21 #include <stdlib.h>
  22 #include <unistd.h>
  23 #include <stdio.h>
  24
  25 #if TIME_WITH_SYS_TIME
  26 # include <sys/time.h>
  27 # include <time.h>
  28 #else
  29 # if HAVE_SYS_TIME_H
  30 #  include <sys/time.h>
  31 # else
  32 #  include <time.h>
  33 # endif
  34 #endif
  35
  36 #include <ctype.h>
  37 #include <string.h>
  38 #include <errno.h>
  39 #include <sys/types.h>
  40 #include <sys/stat.h>
  41 #include <expat.h>
  42 #include <curl/curl.h>
  43 #include <libcitadel.h>
  44 #include "citadel.h"
  45 #include "server.h"
  46 #include "citserver.h"
  47 #include "support.h"
  48 #include "config.h"
  49 #include "threads.h"
  50 #include "ctdl_module.h"
  51 #include "msgbase.h"
  52 #include "parsedate.h"
  53 #include "database.h"
  54 #include "citadel_dirs.h"
  55 #include "md5.h"
  56 #include "context.h"
  57 #include "event_client.h"
  58 #include "rss_atom_parser.h"
  59
  60
  61 #define TMP_MSGDATA 0xFF
  62 #define TMP_SHORTER_URL_OFFSET 0xFE
  63 #define TMP_SHORTER_URLS 0xFD
  64
  65 citthread_mutex_t RSSQueueMutex; /* locks the access to the following vars: */
  66 HashList *RSSQueueRooms = NULL; /* rss_room_counter */
  67 HashList *RSSFetchUrls = NULL; /* -> rss_aggregator; ->RefCount access to be locked too. */
  68
  69 eNextState RSSAggregatorTerminate(AsyncIO *IO);
  70
  71
  72 struct rssnetcfg *rnclist = NULL;
  73 void AppendLink(StrBuf *Message, StrBuf *link, StrBuf *LinkTitle, const char *Title)
  74 {
  75         if (StrLength(link) > 0)
  76         {
  77                 StrBufAppendBufPlain(Message, HKEY("<a href=\""), 0);
  78                 StrBufAppendBuf(Message, link, 0);
  79                 StrBufAppendBufPlain(Message, HKEY("\">"), 0);
  80                 if (StrLength(LinkTitle) > 0)
  81                         StrBufAppendBuf(Message, LinkTitle, 0);
  82                 else if ((Title != NULL) && !IsEmptyStr(Title))
  83                         StrBufAppendBufPlain(Message, Title, -1, 0);
  84                 else
  85                         StrBufAppendBuf(Message, link, 0);
  86                 StrBufAppendBufPlain(Message, HKEY("</a><br>\n"), 0);
  87         }
  88 }
  89 typedef struct __networker_save_message {
  90         AsyncIO IO;
  91         struct CtdlMessage *Msg;
  92         struct recptypes *recp;
  93         rss_aggregator *Cfg;
  94         StrBuf *MsgGUID;
  95         StrBuf *Message;
  96         struct UseTable ut;
  97 } networker_save_message;
  98
  99
 100 void DeleteRoomReference(long QRnumber)
 101 {
 102         HashPos *At;
 103         long HKLen;
 104         const char *HK;
 105         void *vData = NULL;
 106         rss_room_counter *pRoomC;
 107
 108         At = GetNewHashPos(RSSQueueRooms, 0);
 109
 110         GetHashPosFromKey(RSSQueueRooms, LKEY(QRnumber), At);
 111         GetHashPos(RSSQueueRooms, At, &HKLen, &HK, &vData);
 112         if (vData != NULL)
 113         {
 114                 pRoomC = (rss_room_counter *) vData;
 115                 pRoomC->count --;
 116                 if (pRoomC->count == 0)
 117                         DeleteEntryFromHash(RSSQueueRooms, At);
 118         }
 119         DeleteHashPos(&At);
 120 }
 121
 122 void UnlinkRooms(rss_aggregator *Cfg)
 123 {
 124
 125         DeleteRoomReference(Cfg->QRnumber);
 126         if (Cfg->OtherQRnumbers != NULL)
 127         {
 128                 long HKLen;
 129                 const char *HK;
 130                 HashPos *At;
 131                 void *vData;
 132
 133                 At = GetNewHashPos(Cfg->OtherQRnumbers, 0);
 134                 while (GetNextHashPos(Cfg->OtherQRnumbers, At, &HKLen, &HK, &vData) &&
 135                        (vData != NULL))
 136                 {
 137                         long *lData = (long*) vData;
 138                         DeleteRoomReference(*lData);
 139                 }
 140
 141                 DeleteHashPos(&At);
 142         }
 143
 144 }
 145
 146 void UnlinkRSSAggregator(rss_aggregator *Cfg)
 147 {
 148         HashPos *At;
 149
 150         UnlinkRooms(Cfg);
 151
 152         At = GetNewHashPos(RSSFetchUrls, 0);
 153         if (GetHashPosFromKey(RSSFetchUrls, SKEY(Cfg->Url), At) == 0)
 154         {
 155                 DeleteEntryFromHash(RSSFetchUrls, At);
 156         }
 157         DeleteHashPos(&At);
 158 }
 159
 160 eNextState FreeNetworkSaveMessage (AsyncIO *IO)
 161 {
 162         networker_save_message *Ctx = (networker_save_message *) IO->Data;
 163
 164         citthread_mutex_lock(&RSSQueueMutex);
 165         Ctx->Cfg->RefCount --;
 166
 167         if (Ctx->Cfg->RefCount == 0)
 168         {
 169                 UnlinkRSSAggregator(Ctx->Cfg);
 170
 171         }
 172         citthread_mutex_unlock(&RSSQueueMutex);
 173
 174         CtdlFreeMessage(Ctx->Msg);
 175         free_recipients(Ctx->recp);
 176         FreeStrBuf(&Ctx->Message);
 177         FreeStrBuf(&Ctx->MsgGUID);
 178         free(Ctx);
 179         return eAbort;
 180 }
 181
 182 eNextState AbortNetworkSaveMessage (AsyncIO *IO)
 183 {
 184         return eAbort; ///TODO
 185 }
 186
 187 eNextState RSSSaveMessage(AsyncIO *IO)
 188 {
 189         networker_save_message *Ctx = (networker_save_message *) IO->Data;
 190
 191         Ctx->Msg->cm_fields['M'] = SmashStrBuf(&Ctx->Message);
 192
 193         CtdlSubmitMsg(Ctx->Msg, Ctx->recp, NULL, 0);
 194
 195         /* write the uidl to the use table so we don't store this item again */
 196         cdb_store(CDB_USETABLE, SKEY(Ctx->MsgGUID), &Ctx->ut, sizeof(struct UseTable) );
 197
 198         return eTerminateConnection;
 199 }
 200
 201 // TODO: relink me:     ExpandShortUrls(ri->description);
 202
 203 eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO)
 204 {
 205         struct cdbdata *cdbut;
 206         networker_save_message *Ctx = (networker_save_message *) IO->Data;
 207
 208         /* Find out if we've already seen this item */
 209         strcpy(Ctx->ut.ut_msgid, ChrPtr(Ctx->MsgGUID)); /// TODO
 210         Ctx->ut.ut_timestamp = time(NULL);
 211
 212         cdbut = cdb_fetch(CDB_USETABLE, SKEY(Ctx->MsgGUID));
 213 #ifndef DEBUG_RSS
 214         if (cdbut != NULL) {
 215                 /* Item has already been seen */
 216                 syslog(LOG_DEBUG, "%s has already been seen\n", ChrPtr(Ctx->MsgGUID));
 217                 cdb_free(cdbut);
 218
 219                 /* rewrite the record anyway, to update the timestamp */
 220                 cdb_store(CDB_USETABLE,
 221                           SKEY(Ctx->MsgGUID),
 222                           &Ctx->ut, sizeof(struct UseTable) );
 223                 return eAbort;
 224         }
 225         else
 226 #endif
 227         {
 228                 NextDBOperation(IO, RSSSaveMessage);
 229                 return eSendMore;
 230         }
 231 }
 232 void RSSQueueSaveMessage(struct CtdlMessage *Msg, struct recptypes *recp, StrBuf *MsgGUID, StrBuf *MessageBody, rss_aggregator *Cfg)
 233 {
 234         networker_save_message *Ctx;
 235
 236         Ctx = (networker_save_message *) malloc(sizeof(networker_save_message));
 237         memset(Ctx, 0, sizeof(networker_save_message));
 238
 239         Ctx->MsgGUID = MsgGUID;
 240         Ctx->Message = MessageBody;
 241         Ctx->Msg = Msg;
 242         Ctx->Cfg = Cfg;
 243         Ctx->recp = recp;
 244         Ctx->IO.Data = Ctx;
 245         Ctx->IO.CitContext = CloneContext(CC);
 246         Ctx->IO.Terminate = FreeNetworkSaveMessage;
 247         Ctx->IO.ShutdownAbort = AbortNetworkSaveMessage;
 248         QueueDBOperation(&Ctx->IO, RSS_FetchNetworkUsetableEntry);
 249 }
 250
 251
 252 /*
 253  * Commit a fetched and parsed RSS item to disk
 254  */
 255 void rss_save_item(rss_item *ri, rss_aggregator *Cfg)
 256 {
 257
 258         struct MD5Context md5context;
 259         u_char rawdigest[MD5_DIGEST_LEN];
 260         struct CtdlMessage *msg;
 261         struct recptypes *recp = NULL;
 262         int msglen = 0;
 263         StrBuf *Message;
 264         StrBuf *guid;
 265         StrBuf *Buf;
 266
 267         recp = (struct recptypes *) malloc(sizeof(struct recptypes));
 268         if (recp == NULL) return;
 269         memset(recp, 0, sizeof(struct recptypes));
 270         Buf = NewStrBufDup(Cfg->rooms);
 271         recp->recp_room = SmashStrBuf(&Buf);
 272         recp->num_room = Cfg->roomlist_parts;
 273         recp->recptypes_magic = RECPTYPES_MAGIC;
 274
 275         Cfg->RefCount ++;
 276         /* Construct a GUID to use in the S_USETABLE table.
 277          * If one is not present in the item itself, make one up.
 278          */
 279         if (ri->guid != NULL) {
 280                 StrBufSpaceToBlank(ri->guid);
 281                 StrBufTrim(ri->guid);
 282                 guid = NewStrBufPlain(HKEY("rss/"));
 283                 StrBufAppendBuf(guid, ri->guid, 0);
 284         }
 285         else {
 286                 MD5Init(&md5context);
 287                 if (ri->title != NULL) {
 288                         MD5Update(&md5context, (const unsigned char*)ChrPtr(ri->title), StrLength(ri->title));
 289                 }
 290                 if (ri->link != NULL) {
 291                         MD5Update(&md5context, (const unsigned char*)ChrPtr(ri->link), StrLength(ri->link));
 292                 }
 293                 MD5Final(rawdigest, &md5context);
 294                 guid = NewStrBufPlain(NULL, MD5_DIGEST_LEN * 2 + 12 /* _rss2ctdl*/);
 295                 StrBufHexEscAppend(guid, NULL, rawdigest, MD5_DIGEST_LEN);
 296                 StrBufAppendBufPlain(guid, HKEY("_rss2ctdl"), 0);
 297         }
 298
 299         /* translate Item into message. */
 300         syslog(LOG_DEBUG, "RSS: translating item...\n");
 301         if (ri->description == NULL) ri->description = NewStrBufPlain(HKEY(""));
 302         StrBufSpaceToBlank(ri->description);
 303         msg = malloc(sizeof(struct CtdlMessage));
 304         memset(msg, 0, sizeof(struct CtdlMessage));
 305         msg->cm_magic = CTDLMESSAGE_MAGIC;
 306         msg->cm_anon_type = MES_NORMAL;
 307         msg->cm_format_type = FMT_RFC822;
 308
 309         if (ri->guid != NULL) {
 310                 msg->cm_fields['E'] = strdup(ChrPtr(ri->guid));
 311         }
 312
 313         if (ri->author_or_creator != NULL) {
 314                 char *From;
 315                 StrBuf *Encoded = NULL;
 316                 int FromAt;
 317
 318                 From = html_to_ascii(ChrPtr(ri->author_or_creator),
 319                                      StrLength(ri->author_or_creator),
 320                                      512, 0);
 321                 StrBufPlain(ri->author_or_creator, From, -1);
 322                 StrBufTrim(ri->author_or_creator);
 323                 free(From);
 324
 325                 FromAt = strchr(ChrPtr(ri->author_or_creator), '@') != NULL;
 326                 if (!FromAt && StrLength (ri->author_email) > 0)
 327                 {
 328                         StrBufRFC2047encode(&Encoded, ri->author_or_creator);
 329                         msg->cm_fields['A'] = SmashStrBuf(&Encoded);
 330                         msg->cm_fields['P'] = SmashStrBuf(&ri->author_email);
 331                 }
 332                 else
 333                 {
 334                         if (FromAt)
 335                         {
 336                                 msg->cm_fields['A'] = SmashStrBuf(&ri->author_or_creator);
 337                                 msg->cm_fields['P'] = strdup(msg->cm_fields['A']);
 338                         }
 339                         else
 340                         {
 341                                 StrBufRFC2047encode(&Encoded, ri->author_or_creator);
 342                                 msg->cm_fields['A'] = SmashStrBuf(&Encoded);
 343                                 msg->cm_fields['P'] = strdup("rss@localhost");
 344                         }
 345                 }
 346         }
 347         else {
 348                 msg->cm_fields['A'] = strdup("rss");
 349         }
 350
 351         msg->cm_fields['N'] = strdup(NODENAME);
 352         if (ri->title != NULL) {
 353                 long len;
 354                 char *Sbj;
 355                 StrBuf *Encoded, *QPEncoded;
 356
 357                 QPEncoded = NULL;
 358                 StrBufSpaceToBlank(ri->title);
 359                 len = StrLength(ri->title);
 360                 Sbj = html_to_ascii(ChrPtr(ri->title), len, 512, 0);
 361                 len = strlen(Sbj);
 362                 if (Sbj[len - 1] == '\n')
 363                 {
 364                         len --;
 365                         Sbj[len] = '\0';
 366                 }
 367                 Encoded = NewStrBufPlain(Sbj, len);
 368                 free(Sbj);
 369
 370                 StrBufTrim(Encoded);
 371                 StrBufRFC2047encode(&QPEncoded, Encoded);
 372
 373                 msg->cm_fields['U'] = SmashStrBuf(&QPEncoded);
 374                 FreeStrBuf(&Encoded);
 375         }
 376         msg->cm_fields['T'] = malloc(64);
 377         snprintf(msg->cm_fields['T'], 64, "%ld", ri->pubdate);
 378         if (ri->channel_title != NULL) {
 379                 if (StrLength(ri->channel_title) > 0) {
 380                         msg->cm_fields['O'] = strdup(ChrPtr(ri->channel_title));
 381                 }
 382         }
 383         if (ri->link == NULL)
 384                 ri->link = NewStrBufPlain(HKEY(""));
 385
 386 #if 0 /* temporarily disable shorter urls. */
 387         msg->cm_fields[TMP_SHORTER_URLS] = GetShorterUrls(ri->description);
 388 #endif
 389
 390         msglen += 1024 + StrLength(ri->link) + StrLength(ri->description) ;
 391
 392         Message = NewStrBufPlain(NULL, StrLength(ri->description));
 393
 394         StrBufPlain(Message, HKEY(
 395                             "Content-type: text/html; charset=\"UTF-8\"\r\n\r\n"
 396                             "<html><body>\n"));
 397 #if 0 /* disable shorter url for now. */
 398         msg->cm_fields[TMP_SHORTER_URL_OFFSET] = StrLength(Message);
 399 #endif
 400         StrBufAppendBuf(Message, ri->description, 0);
 401         StrBufAppendBufPlain(Message, HKEY("<br><br>\n"), 0);
 402
 403         AppendLink(Message, ri->link, ri->linkTitle, NULL);
 404         AppendLink(Message, ri->reLink, ri->reLinkTitle, "Reply to this");
 405         StrBufAppendBufPlain(Message, HKEY("</body></html>\n"), 0);
 406
 407         RSSQueueSaveMessage(msg, recp, guid, Message, Cfg);
 408 }
 409
 410
 411
 412 /*
 413  * Begin a feed parse
 414  */
 415 int rss_do_fetching(rss_aggregator *Cfg)
 416 {
 417         rss_item *ri;
 418
 419         time_t now;
 420         AsyncIO *IO;
 421
 422         now = time(NULL);
 423
 424         if ((Cfg->next_poll != 0) && (now < Cfg->next_poll))
 425                 return 0;
 426         Cfg->RefCount = 1;
 427
 428         ri = (rss_item*) malloc(sizeof(rss_item));
 429         memset(ri, 0, sizeof(rss_item));
 430         Cfg->Item = ri;
 431         IO = &Cfg->IO;
 432         IO->CitContext = CloneContext(CC);
 433         IO->Data = Cfg;
 434
 435
 436         syslog(LOG_DEBUG, "Fetching RSS feed <%s>\n", ChrPtr(Cfg->Url));
 437         ParseURL(&IO->ConnectMe, Cfg->Url, 80);
 438         CurlPrepareURL(IO->ConnectMe);
 439
 440         if (! evcurl_init(IO,
 441 //                        Ctx,
 442                           NULL,
 443                           "Citadel RSS Client",
 444                           ParseRSSReply,
 445                           RSSAggregatorTerminate))
 446         {
 447                 syslog(LOG_DEBUG, "Unable to initialize libcurl.\n");
 448                 return 0;
 449         }
 450
 451         evcurl_handle_start(IO);
 452         return 1;
 453 }
 454
 455
 456
 457 void DeleteRssCfg(void *vptr)
 458 {
 459         rss_aggregator *rncptr = (rss_aggregator *)vptr;
 460
 461         FreeStrBuf(&rncptr->Url);
 462         FreeStrBuf(&rncptr->rooms);
 463         FreeStrBuf(&rncptr->CData);
 464         FreeStrBuf(&rncptr->Key);
 465
 466         DeleteHash(&rncptr->OtherQRnumbers);
 467
 468         if (rncptr->Item != NULL)
 469         {
 470                 FreeStrBuf(&rncptr->Item->guid);
 471                 FreeStrBuf(&rncptr->Item->title);
 472                 FreeStrBuf(&rncptr->Item->link);
 473                 FreeStrBuf(&rncptr->Item->linkTitle);
 474                 FreeStrBuf(&rncptr->Item->reLink);
 475                 FreeStrBuf(&rncptr->Item->reLinkTitle);
 476                 FreeStrBuf(&rncptr->Item->description);
 477                 FreeStrBuf(&rncptr->Item->channel_title);
 478                 FreeStrBuf(&rncptr->Item->author_or_creator);
 479                 FreeStrBuf(&rncptr->Item->author_url);
 480                 FreeStrBuf(&rncptr->Item->author_email);
 481
 482                 free(rncptr->Item);
 483         }
 484         free(rncptr);
 485 }
 486
 487 eNextState RSSAggregatorTerminate(AsyncIO *IO)
 488 {
 489         rss_aggregator *rncptr = (rss_aggregator *)IO->Data;
 490         HashPos *At;
 491         long HKLen;
 492         const char *HK;
 493         void *vData;
 494
 495         citthread_mutex_lock(&RSSQueueMutex);
 496         rncptr->RefCount --;
 497         if (rncptr->RefCount == 0)
 498         {
 499                 UnlinkRSSAggregator(rncptr);
 500
 501         }
 502         citthread_mutex_unlock(&RSSQueueMutex);
 503 /*
 504         At = GetNewHashPos(RSSFetchUrls, 0);
 505
 506         citthread_mutex_lock(&RSSQueueMutex);
 507         GetHashPosFromKey(RSSFetchUrls, SKEY(rncptr->Url), At);
 508         GetHashPos(RSSFetchUrls, At, &HKLen, &HK, &vData);
 509         DeleteEntryFromHash(RSSFetchUrls, At);
 510         citthread_mutex_unlock(&RSSQueueMutex);
 511
 512         DeleteHashPos(&At);
 513 */
 514         return eAbort;
 515 }
 516
 517 /*
 518  * Scan a room's netconfig to determine whether it is requesting any RSS feeds
 519  */
 520 void rssclient_scan_room(struct ctdlroom *qrbuf, void *data)
 521 {
 522         StrBuf *CfgData;
 523         StrBuf *CfgType;
 524         StrBuf *Line;
 525         rss_room_counter *Count = NULL;
 526         struct stat statbuf;
 527         char filename[PATH_MAX];
 528         int  fd;
 529         int Done;
 530         rss_aggregator *rncptr = NULL;
 531         rss_aggregator *use_this_rncptr = NULL;
 532         void *vptr;
 533         const char *CfgPtr, *lPtr;
 534         const char *Err;
 535
 536         citthread_mutex_lock(&RSSQueueMutex);
 537         if (GetHash(RSSQueueRooms, LKEY(qrbuf->QRnumber), &vptr))
 538         {
 539                 syslog(LOG_DEBUG,
 540                               "rssclient: [%ld] %s already in progress.\n",
 541                               qrbuf->QRnumber,
 542                               qrbuf->QRname);
 543                 citthread_mutex_unlock(&RSSQueueMutex);
 544                 return;
 545         }
 546         citthread_mutex_unlock(&RSSQueueMutex);
 547
 548         assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir);
 549
 550         if (CtdlThreadCheckStop())
 551                 return;
 552
 553         /* Only do net processing for rooms that have netconfigs */
 554         fd = open(filename, 0);
 555         if (fd <= 0) {
 556                 //syslog(LOG_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname);
 557                 return;
 558         }
 559         if (CtdlThreadCheckStop())
 560                 return;
 561         if (fstat(fd, &statbuf) == -1) {
 562                 syslog(LOG_DEBUG, "ERROR: could not stat configfile '%s' - %s\n",
 563                         filename, strerror(errno));
 564                 return;
 565         }
 566         if (CtdlThreadCheckStop())
 567                 return;
 568         CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1);
 569         if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) {
 570                 close(fd);
 571                 FreeStrBuf(&CfgData);
 572                 syslog(LOG_DEBUG, "ERROR: reading config '%s' - %s<br>\n",
 573                         filename, strerror(errno));
 574                 return;
 575         }
 576         close(fd);
 577         if (CtdlThreadCheckStop())
 578                 return;
 579
 580         CfgPtr = NULL;
 581         CfgType = NewStrBuf();
 582         Line = NewStrBufPlain(NULL, StrLength(CfgData));
 583         Done = 0;
 584         while (!Done)
 585         {
 586             Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0;
 587             if (StrLength(Line) > 0)
 588             {
 589                 lPtr = NULL;
 590                 StrBufExtract_NextToken(CfgType, Line, &lPtr, '|');
 591                 if (!strcasecmp("rssclient", ChrPtr(CfgType)))
 592                 {
 593                     if (Count == NULL)
 594                     {
 595                         Count = malloc(sizeof(rss_room_counter));
 596                         Count->count = 0;
 597                     }
 598                     Count->count ++;
 599                     rncptr = (rss_aggregator *) malloc(sizeof(rss_aggregator));
 600                     memset (rncptr, 0, sizeof(rss_aggregator));
 601                     rncptr->roomlist_parts = 1;
 602                     rncptr->Url = NewStrBuf();
 603                     StrBufExtract_NextToken(rncptr->Url, Line, &lPtr, '|');
 604
 605                     citthread_mutex_lock(&RSSQueueMutex);
 606                     GetHash(RSSFetchUrls, SKEY(rncptr->Url), &vptr);
 607                     use_this_rncptr = (rss_aggregator *)vptr;
 608                     if (use_this_rncptr != NULL)
 609                     {
 610                             /* mustn't attach to an active session */
 611                             if (use_this_rncptr->RefCount > 0)
 612                             {
 613                                     DeleteRssCfg(rncptr);
 614                                     Count->count--;
 615                             }
 616                             else
 617                             {
 618                                     long *QRnumber;
 619                                     StrBufAppendBufPlain(use_this_rncptr->rooms,
 620                                                          qrbuf->QRname,
 621                                                          -1, 0);
 622                                     if (use_this_rncptr->roomlist_parts == 1)
 623                                     {
 624                                             use_this_rncptr->OtherQRnumbers = NewHash(1, lFlathash);
 625                                     }
 626                                     QRnumber = (long*)malloc(sizeof(long));
 627                                     *QRnumber = qrbuf->QRnumber;
 628                                     Put(use_this_rncptr->OtherQRnumbers, LKEY(qrbuf->QRnumber), QRnumber, NULL);
 629                                     use_this_rncptr->roomlist_parts++;
 630                             }
 631                             citthread_mutex_unlock(&RSSQueueMutex);
 632                             continue;
 633                     }
 634                     citthread_mutex_unlock(&RSSQueueMutex);
 635
 636                     rncptr->ItemType = RSS_UNSET;
 637
 638                     rncptr->rooms = NewStrBufPlain(qrbuf->QRname, -1);
 639
 640                     citthread_mutex_lock(&RSSQueueMutex);
 641                     Put(RSSFetchUrls, SKEY(rncptr->Url), rncptr, DeleteRssCfg);
 642                     citthread_mutex_unlock(&RSSQueueMutex);
 643                 }
 644             }
 645         }
 646         if (Count != NULL)
 647         {
 648                 Count->QRnumber = qrbuf->QRnumber;
 649                 citthread_mutex_lock(&RSSQueueMutex);
 650                 syslog(LOG_DEBUG, "rssclient: [%ld] %s now starting.\n",
 651                               qrbuf->QRnumber, qrbuf->QRname);
 652                 Put(RSSQueueRooms, LKEY(qrbuf->QRnumber), Count, NULL);
 653                 citthread_mutex_unlock(&RSSQueueMutex);
 654         }
 655         FreeStrBuf(&CfgData);
 656         FreeStrBuf(&CfgType);
 657         FreeStrBuf(&Line);
 658 }
 659
 660 /*
 661  * Scan for rooms that have RSS client requests configured
 662  */
 663 void rssclient_scan(void) {
 664         static int doing_rssclient = 0;
 665         rss_aggregator *rptr = NULL;
 666         void *vrptr = NULL;
 667         HashPos  *it;
 668         long len;
 669         const char *Key;
 670
 671         /*
 672          * This is a simple concurrency check to make sure only one rssclient run
 673          * is done at a time.  We could do this with a mutex, but since we
 674          * don't really require extremely fine granularity here, we'll do it
 675          * with a static variable instead.
 676          */
 677         if (doing_rssclient) return;
 678         doing_rssclient = 1;
 679
 680         syslog(LOG_DEBUG, "rssclient started\n");
 681         CtdlForEachRoom(rssclient_scan_room, NULL);
 682
 683         citthread_mutex_lock(&RSSQueueMutex);
 684
 685         it = GetNewHashPos(RSSFetchUrls, 0);
 686         while (GetNextHashPos(RSSFetchUrls, it, &len, &Key, &vrptr) &&
 687                (vrptr != NULL)) {
 688                 rptr = (rss_aggregator *)vrptr;
 689                 if (rptr->RefCount == 0)
 690                         if (!rss_do_fetching(rptr))
 691                                 UnlinkRSSAggregator(rptr);
 692         }
 693         DeleteHashPos(&it);
 694         citthread_mutex_unlock(&RSSQueueMutex);
 695
 696         syslog(LOG_DEBUG, "rssclient ended\n");
 697         doing_rssclient = 0;
 698         return;
 699 }
 700
 701 void rss_cleanup(void)
 702 {
 703         citthread_mutex_destroy(&RSSQueueMutex);
 704         DeleteHash(&RSSFetchUrls);
 705         DeleteHash(&RSSQueueRooms);
 706 }
 707
 708
 709 CTDL_MODULE_INIT(rssclient)
 710 {
 711         if (threading)
 712         {
 713                 citthread_mutex_init(&RSSQueueMutex, NULL);
 714                 RSSQueueRooms = NewHash(1, lFlathash);
 715                 RSSFetchUrls = NewHash(1, NULL);
 716                 syslog(LOG_INFO, "%s\n", curl_version());
 717                 CtdlRegisterSessionHook(rssclient_scan, EVT_TIMER);
 718                 CtdlRegisterCleanupHook(rss_cleanup);
 719         }
 720         return "rssclient";
 721 }