EV: fix possible nullpointer access in last commit.
[citadel] / citadel / modules / rssclient / serv_rssclient.c
1 /*
2  * Bring external RSS feeds into rooms.
3  *
4  * Copyright (c) 2007-2012 by the citadel.org team
5  *
6  * This program is open source software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 3.
8  *
9  * This program is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
12  * GNU General Public License for more details.
13  */
14
15 #include <stdlib.h>
16 #include <unistd.h>
17 #include <stdio.h>
18
19 #if TIME_WITH_SYS_TIME
20 # include <sys/time.h>
21 # include <time.h>
22 #else
23 # if HAVE_SYS_TIME_H
24 #include <sys/time.h>
25 # else
26 #include <time.h>
27 # endif
28 #endif
29
30 #include <ctype.h>
31 #include <string.h>
32 #include <errno.h>
33 #include <sys/types.h>
34 #include <sys/stat.h>
35 #include <expat.h>
36 #include <curl/curl.h>
37 #include <libcitadel.h>
38 #include "citadel.h"
39 #include "server.h"
40 #include "citserver.h"
41 #include "support.h"
42 #include "config.h"
43 #include "threads.h"
44 #include "ctdl_module.h"
45 #include "msgbase.h"
46 #include "parsedate.h"
47 #include "database.h"
48 #include "citadel_dirs.h"
49 #include "md5.h"
50 #include "context.h"
51 #include "event_client.h"
52 #include "rss_atom_parser.h"
53
54
55 #define TMP_MSGDATA 0xFF
56 #define TMP_SHORTER_URL_OFFSET 0xFE
57 #define TMP_SHORTER_URLS 0xFD
58
59 time_t last_run = 0L;
60
61 pthread_mutex_t RSSQueueMutex; /* locks the access to the following vars: */
62 HashList *RSSQueueRooms = NULL; /* rss_room_counter */
63 HashList *RSSFetchUrls = NULL; /*->rss_aggregator;->RefCount access locked*/
64
65 eNextState RSSAggregator_Terminate(AsyncIO *IO);
66 eNextState RSSAggregator_TerminateDB(AsyncIO *IO);
67 eNextState RSSAggregator_ShutdownAbort(AsyncIO *IO);
68 struct CitContext rss_CC;
69
70 struct rssnetcfg *rnclist = NULL;
71 int RSSClientDebugEnabled = 0;
72 #define N ((rss_aggregator*)IO->Data)->QRnumber
73
74 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (RSSClientDebugEnabled != 0))
75
76 #define EVRSSC_syslog(LEVEL, FORMAT, ...)                               \
77         DBGLOG(LEVEL) syslog(LEVEL,                                     \
78                              "IO[%ld]CC[%d][%ld]RSS" FORMAT,            \
79                              IO->ID, CCID, N, __VA_ARGS__)
80
81 #define EVRSSCM_syslog(LEVEL, FORMAT)                                   \
82         DBGLOG(LEVEL) syslog(LEVEL,                                     \
83                              "IO[%ld]CC[%d][%ld]RSS" FORMAT,            \
84                              IO->ID, CCID, N)
85
86 #define EVRSSQ_syslog(LEVEL, FORMAT, ...)                               \
87         DBGLOG(LEVEL) syslog(LEVEL, "RSS" FORMAT,                       \
88                              __VA_ARGS__)
89 #define EVRSSQM_syslog(LEVEL, FORMAT)                   \
90         DBGLOG(LEVEL) syslog(LEVEL, "RSS" FORMAT)
91
92 #define EVRSSCSM_syslog(LEVEL, FORMAT)                                  \
93         DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]RSS" FORMAT,           \
94                              IO->ID, N)
95
96 typedef enum _RSSState {
97         eRSSCreated,
98         eRSSFetching,
99         eRSSFailure,
100         eRSSParsing,
101         eRSSUT
102 } RSSState;
103 ConstStr RSSStates[] = {
104         {HKEY("Aggregator created")},
105         {HKEY("Fetching content")},
106         {HKEY("Failed")},
107         {HKEY("parsing content")},
108         {HKEY("checking usetable")}
109 };
110
111 static void SetRSSState(AsyncIO *IO, RSSState State)
112 {
113         CitContext* CCC = IO->CitContext;
114         if (CCC != NULL)
115                 memcpy(CCC->cs_clientname, RSSStates[State].Key, RSSStates[State].len + 1);
116 }
117
118 void DeleteRoomReference(long QRnumber)
119 {
120         HashPos *At;
121         long HKLen;
122         const char *HK;
123         void *vData = NULL;
124         rss_room_counter *pRoomC;
125
126         At = GetNewHashPos(RSSQueueRooms, 0);
127
128         if (GetHashPosFromKey(RSSQueueRooms, LKEY(QRnumber), At))
129         {
130                 GetHashPos(RSSQueueRooms, At, &HKLen, &HK, &vData);
131                 if (vData != NULL)
132                 {
133                         pRoomC = (rss_room_counter *) vData;
134                         pRoomC->count --;
135                         if (pRoomC->count == 0)
136                                 DeleteEntryFromHash(RSSQueueRooms, At);
137                 }
138         }
139         DeleteHashPos(&At);
140 }
141
142 void UnlinkRooms(rss_aggregator *RSSAggr)
143 {
144         DeleteRoomReference(RSSAggr->QRnumber);
145         if (RSSAggr->OtherQRnumbers != NULL)
146         {
147                 long HKLen;
148                 const char *HK;
149                 HashPos *At;
150                 void *vData;
151
152                 At = GetNewHashPos(RSSAggr->OtherQRnumbers, 0);
153                 while (! server_shutting_down &&
154                        GetNextHashPos(RSSAggr->OtherQRnumbers,
155                                       At,
156                                       &HKLen, &HK,
157                                       &vData) &&
158                        (vData != NULL))
159                 {
160                         long *lData = (long*) vData;
161                         DeleteRoomReference(*lData);
162                 }
163
164                 DeleteHashPos(&At);
165         }
166 }
167
168 void UnlinkRSSAggregator(rss_aggregator *RSSAggr)
169 {
170         HashPos *At;
171
172         pthread_mutex_lock(&RSSQueueMutex);
173         UnlinkRooms(RSSAggr);
174
175         At = GetNewHashPos(RSSFetchUrls, 0);
176         if (GetHashPosFromKey(RSSFetchUrls, SKEY(RSSAggr->Url), At))
177         {
178                 DeleteEntryFromHash(RSSFetchUrls, At);
179         }
180         DeleteHashPos(&At);
181         last_run = time(NULL);
182         pthread_mutex_unlock(&RSSQueueMutex);
183 }
184
185 void DeleteRssCfg(void *vptr)
186 {
187         rss_aggregator *RSSAggr = (rss_aggregator *)vptr;
188         AsyncIO *IO = &RSSAggr->IO;
189
190         if (IO->CitContext != NULL)
191                 EVRSSCM_syslog(LOG_DEBUG, "RSS: destroying\n");
192
193         FreeStrBuf(&RSSAggr->Url);
194         FreeStrBuf(&RSSAggr->rooms);
195         FreeStrBuf(&RSSAggr->CData);
196         FreeStrBuf(&RSSAggr->Key);
197         DeleteHash(&RSSAggr->OtherQRnumbers);
198
199         DeleteHashPos (&RSSAggr->Pos);
200         DeleteHash (&RSSAggr->Messages);
201         if (RSSAggr->recp.recp_room != NULL)
202                 free(RSSAggr->recp.recp_room);
203
204
205         if (RSSAggr->Item != NULL)
206         {
207                 flush_rss_item(RSSAggr->Item);
208
209                 free(RSSAggr->Item);
210         }
211
212         FreeAsyncIOContents(&RSSAggr->IO);
213         memset(RSSAggr, 0, sizeof(rss_aggregator));
214         free(RSSAggr);
215 }
216
217 eNextState RSSAggregator_Terminate(AsyncIO *IO)
218 {
219         rss_aggregator *RSSAggr = (rss_aggregator *)IO->Data;
220
221         EVRSSCM_syslog(LOG_DEBUG, "RSS: Terminating.\n");
222
223         StopCurlWatchers(IO);
224         UnlinkRSSAggregator(RSSAggr);
225         return eAbort;
226 }
227
228 eNextState RSSAggregator_TerminateDB(AsyncIO *IO)
229 {
230         rss_aggregator *RSSAggr = (rss_aggregator *)IO->Data;
231
232         EVRSSCM_syslog(LOG_DEBUG, "RSS: Terminating.\n");
233
234
235         StopDBWatchers(&RSSAggr->IO);
236         UnlinkRSSAggregator(RSSAggr);
237         return eAbort;
238 }
239
240 eNextState RSSAggregator_ShutdownAbort(AsyncIO *IO)
241 {
242         const char *pUrl;
243         rss_aggregator *RSSAggr = (rss_aggregator *)IO->Data;
244
245         pUrl = IO->ConnectMe->PlainUrl;
246         if (pUrl == NULL)
247                 pUrl = "";
248
249         EVRSSC_syslog(LOG_DEBUG, "RSS: Aborting by shutdown: %s.\n", pUrl);
250
251         StopCurlWatchers(IO);
252         UnlinkRSSAggregator(RSSAggr);
253         return eAbort;
254 }
255
256 void AppendLink(StrBuf *Message,
257                 StrBuf *link,
258                 StrBuf *LinkTitle,
259                 const char *Title)
260 {
261         if (StrLength(link) > 0)
262         {
263                 StrBufAppendBufPlain(Message, HKEY("<a href=\""), 0);
264                 StrBufAppendBuf(Message, link, 0);
265                 StrBufAppendBufPlain(Message, HKEY("\">"), 0);
266                 if (StrLength(LinkTitle) > 0)
267                         StrBufAppendBuf(Message, LinkTitle, 0);
268                 else if ((Title != NULL) && !IsEmptyStr(Title))
269                         StrBufAppendBufPlain(Message, Title, -1, 0);
270                 else
271                         StrBufAppendBuf(Message, link, 0);
272                 StrBufAppendBufPlain(Message, HKEY("</a><br>\n"), 0);
273         }
274 }
275
276
277 void rss_format_item(networker_save_message *SaveMsg)
278 {
279         StrBuf *Message;
280         int msglen = 0;
281
282         if (SaveMsg->author_or_creator != NULL) {
283
284                 char *From;
285                 StrBuf *Encoded = NULL;
286                 int FromAt;
287
288                 From = html_to_ascii(ChrPtr(SaveMsg->author_or_creator),
289                                      StrLength(SaveMsg->author_or_creator),
290                                      512, 0);
291                 StrBufPlain(SaveMsg->author_or_creator, From, -1);
292                 StrBufTrim(SaveMsg->author_or_creator);
293                 free(From);
294
295                 FromAt = strchr(ChrPtr(SaveMsg->author_or_creator), '@') != NULL;
296                 if (!FromAt && StrLength (SaveMsg->author_email) > 0)
297                 {
298                         StrBufRFC2047encode(&Encoded, SaveMsg->author_or_creator);
299                         SaveMsg->Msg.cm_fields['A'] = SmashStrBuf(&Encoded);
300                         SaveMsg->Msg.cm_fields['P'] =
301                                 SmashStrBuf(&SaveMsg->author_email);
302                 }
303                 else
304                 {
305                         if (FromAt)
306                         {
307                                 SaveMsg->Msg.cm_fields['A'] =
308                                         SmashStrBuf(&SaveMsg->author_or_creator);
309                                 SaveMsg->Msg.cm_fields['P'] =
310                                         strdup(SaveMsg->Msg.cm_fields['A']);
311                         }
312                         else
313                         {
314                                 StrBufRFC2047encode(&Encoded,
315                                                     SaveMsg->author_or_creator);
316                                 SaveMsg->Msg.cm_fields['A'] =
317                                         SmashStrBuf(&Encoded);
318                                 SaveMsg->Msg.cm_fields['P'] =
319                                         strdup("rss@localhost");
320
321                         }
322                 }
323         }
324         else {
325                 SaveMsg->Msg.cm_fields['A'] = strdup("rss");
326         }
327
328         SaveMsg->Msg.cm_fields['N'] = strdup(NODENAME);
329         if (SaveMsg->title != NULL) {
330                 long len;
331                 char *Sbj;
332                 StrBuf *Encoded, *QPEncoded;
333
334                 QPEncoded = NULL;
335                 StrBufSpaceToBlank(SaveMsg->title);
336                 len = StrLength(SaveMsg->title);
337                 Sbj = html_to_ascii(ChrPtr(SaveMsg->title), len, 512, 0);
338                 len = strlen(Sbj);
339                 if ((len > 0) && (Sbj[len - 1] == '\n'))
340                 {
341                         len --;
342                         Sbj[len] = '\0';
343                 }
344                 Encoded = NewStrBufPlain(Sbj, len);
345                 free(Sbj);
346
347                 StrBufTrim(Encoded);
348                 StrBufRFC2047encode(&QPEncoded, Encoded);
349
350                 SaveMsg->Msg.cm_fields['U'] = SmashStrBuf(&QPEncoded);
351                 FreeStrBuf(&Encoded);
352         }
353         if (SaveMsg->link == NULL)
354                 SaveMsg->link = NewStrBufPlain(HKEY(""));
355
356 #if 0 /* temporarily disable shorter urls. */
357         SaveMsg->Msg.cm_fields[TMP_SHORTER_URLS] =
358                 GetShorterUrls(SaveMsg->description);
359 #endif
360
361         msglen += 1024 + StrLength(SaveMsg->link) + StrLength(SaveMsg->description) ;
362
363         Message = NewStrBufPlain(NULL, msglen);
364
365         StrBufPlain(Message, HKEY(
366                             "Content-type: text/html; charset=\"UTF-8\"\r\n\r\n"
367                             "<html><body>\n"));
368 #if 0 /* disable shorter url for now. */
369         SaveMsg->Msg.cm_fields[TMP_SHORTER_URL_OFFSET] = StrLength(Message);
370 #endif
371         StrBufAppendBuf(Message, SaveMsg->description, 0);
372         StrBufAppendBufPlain(Message, HKEY("<br><br>\n"), 0);
373
374         AppendLink(Message, SaveMsg->link, SaveMsg->linkTitle, NULL);
375         AppendLink(Message, SaveMsg->reLink, SaveMsg->reLinkTitle, "Reply to this");
376         StrBufAppendBufPlain(Message, HKEY("</body></html>\n"), 0);
377
378
379         SaveMsg->Message = Message;
380 }
381
382 eNextState RSSSaveMessage(AsyncIO *IO)
383 {
384         long len;
385         const char *Key;
386         rss_aggregator *RSSAggr = (rss_aggregator *) IO->Data;
387
388         rss_format_item(RSSAggr->ThisMsg);
389
390         RSSAggr->ThisMsg->Msg.cm_fields['M'] =
391                 SmashStrBuf(&RSSAggr->ThisMsg->Message);
392
393         CtdlSubmitMsg(&RSSAggr->ThisMsg->Msg, &RSSAggr->recp, NULL, 0);
394
395         /* write the uidl to the use table so we don't store this item again */
396
397         CheckIfAlreadySeen("RSS Item Insert", RSSAggr->ThisMsg->MsgGUID, IO->Now, 0, eWrite, IO->ID, CCID);
398
399         if (GetNextHashPos(RSSAggr->Messages,
400                            RSSAggr->Pos,
401                            &len, &Key,
402                            (void**) &RSSAggr->ThisMsg))
403                 return NextDBOperation(IO, RSS_FetchNetworkUsetableEntry);
404         else
405                 return eAbort;
406 }
407
408 eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO)
409 {
410         const char *Key;
411         long len;
412         rss_aggregator *Ctx = (rss_aggregator *) IO->Data;
413
414         /* Find out if we've already seen this item */
415 // todo: expiry?
416 #ifndef DEBUG_RSS
417         SetRSSState(IO, eRSSUT);
418         if (CheckIfAlreadySeen("RSS Item Seen",
419                                Ctx->ThisMsg->MsgGUID,
420                                IO->Now,
421                                IO->Now - USETABLE_ANTIEXPIRE,
422                                eCheckUpdate,
423                                IO->ID, CCID)
424             != 0)
425         {
426                 /* Item has already been seen */
427                 EVRSSC_syslog(LOG_DEBUG,
428                           "%s has already been seen\n",
429                           ChrPtr(Ctx->ThisMsg->MsgGUID));
430                 SetRSSState(IO, eRSSParsing);
431
432                 if (GetNextHashPos(Ctx->Messages,
433                                    Ctx->Pos,
434                                    &len, &Key,
435                                    (void**) &Ctx->ThisMsg))
436                         return NextDBOperation(
437                                 IO,
438                                 RSS_FetchNetworkUsetableEntry);
439                 else
440                         return eAbort;
441         }
442         else
443 #endif
444         {
445                 SetRSSState(IO, eRSSParsing);
446
447                 NextDBOperation(IO, RSSSaveMessage);
448                 return eSendMore;
449         }
450         return eSendMore;
451 }
452
453 eNextState RSSAggregator_AnalyseReply(AsyncIO *IO)
454 {
455         u_char rawdigest[MD5_DIGEST_LEN];
456         struct MD5Context md5context;
457         StrBuf *guid;
458         rss_aggregator *Ctx = (rss_aggregator *) IO->Data;
459
460         if (IO->HttpReq.httpcode != 200)
461         {
462                 StrBuf *ErrMsg;
463                 long lens[2];
464                 const char *strs[2];
465
466                 SetRSSState(IO, eRSSFailure);
467                 ErrMsg = NewStrBuf();
468                 EVRSSC_syslog(LOG_ALERT, "need a 200, got a %ld !\n",
469                               IO->HttpReq.httpcode);
470                 
471                 strs[0] = ChrPtr(Ctx->Url);
472                 lens[0] = StrLength(Ctx->Url);
473
474                 strs[1] = ChrPtr(Ctx->rooms);
475                 lens[1] = StrLength(Ctx->rooms);
476                 StrBufPrintf(ErrMsg,
477                              "Error while RSS-Aggregation Run of %s\n"
478                              " need a 200, got a %ld !\n"
479                              " Response text was: \n"
480                              " \n %s\n",
481                              ChrPtr(Ctx->Url),
482                              IO->HttpReq.httpcode,
483                              ChrPtr(IO->HttpReq.ReplyData));
484                 CtdlAideFPMessage(
485                         ChrPtr(ErrMsg),
486                         "RSS Aggregation run failure",
487                         2, strs, (long*) &lens,
488                         IO->Now,
489                         IO->ID, CCID);
490                 
491                 FreeStrBuf(&ErrMsg);
492                 EVRSSC_syslog(LOG_DEBUG,
493                               "RSS feed returned an invalid http status code. <%s><HTTP %ld>\n",
494                               ChrPtr(Ctx->Url),
495                               IO->HttpReq.httpcode);
496                 return eAbort;
497         }
498         SetRSSState(IO, eRSSUT);
499
500         MD5Init(&md5context);
501
502         MD5Update(&md5context,
503                   (const unsigned char*)SKEY(IO->HttpReq.ReplyData));
504
505         MD5Update(&md5context,
506                   (const unsigned char*)SKEY(Ctx->Url));
507
508         MD5Final(rawdigest, &md5context);
509         guid = NewStrBufPlain(NULL,
510                               MD5_DIGEST_LEN * 2 + 12 /* _rss2ctdl*/);
511         StrBufHexEscAppend(guid, NULL, rawdigest, MD5_DIGEST_LEN);
512         StrBufAppendBufPlain(guid, HKEY("_rssFM"), 0);
513         if (StrLength(guid) > 40)
514                 StrBufCutAt(guid, 40, NULL);
515         /* Find out if we've already seen this item */
516
517 #ifndef DEBUG_RSS
518
519         if (CheckIfAlreadySeen("RSS Whole",
520                                guid,
521                                IO->Now,
522                                IO->Now - USETABLE_ANTIEXPIRE,
523                                eCheckUpdate,
524                                IO->ID, CCID)
525             != 0)
526         {
527                 FreeStrBuf(&guid);
528
529                 EVRSSC_syslog(LOG_DEBUG, "RSS feed already seen. <%s>\n", ChrPtr(Ctx->Url));
530                 return eAbort;
531         }
532         FreeStrBuf(&guid);
533 #endif
534         SetRSSState(IO, eRSSParsing);
535         return RSSAggregator_ParseReply(IO);
536 }
537
538 eNextState RSSAggregator_FinishHttp(AsyncIO *IO)
539 {
540         return QueueDBOperation(IO, RSSAggregator_AnalyseReply);
541 }
542
543 /*
544  * Begin a feed parse
545  */
546 int rss_do_fetching(rss_aggregator *RSSAggr)
547 {
548         AsyncIO         *IO = &RSSAggr->IO;
549         rss_item *ri;
550         time_t now;
551
552         now = time(NULL);
553
554         if ((RSSAggr->next_poll != 0) && (now < RSSAggr->next_poll))
555                 return 0;
556
557         ri = (rss_item*) malloc(sizeof(rss_item));
558         memset(ri, 0, sizeof(rss_item));
559         RSSAggr->Item = ri;
560
561         if (! InitcURLIOStruct(&RSSAggr->IO,
562                                RSSAggr,
563                                "Citadel RSS Client",
564                                RSSAggregator_FinishHttp,
565                                RSSAggregator_Terminate,
566                                RSSAggregator_TerminateDB,
567                                RSSAggregator_ShutdownAbort))
568         {
569                 EVRSSCM_syslog(LOG_ALERT, "Unable to initialize libcurl.\n");
570                 return 0;
571         }
572         SetRSSState(IO, eRSSCreated);
573
574         safestrncpy(((CitContext*)RSSAggr->IO.CitContext)->cs_host,
575                     ChrPtr(RSSAggr->Url),
576                     sizeof(((CitContext*)RSSAggr->IO.CitContext)->cs_host));
577
578         EVRSSC_syslog(LOG_DEBUG, "Fetching RSS feed <%s>\n", ChrPtr(RSSAggr->Url));
579         ParseURL(&RSSAggr->IO.ConnectMe, RSSAggr->Url, 80);
580         CurlPrepareURL(RSSAggr->IO.ConnectMe);
581
582         SetRSSState(IO, eRSSFetching);
583         QueueCurlContext(&RSSAggr->IO);
584         return 1;
585 }
586
587 /*
588  * Scan a room's netconfig to determine whether it is requesting any RSS feeds
589  */
590 void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
591 {
592         const RoomNetCfgLine *pLine;
593         rss_aggregator *RSSAggr = NULL;
594         rss_aggregator *use_this_RSSAggr = NULL;
595         void *vptr;
596
597         pthread_mutex_lock(&RSSQueueMutex);
598         if (GetHash(RSSQueueRooms, LKEY(qrbuf->QRnumber), &vptr))
599         {
600                 EVRSSQ_syslog(LOG_DEBUG,
601                               "rssclient: [%ld] %s already in progress.\n",
602                               qrbuf->QRnumber,
603                               qrbuf->QRname);
604                 pthread_mutex_unlock(&RSSQueueMutex);
605                 return;
606         }
607         pthread_mutex_unlock(&RSSQueueMutex);
608
609         if (server_shutting_down) return;
610
611         pLine = OneRNCFG->NetConfigs[rssclient];
612
613         while (pLine != NULL)
614         {
615                 const char *lPtr = NULL;
616
617                 RSSAggr = (rss_aggregator *) malloc(
618                         sizeof(rss_aggregator));
619
620                 memset (RSSAggr, 0, sizeof(rss_aggregator));
621                 RSSAggr->QRnumber = qrbuf->QRnumber;
622                 RSSAggr->roomlist_parts = 1;
623                 RSSAggr->Url = NewStrBufPlain(NULL, StrLength(pLine->Value[0]));
624                 StrBufExtract_NextToken(RSSAggr->Url,
625                                         pLine->Value[0],
626                                         &lPtr,
627                                         '|');
628
629                 pthread_mutex_lock(&RSSQueueMutex);
630                 GetHash(RSSFetchUrls,
631                         SKEY(RSSAggr->Url),
632                         &vptr);
633
634                 use_this_RSSAggr = (rss_aggregator *)vptr;
635                 if (use_this_RSSAggr != NULL)
636                 {
637                         long *QRnumber;
638                         StrBufAppendBufPlain(
639                                 use_this_RSSAggr->rooms,
640                                 qrbuf->QRname,
641                                 -1, 0);
642                         if (use_this_RSSAggr->roomlist_parts==1)
643                         {
644                                 use_this_RSSAggr->OtherQRnumbers
645                                         = NewHash(1, lFlathash);
646                         }
647                         QRnumber = (long*)malloc(sizeof(long));
648                         *QRnumber = qrbuf->QRnumber;
649                         Put(use_this_RSSAggr->OtherQRnumbers,
650                             LKEY(qrbuf->QRnumber),
651                             QRnumber,
652                             NULL);
653                         use_this_RSSAggr->roomlist_parts++;
654
655                         pthread_mutex_unlock(&RSSQueueMutex);
656
657                         FreeStrBuf(&RSSAggr->Url);
658                         free(RSSAggr);
659                         RSSAggr = NULL;
660                         pLine = pLine->next;
661                         continue;
662                 }
663                 pthread_mutex_unlock(&RSSQueueMutex);
664
665                 RSSAggr->ItemType = RSS_UNSET;
666
667                 RSSAggr->rooms = NewStrBufPlain(
668                         qrbuf->QRname, -1);
669
670                 pthread_mutex_lock(&RSSQueueMutex);
671
672                 Put(RSSFetchUrls,
673                     SKEY(RSSAggr->Url),
674                     RSSAggr,
675                     DeleteRssCfg);
676
677                 pthread_mutex_unlock(&RSSQueueMutex);
678                 pLine = pLine->next;
679         }
680 }
681
682 /*
683  * Scan for rooms that have RSS client requests configured
684  */
685 void rssclient_scan(void) {
686         int RSSRoomCount, RSSCount;
687         rss_aggregator *rptr = NULL;
688         void *vrptr = NULL;
689         HashPos *it;
690         long len;
691         const char *Key;
692         time_t now = time(NULL);
693
694         /* Run no more than once every 15 minutes. */
695         if ((now - last_run) < 900) {
696                 EVRSSQ_syslog(LOG_DEBUG,
697                               "Client: polling interval not yet reached; last run was %ldm%lds ago",
698                               ((now - last_run) / 60),
699                               ((now - last_run) % 60)
700                 );
701                 return;
702         }
703
704         /*
705          * This is a simple concurrency check to make sure only one rssclient
706          * run is done at a time.
707          */
708         pthread_mutex_lock(&RSSQueueMutex);
709         RSSCount = GetCount(RSSFetchUrls);
710         RSSRoomCount = GetCount(RSSQueueRooms);
711         pthread_mutex_unlock(&RSSQueueMutex);
712
713         if ((RSSRoomCount > 0) || (RSSCount > 0)) {
714                 EVRSSQ_syslog(LOG_DEBUG,
715                               "rssclient: concurrency check failed; %d rooms and %d url's are queued",
716                               RSSRoomCount, RSSCount
717                         );
718                 return;
719         }
720
721         become_session(&rss_CC);
722         EVRSSQM_syslog(LOG_DEBUG, "rssclient started");
723         CtdlForEachNetCfgRoom(rssclient_scan_room, NULL, rssclient);
724
725         if (GetCount(RSSFetchUrls) > 0)
726         {
727                 pthread_mutex_lock(&RSSQueueMutex);
728                 EVRSSQ_syslog(LOG_DEBUG,
729                                "rssclient starting %d Clients",
730                                GetCount(RSSFetchUrls));
731                 
732                 it = GetNewHashPos(RSSFetchUrls, 0);
733                 while (!server_shutting_down &&
734                        GetNextHashPos(RSSFetchUrls, it, &len, &Key, &vrptr) &&
735                        (vrptr != NULL)) {
736                         rptr = (rss_aggregator *)vrptr;
737                         if (!rss_do_fetching(rptr))
738                                 UnlinkRSSAggregator(rptr);
739                 }
740                 DeleteHashPos(&it);
741                 pthread_mutex_unlock(&RSSQueueMutex);
742         }
743         else
744                 EVRSSQM_syslog(LOG_DEBUG, "Nothing to do.");
745
746         EVRSSQM_syslog(LOG_DEBUG, "rssclient ended\n");
747         return;
748 }
749
750 void rss_cleanup(void)
751 {
752         /* citthread_mutex_destroy(&RSSQueueMutex); TODO */
753         DeleteHash(&RSSFetchUrls);
754         DeleteHash(&RSSQueueRooms);
755 }
756
757 void LogDebugEnableRSSClient(const int n)
758 {
759         RSSClientDebugEnabled = n;
760 }
761
762 CTDL_MODULE_INIT(rssclient)
763 {
764         if (!threading)
765         {
766                 CtdlREGISTERRoomCfgType(rssclient, ParseGeneric, 0, 1, SerializeGeneric, DeleteGenericCfgLine); /// todo: implement rss specific parser
767                 pthread_mutex_init(&RSSQueueMutex, NULL);
768                 RSSQueueRooms = NewHash(1, lFlathash);
769                 RSSFetchUrls = NewHash(1, NULL);
770                 syslog(LOG_INFO, "%s\n", curl_version());
771                 CtdlRegisterSessionHook(rssclient_scan, EVT_TIMER, PRIO_AGGR + 300);
772                 CtdlRegisterEVCleanupHook(rss_cleanup);
773                 CtdlRegisterDebugFlagHook(HKEY("rssclient"), LogDebugEnableRSSClient, &RSSClientDebugEnabled);
774         }
775         else
776         {
777                 CtdlFillSystemContext(&rss_CC, "rssclient");
778         }
779         return "rssclient";
780 }