2 * Bring external RSS feeds into rooms.
4 * Copyright (c) 2007-2016 by the citadel.org team
6 * This program is open source software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 3.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.See the
12 * GNU General Public License for more details.
19 #if TIME_WITH_SYS_TIME
20 # include <sys/time.h>
33 #include <sys/types.h>
36 #include <curl/curl.h>
37 #include <libcitadel.h>
40 #include "citserver.h"
44 #include "ctdl_module.h"
46 #include "parsedate.h"
48 #include "citadel_dirs.h"
51 #include "event_client.h"
52 #include "rss_atom_parser.h"
55 #define TMP_MSGDATA 0xFF
56 #define TMP_SHORTER_URL_OFFSET 0xFE
57 #define TMP_SHORTER_URLS 0xFD
61 pthread_mutex_t RSSQueueMutex; /* locks the access to the following vars: */
62 HashList *RSSQueueRooms = NULL; /* rss_room_counter */
63 HashList *RSSFetchUrls = NULL; /*->rss_aggregator;->RefCount access locked*/
65 eNextState RSSAggregator_Terminate(AsyncIO *IO);
66 eNextState RSSAggregator_TerminateDB(AsyncIO *IO);
67 eNextState RSSAggregator_ShutdownAbort(AsyncIO *IO);
68 struct CitContext rss_CC;
70 struct rssnetcfg *rnclist = NULL;
71 int RSSClientDebugEnabled = 0;
72 #define N ((rss_aggregator*)IO->Data)->Cfg.QRnumber
74 #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (RSSClientDebugEnabled != 0))
76 #define EVRSSC_syslog(LEVEL, FORMAT, ...) \
77 DBGLOG(LEVEL) syslog(LEVEL, \
78 "%s[%ld]CC[%d][%ld]RSS" FORMAT, \
79 IOSTR, IO->ID, CCID, N, __VA_ARGS__)
81 #define EVRSSCM_syslog(LEVEL, FORMAT) \
82 DBGLOG(LEVEL) syslog(LEVEL, \
83 "%s[%ld]CC[%d][%ld]RSS" FORMAT, \
84 IOSTR, IO->ID, CCID, N)
86 #define EVRSSQ_syslog(LEVEL, FORMAT, ...) \
87 DBGLOG(LEVEL) syslog(LEVEL, "RSS" FORMAT, \
89 #define EVRSSQM_syslog(LEVEL, FORMAT) \
90 DBGLOG(LEVEL) syslog(LEVEL, "RSS" FORMAT)
92 #define EVRSSCSM_syslog(LEVEL, FORMAT) \
93 DBGLOG(LEVEL) syslog(LEVEL, "%s[%ld][%ld]RSS" FORMAT, \
96 typedef enum _RSSState {
103 ConstStr RSSStates[] = {
104 {HKEY("Aggregator created")},
105 {HKEY("Fetching content")},
107 {HKEY("parsing content")},
108 {HKEY("checking usetable")}
112 static size_t GetLocationString( void *ptr, size_t size, size_t nmemb, void *userdata)
114 #define LOCATION "location"
115 if (strncasecmp((char*)ptr, LOCATION, sizeof(LOCATION) - 1) == 0)
117 AsyncIO *IO = (AsyncIO *) userdata;
118 rss_aggregator *RSSAggr = (rss_aggregator *)IO->Data;
120 char *pch = (char*) ptr;
123 pche = pch + (size * nmemb);
124 pch += sizeof(LOCATION);
126 while (isspace(*pch) || (*pch == ':'))
129 while (isspace(*pche) || (*pche == '\0'))
131 if (RSSAggr->RedirectUrl == NULL) {
132 RSSAggr->RedirectUrl = NewStrBufPlain(pch, pche - pch + 1);
135 FlushStrBuf(RSSAggr->RedirectUrl);
136 StrBufPlain(RSSAggr->RedirectUrl, pch, pche - pch + 1);
143 static void SetRSSState(AsyncIO *IO, RSSState State)
145 CitContext* CCC = IO->CitContext;
147 memcpy(CCC->cs_clientname, RSSStates[State].Key, RSSStates[State].len + 1);
152 void DeleteRoomReference(long QRnumber)
158 rss_room_counter *pRoomC;
160 At = GetNewHashPos(RSSQueueRooms, 0);
162 if (GetHashPosFromKey(RSSQueueRooms, LKEY(QRnumber), At))
164 GetHashPos(RSSQueueRooms, At, &HKLen, &HK, &vData);
167 pRoomC = (rss_room_counter *) vData;
169 if (pRoomC->count == 0)
170 DeleteEntryFromHash(RSSQueueRooms, At);
176 void UnlinkRooms(rss_aggregator *RSSAggr)
178 DeleteRoomReference(RSSAggr->Cfg.QRnumber);
179 if (RSSAggr->OtherQRnumbers != NULL)
186 At = GetNewHashPos(RSSAggr->OtherQRnumbers, 0);
187 while (! server_shutting_down &&
188 GetNextHashPos(RSSAggr->OtherQRnumbers,
194 pRSSConfig *Data = (pRSSConfig*) vData;
195 DeleteRoomReference(Data->QRnumber);
202 void UnlinkRSSAggregator(rss_aggregator *RSSAggr)
206 pthread_mutex_lock(&RSSQueueMutex);
207 UnlinkRooms(RSSAggr);
209 At = GetNewHashPos(RSSFetchUrls, 0);
210 if (GetHashPosFromKey(RSSFetchUrls, SKEY(RSSAggr->Url), At))
212 DeleteEntryFromHash(RSSFetchUrls, At);
215 last_run = time(NULL);
216 pthread_mutex_unlock(&RSSQueueMutex);
219 void DeleteRssCfg(void *vptr)
221 rss_aggregator *RSSAggr = (rss_aggregator *)vptr;
222 AsyncIO *IO = &RSSAggr->IO;
224 if (IO->CitContext != NULL) {
225 EVRSSCM_syslog(LOG_DEBUG, "RSS: destroying");
228 FreeStrBuf(&RSSAggr->Url);
229 FreeStrBuf(&RSSAggr->RedirectUrl);
230 FreeStrBuf(&RSSAggr->rooms);
231 FreeStrBuf(&RSSAggr->CData);
232 FreeStrBuf(&RSSAggr->Key);
233 DeleteHash(&RSSAggr->OtherQRnumbers);
235 DeleteHashPos (&RSSAggr->Pos);
236 DeleteHash (&RSSAggr->Messages);
237 if (RSSAggr->recp.recp_room != NULL)
238 free(RSSAggr->recp.recp_room);
241 if (RSSAggr->Item != NULL)
243 flush_rss_item(RSSAggr->Item);
248 FreeAsyncIOContents(&RSSAggr->IO);
249 memset(RSSAggr, 0, sizeof(rss_aggregator));
253 eNextState RSSAggregator_Terminate(AsyncIO *IO)
255 rss_aggregator *RSSAggr = (rss_aggregator *)IO->Data;
257 EVRSSCM_syslog(LOG_DEBUG, "RSS: Terminating.");
259 StopCurlWatchers(IO);
260 UnlinkRSSAggregator(RSSAggr);
264 eNextState RSSAggregator_TerminateDB(AsyncIO *IO)
266 rss_aggregator *RSSAggr = (rss_aggregator *)IO->Data;
268 EVRSSCM_syslog(LOG_DEBUG, "RSS: Terminating.");
271 StopDBWatchers(&RSSAggr->IO);
272 UnlinkRSSAggregator(RSSAggr);
276 eNextState RSSAggregator_ShutdownAbort(AsyncIO *IO)
279 rss_aggregator *RSSAggr = (rss_aggregator *)IO->Data;
281 pUrl = IO->ConnectMe->PlainUrl;
285 EVRSSC_syslog(LOG_DEBUG, "RSS: Aborting by shutdown: %s.", pUrl);
287 StopCurlWatchers(IO);
288 UnlinkRSSAggregator(RSSAggr);
292 void AppendLink(StrBuf *Message,
297 if (StrLength(link) > 0)
299 StrBufAppendBufPlain(Message, HKEY("<a href=\""), 0);
300 StrBufAppendBuf(Message, link, 0);
301 StrBufAppendBufPlain(Message, HKEY("\">"), 0);
302 if (StrLength(LinkTitle) > 0)
303 StrBufAppendBuf(Message, LinkTitle, 0);
304 else if ((Title != NULL) && !IsEmptyStr(Title))
305 StrBufAppendBufPlain(Message, Title, -1, 0);
307 StrBufAppendBuf(Message, link, 0);
308 StrBufAppendBufPlain(Message, HKEY("</a><br>\n"), 0);
313 int rss_format_item(AsyncIO *IO, networker_save_message *SaveMsg)
318 if (StrLength(SaveMsg->description) +
319 StrLength(SaveMsg->link) +
320 StrLength(SaveMsg->linkTitle) +
321 StrLength(SaveMsg->reLink) +
322 StrLength(SaveMsg->reLinkTitle) +
323 StrLength(SaveMsg->title) == 0)
325 EVRSSCM_syslog(LOG_INFO, "Refusing to save empty message.");
329 CM_Flush(&SaveMsg->Msg);
331 if (SaveMsg->author_or_creator != NULL) {
334 StrBuf *Encoded = NULL;
337 From = html_to_ascii(ChrPtr(SaveMsg->author_or_creator),
338 StrLength(SaveMsg->author_or_creator),
340 StrBufPlain(SaveMsg->author_or_creator, From, -1);
341 StrBufTrim(SaveMsg->author_or_creator);
344 FromAt = strchr(ChrPtr(SaveMsg->author_or_creator), '@') != NULL;
345 if (!FromAt && StrLength (SaveMsg->author_email) > 0)
347 StrBufRFC2047encode(&Encoded, SaveMsg->author_or_creator);
348 CM_SetAsFieldSB(&SaveMsg->Msg, eAuthor, &Encoded);
349 CM_SetAsFieldSB(&SaveMsg->Msg, eMessagePath, &SaveMsg->author_email);
355 CM_SetAsFieldSB(&SaveMsg->Msg, eAuthor, &SaveMsg->author_or_creator);
356 CM_CopyField(&SaveMsg->Msg, eMessagePath, eAuthor);
360 StrBufRFC2047encode(&Encoded,
361 SaveMsg->author_or_creator);
362 CM_SetAsFieldSB(&SaveMsg->Msg, eAuthor, &Encoded);
363 CM_SetField(&SaveMsg->Msg, eMessagePath, HKEY("rss@localhost"));
369 CM_SetField(&SaveMsg->Msg, eAuthor, HKEY("rss"));
372 CM_SetField(&SaveMsg->Msg, eNodeName, CtdlGetConfigStr("c_nodename"), strlen(CtdlGetConfigStr("c_nodename")));
373 if (SaveMsg->title != NULL) {
376 StrBuf *Encoded, *QPEncoded;
379 StrBufSpaceToBlank(SaveMsg->title);
380 len = StrLength(SaveMsg->title);
381 Sbj = html_to_ascii(ChrPtr(SaveMsg->title), len, 512, 0);
382 if (!IsEmptyStr(Sbj)) {
384 if ((Sbj[len - 1] == '\n'))
389 Encoded = NewStrBufPlain(Sbj, len);
393 StrBufRFC2047encode(&QPEncoded, Encoded);
395 CM_SetAsFieldSB(&SaveMsg->Msg, eMsgSubject, &QPEncoded);
396 FreeStrBuf(&Encoded);
402 if (SaveMsg->link == NULL)
403 SaveMsg->link = NewStrBufPlain(HKEY(""));
405 #if 0 /* temporarily disable shorter urls. */
406 SaveMsg->Msg.cm_fields[TMP_SHORTER_URLS] =
407 GetShorterUrls(SaveMsg->description);
410 msglen += 1024 + StrLength(SaveMsg->link) + StrLength(SaveMsg->description) ;
412 Message = NewStrBufPlain(NULL, msglen);
414 StrBufPlain(Message, HKEY(
415 "Content-type: text/html; charset=\"UTF-8\"\r\n\r\n"
417 #if 0 /* disable shorter url for now. */
418 SaveMsg->Msg.cm_fields[TMP_SHORTER_URL_OFFSET] = StrLength(Message);
420 StrBufAppendBuf(Message, SaveMsg->description, 0);
421 StrBufAppendBufPlain(Message, HKEY("<br><br>\n"), 0);
423 AppendLink(Message, SaveMsg->link, SaveMsg->linkTitle, NULL);
424 AppendLink(Message, SaveMsg->reLink, SaveMsg->reLinkTitle, "Reply to this");
425 StrBufAppendBufPlain(Message, HKEY("</body></html>\n"), 0);
427 SaveMsg->Message = Message;
431 eNextState RSSSaveMessage(AsyncIO *IO)
435 rss_aggregator *RSSAggr = (rss_aggregator *) IO->Data;
437 if (rss_format_item(IO, RSSAggr->ThisMsg))
439 CM_SetAsFieldSB(&RSSAggr->ThisMsg->Msg, eMesageText, &RSSAggr->ThisMsg->Message);
440 CtdlSubmitMsg(&RSSAggr->ThisMsg->Msg, &RSSAggr->recp, NULL, 0);
442 /* write the uidl to the use table so we don't store this item again */
443 CheckIfAlreadySeen("RSS Item Insert", RSSAggr->ThisMsg->MsgGUID, EvGetNow(IO), 0, eWrite, CCID, IO->ID);
446 if (GetNextHashPos(RSSAggr->Messages,
449 (void**) &RSSAggr->ThisMsg))
450 return NextDBOperation(IO, RSS_FetchNetworkUsetableEntry);
455 eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO)
457 static const time_t antiExpire = USETABLE_ANTIEXPIRE_HIRES;
459 time_t seenstamp = 0;
462 rss_aggregator *Ctx = (rss_aggregator *) IO->Data;
464 /* Find out if we've already seen this item */
466 SetRSSState(IO, eRSSUT);
467 seenstamp = CheckIfAlreadySeen("RSS Item Seen",
468 Ctx->ThisMsg->MsgGUID,
475 /* Item has already been seen */
476 EVRSSC_syslog(LOG_DEBUG,
477 "%s has already been seen - %ld < %ld",
478 ChrPtr(Ctx->ThisMsg->MsgGUID),
479 seenstamp, antiExpire);
481 SetRSSState(IO, eRSSParsing);
483 if (GetNextHashPos(Ctx->Messages,
486 (void**) &Ctx->ThisMsg))
487 return NextDBOperation(
489 RSS_FetchNetworkUsetableEntry);
496 /* Item has already been seen */
497 EVRSSC_syslog(LOG_DEBUG,
498 "%s Parsing - %ld >= %ld",
499 ChrPtr(Ctx->ThisMsg->MsgGUID),
500 seenstamp, antiExpire);
501 SetRSSState(IO, eRSSParsing);
503 NextDBOperation(IO, RSSSaveMessage);
509 void UpdateLastKnownGood(pRSSConfig *pCfg, time_t now)
511 OneRoomNetCfg *pRNCfg;
512 begin_critical_section(S_NETCONFIGS);
513 pRNCfg = CtdlGetNetCfgForRoom(pCfg->QRnumber);
516 RSSCfgLine *RSSCfg = (RSSCfgLine *)pRNCfg->NetConfigs[rssclient];
518 while (RSSCfg != NULL)
520 if (RSSCfg == pCfg->pCfg)
523 RSSCfg = RSSCfg->next;
527 RSSCfg->last_known_good = now;
530 SaveRoomNetConfigFile(pRNCfg, pCfg->QRnumber);
531 FreeRoomNetworkStruct(&pRNCfg);
532 end_critical_section(S_NETCONFIGS);
535 eNextState RSSAggregator_AnalyseReply(AsyncIO *IO)
541 u_char rawdigest[MD5_DIGEST_LEN];
542 struct MD5Context md5context;
544 rss_aggregator *Ctx = (rss_aggregator *) IO->Data;
547 if ((IO->HttpReq.httpcode >= 300) && (IO->HttpReq.httpcode < 400) && (Ctx->RedirectUrl != NULL))
553 SetRSSState(IO, eRSSFailure);
554 ErrMsg = NewStrBuf();
556 EVRSSC_syslog(LOG_INFO, "need a 200, got a %ld !",
557 IO->HttpReq.httpcode);
559 strs[0] = ChrPtr(Ctx->Url);
560 lens[0] = StrLength(Ctx->Url);
562 strs[1] = ChrPtr(Ctx->rooms);
563 lens[1] = StrLength(Ctx->rooms);
565 if (IO->HttpReq.CurlError == NULL)
566 IO->HttpReq.CurlError = "";
569 "Error while RSS-Aggregation Run of %s\n"
570 " need a 200, got a %ld !\n"
571 " Curl Error message: \n%s / %s\n"
572 " Redirect header points to: %s\n"
573 " Response text was: \n"
576 IO->HttpReq.httpcode,
578 IO->HttpReq.CurlError,
579 ChrPtr(Ctx->RedirectUrl),
580 ChrPtr(IO->HttpReq.ReplyData)
585 "RSS Aggregation run failure",
586 2, strs, (long*) &lens,
592 EVRSSC_syslog(LOG_DEBUG,
593 "RSS feed returned an invalid http status code. <%s><HTTP %ld>",
595 IO->HttpReq.httpcode);
598 else if (IO->HttpReq.httpcode != 200)
604 SetRSSState(IO, eRSSFailure);
605 ErrMsg = NewStrBuf();
607 EVRSSC_syslog(LOG_ALERT, "need a 200, got a %ld !",
608 IO->HttpReq.httpcode);
610 strs[0] = ChrPtr(Ctx->Url);
611 lens[0] = StrLength(Ctx->Url);
613 strs[1] = ChrPtr(Ctx->rooms);
614 lens[1] = StrLength(Ctx->rooms);
616 if (IO->HttpReq.CurlError == NULL)
617 IO->HttpReq.CurlError = "";
620 "Error while RSS-Aggregation Run of %s\n"
621 " need a 200, got a %ld !\n"
622 " Curl Error message: \n%s / %s\n"
623 " Response text was: \n"
626 IO->HttpReq.httpcode,
628 IO->HttpReq.CurlError,
629 ChrPtr(IO->HttpReq.ReplyData)
634 "RSS Aggregation run failure",
635 2, strs, (long*) &lens,
641 EVRSSC_syslog(LOG_DEBUG,
642 "RSS feed returned an invalid http status code. <%s><HTTP %ld>",
644 IO->HttpReq.httpcode);
652 UpdateLastKnownGood (pCfg, EvGetNow(IO));
653 if ((Ctx->roomlist_parts > 1) &&
656 it = GetNewHashPos(RSSFetchUrls, 0);
661 if (GetNextHashPos(Ctx->OtherQRnumbers, it, &len, &Key, &vptr))
671 SetRSSState(IO, eRSSUT);
673 MD5Init(&md5context);
675 MD5Update(&md5context,
676 (const unsigned char*)SKEY(IO->HttpReq.ReplyData));
678 MD5Update(&md5context,
679 (const unsigned char*)SKEY(Ctx->Url));
681 MD5Final(rawdigest, &md5context);
682 guid = NewStrBufPlain(NULL,
683 MD5_DIGEST_LEN * 2 + 12 /* _rss2ctdl*/);
684 StrBufHexEscAppend(guid, NULL, rawdigest, MD5_DIGEST_LEN);
685 StrBufAppendBufPlain(guid, HKEY("_rssFM"), 0);
686 if (StrLength(guid) > 40)
687 StrBufCutAt(guid, 40, NULL);
688 /* Find out if we've already seen this item */
692 if (CheckIfAlreadySeen("RSS Whole",
695 EvGetNow(IO) - USETABLE_ANTIEXPIRE,
702 EVRSSC_syslog(LOG_DEBUG, "RSS feed already seen. <%s>", ChrPtr(Ctx->Url));
707 SetRSSState(IO, eRSSParsing);
708 return RSSAggregator_ParseReply(IO);
711 eNextState RSSAggregator_FinishHttp(AsyncIO *IO)
713 return CurlQueueDBOperation(IO, RSSAggregator_AnalyseReply);
719 int rss_do_fetching(rss_aggregator *RSSAggr)
721 AsyncIO *IO = &RSSAggr->IO;
730 if ((RSSAggr->next_poll != 0) && (now < RSSAggr->next_poll))
733 ri = (rss_item*) malloc(sizeof(rss_item));
734 memset(ri, 0, sizeof(rss_item));
737 if (! InitcURLIOStruct(&RSSAggr->IO,
739 "Citadel RSS Client",
740 RSSAggregator_FinishHttp,
741 RSSAggregator_Terminate,
742 RSSAggregator_TerminateDB,
743 RSSAggregator_ShutdownAbort))
745 EVRSSCM_syslog(LOG_ALERT, "Unable to initialize libcurl.");
748 chnd = IO->HttpReq.chnd;
750 OPT(HEADERFUNCTION, GetLocationString);
751 SetRSSState(IO, eRSSCreated);
753 safestrncpy(((CitContext*)RSSAggr->IO.CitContext)->cs_host,
754 ChrPtr(RSSAggr->Url),
755 sizeof(((CitContext*)RSSAggr->IO.CitContext)->cs_host));
757 EVRSSC_syslog(LOG_DEBUG, "Fetching RSS feed <%s>", ChrPtr(RSSAggr->Url));
758 ParseURL(&RSSAggr->IO.ConnectMe, RSSAggr->Url, 80);
759 CurlPrepareURL(RSSAggr->IO.ConnectMe);
761 SetRSSState(IO, eRSSFetching);
762 QueueCurlContext(&RSSAggr->IO);
767 * Scan a room's netconfig to determine whether it is requesting any RSS feeds
769 void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
771 const RSSCfgLine *RSSCfg = (RSSCfgLine *)OneRNCFG->NetConfigs[rssclient];
772 rss_aggregator *RSSAggr = NULL;
773 rss_aggregator *use_this_RSSAggr = NULL;
776 EVRSSQ_syslog(LOG_DEBUG, "rssclient_scan_room(%s)", qrbuf->QRname);
777 pthread_mutex_lock(&RSSQueueMutex);
778 if (GetHash(RSSQueueRooms, LKEY(qrbuf->QRnumber), &vptr))
780 EVRSSQ_syslog(LOG_DEBUG,
781 "rssclient: [%ld] %s already in progress.",
784 pthread_mutex_unlock(&RSSQueueMutex);
787 pthread_mutex_unlock(&RSSQueueMutex);
789 if (server_shutting_down) return;
791 while (RSSCfg != NULL)
793 pthread_mutex_lock(&RSSQueueMutex);
794 GetHash(RSSFetchUrls, SKEY(RSSCfg->Url), &vptr);
796 use_this_RSSAggr = (rss_aggregator *)vptr;
797 if (use_this_RSSAggr != NULL)
801 StrBufAppendBufPlain(use_this_RSSAggr->rooms, qrbuf->QRname, -1, 0);
802 if (use_this_RSSAggr->roomlist_parts==1)
804 use_this_RSSAggr->OtherQRnumbers
805 = NewHash(1, lFlathash);
808 pRSSCfg = (pRSSConfig *) malloc(sizeof(pRSSConfig));
810 pRSSCfg->QRnumber = qrbuf->QRnumber;
811 pRSSCfg->pCfg = RSSCfg;
813 Put(use_this_RSSAggr->OtherQRnumbers,
814 LKEY(qrbuf->QRnumber),
817 use_this_RSSAggr->roomlist_parts++;
819 pthread_mutex_unlock(&RSSQueueMutex);
821 RSSCfg = RSSCfg->next;
824 pthread_mutex_unlock(&RSSQueueMutex);
826 RSSAggr = (rss_aggregator *) malloc(
827 sizeof(rss_aggregator));
829 memset (RSSAggr, 0, sizeof(rss_aggregator));
830 RSSAggr->Cfg.QRnumber = qrbuf->QRnumber;
831 RSSAggr->Cfg.pCfg = RSSCfg;
832 RSSAggr->roomlist_parts = 1;
833 RSSAggr->Url = NewStrBufDup(RSSCfg->Url);
835 RSSAggr->ItemType = RSS_UNSET;
837 RSSAggr->rooms = NewStrBufPlain(
840 pthread_mutex_lock(&RSSQueueMutex);
847 pthread_mutex_unlock(&RSSQueueMutex);
848 RSSCfg = RSSCfg->next;
853 * Scan for rooms that have RSS client requests configured
855 void rssclient_scan(void) {
856 int RSSRoomCount, RSSCount;
857 rss_aggregator *rptr = NULL;
862 time_t now = time(NULL);
864 /* Run no more than once every 15 minutes. */
865 if ((now - last_run) < 900) {
866 EVRSSQ_syslog(LOG_DEBUG,
867 "Client: polling interval not yet reached; last run was %ldm%lds ago",
868 ((now - last_run) / 60),
869 ((now - last_run) % 60)
875 * This is a simple concurrency check to make sure only one rssclient
876 * run is done at a time.
878 pthread_mutex_lock(&RSSQueueMutex);
879 RSSCount = GetCount(RSSFetchUrls);
880 RSSRoomCount = GetCount(RSSQueueRooms);
881 pthread_mutex_unlock(&RSSQueueMutex);
883 if ((RSSRoomCount > 0) || (RSSCount > 0)) {
884 EVRSSQ_syslog(LOG_DEBUG,
885 "rssclient: concurrency check failed; %d rooms and %d url's are queued",
886 RSSRoomCount, RSSCount
892 become_session(&rss_CC);
893 EVRSSQM_syslog(LOG_DEBUG, "rssclient started");
894 CtdlForEachNetCfgRoom(rssclient_scan_room, NULL);
896 if (GetCount(RSSFetchUrls) > 0)
898 pthread_mutex_lock(&RSSQueueMutex);
899 EVRSSQ_syslog(LOG_DEBUG,
900 "rssclient starting %d Clients",
901 GetCount(RSSFetchUrls));
903 it = GetNewHashPos(RSSFetchUrls, 0);
904 while (!server_shutting_down &&
905 GetNextHashPos(RSSFetchUrls, it, &len, &Key, &vrptr) &&
907 rptr = (rss_aggregator *)vrptr;
908 if (!rss_do_fetching(rptr))
909 UnlinkRSSAggregator(rptr);
912 pthread_mutex_unlock(&RSSQueueMutex);
915 EVRSSQM_syslog(LOG_DEBUG, "Nothing to do.");
917 EVRSSQM_syslog(LOG_DEBUG, "rssclient ended");
921 void rss_cleanup(void)
923 /* citthread_mutex_destroy(&RSSQueueMutex); TODO */
924 DeleteHash(&RSSFetchUrls);
925 DeleteHash(&RSSQueueRooms);
928 void LogDebugEnableRSSClient(const int n)
930 RSSClientDebugEnabled = n;
934 typedef struct __RSSVetoInfo {
940 void rssclient_veto_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
942 RSSVetoInfo *Info = (RSSVetoInfo *) data;
943 const RSSCfgLine *RSSCfg = (RSSCfgLine *)OneRNCFG->NetConfigs[rssclient];
945 while (RSSCfg != NULL)
947 if ((RSSCfg->last_known_good != 0) &&
948 (RSSCfg->last_known_good + USETABLE_ANTIEXPIRE < Info->Now))
950 StrBufAppendPrintf(Info->ErrMsg,
951 "RSS feed not seen for a %d days:: <",
952 (Info->Now - RSSCfg->last_known_good) / (24 * 60 * 60));
954 StrBufAppendBuf(Info->ErrMsg, RSSCfg->Url, 0);
955 StrBufAppendBufPlain(Info->ErrMsg, HKEY(">\n"), 0);
957 RSSCfg = RSSCfg->next;
961 int RSSCheckUsetableVeto(StrBuf *ErrMsg)
965 Info.ErrMsg = ErrMsg;
966 Info.Now = time (NULL);
969 CtdlForEachNetCfgRoom(rssclient_veto_scan_room, &Info);
977 void ParseRSSClientCfgLine(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *OneRNCFG)
981 RSSCfg = (RSSCfgLine *) malloc (sizeof(RSSCfgLine));
982 RSSCfg->Url = NewStrBufPlain (NULL, StrLength (Line));
985 StrBufExtract_NextToken(RSSCfg->Url, Line, &LinePos, '|');
986 RSSCfg->last_known_good = StrBufExtractNext_long(Line, &LinePos, '|');
989 RSSCfg->next = (RSSCfgLine *)OneRNCFG->NetConfigs[ThisOne->C];
990 OneRNCFG->NetConfigs[ThisOne->C] = (RoomNetCfgLine*) RSSCfg;
993 void SerializeRSSClientCfgLine(const CfgLineType *ThisOne, StrBuf *OutputBuffer, OneRoomNetCfg *RNCfg, RoomNetCfgLine *data)
995 RSSCfgLine *RSSCfg = (RSSCfgLine*) data;
997 StrBufAppendBufPlain(OutputBuffer, CKEY(ThisOne->Str), 0);
998 StrBufAppendBufPlain(OutputBuffer, HKEY("|"), 0);
999 StrBufAppendBufPlain(OutputBuffer, SKEY(RSSCfg->Url), 0);
1000 StrBufAppendPrintf(OutputBuffer, "|%ld\n", RSSCfg->last_known_good);
1003 void DeleteRSSClientCfgLine(const CfgLineType *ThisOne, RoomNetCfgLine **data)
1005 RSSCfgLine *RSSCfg = (RSSCfgLine*) *data;
1007 FreeStrBuf(&RSSCfg->Url);
1013 CTDL_MODULE_INIT(rssclient)
1017 CtdlRegisterTDAPVetoHook (RSSCheckUsetableVeto, CDB_USETABLE, 0);
1019 CtdlREGISTERRoomCfgType(rssclient, ParseRSSClientCfgLine, 0, 1, SerializeRSSClientCfgLine, DeleteRSSClientCfgLine);
1020 pthread_mutex_init(&RSSQueueMutex, NULL);
1021 RSSQueueRooms = NewHash(1, lFlathash);
1022 RSSFetchUrls = NewHash(1, NULL);
1023 syslog(LOG_INFO, "%s\n", curl_version());
1024 CtdlRegisterSessionHook(rssclient_scan, EVT_TIMER, PRIO_AGGR + 300);
1025 CtdlRegisterEVCleanupHook(rss_cleanup);
1026 CtdlRegisterDebugFlagHook(HKEY("rssclient"), LogDebugEnableRSSClient, &RSSClientDebugEnabled);
1030 CtdlFillSystemContext(&rss_CC, "rssclient");