X-Git-Url: https://code.citadel.org/?a=blobdiff_plain;f=citadel%2Fmodules%2Frssclient%2Fserv_rssclient.c;h=c2dfa85e1ad1149aff91919df492ff68edade82a;hb=03cd7985fad9d51530f613304869f3b7960b9f81;hp=d15613d4907d591f0b5fa98d376b9bcbb0533e92;hpb=63dc1de06b047b4be691541935e98845457c4c04;p=citadel.git diff --git a/citadel/modules/rssclient/serv_rssclient.c b/citadel/modules/rssclient/serv_rssclient.c index d15613d49..c2dfa85e1 100644 --- a/citadel/modules/rssclient/serv_rssclient.c +++ b/citadel/modules/rssclient/serv_rssclient.c @@ -1,7 +1,7 @@ /* * Bring external RSS feeds into rooms. * - * Copyright (c) 2007-2012 by the citadel.org team + * Copyright (c) 2007-2016 by the citadel.org team * * This program is open source software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 3. @@ -69,30 +69,10 @@ struct CitContext rss_CC; struct rssnetcfg *rnclist = NULL; int RSSClientDebugEnabled = 0; -#define N ((rss_aggregator*)IO->Data)->QRnumber +#define N ((rss_aggregator*)IO->Data)->Cfg.QRnumber #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (RSSClientDebugEnabled != 0)) -#define EVRSSC_syslog(LEVEL, FORMAT, ...) \ - DBGLOG(LEVEL) syslog(LEVEL, \ - "IO[%ld]CC[%d][%ld]RSS" FORMAT, \ - IO->ID, CCID, N, __VA_ARGS__) - -#define EVRSSCM_syslog(LEVEL, FORMAT) \ - DBGLOG(LEVEL) syslog(LEVEL, \ - "IO[%ld]CC[%d][%ld]RSS" FORMAT, \ - IO->ID, CCID, N) - -#define EVRSSQ_syslog(LEVEL, FORMAT, ...) \ - DBGLOG(LEVEL) syslog(LEVEL, "RSS" FORMAT, \ - __VA_ARGS__) -#define EVRSSQM_syslog(LEVEL, FORMAT) \ - DBGLOG(LEVEL) syslog(LEVEL, "RSS" FORMAT) - -#define EVRSSCSM_syslog(LEVEL, FORMAT) \ - DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]RSS" FORMAT, \ - IO->ID, N) - typedef enum _RSSState { eRSSCreated, eRSSFetching, @@ -108,13 +88,47 @@ ConstStr RSSStates[] = { {HKEY("checking usetable")} }; + +static size_t GetLocationString( void *ptr, size_t size, size_t nmemb, void *userdata) +{ +#define LOCATION "location" + if (strncasecmp((char*)ptr, LOCATION, sizeof(LOCATION) - 1) == 0) + { + AsyncIO *IO = (AsyncIO *) userdata; + rss_aggregator *RSSAggr = (rss_aggregator *)IO->Data; + + char *pch = (char*) ptr; + char *pche; + + pche = pch + (size * nmemb); + pch += sizeof(LOCATION); + + while (isspace(*pch) || (*pch == ':')) + pch ++; + + while (isspace(*pche) || (*pche == '\0')) + pche--; + if (RSSAggr->RedirectUrl == NULL) { + RSSAggr->RedirectUrl = NewStrBufPlain(pch, pche - pch + 1); + } + else { + FlushStrBuf(RSSAggr->RedirectUrl); + StrBufPlain(RSSAggr->RedirectUrl, pch, pche - pch + 1); + } + } + return size * nmemb; +} + + static void SetRSSState(AsyncIO *IO, RSSState State) { CitContext* CCC = IO->CitContext; - if (CCC != NULL) + if (CCC != NULL) { memcpy(CCC->cs_clientname, RSSStates[State].Key, RSSStates[State].len + 1); + } } + void DeleteRoomReference(long QRnumber) { HashPos *At; @@ -141,7 +155,7 @@ void DeleteRoomReference(long QRnumber) void UnlinkRooms(rss_aggregator *RSSAggr) { - DeleteRoomReference(RSSAggr->QRnumber); + DeleteRoomReference(RSSAggr->Cfg.QRnumber); if (RSSAggr->OtherQRnumbers != NULL) { long HKLen; @@ -157,8 +171,8 @@ void UnlinkRooms(rss_aggregator *RSSAggr) &vData) && (vData != NULL)) { - long *lData = (long*) vData; - DeleteRoomReference(*lData); + pRSSConfig *Data = (pRSSConfig*) vData; + DeleteRoomReference(Data->QRnumber); } DeleteHashPos(&At); @@ -187,10 +201,12 @@ void DeleteRssCfg(void *vptr) rss_aggregator *RSSAggr = (rss_aggregator *)vptr; AsyncIO *IO = &RSSAggr->IO; - if (IO->CitContext != NULL) - EVRSSCM_syslog(LOG_DEBUG, "RSS: destroying\n"); + if (IO->CitContext != NULL) { + syslog(LOG_DEBUG, "RSS: destroying\n"); + } FreeStrBuf(&RSSAggr->Url); + FreeStrBuf(&RSSAggr->RedirectUrl); FreeStrBuf(&RSSAggr->rooms); FreeStrBuf(&RSSAggr->CData); FreeStrBuf(&RSSAggr->Key); @@ -218,7 +234,7 @@ eNextState RSSAggregator_Terminate(AsyncIO *IO) { rss_aggregator *RSSAggr = (rss_aggregator *)IO->Data; - EVRSSCM_syslog(LOG_DEBUG, "RSS: Terminating.\n"); + syslog(LOG_DEBUG, "RSS: Terminating."); StopCurlWatchers(IO); UnlinkRSSAggregator(RSSAggr); @@ -229,7 +245,7 @@ eNextState RSSAggregator_TerminateDB(AsyncIO *IO) { rss_aggregator *RSSAggr = (rss_aggregator *)IO->Data; - EVRSSCM_syslog(LOG_DEBUG, "RSS: Terminating.\n"); + syslog(LOG_DEBUG, "RSS: Terminating."); StopDBWatchers(&RSSAggr->IO); @@ -246,7 +262,7 @@ eNextState RSSAggregator_ShutdownAbort(AsyncIO *IO) if (pUrl == NULL) pUrl = ""; - EVRSSC_syslog(LOG_DEBUG, "RSS: Aborting by shutdown: %s.\n", pUrl); + syslog(LOG_DEBUG, "RSS: Aborting by shutdown: %s.", pUrl); StopCurlWatchers(IO); UnlinkRSSAggregator(RSSAggr); @@ -274,11 +290,24 @@ void AppendLink(StrBuf *Message, } -void rss_format_item(networker_save_message *SaveMsg) +int rss_format_item(AsyncIO *IO, networker_save_message *SaveMsg) { StrBuf *Message; int msglen = 0; + if (StrLength(SaveMsg->description) + + StrLength(SaveMsg->link) + + StrLength(SaveMsg->linkTitle) + + StrLength(SaveMsg->reLink) + + StrLength(SaveMsg->reLinkTitle) + + StrLength(SaveMsg->title) == 0) + { + syslog(LOG_INFO, "Refusing to save empty message."); + return 0; + } + + CM_Flush(&SaveMsg->Msg); + if (SaveMsg->author_or_creator != NULL) { char *From; @@ -296,36 +325,31 @@ void rss_format_item(networker_save_message *SaveMsg) if (!FromAt && StrLength (SaveMsg->author_email) > 0) { StrBufRFC2047encode(&Encoded, SaveMsg->author_or_creator); - SaveMsg->Msg.cm_fields['A'] = SmashStrBuf(&Encoded); - SaveMsg->Msg.cm_fields['P'] = - SmashStrBuf(&SaveMsg->author_email); + CM_SetAsFieldSB(&SaveMsg->Msg, eAuthor, &Encoded); + CM_SetAsFieldSB(&SaveMsg->Msg, eMessagePath, &SaveMsg->author_email); } else { if (FromAt) { - SaveMsg->Msg.cm_fields['A'] = - SmashStrBuf(&SaveMsg->author_or_creator); - SaveMsg->Msg.cm_fields['P'] = - strdup(SaveMsg->Msg.cm_fields['A']); + CM_SetAsFieldSB(&SaveMsg->Msg, eAuthor, &SaveMsg->author_or_creator); + CM_CopyField(&SaveMsg->Msg, eMessagePath, eAuthor); } else { StrBufRFC2047encode(&Encoded, SaveMsg->author_or_creator); - SaveMsg->Msg.cm_fields['A'] = - SmashStrBuf(&Encoded); - SaveMsg->Msg.cm_fields['P'] = - strdup("rss@localhost"); + CM_SetAsFieldSB(&SaveMsg->Msg, eAuthor, &Encoded); + CM_SetField(&SaveMsg->Msg, eMessagePath, HKEY("rss@localhost")); } } } else { - SaveMsg->Msg.cm_fields['A'] = strdup("rss"); + CM_SetField(&SaveMsg->Msg, eAuthor, HKEY("rss")); } - SaveMsg->Msg.cm_fields['N'] = strdup(NODENAME); + CM_SetField(&SaveMsg->Msg, eNodeName, CtdlGetConfigStr("c_nodename"), strlen(CtdlGetConfigStr("c_nodename"))); if (SaveMsg->title != NULL) { long len; char *Sbj; @@ -335,20 +359,25 @@ void rss_format_item(networker_save_message *SaveMsg) StrBufSpaceToBlank(SaveMsg->title); len = StrLength(SaveMsg->title); Sbj = html_to_ascii(ChrPtr(SaveMsg->title), len, 512, 0); - len = strlen(Sbj); - if ((len > 0) && (Sbj[len - 1] == '\n')) - { - len --; - Sbj[len] = '\0'; - } - Encoded = NewStrBufPlain(Sbj, len); - free(Sbj); - - StrBufTrim(Encoded); - StrBufRFC2047encode(&QPEncoded, Encoded); + if (!IsEmptyStr(Sbj)) { + len = strlen(Sbj); + if ((Sbj[len - 1] == '\n')) + { + len --; + Sbj[len] = '\0'; + } + Encoded = NewStrBufPlain(Sbj, len); + - SaveMsg->Msg.cm_fields['U'] = SmashStrBuf(&QPEncoded); - FreeStrBuf(&Encoded); + StrBufTrim(Encoded); + StrBufRFC2047encode(&QPEncoded, Encoded); + + CM_SetAsFieldSB(&SaveMsg->Msg, eMsgSubject, &QPEncoded); + FreeStrBuf(&Encoded); + } + if (Sbj != NULL) { + free(Sbj); + } } if (SaveMsg->link == NULL) SaveMsg->link = NewStrBufPlain(HKEY("")); @@ -375,8 +404,8 @@ void rss_format_item(networker_save_message *SaveMsg) AppendLink(Message, SaveMsg->reLink, SaveMsg->reLinkTitle, "Reply to this"); StrBufAppendBufPlain(Message, HKEY("\n"), 0); - SaveMsg->Message = Message; + return 1; } eNextState RSSSaveMessage(AsyncIO *IO) @@ -385,16 +414,17 @@ eNextState RSSSaveMessage(AsyncIO *IO) const char *Key; rss_aggregator *RSSAggr = (rss_aggregator *) IO->Data; - rss_format_item(RSSAggr->ThisMsg); - - RSSAggr->ThisMsg->Msg.cm_fields['M'] = - SmashStrBuf(&RSSAggr->ThisMsg->Message); - - CtdlSubmitMsg(&RSSAggr->ThisMsg->Msg, &RSSAggr->recp, NULL, 0); - - /* write the uidl to the use table so we don't store this item again */ + if (rss_format_item(IO, RSSAggr->ThisMsg)) + { + CM_SetAsFieldSB(&RSSAggr->ThisMsg->Msg, eMesageText, + &RSSAggr->ThisMsg->Message); - CheckIfAlreadySeen("RSS Item Insert", RSSAggr->ThisMsg->MsgGUID, IO->Now, 0, eWrite, IO->ID, CCID); + CtdlSubmitMsg(&RSSAggr->ThisMsg->Msg, &RSSAggr->recp, NULL, 0); + + /* write the uidl to the use table so we don't store this item again */ + + CheckIfAlreadySeen("RSS Item Insert", RSSAggr->ThisMsg->MsgGUID, EvGetNow(IO), 0, eWrite, CCID, IO->ID); + } if (GetNextHashPos(RSSAggr->Messages, RSSAggr->Pos, @@ -407,26 +437,27 @@ eNextState RSSSaveMessage(AsyncIO *IO) eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO) { + static const time_t antiExpire = USETABLE_ANTIEXPIRE_HIRES; +#ifndef DEBUG_RSS + time_t seenstamp = 0; const char *Key; long len; rss_aggregator *Ctx = (rss_aggregator *) IO->Data; /* Find out if we've already seen this item */ // todo: expiry? -#ifndef DEBUG_RSS SetRSSState(IO, eRSSUT); - if (CheckIfAlreadySeen("RSS Item Seen", - Ctx->ThisMsg->MsgGUID, - IO->Now, - IO->Now - USETABLE_ANTIEXPIRE, - eCheckUpdate, - IO->ID, CCID) - != 0) + seenstamp = CheckIfAlreadySeen("RSS Item Seen", + Ctx->ThisMsg->MsgGUID, + EvGetNow(IO), + antiExpire, + eCheckUpdate, + CCID, IO->ID); + if (seenstamp != 0) { /* Item has already been seen */ - EVRSSC_syslog(LOG_DEBUG, - "%s has already been seen\n", - ChrPtr(Ctx->ThisMsg->MsgGUID)); + syslog(LOG_DEBUG, "%s has already been seen - %ld < %ld", ChrPtr(Ctx->ThisMsg->MsgGUID), seenstamp, antiExpire); + SetRSSState(IO, eRSSParsing); if (GetNextHashPos(Ctx->Messages, @@ -442,6 +473,11 @@ eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO) else #endif { + /* Item has already been seen */ + syslog(LOG_DEBUG, + "%s Parsing - %ld >= %ld", + ChrPtr(Ctx->ThisMsg->MsgGUID), + seenstamp, antiExpire); SetRSSState(IO, eRSSParsing); NextDBOperation(IO, RSSSaveMessage); @@ -450,14 +486,45 @@ eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO) return eSendMore; } +void UpdateLastKnownGood(pRSSConfig *pCfg, time_t now) +{ + OneRoomNetCfg *pRNCfg; + begin_critical_section(S_NETCONFIGS); + pRNCfg = CtdlGetNetCfgForRoom(pCfg->QRnumber); + if (pRNCfg != NULL) + { + RSSCfgLine *RSSCfg = (RSSCfgLine *)pRNCfg->NetConfigs[rssclient]; + + while (RSSCfg != NULL) + { + if (RSSCfg == pCfg->pCfg) + break; + + RSSCfg = RSSCfg->next; + } + if (RSSCfg != NULL) + { + RSSCfg->last_known_good = now; + } + } + SaveRoomNetConfigFile(pRNCfg, pCfg->QRnumber); + FreeRoomNetworkStruct(&pRNCfg); + end_critical_section(S_NETCONFIGS); +} + eNextState RSSAggregator_AnalyseReply(AsyncIO *IO) { + HashPos *it = NULL; + long len; + const char *Key; + pRSSConfig *pCfg; u_char rawdigest[MD5_DIGEST_LEN]; struct MD5Context md5context; StrBuf *guid; rss_aggregator *Ctx = (rss_aggregator *) IO->Data; - if (IO->HttpReq.httpcode != 200) + + if ((IO->HttpReq.httpcode >= 300) && (IO->HttpReq.httpcode < 400) && (Ctx->RedirectUrl != NULL)) { StrBuf *ErrMsg; long lens[2]; @@ -465,36 +532,122 @@ eNextState RSSAggregator_AnalyseReply(AsyncIO *IO) SetRSSState(IO, eRSSFailure); ErrMsg = NewStrBuf(); - EVRSSC_syslog(LOG_ALERT, "need a 200, got a %ld !\n", - IO->HttpReq.httpcode); + if (IO) { + syslog(LOG_INFO, "need a 200, got a %ld !", IO->HttpReq.httpcode); + } + strs[0] = ChrPtr(Ctx->Url); + lens[0] = StrLength(Ctx->Url); + + strs[1] = ChrPtr(Ctx->rooms); + lens[1] = StrLength(Ctx->rooms); + + if (IO->HttpReq.CurlError == NULL) + IO->HttpReq.CurlError = ""; + + StrBufPrintf(ErrMsg, + "Error while RSS-Aggregation Run of %s\n" + " need a 200, got a %ld !\n" + " Curl Error message: \n%s / %s\n" + " Redirect header points to: %s\n" + " Response text was: \n" + " \n %s\n", + ChrPtr(Ctx->Url), + IO->HttpReq.httpcode, + IO->HttpReq.errdesc, + IO->HttpReq.CurlError, + ChrPtr(Ctx->RedirectUrl), + ChrPtr(IO->HttpReq.ReplyData) + ); + + CtdlAideFPMessage( + ChrPtr(ErrMsg), + "RSS Aggregation run failure", + 2, strs, (long*) &lens, + CCID, + IO->ID, + EvGetNow(IO)); + FreeStrBuf(&ErrMsg); + syslog(LOG_DEBUG, + "RSS feed returned an invalid http status code. <%s>", + ChrPtr(Ctx->Url), + IO->HttpReq.httpcode + ); + return eAbort; + } + else if (IO->HttpReq.httpcode != 200) + { + StrBuf *ErrMsg; + long lens[2]; + const char *strs[2]; + + SetRSSState(IO, eRSSFailure); + ErrMsg = NewStrBuf(); + if (IO) { + syslog(LOG_INFO, "need a 200, got a %ld !", IO->HttpReq.httpcode); + } strs[0] = ChrPtr(Ctx->Url); lens[0] = StrLength(Ctx->Url); strs[1] = ChrPtr(Ctx->rooms); lens[1] = StrLength(Ctx->rooms); + + if (IO->HttpReq.CurlError == NULL) + IO->HttpReq.CurlError = ""; + StrBufPrintf(ErrMsg, "Error while RSS-Aggregation Run of %s\n" " need a 200, got a %ld !\n" + " Curl Error message: \n%s / %s\n" " Response text was: \n" " \n %s\n", ChrPtr(Ctx->Url), IO->HttpReq.httpcode, - ChrPtr(IO->HttpReq.ReplyData)); + IO->HttpReq.errdesc, + IO->HttpReq.CurlError, + ChrPtr(IO->HttpReq.ReplyData) + ); + CtdlAideFPMessage( ChrPtr(ErrMsg), "RSS Aggregation run failure", 2, strs, (long*) &lens, - IO->Now, - IO->ID, CCID); + CCID, + IO->ID, + EvGetNow(IO)); FreeStrBuf(&ErrMsg); - EVRSSC_syslog(LOG_DEBUG, - "RSS feed returned an invalid http status code. <%s>\n", + syslog(LOG_DEBUG, + "RSS feed returned an invalid http status code. <%s>", ChrPtr(Ctx->Url), - IO->HttpReq.httpcode); + IO->HttpReq.httpcode + ); return eAbort; } + + pCfg = &Ctx->Cfg; + + while (pCfg != NULL) + { + UpdateLastKnownGood (pCfg, EvGetNow(IO)); + if ((Ctx->roomlist_parts > 1) && + (it == NULL)) + { + it = GetNewHashPos(RSSFetchUrls, 0); + } + if (it != NULL) + { + void *vptr; + if (GetNextHashPos(Ctx->OtherQRnumbers, it, &len, &Key, &vptr)) + pCfg = vptr; + else + pCfg = NULL; + } + else + pCfg = NULL; + } + DeleteHashPos (&it); + SetRSSState(IO, eRSSUT); MD5Init(&md5context); @@ -518,15 +671,15 @@ eNextState RSSAggregator_AnalyseReply(AsyncIO *IO) if (CheckIfAlreadySeen("RSS Whole", guid, - IO->Now, - IO->Now - USETABLE_ANTIEXPIRE, - eCheckUpdate, - IO->ID, CCID) + EvGetNow(IO), + EvGetNow(IO) - USETABLE_ANTIEXPIRE, + eUpdate, + CCID, IO->ID) != 0) { FreeStrBuf(&guid); - EVRSSC_syslog(LOG_DEBUG, "RSS feed already seen. <%s>\n", ChrPtr(Ctx->Url)); + syslog(LOG_DEBUG, "RSS feed already seen. <%s>", ChrPtr(Ctx->Url)); return eAbort; } FreeStrBuf(&guid); @@ -537,7 +690,7 @@ eNextState RSSAggregator_AnalyseReply(AsyncIO *IO) eNextState RSSAggregator_FinishHttp(AsyncIO *IO) { - return QueueDBOperation(IO, RSSAggregator_AnalyseReply); + return CurlQueueDBOperation(IO, RSSAggregator_AnalyseReply); } /* @@ -548,6 +701,9 @@ int rss_do_fetching(rss_aggregator *RSSAggr) AsyncIO *IO = &RSSAggr->IO; rss_item *ri; time_t now; + CURLcode sta; + CURL *chnd; + now = time(NULL); @@ -566,16 +722,19 @@ int rss_do_fetching(rss_aggregator *RSSAggr) RSSAggregator_TerminateDB, RSSAggregator_ShutdownAbort)) { - EVRSSCM_syslog(LOG_ALERT, "Unable to initialize libcurl.\n"); + syslog(LOG_INFO, "Unable to initialize libcurl."); return 0; } + chnd = IO->HttpReq.chnd; + OPT(HEADERDATA, IO); + OPT(HEADERFUNCTION, GetLocationString); SetRSSState(IO, eRSSCreated); safestrncpy(((CitContext*)RSSAggr->IO.CitContext)->cs_host, ChrPtr(RSSAggr->Url), sizeof(((CitContext*)RSSAggr->IO.CitContext)->cs_host)); - EVRSSC_syslog(LOG_DEBUG, "Fetching RSS feed <%s>\n", ChrPtr(RSSAggr->Url)); + syslog(LOG_DEBUG, "Fetching RSS feed <%s>", ChrPtr(RSSAggr->Url)); ParseURL(&RSSAggr->IO.ConnectMe, RSSAggr->Url, 80); CurlPrepareURL(RSSAggr->IO.ConnectMe); @@ -589,16 +748,17 @@ int rss_do_fetching(rss_aggregator *RSSAggr) */ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG) { - const RoomNetCfgLine *pLine; + const RSSCfgLine *RSSCfg = (RSSCfgLine *)OneRNCFG->NetConfigs[rssclient]; rss_aggregator *RSSAggr = NULL; rss_aggregator *use_this_RSSAggr = NULL; void *vptr; + syslog(LOG_DEBUG, "rssclient_scan_room(%s)", qrbuf->QRname); pthread_mutex_lock(&RSSQueueMutex); if (GetHash(RSSQueueRooms, LKEY(qrbuf->QRnumber), &vptr)) { - EVRSSQ_syslog(LOG_DEBUG, - "rssclient: [%ld] %s already in progress.\n", + syslog(LOG_DEBUG, + "rssclient: [%ld] %s already in progress.", qrbuf->QRnumber, qrbuf->QRname); pthread_mutex_unlock(&RSSQueueMutex); @@ -608,33 +768,18 @@ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneR if (server_shutting_down) return; - pLine = OneRNCFG->NetConfigs[rssclient]; - - while (pLine != NULL) + while (RSSCfg != NULL) { - const char *lPtr = NULL; - - RSSAggr = (rss_aggregator *) malloc( - sizeof(rss_aggregator)); - - memset (RSSAggr, 0, sizeof(rss_aggregator)); - RSSAggr->QRnumber = qrbuf->QRnumber; - RSSAggr->roomlist_parts = 1; - RSSAggr->Url = NewStrBufPlain(NULL, StrLength(pLine->Value[0])); - StrBufExtract_NextToken(RSSAggr->Url, - pLine->Value[0], - &lPtr, - '|'); - pthread_mutex_lock(&RSSQueueMutex); GetHash(RSSFetchUrls, - SKEY(RSSAggr->Url), + SKEY(RSSCfg->Url), &vptr); use_this_RSSAggr = (rss_aggregator *)vptr; if (use_this_RSSAggr != NULL) { - long *QRnumber; + pRSSConfig *pRSSCfg; + StrBufAppendBufPlain( use_this_RSSAggr->rooms, qrbuf->QRname, @@ -644,24 +789,34 @@ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneR use_this_RSSAggr->OtherQRnumbers = NewHash(1, lFlathash); } - QRnumber = (long*)malloc(sizeof(long)); - *QRnumber = qrbuf->QRnumber; + + pRSSCfg = (pRSSConfig *) malloc(sizeof(pRSSConfig)); + + pRSSCfg->QRnumber = qrbuf->QRnumber; + pRSSCfg->pCfg = RSSCfg; + Put(use_this_RSSAggr->OtherQRnumbers, LKEY(qrbuf->QRnumber), - QRnumber, + pRSSCfg, NULL); use_this_RSSAggr->roomlist_parts++; pthread_mutex_unlock(&RSSQueueMutex); - FreeStrBuf(&RSSAggr->Url); - free(RSSAggr); - RSSAggr = NULL; - pLine = pLine->next; + RSSCfg = RSSCfg->next; continue; } pthread_mutex_unlock(&RSSQueueMutex); + RSSAggr = (rss_aggregator *) malloc( + sizeof(rss_aggregator)); + + memset (RSSAggr, 0, sizeof(rss_aggregator)); + RSSAggr->Cfg.QRnumber = qrbuf->QRnumber; + RSSAggr->Cfg.pCfg = RSSCfg; + RSSAggr->roomlist_parts = 1; + RSSAggr->Url = NewStrBufDup(RSSCfg->Url); + RSSAggr->ItemType = RSS_UNSET; RSSAggr->rooms = NewStrBufPlain( @@ -675,7 +830,7 @@ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneR DeleteRssCfg); pthread_mutex_unlock(&RSSQueueMutex); - pLine = pLine->next; + RSSCfg = RSSCfg->next; } } @@ -693,7 +848,7 @@ void rssclient_scan(void) { /* Run no more than once every 15 minutes. */ if ((now - last_run) < 900) { - EVRSSQ_syslog(LOG_DEBUG, + syslog(LOG_DEBUG, "Client: polling interval not yet reached; last run was %ldm%lds ago", ((now - last_run) / 60), ((now - last_run) % 60) @@ -711,7 +866,7 @@ void rssclient_scan(void) { pthread_mutex_unlock(&RSSQueueMutex); if ((RSSRoomCount > 0) || (RSSCount > 0)) { - EVRSSQ_syslog(LOG_DEBUG, + syslog(LOG_DEBUG, "rssclient: concurrency check failed; %d rooms and %d url's are queued", RSSRoomCount, RSSCount ); @@ -719,13 +874,13 @@ void rssclient_scan(void) { } become_session(&rss_CC); - EVRSSQM_syslog(LOG_DEBUG, "rssclient started"); + syslog(LOG_DEBUG, "rssclient started"); CtdlForEachNetCfgRoom(rssclient_scan_room, NULL, rssclient); if (GetCount(RSSFetchUrls) > 0) { pthread_mutex_lock(&RSSQueueMutex); - EVRSSQ_syslog(LOG_DEBUG, + syslog(LOG_DEBUG, "rssclient starting %d Clients", GetCount(RSSFetchUrls)); @@ -740,10 +895,11 @@ void rssclient_scan(void) { DeleteHashPos(&it); pthread_mutex_unlock(&RSSQueueMutex); } - else - EVRSSQM_syslog(LOG_DEBUG, "Nothing to do."); + else { + syslog(LOG_DEBUG, "Nothing to do."); + } - EVRSSQM_syslog(LOG_DEBUG, "rssclient ended\n"); + syslog(LOG_DEBUG, "rssclient ended"); return; } @@ -759,11 +915,93 @@ void LogDebugEnableRSSClient(const int n) RSSClientDebugEnabled = n; } + +typedef struct __RSSVetoInfo { + StrBuf *ErrMsg; + time_t Now; + int Veto; +}RSSVetoInfo; + +void rssclient_veto_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG) +{ + RSSVetoInfo *Info = (RSSVetoInfo *) data; + const RSSCfgLine *RSSCfg = (RSSCfgLine *)OneRNCFG->NetConfigs[rssclient]; + + while (RSSCfg != NULL) + { + if ((RSSCfg->last_known_good != 0) && + (RSSCfg->last_known_good + USETABLE_ANTIEXPIRE < Info->Now)) + { + StrBufAppendPrintf(Info->ErrMsg, + "RSS feed not seen for a %d days:: <", + (Info->Now - RSSCfg->last_known_good) / (24 * 60 * 60)); + + StrBufAppendBuf(Info->ErrMsg, RSSCfg->Url, 0); + StrBufAppendBufPlain(Info->ErrMsg, HKEY(">\n"), 0); + } + RSSCfg = RSSCfg->next; + } +} + +int RSSCheckUsetableVeto(StrBuf *ErrMsg) +{ + RSSVetoInfo Info; + + Info.ErrMsg = ErrMsg; + Info.Now = time (NULL); + Info.Veto = 0; + + CtdlForEachNetCfgRoom(rssclient_veto_scan_room, &Info, rssclient); + + return Info.Veto;; +} + + + + +void ParseRSSClientCfgLine(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *OneRNCFG) +{ + RSSCfgLine *RSSCfg; + + RSSCfg = (RSSCfgLine *) malloc (sizeof(RSSCfgLine)); + RSSCfg->Url = NewStrBufPlain (NULL, StrLength (Line)); + + + StrBufExtract_NextToken(RSSCfg->Url, Line, &LinePos, '|'); + RSSCfg->last_known_good = StrBufExtractNext_long(Line, &LinePos, '|'); + + + RSSCfg->next = (RSSCfgLine *)OneRNCFG->NetConfigs[ThisOne->C]; + OneRNCFG->NetConfigs[ThisOne->C] = (RoomNetCfgLine*) RSSCfg; +} + +void SerializeRSSClientCfgLine(const CfgLineType *ThisOne, StrBuf *OutputBuffer, OneRoomNetCfg *RNCfg, RoomNetCfgLine *data) +{ + RSSCfgLine *RSSCfg = (RSSCfgLine*) data; + + StrBufAppendBufPlain(OutputBuffer, CKEY(ThisOne->Str), 0); + StrBufAppendBufPlain(OutputBuffer, HKEY("|"), 0); + StrBufAppendBufPlain(OutputBuffer, SKEY(RSSCfg->Url), 0); + StrBufAppendPrintf(OutputBuffer, "|%ld\n", RSSCfg->last_known_good); +} + +void DeleteRSSClientCfgLine(const CfgLineType *ThisOne, RoomNetCfgLine **data) +{ + RSSCfgLine *RSSCfg = (RSSCfgLine*) *data; + + FreeStrBuf(&RSSCfg->Url); + free(*data); + *data = NULL; +} + + CTDL_MODULE_INIT(rssclient) { if (!threading) { - CtdlREGISTERRoomCfgType(rssclient, ParseGeneric, 0, 1, SerializeGeneric, DeleteGenericCfgLine); /// todo: implement rss specific parser + CtdlRegisterTDAPVetoHook (RSSCheckUsetableVeto, CDB_USETABLE, 0); + + CtdlREGISTERRoomCfgType(rssclient, ParseRSSClientCfgLine, 0, 1, SerializeRSSClientCfgLine, DeleteRSSClientCfgLine); pthread_mutex_init(&RSSQueueMutex, NULL); RSSQueueRooms = NewHash(1, lFlathash); RSSFetchUrls = NewHash(1, NULL);