X-Git-Url: https://code.citadel.org/?a=blobdiff_plain;f=citadel%2Fmodules%2Frssclient%2Fserv_rssclient.c;h=337ace94a765555d259a8c44b1a7f49617bb710f;hb=30a4090b04dff1084df3789efe78afd1e2bf6d90;hp=5f1357603df48ffdb838948c6f301a4e5fa308a6;hpb=65fdf25a39d0ab07270233c1059c53e437c98639;p=citadel.git diff --git a/citadel/modules/rssclient/serv_rssclient.c b/citadel/modules/rssclient/serv_rssclient.c index 5f1357603..337ace94a 100644 --- a/citadel/modules/rssclient/serv_rssclient.c +++ b/citadel/modules/rssclient/serv_rssclient.c @@ -75,13 +75,13 @@ int RSSClientDebugEnabled = 0; #define EVRSSC_syslog(LEVEL, FORMAT, ...) \ DBGLOG(LEVEL) syslog(LEVEL, \ - "IO[%ld]CC[%d][%ld]RSS" FORMAT, \ - IO->ID, CCID, N, __VA_ARGS__) + "%s[%ld]CC[%d][%ld]RSS" FORMAT, \ + IOSTR, IO->ID, CCID, N, __VA_ARGS__) #define EVRSSCM_syslog(LEVEL, FORMAT) \ DBGLOG(LEVEL) syslog(LEVEL, \ - "IO[%ld]CC[%d][%ld]RSS" FORMAT, \ - IO->ID, CCID, N) + "%s[%ld]CC[%d][%ld]RSS" FORMAT, \ + IOSTR, IO->ID, CCID, N) #define EVRSSQ_syslog(LEVEL, FORMAT, ...) \ DBGLOG(LEVEL) syslog(LEVEL, "RSS" FORMAT, \ @@ -90,8 +90,8 @@ int RSSClientDebugEnabled = 0; DBGLOG(LEVEL) syslog(LEVEL, "RSS" FORMAT) #define EVRSSCSM_syslog(LEVEL, FORMAT) \ - DBGLOG(LEVEL) syslog(LEVEL, "IO[%ld][%ld]RSS" FORMAT, \ - IO->ID, N) + DBGLOG(LEVEL) syslog(LEVEL, "%s[%ld][%ld]RSS" FORMAT, \ + IOSTR, IO->ID, N) typedef enum _RSSState { eRSSCreated, @@ -108,6 +108,37 @@ ConstStr RSSStates[] = { {HKEY("checking usetable")} }; + +static size_t GetLocationString( void *ptr, size_t size, size_t nmemb, void *userdata) +{ +#define LOCATION "location" + if (strncasecmp((char*)ptr, LOCATION, sizeof(LOCATION) - 1) == 0) + { + AsyncIO *IO = (AsyncIO *) userdata; + rss_aggregator *RSSAggr = (rss_aggregator *)IO->Data; + + char *pch = (char*) ptr; + char *pche; + + pche = pch + (size * nmemb); + pch += sizeof(LOCATION); + + while (isspace(*pch) || (*pch == ':')) + pch ++; + + while (isspace(*pche) || (*pche == '\0')) + pche--; + if (RSSAggr->RedirectUrl == NULL) { + RSSAggr->RedirectUrl = NewStrBufPlain(pch, pche - pch + 1); + } + else { + FlushStrBuf(RSSAggr->RedirectUrl); + StrBufPlain(RSSAggr->RedirectUrl, pch, pche - pch + 1); + } + } + return size * nmemb; +} + static void SetRSSState(AsyncIO *IO, RSSState State) { CitContext* CCC = IO->CitContext; @@ -191,6 +222,7 @@ void DeleteRssCfg(void *vptr) EVRSSCM_syslog(LOG_DEBUG, "RSS: destroying\n"); FreeStrBuf(&RSSAggr->Url); + FreeStrBuf(&RSSAggr->RedirectUrl); FreeStrBuf(&RSSAggr->rooms); FreeStrBuf(&RSSAggr->CData); FreeStrBuf(&RSSAggr->Key); @@ -274,11 +306,24 @@ void AppendLink(StrBuf *Message, } -void rss_format_item(networker_save_message *SaveMsg) +int rss_format_item(AsyncIO *IO, networker_save_message *SaveMsg) { StrBuf *Message; int msglen = 0; + if (StrLength(SaveMsg->description) + + StrLength(SaveMsg->link) + + StrLength(SaveMsg->linkTitle) + + StrLength(SaveMsg->reLink) + + StrLength(SaveMsg->reLinkTitle) + + StrLength(SaveMsg->title) == 0) + { + EVRSSCM_syslog(LOG_INFO, "Refusing to save empty message."); + return 0; + } + + CM_Flush(&SaveMsg->Msg); + if (SaveMsg->author_or_creator != NULL) { char *From; @@ -296,36 +341,31 @@ void rss_format_item(networker_save_message *SaveMsg) if (!FromAt && StrLength (SaveMsg->author_email) > 0) { StrBufRFC2047encode(&Encoded, SaveMsg->author_or_creator); - SaveMsg->Msg.cm_fields['A'] = SmashStrBuf(&Encoded); - SaveMsg->Msg.cm_fields['P'] = - SmashStrBuf(&SaveMsg->author_email); + CM_SetAsFieldSB(&SaveMsg->Msg, eAuthor, &Encoded); + CM_SetAsFieldSB(&SaveMsg->Msg, eMessagePath, &SaveMsg->author_email); } else { if (FromAt) { - SaveMsg->Msg.cm_fields['A'] = - SmashStrBuf(&SaveMsg->author_or_creator); - SaveMsg->Msg.cm_fields['P'] = - strdup(SaveMsg->Msg.cm_fields['A']); + CM_SetAsFieldSB(&SaveMsg->Msg, eAuthor, &SaveMsg->author_or_creator); + CM_CopyField(&SaveMsg->Msg, eMessagePath, eAuthor); } else { StrBufRFC2047encode(&Encoded, SaveMsg->author_or_creator); - SaveMsg->Msg.cm_fields['A'] = - SmashStrBuf(&Encoded); - SaveMsg->Msg.cm_fields['P'] = - strdup("rss@localhost"); + CM_SetAsFieldSB(&SaveMsg->Msg, eAuthor, &Encoded); + CM_SetField(&SaveMsg->Msg, eMessagePath, HKEY("rss@localhost")); } } } else { - SaveMsg->Msg.cm_fields['A'] = strdup("rss"); + CM_SetField(&SaveMsg->Msg, eAuthor, HKEY("rss")); } - SaveMsg->Msg.cm_fields['N'] = strdup(NODENAME); + CM_SetField(&SaveMsg->Msg, eNodeName, CFG_KEY(c_nodename)); if (SaveMsg->title != NULL) { long len; char *Sbj; @@ -347,7 +387,7 @@ void rss_format_item(networker_save_message *SaveMsg) StrBufTrim(Encoded); StrBufRFC2047encode(&QPEncoded, Encoded); - SaveMsg->Msg.cm_fields['U'] = SmashStrBuf(&QPEncoded); + CM_SetAsFieldSB(&SaveMsg->Msg, eMsgSubject, &QPEncoded); FreeStrBuf(&Encoded); } if (SaveMsg->link == NULL) @@ -375,8 +415,8 @@ void rss_format_item(networker_save_message *SaveMsg) AppendLink(Message, SaveMsg->reLink, SaveMsg->reLinkTitle, "Reply to this"); StrBufAppendBufPlain(Message, HKEY("\n"), 0); - SaveMsg->Message = Message; + return 1; } eNextState RSSSaveMessage(AsyncIO *IO) @@ -385,16 +425,17 @@ eNextState RSSSaveMessage(AsyncIO *IO) const char *Key; rss_aggregator *RSSAggr = (rss_aggregator *) IO->Data; - rss_format_item(RSSAggr->ThisMsg); - - RSSAggr->ThisMsg->Msg.cm_fields['M'] = - SmashStrBuf(&RSSAggr->ThisMsg->Message); - - CtdlSubmitMsg(&RSSAggr->ThisMsg->Msg, &RSSAggr->recp, NULL, 0); - - /* write the uidl to the use table so we don't store this item again */ + if (rss_format_item(IO, RSSAggr->ThisMsg)) + { + CM_SetAsFieldSB(&RSSAggr->ThisMsg->Msg, eMesageText, + &RSSAggr->ThisMsg->Message); - CheckIfAlreadySeen("RSS Item Insert", RSSAggr->ThisMsg->MsgGUID, IO->Now, 0, eWrite, IO->ID, CCID); + CtdlSubmitMsg(&RSSAggr->ThisMsg->Msg, &RSSAggr->recp, NULL, 0); + + /* write the uidl to the use table so we don't store this item again */ + + CheckIfAlreadySeen("RSS Item Insert", RSSAggr->ThisMsg->MsgGUID, EvGetNow(IO), 0, eWrite, CCID, IO->ID); + } if (GetNextHashPos(RSSAggr->Messages, RSSAggr->Pos, @@ -407,26 +448,30 @@ eNextState RSSSaveMessage(AsyncIO *IO) eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO) { + static const time_t antiExpire = USETABLE_ANTIEXPIRE_HIRES; +#ifndef DEBUG_RSS + time_t seenstamp = 0; const char *Key; long len; rss_aggregator *Ctx = (rss_aggregator *) IO->Data; /* Find out if we've already seen this item */ // todo: expiry? -#ifndef DEBUG_RSS SetRSSState(IO, eRSSUT); - if (CheckIfAlreadySeen("RSS Item Seen", - Ctx->ThisMsg->MsgGUID, - IO->Now, - IO->Now - USETABLE_ANTIEXPIRE, - eCheckUpdate, - IO->ID, CCID) - != 0) + seenstamp = CheckIfAlreadySeen("RSS Item Seen", + Ctx->ThisMsg->MsgGUID, + EvGetNow(IO), + antiExpire, + eCheckUpdate, + CCID, IO->ID); + if (seenstamp != 0) { /* Item has already been seen */ EVRSSC_syslog(LOG_DEBUG, - "%s has already been seen\n", - ChrPtr(Ctx->ThisMsg->MsgGUID)); + "%s has already been seen - %ld < %ld", + ChrPtr(Ctx->ThisMsg->MsgGUID), + seenstamp, antiExpire); + SetRSSState(IO, eRSSParsing); if (GetNextHashPos(Ctx->Messages, @@ -442,6 +487,11 @@ eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO) else #endif { + /* Item has already been seen */ + EVRSSC_syslog(LOG_DEBUG, + "%s Parsing - %ld >= %ld", + ChrPtr(Ctx->ThisMsg->MsgGUID), + seenstamp, antiExpire); SetRSSState(IO, eRSSParsing); NextDBOperation(IO, RSSSaveMessage); @@ -487,36 +537,97 @@ eNextState RSSAggregator_AnalyseReply(AsyncIO *IO) StrBuf *guid; rss_aggregator *Ctx = (rss_aggregator *) IO->Data; - if (IO->HttpReq.httpcode != 200) - { + + if ((IO->HttpReq.httpcode >= 300) && + (IO->HttpReq.httpcode < 400) && + (Ctx->RedirectUrl != NULL)) { + StrBuf *ErrMsg; long lens[2]; const char *strs[2]; SetRSSState(IO, eRSSFailure); ErrMsg = NewStrBuf(); - EVRSSC_syslog(LOG_ALERT, "need a 200, got a %ld !\n", + if (IO) EVRSSC_syslog(LOG_ALERT, "need a 200, got a %ld !\n", IO->HttpReq.httpcode); + strs[0] = ChrPtr(Ctx->Url); + lens[0] = StrLength(Ctx->Url); + + strs[1] = ChrPtr(Ctx->rooms); + lens[1] = StrLength(Ctx->rooms); + + if (IO->HttpReq.CurlError == NULL) + IO->HttpReq.CurlError = ""; + + StrBufPrintf(ErrMsg, + "Error while RSS-Aggregation Run of %s\n" + " need a 200, got a %ld !\n" + " Curl Error message: \n%s / %s\n" + " Redirect header points to: %s\n" + " Response text was: \n" + " \n %s\n", + ChrPtr(Ctx->Url), + IO->HttpReq.httpcode, + IO->HttpReq.errdesc, + IO->HttpReq.CurlError, + ChrPtr(Ctx->RedirectUrl), + ChrPtr(IO->HttpReq.ReplyData) + ); + + CtdlAideFPMessage( + ChrPtr(ErrMsg), + "RSS Aggregation run failure", + 2, strs, (long*) &lens, + CCID, + IO->ID, + EvGetNow(IO)); + FreeStrBuf(&ErrMsg); + EVRSSC_syslog(LOG_DEBUG, + "RSS feed returned an invalid http status code. <%s>\n", + ChrPtr(Ctx->Url), + IO->HttpReq.httpcode); + return eAbort; + } + else if (IO->HttpReq.httpcode != 200) + { + StrBuf *ErrMsg; + long lens[2]; + const char *strs[2]; + + SetRSSState(IO, eRSSFailure); + ErrMsg = NewStrBuf(); + if (IO) EVRSSC_syslog(LOG_ALERT, "need a 200, got a %ld !\n", + IO->HttpReq.httpcode); strs[0] = ChrPtr(Ctx->Url); lens[0] = StrLength(Ctx->Url); strs[1] = ChrPtr(Ctx->rooms); lens[1] = StrLength(Ctx->rooms); + + if (IO->HttpReq.CurlError == NULL) + IO->HttpReq.CurlError = ""; + StrBufPrintf(ErrMsg, "Error while RSS-Aggregation Run of %s\n" " need a 200, got a %ld !\n" + " Curl Error message: \n%s / %s\n" " Response text was: \n" " \n %s\n", ChrPtr(Ctx->Url), IO->HttpReq.httpcode, - ChrPtr(IO->HttpReq.ReplyData)); + IO->HttpReq.errdesc, + IO->HttpReq.CurlError, + ChrPtr(IO->HttpReq.ReplyData) + ); + CtdlAideFPMessage( ChrPtr(ErrMsg), "RSS Aggregation run failure", 2, strs, (long*) &lens, - IO->Now, - IO->ID, CCID); + CCID, + IO->ID, + EvGetNow(IO)); FreeStrBuf(&ErrMsg); EVRSSC_syslog(LOG_DEBUG, @@ -530,7 +641,7 @@ eNextState RSSAggregator_AnalyseReply(AsyncIO *IO) while (pCfg != NULL) { - UpdateLastKnownGood (pCfg, IO->Now); + UpdateLastKnownGood (pCfg, EvGetNow(IO)); if ((Ctx->roomlist_parts > 1) && (it == NULL)) { @@ -572,10 +683,10 @@ eNextState RSSAggregator_AnalyseReply(AsyncIO *IO) if (CheckIfAlreadySeen("RSS Whole", guid, - IO->Now, - IO->Now - USETABLE_ANTIEXPIRE, - eCheckUpdate, - IO->ID, CCID) + EvGetNow(IO), + EvGetNow(IO) - USETABLE_ANTIEXPIRE, + eUpdate, + CCID, IO->ID) != 0) { FreeStrBuf(&guid); @@ -591,7 +702,6 @@ eNextState RSSAggregator_AnalyseReply(AsyncIO *IO) eNextState RSSAggregator_FinishHttp(AsyncIO *IO) { - StopCurlWatchers(IO); return CurlQueueDBOperation(IO, RSSAggregator_AnalyseReply); } @@ -603,6 +713,9 @@ int rss_do_fetching(rss_aggregator *RSSAggr) AsyncIO *IO = &RSSAggr->IO; rss_item *ri; time_t now; + CURLcode sta; + CURL *chnd; + now = time(NULL); @@ -624,6 +737,9 @@ int rss_do_fetching(rss_aggregator *RSSAggr) EVRSSCM_syslog(LOG_ALERT, "Unable to initialize libcurl.\n"); return 0; } + chnd = IO->HttpReq.chnd; + OPT(HEADERDATA, IO); + OPT(HEADERFUNCTION, GetLocationString); SetRSSState(IO, eRSSCreated); safestrncpy(((CitContext*)RSSAggr->IO.CitContext)->cs_host,