X-Git-Url: https://code.citadel.org/?a=blobdiff_plain;f=citadel%2Fmodules%2Frssclient%2Fserv_rssclient.c;h=1e43e96c765977ba9438338292fcf939b2590c8a;hb=2e82ffdd5614745a54c729a48bc640c2aaee6405;hp=b55260624f29621aaade2b1abba09bb899e41ced;hpb=9dbc5955d83d3790c797f4485f3bfd120da835cf;p=citadel.git diff --git a/citadel/modules/rssclient/serv_rssclient.c b/citadel/modules/rssclient/serv_rssclient.c index b55260624..1e43e96c7 100644 --- a/citadel/modules/rssclient/serv_rssclient.c +++ b/citadel/modules/rssclient/serv_rssclient.c @@ -164,7 +164,9 @@ void DeleteRssCfg(void *vptr) { rss_aggregator *RSSAggr = (rss_aggregator *)vptr; AsyncIO *IO = &RSSAggr->IO; - EVRSSCM_syslog(LOG_DEBUG, "RSS: destroying\n"); + + if (IO->CitContext != NULL) + EVRSSCM_syslog(LOG_DEBUG, "RSS: destroying\n"); FreeStrBuf(&RSSAggr->Url); FreeStrBuf(&RSSAggr->rooms); @@ -438,10 +440,32 @@ eNextState RSSAggregator_AnalyseReply(AsyncIO *IO) if (IO->HttpReq.httpcode != 200) { + StrBuf *ErrMsg; + long lens[2]; + const char *strs[2]; + ErrMsg = NewStrBuf(); EVRSSC_syslog(LOG_ALERT, "need a 200, got a %ld !\n", IO->HttpReq.httpcode); -// TODO: aide error message with rate limit + + strs[0] = ChrPtr(Ctx->Url); + lens[0] = StrLength(Ctx->Url); + + strs[1] = ChrPtr(Ctx->rooms); + lens[1] = StrLength(Ctx->rooms); + StrBufPrintf(ErrMsg, + "Error while RSS-Aggregation Run of %s\n" + " need a 200, got a %ld !\n" + " Response text was: \n" + " \n %s\n", + ChrPtr(Ctx->Url), + IO->HttpReq.httpcode, + ChrPtr(IO->HttpReq.ReplyData)); + CtdlAideFPMessage( + ChrPtr(ErrMsg), + "RSS Aggregation run failure", + 2, strs, (long*) &lens); + FreeStrBuf(&ErrMsg); return eAbort; } @@ -461,24 +485,33 @@ eNextState RSSAggregator_AnalyseReply(AsyncIO *IO) if (StrLength(guid) > 40) StrBufCutAt(guid, 40, NULL); /* Find out if we've already seen this item */ - memcpy(ut.ut_msgid, SKEY(guid)); - ut.ut_timestamp = time(NULL); - cdbut = cdb_fetch(CDB_USETABLE, SKEY(guid)); #ifndef DEBUG_RSS + cdbut = cdb_fetch(CDB_USETABLE, SKEY(guid)); if (cdbut != NULL) { - /* Item has already been seen */ - EVRSSC_syslog(LOG_DEBUG, - "%s has already been seen\n", - ChrPtr(Ctx->Url)); + memcpy(&ut, cdbut->ptr, + ((cdbut->len > sizeof(struct UseTable)) ? + sizeof(struct UseTable) : cdbut->len)); + + if (IO->Now - ut.ut_timestamp > + 60 * 60 * 24 * 4) + { + /* Item has already been seen in the last 4 days */ + EVRSSC_syslog(LOG_DEBUG, + "%s has already been seen\n", + ChrPtr(Ctx->Url)); + } cdb_free(cdbut); } + memcpy(ut.ut_msgid, SKEY(guid)); + ut.ut_timestamp = IO->Now; + /* rewrite the record anyway, to update the timestamp */ cdb_store(CDB_USETABLE, SKEY(guid), &ut, sizeof(struct UseTable) ); - + FreeStrBuf(&guid); if (cdbut != NULL) return eAbort; #endif return RSSAggregator_ParseReply(IO); @@ -534,21 +567,13 @@ int rss_do_fetching(rss_aggregator *RSSAggr) /* * Scan a room's netconfig to determine whether it is requesting any RSS feeds */ -void rssclient_scan_room(struct ctdlroom *qrbuf, void *data) +void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG) { - StrBuf *CfgData=NULL; - StrBuf *CfgType; - StrBuf *Line; + const RoomNetCfgLine *pLine; rss_room_counter *Count = NULL; - struct stat statbuf; - char filename[PATH_MAX]; - int fd; - int Done; rss_aggregator *RSSAggr = NULL; rss_aggregator *use_this_RSSAggr = NULL; void *vptr; - const char *CfgPtr, *lPtr; - const char *Err; pthread_mutex_lock(&RSSQueueMutex); if (GetHash(RSSQueueRooms, LKEY(qrbuf->QRnumber), &vptr)) @@ -562,143 +587,84 @@ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data) } pthread_mutex_unlock(&RSSQueueMutex); - assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir); - - if (server_shutting_down) - return; - - /* Only do net processing for rooms that have netconfigs */ - fd = open(filename, 0); - if (fd <= 0) { - /* syslog(LOG_DEBUG, - "rssclient: %s no config.\n", - qrbuf->QRname); */ - return; - } + if (server_shutting_down) return; - if (server_shutting_down) - return; - - if (fstat(fd, &statbuf) == -1) { - EVRSSQ_syslog(LOG_DEBUG, - "ERROR: could not stat configfile '%s' - %s\n", - filename, - strerror(errno)); - return; - } + pLine = OneRNCFG->NetConfigs[rssclient]; - if (server_shutting_down) - return; + while (pLine != NULL) + { + const char *lPtr = NULL; - CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1); + if (Count == NULL) + { + Count = malloc( + sizeof(rss_room_counter)); + Count->count = 0; + } + Count->count ++; + RSSAggr = (rss_aggregator *) malloc( + sizeof(rss_aggregator)); + + memset (RSSAggr, 0, sizeof(rss_aggregator)); + RSSAggr->QRnumber = qrbuf->QRnumber; + RSSAggr->roomlist_parts = 1; + RSSAggr->Url = NewStrBufPlain(NULL, StrLength(pLine->Value[0])); + StrBufExtract_NextToken(RSSAggr->Url, + pLine->Value[0], + &lPtr, + '|'); - if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) { - close(fd); - FreeStrBuf(&CfgData); - EVRSSQ_syslog(LOG_ERR, "ERROR: reading config '%s' - %s
\n", - filename, strerror(errno)); - return; - } - close(fd); - if (server_shutting_down) - return; + pthread_mutex_lock(&RSSQueueMutex); + GetHash(RSSFetchUrls, + SKEY(RSSAggr->Url), + &vptr); - CfgPtr = NULL; - CfgType = NewStrBuf(); - Line = NewStrBufPlain(NULL, StrLength(CfgData)); - Done = 0; - while (!Done) - { - Done = StrBufSipLine(Line, CfgData, &CfgPtr) == 0; - if (StrLength(Line) > 0) + use_this_RSSAggr = (rss_aggregator *)vptr; + if (use_this_RSSAggr != NULL) { - lPtr = NULL; - StrBufExtract_NextToken(CfgType, Line, &lPtr, '|'); - if (!strcasecmp("rssclient", ChrPtr(CfgType))) + long *QRnumber; + StrBufAppendBufPlain( + use_this_RSSAggr->rooms, + qrbuf->QRname, + -1, 0); + if (use_this_RSSAggr->roomlist_parts==1) { - if (Count == NULL) - { - Count = malloc( - sizeof(rss_room_counter)); - Count->count = 0; - } - Count->count ++; - RSSAggr = (rss_aggregator *) malloc( - sizeof(rss_aggregator)); - - memset (RSSAggr, 0, sizeof(rss_aggregator)); - RSSAggr->QRnumber = qrbuf->QRnumber; - RSSAggr->roomlist_parts = 1; - RSSAggr->Url = NewStrBuf(); - - StrBufExtract_NextToken(RSSAggr->Url, - Line, - &lPtr, - '|'); - - pthread_mutex_lock(&RSSQueueMutex); - GetHash(RSSFetchUrls, - SKEY(RSSAggr->Url), - &vptr); - - use_this_RSSAggr = (rss_aggregator *)vptr; - if (use_this_RSSAggr != NULL) - { - long *QRnumber; - StrBufAppendBufPlain( - use_this_RSSAggr->rooms, - qrbuf->QRname, - -1, 0); - if (use_this_RSSAggr->roomlist_parts==1) - { - use_this_RSSAggr->OtherQRnumbers - = NewHash(1, lFlathash); - } - QRnumber = (long*)malloc(sizeof(long)); - *QRnumber = qrbuf->QRnumber; - Put(use_this_RSSAggr->OtherQRnumbers, - LKEY(qrbuf->QRnumber), - QRnumber, - NULL); - use_this_RSSAggr->roomlist_parts++; - - pthread_mutex_unlock(&RSSQueueMutex); - - FreeStrBuf(&RSSAggr->Url); - free(RSSAggr); - RSSAggr = NULL; - continue; - } - pthread_mutex_unlock(&RSSQueueMutex); - - RSSAggr->ItemType = RSS_UNSET; - - RSSAggr->rooms = NewStrBufPlain( - qrbuf->QRname, -1); - - pthread_mutex_lock(&RSSQueueMutex); - - Put(RSSFetchUrls, - SKEY(RSSAggr->Url), - RSSAggr, - DeleteRssCfg); - - pthread_mutex_unlock(&RSSQueueMutex); + use_this_RSSAggr->OtherQRnumbers + = NewHash(1, lFlathash); } + QRnumber = (long*)malloc(sizeof(long)); + *QRnumber = qrbuf->QRnumber; + Put(use_this_RSSAggr->OtherQRnumbers, + LKEY(qrbuf->QRnumber), + QRnumber, + NULL); + use_this_RSSAggr->roomlist_parts++; + + pthread_mutex_unlock(&RSSQueueMutex); + + FreeStrBuf(&RSSAggr->Url); + free(RSSAggr); + RSSAggr = NULL; + pLine = pLine->next; + continue; } - } - if (Count != NULL) - { - Count->QRnumber = qrbuf->QRnumber; + pthread_mutex_unlock(&RSSQueueMutex); + + RSSAggr->ItemType = RSS_UNSET; + + RSSAggr->rooms = NewStrBufPlain( + qrbuf->QRname, -1); + pthread_mutex_lock(&RSSQueueMutex); - EVRSSQ_syslog(LOG_DEBUG, "client: [%ld] %s now starting.\n", - qrbuf->QRnumber, qrbuf->QRname); - Put(RSSQueueRooms, LKEY(qrbuf->QRnumber), Count, NULL); + + Put(RSSFetchUrls, + SKEY(RSSAggr->Url), + RSSAggr, + DeleteRssCfg); + pthread_mutex_unlock(&RSSQueueMutex); + pLine = pLine->next; } - FreeStrBuf(&CfgData); - FreeStrBuf(&CfgType); - FreeStrBuf(&Line); } /* @@ -742,7 +708,7 @@ void rssclient_scan(void) { become_session(&rss_CC); EVRSSQM_syslog(LOG_DEBUG, "rssclient started\n"); - CtdlForEachRoom(rssclient_scan_room, NULL); + CtdlForEachNetCfgRoom(rssclient_scan_room, NULL, rssclient); pthread_mutex_lock(&RSSQueueMutex); @@ -775,9 +741,9 @@ void LogDebugEnableRSSClient(const int n) CTDL_MODULE_INIT(rssclient) { - if (threading) + if (!threading) { - CtdlFillSystemContext(&rss_CC, "rssclient"); + CtdlREGISTERRoomCfgType(rssclient, ParseGeneric, 0, 1, SerializeGeneric, DeleteGenericCfgLine); /// todo: implement rss specific parser pthread_mutex_init(&RSSQueueMutex, NULL); RSSQueueRooms = NewHash(1, lFlathash); RSSFetchUrls = NewHash(1, NULL); @@ -786,5 +752,9 @@ CTDL_MODULE_INIT(rssclient) CtdlRegisterEVCleanupHook(rss_cleanup); CtdlRegisterDebugFlagHook(HKEY("rssclient"), LogDebugEnableRSSClient, &RSSClientDebugEnabled); } + else + { + CtdlFillSystemContext(&rss_CC, "rssclient"); + } return "rssclient"; }