RSS: some more debug logging.

[citadel.git] / citadel / modules / rssclient / serv_rssclient.c

diff --git a/citadel/modules/rssclient/serv_rssclient.c b/citadel/modules/rssclient/serv_rssclient.c
index 1e43e96c765977ba9438338292fcf939b2590c8a..74a6722b74a56053f56f1af68d3c3114c7b47e67 100644 (file)
--- a/citadel/modules/rssclient/serv_rssclient.c
+++ b/citadel/modules/rssclient/serv_rssclient.c
@@ -371,10 +371,8 @@ eNextState RSSSaveMessage(AsyncIO *IO)
        CtdlSubmitMsg(&RSSAggr->ThisMsg->Msg, &RSSAggr->recp, NULL, 0);
 
        /* write the uidl to the use table so we don't store this item again */
-       cdb_store(CDB_USETABLE,
-                 SKEY(RSSAggr->ThisMsg->MsgGUID),
-                 &RSSAggr->ThisMsg->ut,
-                 sizeof(struct UseTable) );
+
+       CheckIfAlreadySeen("RSS Item Insert", RSSAggr->ThisMsg->MsgGUID, IO->Now, 0, eWrite, IO->ID, CCID);
 
        if (GetNextHashPos(RSSAggr->Messages,
                           RSSAggr->Pos,
@@ -389,27 +387,23 @@ eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO)
 {
        const char *Key;
        long len;
-       struct cdbdata *cdbut;
        rss_aggregator *Ctx = (rss_aggregator *) IO->Data;
 
        /* Find out if we've already seen this item */
-       strcpy(Ctx->ThisMsg->ut.ut_msgid,
-              ChrPtr(Ctx->ThisMsg->MsgGUID)); /// TODO
-       Ctx->ThisMsg->ut.ut_timestamp = time(NULL);
-
-       cdbut = cdb_fetch(CDB_USETABLE, SKEY(Ctx->ThisMsg->MsgGUID));
+// todo: expiry?
 #ifndef DEBUG_RSS
-       if (cdbut != NULL) {
+       if (CheckIfAlreadySeen("RSS Item Seen",
+                              Ctx->ThisMsg->MsgGUID,
+                              IO->Now,
+                              IO->Now - USETABLE_ANTIEXPIRE,
+                              eCheckUpdate,
+                              IO->ID, CCID)
+           != 0)
+       {
                /* Item has already been seen */
                EVRSSC_syslog(LOG_DEBUG,
                          "%s has already been seen\n",
                          ChrPtr(Ctx->ThisMsg->MsgGUID));
-               cdb_free(cdbut);
-
-               /* rewrite the record anyway, to update the timestamp */
-               cdb_store(CDB_USETABLE,
-                         SKEY(Ctx->ThisMsg->MsgGUID),
-                         &Ctx->ThisMsg->ut, sizeof(struct UseTable) );
 
                if (GetNextHashPos(Ctx->Messages,
                                   Ctx->Pos,
@@ -427,15 +421,14 @@ eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO)
                NextDBOperation(IO, RSSSaveMessage);
                return eSendMore;
        }
+       return eSendMore;
 }
 
 eNextState RSSAggregator_AnalyseReply(AsyncIO *IO)
 {
-       struct UseTable ut;
        u_char rawdigest[MD5_DIGEST_LEN];
        struct MD5Context md5context;
        StrBuf *guid;
-       struct cdbdata *cdbut;
        rss_aggregator *Ctx = (rss_aggregator *) IO->Data;
 
        if (IO->HttpReq.httpcode != 200)
@@ -464,8 +457,15 @@ eNextState RSSAggregator_AnalyseReply(AsyncIO *IO)
                CtdlAideFPMessage(
                        ChrPtr(ErrMsg),
                        "RSS Aggregation run failure",
-                       2, strs, (long*) &lens);
+                       2, strs, (long*) &lens,
+                       IO->Now,
+                       IO->ID, CCID);
+               
                FreeStrBuf(&ErrMsg);
+               EVRSSC_syslog(LOG_DEBUG,
+                             "RSS feed returned an invalid http status code. <%s><HTTP %ld>\n",
+                             ChrPtr(Ctx->Url),
+                             IO->HttpReq.httpcode);
                return eAbort;
        }
 
@@ -487,32 +487,21 @@ eNextState RSSAggregator_AnalyseReply(AsyncIO *IO)
        /* Find out if we've already seen this item */
 
 #ifndef DEBUG_RSS
-       cdbut = cdb_fetch(CDB_USETABLE, SKEY(guid));
-       if (cdbut != NULL) {
-                memcpy(&ut, cdbut->ptr,
-                       ((cdbut->len > sizeof(struct UseTable)) ?
-                        sizeof(struct UseTable) : cdbut->len));
-
-               if (IO->Now - ut.ut_timestamp  > 
-                   60 * 60 * 24 * 4)
-               {
-                       /* Item has already been seen in the last 4 days */
-                       EVRSSC_syslog(LOG_DEBUG,
-                                     "%s has already been seen\n",
-                                     ChrPtr(Ctx->Url));
-               }
-               cdb_free(cdbut);
-       }
 
-       memcpy(ut.ut_msgid, SKEY(guid));
-       ut.ut_timestamp = IO->Now;
+       if (CheckIfAlreadySeen("RSS Whole",
+                              guid,
+                              IO->Now,
+                              IO->Now - USETABLE_ANTIEXPIRE,
+                              eCheckUpdate,
+                              IO->ID, CCID)
+           != 0)
+       {
+               FreeStrBuf(&guid);
 
-       /* rewrite the record anyway, to update the timestamp */
-       cdb_store(CDB_USETABLE,
-                 SKEY(guid),
-                 &ut, sizeof(struct UseTable) );
+               EVRSSC_syslog(LOG_DEBUG, "RSS feed already seen. <%s>\n", ChrPtr(Ctx->Url));
+               return eAbort;
+       }
        FreeStrBuf(&guid);
-       if (cdbut != NULL) return eAbort;
 #endif
        return RSSAggregator_ParseReply(IO);
 }
@@ -570,7 +559,6 @@ int rss_do_fetching(rss_aggregator *RSSAggr)
 void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG)
 {
        const RoomNetCfgLine *pLine;
-       rss_room_counter *Count = NULL;
        rss_aggregator *RSSAggr = NULL;
        rss_aggregator *use_this_RSSAggr = NULL;
        void *vptr;
@@ -595,13 +583,6 @@ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneR
        {
                const char *lPtr = NULL;
 
-               if (Count == NULL)
-               {
-                       Count = malloc(
-                               sizeof(rss_room_counter));
-                       Count->count = 0;
-               }
-               Count->count ++;
                RSSAggr = (rss_aggregator *) malloc(
                        sizeof(rss_aggregator));
 
@@ -707,21 +688,29 @@ void rssclient_scan(void) {
        }
 
        become_session(&rss_CC);
-       EVRSSQM_syslog(LOG_DEBUG, "rssclient started\n");
+       EVRSSQM_syslog(LOG_DEBUG, "rssclient started");
        CtdlForEachNetCfgRoom(rssclient_scan_room, NULL, rssclient);
 
-       pthread_mutex_lock(&RSSQueueMutex);
-
-       it = GetNewHashPos(RSSFetchUrls, 0);
-       while (!server_shutting_down &&
-              GetNextHashPos(RSSFetchUrls, it, &len, &Key, &vrptr) &&
-              (vrptr != NULL)) {
-               rptr = (rss_aggregator *)vrptr;
-               if (!rss_do_fetching(rptr))
-                       UnlinkRSSAggregator(rptr);
+       if (GetCount(RSSFetchUrls) > 0)
+       {
+               pthread_mutex_lock(&RSSQueueMutex);
+               EVRSSQ_syslog(LOG_DEBUG,
+                              "rssclient starting %d Clients",
+                              GetCount(RSSFetchUrls));
+               
+               it = GetNewHashPos(RSSFetchUrls, 0);
+               while (!server_shutting_down &&
+                      GetNextHashPos(RSSFetchUrls, it, &len, &Key, &vrptr) &&
+                      (vrptr != NULL)) {
+                       rptr = (rss_aggregator *)vrptr;
+                       if (!rss_do_fetching(rptr))
+                               UnlinkRSSAggregator(rptr);
+               }
+               DeleteHashPos(&it);
+               pthread_mutex_unlock(&RSSQueueMutex);
        }
-       DeleteHashPos(&it);
-       pthread_mutex_unlock(&RSSQueueMutex);
+       else
+               EVRSSQM_syslog(LOG_DEBUG, "Nothing to do.");
 
        EVRSSQM_syslog(LOG_DEBUG, "rssclient ended\n");
        return;