From 10a238f3ecfe0a3a197c267442098770c66d364d Mon Sep 17 00:00:00 2001 From: Wilfried Goesgens Date: Mon, 27 Oct 2014 22:00:15 +0100 Subject: [PATCH] Adjust handling of seen events, add debug functions to retrieve & set seen stamps --- citadel/database.c | 64 ++++++++++++++++++++-- citadel/modules/rssclient/serv_rssclient.c | 27 ++++++--- citadel/msgbase.c | 43 +++++++++------ 3 files changed, 101 insertions(+), 33 deletions(-) diff --git a/citadel/database.c b/citadel/database.c index 3f5123f01..e89783f69 100644 --- a/citadel/database.c +++ b/citadel/database.c @@ -892,6 +892,12 @@ int SeentDebugEnabled = 0; "%s[%ld]CC[%ld] SEEN[%s][%d] " FORMAT, \ IOSTR, ioid, ccid, Facility, cType) +#define SEEN_syslog(LEVEL, FORMAT, ...) \ + DBGLOG(LEVEL) syslog(LEVEL, \ + "%s[%ld]CC[%ld] SEEN[%s][%d] " FORMAT, \ + IOSTR, ioid, ccid, Facility, cType, \ + __VA_ARGS__) + time_t CheckIfAlreadySeen(const char *Facility, StrBuf *guid, time_t now, @@ -906,24 +912,23 @@ time_t CheckIfAlreadySeen(const char *Facility, if (cType != eWrite) { - SEENM_syslog(LOG_DEBUG, "Loading"); + SEEN_syslog(LOG_DEBUG, "Loading [%s]", ChrPtr(guid)); cdbut = cdb_fetch(CDB_USETABLE, SKEY(guid)); if ((cdbut != NULL) && (cdbut->ptr != NULL)) { memcpy(&ut, cdbut->ptr, ((cdbut->len > sizeof(struct UseTable)) ? sizeof(struct UseTable) : cdbut->len)); - InDBTimeStamp = ut.ut_timestamp; + InDBTimeStamp = now - ut.ut_timestamp; if (InDBTimeStamp < antiexpire) { - SEENM_syslog(LOG_DEBUG, "Found - Not expired."); + SEEN_syslog(LOG_DEBUG, "Found - Not expired %ld < %ld", InDBTimeStamp, antiexpire); cdb_free(cdbut); return InDBTimeStamp; } else { - SEENM_syslog(LOG_DEBUG, "Found - Expired."); - InDBTimeStamp = ut.ut_timestamp; + SEEN_syslog(LOG_DEBUG, "Found - Expired. %ld >= %ld", InDBTimeStamp, antiexpire); cdb_free(cdbut); } } @@ -941,7 +946,7 @@ time_t CheckIfAlreadySeen(const char *Facility, memcpy(ut.ut_msgid, SKEY(guid)); ut.ut_timestamp = now; - SEENM_syslog(LOG_DEBUG, "Saving"); + SEENM_syslog(LOG_DEBUG, "Saving new Timestamp"); /* rewrite the record anyway, to update the timestamp */ cdb_store(CDB_USETABLE, SKEY(guid), @@ -952,6 +957,52 @@ time_t CheckIfAlreadySeen(const char *Facility, } +void cmd_rsen(char *argbuf) { + char Token[SIZ]; + long TLen; + char Time[SIZ]; + + struct UseTable ut; + struct cdbdata *cdbut; + + if (CtdlAccessCheck(ac_aide)) return; + + TLen = extract_token(Token, argbuf, 1, '|', sizeof Token); + if (strncmp(argbuf, "GET", 3) == 0) { + cdbut = cdb_fetch(CDB_USETABLE, Token, TLen); + if (cdbut != NULL) { + memcpy(&ut, cdbut->ptr, + ((cdbut->len > sizeof(struct UseTable)) ? + sizeof(struct UseTable) : cdbut->len)); + + cprintf("%d %ld\n", CIT_OK, ut.ut_timestamp); + } + else { + cprintf("%d not found\n", ERROR + NOT_HERE); + } + + } + else if (strncmp(argbuf, "SET", 3) == 0) { + memcpy(ut.ut_msgid, Token, TLen); + extract_token(Time, argbuf, 2, '|', sizeof Time); + ut.ut_timestamp = atol(Time); + cdb_store(CDB_USETABLE, + Token, TLen, + &ut, sizeof(struct UseTable) ); + cprintf("%d token updated\n", CIT_OK); + } + else if (strncmp(argbuf, "DEL", 3) == 0) { + if (cdb_delete(CDB_USETABLE, Token, TLen)) + cprintf("%d not found\n", ERROR + NOT_HERE); + else + cprintf("%d deleted.\n", CIT_OK); + + } + else { + cprintf("%d Usage: [GET|SET|DEL]|Token|timestamp\n", ERROR); + } + +} void LogDebugEnableSeenEnable(const int n) { SeentDebugEnabled = n; @@ -962,6 +1013,7 @@ CTDL_MODULE_INIT(database) if (!threading) { CtdlRegisterDebugFlagHook(HKEY("SeenDebug"), LogDebugEnableSeenEnable, &SeentDebugEnabled); + CtdlRegisterProtoHook(cmd_rsen, "RSEN", "manipulate Aggregators seen database"); } /* return our module id for the log */ diff --git a/citadel/modules/rssclient/serv_rssclient.c b/citadel/modules/rssclient/serv_rssclient.c index 743948b09..784f34b60 100644 --- a/citadel/modules/rssclient/serv_rssclient.c +++ b/citadel/modules/rssclient/serv_rssclient.c @@ -416,6 +416,8 @@ eNextState RSSSaveMessage(AsyncIO *IO) eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO) { + static const time_t antiExpire = USETABLE_ANTIEXPIRE_HIRES; + time_t seenstamp = 0; #ifndef DEBUG_RSS const char *Key; long len; @@ -424,18 +426,20 @@ eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO) /* Find out if we've already seen this item */ // todo: expiry? SetRSSState(IO, eRSSUT); - if (CheckIfAlreadySeen("RSS Item Seen", - Ctx->ThisMsg->MsgGUID, - IO->Now, - IO->Now - USETABLE_ANTIEXPIRE_HIRES, - eCheckUpdate, - CCID, IO->ID) - != 0) + seenstamp = CheckIfAlreadySeen("RSS Item Seen", + Ctx->ThisMsg->MsgGUID, + IO->Now, + antiExpire, + eCheckUpdate, + CCID, IO->ID); + if (seenstamp < antiExpire) { /* Item has already been seen */ EVRSSC_syslog(LOG_DEBUG, - "%s has already been seen\n", - ChrPtr(Ctx->ThisMsg->MsgGUID)); + "%s has already been seen - %ld < %ld", + ChrPtr(Ctx->ThisMsg->MsgGUID), + seenstamp, antiExpire); + SetRSSState(IO, eRSSParsing); if (GetNextHashPos(Ctx->Messages, @@ -451,6 +455,11 @@ eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO) else #endif { + /* Item has already been seen */ + EVRSSC_syslog(LOG_DEBUG, + "%s Parsing - %ld >= %ld", + ChrPtr(Ctx->ThisMsg->MsgGUID), + seenstamp, antiExpire); SetRSSState(IO, eRSSParsing); NextDBOperation(IO, RSSSaveMessage); diff --git a/citadel/msgbase.c b/citadel/msgbase.c index 4c1bd70ed..4d1881c6b 100644 --- a/citadel/msgbase.c +++ b/citadel/msgbase.c @@ -3061,7 +3061,8 @@ void flood_protect_quickie_message(const char *from, StrBuf *guid; char timestamp[64]; long tslen; - time_t tsday = NOW / (8*60*60); /* just care for a day... */ + static const time_t tsday = (8*60*60); /* just care for a day... */ + time_t seenstamp; tslen = snprintf(timestamp, sizeof(timestamp), "%ld", tsday); MD5Init(&md5context); @@ -3080,29 +3081,35 @@ void flood_protect_quickie_message(const char *from, if (StrLength(guid) > 40) StrBufCutAt(guid, 40, NULL); - if (CheckIfAlreadySeen("FPAideMessage", - guid, - NOW, - tsday, - eUpdate, - ccid, - ioid)!= 0) + seenstamp = CheckIfAlreadySeen("FPAideMessage", + guid, + NOW, + tsday, + eUpdate, + ccid, + ioid); + if (seenstamp < tsday) { FreeStrBuf(&guid); /* yes, we did. flood protection kicks in. */ syslog(LOG_DEBUG, - "not sending message again\n"); + "not sending message again - %ld < %ld \n", seenstamp, tsday); return; } - FreeStrBuf(&guid); - /* no, this message isn't sent recently; go ahead. */ - quickie_message(from, - fromaddr, - to, - room, - text, - format_type, - subject); + else + { + syslog(LOG_DEBUG, + "sending message. %ld >= %ld", seenstamp, tsday); + FreeStrBuf(&guid); + /* no, this message isn't sent recently; go ahead. */ + quickie_message(from, + fromaddr, + to, + room, + text, + format_type, + subject); + } } -- 2.30.2