From: Wilfried Goesgens Date: Tue, 14 May 2013 22:31:15 +0000 (+0200) Subject: TDAP: add facility to protect usetable entries from being deleted while still needed. X-Git-Tag: v8.20~49 X-Git-Url: https://code.citadel.org/?p=citadel.git;a=commitdiff_plain;h=d262741cce924786b2de5f3f10f5470e5d45743e TDAP: add facility to protect usetable entries from being deleted while still needed. --- diff --git a/citadel/include/ctdl_module.h b/citadel/include/ctdl_module.h index a96cdffe5..ab182b362 100644 --- a/citadel/include/ctdl_module.h +++ b/citadel/include/ctdl_module.h @@ -120,6 +120,10 @@ #define PRIO_UNSTEALTH 45000 /* Priorities for EVT_STEALTH */ #define PRIO_STEALTH 50000 +void CtdlRegisterTDAPVetoHook(int (*fcn_ptr)(StrBuf*), int EventType, int Priority); +void CtdlUnregisterTDAPVetoHook(int (*fcn_ptr) (StrBuf*), int EventType); + + void CtdlRegisterSessionHook(void (*fcn_ptr)(void), int EventType, int Priority); void CtdlUnregisterSessionHook(void (*fcn_ptr)(void), int EventType); void CtdlShutdownServiceHooks(void); diff --git a/citadel/modules/expire/serv_expire.c b/citadel/modules/expire/serv_expire.c index e03d43b6a..7da3ef84e 100644 --- a/citadel/modules/expire/serv_expire.c +++ b/citadel/modules/expire/serv_expire.c @@ -694,7 +694,7 @@ int PurgeVisits(void) { * Purge the use table of old entries. * */ -int PurgeUseTable(void) { +int PurgeUseTable(StrBuf *ErrMsg) { int purged = 0; struct cdbdata *cdbut; struct UseTable ut; @@ -702,6 +702,12 @@ int PurgeUseTable(void) { struct UPurgeList *uptr; /* Phase 1: traverse through the table, discovering old records... */ + if (CheckTDAPVeto(CDB_USETABLE, ErrMsg)) + { + syslog(LOG_DEBUG, "Purge use table: VETO!"); + return 0; + } + syslog(LOG_DEBUG, "Purge use table: phase 1"); cdb_rewind(CDB_USETABLE); while(cdbut = cdb_next_item(CDB_USETABLE), cdbut != NULL) { @@ -899,8 +905,13 @@ void purge_databases(void) if (!server_shutting_down) { - retval = PurgeUseTable(); + StrBuf *ErrMsg; + + ErrMsg = NewStrBuf (); + retval = PurgeUseTable(ErrMsg); syslog(LOG_NOTICE, "Purged %d entries from the use table.", retval); +////TODO: fix errmsg + FreeStrBuf(&ErrMsg); } if (!server_shutting_down) diff --git a/citadel/modules/rssclient/rss_atom_parser.c b/citadel/modules/rssclient/rss_atom_parser.c index c45e9358d..0a09f2c37 100644 --- a/citadel/modules/rssclient/rss_atom_parser.c +++ b/citadel/modules/rssclient/rss_atom_parser.c @@ -62,7 +62,7 @@ void rss_remember_item(rss_item *ri, rss_aggregator *Cfg); int RSSAtomParserDebugEnabled = 0; -#define N ((rss_aggregator*)IO->Data)->QRnumber +#define N ((rss_aggregator*)IO->Data)->Cfg.QRnumber #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (RSSAtomParserDebugEnabled != 0)) diff --git a/citadel/modules/rssclient/rss_atom_parser.h b/citadel/modules/rssclient/rss_atom_parser.h index 3bc4c7826..07df846f4 100644 --- a/citadel/modules/rssclient/rss_atom_parser.h +++ b/citadel/modules/rssclient/rss_atom_parser.h @@ -79,6 +79,18 @@ typedef struct __networker_save_message { StrBuf *reLinkTitle; } networker_save_message; +typedef struct RSSCfgLine RSSCfgLine; +struct RSSCfgLine { + StrBuf *Url; + time_t last_known_good; + RSSCfgLine *Next; +}; + +typedef struct __pRSSConfig { + const RSSCfgLine *pCfg; + long QRnumber; +}pRSSConfig; + struct rss_aggregator { AsyncIO IO; XML_Parser xp; @@ -90,7 +102,7 @@ struct rss_aggregator { time_t next_poll; StrBuf *Url; StrBuf *rooms; - long QRnumber; + pRSSConfig Cfg; HashList *OtherQRnumbers; StrBuf *CData; diff --git a/citadel/modules/rssclient/serv_rssclient.c b/citadel/modules/rssclient/serv_rssclient.c index d15613d49..b2a069fd6 100644 --- a/citadel/modules/rssclient/serv_rssclient.c +++ b/citadel/modules/rssclient/serv_rssclient.c @@ -69,7 +69,7 @@ struct CitContext rss_CC; struct rssnetcfg *rnclist = NULL; int RSSClientDebugEnabled = 0; -#define N ((rss_aggregator*)IO->Data)->QRnumber +#define N ((rss_aggregator*)IO->Data)->Cfg.QRnumber #define DBGLOG(LEVEL) if ((LEVEL != LOG_DEBUG) || (RSSClientDebugEnabled != 0)) @@ -141,7 +141,7 @@ void DeleteRoomReference(long QRnumber) void UnlinkRooms(rss_aggregator *RSSAggr) { - DeleteRoomReference(RSSAggr->QRnumber); + DeleteRoomReference(RSSAggr->Cfg.QRnumber); if (RSSAggr->OtherQRnumbers != NULL) { long HKLen; @@ -157,8 +157,8 @@ void UnlinkRooms(rss_aggregator *RSSAggr) &vData) && (vData != NULL)) { - long *lData = (long*) vData; - DeleteRoomReference(*lData); + pRSSConfig *Data = (pRSSConfig*) vData; + DeleteRoomReference(Data->QRnumber); } DeleteHashPos(&At); @@ -450,8 +450,38 @@ eNextState RSS_FetchNetworkUsetableEntry(AsyncIO *IO) return eSendMore; } +void UpdateLastKnownGood(pRSSConfig *pCfg, time_t now) +{ + OneRoomNetCfg* pRNCfg; + begin_critical_section(S_NETCONFIGS); + pRNCfg = CtdlGetNetCfgForRoom (pCfg->QRnumber); + if (pRNCfg != NULL) + { + RSSCfgLine *RSSCfg = (RSSCfgLine *)pRNCfg->NetConfigs[rssclient]; + + while (RSSCfg != NULL) + { + if (RSSCfg == pCfg->pCfg) + break; + + RSSCfg = RSSCfg->Next; + } + if (RSSCfg != NULL) + { + pRNCfg->changed = 1; + RSSCfg->last_known_good = now; + } + } + + end_critical_section(S_NETCONFIGS); +} + eNextState RSSAggregator_AnalyseReply(AsyncIO *IO) { + HashPos *it = NULL; + long len; + const char *Key; + pRSSConfig *pCfg; u_char rawdigest[MD5_DIGEST_LEN]; struct MD5Context md5context; StrBuf *guid; @@ -495,6 +525,28 @@ eNextState RSSAggregator_AnalyseReply(AsyncIO *IO) IO->HttpReq.httpcode); return eAbort; } + + pCfg = &Ctx->Cfg; + + while (pCfg != NULL) + { + UpdateLastKnownGood (pCfg, IO->Now); + if ((Ctx->roomlist_parts > 1) && + (it == NULL)) + { + it = GetNewHashPos(RSSFetchUrls, 0); + } + if (it != NULL) + { + void *vptr; + GetNextHashPos(Ctx->OtherQRnumbers, it, &len, &Key, &vptr); + pCfg = vptr; + } + else + pCfg = NULL; + } + DeleteHashPos (&it); + SetRSSState(IO, eRSSUT); MD5Init(&md5context); @@ -589,7 +641,7 @@ int rss_do_fetching(rss_aggregator *RSSAggr) */ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG) { - const RoomNetCfgLine *pLine; + const RSSCfgLine *RSSCfg = (RSSCfgLine *)OneRNCFG->NetConfigs[rssclient]; rss_aggregator *RSSAggr = NULL; rss_aggregator *use_this_RSSAggr = NULL; void *vptr; @@ -608,33 +660,18 @@ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneR if (server_shutting_down) return; - pLine = OneRNCFG->NetConfigs[rssclient]; - - while (pLine != NULL) + while (RSSCfg != NULL) { - const char *lPtr = NULL; - - RSSAggr = (rss_aggregator *) malloc( - sizeof(rss_aggregator)); - - memset (RSSAggr, 0, sizeof(rss_aggregator)); - RSSAggr->QRnumber = qrbuf->QRnumber; - RSSAggr->roomlist_parts = 1; - RSSAggr->Url = NewStrBufPlain(NULL, StrLength(pLine->Value[0])); - StrBufExtract_NextToken(RSSAggr->Url, - pLine->Value[0], - &lPtr, - '|'); - pthread_mutex_lock(&RSSQueueMutex); GetHash(RSSFetchUrls, - SKEY(RSSAggr->Url), + SKEY(RSSCfg->Url), &vptr); use_this_RSSAggr = (rss_aggregator *)vptr; if (use_this_RSSAggr != NULL) { - long *QRnumber; + pRSSConfig *pRSSCfg; + StrBufAppendBufPlain( use_this_RSSAggr->rooms, qrbuf->QRname, @@ -644,24 +681,34 @@ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneR use_this_RSSAggr->OtherQRnumbers = NewHash(1, lFlathash); } - QRnumber = (long*)malloc(sizeof(long)); - *QRnumber = qrbuf->QRnumber; + + pRSSCfg = (pRSSConfig *) malloc(sizeof(pRSSConfig)); + + pRSSCfg->QRnumber = qrbuf->QRnumber; + pRSSCfg->pCfg = RSSCfg; + Put(use_this_RSSAggr->OtherQRnumbers, LKEY(qrbuf->QRnumber), - QRnumber, + pRSSCfg, NULL); use_this_RSSAggr->roomlist_parts++; pthread_mutex_unlock(&RSSQueueMutex); - FreeStrBuf(&RSSAggr->Url); - free(RSSAggr); - RSSAggr = NULL; - pLine = pLine->next; + RSSCfg = RSSCfg->Next; continue; } pthread_mutex_unlock(&RSSQueueMutex); + RSSAggr = (rss_aggregator *) malloc( + sizeof(rss_aggregator)); + + memset (RSSAggr, 0, sizeof(rss_aggregator)); + RSSAggr->Cfg.QRnumber = qrbuf->QRnumber; + RSSAggr->Cfg.pCfg = RSSCfg; + RSSAggr->roomlist_parts = 1; + RSSAggr->Url = NewStrBufDup(RSSCfg->Url); + RSSAggr->ItemType = RSS_UNSET; RSSAggr->rooms = NewStrBufPlain( @@ -675,7 +722,7 @@ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneR DeleteRssCfg); pthread_mutex_unlock(&RSSQueueMutex); - pLine = pLine->next; + RSSCfg = RSSCfg->Next; } } @@ -759,11 +806,93 @@ void LogDebugEnableRSSClient(const int n) RSSClientDebugEnabled = n; } + +typedef struct __RSSVetoInfo { + StrBuf *ErrMsg; + time_t Now; + int Veto; +}RSSVetoInfo; + +void rssclient_veto_scan_room(struct ctdlroom *qrbuf, void *data, OneRoomNetCfg *OneRNCFG) +{ + RSSVetoInfo *Info = (RSSVetoInfo *) data; + const RSSCfgLine *RSSCfg = (RSSCfgLine *)OneRNCFG->NetConfigs[rssclient]; + + while (RSSCfg != NULL) + { + if ((RSSCfg->last_known_good != 0) && + (RSSCfg->last_known_good + USETABLE_ANTIEXPIRE < Info->Now)) + { + StrBufAppendPrintf(Info->ErrMsg, + "RSS feed not seen for a %d days:: <", + (Info->Now - RSSCfg->last_known_good) / (24 * 60 * 60)); + + StrBufAppendBuf(Info->ErrMsg, RSSCfg->Url, 0); + StrBufAppendBufPlain(Info->ErrMsg, HKEY(">\n"), 0); + } + RSSCfg = RSSCfg->Next; + } +} + +int RSSCheckUsetableVeto(StrBuf *ErrMsg) +{ + RSSVetoInfo Info; + + Info.ErrMsg = ErrMsg; + Info.Now = time (NULL); + Info.Veto = 0; + + CtdlForEachNetCfgRoom(rssclient_veto_scan_room, &Info, rssclient); + + return Info.Veto;; +} + + + + +void ParseRSSClientCfgLine(const CfgLineType *ThisOne, StrBuf *Line, const char *LinePos, OneRoomNetCfg *OneRNCFG) +{ + RSSCfgLine *RSSCfg; + + RSSCfg = (RSSCfgLine *) malloc (sizeof(RSSCfgLine)); + RSSCfg->Url = NewStrBufPlain (NULL, StrLength (Line)); + + + StrBufExtract_NextToken(RSSCfg->Url, Line, &LinePos, '|'); + RSSCfg->last_known_good = StrBufExtractNext_long(Line, &LinePos, '|'); + + + RSSCfg->Next = (RSSCfgLine *)OneRNCFG->NetConfigs[ThisOne->C]; + OneRNCFG->NetConfigs[ThisOne->C] = (RoomNetCfgLine*) RSSCfg; +} + +void SerializeRSSClientCfgLine(const CfgLineType *ThisOne, StrBuf *OutputBuffer, OneRoomNetCfg *RNCfg, RoomNetCfgLine *data) +{ + RSSCfgLine *RSSCfg = (RSSCfgLine*) data; + + StrBufAppendBufPlain(OutputBuffer, CKEY(ThisOne->Str), 0); + StrBufAppendBufPlain(OutputBuffer, HKEY("|"), 0); + StrBufAppendBufPlain(OutputBuffer, SKEY(RSSCfg->Url), 0); + StrBufAppendPrintf(OutputBuffer, "|%ld\n", RSSCfg->last_known_good); +} + +void DeleteRSSClientCfgLine(const CfgLineType *ThisOne, RoomNetCfgLine **data) +{ + RSSCfgLine *RSSCfg = (RSSCfgLine*) data; + + FreeStrBuf(&RSSCfg->Url); + free(*data); + *data = NULL; +} + + CTDL_MODULE_INIT(rssclient) { if (!threading) { - CtdlREGISTERRoomCfgType(rssclient, ParseGeneric, 0, 1, SerializeGeneric, DeleteGenericCfgLine); /// todo: implement rss specific parser + CtdlRegisterTDAPVetoHook (RSSCheckUsetableVeto, CDB_USETABLE, 0); + + CtdlREGISTERRoomCfgType(rssclient, ParseRSSClientCfgLine, 0, 1, SerializeRSSClientCfgLine, DeleteRSSClientCfgLine); pthread_mutex_init(&RSSQueueMutex, NULL); RSSQueueRooms = NewHash(1, lFlathash); RSSFetchUrls = NewHash(1, NULL); diff --git a/citadel/serv_extensions.c b/citadel/serv_extensions.c index a4c58d048..9f0bd25dc 100644 --- a/citadel/serv_extensions.c +++ b/citadel/serv_extensions.c @@ -56,7 +56,8 @@ struct LogFunctionHook { int loglevel; void (*h_function_pointer) (char *); }; -extern LogFunctionHook *LogHookTable; + +LogFunctionHook *LogHookTable = NULL; typedef struct FixedOutputHook FixedOutputHook; struct FixedOutputHook { @@ -64,7 +65,23 @@ struct FixedOutputHook { char content_type[64]; void (*h_function_pointer) (char *, int); }; -extern FixedOutputHook *FixedOutputTable; +FixedOutputHook *FixedOutputTable = NULL; + + + +/* + * TDAPVetoHookFunctionHook extensions are used for any type of hook for which + * may prevent the autopurger to run for this specific data class. + * the function should at least LOG_INFO that it does so. + */ +typedef struct TDAPVetoHookFunctionHook TDAPVetoHookFunctionHook; +struct TDAPVetoHookFunctionHook { + TDAPVetoHookFunctionHook *next; + int Priority; + int (*h_function_pointer) (StrBuf *); + int eventtype; +}; +TDAPVetoHookFunctionHook *TDAPVetoHookTable = NULL; @@ -81,8 +98,7 @@ struct SessionFunctionHook { void (*h_function_pointer) (void); int eventtype; }; -extern SessionFunctionHook *SessionHookTable; - +SessionFunctionHook *SessionHookTable = NULL; /* * UserFunctionHook extensions are used for any type of hook which implements @@ -95,7 +111,7 @@ struct UserFunctionHook { void (*h_function_pointer) (struct ctdluser *usbuf); int eventtype; }; -extern UserFunctionHook *UserHookTable; +UserFunctionHook *UserHookTable = NULL; /* * MessageFunctionHook extensions are used for hooks which implement handlers @@ -107,7 +123,7 @@ struct MessageFunctionHook { int (*h_function_pointer) (struct CtdlMessage *msg); int eventtype; }; -extern MessageFunctionHook *MessageHookTable; +MessageFunctionHook *MessageHookTable = NULL; /* @@ -119,7 +135,7 @@ struct NetprocFunctionHook { NetprocFunctionHook *next; int (*h_function_pointer) (struct CtdlMessage *msg, char *target_room); }; -extern NetprocFunctionHook *NetprocHookTable; +NetprocFunctionHook *NetprocHookTable = NULL; /* @@ -131,7 +147,7 @@ struct DeleteFunctionHook { DeleteFunctionHook *next; void (*h_function_pointer) (char *target_room, long msgnum); }; -extern DeleteFunctionHook *DeleteHookTable; +DeleteFunctionHook *DeleteHookTable = NULL; /* @@ -146,7 +162,7 @@ struct XmsgFunctionHook { int (*h_function_pointer) (char *, char *, char *, char *); int order; }; -extern XmsgFunctionHook *XmsgHookTable; +XmsgFunctionHook *XmsgHookTable = NULL; @@ -160,7 +176,7 @@ struct RoomFunctionHook { RoomFunctionHook *next; int (*fcn_ptr) (struct ctdlroom *); }; -extern RoomFunctionHook *RoomHookTable; +RoomFunctionHook *RoomHookTable = NULL; @@ -170,21 +186,12 @@ struct SearchFunctionHook { void (*fcn_ptr) (int *, long **, const char *); char *name; }; -extern SearchFunctionHook *SearchFunctionHookTable; - +SearchFunctionHook *SearchFunctionHookTable = NULL; CleanupFunctionHook *CleanupHookTable = NULL; CleanupFunctionHook *EVCleanupHookTable = NULL; -SessionFunctionHook *SessionHookTable = NULL; -UserFunctionHook *UserHookTable = NULL; -XmsgFunctionHook *XmsgHookTable = NULL; -MessageFunctionHook *MessageHookTable = NULL; -NetprocFunctionHook *NetprocHookTable = NULL; -DeleteFunctionHook *DeleteHookTable = NULL; + ServiceFunctionHook *ServiceHookTable = NULL; -FixedOutputHook *FixedOutputTable = NULL; -RoomFunctionHook *RoomHookTable = NULL; -SearchFunctionHook *SearchFunctionHookTable = NULL; typedef struct ProtoFunctionHook ProtoFunctionHook; struct ProtoFunctionHook { @@ -586,6 +593,75 @@ void CtdlDestroyEVCleanupHooks(void) EVCleanupHookTable = NULL; } +void CtdlRegisterTDAPVetoHook(int (*fcn_ptr) (StrBuf*), int EventType, int Priority) +{ + TDAPVetoHookFunctionHook *newfcn; + + newfcn = (TDAPVetoHookFunctionHook *) + malloc(sizeof(TDAPVetoHookFunctionHook)); + newfcn->Priority = Priority; + newfcn->h_function_pointer = fcn_ptr; + newfcn->eventtype = EventType; + + TDAPVetoHookFunctionHook **pfcn; + pfcn = &TDAPVetoHookTable; + while ((*pfcn != NULL) && + ((*pfcn)->Priority < newfcn->Priority) && + ((*pfcn)->next != NULL)) + pfcn = &(*pfcn)->next; + + newfcn->next = *pfcn; + *pfcn = newfcn; + + MOD_syslog(LOG_DEBUG, "Registered a new TDAP Veto function (type %d Priority %d)\n", + EventType, Priority); +} + + +void CtdlUnregisterTDAPVetoHook(int (*fcn_ptr) (StrBuf*), int EventType) +{ + TDAPVetoHookFunctionHook *cur, *p, *last; + last = NULL; + cur = TDAPVetoHookTable; + while (cur != NULL) { + if ((fcn_ptr == cur->h_function_pointer) && + (EventType == cur->eventtype)) + { + MOD_syslog(LOG_DEBUG, "Unregistered TDAP Veto function (type %d)\n", + EventType); + p = cur->next; + + free(cur); + cur = NULL; + + if (last != NULL) + last->next = p; + else + TDAPVetoHookTable = p; + cur = p; + } + else { + last = cur; + cur = cur->next; + } + } +} + +void CtdlDestroyTDAPVetoHooks(void) +{ + TDAPVetoHookFunctionHook *cur, *p; + + cur = TDAPVetoHookTable; + while (cur != NULL) + { + MODM_syslog(LOG_DEBUG, "Destroyed TDAP Veto function\n"); + p = cur->next; + free(cur); + cur = p; + } + TDAPVetoHookTable = NULL; +} + void CtdlRegisterSessionHook(void (*fcn_ptr) (void), int EventType, int Priority) { @@ -1330,6 +1406,18 @@ void CtdlModuleDoSearch(int *num_msgs, long **search_msgs, const char *search_st *num_msgs = 0; } +int CheckTDAPVeto (int DBType, StrBuf *ErrMsg) +{ + int Result = 0; + TDAPVetoHookFunctionHook *fcn = NULL; + + for (fcn = TDAPVetoHookTable; (fcn != NULL) && (Result == 0); fcn = fcn->next) { + if (fcn->eventtype == DBType) { + Result = (*fcn->h_function_pointer) (ErrMsg); + } + } + return Result; +} void PerformSessionHooks(int EventType) { diff --git a/citadel/serv_extensions.h b/citadel/serv_extensions.h index f4f720144..de8502648 100644 --- a/citadel/serv_extensions.h +++ b/citadel/serv_extensions.h @@ -63,6 +63,9 @@ char *Dynamic_Module_Init(void); void CtdlDestroySessionHooks(void); void PerformSessionHooks(int EventType); +int CheckTDAPVeto (int DBType, StrBuf *ErrMsg); +void CtdlDestroyTDAPVetoHooks(void); + void CtdlDestroyUserHooks(void); void PerformUserHooks(struct ctdluser *usbuf, int EventType); diff --git a/citadel/sysdep.c b/citadel/sysdep.c index 01e7e831d..a6b225fea 100644 --- a/citadel/sysdep.c +++ b/citadel/sysdep.c @@ -979,6 +979,7 @@ void sysdep_master_cleanup(void) { CtdlDestroyCleanupHooks(); CtdlDestroyFixedOutputHooks(); CtdlDestroySessionHooks(); + CtdlDestroyTDAPVetoHooks(); CtdlDestroyServiceHook(); CtdlDestroyRoomHooks(); CtdlDestroySearchHooks(); diff --git a/citadel/utils/chkpwd.c b/citadel/utils/chkpwd.c index 19c97c832..4dfc594e2 100644 --- a/citadel/utils/chkpwd.c +++ b/citadel/utils/chkpwd.c @@ -22,6 +22,8 @@ #include #include +#include + #include "auth.h" #include "config.h" #include "citadel_dirs.h"