From cdd014a096900b9abc851eb391b465395707134c Mon Sep 17 00:00:00 2001 From: Art Cancro Date: Wed, 8 Mar 2017 19:08:43 -0500 Subject: [PATCH] Eager-zero and compress CDB_USETABLE records to save disk space. --- citadel/database.c | 37 ++++++++++---------- citadel/database.h | 15 ++++---- citadel/modules/network/serv_netspool.c | 9 ++--- citadel/modules/pop3client/serv_pop3client.c | 19 ++++------ citadel/modules/rssclient/serv_rssclient.c | 2 +- citadel/msgbase.c | 14 ++------ 6 files changed, 39 insertions(+), 57 deletions(-) diff --git a/citadel/database.c b/citadel/database.c index c89d3bd8f..e71d06976 100644 --- a/citadel/database.c +++ b/citadel/database.c @@ -515,8 +515,8 @@ int cdb_store(int cdb, const void *ckey, int ckeylen, void *cdata, int cdatalen) ddata.size = cdatalen; ddata.data = cdata; - /* Only compress Visit records. Everything else is uncompressed. */ - if (cdb == CDB_VISIT) { + /* Only compress Visit and UseTable records. Everything else is uncompressed. */ + if ( (cdb == CDB_VISIT) || (cdb == CDB_USETABLE) ) { compressing = 1; zheader.magic = COMPRESS_MAGIC; zheader.uncompressed_len = cdatalen; @@ -869,26 +869,31 @@ void cdb_trunc(int cdb) } } -time_t CheckIfAlreadySeen(const char *Facility, - StrBuf *guid, - time_t now, - time_t antiexpire, - eCheckType cType, - long ccid, - long ioid) + +//time_t CheckIfAlreadySeen(const char *Facility, + //StrBuf *guid, + //time_t now, + //time_t antiexpire, + //eCheckType cType, + //long ccid, + //long ioid +//) { + + +time_t CheckIfAlreadySeen(StrBuf *guid, time_t now, time_t antiexpire, eCheckType cType) { time_t InDBTimeStamp = 0; struct UseTable ut; struct cdbdata *cdbut; + memset(&ut, 0, sizeof(struct UseTable)); // zeroing it out makes it compress better + if (cType != eWrite) { syslog(LOG_DEBUG, "Loading [%s]", ChrPtr(guid)); cdbut = cdb_fetch(CDB_USETABLE, SKEY(guid)); if ((cdbut != NULL) && (cdbut->ptr != NULL)) { - memcpy(&ut, cdbut->ptr, - ((cdbut->len > sizeof(struct UseTable)) ? - sizeof(struct UseTable) : cdbut->len)); + memcpy(&ut, cdbut->ptr, ((cdbut->len > sizeof(struct UseTable)) ? sizeof(struct UseTable) : cdbut->len)); InDBTimeStamp = now - ut.ut_timestamp; if (InDBTimeStamp < antiexpire) @@ -919,13 +924,9 @@ time_t CheckIfAlreadySeen(const char *Facility, memcpy(ut.ut_msgid, SKEY(guid)); ut.ut_timestamp = now; - syslog(LOG_DEBUG, "Saving new Timestamp"); /* rewrite the record anyway, to update the timestamp */ - cdb_store(CDB_USETABLE, - SKEY(guid), - &ut, sizeof(struct UseTable) ); - - syslog(LOG_DEBUG, "Done Saving"); + syslog(LOG_DEBUG, "(re)writing usetable record"); + cdb_store(CDB_USETABLE, SKEY(guid), &ut, sizeof(struct UseTable)); return InDBTimeStamp; } diff --git a/citadel/database.h b/citadel/database.h index e7acb31fb..362d1724a 100644 --- a/citadel/database.h +++ b/citadel/database.h @@ -60,13 +60,14 @@ typedef enum __eCheckType { eWrite /* write this to DB, unconditional. */ }eCheckType; -time_t CheckIfAlreadySeen(const char *Facility, - StrBuf *guid, - time_t now, - time_t antiexpire, - eCheckType cType, - long ccid, - long ioid); +//time_t CheckIfAlreadySeen(const char *Facility, + //StrBuf *guid, + //time_t now, + //time_t antiexpire, + //eCheckType cType, + //long ccid, + //long ioid); +time_t CheckIfAlreadySeen(StrBuf *guid, time_t now, time_t antiexpire, eCheckType cType); #endif /* DATABASE_H */ diff --git a/citadel/modules/network/serv_netspool.c b/citadel/modules/network/serv_netspool.c index ab0aa9f99..5d7daa7f0 100644 --- a/citadel/modules/network/serv_netspool.c +++ b/citadel/modules/network/serv_netspool.c @@ -2,7 +2,7 @@ * This module handles shared rooms, inter-Citadel mail, and outbound * mailing list processing. * - * Copyright (c) 2000-2016 by the citadel.org team + * Copyright (c) 2000-2017 by the citadel.org team * * This program is open source software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, version 3. @@ -520,12 +520,7 @@ int network_usetable(struct CtdlMessage *msg) } } now = time(NULL); - if (CheckIfAlreadySeen("Networker Import", - msgid, - now, 0, - eUpdate, - CCC->cs_pid, 0) != 0) - { + if (CheckIfAlreadySeen(msgid, now, 0, eUpdate)) { FreeStrBuf(&msgid); return(1); } diff --git a/citadel/modules/pop3client/serv_pop3client.c b/citadel/modules/pop3client/serv_pop3client.c index 08afeefa4..358342f47 100644 --- a/citadel/modules/pop3client/serv_pop3client.c +++ b/citadel/modules/pop3client/serv_pop3client.c @@ -374,12 +374,11 @@ eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO) RecvMsg->CurrMsg = (FetchItem*)vData; - seenstamp = CheckIfAlreadySeen("POP3 Item Seen", - RecvMsg->CurrMsg->MsgUID, - EvGetNow(IO), - EvGetNow(IO) - USETABLE_ANTIEXPIRE, - eCheckUpdate, - IO->ID, CCID); + seenstamp = CheckIfAlreadySeen(RecvMsg->CurrMsg->MsgUID, + EvGetNow(IO), + EvGetNow(IO) - USETABLE_ANTIEXPIRE, + eCheckUpdate + ); if (seenstamp != 0) { /* Item has already been seen */ @@ -526,13 +525,7 @@ eNextState POP3C_StoreMsgRead(AsyncIO *IO) SetPOP3State(IO, eStoreMsg); syslog(LOG_DEBUG, "MARKING: %s as seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID)); - CheckIfAlreadySeen("POP3 Item Seen", - RecvMsg->CurrMsg->MsgUID, - EvGetNow(IO), - EvGetNow(IO) - USETABLE_ANTIEXPIRE, - eWrite, - IO->ID, CCID - ); + CheckIfAlreadySeen(RecvMsg->CurrMsg->MsgUID, EvGetNow(IO), EvGetNow(IO) - USETABLE_ANTIEXPIRE, eWrite); return DBQueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages); } diff --git a/citadel/modules/rssclient/serv_rssclient.c b/citadel/modules/rssclient/serv_rssclient.c index 9be08531b..6c9d364bb 100644 --- a/citadel/modules/rssclient/serv_rssclient.c +++ b/citadel/modules/rssclient/serv_rssclient.c @@ -134,7 +134,7 @@ void rss_end_element(void *data, const char *el) // check the use table StrBuf *u = NewStrBuf(); StrBufAppendPrintf(u, "rss/%s", r->item_id); - time_t already_seen = CheckIfAlreadySeen("rss", u, time(NULL), 604800, eUpdate, 0, 0); + time_t already_seen = CheckIfAlreadySeen(u, time(NULL), 604800, eUpdate); FreeStrBuf(&u); if (already_seen == 0) { diff --git a/citadel/msgbase.c b/citadel/msgbase.c index 20e0a39b3..88b21f5ad 100644 --- a/citadel/msgbase.c +++ b/citadel/msgbase.c @@ -3131,25 +3131,17 @@ void flood_protect_quickie_message(const char *from, if (StrLength(guid) > 40) StrBufCutAt(guid, 40, NULL); - seenstamp = CheckIfAlreadySeen("FPAideMessage", - guid, - NOW, - tsday, - eUpdate, - ccid, - ioid); + seenstamp = CheckIfAlreadySeen(guid, NOW, tsday, eUpdate); if ((seenstamp > 0) && (seenstamp < tsday)) { FreeStrBuf(&guid); /* yes, we did. flood protection kicks in. */ - syslog(LOG_DEBUG, - "not sending message again - %ld < %ld \n", seenstamp, tsday); + syslog(LOG_DEBUG, "not sending message again - %ld < %ld \n", seenstamp, tsday); return; } else { - syslog(LOG_DEBUG, - "sending message. %ld >= %ld", seenstamp, tsday); + syslog(LOG_DEBUG, "sending message. %ld >= %ld", seenstamp, tsday); FreeStrBuf(&guid); /* no, this message isn't sent recently; go ahead. */ quickie_message(from, -- 2.30.2