Eager-zero and compress CDB_USETABLE records to save disk space.

diff --git a/citadel/database.c b/citadel/database.c
index c89d3bd8f73bdc444cf2b58c37d050d6d73073ea..e71d069768d1863f2a92bbe50e0ad575cb290684 100644 (file)
--- a/citadel/database.c
+++ b/citadel/database.c
@@ -515,8 +515,8 @@ int cdb_store(int cdb, const void *ckey, int ckeylen, void *cdata, int cdatalen)
        ddata.size = cdatalen;
        ddata.data = cdata;
 
-       /* Only compress Visit records.  Everything else is uncompressed. */
-       if (cdb == CDB_VISIT) {
+       /* Only compress Visit and UseTable records.  Everything else is uncompressed. */
+       if ( (cdb == CDB_VISIT) || (cdb == CDB_USETABLE) ) {
                compressing = 1;
                zheader.magic = COMPRESS_MAGIC;
                zheader.uncompressed_len = cdatalen;
@@ -869,26 +869,31 @@ void cdb_trunc(int cdb)
        }
 }
 
-time_t CheckIfAlreadySeen(const char *Facility,
-                         StrBuf *guid,
-                         time_t now,
-                         time_t antiexpire,
-                         eCheckType cType,
-                         long ccid,
-                         long ioid)
+
+//time_t CheckIfAlreadySeen(const char *Facility,
+                         //StrBuf *guid,
+                         //time_t now,
+                         //time_t antiexpire,
+                         //eCheckType cType,
+                         //long ccid,
+                         //long ioid
+//) {
+
+
+time_t CheckIfAlreadySeen(StrBuf *guid, time_t now, time_t antiexpire, eCheckType cType)
 {
        time_t InDBTimeStamp = 0;
        struct UseTable ut;
        struct cdbdata *cdbut;
 
+       memset(&ut, 0, sizeof(struct UseTable));        // zeroing it out makes it compress better
+
        if (cType != eWrite)
        {
                syslog(LOG_DEBUG, "Loading [%s]", ChrPtr(guid));
                cdbut = cdb_fetch(CDB_USETABLE, SKEY(guid));
                if ((cdbut != NULL) && (cdbut->ptr != NULL)) {
-                       memcpy(&ut, cdbut->ptr,
-                              ((cdbut->len > sizeof(struct UseTable)) ?
-                               sizeof(struct UseTable) : cdbut->len));
+                       memcpy(&ut, cdbut->ptr, ((cdbut->len > sizeof(struct UseTable)) ?  sizeof(struct UseTable) : cdbut->len));
                        InDBTimeStamp = now - ut.ut_timestamp;
 
                        if (InDBTimeStamp < antiexpire)
@@ -919,13 +924,9 @@ time_t CheckIfAlreadySeen(const char *Facility,
        memcpy(ut.ut_msgid, SKEY(guid));
        ut.ut_timestamp = now;
 
-       syslog(LOG_DEBUG, "Saving new Timestamp");
        /* rewrite the record anyway, to update the timestamp */
-       cdb_store(CDB_USETABLE,
-                 SKEY(guid),
-                 &ut, sizeof(struct UseTable) );
-
-       syslog(LOG_DEBUG, "Done Saving");
+       syslog(LOG_DEBUG, "(re)writing usetable record");
+       cdb_store(CDB_USETABLE, SKEY(guid), &ut, sizeof(struct UseTable));
        return InDBTimeStamp;
 }
 

diff --git a/citadel/database.h b/citadel/database.h
index e7acb31fbd9b44398a930a4c4a12fa069c1780f0..362d1724a5b5c575b779ba225e7329a4932faab5 100644 (file)
--- a/citadel/database.h
+++ b/citadel/database.h
@@ -60,13 +60,14 @@ typedef enum __eCheckType {
        eWrite         /* write this to DB, unconditional. */
 }eCheckType;
 
-time_t CheckIfAlreadySeen(const char *Facility,
-                         StrBuf *guid,
-                         time_t now,
-                         time_t antiexpire,
-                         eCheckType cType,
-                         long ccid,
-                         long ioid);
+//time_t CheckIfAlreadySeen(const char *Facility,
+                         //StrBuf *guid,
+                         //time_t now,
+                         //time_t antiexpire,
+                         //eCheckType cType,
+                         //long ccid,
+                         //long ioid);
+time_t CheckIfAlreadySeen(StrBuf *guid, time_t now, time_t antiexpire, eCheckType cType);
 
 
 #endif /* DATABASE_H */

diff --git a/citadel/modules/network/serv_netspool.c b/citadel/modules/network/serv_netspool.c
index ab0aa9f9963dd6032d632c6261607f89d953b343..5d7daa7f0a808dcd75cabeab2d76914d0bc419d7 100644 (file)
--- a/citadel/modules/network/serv_netspool.c
+++ b/citadel/modules/network/serv_netspool.c
@@ -2,7 +2,7 @@
  * This module handles shared rooms, inter-Citadel mail, and outbound
  * mailing list processing.
  *
- * Copyright (c) 2000-2016 by the citadel.org team
+ * Copyright (c) 2000-2017 by the citadel.org team
  *
  * This program is open source software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License, version 3.
@@ -520,12 +520,7 @@ int network_usetable(struct CtdlMessage *msg)
                }
        }
        now = time(NULL);
-       if (CheckIfAlreadySeen("Networker Import",
-                              msgid,
-                              now, 0,
-                              eUpdate,
-                              CCC->cs_pid, 0) != 0)
-       {
+       if (CheckIfAlreadySeen(msgid, now, 0, eUpdate)) {
                FreeStrBuf(&msgid);
                return(1);
        }

diff --git a/citadel/modules/pop3client/serv_pop3client.c b/citadel/modules/pop3client/serv_pop3client.c
index 08afeefa46f3baafa880ad5365124cd3e68f6726..358342f47215ff5e6815a5d1e7f41bc2cf9da754 100644 (file)
--- a/citadel/modules/pop3client/serv_pop3client.c
+++ b/citadel/modules/pop3client/serv_pop3client.c
@@ -374,12 +374,11 @@ eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO)
 
                RecvMsg->CurrMsg = (FetchItem*)vData;
 
-               seenstamp = CheckIfAlreadySeen("POP3 Item Seen",
-                                              RecvMsg->CurrMsg->MsgUID,
-                                              EvGetNow(IO),
-                                              EvGetNow(IO) - USETABLE_ANTIEXPIRE,
-                                              eCheckUpdate,
-                                              IO->ID, CCID);
+               seenstamp = CheckIfAlreadySeen(RecvMsg->CurrMsg->MsgUID,
+                       EvGetNow(IO),
+                       EvGetNow(IO) - USETABLE_ANTIEXPIRE,
+                       eCheckUpdate
+               );
                if (seenstamp != 0)
                {
                        /* Item has already been seen */
@@ -526,13 +525,7 @@ eNextState POP3C_StoreMsgRead(AsyncIO *IO)
        SetPOP3State(IO, eStoreMsg);
 
        syslog(LOG_DEBUG, "MARKING: %s as seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID));
-       CheckIfAlreadySeen("POP3 Item Seen",
-               RecvMsg->CurrMsg->MsgUID,
-               EvGetNow(IO),
-               EvGetNow(IO) - USETABLE_ANTIEXPIRE,
-               eWrite,
-               IO->ID, CCID
-       );
+       CheckIfAlreadySeen(RecvMsg->CurrMsg->MsgUID, EvGetNow(IO), EvGetNow(IO) - USETABLE_ANTIEXPIRE, eWrite);
 
        return DBQueueEventContext(&RecvMsg->IO, POP3_C_ReAttachToFetchMessages);
 }

diff --git a/citadel/modules/rssclient/serv_rssclient.c b/citadel/modules/rssclient/serv_rssclient.c
index 9be08531bda2824f751953020e4d1a9653493e35..6c9d364bbe83d90eab250483d5741f657566a914 100644 (file)
--- a/citadel/modules/rssclient/serv_rssclient.c
+++ b/citadel/modules/rssclient/serv_rssclient.c
@@ -134,7 +134,7 @@ void rss_end_element(void *data, const char *el)
                        // check the use table
                        StrBuf *u = NewStrBuf();
                        StrBufAppendPrintf(u, "rss/%s", r->item_id);
-                       time_t already_seen = CheckIfAlreadySeen("rss", u, time(NULL), 604800, eUpdate, 0, 0);
+                       time_t already_seen = CheckIfAlreadySeen(u, time(NULL), 604800, eUpdate);
                        FreeStrBuf(&u);
 
                        if (already_seen == 0) {

diff --git a/citadel/msgbase.c b/citadel/msgbase.c
index 20e0a39b3f268dffda765ead7e7b0178faeed137..88b21f5ad91fad6db0d4645fca428f3ce3d46ea5 100644 (file)
--- a/citadel/msgbase.c
+++ b/citadel/msgbase.c
@@ -3131,25 +3131,17 @@ void flood_protect_quickie_message(const char *from,
        if (StrLength(guid) > 40)
                StrBufCutAt(guid, 40, NULL);
 
-       seenstamp = CheckIfAlreadySeen("FPAideMessage",
-                                      guid,
-                                      NOW,
-                                      tsday,
-                                      eUpdate,
-                                      ccid,
-                                      ioid);
+       seenstamp = CheckIfAlreadySeen(guid, NOW, tsday, eUpdate);
        if ((seenstamp > 0) && (seenstamp < tsday))
        {
                FreeStrBuf(&guid);
                /* yes, we did. flood protection kicks in. */
-               syslog(LOG_DEBUG,
-                          "not sending message again - %ld < %ld \n", seenstamp, tsday);
+               syslog(LOG_DEBUG, "not sending message again - %ld < %ld \n", seenstamp, tsday);
                return;
        }
        else
        {
-               syslog(LOG_DEBUG,
-                          "sending message. %ld >= %ld", seenstamp, tsday);
+               syslog(LOG_DEBUG, "sending message. %ld >= %ld", seenstamp, tsday);
                FreeStrBuf(&guid);
                /* no, this message isn't sent recently; go ahead. */
                quickie_message(from,