X-Git-Url: https://code.citadel.org/?a=blobdiff_plain;f=citadel%2Fdatabase.c;h=2f6a5b5cedfe41f1b17fa58a0aba7a0cc037ef05;hb=8e944083763c9ddcb32d763cf8f19c966d01f873;hp=9a22c8d2c62a6ca7af9c819fdb879946db3b2db4;hpb=a55b22e1215903d726353fb6b9f7c3608eb61c0d;p=citadel.git diff --git a/citadel/database.c b/citadel/database.c index 9a22c8d2c..2f6a5b5ce 100644 --- a/citadel/database.c +++ b/citadel/database.c @@ -1,7 +1,7 @@ /* * This is a data store backend for the Citadel server which uses Berkeley DB. * - * Copyright (c) 1987-2018 by the citadel.org team + * Copyright (c) 1987-2021 by the citadel.org team * * This program is open source software; you can redistribute it and/or * modify it under the terms of the GNU General Public License version 3. @@ -33,13 +33,7 @@ #include #include -#ifdef HAVE_DB_H #include -#elif defined(HAVE_DB4_DB_H) -#include -#else -#error Neither nor was found by configure. Install db5-devel. -#endif #if DB_VERSION_MAJOR < 5 #error Citadel requires Berkeley DB v5.0 or newer. Please upgrade. @@ -56,8 +50,7 @@ static DB *dbp[MAXCDB]; /* One DB handle for each Citadel database */ static DB_ENV *dbenv; /* The DB environment (global) */ -void cdb_abort(void) -{ +void cdb_abort(void) { syslog(LOG_DEBUG, "db: citserver is stopping in order to prevent data loss. uid=%d gid=%d euid=%d egid=%d", getuid(), getgid(), geteuid(), getegid() ); @@ -66,8 +59,7 @@ void cdb_abort(void) /* Verbose logging callback */ -void cdb_verbose_log(const DB_ENV * dbenv, const char *msg) -{ +void cdb_verbose_log(const DB_ENV * dbenv, const char *msg) { if (!IsEmptyStr(msg)) { syslog(LOG_DEBUG, "db: %s", msg); } @@ -75,15 +67,13 @@ void cdb_verbose_log(const DB_ENV * dbenv, const char *msg) /* Verbose logging callback */ -void cdb_verbose_err(const DB_ENV * dbenv, const char *errpfx, const char *msg) -{ +void cdb_verbose_err(const DB_ENV * dbenv, const char *errpfx, const char *msg) { syslog(LOG_ERR, "db: %s", msg); } -/* just a little helper function */ -static void txabort(DB_TXN * tid) -{ +/* wrapper for txn_abort() that logs/aborts on error */ +static void txabort(DB_TXN *tid) { int ret; ret = tid->abort(tid); @@ -95,9 +85,8 @@ static void txabort(DB_TXN * tid) } -/* this one is even more helpful than the last. */ -static void txcommit(DB_TXN * tid) -{ +/* wrapper for txn_commit() that logs/aborts on error */ +static void txcommit(DB_TXN *tid) { int ret; ret = tid->commit(tid, 0); @@ -109,9 +98,8 @@ static void txcommit(DB_TXN * tid) } -/* are you sensing a pattern yet? */ -static void txbegin(DB_TXN ** tid) -{ +/* wrapper for txn_begin() that logs/aborts on error */ +static void txbegin(DB_TXN **tid) { int ret; ret = dbenv->txn_begin(dbenv, NULL, tid, 0); @@ -123,14 +111,13 @@ static void txbegin(DB_TXN ** tid) } -static void dbpanic(DB_ENV * env, int errval) -{ +/* panic callback */ +static void dbpanic(DB_ENV * env, int errval) { syslog(LOG_ERR, "db: PANIC: %s", db_strerror(errval)); } -static void cclose(DBC * cursor) -{ +static void cclose(DBC * cursor) { int ret; if ((ret = cursor->c_close(cursor))) { @@ -140,8 +127,7 @@ static void cclose(DBC * cursor) } -static void bailIfCursor(DBC ** cursors, const char *msg) -{ +static void bailIfCursor(DBC ** cursors, const char *msg) { int i; for (i = 0; i < MAXCDB; i++) @@ -152,8 +138,7 @@ static void bailIfCursor(DBC ** cursors, const char *msg) } -void cdb_check_handles(void) -{ +void cdb_check_handles(void) { bailIfCursor(TSD->cursors, "in check_handles"); if (TSD->tid != NULL) { @@ -163,53 +148,10 @@ void cdb_check_handles(void) } -/* - * Cull the database logs by removing log files that are no longer needed. - * Note: this code will be removed once we are confident that Berkeley DB can do this on its own. - */ -void cdb_cull_logs(void) -{ -#ifndef DB_LOG_AUTO_REMOVE - u_int32_t flags; - int ret; - char **file, **list; - char errmsg[SIZ]; - - flags = DB_ARCH_ABS; - - /* Get the list of names. */ - if ((ret = dbenv->log_archive(dbenv, &list, flags)) != 0) { - syslog(LOG_ERR, "db: cdb_cull_logs() %s", db_strerror(ret)); - return; - } - - /* Print the list of names. */ - if (list != NULL) { - for (file = list; *file != NULL; ++file) { - syslog(LOG_DEBUG, "db: deleting log %s", *file); - ret = unlink(*file); - if (ret != 0) { - snprintf(errmsg, sizeof(errmsg), - " ** ERROR **\n \n \n " - "Citadel was unable to delete the " - "database log file '%s' because of the " - "following error:\n \n %s\n \n" - " This log file is no longer in use " - "and may be safely deleted.\n", *file, strerror(errno)); - CtdlAideMessage(errmsg, "Database Warning Message"); - } - } - free(list); - } -#endif /* DB_LOG_AUTO_REMOVE */ -} - - /* * Request a checkpoint of the database. Called once per minute by the thread manager. */ -void cdb_checkpoint(void) -{ +void cdb_checkpoint(void) { int ret; syslog(LOG_DEBUG, "db: -- checkpoint --"); @@ -222,18 +164,11 @@ void cdb_checkpoint(void) /* After a successful checkpoint, we can cull the unused logs */ if (CtdlGetConfigInt("c_auto_cull")) { -#ifdef DB_LOG_AUTO_REMOVE - // This version of Berkeley DB can cull the logs on its own! ret = dbenv->log_set_config(dbenv, DB_LOG_AUTO_REMOVE, 1); -#else - cdb_cull_logs(); // Citadel knows how to do it too. -#endif } -#ifdef DB_LOG_AUTO_REMOVE else { ret = dbenv->log_set_config(dbenv, DB_LOG_AUTO_REMOVE, 0); } -#endif } @@ -243,8 +178,7 @@ void cdb_checkpoint(void) * critical section here, because there aren't any active threads * manipulating the database yet. */ -void open_databases(void) -{ +void open_databases(void) { int ret; int i; char dbfilename[32]; @@ -257,20 +191,19 @@ void open_databases(void) syslog(LOG_DEBUG, "db: Linked zlib: %s", zlibVersion()); /* - * Silently try to create the database subdirectory. If it's - * already there, no problem. + * Silently try to create the database subdirectory. If it's already there, no problem. */ if ((mkdir(ctdl_data_dir, 0700) != 0) && (errno != EEXIST)) { syslog(LOG_ERR, "db: unable to create database directory [%s]: %m", ctdl_data_dir); } if (chmod(ctdl_data_dir, 0700) != 0) { - syslog(LOG_ERR, "db: unable to set database directory accessrights [%s]: %m", ctdl_data_dir); + syslog(LOG_ERR, "db: unable to set database directory permissions [%s]: %m", ctdl_data_dir); } if (chown(ctdl_data_dir, CTDLUID, (-1)) != 0) { syslog(LOG_ERR, "db: unable to set the owner for [%s]: %m", ctdl_data_dir); } - syslog(LOG_DEBUG, "db: Setting up DB environment\n"); - /* db_env_set_func_yield((int (*)(u_long, u_long))sched_yield); */ + syslog(LOG_DEBUG, "db: Setting up DB environment"); + // db_env_set_func_yield((int (*)(u_long, u_long))sched_yield); ret = db_env_create(&dbenv, 0); if (ret) { syslog(LOG_ERR, "db: db_env_create: %s", db_strerror(ret)); @@ -306,19 +239,19 @@ void open_databases(void) flags = DB_CREATE | DB_INIT_MPOOL | DB_PRIVATE | DB_INIT_TXN | DB_INIT_LOCK | DB_THREAD | DB_INIT_LOG; syslog(LOG_DEBUG, "db: dbenv->open(dbenv, %s, %d, 0)", ctdl_data_dir, flags); - ret = dbenv->open(dbenv, ctdl_data_dir, flags, 0); + ret = dbenv->open(dbenv, ctdl_data_dir, flags, 0); // try opening the database cleanly if (ret == DB_RUNRECOVERY) { syslog(LOG_ERR, "db: dbenv->open: %s", db_strerror(ret)); syslog(LOG_ERR, "db: attempting recovery..."); flags |= DB_RECOVER; - ret = dbenv->open(dbenv, ctdl_data_dir, flags, 0); + ret = dbenv->open(dbenv, ctdl_data_dir, flags, 0); // try recovery } if (ret == DB_RUNRECOVERY) { syslog(LOG_ERR, "db: dbenv->open: %s", db_strerror(ret)); syslog(LOG_ERR, "db: attempting catastrophic recovery..."); flags &= ~DB_RECOVER; flags |= DB_RECOVER_FATAL; - ret = dbenv->open(dbenv, ctdl_data_dir, flags, 0); + ret = dbenv->open(dbenv, ctdl_data_dir, flags, 0); // try catastrophic recovery } if (ret) { syslog(LOG_ERR, "db: dbenv->open: %s", db_strerror(ret)); @@ -328,43 +261,32 @@ void open_databases(void) } syslog(LOG_INFO, "db: mounting databases"); - for (i = 0; i < MAXCDB; ++i) { - - /* Create a database handle */ - ret = db_create(&dbp[i], dbenv, 0); + ret = db_create(&dbp[i], dbenv, 0); // Create a database handle if (ret) { syslog(LOG_ERR, "db: db_create: %s", db_strerror(ret)); syslog(LOG_ERR, "db: exit code %d", ret); exit(CTDLEXIT_DB); } - - /* Arbitrary names for our tables -- we reference them by - * number, so we don't have string names for them. - */ - snprintf(dbfilename, sizeof dbfilename, "cdb.%02x", i); - + snprintf(dbfilename, sizeof dbfilename, "cdb.%02x", i); // table names by number ret = dbp[i]->open(dbp[i], NULL, dbfilename, NULL, DB_BTREE, DB_CREATE | DB_AUTO_COMMIT | DB_THREAD, 0600); if (ret) { syslog(LOG_ERR, "db: db_open[%02x]: %s", i, db_strerror(ret)); if (ret == ENOMEM) { - syslog(LOG_ERR, - "db: You may need to tune your database; please read http://www.citadel.org/doku.php?id=faq:troubleshooting:out_of_lock_entries for more information."); + syslog(LOG_ERR, "db: You may need to tune your database; please read http://www.citadel.org/doku.php?id=faq:troubleshooting:out_of_lock_entries for more information."); } syslog(LOG_ERR, "db: exit code %d", ret); exit(CTDLEXIT_DB); } } - } -/* Make sure we own all the files, because in a few milliseconds - * we're going to drop root privs. +/* + * Make sure we own all the files, because in a few milliseconds we're going to drop root privs. */ -void cdb_chmod_data(void) -{ +void cdb_chmod_data(void) { DIR *dp; struct dirent *d; char filename[PATH_MAX]; @@ -382,8 +304,6 @@ void cdb_chmod_data(void) } closedir(dp); } - - syslog(LOG_DEBUG, "db: open_databases() finished"); } @@ -391,14 +311,13 @@ void cdb_chmod_data(void) * Close all of the db database files we've opened. This can be done * in a loop, since it's just a bunch of closes. */ -void close_databases(void) -{ - int a; +void close_databases(void) { + int i; int ret; syslog(LOG_INFO, "db: performing final checkpoint"); if ((ret = dbenv->txn_checkpoint(dbenv, 0, 0, 0))) { - syslog(LOG_ERR, "txn_checkpoint: %s", db_strerror(ret)); + syslog(LOG_ERR, "db: txn_checkpoint: %s", db_strerror(ret)); } syslog(LOG_INFO, "db: flushing the database logs"); @@ -406,21 +325,23 @@ void close_databases(void) syslog(LOG_ERR, "db: log_flush: %s", db_strerror(ret)); } - /* print some statistics... */ -#ifdef DB_STAT_ALL - dbenv->lock_stat_print(dbenv, DB_STAT_ALL); -#endif - /* close the tables */ - for (a = 0; a < MAXCDB; ++a) { - syslog(LOG_INFO, "db: closing database %02x", a); - ret = dbp[a]->close(dbp[a], 0); + syslog(LOG_INFO, "db: closing databases"); + for (i = 0; i < MAXCDB; ++i) { + syslog(LOG_INFO, "db: closing database %02x", i); + ret = dbp[i]->close(dbp[i], 0); if (ret) { syslog(LOG_ERR, "db: db_close: %s", db_strerror(ret)); } } + // This seemed nifty at the time but did anyone really look at it? + // #ifdef DB_STAT_ALL + // /* print some statistics... */ + // dbenv->lock_stat_print(dbenv, DB_STAT_ALL); + // #endif + /* Close the handle. */ ret = dbenv->close(dbenv, 0); if (ret) { @@ -432,8 +353,7 @@ void close_databases(void) /* * Decompress a database item if it was compressed on disk */ -void cdb_decompress_if_necessary(struct cdbdata *cdb) -{ +void cdb_decompress_if_necessary(struct cdbdata *cdb) { static int magic = COMPRESS_MAGIC; if ((cdb == NULL) || (cdb->ptr == NULL) || (cdb->len < sizeof(magic)) || (memcmp(cdb->ptr, &magic, sizeof(magic)))) { @@ -450,8 +370,9 @@ void cdb_decompress_if_necessary(struct cdbdata *cdb) memset(&zheader, 0, sizeof(struct CtdlCompressHeader)); cplen = sizeof(struct CtdlCompressHeader); - if (sizeof(struct CtdlCompressHeader) > cdb->len) + if (sizeof(struct CtdlCompressHeader) > cdb->len) { cplen = cdb->len; + } memcpy(&zheader, cdb->ptr, cplen); compressed_data = cdb->ptr; @@ -477,13 +398,11 @@ void cdb_decompress_if_necessary(struct cdbdata *cdb) * Store a piece of data. Returns 0 if the operation was successful. If a * key already exists it should be overwritten. */ -int cdb_store(int cdb, const void *ckey, int ckeylen, void *cdata, int cdatalen) -{ +int cdb_store(int cdb, const void *ckey, int ckeylen, void *cdata, int cdatalen) { DBT dkey, ddata; - DB_TXN *tid; + DB_TXN *tid = NULL; int ret = 0; - struct CtdlCompressHeader zheader; char *compressed_data = NULL; int compressing = 0; @@ -517,11 +436,11 @@ int cdb_store(int cdb, const void *ckey, int ckeylen, void *cdata, int cdatalen) } if (TSD->tid != NULL) { - ret = dbp[cdb]->put(dbp[cdb], /* db */ - TSD->tid, /* transaction ID */ - &dkey, /* key */ - &ddata, /* data */ - 0 /* flags */ + ret = dbp[cdb]->put(dbp[cdb], // db + TSD->tid, // transaction ID + &dkey, // key + &ddata, // data + 0 // flags ); if (ret) { syslog(LOG_EMERG, "db: cdb_store(%d): %s", cdb, db_strerror(ret)); @@ -537,11 +456,11 @@ int cdb_store(int cdb, const void *ckey, int ckeylen, void *cdata, int cdatalen) retry: txbegin(&tid); - if ((ret = dbp[cdb]->put(dbp[cdb], /* db */ - tid, /* transaction ID */ - &dkey, /* key */ - &ddata, /* data */ - 0))) { /* flags */ + if ((ret = dbp[cdb]->put(dbp[cdb], // db + tid, // transaction ID + &dkey, // key + &ddata, // data + 0))) { // flags if (ret == DB_LOCK_DEADLOCK) { txabort(tid); goto retry; @@ -564,9 +483,7 @@ int cdb_store(int cdb, const void *ckey, int ckeylen, void *cdata, int cdatalen) /* * Delete a piece of data. Returns 0 if the operation was successful. */ -int cdb_delete(int cdb, void *key, int keylen) -{ - +int cdb_delete(int cdb, void *key, int keylen) { DBT dkey; DB_TXN *tid; int ret; @@ -605,8 +522,7 @@ int cdb_delete(int cdb, void *key, int keylen) } -static DBC *localcursor(int cdb) -{ +static DBC *localcursor(int cdb) { int ret; DBC *curs; @@ -631,8 +547,7 @@ static DBC *localcursor(int cdb) * a struct cdbdata which it is the caller's responsibility to free later on * using the cdb_free() routine. */ -struct cdbdata *cdb_fetch(int cdb, const void *key, int keylen) -{ +struct cdbdata *cdb_fetch(int cdb, const void *key, int keylen) { struct cdbdata *tempcdb; DBT dkey, dret; int ret; @@ -644,21 +559,18 @@ struct cdbdata *cdb_fetch(int cdb, const void *key, int keylen) if (TSD->tid != NULL) { memset(&dret, 0, sizeof(DBT)); dret.flags = DB_DBT_MALLOC; - ret = dbp[cdb]->get(dbp[cdb], TSD->tid, &dkey, &dret, 0); + ret = dbp[cdb]->get(dbp[cdb], TSD->tid, &dkey, &dret, 0); // crashing here } else { DBC *curs; do { memset(&dret, 0, sizeof(DBT)); dret.flags = DB_DBT_MALLOC; - curs = localcursor(cdb); - ret = curs->c_get(curs, &dkey, &dret, DB_SET); cclose(curs); } while (ret == DB_LOCK_DEADLOCK); - } if ((ret != 0) && (ret != DB_NOTFOUND)) { @@ -693,8 +605,7 @@ struct cdbdata *cdb_fetch(int cdb, const void *key, int keylen) * pointer elsewhere and then setting 'ptr' to NULL. cdb_free() will then * avoid freeing it. */ -void cdb_free(struct cdbdata *cdb) -{ +void cdb_free(struct cdbdata *cdb) { if (cdb->ptr) { free(cdb->ptr); } @@ -702,8 +613,7 @@ void cdb_free(struct cdbdata *cdb) } -void cdb_close_cursor(int cdb) -{ +void cdb_close_cursor(int cdb) { if (TSD->cursors[cdb] != NULL) { cclose(TSD->cursors[cdb]); } @@ -717,8 +627,7 @@ void cdb_close_cursor(int cdb) * (There is guaranteed to be no more than one traversal in * progress per thread at any given time.) */ -void cdb_rewind(int cdb) -{ +void cdb_rewind(int cdb) { int ret = 0; if (TSD->cursors[cdb] != NULL) { @@ -742,8 +651,7 @@ void cdb_rewind(int cdb) * Fetch the next item in a sequential search. Returns a pointer to a * cdbdata structure, or NULL if we've hit the end. */ -struct cdbdata *cdb_next_item(int cdb) -{ +struct cdbdata *cdb_next_item(int cdb) { DBT key, data; struct cdbdata *cdbret; int ret = 0; @@ -776,9 +684,7 @@ struct cdbdata *cdb_next_item(int cdb) /* * Transaction-based stuff. I'm writing this as I bake cookies... */ -void cdb_begin_transaction(void) -{ - +void cdb_begin_transaction(void) { bailIfCursor(TSD->cursors, "can't begin transaction during r/o cursor"); if (TSD->tid != NULL) { @@ -790,8 +696,7 @@ void cdb_begin_transaction(void) } -void cdb_end_transaction(void) -{ +void cdb_end_transaction(void) { int i; for (i = 0; i < MAXCDB; i++) @@ -815,8 +720,7 @@ void cdb_end_transaction(void) /* * Truncate (delete every record) */ -void cdb_trunc(int cdb) -{ +void cdb_trunc(int cdb) { /* DB_TXN *tid; */ int ret; u_int32_t count; @@ -840,8 +744,7 @@ void cdb_trunc(int cdb) } else { syslog(LOG_EMERG, "db: cdb_truncate(%d): %s", cdb, db_strerror(ret)); if (ret == ENOMEM) { - syslog(LOG_EMERG, - "db: You may need to tune your database; please read http://www.citadel.org/doku.php?id=faq:troubleshooting:out_of_lock_entries for more information."); + syslog(LOG_EMERG, "db: You may need to tune your database; please read http://www.citadel.org/doku.php?id=faq:troubleshooting:out_of_lock_entries for more information."); } exit(CTDLEXIT_DB); } @@ -855,8 +758,7 @@ void cdb_trunc(int cdb) /* * compact (defragment) the database , possibly returning space back to the underlying filesystem */ -void cdb_compact(void) -{ +void cdb_compact(void) { int ret; int i; @@ -875,8 +777,7 @@ void cdb_compact(void) // Has an item already been seen (is it in the CDB_USETABLE) ? // Returns 0 if it hasn't, 1 if it has // In either case, writes the item to the database for next time. -int CheckIfAlreadySeen(StrBuf *guid) -{ +int CheckIfAlreadySeen(StrBuf *guid) { int found = 0; struct UseTable ut; struct cdbdata *cdbut; @@ -897,53 +798,10 @@ int CheckIfAlreadySeen(StrBuf *guid) } -void cmd_rsen(char *argbuf) -{ - char Token[SIZ]; - long TLen; - char Time[SIZ]; - - struct UseTable ut; - struct cdbdata *cdbut; - - if (CtdlAccessCheck(ac_aide)) { - return; - } - - TLen = extract_token(Token, argbuf, 1, '|', sizeof Token); - if (strncmp(argbuf, "GET", 3) == 0) { - cdbut = cdb_fetch(CDB_USETABLE, Token, TLen); - if (cdbut != NULL) { - memcpy(&ut, cdbut->ptr, ((cdbut->len > sizeof(struct UseTable)) ? sizeof(struct UseTable) : cdbut->len)); - - cprintf("%d %ld\n", CIT_OK, ut.ut_timestamp); - } else { - cprintf("%d not found\n", ERROR + NOT_HERE); - } - - } else if (strncmp(argbuf, "SET", 3) == 0) { - memcpy(ut.ut_msgid, Token, TLen); - extract_token(Time, argbuf, 2, '|', sizeof Time); - ut.ut_timestamp = atol(Time); - cdb_store(CDB_USETABLE, Token, TLen, &ut, sizeof(struct UseTable)); - cprintf("%d token updated\n", CIT_OK); - } else if (strncmp(argbuf, "DEL", 3) == 0) { - if (cdb_delete(CDB_USETABLE, Token, TLen)) - cprintf("%d not found\n", ERROR + NOT_HERE); - else - cprintf("%d deleted.\n", CIT_OK); - - } else { - cprintf("%d Usage: [GET|SET|DEL]|Token|timestamp\n", ERROR); - } - -} - - CTDL_MODULE_INIT(database) { if (!threading) { - CtdlRegisterProtoHook(cmd_rsen, "RSEN", "manipulate Aggregators seen database"); + // nothing to do here } /* return our module id for the log */