From 0f29bb4d04dd7a16f31f008fea15d2d69068f708 Mon Sep 17 00:00:00 2001
From: Art Cancro <ajc@citadel.org>
Date: Wed, 30 Aug 2023 15:27:39 -0400
Subject: [PATCH] Wrapping indexer transactions inside a critical section seems
 to make it not deadlock

---
 .../server/backends/berkeley_db/berkeley_db.c | 39 +++++++++++--------
 citadel/server/citadel_defs.h                 |  1 +
 .../server/modules/fulltext/serv_fulltext.c   |  3 +-
 citadel/utils/loadtest.c                      |  2 +-
 4 files changed, 26 insertions(+), 19 deletions(-)

diff --git a/citadel/server/backends/berkeley_db/berkeley_db.c b/citadel/server/backends/berkeley_db/berkeley_db.c
index b025f0953..b36c3c105 100644
--- a/citadel/server/backends/berkeley_db/berkeley_db.c
+++ b/citadel/server/backends/berkeley_db/berkeley_db.c
@@ -420,17 +420,21 @@ int bdb_store(int cdb, const void *ckey, int ckeylen, void *cdata, int cdatalen)
 		bdb_begin_transaction();		// create our own for this operation.
 	}
 
-	ret = bdb_table[cdb]->put(bdb_table[cdb],	// db
-			    TSD->tid,			// transaction ID
-			    &dkey,			// key
-			    &ddata,			// data
-			    0				// flags
-	);
-
-	if (ret) {
-		syslog(LOG_ERR, "bdb: bdb_store(%02x): %s", cdb, db_strerror(ret));
-		bdb_abort();
-	}
+	do {
+		ret = bdb_table[cdb]->put(bdb_table[cdb],	// db
+			    	TSD->tid,			// transaction ID
+			    	&dkey,			// key
+			    	&ddata,			// data
+			    	0				// flags
+		);
+		if ((ret != 0) && (ret != DB_LOCK_DEADLOCK)) {
+			syslog(LOG_ERR, "bdb: bdb_store(%02x): error %d: %s", cdb, ret, db_strerror(ret));
+			bdb_abort();
+		}
+		if (ret == DB_LOCK_DEADLOCK) {
+			syslog(LOG_DEBUG, "bdb: bdb_store(%02x): would deadlock, trying again", cdb);
+		}
+	} while (ret == DB_LOCK_DEADLOCK);
 
 	if (!existing_txn) {
 		bdb_end_transaction();
@@ -497,12 +501,14 @@ struct cdbdata bdb_fetch(int cdb, const void *key, int keylen) {
 	dkey.data = (void *) key;
 
 	TSD->dbdata[cdb].flags = DB_DBT_REALLOC;
-	ret = bdb_table[cdb]->get(bdb_table[cdb], TSD->tid, &dkey, &TSD->dbdata[cdb], 0);
 
-	if ((ret != 0) && (ret != DB_NOTFOUND)) {
-		syslog(LOG_ERR, "bdb: bdb_fetch(%d): %s", cdb, db_strerror(ret));
-		bdb_abort();
-	}
+	do {
+		ret = bdb_table[cdb]->get(bdb_table[cdb], TSD->tid, &dkey, &TSD->dbdata[cdb], 0);
+		if ((ret != 0) && (ret != DB_NOTFOUND) && (ret != DB_LOCK_DEADLOCK)) {
+			syslog(LOG_ERR, "bdb: bdb_fetch(%d): error %d: %s", cdb, ret, db_strerror(ret));
+			bdb_abort();
+		}
+	} while (ret == DB_LOCK_DEADLOCK);
 
 	if (ret == 0) {
 		bdb_decompress_if_necessary(&TSD->dbdata[cdb]);
@@ -635,7 +641,6 @@ void bdb_tick(void) {
 	}
 }
 
-
 // Calling this function activates the Berkeley DB back end.
 void bdb_init_backend(void) {
 
diff --git a/citadel/server/citadel_defs.h b/citadel/server/citadel_defs.h
index a879c4631..6a16bfda8 100644
--- a/citadel/server/citadel_defs.h
+++ b/citadel/server/citadel_defs.h
@@ -170,6 +170,7 @@ enum {
 	S_IM_LOGS,
 	S_OPENSSL,
 	S_SMTPQUEUE,
+	S_INDEXER,
 	MAX_SEMAPHORES
 };
 
diff --git a/citadel/server/modules/fulltext/serv_fulltext.c b/citadel/server/modules/fulltext/serv_fulltext.c
index 994e856bb..caa5f17e7 100644
--- a/citadel/server/modules/fulltext/serv_fulltext.c
+++ b/citadel/server/modules/fulltext/serv_fulltext.c
@@ -102,6 +102,7 @@ void ft_index_message(long msgnum, int op) {
 	);
 
 	if (array_len(tokens_in_this_message) > 0) {
+		begin_critical_section(S_INDEXER);
 		cdb_begin_transaction();
 		for (i=0; i<array_len(tokens_in_this_message); ++i) {
 
@@ -122,7 +123,6 @@ void ft_index_message(long msgnum, int op) {
 				memcpy(newbucket, cdb_bucket.ptr, cdb_bucket.len);
 				for (j=0; j<nmsgs; ++j) {
 					if ((newbucket[j] == msgnum) || (newbucket[j] == 0)) {
-						fprintf(stderr, "removing %ld\n", newbucket[j]);
 						memcpy(&newbucket[j], &newbucket[j+1], ((nmsgs-j)*sizeof(long)));
 						--j;
 						--nmsgs;
@@ -137,6 +137,7 @@ void ft_index_message(long msgnum, int op) {
 			if (server_shutting_down) break;
 		}
 		cdb_end_transaction();
+		end_critical_section(S_INDEXER);
 		CtdlSetConfigLong("MMfulltext", msgnum);
 	}
 	array_free(tokens_in_this_message);
diff --git a/citadel/utils/loadtest.c b/citadel/utils/loadtest.c
index 5cd8253db..c3acb0589 100644
--- a/citadel/utils/loadtest.c
+++ b/citadel/utils/loadtest.c
@@ -292,7 +292,7 @@ void do_stuff(int serv_sock) {
 
 	long ops = 0;
 	while(1) {
-		alarm(30);
+		//alarm(30);
 		perform_random_thing(serv_sock);
 		printf("\033[%d;11H\033[33m%8ld\033[0m", 7+serv_sock, ++ops);
 		fflush(stdout);
-- 
2.39.2