From acfcc89045bbdc80ae7614ca3ee38303ae98f08b Mon Sep 17 00:00:00 2001 From: Art Cancro Date: Wed, 1 Jun 2005 05:23:26 +0000 Subject: [PATCH] * The full text indexer now runs in its own dedicated thread instead of in the housekeeping thread. The main indexer loop now also has the ability to save its place and bail out early when it discovers that the server is trying to shut down. The main server loop will pthread_join() the indexer thread and patiently wait for it to complete before exiting. These changes all put together mean that citserver will not hang when it is terminated during an indexing operation. --- citadel/ChangeLog | 1 + citadel/citserver.c | 4 ++++ citadel/serv_fulltext.c | 34 ++++++++++++++++++++++++++- citadel/serv_fulltext.h | 1 + citadel/server_main.c | 3 +++ citadel/sysconfig.h | 2 ++ citadel/sysdep.c | 51 +++++++++++++++++++++++++++++++++++------ citadel/sysdep_decls.h | 5 ++++ 8 files changed, 93 insertions(+), 8 deletions(-) diff --git a/citadel/ChangeLog b/citadel/ChangeLog index 7d417c34f..3c075a649 100644 --- a/citadel/ChangeLog +++ b/citadel/ChangeLog @@ -6764,3 +6764,4 @@ Sat Jul 11 00:20:48 EDT 1998 Nathan Bryant Fri Jul 10 1998 Art Cancro * Initial CVS import + diff --git a/citadel/citserver.c b/citadel/citserver.c index 473e256b3..1e32ea54b 100644 --- a/citadel/citserver.c +++ b/citadel/citserver.c @@ -148,6 +148,10 @@ void master_cleanup(int exitcode) { (*fcn->h_function_pointer)(); } + /* Shut down the indexer thread */ + lprintf(CTDL_INFO, "Waiting for the indexer thread to shut down\n"); + pthread_join(indexer_thread_tid, NULL); + /* Close databases */ lprintf(CTDL_INFO, "Closing databases\n"); close_databases(); diff --git a/citadel/serv_fulltext.c b/citadel/serv_fulltext.c index d2384ab4f..61e2085bd 100644 --- a/citadel/serv_fulltext.c +++ b/citadel/serv_fulltext.c @@ -253,6 +253,14 @@ void do_fulltext_indexing(void) { last_progress = time(NULL); } ft_index_message(ft_newmsgs[i], 1); + + /* Check to see if we need to quit early */ + if (time_to_die) { + lprintf(CTDL_DEBUG, "Indexer quitting early\n"); + ft_newhighest = ft_newmsgs[i]; + break; + } + } free(ft_newmsgs); @@ -271,6 +279,31 @@ void do_fulltext_indexing(void) { return; } +/* + * Main loop for the indexer thread. + */ +void *indexer_thread(void *arg) { + struct CitContext indexerCC; + + lprintf(CTDL_DEBUG, "indexer_thread() initializing\n"); + + memset(&indexerCC, 0, sizeof(struct CitContext)); + indexerCC.internal_pgm = 1; + indexerCC.cs_pid = 0; + pthread_setspecific(MyConKey, (void *)&indexerCC ); + + cdb_allocate_tsd(); + + while (!time_to_die) { + do_fulltext_indexing(); + sleep(1); + } + + lprintf(CTDL_DEBUG, "indexer_thread() exiting\n"); + pthread_exit(NULL); +} + + /* * API call to perform searches. @@ -378,7 +411,6 @@ void cmd_srch(char *argbuf) { char *serv_fulltext_init(void) { - CtdlRegisterSessionHook(do_fulltext_indexing, EVT_TIMER); CtdlRegisterProtoHook(cmd_srch, "SRCH", "Full text search"); return "$Id$"; } diff --git a/citadel/serv_fulltext.h b/citadel/serv_fulltext.h index c083af261..5a106f7af 100644 --- a/citadel/serv_fulltext.h +++ b/citadel/serv_fulltext.h @@ -5,3 +5,4 @@ void ft_index_message(long msgnum, int op); void ft_search(int *fts_num_msgs, long **fts_msgs, char *search_string); +void *indexer_thread(void *arg); diff --git a/citadel/server_main.c b/citadel/server_main.c index 75c6c7137..fdf8e1589 100644 --- a/citadel/server_main.c +++ b/citadel/server_main.c @@ -241,6 +241,9 @@ int main(int argc, char **argv) } end_critical_section(S_WORKER_LIST); + /* Create the indexer thread. */ + create_indexer_thread(); + /* Now this thread can become a worker as well. */ worker_thread(NULL); diff --git a/citadel/sysconfig.h b/citadel/sysconfig.h index bdb78502d..9c233555c 100644 --- a/citadel/sysconfig.h +++ b/citadel/sysconfig.h @@ -124,3 +124,5 @@ #define CTDL_KEY_PATH CTDL_CRYPTO_DIR "/citadel.key" #define CTDL_CSR_PATH CTDL_CRYPTO_DIR "/citadel.csr" #define CTDL_CER_PATH CTDL_CRYPTO_DIR "/citadel.cer" + +#define THREADSTACKSIZE 1048576 diff --git a/citadel/sysdep.c b/citadel/sysdep.c index 30247b961..b71c068a2 100644 --- a/citadel/sysdep.c +++ b/citadel/sysdep.c @@ -66,6 +66,7 @@ #include "housekeeping.h" #include "tools.h" #include "serv_crypto.h" +#include "serv_fulltext.h" #ifdef HAVE_SYS_SELECT_H #include @@ -97,6 +98,7 @@ struct CitContext masterCC; time_t last_purge = 0; /* Last dead session purge */ static int num_threads = 0; /* Current number of threads */ int num_sessions = 0; /* Current number of sessions */ +pthread_t indexer_thread_tid; int syslog_facility = (-1); int enable_syslog = 0; @@ -410,8 +412,7 @@ struct CitContext *MyContext(void) { * Initialize a new context and place it in the list. The session number * used to be the PID (which is why it's called cs_pid), but that was when we * had one process per session. Now we just assign them sequentially, starting - * at 1 (don't change it to 0 because masterCC uses 0) and re-using them when - * sessions terminate. + * at 1 (don't change it to 0 because masterCC uses 0). */ struct CitContext *CreateNewContext(void) { struct CitContext *me; @@ -787,12 +788,13 @@ void create_worker(void) { return; } - /* Our per-thread stacks need to be bigger than the default size, otherwise - * the MIME parser crashes on FreeBSD, and the IMAP service crashes on - * 64-bit Linux. + /* Our per-thread stacks need to be bigger than the default size, + * otherwise the MIME parser crashes on FreeBSD, and the IMAP service + * crashes on 64-bit Linux. */ - if ((ret = pthread_attr_setstacksize(&attr, 1024 * 1024))) { - lprintf(CTDL_EMERG, "pthread_attr_setstacksize: %s\n", strerror(ret)); + if ((ret = pthread_attr_setstacksize(&attr, THREADSTACKSIZE))) { + lprintf(CTDL_EMERG, "pthread_attr_setstacksize: %s\n", + strerror(ret)); time_to_die = -1; pthread_attr_destroy(&attr); return; @@ -811,6 +813,41 @@ void create_worker(void) { } +/* + * Create the indexer thread and begin its operation. + */ +void create_indexer_thread(void) { + int ret; + pthread_attr_t attr; + + if ((ret = pthread_attr_init(&attr))) { + lprintf(CTDL_EMERG, "pthread_attr_init: %s\n", strerror(ret)); + time_to_die = -1; + return; + } + + /* Our per-thread stacks need to be bigger than the default size, + * otherwise the MIME parser crashes on FreeBSD, and the IMAP service + * crashes on 64-bit Linux. + */ + if ((ret = pthread_attr_setstacksize(&attr, THREADSTACKSIZE))) { + lprintf(CTDL_EMERG, "pthread_attr_setstacksize: %s\n", + strerror(ret)); + time_to_die = -1; + pthread_attr_destroy(&attr); + return; + } + + if ((ret = pthread_create(&indexer_thread_tid, &attr, indexer_thread, NULL) != 0)) + { + lprintf(CTDL_ALERT, "Can't create indexer thread: %s\n", + strerror(ret)); + } + + pthread_attr_destroy(&attr); +} + + /* * Purge all sessions which have the 'kill_me' flag set. diff --git a/citadel/sysdep_decls.h b/citadel/sysdep_decls.h index a4d574435..99071ec3c 100644 --- a/citadel/sysdep_decls.h +++ b/citadel/sysdep_decls.h @@ -41,6 +41,8 @@ void lprintf (enum LogLevel loglevel, const char *format, ...); void cprintf (const char *format, ...); #endif +extern pthread_key_t MyConKey; /* TSD key for MyContext() */ + extern int enable_syslog; void init_sysdep (void); @@ -95,3 +97,6 @@ void tracked_free(void *ptr); char *tracked_strdup(const char *s, char *file, int line); void dump_heap(void); #endif + +void create_indexer_thread(void); +extern pthread_t indexer_thread_tid; -- 2.30.2