A couple of minor speed ups and the beginings of using signals in
[citadel.git] / citadel / sysdep.c
index 85854cd40812b6b0390f066c422363879341d249..a0ccfc8f10fb01e07d1c1550c84fb420ebf48cb8 100644 (file)
  *
  */
 
-#ifdef DLL_EXPORT
-#define IN_LIBCIT
-#endif
-
 #include "sysdep.h"
 #include <stdlib.h>
 #include <unistd.h>
@@ -27,6 +23,7 @@
 #include <sys/stat.h>
 #include <sys/wait.h>
 #include <sys/socket.h>
+#include <syslog.h>
 #include <sys/syslog.h>
 
 #if TIME_WITH_SYS_TIME
@@ -43,6 +40,7 @@
 #include <limits.h>
 #include <sys/resource.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <arpa/inet.h>
 #include <netdb.h>
 #include <sys/un.h>
 #ifdef HAVE_PTHREAD_H
 #include <pthread.h>
 #endif
+#include <libcitadel.h>
 #include "citadel.h"
 #include "server.h"
-#include "serv_extensions.h"
 #include "sysdep_decls.h"
 #include "citserver.h"
 #include "support.h"
 #include "config.h"
 #include "database.h"
 #include "housekeeping.h"
-#include "tools.h"
-#include "serv_crypto.h"
+#include "modules/crypto/serv_crypto.h"        /* Needed for init_ssl, client_write_ssl, client_read_ssl, destruct_ssl */
+#include "ecrash.h"
 
 #ifdef HAVE_SYS_SELECT_H
 #include <sys/select.h>
@@ -74,6 +72,7 @@
 #include "snprintf.h"
 #endif
 
+#include "ctdl_module.h"
 
 #ifdef DEBUG_MEMORY_LEAKS
 struct igheap {
@@ -95,37 +94,46 @@ int verbosity = DEFAULT_VERBOSITY;          /* Logging level */
 struct CitContext masterCC;
 time_t last_purge = 0;                         /* Last dead session purge */
 static int num_threads = 0;                    /* Current number of threads */
+static int num_workers = 0;                    /* Current number of worker threads */
 int num_sessions = 0;                          /* Current number of sessions */
 
-int syslog_facility = (-1);
+int syslog_facility = LOG_DAEMON;
+int enable_syslog = 0;
+
+
+/*
+ * Create an interface to lprintf that follows the coding convention.
+ * This is here until such time as we have replaced all calls to lprintf with CtdlLogPrintf
+ */
+void CtdlLogPrintf(enum LogLevel loglevel, const char *format, ...)
+{
+       va_list arg_ptr;
+       va_start(arg_ptr, format);
+       vlprintf(loglevel, format, arg_ptr);
+       va_end(arg_ptr);
+}
 
 
 /*
  * lprintf()  ...   Write logging information
- * 
- * Note: the variable "buf" below needs to be large enough to handle any
- * log data sent through this function.  BE CAREFUL!
  */
-extern int running_as_daemon;
-static int enable_syslog = 1;
 void lprintf(enum LogLevel loglevel, const char *format, ...) {   
        va_list arg_ptr;
+       va_start(arg_ptr, format);
+       vlprintf(loglevel, format, arg_ptr);
+       va_end(arg_ptr);
+}
+
+void vlprintf(enum LogLevel loglevel, const char *format, va_list arg_ptr)
+{
+       char buf[SIZ], buf2[SIZ];
 
        if (enable_syslog) {
-               va_start(arg_ptr, format);
-                       vsyslog(loglevel, format, arg_ptr);
-               va_end(arg_ptr);
+               vsyslog((syslog_facility | loglevel), format, arg_ptr);
        }
 
-       if (enable_syslog && LogHookTable == 0) return;
-
-       /* legacy output code; hooks get processed first */
-       char buf[SIZ];
-       va_start(arg_ptr, format);   
-               vsnprintf(buf, sizeof(buf), format, arg_ptr);   
-       va_end(arg_ptr);   
-       PerformLogHooks(loglevel, buf);
-
+       /* stderr output code */
        if (enable_syslog || running_as_daemon) return;
 
        /* if we run in forground and syslog is disabled, log to terminal */
@@ -139,19 +147,22 @@ void lprintf(enum LogLevel loglevel, const char *format, ...) {
                unixtime = tv.tv_sec;
                localtime_r(&unixtime, &tim);
                if (CC->cs_pid != 0) {
-                       fprintf(stderr,
-                               "%04d/%02d/%02d %2d:%02d:%02d.%06ld [%3d] %s",
+                       sprintf(buf,
+                               "%04d/%02d/%02d %2d:%02d:%02d.%06ld [%3d] ",
                                tim.tm_year + 1900, tim.tm_mon + 1,
                                tim.tm_mday, tim.tm_hour, tim.tm_min,
                                tim.tm_sec, (long)tv.tv_usec,
-                               CC->cs_pid, buf);
+                               CC->cs_pid);
                } else {
-                       fprintf(stderr,
-                               "%04d/%02d/%02d %2d:%02d:%02d.%06ld %s",
+                       sprintf(buf,
+                               "%04d/%02d/%02d %2d:%02d:%02d.%06ld ",
                                tim.tm_year + 1900, tim.tm_mon + 1,
                                tim.tm_mday, tim.tm_hour, tim.tm_min,
-                               tim.tm_sec, (long)tv.tv_usec, buf);
+                               tim.tm_sec, (long)tv.tv_usec);
                }
+               vsprintf(buf2, format, arg_ptr);   
+
+               fprintf(stderr, "%s%s", buf, buf2);
                fflush(stderr);
        }
 }   
@@ -159,26 +170,47 @@ void lprintf(enum LogLevel loglevel, const char *format, ...) {
 
 
 /*
- * We used to use master_cleanup() as a signal handler to shut down the server.
- * however, master_cleanup() and the functions it calls do some things that
- * aren't such a good idea to do from a signal handler: acquiring mutexes,
- * playing with signal masks on BSDI systems, etc. so instead we install the
- * following signal handler to set a global variable to inform the main loop
- * that it's time to call master_cleanup() and exit.
+ * Signal handler to shut down the server.
  */
 
-volatile int time_to_die = 0;
+volatile int exit_signal = 0;
+volatile int shutdown_and_halt = 0;
+volatile int restart_server = 0;
+volatile int running_as_daemon = 0;
 
 static RETSIGTYPE signal_cleanup(int signum) {
-       time_to_die = 1;
+       CtdlLogPrintf(CTDL_DEBUG, "Caught signal %d; shutting down.\n", signum);
+#ifdef THREADS_USESIGNALS
+       if (CT)
+       {
+               CtdlLogPrintf(CTDL_DEBUG, "Thread \"%s\" caught signal %d.\n", CT->name, signum);
+               CT->signal = signum;
+       }
+       else
+#endif
+               exit_signal = signum;
+}
+
+
+
+
+void InitialiseSemaphores(void)
+{
+       int i;
+
+       /* Set up a bunch of semaphores to be used for critical sections */
+       for (i=0; i<MAX_SEMAPHORES; ++i) {
+               pthread_mutex_init(&Critters[i], NULL);
+       }
 }
 
 
+
 /*
  * Some initialization stuff...
  */
 void init_sysdep(void) {
-       int i;
+       sigset_t set;
 
        /* Avoid vulnerabilities related to FD_SETSIZE if we can. */
 #ifdef FD_SETSIZE
@@ -196,11 +228,6 @@ void init_sysdep(void) {
        init_ssl();
 #endif
 
-       /* Set up a bunch of semaphores to be used for critical sections */
-       for (i=0; i<MAX_SEMAPHORES; ++i) {
-               pthread_mutex_init(&Critters[i], NULL);
-       }
-
        /*
         * Set up a place to put thread-specific data.
         * We only need a single pointer per thread - it points to the
@@ -208,17 +235,31 @@ void init_sysdep(void) {
         * session to which the calling thread is currently bound.
         */
        if (pthread_key_create(&MyConKey, NULL) != 0) {
-               lprintf(CTDL_CRIT, "Can't create TSD key!!  %s\n", strerror(errno));
+               CtdlLogPrintf(CTDL_CRIT, "Can't create TSD key: %s\n",
+                       strerror(errno));
        }
 
        /*
         * The action for unexpected signals and exceptions should be to
         * call signal_cleanup() to gracefully shut down the server.
         */
+       sigemptyset(&set);
+       sigaddset(&set, SIGINT);
+       sigaddset(&set, SIGQUIT);
+       sigaddset(&set, SIGHUP);
+       sigaddset(&set, SIGTERM);
+       // sigaddset(&set, SIGSEGV);    commented out because
+       // sigaddset(&set, SIGILL);     we want core dumps
+       // sigaddset(&set, SIGBUS);
+       sigprocmask(SIG_UNBLOCK, &set, NULL);
+
        signal(SIGINT, signal_cleanup);
        signal(SIGQUIT, signal_cleanup);
        signal(SIGHUP, signal_cleanup);
        signal(SIGTERM, signal_cleanup);
+       // signal(SIGSEGV, signal_cleanup);     commented out because
+       // signal(SIGILL, signal_cleanup);      we want core dumps
+       // signal(SIGBUS, signal_cleanup);
 
        /*
         * Do not shut down the server on broken pipe signals, otherwise the
@@ -229,12 +270,35 @@ void init_sysdep(void) {
 }
 
 
+
+/*
+ * Obtain a semaphore lock to begin a critical section.
+ * but only if no one else has one
+ */
+int try_critical_section(int which_one)
+{
+       /* For all types of critical sections except those listed here,
+        * ensure nobody ever tries to do a critical section within a
+        * transaction; this could lead to deadlock.
+        */
+       if (    (which_one != S_FLOORCACHE)
+#ifdef DEBUG_MEMORY_LEAKS
+               && (which_one != S_DEBUGMEMLEAKS)
+#endif
+               && (which_one != S_RPLIST)
+       ) {
+               cdb_check_handles();
+       }
+       return (pthread_mutex_trylock(&Critters[which_one]));
+}
+
+
 /*
  * Obtain a semaphore lock to begin a critical section.
  */
 void begin_critical_section(int which_one)
 {
-       /* lprintf(CTDL_DEBUG, "begin_critical_section(%d)\n", which_one); */
+       /* CtdlLogPrintf(CTDL_DEBUG, "begin_critical_section(%d)\n", which_one); */
 
        /* For all types of critical sections except those listed here,
         * ensure nobody ever tries to do a critical section within a
@@ -244,6 +308,7 @@ void begin_critical_section(int which_one)
 #ifdef DEBUG_MEMORY_LEAKS
                && (which_one != S_DEBUGMEMLEAKS)
 #endif
+               && (which_one != S_RPLIST)
        ) {
                cdb_check_handles();
        }
@@ -265,7 +330,7 @@ void end_critical_section(int which_one)
  * a TCP port.  The server shuts down if the bind fails.
  *
  */
-int ig_tcp_server(char *ip_addr, int port_number, int queue_len)
+int ig_tcp_server(char *ip_addr, int port_number, int queue_len, char **errormessage)
 {
        struct sockaddr_in sin;
        int s, i;
@@ -284,15 +349,18 @@ int ig_tcp_server(char *ip_addr, int port_number, int queue_len)
                sin.sin_addr.s_addr = inet_addr(ip_addr);
        }
                                                                                
-       if (sin.sin_addr.s_addr == INADDR_NONE) {
+       if (sin.sin_addr.s_addr == !INADDR_ANY) {
                sin.sin_addr.s_addr = INADDR_ANY;
        }
 
        s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
 
        if (s < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't create a socket: %s\n",
-                       strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                                "citserver: Can't create a socket: %s",
+                                strerror(errno));
+               CtdlLogPrintf(CTDL_EMERG, "%s\n", *errormessage);
                return(-1);
        }
 
@@ -300,14 +368,32 @@ int ig_tcp_server(char *ip_addr, int port_number, int queue_len)
        setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &i, sizeof(i));
 
        if (bind(s, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't bind: %s\n",
-                       strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                                "citserver: Can't bind: %s",
+                                strerror(errno));
+               CtdlLogPrintf(CTDL_EMERG, "%s\n", *errormessage);
+               close(s);
+               return(-1);
+       }
+
+       /* set to nonblock - we need this for some obscure situations */
+       if (fcntl(s, F_SETFL, O_NONBLOCK) < 0) {
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                                "citserver: Can't set socket to non-blocking: %s",
+                                strerror(errno));
+               CtdlLogPrintf(CTDL_EMERG, "%s\n", *errormessage);
                close(s);
                return(-1);
        }
 
        if (listen(s, actual_queue_len) < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't listen: %s\n", strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                                "citserver: Can't listen: %s",
+                                strerror(errno));
+               CtdlLogPrintf(CTDL_EMERG, "%s\n", *errormessage);
                close(s);
                return(-1);
        }
@@ -320,7 +406,7 @@ int ig_tcp_server(char *ip_addr, int port_number, int queue_len)
 /*
  * Create a Unix domain socket and listen on it
  */
-int ig_uds_server(char *sockpath, int queue_len)
+int ig_uds_server(char *sockpath, int queue_len, char **errormessage)
 {
        struct sockaddr_un addr;
        int s;
@@ -332,8 +418,10 @@ int ig_uds_server(char *sockpath, int queue_len)
 
        i = unlink(sockpath);
        if (i != 0) if (errno != ENOENT) {
-               lprintf(CTDL_EMERG, "citserver: can't unlink %s: %s\n",
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, "citserver: can't unlink %s: %s",
                        sockpath, strerror(errno));
+               CtdlLogPrintf(CTDL_EMERG, "%s\n", *errormessage);
                return(-1);
        }
 
@@ -343,23 +431,44 @@ int ig_uds_server(char *sockpath, int queue_len)
 
        s = socket(AF_UNIX, SOCK_STREAM, 0);
        if (s < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't create a socket: %s\n",
-                       strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                        "citserver: Can't create a socket: %s",
+                        strerror(errno));
+               CtdlLogPrintf(CTDL_EMERG, "%s\n", *errormessage);
                return(-1);
        }
 
        if (bind(s, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't bind: %s\n",
-                       strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                        "citserver: Can't bind: %s",
+                        strerror(errno));
+               CtdlLogPrintf(CTDL_EMERG, "%s\n", *errormessage);
+               return(-1);
+       }
+
+       /* set to nonblock - we need this for some obscure situations */
+       if (fcntl(s, F_SETFL, O_NONBLOCK) < 0) {
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                        "citserver: Can't set socket to non-blocking: %s",
+                        strerror(errno));
+               CtdlLogPrintf(CTDL_EMERG, "%s\n", *errormessage);
+               close(s);
                return(-1);
        }
 
        if (listen(s, actual_queue_len) < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't listen: %s\n", strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                        "citserver: Can't listen: %s",
+                        strerror(errno));
+               CtdlLogPrintf(CTDL_EMERG, "%s\n", *errormessage);
                return(-1);
        }
 
-       chmod(sockpath, 0777);
+       chmod(sockpath, S_ISGID|S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IWOTH|S_IXOTH);
        return(s);
 }
 
@@ -386,15 +495,15 @@ struct CitContext *MyContext(void) {
  * Initialize a new context and place it in the list.  The session number
  * used to be the PID (which is why it's called cs_pid), but that was when we
  * had one process per session.  Now we just assign them sequentially, starting
- * at 1 (don't change it to 0 because masterCC uses 0) and re-using them when
- * sessions terminate.
+ * at 1 (don't change it to 0 because masterCC uses 0).
  */
 struct CitContext *CreateNewContext(void) {
-       struct CitContext *me, *ptr;
+       struct CitContext *me;
+       static int next_pid = 0;
 
        me = (struct CitContext *) malloc(sizeof(struct CitContext));
        if (me == NULL) {
-               lprintf(CTDL_ALERT, "citserver: can't allocate memory!!\n");
+               CtdlLogPrintf(CTDL_ALERT, "citserver: can't allocate memory!!\n");
                return NULL;
        }
        memset(me, 0, sizeof(struct CitContext));
@@ -405,52 +514,71 @@ struct CitContext *CreateNewContext(void) {
         */
        me->state = CON_EXECUTING;
 
-
        /*
         * Generate a unique session number and insert this context into
         * the list.
         */
        begin_critical_section(S_SESSION_TABLE);
-
-       if (ContextList == NULL) {
-               ContextList = me;
-               me->cs_pid = 1;
-               me->next = NULL;
-       }
-
-       else if (ContextList->cs_pid > 1) {
-               me->next = ContextList;
-               ContextList = me;
-               me->cs_pid = 1;
-       }
-
-       else {
-               for (ptr = ContextList; ptr != NULL; ptr = ptr->next) {
-                       if (ptr->next == NULL) {
-                               ptr->next = me;
-                               me->cs_pid = ptr->cs_pid + 1;
-                               me->next = NULL;
-                               goto DONE;
-                       }
-                       else if (ptr->next->cs_pid > (ptr->cs_pid+1)) {
-                               me->next = ptr->next;
-                               ptr->next = me;
-                               me->cs_pid = ptr->cs_pid + 1;
-                               goto DONE;
-                       }
-               }
+       me->cs_pid = ++next_pid;
+       me->prev = NULL;
+       me->next = ContextList;
+       ContextList = me;
+       if (me->next != NULL) {
+               me->next->prev = me;
        }
-
-DONE:  ++num_sessions;
+       ++num_sessions;
        end_critical_section(S_SESSION_TABLE);
        return(me);
 }
 
 
 /*
- * buffer_output() ... tell client_write to buffer all output until
- *                  instructed to dump it all out later
+ * The following functions implement output buffering. If the kernel supplies
+ * native TCP buffering (Linux & *BSD), use that; otherwise, emulate it with
+ * user-space buffering.
  */
+#ifndef HAVE_DARWIN
+#ifdef TCP_CORK
+#      define HAVE_TCP_BUFFERING
+#else
+#      ifdef TCP_NOPUSH
+#              define HAVE_TCP_BUFFERING
+#              define TCP_CORK TCP_NOPUSH
+#      endif
+#endif /* TCP_CORK */
+#endif /* HAVE_DARWIN */
+
+#ifdef HAVE_TCP_BUFFERING
+static unsigned on = 1, off = 0;
+void buffer_output(void) {
+       struct CitContext *ctx = MyContext();
+       setsockopt(ctx->client_socket, IPPROTO_TCP, TCP_CORK, &on, 4);
+       ctx->buffering = 1;
+}
+
+void unbuffer_output(void) {
+       struct CitContext *ctx = MyContext();
+       setsockopt(ctx->client_socket, IPPROTO_TCP, TCP_CORK, &off, 4);
+       ctx->buffering = 0;
+}
+
+void flush_output(void) {
+       struct CitContext *ctx = MyContext();
+       setsockopt(ctx->client_socket, IPPROTO_TCP, TCP_CORK, &off, 4);
+       setsockopt(ctx->client_socket, IPPROTO_TCP, TCP_CORK, &on, 4);
+}
+#else 
+#ifdef HAVE_DARWIN
+/* Stub functions for Darwin/OS X where TCP buffering isn't liked at all */
+void buffer_output(void) {
+       CC->buffering = 0;
+}
+void unbuffer_output(void) {
+       CC->buffering = 0;
+}
+void flush_output(void) {
+}
+#else
 void buffer_output(void) {
        if (CC->buffering == 0) {
                CC->buffering = 1;
@@ -459,9 +587,6 @@ void buffer_output(void) {
        }
 }
 
-/*
- * flush_output()  ...   dump out all that output we've been buffering.
- */
 void flush_output(void) {
        if (CC->buffering == 1) {
                client_write(CC->output_buffer, CC->buffer_len);
@@ -469,9 +594,6 @@ void flush_output(void) {
        }
 }
 
-/*
- * unbuffer_output()  ...  stop buffering output.
- */
 void unbuffer_output(void) {
        if (CC->buffering == 1) {
                CC->buffering = 0;
@@ -482,6 +604,8 @@ void unbuffer_output(void) {
                CC->output_buffer = NULL;
        }
 }
+#endif /* HAVE_DARWIN */
+#endif /* HAVE_TCP_BUFFERING */
 
 
 
@@ -492,46 +616,55 @@ void client_write(char *buf, int nbytes)
 {
        int bytes_written = 0;
        int retval;
-       int sock;
+#ifndef HAVE_TCP_BUFFERING
        int old_buffer_len = 0;
-
-       if (CC->redirect_fp != NULL) {
-               fwrite(buf, (size_t)nbytes, (size_t)1, CC->redirect_fp);
+#endif
+       t_context *Ctx;
+
+       Ctx = CC;
+       if (Ctx->redirect_buffer != NULL) {
+               if ((Ctx->redirect_len + nbytes + 2) >= Ctx->redirect_alloc) {
+                       Ctx->redirect_alloc = (Ctx->redirect_alloc * 2) + nbytes;
+                       Ctx->redirect_buffer = realloc(Ctx->redirect_buffer,
+                                               Ctx->redirect_alloc);
+               }
+               memcpy(&Ctx->redirect_buffer[Ctx->redirect_len], buf, nbytes);
+               Ctx->redirect_len += nbytes;
+               Ctx->redirect_buffer[Ctx->redirect_len] = 0;
                return;
        }
 
-       if (CC->redirect_sock > 0) {
-               sock = CC->redirect_sock;       /* and continue below... */
-       }
-       else {
-               sock = CC->client_socket;
-       }
-
+#ifndef HAVE_TCP_BUFFERING
        /* If we're buffering for later, do that now. */
-       if (CC->buffering) {
-               old_buffer_len = CC->buffer_len;
-               CC->buffer_len += nbytes;
-               CC->output_buffer = realloc(CC->output_buffer, CC->buffer_len);
-               memcpy(&CC->output_buffer[old_buffer_len], buf, nbytes);
+       if (Ctx->buffering) {
+               old_buffer_len = Ctx->buffer_len;
+               Ctx->buffer_len += nbytes;
+               Ctx->output_buffer = realloc(Ctx->output_buffer, Ctx->buffer_len);
+               memcpy(&Ctx->output_buffer[old_buffer_len], buf, nbytes);
                return;
        }
+#endif
 
        /* Ok, at this point we're not buffering.  Go ahead and write. */
 
 #ifdef HAVE_OPENSSL
-       if (CC->redirect_ssl) {
+       if (Ctx->redirect_ssl) {
                client_write_ssl(buf, nbytes);
                return;
        }
 #endif
 
        while (bytes_written < nbytes) {
-               retval = write(sock, &buf[bytes_written],
+               retval = write(Ctx->client_socket, &buf[bytes_written],
                        nbytes - bytes_written);
                if (retval < 1) {
-                       lprintf(CTDL_ERR, "client_write() failed: %s\n",
-                               strerror(errno));
-                       if (sock == CC->client_socket) CC->kill_me = 1;
+                       CtdlLogPrintf(CTDL_ERR,
+                               "client_write(%d bytes) failed: %s (%d)\n",
+                               nbytes - bytes_written,
+                               strerror(errno), errno);
+                       cit_backtrace();
+                       // CtdlLogPrintf(CTDL_DEBUG, "Tried to send: %s",  &buf[bytes_written]);
+                       Ctx->kill_me = 1;
                        return;
                }
                bytes_written = bytes_written + retval;
@@ -546,7 +679,7 @@ void client_write(char *buf, int nbytes)
  */
 void cprintf(const char *format, ...) {   
        va_list arg_ptr;   
-       char buf[SIZ];   
+       char buf[1024];   
    
        va_start(arg_ptr, format);   
        if (vsnprintf(buf, sizeof buf, format, arg_ptr) == -1)
@@ -568,6 +701,7 @@ int client_read_to(char *buf, int bytes, int timeout)
 {
        int len,rlen;
        fd_set rfds;
+       int fd;
        struct timeval tv;
        int retval;
 
@@ -577,23 +711,23 @@ int client_read_to(char *buf, int bytes, int timeout)
        }
 #endif
        len = 0;
+       fd = CC->client_socket;
        while(len<bytes) {
                FD_ZERO(&rfds);
-               FD_SET(CC->client_socket, &rfds);
+               FD_SET(fd, &rfds);
                tv.tv_sec = timeout;
                tv.tv_usec = 0;
 
-               retval = select( (CC->client_socket)+1, 
-                                       &rfds, NULL, NULL, &tv);
+               retval = select( (fd)+1, 
+                                &rfds, NULL, NULL, &tv);
 
-               if (FD_ISSET(CC->client_socket, &rfds) == 0) {
+               if (FD_ISSET(fd, &rfds) == 0) {
                        return(0);
                }
 
-               rlen = read(CC->client_socket, &buf[len], bytes-len);
+               rlen = read(fd, &buf[len], bytes-len);
                if (rlen<1) {
-                       lprintf(CTDL_ERR, "client_read() failed: %s\n",
-                               strerror(errno));
+                       /* The socket has been disconnected! */
                        CC->kill_me = 1;
                        return(-1);
                }
@@ -636,23 +770,60 @@ int client_getln(char *buf, int bufsize)
                while (buf[i] != '\n' && retval == 1)
                        retval = client_read(&buf[i], 1);
 
-       /* Strip the trailing newline and any trailing nonprintables (cr's)
+       /* Strip the trailing LF, and the trailing CR if present.
         */
        buf[i] = 0;
-       while ((strlen(buf)>0)&&(!isprint(buf[strlen(buf)-1])))
-               buf[strlen(buf)-1] = 0;
-       if (retval < 0) strcpy(buf, "000");
+       while ( (i > 0)
+               && ( (buf[i - 1]==13)
+                    || ( buf[i - 1]==10)) ) {
+               i--;
+               buf[i] = 0;
+       }
+       if (retval < 0) safestrncpy(&buf[i], "000", bufsize - i);
        return(retval);
 }
 
 
+/*
+ * Cleanup any contexts that are left lying around
+ */
+void context_cleanup(void)
+{
+       struct CitContext *ptr = NULL;
+       struct CitContext *rem = NULL;
+
+       /*
+        * Clean up the contexts.
+        * There are no threads so no critical_section stuff is needed.
+        */
+       ptr = ContextList;
+       
+       /* We need to update the ContextList because some modules may want to itterate it
+        * Question is should we NULL it before iterating here or should we just keep updating it
+        * as we remove items?
+        *
+        * Answer is to NULL it first to prevent modules from doing any actions on the list at all
+        */
+       ContextList=NULL;
+       while (ptr != NULL){
+               /* Remove the session from the active list */
+               rem = ptr->next;
+               --num_sessions;
+               
+               lprintf(CTDL_DEBUG, "Purging session %d\n", ptr->cs_pid);
+               RemoveContext(ptr);
+               free (ptr);
+               ptr = rem;
+       }
+}
+
 
 /*
  * The system-dependent part of master_cleanup() - close the master socket.
  */
 void sysdep_master_cleanup(void) {
        struct ServiceFunctionHook *serviceptr;
-
+       
        /*
         * close all protocol master sockets
         */
@@ -660,11 +831,11 @@ void sysdep_master_cleanup(void) {
            serviceptr = serviceptr->next ) {
 
                if (serviceptr->tcp_port > 0)
-                       lprintf(CTDL_INFO, "Closing listener on port %d\n",
+                       CtdlLogPrintf(CTDL_INFO, "Closing listener on port %d\n",
                                serviceptr->tcp_port);
 
                if (serviceptr->sockpath != NULL)
-                       lprintf(CTDL_INFO, "Closing listener on '%s'\n",
+                       CtdlLogPrintf(CTDL_INFO, "Closing listener on '%s'\n",
                                serviceptr->sockpath);
 
                close(serviceptr->msock);
@@ -674,9 +845,31 @@ void sysdep_master_cleanup(void) {
                        unlink(serviceptr->sockpath);
                }
        }
+       
+       context_cleanup();
+       
+#ifdef HAVE_OPENSSL
+       destruct_ssl();
+#endif
+       CtdlDestroyProtoHooks();
+       CtdlDestroyDeleteHooks();
+       CtdlDestroyXmsgHooks();
+       CtdlDestroyNetprocHooks();
+       CtdlDestroyUserHooks();
+       CtdlDestroyMessageHook();
+       CtdlDestroyCleanupHooks();
+       CtdlDestroyFixedOutputHooks();  
+       CtdlDestroySessionHooks();
+       CtdlDestroyServiceHook();
+       CtdlDestroyRoomHooks();
+       CtdlDestroyDirectoryServiceFuncs();
+       #ifdef HAVE_BACKTRACE
+       eCrash_Uninit();
+       #endif
 }
 
 
+
 /*
  * Terminate another session.
  * (This could justifiably be moved out of sysdep.c because it
@@ -694,19 +887,103 @@ void kill_session(int session_to_kill) {
        end_critical_section(S_SESSION_TABLE);
 }
 
-
+pid_t current_child;
+void graceful_shutdown(int signum) {
+       kill(current_child, signum);
+       unlink(file_pid_file);
+       exit(0);
+}
 
 
 /*
  * Start running as a daemon.
  */
 void start_daemon(int unused) {
-       close(0); close(1); close(2);
-       if (fork()) exit(0);
+       int status = 0;
+       pid_t child = 0;
+       FILE *fp;
+       int do_restart = 0;
+
+       current_child = 0;
+
+       /* Close stdin/stdout/stderr and replace them with /dev/null.
+        * We don't just call close() because we don't want these fd's
+        * to be reused for other files.
+        */
+       chdir(ctdl_run_dir);
+
+       child = fork();
+       if (child != 0) {
+               exit(0);
+       }
+       
+       signal(SIGHUP, SIG_IGN);
+       signal(SIGINT, SIG_IGN);
+       signal(SIGQUIT, SIG_IGN);
+
        setsid();
-       signal(SIGHUP,SIG_IGN);
-       signal(SIGINT,SIG_IGN);
-       signal(SIGQUIT,SIG_IGN);
+       umask(0);
+        freopen("/dev/null", "r", stdin);
+        freopen("/dev/null", "w", stdout);
+        freopen("/dev/null", "w", stderr);
+
+       do {
+               current_child = fork();
+
+               signal(SIGTERM, graceful_shutdown);
+       
+               if (current_child < 0) {
+                       perror("fork");
+                       exit(errno);
+               }
+       
+               else if (current_child == 0) {
+                       return; /* continue starting citadel. */
+               }
+       
+               else {
+                       fp = fopen(file_pid_file, "w");
+                       if (fp != NULL) {
+               /*
+                * NB.. The pid file contains the pid of the actual server.
+                * This is not the pid of the watcher process
+                */
+                               fprintf(fp, ""F_PID_T"\n", current_child);
+                               fclose(fp);
+                       }
+                       waitpid(current_child, &status, 0);
+               }
+
+               do_restart = 0;
+
+               /* Did the main process exit with an actual exit code? */
+               if (WIFEXITED(status)) {
+
+                       /* Exit code 0 means the watcher should exit */
+                       if (WEXITSTATUS(status) == 0) {
+                               do_restart = 0;
+                       }
+
+                       /* Exit code 101-109 means the watcher should exit */
+                       else if ( (WEXITSTATUS(status) >= 101) && (WEXITSTATUS(status) <= 109) ) {
+                               do_restart = 0;
+                       }
+
+                       /* Any other exit code means we should restart. */
+                       else {
+                               do_restart = 1;
+                       }
+               }
+
+               /* Any other type of termination (signals, etc.) should also restart. */
+               else {
+                       do_restart = 1;
+               }
+
+       } while (do_restart);
+
+       unlink(file_pid_file);
+       exit(WEXITSTATUS(status));
 }
 
 
@@ -732,142 +1009,1141 @@ int convert_login(char NameToConvert[]) {
        }
 }
 
-struct worker_node *worker_list = NULL;
 
 
 /*
- * create a worker thread. this function must always be called from within
- * an S_WORKER_LIST critical section!
+ * New thread interface.
+ * To create a thread you must call one of the create thread functions.
+ * You must pass it the address of (a pointer to a CtdlThreadNode initialised to NULL) like this
+ * struct CtdlThreadNode *node = NULL;
+ * pass in &node
+ * If the thread is created *node will point to the thread control structure for the created thread.
+ * If the thread creation fails *node remains NULL
+ * Do not free the memory pointed to by *node, it doesn't belong to you.
+ * This new interface duplicates much of the eCrash stuff. We should go for closer integration since that would
+ * remove the need for the calls to eCrashRegisterThread and friends
  */
-void create_worker(void) {
-       int ret;
-       struct worker_node *n;
-       pthread_attr_t attr;
 
-       n = malloc(sizeof(struct worker_node));
-       if (n == NULL) {
-               lprintf(CTDL_EMERG, "can't allocate worker_node, exiting\n");
-               time_to_die = -1;
-               return;
-       }
 
-       if ((ret = pthread_attr_init(&attr))) {
-               lprintf(CTDL_EMERG, "pthread_attr_init: %s\n", strerror(ret));
-               time_to_die = -1;
-               return;
-       }
+struct CtdlThreadNode *CtdlThreadList = NULL;
+struct CtdlThreadNode *CtdlThreadSchedList = NULL;
 
-       /* Our per-thread stacks need to be bigger than the default size, otherwise
-        * the MIME parser crashes on FreeBSD, and the IMAP service crashes on
-        * 64-bit Linux.
-        */
-       if ((ret = pthread_attr_setstacksize(&attr, 1024 * 1024))) {
-               lprintf(CTDL_EMERG, "pthread_attr_setstacksize: %s\n", strerror(ret));
-               time_to_die = -1;
-               pthread_attr_destroy(&attr);
-               return;
+/*
+ * Condition variable and Mutex for thread garbage collection
+ */
+/*static pthread_mutex_t thread_gc_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t thread_gc_cond = PTHREAD_COND_INITIALIZER;
+*/
+static pthread_t GC_thread;
+static char *CtdlThreadStates[CTDL_THREAD_LAST_STATE];
+double CtdlThreadLoadAvg = 0;
+double CtdlThreadWorkerAvg = 0;
+pthread_key_t ThreadKey;
+
+/*
+ * A function to destroy the TSD
+ */
+static void ctdl_thread_internal_dest_tsd(void *arg)
+{
+       if (arg != NULL) {
+               check_handles(arg);
+               free(arg);
        }
+}
 
-       if ((ret = pthread_create(&n->tid, &attr, worker_thread, NULL) != 0))
-       {
 
-               lprintf(CTDL_ALERT, "Can't create worker thread: %s\n",
+/*
+ * A function to initialise the thread TSD
+ */
+void ctdl_thread_internal_init_tsd(void)
+{
+       int ret;
+       
+       if ((ret = pthread_key_create(&ThreadKey, ctdl_thread_internal_dest_tsd))) {
+               lprintf(CTDL_EMERG, "pthread_key_create: %s\n",
                        strerror(ret));
+               exit(CTDLEXIT_DB);
        }
-
-       n->next = worker_list;
-       worker_list = n;
-       pthread_attr_destroy(&attr);
 }
 
-
-
 /*
- * Purge all sessions which have the 'kill_me' flag set.
- * This function has code to prevent it from running more than once every
- * few seconds, because running it after every single unbind would waste a lot
- * of CPU time and keep the context list locked too much.  To force it to run
- * anyway, set "force" to nonzero.
- *
+ * Ensure that we have a key for thread-specific data. 
  *
- * After that's done, we raise the size of the worker thread pool
- * if such an action is appropriate.
+ * This should be called immediately after startup by any thread 
+ * 
  */
-void dead_session_purge(int force) {
-       struct CitContext *ptr, *rem;
+void CtdlThreadAllocTSD(void)
+{
+       ThreadTSD *tsd;
 
-       if (force == 0) {
-               if ( (time(NULL) - last_purge) < 5 ) {
-                       return; /* Too soon, go away */
-               }
-       }
-       time(&last_purge);
+       if (pthread_getspecific(ThreadKey) != NULL)
+               return;
 
-       do {
-               rem = NULL;
-               begin_critical_section(S_SESSION_TABLE);
-               for (ptr = ContextList; ptr != NULL; ptr = ptr->next) {
-                       if ( (ptr->state == CON_IDLE) && (ptr->kill_me) ) {
-                               rem = ptr;
-                       }
-               }
-               end_critical_section(S_SESSION_TABLE);
+       tsd = malloc(sizeof(ThreadTSD));
 
-               /* RemoveContext() enters its own S_SESSION_TABLE critical
-                * section, so we have to do it like this.
-                */     
-               if (rem != NULL) {
-                       lprintf(CTDL_DEBUG, "Purging session %d\n", rem->cs_pid);
-                       RemoveContext(rem);
-               }
+       tsd->tid = NULL;
 
-       } while (rem != NULL);
+       memset(tsd->cursors, 0, sizeof tsd->cursors);
+       tsd->self = NULL;
+       
+       pthread_setspecific(ThreadKey, tsd);
+}
 
-       /* Raise the size of the worker thread pool if necessary. */
 
-       if ( (num_sessions > num_threads)
-          && (num_threads < config.c_max_workers) ) {
-               begin_critical_section(S_WORKER_LIST);
-               create_worker();
-               end_critical_section(S_WORKER_LIST);
-       }
+void ctdl_thread_internal_free_tsd(void)
+{
+       ctdl_thread_internal_dest_tsd(pthread_getspecific(ThreadKey));
+       pthread_setspecific(ThreadKey, NULL);
 }
 
 
+void ctdl_thread_internal_cleanup(void)
+{
+       int i;
+       struct CtdlThreadNode *this_thread, *that_thread;
+       
+       for (i=0; i<CTDL_THREAD_LAST_STATE; i++)
+       {
+               free (CtdlThreadStates[i]);
+       }
+       
+       /* Clean up the scheduled thread list */
+       this_thread = CtdlThreadSchedList;
+       while (this_thread)
+       {
+               that_thread = this_thread;
+               this_thread = this_thread->next;
+               pthread_mutex_destroy(&that_thread->ThreadMutex);
+               pthread_cond_destroy(&that_thread->ThreadCond);
+               pthread_mutex_destroy(&that_thread->SleepMutex);
+               pthread_cond_destroy(&that_thread->SleepCond);
+               pthread_attr_destroy(&that_thread->attr);
+               free(that_thread);
+       }
+       ctdl_thread_internal_free_tsd();
+}
+
+void ctdl_thread_internal_init(void)
+{
+       struct CtdlThreadNode *this_thread;
+       int ret = 0;
+       
+       GC_thread = pthread_self();
+       CtdlThreadStates[CTDL_THREAD_INVALID] = strdup ("Invalid Thread");
+       CtdlThreadStates[CTDL_THREAD_VALID] = strdup("Valid Thread");
+       CtdlThreadStates[CTDL_THREAD_CREATE] = strdup("Thread being Created");
+       CtdlThreadStates[CTDL_THREAD_CANCELLED] = strdup("Thread Cancelled");
+       CtdlThreadStates[CTDL_THREAD_EXITED] = strdup("Thread Exited");
+       CtdlThreadStates[CTDL_THREAD_STOPPING] = strdup("Thread Stopping");
+       CtdlThreadStates[CTDL_THREAD_STOP_REQ] = strdup("Thread Stop Requested");
+       CtdlThreadStates[CTDL_THREAD_SLEEPING] = strdup("Thread Sleeping");
+       CtdlThreadStates[CTDL_THREAD_RUNNING] = strdup("Thread Running");
+       CtdlThreadStates[CTDL_THREAD_BLOCKED] = strdup("Thread Blocked");
+       
+       /* Get ourself a thread entry */
+       this_thread = malloc(sizeof(struct CtdlThreadNode));
+       if (this_thread == NULL) {
+               CtdlLogPrintf(CTDL_EMERG, "Thread system, can't allocate CtdlThreadNode, exiting\n");
+               return;
+       }
+       // Ensuring this is zero'd means we make sure the thread doesn't start doing its thing until we are ready.
+       memset (this_thread, 0, sizeof(struct CtdlThreadNode));
+       
+       pthread_mutex_init (&(this_thread->ThreadMutex), NULL);
+       pthread_cond_init (&(this_thread->ThreadCond), NULL);
+       pthread_mutex_init (&(this_thread->SleepMutex), NULL);
+       pthread_cond_init (&(this_thread->SleepCond), NULL);
+       
+       /* We are garbage collector so create us as running */
+       this_thread->state = CTDL_THREAD_RUNNING;
+       
+       if ((ret = pthread_attr_init(&this_thread->attr))) {
+               CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_init: %s\n", strerror(ret));
+               free(this_thread);
+               return;
+       }
 
+       this_thread->name = "Garbage Collection Thread";
+       
+       this_thread->tid = GC_thread;
+       CT = this_thread;
+       
+       num_threads++;  // Increase the count of threads in the system.
+
+       this_thread->next = CtdlThreadList;
+       CtdlThreadList = this_thread;
+       if (this_thread->next)
+               this_thread->next->prev = this_thread;
+       /* Set up start times */
+       gettimeofday(&this_thread->start_time, NULL);           /* Time this thread started */
+       memcpy(&this_thread->last_state_change, &this_thread->start_time, sizeof (struct timeval));     /* Changed state so mark it. */
+}
 
 
 /*
- * Redirect a session's output to a file or socket.
- * This function may be called with a file handle *or* a socket (but not
- * both).  Call with neither to return output to its normal client socket.
+ * A function to update a threads load averages
  */
-void CtdlRedirectOutput(FILE *fp, int sock) {
-
-       if (fp != NULL) CC->redirect_fp = fp;
-       else CC->redirect_fp = NULL;
-
-       if (sock > 0) CC->redirect_sock = sock;
-       else CC->redirect_sock = (-1);
-
+ void ctdl_thread_internal_update_avgs(struct CtdlThreadNode *this_thread)
+ {
+       struct timeval now, result;
+       double last_duration;
+
+       gettimeofday(&now, NULL);
+       timersub(&now, &(this_thread->last_state_change), &result);
+       pthread_mutex_lock(&this_thread->ThreadMutex);
+       // result now has a timeval for the time we spent in the last state since we last updated
+       last_duration = (double)result.tv_sec + ((double)result.tv_usec / (double) 1000000);
+       if (this_thread->state == CTDL_THREAD_SLEEPING)
+               this_thread->avg_sleeping += last_duration;
+       if (this_thread->state == CTDL_THREAD_RUNNING)
+               this_thread->avg_running += last_duration;
+       if (this_thread->state == CTDL_THREAD_BLOCKED)
+               this_thread->avg_blocked += last_duration;
+       memcpy (&this_thread->last_state_change, &now, sizeof (struct timeval));
+       pthread_mutex_unlock(&this_thread->ThreadMutex);
 }
 
-
 /*
- * masterCC is the context we use when not attached to a session.  This
- * function initializes it.
+ * A function to chenge the state of a thread
  */
-void InitializeMasterCC(void) {
-       memset(&masterCC, 0, sizeof(struct CitContext));
-       masterCC.internal_pgm = 1;
-       masterCC.cs_pid = 0;
+void ctdl_thread_internal_change_state (struct CtdlThreadNode *this_thread, enum CtdlThreadState new_state)
+{
+       /*
+        * Wether we change state or not we need update the load values
+        */
+       ctdl_thread_internal_update_avgs(this_thread);
+       pthread_mutex_lock(&this_thread->ThreadMutex); /* To prevent race condition of a sleeping thread */
+       if ((new_state == CTDL_THREAD_STOP_REQ) && (this_thread->state > CTDL_THREAD_STOP_REQ))
+               this_thread->state = new_state;
+       if (((new_state == CTDL_THREAD_SLEEPING) || (new_state == CTDL_THREAD_BLOCKED)) && (this_thread->state == CTDL_THREAD_RUNNING))
+               this_thread->state = new_state;
+       if ((new_state == CTDL_THREAD_RUNNING) && ((this_thread->state == CTDL_THREAD_SLEEPING) || (this_thread->state == CTDL_THREAD_BLOCKED)))
+               this_thread->state = new_state;
+       pthread_mutex_unlock(&this_thread->ThreadMutex);
 }
 
 
+/*
+ * A function to tell all threads to exit
+ */
+void CtdlThreadStopAll(void)
+{
+       //FIXME: The signalling of the condition should not be in the critical_section
+       // We need to build a list of threads we are going to signal and then signal them afterwards
+       
+       struct CtdlThreadNode *this_thread;
+       
+       begin_critical_section(S_THREAD_LIST);
+       this_thread = CtdlThreadList;
+       while(this_thread)
+       {
+#ifdef THREADS_USESIGNALS
+               pthread_kill(this_thread->tid, SIGHUP);
+#endif
+               ctdl_thread_internal_change_state (this_thread, CTDL_THREAD_STOP_REQ);
+               pthread_cond_signal(&this_thread->ThreadCond);
+               pthread_cond_signal(&this_thread->SleepCond);
+               CtdlLogPrintf(CTDL_DEBUG, "Thread system stopping thread \"%s\" (%ld).\n", this_thread->name, this_thread->tid);
+               this_thread = this_thread->next;
+       }
+       end_critical_section(S_THREAD_LIST);
+}
+
+
+/*
+ * A function to wake up all sleeping threads
+ */
+void CtdlThreadWakeAll(void)
+{
+       struct CtdlThreadNode *this_thread;
+       
+       CtdlLogPrintf(CTDL_DEBUG, "Thread system waking all threads.\n");
+       
+       begin_critical_section(S_THREAD_LIST);
+       this_thread = CtdlThreadList;
+       while(this_thread)
+       {
+               if (!this_thread->thread_func)
+               {
+                       pthread_cond_signal(&this_thread->ThreadCond);
+                       pthread_cond_signal(&this_thread->SleepCond);
+               }
+               this_thread = this_thread->next;
+       }
+       end_critical_section(S_THREAD_LIST);
+}
+
+
+/*
+ * A function to return the number of threads running in the system
+ */
+int CtdlThreadGetCount(void)
+{
+       return  num_threads;
+}
+
+int CtdlThreadGetWorkers(void)
+{
+       return  num_workers;
+}
+
+double CtdlThreadGetWorkerAvg(void)
+{
+       double ret;
+       
+       begin_critical_section(S_THREAD_LIST);
+       ret =  CtdlThreadWorkerAvg;
+       end_critical_section(S_THREAD_LIST);
+       return ret;
+}
+
+double CtdlThreadGetLoadAvg(void)
+{
+       double ret;
+       
+       begin_critical_section(S_THREAD_LIST);
+       ret =  CtdlThreadLoadAvg;
+       end_critical_section(S_THREAD_LIST);
+       return ret;
+}
+
+
+
+
+/*
+ * A function to rename a thread
+ * Returns a const char *
+ */
+const char *CtdlThreadName(const char *name)
+{
+       const char *old_name;
+       
+       if (!CT)
+       {
+               CtdlLogPrintf(CTDL_WARNING, "Thread system WARNING. Attempt to CtdlThreadRename() a non thread. %s\n", name);
+               return NULL;
+       }
+// FIXME: do we need this lock? I think not since the pointer asignmaent should be atomic
+       pthread_mutex_lock(&CT->ThreadMutex);
+       old_name = CT->name;
+       if (name)
+               CT->name = name;
+       pthread_mutex_unlock(&CT->ThreadMutex);
+       return (old_name);
+}      
+
+
+/*
+ * A function to force a thread to exit
+ */
+void CtdlThreadCancel(struct CtdlThreadNode *thread)
+{
+       struct CtdlThreadNode *this_thread;
+       
+       if (!thread)
+               this_thread = CT;
+       else
+               this_thread = thread;
+       if (!this_thread)
+       {
+               CtdlLogPrintf(CTDL_EMERG, "Thread system PANIC. Attempt to CtdlThreadCancel() a non thread.\n");
+               CtdlThreadStopAll();
+               return;
+       }
+       
+       if (!this_thread->thread_func)
+       {
+               CtdlLogPrintf(CTDL_EMERG, "Thread system PANIC. Attempt to CtdlThreadCancel() the garbage collector.\n");
+               CtdlThreadStopAll();
+               return;
+       }
+       
+       ctdl_thread_internal_change_state (this_thread, CTDL_THREAD_CANCELLED);
+       pthread_cancel(this_thread->tid);
+}
+
+
+
+/*
+ * A function for a thread to check if it has been asked to stop
+ */
+int CtdlThreadCheckStop(void)
+{
+       int state;
+       
+       if (!CT)
+       {
+               CtdlLogPrintf(CTDL_EMERG, "Thread system PANIC, CtdlThreadCheckStop() called by a non thread.\n");
+               CtdlThreadStopAll();
+               return -1;
+       }
+       
+       state = CT->state;
+
+#ifdef THREADS_USERSIGNALS
+       if (CT->signal)
+               CtdlLogPrintf(CTDL_DEBUG, "Thread \"%s\" caught signal %d.\n", CT->name, CT->signal);
+#endif
+       pthread_mutex_lock(&CT->ThreadMutex);
+       if(state == CTDL_THREAD_STOP_REQ)
+       {
+               CT->state = CTDL_THREAD_STOPPING;
+               pthread_mutex_unlock(&CT->ThreadMutex);
+               return -1;
+       }
+       else if((state < CTDL_THREAD_STOP_REQ) && (state > CTDL_THREAD_CREATE))
+       {
+               pthread_mutex_unlock(&CT->ThreadMutex);
+               return -1;
+       }
+       pthread_mutex_unlock(&CT->ThreadMutex);
+       return 0;
+}
+
+
+/*
+ * A function to ask a thread to exit
+ * The thread must call CtdlThreadCheckStop() periodically to determine if it should exit
+ */
+void CtdlThreadStop(struct CtdlThreadNode *thread)
+{
+       struct CtdlThreadNode *this_thread;
+       
+       if (!thread)
+               this_thread = CT;
+       else
+               this_thread = thread;
+       if (!this_thread)
+               return;
+       if (!(this_thread->thread_func))
+               return;         // Don't stop garbage collector
+#ifdef THREADS_USESIGNALS
+       pthread_kill(this_thread->tid, SIGHUP); 
+#endif
+       ctdl_thread_internal_change_state (this_thread, CTDL_THREAD_STOP_REQ);
+       pthread_cond_signal(&this_thread->ThreadCond);
+       pthread_cond_signal(&this_thread->SleepCond);
+}
+
+/*
+ * So we now have a sleep command that works with threads but it is in seconds
+ */
+void CtdlThreadSleep(int secs)
+{
+       struct timespec wake_time;
+       struct timeval time_now;
+       
+       
+       if (!CT)
+       {
+               CtdlLogPrintf(CTDL_WARNING, "CtdlThreadSleep() called by something that is not a thread. Should we die?\n");
+               return;
+       }
+       
+       memset (&wake_time, 0, sizeof(struct timespec));
+       gettimeofday(&time_now, NULL);
+       wake_time.tv_sec = time_now.tv_sec + secs;
+       wake_time.tv_nsec = time_now.tv_usec * 10;
+
+       ctdl_thread_internal_change_state (CT, CTDL_THREAD_SLEEPING);
+       
+       pthread_mutex_lock(&CT->ThreadMutex); /* Prevent something asking us to awaken before we've gone to sleep */
+       pthread_cond_timedwait(&CT->SleepCond, &CT->ThreadMutex, &wake_time);
+       pthread_mutex_unlock(&CT->ThreadMutex);
+       
+       ctdl_thread_internal_change_state (CT, CTDL_THREAD_RUNNING);
+}
+
+
+/*
+ * Routine to clean up our thread function on exit
+ */
+static void ctdl_internal_thread_cleanup(void *arg)
+{
+       /*
+        * In here we were called by the current thread because it is exiting
+        * NB. WE ARE THE CURRENT THREAD
+        */
+       CtdlLogPrintf(CTDL_NOTICE, "Thread \"%s\" (%ld) exited.\n", CT->name, CT->tid);
+       
+       #ifdef HAVE_BACKTRACE
+       eCrash_UnregisterThread();
+       #endif
+       
+       pthread_mutex_lock(&CT->ThreadMutex);
+       CT->state = CTDL_THREAD_EXITED; // needs to be last thing else house keeping will unlink us too early
+       pthread_mutex_unlock(&CT->ThreadMutex);
+}
+
+/*
+ * A quick function to show the load averages
+ */
+void ctdl_thread_internal_calc_loadavg(void)
+{
+       struct CtdlThreadNode *that_thread;
+       double load_avg, worker_avg;
+       int workers = 0;
+
+       that_thread = CtdlThreadList;
+       load_avg = 0;
+       worker_avg = 0;
+       while(that_thread)
+       {
+               /* Update load averages */
+               ctdl_thread_internal_update_avgs(that_thread);
+               pthread_mutex_lock(&that_thread->ThreadMutex);
+               that_thread->load_avg = that_thread->avg_sleeping + that_thread->avg_running + that_thread->avg_blocked;
+               that_thread->load_avg = that_thread->avg_running / that_thread->load_avg * 100;
+               that_thread->avg_sleeping /= 2;
+               that_thread->avg_running /= 2;
+               that_thread->avg_blocked /= 2;
+               load_avg += that_thread->load_avg;
+               if (that_thread->flags & CTDLTHREAD_WORKER)
+               {
+                       worker_avg += that_thread->load_avg;
+                       workers++;
+               }
+#ifdef WITH_THREADLOG
+               CtdlLogPrintf(CTDL_DEBUG, "CtdlThread, \"%s\" (%ld) \"%s\" %f %f %f %f.\n",
+                       that_thread->name,
+                       that_thread->tid,
+                       CtdlThreadStates[that_thread->state],
+                       that_thread->avg_sleeping,
+                       that_thread->avg_running,
+                       that_thread->avg_blocked,
+                       that_thread->load_avg);
+#endif
+               pthread_mutex_unlock(&that_thread->ThreadMutex);
+               that_thread = that_thread->next;
+       }
+       CtdlThreadLoadAvg = load_avg/num_threads;
+       CtdlThreadWorkerAvg = worker_avg/workers;
+#ifdef WITH_THREADLOG
+       CtdlLogPrintf(CTDL_INFO, "System load average %f, workers averag %f, threads %d, workers %d, sessions %d\n", CtdlThreadLoadAvg, CtdlThreadWorkerAvg, num_threads, num_workers, num_sessions);
+#endif
+}
+
+
+/*
+ * Garbage collection routine.
+ * Gets called by main() in a loop to clean up the thread list periodically.
+ */
+void CtdlThreadGC (void)
+{
+       struct CtdlThreadNode *this_thread, *that_thread;
+       int workers = 0;
+       int ret=0;
+       
+       begin_critical_section(S_THREAD_LIST);
+       
+       /* Handle exiting of garbage collector thread */
+       if(num_threads == 1)
+               CtdlThreadList->state = CTDL_THREAD_EXITED;
+       
+#ifdef WITH_THREADLOG
+       CtdlLogPrintf(CTDL_DEBUG, "Thread system running garbage collection.\n");
+#endif
+       /*
+        * Woke up to do garbage collection
+        */
+       this_thread = CtdlThreadList;
+       while(this_thread)
+       {
+               that_thread = this_thread;
+               this_thread = this_thread->next;
+               
+               /* Do we need to clean up this thread? */
+               pthread_mutex_lock(&that_thread->ThreadMutex);
+               if (that_thread->state != CTDL_THREAD_EXITED)
+               {
+                       if(that_thread->flags & CTDLTHREAD_WORKER)
+                               workers++;      /* Sanity check on number of worker threads */
+                       pthread_mutex_unlock(&that_thread->ThreadMutex);
+                       continue;
+               }
+               
+               if (pthread_equal(that_thread->tid, pthread_self()) && that_thread->thread_func)
+               {       /* Sanity check */
+                       pthread_mutex_unlock(&that_thread->ThreadMutex);
+                       end_critical_section(S_THREAD_LIST);
+                       CtdlLogPrintf(CTDL_EMERG, "Thread system PANIC, a thread is trying to clean up after itself.\n");
+                       abort();
+                       return;
+               }
+               
+               if (num_threads <= 0)
+               {       /* Sanity check */
+                       pthread_mutex_unlock(&that_thread->ThreadMutex);
+                       end_critical_section(S_THREAD_LIST);
+                       CtdlLogPrintf(CTDL_EMERG, "Thread system PANIC, num_threads <= 0 and trying to do Garbage Collection.\n");
+                       abort();
+                       return;
+               }
+
+               if(that_thread->flags & CTDLTHREAD_WORKER)
+                       num_workers--;  /* This is a wroker thread so reduce the count. */
+               num_threads--;
+               /* If we are unlinking the list head then the next becomes the list head */
+               if(that_thread->prev)
+                       that_thread->prev->next = that_thread->next;
+               else
+                       CtdlThreadList = that_thread->next;
+               if(that_thread->next)
+                       that_thread->next->prev = that_thread->prev;
+               
+               pthread_mutex_unlock(&that_thread->ThreadMutex);
+               pthread_cond_signal(&that_thread->ThreadCond);
+               pthread_cond_signal(&that_thread->SleepCond);   // Make sure this thread is awake
+               pthread_mutex_lock(&that_thread->ThreadMutex);  // Make sure it has done what its doing
+               pthread_mutex_unlock(&that_thread->ThreadMutex);
+               /*
+                * Join on the thread to do clean up and prevent memory leaks
+                * Also makes sure the thread has cleaned up after itself before we remove it from the list
+                * We can join on the garbage collector thread the join should just return EDEADLCK
+                */
+               ret = pthread_join (that_thread->tid, NULL);
+               if (ret == EDEADLK)
+                       CtdlLogPrintf(CTDL_DEBUG, "Garbage collection on own thread.\n");
+               else if (ret == EINVAL)
+                       CtdlLogPrintf(CTDL_DEBUG, "Garbage collection, that thread already joined on.\n");
+               else if (ret == ESRCH)
+                       CtdlLogPrintf(CTDL_DEBUG, "Garbage collection, no thread to join on.\n");
+               else if (ret != 0)
+                       CtdlLogPrintf(CTDL_DEBUG, "Garbage collection, pthread_join returned an unknown error.\n");
+               /*
+                * Now we own that thread entry
+                */
+               CtdlLogPrintf(CTDL_INFO, "Garbage Collection for thread \"%s\" (%ld).\n", that_thread->name, that_thread->tid);
+               pthread_mutex_destroy(&that_thread->ThreadMutex);
+               pthread_cond_destroy(&that_thread->ThreadCond);
+               pthread_mutex_destroy(&that_thread->SleepMutex);
+               pthread_cond_destroy(&that_thread->SleepCond);
+               pthread_attr_destroy(&that_thread->attr);
+               free(that_thread);
+       }
+       
+       /* Sanity check number of worker threads */
+       if (workers != num_workers)
+       {
+               end_critical_section(S_THREAD_LIST);
+               CtdlLogPrintf(CTDL_EMERG,
+                       "Thread system PANIC, discrepancy in number of worker threads. Counted %d, should be %d.\n",
+                       workers, num_workers
+                       );
+               abort();
+       }
+       end_critical_section(S_THREAD_LIST);
+}
+
+
+
+/*
+ * Runtime function for a Citadel Thread.
+ * This initialises the threads environment and then calls the user supplied thread function
+ * Note that this is the REAL thread function and wraps the users thread function.
+ */ 
+static void *ctdl_internal_thread_func (void *arg)
+{
+       struct CtdlThreadNode *this_thread;
+       void *ret = NULL;
+
+       /* lock and unlock the thread list.
+        * This causes this thread to wait until all its creation stuff has finished before it
+        * can continue its execution.
+        */
+       begin_critical_section(S_THREAD_LIST);
+       this_thread = (struct CtdlThreadNode *) arg;
+       gettimeofday(&this_thread->start_time, NULL);           /* Time this thread started */
+       pthread_mutex_lock(&this_thread->ThreadMutex);
+       
+       // Register the cleanup function to take care of when we exit.
+       pthread_cleanup_push(ctdl_internal_thread_cleanup, NULL);
+       // Get our thread data structure
+       CtdlThreadAllocTSD();
+       CT = this_thread;
+       this_thread->pid = getpid();
+       memcpy(&this_thread->last_state_change, &this_thread->start_time, sizeof (struct timeval));     /* Changed state so mark it. */
+       /* Only change to running state if we weren't asked to stop during the create cycle
+        * Other wise there is a window to allow this threads creation to continue to full grown and
+        * therby prevent a shutdown of the server.
+        */
+       pthread_mutex_unlock(&this_thread->ThreadMutex);
+               
+       if (!CtdlThreadCheckStop())
+       {
+               pthread_mutex_lock(&this_thread->ThreadMutex);
+               this_thread->state = CTDL_THREAD_RUNNING;
+               pthread_mutex_unlock(&this_thread->ThreadMutex);
+       }
+       end_critical_section(S_THREAD_LIST);
+       
+       // Register for tracing
+       #ifdef HAVE_BACKTRACE
+       eCrash_RegisterThread(this_thread->name, 0);
+       #endif
+       
+       // Tell the world we are here
+       CtdlLogPrintf(CTDL_NOTICE, "Created a new thread \"%s\" (%ld). \n", this_thread->name, this_thread->tid);
+
+       
+       
+       /*
+        * run the thread to do the work but only if we haven't been asked to stop
+        */
+       if (!CtdlThreadCheckStop())
+               ret = (this_thread->thread_func)(this_thread->user_args);
+       
+       /*
+        * Our thread is exiting either because it wanted to end or because the server is stopping
+        * We need to clean up
+        */
+       pthread_cleanup_pop(1); // Execute our cleanup routine and remove it
+       
+       return(ret);
+}
+
+
+/*
+ * Internal function to create a thread.
+ * Must be called from within a S_THREAD_LIST critical section
+ */ 
+struct CtdlThreadNode *ctdl_internal_create_thread(char *name, long flags, void *(*thread_func) (void *arg), void *args)
+{
+       int ret = 0;
+       struct CtdlThreadNode *this_thread;
+
+       if (num_threads >= 32767)
+       {
+               CtdlLogPrintf(CTDL_EMERG, "Thread system. Thread list full.\n");
+               return NULL;
+       }
+               
+       this_thread = malloc(sizeof(struct CtdlThreadNode));
+       if (this_thread == NULL) {
+               CtdlLogPrintf(CTDL_EMERG, "Thread system, can't allocate CtdlThreadNode, exiting\n");
+               return NULL;
+       }
+       // Ensuring this is zero'd means we make sure the thread doesn't start doing its thing until we are ready.
+       memset (this_thread, 0, sizeof(struct CtdlThreadNode));
+       
+       /* Create the mutex's early so we can use them */
+       pthread_mutex_init (&(this_thread->ThreadMutex), NULL);
+       pthread_cond_init (&(this_thread->ThreadCond), NULL);
+       pthread_mutex_init (&(this_thread->SleepMutex), NULL);
+       pthread_cond_init (&(this_thread->SleepCond), NULL);
+       
+       pthread_mutex_lock(&this_thread->ThreadMutex);
+       
+       this_thread->state = CTDL_THREAD_CREATE;
+       
+       if ((ret = pthread_attr_init(&this_thread->attr))) {
+               pthread_mutex_unlock(&this_thread->ThreadMutex);
+               pthread_mutex_destroy(&(this_thread->ThreadMutex));
+               pthread_cond_destroy(&(this_thread->ThreadCond));
+               pthread_mutex_destroy(&(this_thread->SleepMutex));
+               pthread_cond_destroy(&(this_thread->SleepCond));
+               CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_init: %s\n", strerror(ret));
+               free(this_thread);
+               return NULL;
+       }
+
+       /* Our per-thread stacks need to be bigger than the default size,
+        * otherwise the MIME parser crashes on FreeBSD, and the IMAP service
+        * crashes on 64-bit Linux.
+        */
+       if (flags & CTDLTHREAD_BIGSTACK)
+       {
+#ifdef WITH_THREADLOG
+               CtdlLogPrintf(CTDL_INFO, "Thread system. Creating BIG STACK thread.\n");
+#endif
+               if ((ret = pthread_attr_setstacksize(&this_thread->attr, THREADSTACKSIZE))) {
+                       pthread_mutex_unlock(&this_thread->ThreadMutex);
+                       pthread_mutex_destroy(&(this_thread->ThreadMutex));
+                       pthread_cond_destroy(&(this_thread->ThreadCond));
+                       pthread_mutex_destroy(&(this_thread->SleepMutex));
+                       pthread_cond_destroy(&(this_thread->SleepCond));
+                       pthread_attr_destroy(&this_thread->attr);
+                       CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_setstacksize: %s\n",
+                               strerror(ret));
+                       free(this_thread);
+                       return NULL;
+               }
+       }
+
+       /*
+        * If we got here we are going to create the thread so we must initilise the structure
+        * first because most implimentations of threading can't create it in a stopped state
+        * and it might want to do things with its structure that aren't initialised otherwise.
+        */
+       if(name)
+       {
+               this_thread->name = name;
+       }
+       else
+       {
+               this_thread->name = "Un-named Thread";
+       }
+       
+       this_thread->flags = flags;
+       this_thread->thread_func = thread_func;
+       this_thread->user_args = args;
+       /* Set this new thread with an avg_blocked of 2. We do this so that its creation affects the
+        * load average for the system. If we don't do this then we create a mass of threads at the same time 
+        * because the creation didn't affect the load average.
+        */
+       this_thread->avg_blocked = 2;
+       
+       /*
+        * We pass this_thread into the thread as its args so that it can find out information
+        * about itself and it has a bit of storage space for itself, not to mention that the REAL
+        * thread function needs to finish off the setup of the structure
+        */
+       if ((ret = pthread_create(&this_thread->tid, &this_thread->attr, ctdl_internal_thread_func, this_thread) != 0))
+       {
+
+               CtdlLogPrintf(CTDL_ALERT, "Thread system, Can't create thread: %s\n",
+                       strerror(ret));
+               pthread_mutex_unlock(&this_thread->ThreadMutex);
+               pthread_mutex_destroy(&(this_thread->ThreadMutex));
+               pthread_cond_destroy(&(this_thread->ThreadCond));
+               pthread_mutex_destroy(&(this_thread->SleepMutex));
+               pthread_cond_destroy(&(this_thread->SleepCond));
+               pthread_attr_destroy(&this_thread->attr);
+               free(this_thread);
+               return NULL;
+       }
+       
+       num_threads++;  // Increase the count of threads in the system.
+       if(this_thread->flags & CTDLTHREAD_WORKER)
+               num_workers++;
+
+       this_thread->next = CtdlThreadList;
+       CtdlThreadList = this_thread;
+       if (this_thread->next)
+               this_thread->next->prev = this_thread;
+       
+       pthread_mutex_unlock(&this_thread->ThreadMutex);
+       
+       ctdl_thread_internal_calc_loadavg();
+       return this_thread;
+}
+
+/*
+ * Wrapper function to create a thread
+ * ensures the critical section and other protections are in place.
+ * char *name = name to give to thread, if NULL, use generic name
+ * int flags = flags to determine type of thread and standard facilities
+ */
+struct CtdlThreadNode *CtdlThreadCreate(char *name, long flags, void *(*thread_func) (void *arg), void *args)
+{
+       struct CtdlThreadNode *ret = NULL;
+       
+       begin_critical_section(S_THREAD_LIST);
+       ret = ctdl_internal_create_thread(name, flags, thread_func, args);
+       end_critical_section(S_THREAD_LIST);
+       return ret;
+}
+
+
+
+/*
+ * Internal function to schedule a thread.
+ * Must be called from within a S_THREAD_LIST critical section
+ */ 
+struct CtdlThreadNode *CtdlThreadSchedule(char *name, long flags, void *(*thread_func) (void *arg), void *args, time_t when)
+{
+       int ret = 0;
+       struct CtdlThreadNode *this_thread;
+
+       if (num_threads >= 32767)
+       {
+               CtdlLogPrintf(CTDL_EMERG, "Thread system. Thread list full.\n");
+               return NULL;
+       }
+               
+       this_thread = malloc(sizeof(struct CtdlThreadNode));
+       if (this_thread == NULL) {
+               CtdlLogPrintf(CTDL_EMERG, "Thread system, can't allocate CtdlThreadNode, exiting\n");
+               return NULL;
+       }
+       // Ensuring this is zero'd means we make sure the thread doesn't start doing its thing until we are ready.
+       memset (this_thread, 0, sizeof(struct CtdlThreadNode));
+       
+       /* Create the mutex's early so we can use them */
+       pthread_mutex_init (&(this_thread->ThreadMutex), NULL);
+       pthread_cond_init (&(this_thread->ThreadCond), NULL);
+       pthread_mutex_init (&(this_thread->SleepMutex), NULL);
+       pthread_cond_init (&(this_thread->SleepCond), NULL);
+       
+       this_thread->state = CTDL_THREAD_CREATE;
+       
+       if ((ret = pthread_attr_init(&this_thread->attr))) {
+               pthread_mutex_destroy(&(this_thread->ThreadMutex));
+               pthread_cond_destroy(&(this_thread->ThreadCond));
+               pthread_mutex_destroy(&(this_thread->SleepMutex));
+               pthread_cond_destroy(&(this_thread->SleepCond));
+               CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_init: %s\n", strerror(ret));
+               free(this_thread);
+               return NULL;
+       }
+
+       /* Our per-thread stacks need to be bigger than the default size,
+        * otherwise the MIME parser crashes on FreeBSD, and the IMAP service
+        * crashes on 64-bit Linux.
+        */
+       if (flags & CTDLTHREAD_BIGSTACK)
+       {
+               CtdlLogPrintf(CTDL_INFO, "Thread system. Creating BIG STACK thread.\n");
+               if ((ret = pthread_attr_setstacksize(&this_thread->attr, THREADSTACKSIZE))) {
+                       pthread_mutex_destroy(&(this_thread->ThreadMutex));
+                       pthread_cond_destroy(&(this_thread->ThreadCond));
+                       pthread_mutex_destroy(&(this_thread->SleepMutex));
+                       pthread_cond_destroy(&(this_thread->SleepCond));
+                       pthread_attr_destroy(&this_thread->attr);
+                       CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_setstacksize: %s\n",
+                               strerror(ret));
+                       free(this_thread);
+                       return NULL;
+               }
+       }
+
+       /*
+        * If we got here we are going to create the thread so we must initilise the structure
+        * first because most implimentations of threading can't create it in a stopped state
+        * and it might want to do things with its structure that aren't initialised otherwise.
+        */
+       if(name)
+       {
+               this_thread->name = name;
+       }
+       else
+       {
+               this_thread->name = "Un-named Thread";
+       }
+       
+       this_thread->flags = flags;
+       this_thread->thread_func = thread_func;
+       this_thread->user_args = args;
+       /* Set this new thread with an avg_blocked of 2. We do this so that its creation affects the
+        * load average for the system. If we don't do this then we create a mass of threads at the same time 
+        * because the creation didn't affect the load average.
+        */
+       this_thread->avg_blocked = 2;
+       
+       /*
+        * When to start this thread
+        */
+       this_thread->when = when;
+
+       begin_critical_section(S_SCHEDULE_LIST);
+       this_thread->next = CtdlThreadSchedList;
+       CtdlThreadSchedList = this_thread;
+       if (this_thread->next)
+               this_thread->next->prev = this_thread;
+       end_critical_section(S_SCHEDULE_LIST);
+       
+       return this_thread;
+}
+
+
+
+struct CtdlThreadNode *ctdl_thread_internal_start_scheduled (struct CtdlThreadNode *this_thread)
+{
+       int ret = 0;
+       
+       /*
+        * We pass this_thread into the thread as its args so that it can find out information
+        * about itself and it has a bit of storage space for itself, not to mention that the REAL
+        * thread function needs to finish off the setup of the structure
+        */
+       if ((ret = pthread_create(&this_thread->tid, &this_thread->attr, ctdl_internal_thread_func, this_thread) != 0))
+       {
+
+               CtdlLogPrintf(CTDL_ALERT, "Thread system, Can't create thread: %s\n",
+                       strerror(ret));
+               return NULL;
+       }
+       
+       
+       num_threads++;  // Increase the count of threads in the system.
+       if(this_thread->flags & CTDLTHREAD_WORKER)
+               num_workers++;
+
+       this_thread->next = CtdlThreadList;
+       CtdlThreadList = this_thread;
+       if (this_thread->next)
+               this_thread->next->prev = this_thread;
+       
+       return this_thread;
+}
+
+
+
+void ctdl_thread_internal_check_scheduled(void)
+{
+       struct CtdlThreadNode *this_thread, *that_thread;
+       time_t now;
+       
+       if (try_critical_section(S_SCHEDULE_LIST))
+               return; /* If this list is locked we wait till the next chance */
+       
+       now = time(NULL);
+       
+#ifdef WITH_THREADLOG
+       CtdlLogPrintf(CTDL_DEBUG, "Checking for scheduled threads to start.\n");
+#endif
+
+       this_thread = CtdlThreadSchedList;
+       while(this_thread)
+       {
+               that_thread = this_thread;
+               this_thread = this_thread->next;
+               
+               if (now > that_thread->when)
+               {
+                       /* Unlink from schedule list */
+                       if (that_thread->prev)
+                               that_thread->prev->next = that_thread->next;
+                       else
+                               CtdlThreadSchedList = that_thread->next;
+                       if (that_thread->next)
+                               that_thread->next->prev = that_thread->prev;
+                               
+                       that_thread->next = that_thread->prev = NULL;
+#ifdef WITH_THREADLOG
+                       CtdlLogPrintf(CTDL_DEBUG, "About to start scheduled thread \"%s\".\n", that_thread->name);
+#endif
+                       begin_critical_section(S_THREAD_LIST);
+                       if (CT->state > CTDL_THREAD_STOP_REQ)
+                       {       /* Only start it if the system is not stopping */
+                               pthread_mutex_lock(&that_thread->ThreadMutex);
+                               if (ctdl_thread_internal_start_scheduled (that_thread) == NULL)
+                               {
+#ifdef WITH_THREADLOG
+                       CtdlLogPrintf(CTDL_DEBUG, "Failed to start scheduled thread \"%s\".\n", that_thread->name);
+#endif
+                                       pthread_mutex_unlock(&that_thread->ThreadMutex);
+                                       pthread_mutex_destroy(&(that_thread->ThreadMutex));
+                                       pthread_cond_destroy(&(that_thread->ThreadCond));
+                                       pthread_mutex_destroy(&(that_thread->SleepMutex));
+                                       pthread_cond_destroy(&(that_thread->SleepCond));
+                                       pthread_attr_destroy(&that_thread->attr);
+                                       free(that_thread);
+                               }
+                               else
+                               {
+                                       CtdlLogPrintf(CTDL_INFO, "Thread system, Started a scheduled thread \"%s\" (%ld).\n",
+                                               that_thread->name, that_thread->tid);
+                                       pthread_mutex_unlock(&that_thread->ThreadMutex);
+                                       ctdl_thread_internal_calc_loadavg();
+                               }
+                       }
+                       end_critical_section(S_THREAD_LIST);
+               }
+               else
+               {
+#ifdef WITH_THREADLOG
+                       CtdlLogPrintf(CTDL_DEBUG, "Thread \"%s\" will start in %ld seconds.\n", that_thread->name, that_thread->when - time(NULL));
+#endif
+               }
+       }
+       end_critical_section(S_SCHEDULE_LIST);
+}
+
+
+/*
+ * A warapper function for select so we can show a thread as blocked
+ */
+int CtdlThreadSelect(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout)
+{
+       int ret;
+       
+       ctdl_thread_internal_change_state(CT, CTDL_THREAD_BLOCKED);
+       ret = select(n, readfds, writefds, exceptfds, timeout);
+       ctdl_thread_internal_change_state(CT, CTDL_THREAD_RUNNING);
+       return ret;
+}
+
+/*
+ * Purge all sessions which have the 'kill_me' flag set.
+ * This function has code to prevent it from running more than once every
+ * few seconds, because running it after every single unbind would waste a lot
+ * of CPU time and keep the context list locked too much.  To force it to run
+ * anyway, set "force" to nonzero.
+ */
+void dead_session_purge(int force) {
+       struct CitContext *ptr, *ptr2;          /* general-purpose utility pointer */
+       struct CitContext *rem = NULL;  /* list of sessions to be destroyed */
+       
+       if (force == 0) {
+               if ( (time(NULL) - last_purge) < 5 ) {
+                       return; /* Too soon, go away */
+               }
+       }
+       time(&last_purge);
+
+       if (try_critical_section(S_SESSION_TABLE))
+               return;
+               
+       ptr = ContextList;
+       while (ptr) {
+               ptr2 = ptr;
+               ptr = ptr->next;
+               
+               if ( (ptr2->state == CON_IDLE) && (ptr2->kill_me) ) {
+                       /* Remove the session from the active list */
+                       if (ptr2->prev) {
+                               ptr2->prev->next = ptr2->next;
+                       }
+                       else {
+                               ContextList = ptr2->next;
+                       }
+                       if (ptr2->next) {
+                               ptr2->next->prev = ptr2->prev;
+                       }
+
+                       --num_sessions;
+                       /* And put it on our to-be-destroyed list */
+                       ptr2->next = rem;
+                       rem = ptr2;
+               }
+       }
+       end_critical_section(S_SESSION_TABLE);
+
+       /* Now that we no longer have the session list locked, we can take
+        * our time and destroy any sessions on the to-be-killed list, which
+        * is allocated privately on this thread's stack.
+        */
+       while (rem != NULL) {
+               CtdlLogPrintf(CTDL_DEBUG, "Purging session %d\n", rem->cs_pid);
+               RemoveContext(rem);
+               ptr = rem;
+               rem = rem->next;
+               free(ptr);
+       }
+}
+
+
+
+
+
+/*
+ * masterCC is the context we use when not attached to a session.  This
+ * function initializes it.
+ */
+void InitializeMasterCC(void) {
+       memset(&masterCC, 0, sizeof(struct CitContext));
+       masterCC.internal_pgm = 1;
+       masterCC.cs_pid = 0;
+}
+
+
+
+
+
 
-
-
-
 /*
  * Bind a thread to a context.  (It's inline merely to speed things up.)
  */
@@ -893,12 +2169,9 @@ void *worker_thread(void *arg) {
        struct timeval tv;
        int force_purge = 0;
        int m;
+       
 
-       num_threads++;
-
-       cdb_allocate_tsd();
-
-       while (!time_to_die) {
+       while (!CtdlThreadCheckStop()) {
 
                /* make doubly sure we're not holding any stale db handles
                 * which might cause a deadlock.
@@ -925,7 +2198,9 @@ do_select: force_purge = 0;
                }
                end_critical_section(S_SESSION_TABLE);
 
-               if (bind_me) goto SKIP_SELECT;
+               if (bind_me) {
+                       goto SKIP_SELECT;
+               }
 
                /* If we got this far, it means that there are no sessions
                 * which a previous thread marked for attention, so we go
@@ -942,30 +2217,34 @@ do_select:       force_purge = 0;
                        }
                }
 
-               if (!time_to_die) {
+               if (!CtdlThreadCheckStop()) {
                        tv.tv_sec = 1;          /* wake up every second if no input */
                        tv.tv_usec = 0;
-                       retval = select(highest + 1, &readfds, NULL, NULL, &tv);
+                       retval = CtdlThreadSelect(highest + 1, &readfds, NULL, NULL, &tv);
                }
 
-               if (time_to_die) return(NULL);
+               if (CtdlThreadCheckStop()) return(NULL);
 
                /* Now figure out who made this select() unblock.
                 * First, check for an error or exit condition.
                 */
                if (retval < 0) {
                        if (errno == EBADF) {
-                               lprintf(CTDL_NOTICE, "select() failed: (%s)\n",
+                               CtdlLogPrintf(CTDL_NOTICE, "select() failed: (%s)\n",
                                        strerror(errno));
                                goto do_select;
                        }
                        if (errno != EINTR) {
-                               lprintf(CTDL_EMERG, "Exiting (%s)\n", strerror(errno));
-                               time_to_die = 1;
-                       } else if (!time_to_die)
+                               CtdlLogPrintf(CTDL_EMERG, "Exiting (%s)\n", strerror(errno));
+                               CtdlThreadStopAll();
+                       } else if (!CtdlThreadCheckStop()) {
+                               CtdlLogPrintf(CTDL_DEBUG, "Un handled select failure.\n");
                                goto do_select;
+                       }
+               }
+               else if(retval == 0) {
+                       goto SKIP_SELECT;
                }
-
                /* Next, check to see if it's a new client connecting
                 * on a master socket.
                 */
@@ -974,29 +2253,36 @@ do_select:       force_purge = 0;
 
                        if (FD_ISSET(serviceptr->msock, &readfds)) {
                                ssock = accept(serviceptr->msock, NULL, 0);
-                               if (ssock < 0) {
-                                       lprintf(CTDL_CRIT,
-                                               "citserver: accept(): %s\n",
-                                               strerror(errno));
-                               }
-                               else {
-                                       lprintf(CTDL_DEBUG,
+                               if (ssock >= 0) {
+                                       CtdlLogPrintf(CTDL_DEBUG,
                                                "New client socket %d\n",
                                                ssock);
 
+                                       /* The master socket is non-blocking but the client
+                                        * sockets need to be blocking, otherwise certain
+                                        * operations barf on FreeBSD.  Not a fatal error.
+                                        */
+                                       if (fcntl(ssock, F_SETFL, 0) < 0) {
+                                               CtdlLogPrintf(CTDL_EMERG,
+                                                       "citserver: Can't set socket to blocking: %s\n",
+                                                       strerror(errno));
+                                       }
+
                                        /* New context will be created already
-                                       * set up in the CON_EXECUTING state.
-                                       */
+                                        * set up in the CON_EXECUTING state.
+                                        */
                                        con = CreateNewContext();
 
-                                       /* Assign new socket number to it. */
+                                       /* Assign our new socket number to it. */
                                        con->client_socket = ssock;
                                        con->h_command_function =
                                                serviceptr->h_command_function;
                                        con->h_async_function =
                                                serviceptr->h_async_function;
-
-                                       /* Determine whether local socket */
+                                       con->ServiceName =
+                                               serviceptr->ServiceName;
+                                       
+                                       /* Determine whether it's a local socket */
                                        if (serviceptr->sockpath != NULL)
                                                con->is_local_socket = 1;
        
@@ -1064,9 +2350,7 @@ SKIP_SELECT:
 
                dead_session_purge(force_purge);
                do_housekeeping();
-               check_sched_shutdown();
        }
-
        /* If control reaches this point, the server is shutting down */        
        return(NULL);
 }
@@ -1216,7 +2500,7 @@ void dump_heap(void) {
        struct igheap *thisheap;
 
        for (thisheap = igheap; thisheap != NULL; thisheap = thisheap->next) {
-               lprintf(CTDL_CRIT, "UNFREED: %30s : %d\n",
+               CtdlLogPrintf(CTDL_CRIT, "UNFREED: %30s : %d\n",
                        thisheap->file, thisheap->line);
        }
 }