]> code.citadel.org Git - citadel.git/blobdiff - citadel/threads.c
Prevent cancellation of threads but log that we would have cancelled it.
[citadel.git] / citadel / threads.c
index f099177359b202a9a24dbf3c805c1c9cb1e137d8..fdbf5dced068a5fa3be3daeaf5a367709e078dd4 100644 (file)
@@ -8,6 +8,7 @@
  *
  */
 
+#include <sys/types.h>
 #include <errno.h>
 #include <sys/socket.h>
 #include <unistd.h>
@@ -56,19 +57,13 @@ static int num_workers = 0;                 /* Current number of worker threads */
 CtdlThreadNode *CtdlThreadList = NULL;
 CtdlThreadNode *CtdlThreadSchedList = NULL;
 
-/*
- * Condition variable and Mutex for thread garbage collection
- */
-/*static pthread_mutex_t thread_gc_mutex = PTHREAD_MUTEX_INITIALIZER;
-static pthread_cond_t thread_gc_cond = PTHREAD_COND_INITIALIZER;
-*/
-static pthread_t GC_thread;
+static citthread_t GC_thread;
 static char *CtdlThreadStates[CTDL_THREAD_LAST_STATE];
 double CtdlThreadLoadAvg = 0;
 double CtdlThreadWorkerAvg = 0;
-pthread_key_t ThreadKey;
+citthread_key_t ThreadKey;
 
-pthread_mutex_t Critters[MAX_SEMAPHORES];      /* Things needing locking */
+citthread_mutex_t Critters[MAX_SEMAPHORES];    /* Things needing locking */
 
 
 
@@ -78,7 +73,7 @@ void InitialiseSemaphores(void)
 
        /* Set up a bunch of semaphores to be used for critical sections */
        for (i=0; i<MAX_SEMAPHORES; ++i) {
-               pthread_mutex_init(&Critters[i], NULL);
+               citthread_mutex_init(&Critters[i], NULL);
        }
 }
 
@@ -103,7 +98,7 @@ int try_critical_section(int which_one)
        ) {
                cdb_check_handles();
        }
-       return (pthread_mutex_trylock(&Critters[which_one]));
+       return (citthread_mutex_trylock(&Critters[which_one]));
 }
 
 
@@ -126,7 +121,7 @@ void begin_critical_section(int which_one)
        ) {
                cdb_check_handles();
        }
-       pthread_mutex_lock(&Critters[which_one]);
+       citthread_mutex_lock(&Critters[which_one]);
 }
 
 /*
@@ -134,7 +129,7 @@ void begin_critical_section(int which_one)
  */
 void end_critical_section(int which_one)
 {
-       pthread_mutex_unlock(&Critters[which_one]);
+       citthread_mutex_unlock(&Critters[which_one]);
 }
 
 
@@ -157,9 +152,8 @@ void ctdl_thread_internal_init_tsd(void)
 {
        int ret;
        
-       if ((ret = pthread_key_create(&ThreadKey, ctdl_thread_internal_dest_tsd))) {
-               lprintf(CTDL_EMERG, "pthread_key_create: %s\n",
-                       strerror(ret));
+       if ((ret = citthread_key_create(&ThreadKey, ctdl_thread_internal_dest_tsd))) {
+               lprintf(CTDL_EMERG, "citthread_key_create: %s\n", strerror(ret));
                exit(CTDLEXIT_DB);
        }
 }
@@ -174,7 +168,7 @@ void CtdlThreadAllocTSD(void)
 {
        ThreadTSD *tsd;
 
-       if (pthread_getspecific(ThreadKey) != NULL)
+       if (citthread_getspecific(ThreadKey) != NULL)
                return;
 
        tsd = malloc(sizeof(ThreadTSD));
@@ -184,14 +178,14 @@ void CtdlThreadAllocTSD(void)
        memset(tsd->cursors, 0, sizeof tsd->cursors);
        tsd->self = NULL;
        
-       pthread_setspecific(ThreadKey, tsd);
+       citthread_setspecific(ThreadKey, tsd);
 }
 
 
 void ctdl_thread_internal_free_tsd(void)
 {
-       ctdl_thread_internal_dest_tsd(pthread_getspecific(ThreadKey));
-       pthread_setspecific(ThreadKey, NULL);
+       ctdl_thread_internal_dest_tsd(citthread_getspecific(ThreadKey));
+       citthread_setspecific(ThreadKey, NULL);
 }
 
 
@@ -211,11 +205,11 @@ void ctdl_thread_internal_cleanup(void)
        {
                that_thread = this_thread;
                this_thread = this_thread->next;
-               pthread_mutex_destroy(&that_thread->ThreadMutex);
-               pthread_cond_destroy(&that_thread->ThreadCond);
-               pthread_mutex_destroy(&that_thread->SleepMutex);
-               pthread_cond_destroy(&that_thread->SleepCond);
-               pthread_attr_destroy(&that_thread->attr);
+               citthread_mutex_destroy(&that_thread->ThreadMutex);
+               citthread_cond_destroy(&that_thread->ThreadCond);
+               citthread_mutex_destroy(&that_thread->SleepMutex);
+               citthread_cond_destroy(&that_thread->SleepCond);
+               citthread_attr_destroy(&that_thread->attr);
                free(that_thread);
        }
        ctdl_thread_internal_free_tsd();
@@ -226,7 +220,7 @@ void ctdl_thread_internal_init(void)
        CtdlThreadNode *this_thread;
        int ret = 0;
        
-       GC_thread = pthread_self();
+       GC_thread = citthread_self();
        CtdlThreadStates[CTDL_THREAD_INVALID] = strdup ("Invalid Thread");
        CtdlThreadStates[CTDL_THREAD_VALID] = strdup("Valid Thread");
        CtdlThreadStates[CTDL_THREAD_CREATE] = strdup("Thread being Created");
@@ -247,16 +241,16 @@ void ctdl_thread_internal_init(void)
        // Ensuring this is zero'd means we make sure the thread doesn't start doing its thing until we are ready.
        memset (this_thread, 0, sizeof(CtdlThreadNode));
        
-       pthread_mutex_init (&(this_thread->ThreadMutex), NULL);
-       pthread_cond_init (&(this_thread->ThreadCond), NULL);
-       pthread_mutex_init (&(this_thread->SleepMutex), NULL);
-       pthread_cond_init (&(this_thread->SleepCond), NULL);
+       citthread_mutex_init (&(this_thread->ThreadMutex), NULL);
+       citthread_cond_init (&(this_thread->ThreadCond), NULL);
+       citthread_mutex_init (&(this_thread->SleepMutex), NULL);
+       citthread_cond_init (&(this_thread->SleepCond), NULL);
        
        /* We are garbage collector so create us as running */
        this_thread->state = CTDL_THREAD_RUNNING;
        
-       if ((ret = pthread_attr_init(&this_thread->attr))) {
-               CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_init: %s\n", strerror(ret));
+       if ((ret = citthread_attr_init(&this_thread->attr))) {
+               CtdlLogPrintf(CTDL_EMERG, "Thread system, citthread_attr_init: %s\n", strerror(ret));
                free(this_thread);
                return;
        }
@@ -289,7 +283,7 @@ void ctdl_thread_internal_init(void)
        gettimeofday(&now, NULL);
        timersub(&now, &(this_thread->last_state_change), &result);
        /* I don't think these mutex's are needed here */
-       pthread_mutex_lock(&this_thread->ThreadMutex);
+       citthread_mutex_lock(&this_thread->ThreadMutex);
        // result now has a timeval for the time we spent in the last state since we last updated
        last_duration = (double)result.tv_sec + ((double)result.tv_usec / (double) 1000000);
        if (this_thread->state == CTDL_THREAD_SLEEPING)
@@ -299,7 +293,7 @@ void ctdl_thread_internal_init(void)
        if (this_thread->state == CTDL_THREAD_BLOCKED)
                this_thread->avg_blocked += last_duration;
        memcpy (&this_thread->last_state_change, &now, sizeof (struct timeval));
-       pthread_mutex_unlock(&this_thread->ThreadMutex);
+       citthread_mutex_unlock(&this_thread->ThreadMutex);
 }
 
 /*
@@ -312,14 +306,14 @@ void ctdl_thread_internal_change_state (CtdlThreadNode *this_thread, enum CtdlTh
         */
        ctdl_thread_internal_update_avgs(this_thread);
        /* This mutex not needed here? */
-       pthread_mutex_lock(&this_thread->ThreadMutex); /* To prevent race condition of a sleeping thread */
+       citthread_mutex_lock(&this_thread->ThreadMutex); /* To prevent race condition of a sleeping thread */
        if ((new_state == CTDL_THREAD_STOP_REQ) && (this_thread->state > CTDL_THREAD_STOP_REQ))
                this_thread->state = new_state;
        if (((new_state == CTDL_THREAD_SLEEPING) || (new_state == CTDL_THREAD_BLOCKED)) && (this_thread->state == CTDL_THREAD_RUNNING))
                this_thread->state = new_state;
        if ((new_state == CTDL_THREAD_RUNNING) && ((this_thread->state == CTDL_THREAD_SLEEPING) || (this_thread->state == CTDL_THREAD_BLOCKED)))
                this_thread->state = new_state;
-       pthread_mutex_unlock(&this_thread->ThreadMutex);
+       citthread_mutex_unlock(&this_thread->ThreadMutex);
 }
 
 
@@ -338,12 +332,13 @@ void CtdlThreadStopAll(void)
        while(this_thread)
        {
 #ifdef THREADS_USESIGNALS
-               pthread_kill(this_thread->tid, SIGHUP);
+               citthread_killl(this_thread->tid, SIGHUP);
 #endif
                ctdl_thread_internal_change_state (this_thread, CTDL_THREAD_STOP_REQ);
-               pthread_cond_signal(&this_thread->ThreadCond);
-               pthread_cond_signal(&this_thread->SleepCond);
-               CtdlLogPrintf(CTDL_DEBUG, "Thread system stopping thread \"%s\" (%ld).\n", this_thread->name, this_thread->tid);
+               citthread_cond_signal(&this_thread->ThreadCond);
+               citthread_cond_signal(&this_thread->SleepCond);
+               CtdlLogPrintf(CTDL_DEBUG, "Thread system stopping thread \"%s\" (0x%08lx).\n",
+                       this_thread->name, this_thread->tid);
                this_thread = this_thread->next;
        }
        end_critical_section(S_THREAD_LIST);
@@ -365,8 +360,8 @@ void CtdlThreadWakeAll(void)
        {
                if (!this_thread->thread_func)
                {
-                       pthread_cond_signal(&this_thread->ThreadCond);
-                       pthread_cond_signal(&this_thread->SleepCond);
+                       citthread_cond_signal(&this_thread->ThreadCond);
+                       citthread_cond_signal(&this_thread->SleepCond);
                }
                this_thread = this_thread->next;
        }
@@ -456,11 +451,10 @@ void CtdlThreadCancel(CtdlThreadNode *thread)
        }
        
        ctdl_thread_internal_change_state (this_thread, CTDL_THREAD_CANCELLED);
-       pthread_cancel(this_thread->tid);
+       citthread_cancel(this_thread->tid);
 }
 
 
-
 /*
  * A function for a thread to check if it has been asked to stop
  */
@@ -511,11 +505,11 @@ void CtdlThreadStop(CtdlThreadNode *thread)
        if (!(this_thread->thread_func))
                return;         // Don't stop garbage collector
 #ifdef THREADS_USESIGNALS
-       pthread_kill(this_thread->tid, SIGHUP); 
+       citthread_kill(this_thread->tid, SIGHUP);       
 #endif
        ctdl_thread_internal_change_state (this_thread, CTDL_THREAD_STOP_REQ);
-       pthread_cond_signal(&this_thread->ThreadCond);
-       pthread_cond_signal(&this_thread->SleepCond);
+       citthread_cond_signal(&this_thread->ThreadCond);
+       citthread_cond_signal(&this_thread->SleepCond);
 }
 
 /*
@@ -540,9 +534,9 @@ void CtdlThreadSleep(int secs)
 
        ctdl_thread_internal_change_state (CT, CTDL_THREAD_SLEEPING);
        
-       pthread_mutex_lock(&CT->ThreadMutex); /* Prevent something asking us to awaken before we've gone to sleep */
-       pthread_cond_timedwait(&CT->SleepCond, &CT->ThreadMutex, &wake_time);
-       pthread_mutex_unlock(&CT->ThreadMutex);
+       citthread_mutex_lock(&CT->ThreadMutex); /* Prevent something asking us to awaken before we've gone to sleep */
+       citthread_cond_timedwait(&CT->SleepCond, &CT->ThreadMutex, &wake_time);
+       citthread_mutex_unlock(&CT->ThreadMutex);
        
        ctdl_thread_internal_change_state (CT, CTDL_THREAD_RUNNING);
 }
@@ -557,15 +551,15 @@ static void ctdl_internal_thread_cleanup(void *arg)
         * In here we were called by the current thread because it is exiting
         * NB. WE ARE THE CURRENT THREAD
         */
-       CtdlLogPrintf(CTDL_NOTICE, "Thread \"%s\" (%ld) exited.\n", CT->name, CT->tid);
+       CtdlLogPrintf(CTDL_NOTICE, "Thread \"%s\" (0x%08lx) exited.\n", CT->name, CT->tid);
        
        #ifdef HAVE_BACKTRACE
        eCrash_UnregisterThread();
        #endif
        
-       pthread_mutex_lock(&CT->ThreadMutex);
+       citthread_mutex_lock(&CT->ThreadMutex);
        CT->state = CTDL_THREAD_EXITED; // needs to be last thing else house keeping will unlink us too early
-       pthread_mutex_unlock(&CT->ThreadMutex);
+       citthread_mutex_unlock(&CT->ThreadMutex);
 }
 
 /*
@@ -584,7 +578,7 @@ void ctdl_thread_internal_calc_loadavg(void)
        {
                /* Update load averages */
                ctdl_thread_internal_update_avgs(that_thread);
-               pthread_mutex_lock(&that_thread->ThreadMutex);
+               citthread_mutex_lock(&that_thread->ThreadMutex);
                that_thread->load_avg = (that_thread->avg_sleeping + that_thread->avg_running) / (that_thread->avg_sleeping + that_thread->avg_running + that_thread->avg_blocked) * 100;
                that_thread->avg_sleeping /= 2;
                that_thread->avg_running /= 2;
@@ -605,7 +599,7 @@ void ctdl_thread_internal_calc_loadavg(void)
                        that_thread->avg_blocked,
                        that_thread->load_avg);
 #endif
-               pthread_mutex_unlock(&that_thread->ThreadMutex);
+               citthread_mutex_unlock(&that_thread->ThreadMutex);
                that_thread = that_thread->next;
        }
        CtdlThreadLoadAvg = load_avg/num_threads;
@@ -644,15 +638,30 @@ void CtdlThreadGC (void)
                that_thread = this_thread;
                this_thread = this_thread->next;
                
+               if ((that_thread->state == CTDL_THREAD_STOP_REQ || that_thread->state == CTDL_THREAD_STOPPING)
+                       && (!citthread_equal(that_thread->tid, citthread_self())))
+                               that_thread->stop_ticker++;
+               
+               if (that_thread->stop_ticker == 5)
+               {
+                       CtdlLogPrintf(CTDL_DEBUG, "Thread System: The thread \"%s\" (0x%08lx) failed to self terminate withing 5 ticks. It would be cancelled now.\n", that_thread->name, that_thread->tid);
+                       if ((that_thread->flags & CTDLTHREAD_WORKER) == 0)
+                               CtdlLogPrintf(CTDL_INFO, "Thread System: A non worker thread would have been canceled this may cause message loss.\n");
+//                     that_thread->state = CTDL_THREAD_CANCELLED;
+                       that_thread->stop_ticker++;
+//                     citthread_cancel(that_thread->tid);
+//                     continue;
+               }
+               
                /* Do we need to clean up this thread? */
-               if (that_thread->state != CTDL_THREAD_EXITED)
+               if ((that_thread->state != CTDL_THREAD_EXITED) && (that_thread->state != CTDL_THREAD_CANCELLED))
                {
                        if(that_thread->flags & CTDLTHREAD_WORKER)
                                workers++;      /* Sanity check on number of worker threads */
                        continue;
                }
                
-               if (pthread_equal(that_thread->tid, pthread_self()) && that_thread->thread_func)
+               if (citthread_equal(that_thread->tid, citthread_self()) && that_thread->thread_func)
                {       /* Sanity check */
                        end_critical_section(S_THREAD_LIST);
                        CtdlLogPrintf(CTDL_EMERG, "Thread system PANIC, a thread is trying to clean up after itself.\n");
@@ -679,16 +688,16 @@ void CtdlThreadGC (void)
                if(that_thread->next)
                        that_thread->next->prev = that_thread->prev;
                
-               pthread_cond_signal(&that_thread->ThreadCond);
-               pthread_cond_signal(&that_thread->SleepCond);   // Make sure this thread is awake
-               pthread_mutex_lock(&that_thread->ThreadMutex);  // Make sure it has done what its doing
-               pthread_mutex_unlock(&that_thread->ThreadMutex);
+               citthread_cond_signal(&that_thread->ThreadCond);
+               citthread_cond_signal(&that_thread->SleepCond); // Make sure this thread is awake
+               citthread_mutex_lock(&that_thread->ThreadMutex);        // Make sure it has done what its doing
+               citthread_mutex_unlock(&that_thread->ThreadMutex);
                /*
                 * Join on the thread to do clean up and prevent memory leaks
                 * Also makes sure the thread has cleaned up after itself before we remove it from the list
                 * We can join on the garbage collector thread the join should just return EDEADLCK
                 */
-               ret = pthread_join (that_thread->tid, NULL);
+               ret = citthread_join (that_thread->tid, NULL);
                if (ret == EDEADLK)
                        CtdlLogPrintf(CTDL_DEBUG, "Garbage collection on own thread.\n");
                else if (ret == EINVAL)
@@ -696,16 +705,17 @@ void CtdlThreadGC (void)
                else if (ret == ESRCH)
                        CtdlLogPrintf(CTDL_DEBUG, "Garbage collection, no thread to join on.\n");
                else if (ret != 0)
-                       CtdlLogPrintf(CTDL_DEBUG, "Garbage collection, pthread_join returned an unknown error.\n");
+                       CtdlLogPrintf(CTDL_DEBUG, "Garbage collection, citthread_join returned an unknown error(%d).\n", ret);
                /*
                 * Now we own that thread entry
                 */
-               CtdlLogPrintf(CTDL_INFO, "Garbage Collection for thread \"%s\" (%ld).\n", that_thread->name, that_thread->tid);
-               pthread_mutex_destroy(&that_thread->ThreadMutex);
-               pthread_cond_destroy(&that_thread->ThreadCond);
-               pthread_mutex_destroy(&that_thread->SleepMutex);
-               pthread_cond_destroy(&that_thread->SleepCond);
-               pthread_attr_destroy(&that_thread->attr);
+               CtdlLogPrintf(CTDL_INFO, "Garbage Collection for thread \"%s\" (0x%08lx).\n",
+                       that_thread->name, that_thread->tid);
+               citthread_mutex_destroy(&that_thread->ThreadMutex);
+               citthread_cond_destroy(&that_thread->ThreadCond);
+               citthread_mutex_destroy(&that_thread->SleepMutex);
+               citthread_cond_destroy(&that_thread->SleepCond);
+               citthread_attr_destroy(&that_thread->attr);
                free(that_thread);
        }
        sys_workers = num_workers;
@@ -742,10 +752,10 @@ static void *ctdl_internal_thread_func (void *arg)
        begin_critical_section(S_THREAD_LIST);
        this_thread = (CtdlThreadNode *) arg;
        gettimeofday(&this_thread->start_time, NULL);           /* Time this thread started */
-       pthread_mutex_lock(&this_thread->ThreadMutex);
+//     citthread_mutex_lock(&this_thread->ThreadMutex);
        
        // Register the cleanup function to take care of when we exit.
-       pthread_cleanup_push(ctdl_internal_thread_cleanup, NULL);
+       citthread_cleanup_push(ctdl_internal_thread_cleanup, NULL);
        // Get our thread data structure
        CtdlThreadAllocTSD();
        CT = this_thread;
@@ -755,13 +765,13 @@ static void *ctdl_internal_thread_func (void *arg)
         * Other wise there is a window to allow this threads creation to continue to full grown and
         * therby prevent a shutdown of the server.
         */
-       pthread_mutex_unlock(&this_thread->ThreadMutex);
+//     citthread_mutex_unlock(&this_thread->ThreadMutex);
                
        if (!CtdlThreadCheckStop())
        {
-               pthread_mutex_lock(&this_thread->ThreadMutex);
+               citthread_mutex_lock(&this_thread->ThreadMutex);
                this_thread->state = CTDL_THREAD_RUNNING;
-               pthread_mutex_unlock(&this_thread->ThreadMutex);
+               citthread_mutex_unlock(&this_thread->ThreadMutex);
        }
        end_critical_section(S_THREAD_LIST);
        
@@ -771,9 +781,8 @@ static void *ctdl_internal_thread_func (void *arg)
        #endif
        
        // Tell the world we are here
-       CtdlLogPrintf(CTDL_NOTICE, "Created a new thread \"%s\" (%ld). \n", this_thread->name, this_thread->tid);
-
-       
+       CtdlLogPrintf(CTDL_NOTICE, "Created a new thread \"%s\" (0x%08lx).\n",
+               this_thread->name, this_thread->tid);
        
        /*
         * run the thread to do the work but only if we haven't been asked to stop
@@ -785,53 +794,39 @@ static void *ctdl_internal_thread_func (void *arg)
         * Our thread is exiting either because it wanted to end or because the server is stopping
         * We need to clean up
         */
-       pthread_cleanup_pop(1); // Execute our cleanup routine and remove it
+       citthread_cleanup_pop(1);       // Execute our cleanup routine and remove it
        
        return(ret);
 }
 
 
+
+
 /*
- * Internal function to create a thread.
- * Must be called from within a S_THREAD_LIST critical section
- */ 
-CtdlThreadNode *ctdl_internal_create_thread(char *name, long flags, void *(*thread_func) (void *arg), void *args)
+ * Function to initialise an empty thread structure
+ */
+CtdlThreadNode *ctdl_internal_init_thread_struct(CtdlThreadNode *this_thread, long flags)
 {
        int ret = 0;
-       CtdlThreadNode *this_thread;
-
-       if (num_threads >= 32767)
-       {
-               CtdlLogPrintf(CTDL_EMERG, "Thread system. Thread list full.\n");
-               return NULL;
-       }
-               
-       this_thread = malloc(sizeof(CtdlThreadNode));
-       if (this_thread == NULL) {
-               CtdlLogPrintf(CTDL_EMERG, "Thread system, can't allocate CtdlThreadNode, exiting\n");
-               return NULL;
-       }
+       
        // Ensuring this is zero'd means we make sure the thread doesn't start doing its thing until we are ready.
        memset (this_thread, 0, sizeof(CtdlThreadNode));
        
        /* Create the mutex's early so we can use them */
-       pthread_mutex_init (&(this_thread->ThreadMutex), NULL);
-       pthread_cond_init (&(this_thread->ThreadCond), NULL);
-       pthread_mutex_init (&(this_thread->SleepMutex), NULL);
-       pthread_cond_init (&(this_thread->SleepCond), NULL);
-       
-       pthread_mutex_lock(&this_thread->ThreadMutex);
+       citthread_mutex_init (&(this_thread->ThreadMutex), NULL);
+       citthread_cond_init (&(this_thread->ThreadCond), NULL);
+       citthread_mutex_init (&(this_thread->SleepMutex), NULL);
+       citthread_cond_init (&(this_thread->SleepCond), NULL);
        
        this_thread->state = CTDL_THREAD_CREATE;
        
-       if ((ret = pthread_attr_init(&this_thread->attr))) {
-               pthread_mutex_unlock(&this_thread->ThreadMutex);
-               pthread_mutex_destroy(&(this_thread->ThreadMutex));
-               pthread_cond_destroy(&(this_thread->ThreadCond));
-               pthread_mutex_destroy(&(this_thread->SleepMutex));
-               pthread_cond_destroy(&(this_thread->SleepCond));
-               CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_init: %s\n", strerror(ret));
+       if ((ret = citthread_attr_init(&this_thread->attr))) {
+               citthread_mutex_unlock(&this_thread->ThreadMutex);
+               citthread_mutex_destroy(&(this_thread->ThreadMutex));
+               citthread_cond_destroy(&(this_thread->ThreadCond));
+               citthread_mutex_destroy(&(this_thread->SleepMutex));
+               citthread_cond_destroy(&(this_thread->SleepCond));
+               CtdlLogPrintf(CTDL_EMERG, "Thread system, citthread_attr_init: %s\n", strerror(ret));
                free(this_thread);
                return NULL;
        }
@@ -845,20 +840,59 @@ CtdlThreadNode *ctdl_internal_create_thread(char *name, long flags, void *(*thre
 #ifdef WITH_THREADLOG
                CtdlLogPrintf(CTDL_INFO, "Thread system. Creating BIG STACK thread.\n");
 #endif
-               if ((ret = pthread_attr_setstacksize(&this_thread->attr, THREADSTACKSIZE))) {
-                       pthread_mutex_unlock(&this_thread->ThreadMutex);
-                       pthread_mutex_destroy(&(this_thread->ThreadMutex));
-                       pthread_cond_destroy(&(this_thread->ThreadCond));
-                       pthread_mutex_destroy(&(this_thread->SleepMutex));
-                       pthread_cond_destroy(&(this_thread->SleepCond));
-                       pthread_attr_destroy(&this_thread->attr);
-                       CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_setstacksize: %s\n",
+               if ((ret = citthread_attr_setstacksize(&this_thread->attr, THREADSTACKSIZE))) {
+                       citthread_mutex_unlock(&this_thread->ThreadMutex);
+                       citthread_mutex_destroy(&(this_thread->ThreadMutex));
+                       citthread_cond_destroy(&(this_thread->ThreadCond));
+                       citthread_mutex_destroy(&(this_thread->SleepMutex));
+                       citthread_cond_destroy(&(this_thread->SleepCond));
+                       citthread_attr_destroy(&this_thread->attr);
+                       CtdlLogPrintf(CTDL_EMERG, "Thread system, citthread_attr_setstacksize: %s\n",
                                strerror(ret));
                        free(this_thread);
                        return NULL;
                }
        }
 
+       /* Set this new thread with an avg_blocked of 2. We do this so that its creation affects the
+        * load average for the system. If we don't do this then we create a mass of threads at the same time 
+        * because the creation didn't affect the load average.
+        */
+       this_thread->avg_blocked = 2;
+       
+       return (this_thread);
+}
+
+
+
+/*
+ * Internal function to create a thread.
+ */ 
+CtdlThreadNode *ctdl_internal_create_thread(char *name, long flags, void *(*thread_func) (void *arg), void *args)
+{
+       int ret = 0;
+       CtdlThreadNode *this_thread;
+
+       if (num_threads >= 32767)
+       {
+               CtdlLogPrintf(CTDL_EMERG, "Thread system. Thread list full.\n");
+               return NULL;
+       }
+               
+       this_thread = malloc(sizeof(CtdlThreadNode));
+       if (this_thread == NULL) {
+               CtdlLogPrintf(CTDL_EMERG, "Thread system, can't allocate CtdlThreadNode, exiting\n");
+               return NULL;
+       }
+       
+       /* Initialise the thread structure */
+       if (ctdl_internal_init_thread_struct(this_thread, flags) == NULL)
+       {
+               free(this_thread);
+               CtdlLogPrintf(CTDL_EMERG, "Thread system, can't initialise CtdlThreadNode, exiting\n");
+               return NULL;
+       }
        /*
         * If we got here we are going to create the thread so we must initilise the structure
         * first because most implimentations of threading can't create it in a stopped state
@@ -876,28 +910,26 @@ CtdlThreadNode *ctdl_internal_create_thread(char *name, long flags, void *(*thre
        this_thread->flags = flags;
        this_thread->thread_func = thread_func;
        this_thread->user_args = args;
-       /* Set this new thread with an avg_blocked of 2. We do this so that its creation affects the
-        * load average for the system. If we don't do this then we create a mass of threads at the same time 
-        * because the creation didn't affect the load average.
-        */
-       this_thread->avg_blocked = 2;
        
+//     citthread_mutex_lock(&this_thread->ThreadMutex);
+       
+       begin_critical_section(S_THREAD_LIST);
        /*
         * We pass this_thread into the thread as its args so that it can find out information
         * about itself and it has a bit of storage space for itself, not to mention that the REAL
         * thread function needs to finish off the setup of the structure
         */
-       if ((ret = pthread_create(&this_thread->tid, &this_thread->attr, ctdl_internal_thread_func, this_thread) != 0))
+       if ((ret = citthread_create(&this_thread->tid, &this_thread->attr, ctdl_internal_thread_func, this_thread) != 0))
        {
-
+               end_critical_section(S_THREAD_LIST);
                CtdlLogPrintf(CTDL_ALERT, "Thread system, Can't create thread: %s\n",
                        strerror(ret));
-               pthread_mutex_unlock(&this_thread->ThreadMutex);
-               pthread_mutex_destroy(&(this_thread->ThreadMutex));
-               pthread_cond_destroy(&(this_thread->ThreadCond));
-               pthread_mutex_destroy(&(this_thread->SleepMutex));
-               pthread_cond_destroy(&(this_thread->SleepCond));
-               pthread_attr_destroy(&this_thread->attr);
+               citthread_mutex_unlock(&this_thread->ThreadMutex);
+               citthread_mutex_destroy(&(this_thread->ThreadMutex));
+               citthread_cond_destroy(&(this_thread->ThreadCond));
+               citthread_mutex_destroy(&(this_thread->SleepMutex));
+               citthread_cond_destroy(&(this_thread->SleepCond));
+               citthread_attr_destroy(&this_thread->attr);
                free(this_thread);
                return NULL;
        }
@@ -910,10 +942,11 @@ CtdlThreadNode *ctdl_internal_create_thread(char *name, long flags, void *(*thre
        CtdlThreadList = this_thread;
        if (this_thread->next)
                this_thread->next->prev = this_thread;
+       ctdl_thread_internal_calc_loadavg();
        
-       pthread_mutex_unlock(&this_thread->ThreadMutex);
+//     citthread_mutex_unlock(&this_thread->ThreadMutex);
+       end_critical_section(S_THREAD_LIST);
        
-       ctdl_thread_internal_calc_loadavg();
        return this_thread;
 }
 
@@ -927,9 +960,7 @@ CtdlThreadNode *CtdlThreadCreate(char *name, long flags, void *(*thread_func) (v
 {
        CtdlThreadNode *ret = NULL;
        
-       begin_critical_section(S_THREAD_LIST);
        ret = ctdl_internal_create_thread(name, flags, thread_func, args);
-       end_critical_section(S_THREAD_LIST);
        return ret;
 }
 
@@ -941,7 +972,6 @@ CtdlThreadNode *CtdlThreadCreate(char *name, long flags, void *(*thread_func) (v
  */ 
 CtdlThreadNode *CtdlThreadSchedule(char *name, long flags, void *(*thread_func) (void *arg), void *args, time_t when)
 {
-       int ret = 0;
        CtdlThreadNode *this_thread;
 
        if (num_threads >= 32767)
@@ -955,47 +985,14 @@ CtdlThreadNode *CtdlThreadSchedule(char *name, long flags, void *(*thread_func)
                CtdlLogPrintf(CTDL_EMERG, "Thread system, can't allocate CtdlThreadNode, exiting\n");
                return NULL;
        }
-       // Ensuring this is zero'd means we make sure the thread doesn't start doing its thing until we are ready.
-       memset (this_thread, 0, sizeof(CtdlThreadNode));
-       
-       /* Create the mutex's early so we can use them */
-       pthread_mutex_init (&(this_thread->ThreadMutex), NULL);
-       pthread_cond_init (&(this_thread->ThreadCond), NULL);
-       pthread_mutex_init (&(this_thread->SleepMutex), NULL);
-       pthread_cond_init (&(this_thread->SleepCond), NULL);
-       
-       this_thread->state = CTDL_THREAD_CREATE;
-       
-       if ((ret = pthread_attr_init(&this_thread->attr))) {
-               pthread_mutex_destroy(&(this_thread->ThreadMutex));
-               pthread_cond_destroy(&(this_thread->ThreadCond));
-               pthread_mutex_destroy(&(this_thread->SleepMutex));
-               pthread_cond_destroy(&(this_thread->SleepCond));
-               CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_init: %s\n", strerror(ret));
+       /* Initialise the thread structure */
+       if (ctdl_internal_init_thread_struct(this_thread, flags) == NULL)
+       {
                free(this_thread);
+               CtdlLogPrintf(CTDL_EMERG, "Thread system, can't initialise CtdlThreadNode, exiting\n");
                return NULL;
        }
 
-       /* Our per-thread stacks need to be bigger than the default size,
-        * otherwise the MIME parser crashes on FreeBSD, and the IMAP service
-        * crashes on 64-bit Linux.
-        */
-       if (flags & CTDLTHREAD_BIGSTACK)
-       {
-               CtdlLogPrintf(CTDL_INFO, "Thread system. Creating BIG STACK thread.\n");
-               if ((ret = pthread_attr_setstacksize(&this_thread->attr, THREADSTACKSIZE))) {
-                       pthread_mutex_destroy(&(this_thread->ThreadMutex));
-                       pthread_cond_destroy(&(this_thread->ThreadCond));
-                       pthread_mutex_destroy(&(this_thread->SleepMutex));
-                       pthread_cond_destroy(&(this_thread->SleepCond));
-                       pthread_attr_destroy(&this_thread->attr);
-                       CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_setstacksize: %s\n",
-                               strerror(ret));
-                       free(this_thread);
-                       return NULL;
-               }
-       }
-
        /*
         * If we got here we are going to create the thread so we must initilise the structure
         * first because most implimentations of threading can't create it in a stopped state
@@ -1013,11 +1010,6 @@ CtdlThreadNode *CtdlThreadSchedule(char *name, long flags, void *(*thread_func)
        this_thread->flags = flags;
        this_thread->thread_func = thread_func;
        this_thread->user_args = args;
-       /* Set this new thread with an avg_blocked of 2. We do this so that its creation affects the
-        * load average for the system. If we don't do this then we create a mass of threads at the same time 
-        * because the creation didn't affect the load average.
-        */
-       this_thread->avg_blocked = 2;
        
        /*
         * When to start this thread
@@ -1040,16 +1032,24 @@ CtdlThreadNode *ctdl_thread_internal_start_scheduled (CtdlThreadNode *this_threa
 {
        int ret = 0;
        
+//     citthread_mutex_lock(&that_thread->ThreadMutex);
+       begin_critical_section(S_THREAD_LIST);
        /*
         * We pass this_thread into the thread as its args so that it can find out information
         * about itself and it has a bit of storage space for itself, not to mention that the REAL
         * thread function needs to finish off the setup of the structure
         */
-       if ((ret = pthread_create(&this_thread->tid, &this_thread->attr, ctdl_internal_thread_func, this_thread) != 0))
+       if ((ret = citthread_create(&this_thread->tid, &this_thread->attr, ctdl_internal_thread_func, this_thread) != 0))
        {
-
-               CtdlLogPrintf(CTDL_ALERT, "Thread system, Can't create thread: %s\n",
-                       strerror(ret));
+               end_critical_section(S_THREAD_LIST);
+               CtdlLogPrintf(CTDL_DEBUG, "Failed to start scheduled thread \"%s\": %s\n", this_thread->name, strerror(ret));
+//             citthread_mutex_unlock(&this_thread->ThreadMutex);
+               citthread_mutex_destroy(&(this_thread->ThreadMutex));
+               citthread_cond_destroy(&(this_thread->ThreadCond));
+               citthread_mutex_destroy(&(this_thread->SleepMutex));
+               citthread_cond_destroy(&(this_thread->SleepCond));
+               citthread_attr_destroy(&this_thread->attr);
+               free(this_thread);
                return NULL;
        }
        
@@ -1062,6 +1062,11 @@ CtdlThreadNode *ctdl_thread_internal_start_scheduled (CtdlThreadNode *this_threa
        CtdlThreadList = this_thread;
        if (this_thread->next)
                this_thread->next->prev = this_thread;
+//     citthread_mutex_unlock(&that_thread->ThreadMutex);
+       
+       ctdl_thread_internal_calc_loadavg();
+       end_critical_section(S_THREAD_LIST);
+       
        
        return this_thread;
 }
@@ -1102,39 +1107,24 @@ void ctdl_thread_internal_check_scheduled(void)
 #ifdef WITH_THREADLOG
                        CtdlLogPrintf(CTDL_DEBUG, "About to start scheduled thread \"%s\".\n", that_thread->name);
 #endif
-                       begin_critical_section(S_THREAD_LIST);
                        if (CT->state > CTDL_THREAD_STOP_REQ)
                        {       /* Only start it if the system is not stopping */
-                               pthread_mutex_lock(&that_thread->ThreadMutex);
-                               if (ctdl_thread_internal_start_scheduled (that_thread) == NULL)
+                               if (ctdl_thread_internal_start_scheduled (that_thread))
                                {
 #ifdef WITH_THREADLOG
-                       CtdlLogPrintf(CTDL_DEBUG, "Failed to start scheduled thread \"%s\".\n", that_thread->name);
-#endif
-                                       pthread_mutex_unlock(&that_thread->ThreadMutex);
-                                       pthread_mutex_destroy(&(that_thread->ThreadMutex));
-                                       pthread_cond_destroy(&(that_thread->ThreadCond));
-                                       pthread_mutex_destroy(&(that_thread->SleepMutex));
-                                       pthread_cond_destroy(&(that_thread->SleepCond));
-                                       pthread_attr_destroy(&that_thread->attr);
-                                       free(that_thread);
-                               }
-                               else
-                               {
-                                       CtdlLogPrintf(CTDL_INFO, "Thread system, Started a scheduled thread \"%s\" (%ld).\n",
+                                       CtdlLogPrintf(CTDL_INFO, "Thread system, Started a scheduled thread \"%s\" (%ud).\n",
                                                that_thread->name, that_thread->tid);
-                                       pthread_mutex_unlock(&that_thread->ThreadMutex);
-                                       ctdl_thread_internal_calc_loadavg();
+#endif
                                }
                        }
-                       end_critical_section(S_THREAD_LIST);
                }
+#ifdef WITH_THREADLOG
                else
                {
-#ifdef WITH_THREADLOG
-                       CtdlLogPrintf(CTDL_DEBUG, "Thread \"%s\" will start in %ld seconds.\n", that_thread->name, that_thread->when - time(NULL));
-#endif
+                       CtdlLogPrintf(CTDL_DEBUG, "Thread \"%s\" will start in %ld seconds.\n",
+                               that_thread->name, that_thread->when - time(NULL));
                }
+#endif
        }
        end_critical_section(S_SCHEDULE_LIST);
 }
@@ -1176,7 +1166,7 @@ void go_threading(void)
        /*
         * This thread is now used for garbage collection of other threads in the thread list
         */
-       CtdlLogPrintf(CTDL_INFO, "Startup thread %d becoming garbage collector,\n", pthread_self());
+       CtdlLogPrintf(CTDL_INFO, "Startup thread %d becoming garbage collector,\n", citthread_self());
 
        /*
         * We do a lot of locking and unlocking of the thread list in here.
@@ -1212,20 +1202,20 @@ void go_threading(void)
                        last_worker = CtdlThreadList;
                        while (last_worker)
                        {
-                               pthread_mutex_lock(&last_worker->ThreadMutex);
+                               citthread_mutex_lock(&last_worker->ThreadMutex);
                                if (last_worker->flags & CTDLTHREAD_WORKER && (last_worker->state > CTDL_THREAD_STOPPING) && (last_worker->Context == NULL))
                                {
-                                       pthread_mutex_unlock(&last_worker->ThreadMutex);
+                                       citthread_mutex_unlock(&last_worker->ThreadMutex);
                                        break;
                                }
-                               pthread_mutex_unlock(&last_worker->ThreadMutex);
+                               citthread_mutex_unlock(&last_worker->ThreadMutex);
                                last_worker = last_worker->next;
                        }
                        end_critical_section(S_THREAD_LIST);
                        if (last_worker)
                        {
 #ifdef WITH_THREADLOG
-                               CtdlLogPrintf(CTDL_DEBUG, "Thread system, stopping excess worker thread \"%s\" (%ld).\n",
+                               CtdlLogPrintf(CTDL_DEBUG, "Thread system, stopping excess worker thread \"%s\" (0x%08lx).\n",
                                        last_worker->name,
                                        last_worker->tid
                                        );
@@ -1250,7 +1240,7 @@ void go_threading(void)
                        for (i=0; i<5 ; i++)
                        {
 #ifdef NEW_WORKER
-                               CtdlThreadCreate("Worker Thread",
+                               CtdlThreadCreate("Worker Thread (new)",
                                        CTDLTHREAD_BIGSTACK + CTDLTHREAD_WORKER,
                                        new_worker_thread,
                                        NULL