citadel/threads.c

   1 /*
   2  * $Id: sysdep.c 5882 2007-12-13 19:46:05Z davew $
   3  *
   4  * Citadel "system dependent" stuff.
   5  * See copyright.txt for copyright information.
   6  *
   7  * Here's where we have the Citadel thread implimentation
   8  *
   9  */
  10
  11 #include <errno.h>
  12
  13 #include "threads.h"
  14 #include "ctdl_module.h"
  15 #include "modules_init.h"
  16 #include "housekeeping.h"
  17 #include "config.h"
  18
  19 /*
  20  * New thread interface.
  21  * To create a thread you must call one of the create thread functions.
  22  * You must pass it the address of (a pointer to a CtdlThreadNode initialised to NULL) like this
  23  * struct CtdlThreadNode *node = NULL;
  24  * pass in &node
  25  * If the thread is created *node will point to the thread control structure for the created thread.
  26  * If the thread creation fails *node remains NULL
  27  * Do not free the memory pointed to by *node, it doesn't belong to you.
  28  * This new interface duplicates much of the eCrash stuff. We should go for closer integration since that would
  29  * remove the need for the calls to eCrashRegisterThread and friends
  30  */
  31
  32 static int num_threads = 0;                     /* Current number of threads */
  33 static int num_workers = 0;                     /* Current number of worker threads */
  34
  35 struct CtdlThreadNode *CtdlThreadList = NULL;
  36 struct CtdlThreadNode *CtdlThreadSchedList = NULL;
  37
  38 /*
  39  * Condition variable and Mutex for thread garbage collection
  40  */
  41 /*static pthread_mutex_t thread_gc_mutex = PTHREAD_MUTEX_INITIALIZER;
  42 static pthread_cond_t thread_gc_cond = PTHREAD_COND_INITIALIZER;
  43 */
  44 static pthread_t GC_thread;
  45 static char *CtdlThreadStates[CTDL_THREAD_LAST_STATE];
  46 double CtdlThreadLoadAvg = 0;
  47 double CtdlThreadWorkerAvg = 0;
  48 pthread_key_t ThreadKey;
  49
  50 pthread_mutex_t Critters[MAX_SEMAPHORES];       /* Things needing locking */
  51
  52
  53
  54 void InitialiseSemaphores(void)
  55 {
  56         int i;
  57
  58         /* Set up a bunch of semaphores to be used for critical sections */
  59         for (i=0; i<MAX_SEMAPHORES; ++i) {
  60                 pthread_mutex_init(&Critters[i], NULL);
  61         }
  62 }
  63
  64
  65
  66
  67 /*
  68  * Obtain a semaphore lock to begin a critical section.
  69  * but only if no one else has one
  70  */
  71 int try_critical_section(int which_one)
  72 {
  73         /* For all types of critical sections except those listed here,
  74          * ensure nobody ever tries to do a critical section within a
  75          * transaction; this could lead to deadlock.
  76          */
  77         if (    (which_one != S_FLOORCACHE)
  78 #ifdef DEBUG_MEMORY_LEAKS
  79                 && (which_one != S_DEBUGMEMLEAKS)
  80 #endif
  81                 && (which_one != S_RPLIST)
  82         ) {
  83                 cdb_check_handles();
  84         }
  85         return (pthread_mutex_trylock(&Critters[which_one]));
  86 }
  87
  88
  89 /*
  90  * Obtain a semaphore lock to begin a critical section.
  91  */
  92 void begin_critical_section(int which_one)
  93 {
  94         /* CtdlLogPrintf(CTDL_DEBUG, "begin_critical_section(%d)\n", which_one); */
  95
  96         /* For all types of critical sections except those listed here,
  97          * ensure nobody ever tries to do a critical section within a
  98          * transaction; this could lead to deadlock.
  99          */
 100         if (    (which_one != S_FLOORCACHE)
 101 #ifdef DEBUG_MEMORY_LEAKS
 102                 && (which_one != S_DEBUGMEMLEAKS)
 103 #endif
 104                 && (which_one != S_RPLIST)
 105         ) {
 106                 cdb_check_handles();
 107         }
 108         pthread_mutex_lock(&Critters[which_one]);
 109 }
 110
 111 /*
 112  * Release a semaphore lock to end a critical section.
 113  */
 114 void end_critical_section(int which_one)
 115 {
 116         pthread_mutex_unlock(&Critters[which_one]);
 117 }
 118
 119
 120 /*
 121  * A function to destroy the TSD
 122  */
 123 static void ctdl_thread_internal_dest_tsd(void *arg)
 124 {
 125         if (arg != NULL) {
 126                 check_handles(arg);
 127                 free(arg);
 128         }
 129 }
 130
 131
 132 /*
 133  * A function to initialise the thread TSD
 134  */
 135 void ctdl_thread_internal_init_tsd(void)
 136 {
 137         int ret;
 138
 139         if ((ret = pthread_key_create(&ThreadKey, ctdl_thread_internal_dest_tsd))) {
 140                 lprintf(CTDL_EMERG, "pthread_key_create: %s\n",
 141                         strerror(ret));
 142                 exit(CTDLEXIT_DB);
 143         }
 144 }
 145
 146 /*
 147  * Ensure that we have a key for thread-specific data.
 148  *
 149  * This should be called immediately after startup by any thread
 150  *
 151  */
 152 void CtdlThreadAllocTSD(void)
 153 {
 154         ThreadTSD *tsd;
 155
 156         if (pthread_getspecific(ThreadKey) != NULL)
 157                 return;
 158
 159         tsd = malloc(sizeof(ThreadTSD));
 160
 161         tsd->tid = NULL;
 162
 163         memset(tsd->cursors, 0, sizeof tsd->cursors);
 164         tsd->self = NULL;
 165
 166         pthread_setspecific(ThreadKey, tsd);
 167 }
 168
 169
 170 void ctdl_thread_internal_free_tsd(void)
 171 {
 172         ctdl_thread_internal_dest_tsd(pthread_getspecific(ThreadKey));
 173         pthread_setspecific(ThreadKey, NULL);
 174 }
 175
 176
 177 void ctdl_thread_internal_cleanup(void)
 178 {
 179         int i;
 180         struct CtdlThreadNode *this_thread, *that_thread;
 181
 182         for (i=0; i<CTDL_THREAD_LAST_STATE; i++)
 183         {
 184                 free (CtdlThreadStates[i]);
 185         }
 186
 187         /* Clean up the scheduled thread list */
 188         this_thread = CtdlThreadSchedList;
 189         while (this_thread)
 190         {
 191                 that_thread = this_thread;
 192                 this_thread = this_thread->next;
 193                 pthread_mutex_destroy(&that_thread->ThreadMutex);
 194                 pthread_cond_destroy(&that_thread->ThreadCond);
 195                 pthread_mutex_destroy(&that_thread->SleepMutex);
 196                 pthread_cond_destroy(&that_thread->SleepCond);
 197                 pthread_attr_destroy(&that_thread->attr);
 198                 free(that_thread);
 199         }
 200         ctdl_thread_internal_free_tsd();
 201 }
 202
 203 void ctdl_thread_internal_init(void)
 204 {
 205         struct CtdlThreadNode *this_thread;
 206         int ret = 0;
 207
 208         GC_thread = pthread_self();
 209         CtdlThreadStates[CTDL_THREAD_INVALID] = strdup ("Invalid Thread");
 210         CtdlThreadStates[CTDL_THREAD_VALID] = strdup("Valid Thread");
 211         CtdlThreadStates[CTDL_THREAD_CREATE] = strdup("Thread being Created");
 212         CtdlThreadStates[CTDL_THREAD_CANCELLED] = strdup("Thread Cancelled");
 213         CtdlThreadStates[CTDL_THREAD_EXITED] = strdup("Thread Exited");
 214         CtdlThreadStates[CTDL_THREAD_STOPPING] = strdup("Thread Stopping");
 215         CtdlThreadStates[CTDL_THREAD_STOP_REQ] = strdup("Thread Stop Requested");
 216         CtdlThreadStates[CTDL_THREAD_SLEEPING] = strdup("Thread Sleeping");
 217         CtdlThreadStates[CTDL_THREAD_RUNNING] = strdup("Thread Running");
 218         CtdlThreadStates[CTDL_THREAD_BLOCKED] = strdup("Thread Blocked");
 219
 220         /* Get ourself a thread entry */
 221         this_thread = malloc(sizeof(struct CtdlThreadNode));
 222         if (this_thread == NULL) {
 223                 CtdlLogPrintf(CTDL_EMERG, "Thread system, can't allocate CtdlThreadNode, exiting\n");
 224                 return;
 225         }
 226         // Ensuring this is zero'd means we make sure the thread doesn't start doing its thing until we are ready.
 227         memset (this_thread, 0, sizeof(struct CtdlThreadNode));
 228
 229         pthread_mutex_init (&(this_thread->ThreadMutex), NULL);
 230         pthread_cond_init (&(this_thread->ThreadCond), NULL);
 231         pthread_mutex_init (&(this_thread->SleepMutex), NULL);
 232         pthread_cond_init (&(this_thread->SleepCond), NULL);
 233
 234         /* We are garbage collector so create us as running */
 235         this_thread->state = CTDL_THREAD_RUNNING;
 236
 237         if ((ret = pthread_attr_init(&this_thread->attr))) {
 238                 CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_init: %s\n", strerror(ret));
 239                 free(this_thread);
 240                 return;
 241         }
 242
 243         this_thread->name = "Garbage Collection Thread";
 244
 245         this_thread->tid = GC_thread;
 246         CT = this_thread;
 247
 248         num_threads++;  // Increase the count of threads in the system.
 249
 250         this_thread->next = CtdlThreadList;
 251         CtdlThreadList = this_thread;
 252         if (this_thread->next)
 253                 this_thread->next->prev = this_thread;
 254         /* Set up start times */
 255         gettimeofday(&this_thread->start_time, NULL);           /* Time this thread started */
 256         memcpy(&this_thread->last_state_change, &this_thread->start_time, sizeof (struct timeval));     /* Changed state so mark it. */
 257 }
 258
 259
 260 /*
 261  * A function to update a threads load averages
 262  */
 263  void ctdl_thread_internal_update_avgs(struct CtdlThreadNode *this_thread)
 264  {
 265         struct timeval now, result;
 266         double last_duration;
 267
 268         gettimeofday(&now, NULL);
 269         timersub(&now, &(this_thread->last_state_change), &result);
 270         pthread_mutex_lock(&this_thread->ThreadMutex);
 271         // result now has a timeval for the time we spent in the last state since we last updated
 272         last_duration = (double)result.tv_sec + ((double)result.tv_usec / (double) 1000000);
 273         if (this_thread->state == CTDL_THREAD_SLEEPING)
 274                 this_thread->avg_sleeping += last_duration;
 275         if (this_thread->state == CTDL_THREAD_RUNNING)
 276                 this_thread->avg_running += last_duration;
 277         if (this_thread->state == CTDL_THREAD_BLOCKED)
 278                 this_thread->avg_blocked += last_duration;
 279         memcpy (&this_thread->last_state_change, &now, sizeof (struct timeval));
 280         pthread_mutex_unlock(&this_thread->ThreadMutex);
 281 }
 282
 283 /*
 284  * A function to chenge the state of a thread
 285  */
 286 void ctdl_thread_internal_change_state (struct CtdlThreadNode *this_thread, enum CtdlThreadState new_state)
 287 {
 288         /*
 289          * Wether we change state or not we need update the load values
 290          */
 291         ctdl_thread_internal_update_avgs(this_thread);
 292         pthread_mutex_lock(&this_thread->ThreadMutex); /* To prevent race condition of a sleeping thread */
 293         if ((new_state == CTDL_THREAD_STOP_REQ) && (this_thread->state > CTDL_THREAD_STOP_REQ))
 294                 this_thread->state = new_state;
 295         if (((new_state == CTDL_THREAD_SLEEPING) || (new_state == CTDL_THREAD_BLOCKED)) && (this_thread->state == CTDL_THREAD_RUNNING))
 296                 this_thread->state = new_state;
 297         if ((new_state == CTDL_THREAD_RUNNING) && ((this_thread->state == CTDL_THREAD_SLEEPING) || (this_thread->state == CTDL_THREAD_BLOCKED)))
 298                 this_thread->state = new_state;
 299         pthread_mutex_unlock(&this_thread->ThreadMutex);
 300 }
 301
 302
 303 /*
 304  * A function to tell all threads to exit
 305  */
 306 void CtdlThreadStopAll(void)
 307 {
 308         //FIXME: The signalling of the condition should not be in the critical_section
 309         // We need to build a list of threads we are going to signal and then signal them afterwards
 310
 311         struct CtdlThreadNode *this_thread;
 312
 313         begin_critical_section(S_THREAD_LIST);
 314         this_thread = CtdlThreadList;
 315         while(this_thread)
 316         {
 317 #ifdef THREADS_USESIGNALS
 318                 pthread_kill(this_thread->tid, SIGHUP);
 319 #endif
 320                 ctdl_thread_internal_change_state (this_thread, CTDL_THREAD_STOP_REQ);
 321                 pthread_cond_signal(&this_thread->ThreadCond);
 322                 pthread_cond_signal(&this_thread->SleepCond);
 323                 CtdlLogPrintf(CTDL_DEBUG, "Thread system stopping thread \"%s\" (%ld).\n", this_thread->name, this_thread->tid);
 324                 this_thread = this_thread->next;
 325         }
 326         end_critical_section(S_THREAD_LIST);
 327 }
 328
 329
 330 /*
 331  * A function to wake up all sleeping threads
 332  */
 333 void CtdlThreadWakeAll(void)
 334 {
 335         struct CtdlThreadNode *this_thread;
 336
 337         CtdlLogPrintf(CTDL_DEBUG, "Thread system waking all threads.\n");
 338
 339         begin_critical_section(S_THREAD_LIST);
 340         this_thread = CtdlThreadList;
 341         while(this_thread)
 342         {
 343                 if (!this_thread->thread_func)
 344                 {
 345                         pthread_cond_signal(&this_thread->ThreadCond);
 346                         pthread_cond_signal(&this_thread->SleepCond);
 347                 }
 348                 this_thread = this_thread->next;
 349         }
 350         end_critical_section(S_THREAD_LIST);
 351 }
 352
 353
 354 /*
 355  * A function to return the number of threads running in the system
 356  */
 357 int CtdlThreadGetCount(void)
 358 {
 359         return  num_threads;
 360 }
 361
 362 int CtdlThreadGetWorkers(void)
 363 {
 364         return  num_workers;
 365 }
 366
 367 double CtdlThreadGetWorkerAvg(void)
 368 {
 369         double ret;
 370
 371         begin_critical_section(S_THREAD_LIST);
 372         ret =  CtdlThreadWorkerAvg;
 373         end_critical_section(S_THREAD_LIST);
 374         return ret;
 375 }
 376
 377 double CtdlThreadGetLoadAvg(void)
 378 {
 379         double ret;
 380
 381         begin_critical_section(S_THREAD_LIST);
 382         ret =  CtdlThreadLoadAvg;
 383         end_critical_section(S_THREAD_LIST);
 384         return ret;
 385 }
 386
 387
 388
 389
 390 /*
 391  * A function to rename a thread
 392  * Returns a const char *
 393  */
 394 const char *CtdlThreadName(const char *name)
 395 {
 396         const char *old_name;
 397
 398         if (!CT)
 399         {
 400                 CtdlLogPrintf(CTDL_WARNING, "Thread system WARNING. Attempt to CtdlThreadRename() a non thread. %s\n", name);
 401                 return NULL;
 402         }
 403 // FIXME: do we need this lock? I think not since the pointer asignmaent should be atomic
 404         pthread_mutex_lock(&CT->ThreadMutex);
 405         old_name = CT->name;
 406         if (name)
 407                 CT->name = name;
 408         pthread_mutex_unlock(&CT->ThreadMutex);
 409         return (old_name);
 410 }
 411
 412
 413 /*
 414  * A function to force a thread to exit
 415  */
 416 void CtdlThreadCancel(struct CtdlThreadNode *thread)
 417 {
 418         struct CtdlThreadNode *this_thread;
 419
 420         if (!thread)
 421                 this_thread = CT;
 422         else
 423                 this_thread = thread;
 424         if (!this_thread)
 425         {
 426                 CtdlLogPrintf(CTDL_EMERG, "Thread system PANIC. Attempt to CtdlThreadCancel() a non thread.\n");
 427                 CtdlThreadStopAll();
 428                 return;
 429         }
 430
 431         if (!this_thread->thread_func)
 432         {
 433                 CtdlLogPrintf(CTDL_EMERG, "Thread system PANIC. Attempt to CtdlThreadCancel() the garbage collector.\n");
 434                 CtdlThreadStopAll();
 435                 return;
 436         }
 437
 438         ctdl_thread_internal_change_state (this_thread, CTDL_THREAD_CANCELLED);
 439         pthread_cancel(this_thread->tid);
 440 }
 441
 442
 443
 444 /*
 445  * A function for a thread to check if it has been asked to stop
 446  */
 447 int CtdlThreadCheckStop(void)
 448 {
 449         int state;
 450
 451         if (!CT)
 452         {
 453                 CtdlLogPrintf(CTDL_EMERG, "Thread system PANIC, CtdlThreadCheckStop() called by a non thread.\n");
 454                 CtdlThreadStopAll();
 455                 return -1;
 456         }
 457
 458         state = CT->state;
 459
 460 #ifdef THREADS_USERSIGNALS
 461         if (CT->signal)
 462                 CtdlLogPrintf(CTDL_DEBUG, "Thread \"%s\" caught signal %d.\n", CT->name, CT->signal);
 463 #endif
 464         pthread_mutex_lock(&CT->ThreadMutex);
 465         if(state == CTDL_THREAD_STOP_REQ)
 466         {
 467                 CT->state = CTDL_THREAD_STOPPING;
 468                 pthread_mutex_unlock(&CT->ThreadMutex);
 469                 return -1;
 470         }
 471         else if((state < CTDL_THREAD_STOP_REQ) && (state > CTDL_THREAD_CREATE))
 472         {
 473                 pthread_mutex_unlock(&CT->ThreadMutex);
 474                 return -1;
 475         }
 476         pthread_mutex_unlock(&CT->ThreadMutex);
 477         return 0;
 478 }
 479
 480
 481 /*
 482  * A function to ask a thread to exit
 483  * The thread must call CtdlThreadCheckStop() periodically to determine if it should exit
 484  */
 485 void CtdlThreadStop(struct CtdlThreadNode *thread)
 486 {
 487         struct CtdlThreadNode *this_thread;
 488
 489         if (!thread)
 490                 this_thread = CT;
 491         else
 492                 this_thread = thread;
 493         if (!this_thread)
 494                 return;
 495         if (!(this_thread->thread_func))
 496                 return;         // Don't stop garbage collector
 497 #ifdef THREADS_USESIGNALS
 498         pthread_kill(this_thread->tid, SIGHUP);
 499 #endif
 500         ctdl_thread_internal_change_state (this_thread, CTDL_THREAD_STOP_REQ);
 501         pthread_cond_signal(&this_thread->ThreadCond);
 502         pthread_cond_signal(&this_thread->SleepCond);
 503 }
 504
 505 /*
 506  * So we now have a sleep command that works with threads but it is in seconds
 507  */
 508 void CtdlThreadSleep(int secs)
 509 {
 510         struct timespec wake_time;
 511         struct timeval time_now;
 512
 513
 514         if (!CT)
 515         {
 516                 CtdlLogPrintf(CTDL_WARNING, "CtdlThreadSleep() called by something that is not a thread. Should we die?\n");
 517                 return;
 518         }
 519
 520         memset (&wake_time, 0, sizeof(struct timespec));
 521         gettimeofday(&time_now, NULL);
 522         wake_time.tv_sec = time_now.tv_sec + secs;
 523         wake_time.tv_nsec = time_now.tv_usec * 10;
 524
 525         ctdl_thread_internal_change_state (CT, CTDL_THREAD_SLEEPING);
 526
 527         pthread_mutex_lock(&CT->ThreadMutex); /* Prevent something asking us to awaken before we've gone to sleep */
 528         pthread_cond_timedwait(&CT->SleepCond, &CT->ThreadMutex, &wake_time);
 529         pthread_mutex_unlock(&CT->ThreadMutex);
 530
 531         ctdl_thread_internal_change_state (CT, CTDL_THREAD_RUNNING);
 532 }
 533
 534
 535 /*
 536  * Routine to clean up our thread function on exit
 537  */
 538 static void ctdl_internal_thread_cleanup(void *arg)
 539 {
 540         /*
 541          * In here we were called by the current thread because it is exiting
 542          * NB. WE ARE THE CURRENT THREAD
 543          */
 544         CtdlLogPrintf(CTDL_NOTICE, "Thread \"%s\" (%ld) exited.\n", CT->name, CT->tid);
 545
 546         #ifdef HAVE_BACKTRACE
 547         eCrash_UnregisterThread();
 548         #endif
 549
 550         pthread_mutex_lock(&CT->ThreadMutex);
 551         CT->state = CTDL_THREAD_EXITED; // needs to be last thing else house keeping will unlink us too early
 552         pthread_mutex_unlock(&CT->ThreadMutex);
 553 }
 554
 555 /*
 556  * A quick function to show the load averages
 557  */
 558 void ctdl_thread_internal_calc_loadavg(void)
 559 {
 560         struct CtdlThreadNode *that_thread;
 561         double load_avg, worker_avg;
 562         int workers = 0;
 563
 564         that_thread = CtdlThreadList;
 565         load_avg = 0;
 566         worker_avg = 0;
 567         while(that_thread)
 568         {
 569                 /* Update load averages */
 570                 ctdl_thread_internal_update_avgs(that_thread);
 571                 pthread_mutex_lock(&that_thread->ThreadMutex);
 572                 that_thread->load_avg = that_thread->avg_sleeping + that_thread->avg_running + that_thread->avg_blocked;
 573                 that_thread->load_avg = that_thread->avg_running / that_thread->load_avg * 100;
 574                 that_thread->avg_sleeping /= 2;
 575                 that_thread->avg_running /= 2;
 576                 that_thread->avg_blocked /= 2;
 577                 load_avg += that_thread->load_avg;
 578                 if (that_thread->flags & CTDLTHREAD_WORKER)
 579                 {
 580                         worker_avg += that_thread->load_avg;
 581                         workers++;
 582                 }
 583 #ifdef WITH_THREADLOG
 584                 CtdlLogPrintf(CTDL_DEBUG, "CtdlThread, \"%s\" (%ld) \"%s\" %f %f %f %f.\n",
 585                         that_thread->name,
 586                         that_thread->tid,
 587                         CtdlThreadStates[that_thread->state],
 588                         that_thread->avg_sleeping,
 589                         that_thread->avg_running,
 590                         that_thread->avg_blocked,
 591                         that_thread->load_avg);
 592 #endif
 593                 pthread_mutex_unlock(&that_thread->ThreadMutex);
 594                 that_thread = that_thread->next;
 595         }
 596         CtdlThreadLoadAvg = load_avg/num_threads;
 597         CtdlThreadWorkerAvg = worker_avg/workers;
 598 #ifdef WITH_THREADLOG
 599         CtdlLogPrintf(CTDL_INFO, "System load average %f, workers averag %f, threads %d, workers %d, sessions %d\n", CtdlThreadLoadAvg, CtdlThreadWorkerAvg, num_threads, num_workers, num_sessions);
 600 #endif
 601 }
 602
 603
 604 /*
 605  * Garbage collection routine.
 606  * Gets called by main() in a loop to clean up the thread list periodically.
 607  */
 608 void CtdlThreadGC (void)
 609 {
 610         struct CtdlThreadNode *this_thread, *that_thread;
 611         int workers = 0;
 612         int ret=0;
 613
 614         begin_critical_section(S_THREAD_LIST);
 615
 616         /* Handle exiting of garbage collector thread */
 617         if(num_threads == 1)
 618                 CtdlThreadList->state = CTDL_THREAD_EXITED;
 619
 620 #ifdef WITH_THREADLOG
 621         CtdlLogPrintf(CTDL_DEBUG, "Thread system running garbage collection.\n");
 622 #endif
 623         /*
 624          * Woke up to do garbage collection
 625          */
 626         this_thread = CtdlThreadList;
 627         while(this_thread)
 628         {
 629                 that_thread = this_thread;
 630                 this_thread = this_thread->next;
 631
 632                 /* Do we need to clean up this thread? */
 633                 pthread_mutex_lock(&that_thread->ThreadMutex);
 634                 if (that_thread->state != CTDL_THREAD_EXITED)
 635                 {
 636                         if(that_thread->flags & CTDLTHREAD_WORKER)
 637                                 workers++;      /* Sanity check on number of worker threads */
 638                         pthread_mutex_unlock(&that_thread->ThreadMutex);
 639                         continue;
 640                 }
 641
 642                 if (pthread_equal(that_thread->tid, pthread_self()) && that_thread->thread_func)
 643                 {       /* Sanity check */
 644                         pthread_mutex_unlock(&that_thread->ThreadMutex);
 645                         end_critical_section(S_THREAD_LIST);
 646                         CtdlLogPrintf(CTDL_EMERG, "Thread system PANIC, a thread is trying to clean up after itself.\n");
 647                         abort();
 648                         return;
 649                 }
 650
 651                 if (num_threads <= 0)
 652                 {       /* Sanity check */
 653                         pthread_mutex_unlock(&that_thread->ThreadMutex);
 654                         end_critical_section(S_THREAD_LIST);
 655                         CtdlLogPrintf(CTDL_EMERG, "Thread system PANIC, num_threads <= 0 and trying to do Garbage Collection.\n");
 656                         abort();
 657                         return;
 658                 }
 659
 660                 if(that_thread->flags & CTDLTHREAD_WORKER)
 661                         num_workers--;  /* This is a wroker thread so reduce the count. */
 662                 num_threads--;
 663                 /* If we are unlinking the list head then the next becomes the list head */
 664                 if(that_thread->prev)
 665                         that_thread->prev->next = that_thread->next;
 666                 else
 667                         CtdlThreadList = that_thread->next;
 668                 if(that_thread->next)
 669                         that_thread->next->prev = that_thread->prev;
 670
 671                 pthread_mutex_unlock(&that_thread->ThreadMutex);
 672                 pthread_cond_signal(&that_thread->ThreadCond);
 673                 pthread_cond_signal(&that_thread->SleepCond);   // Make sure this thread is awake
 674                 pthread_mutex_lock(&that_thread->ThreadMutex);  // Make sure it has done what its doing
 675                 pthread_mutex_unlock(&that_thread->ThreadMutex);
 676                 /*
 677                  * Join on the thread to do clean up and prevent memory leaks
 678                  * Also makes sure the thread has cleaned up after itself before we remove it from the list
 679                  * We can join on the garbage collector thread the join should just return EDEADLCK
 680                  */
 681                 ret = pthread_join (that_thread->tid, NULL);
 682                 if (ret == EDEADLK)
 683                         CtdlLogPrintf(CTDL_DEBUG, "Garbage collection on own thread.\n");
 684                 else if (ret == EINVAL)
 685                         CtdlLogPrintf(CTDL_DEBUG, "Garbage collection, that thread already joined on.\n");
 686                 else if (ret == ESRCH)
 687                         CtdlLogPrintf(CTDL_DEBUG, "Garbage collection, no thread to join on.\n");
 688                 else if (ret != 0)
 689                         CtdlLogPrintf(CTDL_DEBUG, "Garbage collection, pthread_join returned an unknown error.\n");
 690                 /*
 691                  * Now we own that thread entry
 692                  */
 693                 CtdlLogPrintf(CTDL_INFO, "Garbage Collection for thread \"%s\" (%ld).\n", that_thread->name, that_thread->tid);
 694                 pthread_mutex_destroy(&that_thread->ThreadMutex);
 695                 pthread_cond_destroy(&that_thread->ThreadCond);
 696                 pthread_mutex_destroy(&that_thread->SleepMutex);
 697                 pthread_cond_destroy(&that_thread->SleepCond);
 698                 pthread_attr_destroy(&that_thread->attr);
 699                 free(that_thread);
 700         }
 701
 702         /* Sanity check number of worker threads */
 703         if (workers != num_workers)
 704         {
 705                 end_critical_section(S_THREAD_LIST);
 706                 CtdlLogPrintf(CTDL_EMERG,
 707                         "Thread system PANIC, discrepancy in number of worker threads. Counted %d, should be %d.\n",
 708                         workers, num_workers
 709                         );
 710                 abort();
 711         }
 712         end_critical_section(S_THREAD_LIST);
 713 }
 714
 715
 716
 717
 718 /*
 719  * Runtime function for a Citadel Thread.
 720  * This initialises the threads environment and then calls the user supplied thread function
 721  * Note that this is the REAL thread function and wraps the users thread function.
 722  */
 723 static void *ctdl_internal_thread_func (void *arg)
 724 {
 725         struct CtdlThreadNode *this_thread;
 726         void *ret = NULL;
 727
 728         /* lock and unlock the thread list.
 729          * This causes this thread to wait until all its creation stuff has finished before it
 730          * can continue its execution.
 731          */
 732         begin_critical_section(S_THREAD_LIST);
 733         this_thread = (struct CtdlThreadNode *) arg;
 734         gettimeofday(&this_thread->start_time, NULL);           /* Time this thread started */
 735         pthread_mutex_lock(&this_thread->ThreadMutex);
 736
 737         // Register the cleanup function to take care of when we exit.
 738         pthread_cleanup_push(ctdl_internal_thread_cleanup, NULL);
 739         // Get our thread data structure
 740         CtdlThreadAllocTSD();
 741         CT = this_thread;
 742         this_thread->pid = getpid();
 743         memcpy(&this_thread->last_state_change, &this_thread->start_time, sizeof (struct timeval));     /* Changed state so mark it. */
 744         /* Only change to running state if we weren't asked to stop during the create cycle
 745          * Other wise there is a window to allow this threads creation to continue to full grown and
 746          * therby prevent a shutdown of the server.
 747          */
 748         pthread_mutex_unlock(&this_thread->ThreadMutex);
 749
 750         if (!CtdlThreadCheckStop())
 751         {
 752                 pthread_mutex_lock(&this_thread->ThreadMutex);
 753                 this_thread->state = CTDL_THREAD_RUNNING;
 754                 pthread_mutex_unlock(&this_thread->ThreadMutex);
 755         }
 756         end_critical_section(S_THREAD_LIST);
 757
 758         // Register for tracing
 759         #ifdef HAVE_BACKTRACE
 760         eCrash_RegisterThread(this_thread->name, 0);
 761         #endif
 762
 763         // Tell the world we are here
 764         CtdlLogPrintf(CTDL_NOTICE, "Created a new thread \"%s\" (%ld). \n", this_thread->name, this_thread->tid);
 765
 766
 767
 768         /*
 769          * run the thread to do the work but only if we haven't been asked to stop
 770          */
 771         if (!CtdlThreadCheckStop())
 772                 ret = (this_thread->thread_func)(this_thread->user_args);
 773
 774         /*
 775          * Our thread is exiting either because it wanted to end or because the server is stopping
 776          * We need to clean up
 777          */
 778         pthread_cleanup_pop(1); // Execute our cleanup routine and remove it
 779
 780         return(ret);
 781 }
 782
 783
 784
 785 /*
 786  * Internal function to create a thread.
 787  * Must be called from within a S_THREAD_LIST critical section
 788  */
 789 struct CtdlThreadNode *ctdl_internal_create_thread(char *name, long flags, void *(*thread_func) (void *arg), void *args)
 790 {
 791         int ret = 0;
 792         struct CtdlThreadNode *this_thread;
 793
 794         if (num_threads >= 32767)
 795         {
 796                 CtdlLogPrintf(CTDL_EMERG, "Thread system. Thread list full.\n");
 797                 return NULL;
 798         }
 799
 800         this_thread = malloc(sizeof(struct CtdlThreadNode));
 801         if (this_thread == NULL) {
 802                 CtdlLogPrintf(CTDL_EMERG, "Thread system, can't allocate CtdlThreadNode, exiting\n");
 803                 return NULL;
 804         }
 805         // Ensuring this is zero'd means we make sure the thread doesn't start doing its thing until we are ready.
 806         memset (this_thread, 0, sizeof(struct CtdlThreadNode));
 807
 808         /* Create the mutex's early so we can use them */
 809         pthread_mutex_init (&(this_thread->ThreadMutex), NULL);
 810         pthread_cond_init (&(this_thread->ThreadCond), NULL);
 811         pthread_mutex_init (&(this_thread->SleepMutex), NULL);
 812         pthread_cond_init (&(this_thread->SleepCond), NULL);
 813
 814         pthread_mutex_lock(&this_thread->ThreadMutex);
 815
 816         this_thread->state = CTDL_THREAD_CREATE;
 817
 818         if ((ret = pthread_attr_init(&this_thread->attr))) {
 819                 pthread_mutex_unlock(&this_thread->ThreadMutex);
 820                 pthread_mutex_destroy(&(this_thread->ThreadMutex));
 821                 pthread_cond_destroy(&(this_thread->ThreadCond));
 822                 pthread_mutex_destroy(&(this_thread->SleepMutex));
 823                 pthread_cond_destroy(&(this_thread->SleepCond));
 824                 CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_init: %s\n", strerror(ret));
 825                 free(this_thread);
 826                 return NULL;
 827         }
 828
 829         /* Our per-thread stacks need to be bigger than the default size,
 830          * otherwise the MIME parser crashes on FreeBSD, and the IMAP service
 831          * crashes on 64-bit Linux.
 832          */
 833         if (flags & CTDLTHREAD_BIGSTACK)
 834         {
 835 #ifdef WITH_THREADLOG
 836                 CtdlLogPrintf(CTDL_INFO, "Thread system. Creating BIG STACK thread.\n");
 837 #endif
 838                 if ((ret = pthread_attr_setstacksize(&this_thread->attr, THREADSTACKSIZE))) {
 839                         pthread_mutex_unlock(&this_thread->ThreadMutex);
 840                         pthread_mutex_destroy(&(this_thread->ThreadMutex));
 841                         pthread_cond_destroy(&(this_thread->ThreadCond));
 842                         pthread_mutex_destroy(&(this_thread->SleepMutex));
 843                         pthread_cond_destroy(&(this_thread->SleepCond));
 844                         pthread_attr_destroy(&this_thread->attr);
 845                         CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_setstacksize: %s\n",
 846                                 strerror(ret));
 847                         free(this_thread);
 848                         return NULL;
 849                 }
 850         }
 851
 852         /*
 853          * If we got here we are going to create the thread so we must initilise the structure
 854          * first because most implimentations of threading can't create it in a stopped state
 855          * and it might want to do things with its structure that aren't initialised otherwise.
 856          */
 857         if(name)
 858         {
 859                 this_thread->name = name;
 860         }
 861         else
 862         {
 863                 this_thread->name = "Un-named Thread";
 864         }
 865
 866         this_thread->flags = flags;
 867         this_thread->thread_func = thread_func;
 868         this_thread->user_args = args;
 869         /* Set this new thread with an avg_blocked of 2. We do this so that its creation affects the
 870          * load average for the system. If we don't do this then we create a mass of threads at the same time
 871          * because the creation didn't affect the load average.
 872          */
 873         this_thread->avg_blocked = 2;
 874
 875         /*
 876          * We pass this_thread into the thread as its args so that it can find out information
 877          * about itself and it has a bit of storage space for itself, not to mention that the REAL
 878          * thread function needs to finish off the setup of the structure
 879          */
 880         if ((ret = pthread_create(&this_thread->tid, &this_thread->attr, ctdl_internal_thread_func, this_thread) != 0))
 881         {
 882
 883                 CtdlLogPrintf(CTDL_ALERT, "Thread system, Can't create thread: %s\n",
 884                         strerror(ret));
 885                 pthread_mutex_unlock(&this_thread->ThreadMutex);
 886                 pthread_mutex_destroy(&(this_thread->ThreadMutex));
 887                 pthread_cond_destroy(&(this_thread->ThreadCond));
 888                 pthread_mutex_destroy(&(this_thread->SleepMutex));
 889                 pthread_cond_destroy(&(this_thread->SleepCond));
 890                 pthread_attr_destroy(&this_thread->attr);
 891                 free(this_thread);
 892                 return NULL;
 893         }
 894
 895         num_threads++;  // Increase the count of threads in the system.
 896         if(this_thread->flags & CTDLTHREAD_WORKER)
 897                 num_workers++;
 898
 899         this_thread->next = CtdlThreadList;
 900         CtdlThreadList = this_thread;
 901         if (this_thread->next)
 902                 this_thread->next->prev = this_thread;
 903
 904         pthread_mutex_unlock(&this_thread->ThreadMutex);
 905
 906         ctdl_thread_internal_calc_loadavg();
 907         return this_thread;
 908 }
 909
 910 /*
 911  * Wrapper function to create a thread
 912  * ensures the critical section and other protections are in place.
 913  * char *name = name to give to thread, if NULL, use generic name
 914  * int flags = flags to determine type of thread and standard facilities
 915  */
 916 struct CtdlThreadNode *CtdlThreadCreate(char *name, long flags, void *(*thread_func) (void *arg), void *args)
 917 {
 918         struct CtdlThreadNode *ret = NULL;
 919
 920         begin_critical_section(S_THREAD_LIST);
 921         ret = ctdl_internal_create_thread(name, flags, thread_func, args);
 922         end_critical_section(S_THREAD_LIST);
 923         return ret;
 924 }
 925
 926
 927
 928 /*
 929  * Internal function to schedule a thread.
 930  * Must be called from within a S_THREAD_LIST critical section
 931  */
 932 struct CtdlThreadNode *CtdlThreadSchedule(char *name, long flags, void *(*thread_func) (void *arg), void *args, time_t when)
 933 {
 934         int ret = 0;
 935         struct CtdlThreadNode *this_thread;
 936
 937         if (num_threads >= 32767)
 938         {
 939                 CtdlLogPrintf(CTDL_EMERG, "Thread system. Thread list full.\n");
 940                 return NULL;
 941         }
 942
 943         this_thread = malloc(sizeof(struct CtdlThreadNode));
 944         if (this_thread == NULL) {
 945                 CtdlLogPrintf(CTDL_EMERG, "Thread system, can't allocate CtdlThreadNode, exiting\n");
 946                 return NULL;
 947         }
 948         // Ensuring this is zero'd means we make sure the thread doesn't start doing its thing until we are ready.
 949         memset (this_thread, 0, sizeof(struct CtdlThreadNode));
 950
 951         /* Create the mutex's early so we can use them */
 952         pthread_mutex_init (&(this_thread->ThreadMutex), NULL);
 953         pthread_cond_init (&(this_thread->ThreadCond), NULL);
 954         pthread_mutex_init (&(this_thread->SleepMutex), NULL);
 955         pthread_cond_init (&(this_thread->SleepCond), NULL);
 956
 957         this_thread->state = CTDL_THREAD_CREATE;
 958
 959         if ((ret = pthread_attr_init(&this_thread->attr))) {
 960                 pthread_mutex_destroy(&(this_thread->ThreadMutex));
 961                 pthread_cond_destroy(&(this_thread->ThreadCond));
 962                 pthread_mutex_destroy(&(this_thread->SleepMutex));
 963                 pthread_cond_destroy(&(this_thread->SleepCond));
 964                 CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_init: %s\n", strerror(ret));
 965                 free(this_thread);
 966                 return NULL;
 967         }
 968
 969         /* Our per-thread stacks need to be bigger than the default size,
 970          * otherwise the MIME parser crashes on FreeBSD, and the IMAP service
 971          * crashes on 64-bit Linux.
 972          */
 973         if (flags & CTDLTHREAD_BIGSTACK)
 974         {
 975                 CtdlLogPrintf(CTDL_INFO, "Thread system. Creating BIG STACK thread.\n");
 976                 if ((ret = pthread_attr_setstacksize(&this_thread->attr, THREADSTACKSIZE))) {
 977                         pthread_mutex_destroy(&(this_thread->ThreadMutex));
 978                         pthread_cond_destroy(&(this_thread->ThreadCond));
 979                         pthread_mutex_destroy(&(this_thread->SleepMutex));
 980                         pthread_cond_destroy(&(this_thread->SleepCond));
 981                         pthread_attr_destroy(&this_thread->attr);
 982                         CtdlLogPrintf(CTDL_EMERG, "Thread system, pthread_attr_setstacksize: %s\n",
 983                                 strerror(ret));
 984                         free(this_thread);
 985                         return NULL;
 986                 }
 987         }
 988
 989         /*
 990          * If we got here we are going to create the thread so we must initilise the structure
 991          * first because most implimentations of threading can't create it in a stopped state
 992          * and it might want to do things with its structure that aren't initialised otherwise.
 993          */
 994         if(name)
 995         {
 996                 this_thread->name = name;
 997         }
 998         else
 999         {
1000                 this_thread->name = "Un-named Thread";
1001         }
1002
1003         this_thread->flags = flags;
1004         this_thread->thread_func = thread_func;
1005         this_thread->user_args = args;
1006         /* Set this new thread with an avg_blocked of 2. We do this so that its creation affects the
1007          * load average for the system. If we don't do this then we create a mass of threads at the same time
1008          * because the creation didn't affect the load average.
1009          */
1010         this_thread->avg_blocked = 2;
1011
1012         /*
1013          * When to start this thread
1014          */
1015         this_thread->when = when;
1016
1017         begin_critical_section(S_SCHEDULE_LIST);
1018         this_thread->next = CtdlThreadSchedList;
1019         CtdlThreadSchedList = this_thread;
1020         if (this_thread->next)
1021                 this_thread->next->prev = this_thread;
1022         end_critical_section(S_SCHEDULE_LIST);
1023
1024         return this_thread;
1025 }
1026
1027
1028
1029 struct CtdlThreadNode *ctdl_thread_internal_start_scheduled (struct CtdlThreadNode *this_thread)
1030 {
1031         int ret = 0;
1032
1033         /*
1034          * We pass this_thread into the thread as its args so that it can find out information
1035          * about itself and it has a bit of storage space for itself, not to mention that the REAL
1036          * thread function needs to finish off the setup of the structure
1037          */
1038         if ((ret = pthread_create(&this_thread->tid, &this_thread->attr, ctdl_internal_thread_func, this_thread) != 0))
1039         {
1040
1041                 CtdlLogPrintf(CTDL_ALERT, "Thread system, Can't create thread: %s\n",
1042                         strerror(ret));
1043                 return NULL;
1044         }
1045
1046
1047         num_threads++;  // Increase the count of threads in the system.
1048         if(this_thread->flags & CTDLTHREAD_WORKER)
1049                 num_workers++;
1050
1051         this_thread->next = CtdlThreadList;
1052         CtdlThreadList = this_thread;
1053         if (this_thread->next)
1054                 this_thread->next->prev = this_thread;
1055
1056         return this_thread;
1057 }
1058
1059
1060
1061 void ctdl_thread_internal_check_scheduled(void)
1062 {
1063         struct CtdlThreadNode *this_thread, *that_thread;
1064         time_t now;
1065
1066         if (try_critical_section(S_SCHEDULE_LIST))
1067                 return; /* If this list is locked we wait till the next chance */
1068
1069         now = time(NULL);
1070
1071 #ifdef WITH_THREADLOG
1072         CtdlLogPrintf(CTDL_DEBUG, "Checking for scheduled threads to start.\n");
1073 #endif
1074
1075         this_thread = CtdlThreadSchedList;
1076         while(this_thread)
1077         {
1078                 that_thread = this_thread;
1079                 this_thread = this_thread->next;
1080
1081                 if (now > that_thread->when)
1082                 {
1083                         /* Unlink from schedule list */
1084                         if (that_thread->prev)
1085                                 that_thread->prev->next = that_thread->next;
1086                         else
1087                                 CtdlThreadSchedList = that_thread->next;
1088                         if (that_thread->next)
1089                                 that_thread->next->prev = that_thread->prev;
1090
1091                         that_thread->next = that_thread->prev = NULL;
1092 #ifdef WITH_THREADLOG
1093                         CtdlLogPrintf(CTDL_DEBUG, "About to start scheduled thread \"%s\".\n", that_thread->name);
1094 #endif
1095                         begin_critical_section(S_THREAD_LIST);
1096                         if (CT->state > CTDL_THREAD_STOP_REQ)
1097                         {       /* Only start it if the system is not stopping */
1098                                 pthread_mutex_lock(&that_thread->ThreadMutex);
1099                                 if (ctdl_thread_internal_start_scheduled (that_thread) == NULL)
1100                                 {
1101 #ifdef WITH_THREADLOG
1102                         CtdlLogPrintf(CTDL_DEBUG, "Failed to start scheduled thread \"%s\".\n", that_thread->name);
1103 #endif
1104                                         pthread_mutex_unlock(&that_thread->ThreadMutex);
1105                                         pthread_mutex_destroy(&(that_thread->ThreadMutex));
1106                                         pthread_cond_destroy(&(that_thread->ThreadCond));
1107                                         pthread_mutex_destroy(&(that_thread->SleepMutex));
1108                                         pthread_cond_destroy(&(that_thread->SleepCond));
1109                                         pthread_attr_destroy(&that_thread->attr);
1110                                         free(that_thread);
1111                                 }
1112                                 else
1113                                 {
1114                                         CtdlLogPrintf(CTDL_INFO, "Thread system, Started a scheduled thread \"%s\" (%ld).\n",
1115                                                 that_thread->name, that_thread->tid);
1116                                         pthread_mutex_unlock(&that_thread->ThreadMutex);
1117                                         ctdl_thread_internal_calc_loadavg();
1118                                 }
1119                         }
1120                         end_critical_section(S_THREAD_LIST);
1121                 }
1122                 else
1123                 {
1124 #ifdef WITH_THREADLOG
1125                         CtdlLogPrintf(CTDL_DEBUG, "Thread \"%s\" will start in %ld seconds.\n", that_thread->name, that_thread->when - time(NULL));
1126 #endif
1127                 }
1128         }
1129         end_critical_section(S_SCHEDULE_LIST);
1130 }
1131
1132
1133 /*
1134  * A warapper function for select so we can show a thread as blocked
1135  */
1136 int CtdlThreadSelect(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout)
1137 {
1138         int ret;
1139
1140         ctdl_thread_internal_change_state(CT, CTDL_THREAD_BLOCKED);
1141         ret = select(n, readfds, writefds, exceptfds, timeout);
1142         ctdl_thread_internal_change_state(CT, CTDL_THREAD_RUNNING);
1143         return ret;
1144 }
1145
1146
1147
1148
1149
1150
1151 void go_threading(void)
1152 {
1153         int i;
1154         struct CtdlThreadNode *last_worker;
1155
1156         /*
1157          * Initialise the thread system
1158          */
1159         ctdl_thread_internal_init();
1160         /*
1161          * Now create a bunch of worker threads.
1162          */
1163 //      CtdlLogPrintf(CTDL_DEBUG, "Starting %d worker threads\n", config.c_min_workers);
1164 //      begin_critical_section(S_THREAD_LIST);
1165 //      i=0;    /* Always start at least 1 worker thread */
1166 //      do
1167 //      {
1168 //              ctdl_internal_create_thread("Worker Thread", CTDLTHREAD_BIGSTACK + CTDLTHREAD_WORKER, worker_thread, NULL);
1169 //      } while (++i < config.c_min_workers);
1170 //      end_critical_section(S_THREAD_LIST);
1171
1172         /* Second call to module init functions now that threading is up */
1173         initialise_modules(1);
1174
1175         /*
1176          * This thread is now used for garbage collection of other threads in the thread list
1177          */
1178         CtdlLogPrintf(CTDL_INFO, "Startup thread %d becoming garbage collector,\n", pthread_self());
1179
1180         /*
1181          * We do a lot of locking and unlocking of the thread list in here.
1182          * We do this so that we can repeatedly release time for other threads
1183          * that may be waiting on the thread list.
1184          * We are a low priority thread so we can afford to do this
1185          */
1186
1187         while (CtdlThreadGetCount())
1188         {
1189                 if (CT->signal)
1190                         exit_signal = CT->signal;
1191                 if (exit_signal)
1192                         CtdlThreadStopAll();
1193                 check_sched_shutdown();
1194                 if (CT->state > CTDL_THREAD_STOP_REQ)
1195                 {
1196                         begin_critical_section(S_THREAD_LIST);
1197                         ctdl_thread_internal_calc_loadavg();
1198                         end_critical_section(S_THREAD_LIST);
1199
1200                         ctdl_thread_internal_check_scheduled(); /* start scheduled threads */
1201                 }
1202
1203                 /* Reduce the size of the worker thread pool if necessary. */
1204                 if ((CtdlThreadGetWorkers() > config.c_min_workers) && (CtdlThreadWorkerAvg < 20) && (CT->state > CTDL_THREAD_STOP_REQ))
1205                 {
1206                         /* Ask a worker thread to stop as we no longer need it */
1207                         begin_critical_section(S_THREAD_LIST);
1208                         last_worker = CtdlThreadList;
1209                         while (last_worker)
1210                         {
1211                                 pthread_mutex_lock(&last_worker->ThreadMutex);
1212                                 if (last_worker->flags & CTDLTHREAD_WORKER && last_worker->state > CTDL_THREAD_STOPPING)
1213                                 {
1214                                         pthread_mutex_unlock(&last_worker->ThreadMutex);
1215                                         break;
1216                                 }
1217                                 pthread_mutex_unlock(&last_worker->ThreadMutex);
1218                                 last_worker = last_worker->next;
1219                         }
1220                         end_critical_section(S_THREAD_LIST);
1221                         if (last_worker)
1222                         {
1223 #ifdef WITH_THREADLOG
1224                                 CtdlLogPrintf(CTDL_DEBUG, "Thread system, stopping excess worker thread \"%s\" (%ld).\n",
1225                                         last_worker->name,
1226                                         last_worker->tid
1227                                         );
1228 #endif
1229                                 CtdlThreadStop(last_worker);
1230                         }
1231                 }
1232
1233                 /*
1234                  * If all our workers are working hard, start some more to help out
1235                  * with things
1236                  */
1237                 /* FIXME: come up with a better way to dynamically alter the number of threads
1238                  * based on the system load
1239                  */
1240 //              if ((CtdlThreadGetWorkers() < config.c_max_workers) && (CtdlThreadGetWorkers() < num_sessions))
1241                 // && (CtdlThreadLoadAvg < 90) )
1242                 if ((((CtdlThreadGetWorkers() < config.c_max_workers) && (CtdlThreadGetWorkerAvg() > 60) && (CtdlThreadGetLoadAvg() < 90) ) || CtdlThreadGetWorkers() < config.c_min_workers) && (CT->state > CTDL_THREAD_STOP_REQ))
1243                 {
1244                         for (i=0; i<5 ; i++)
1245 //                      for (i=0; i< (num_sessions - CtdlThreadGetWorkers()) ; i++)
1246 //                      for (i=0; i< (10 - (55 - CtdlThreadWorkerAvg) / CtdlThreadWorkerAvg / CtdlThreadGetWorkers()) ; i++)
1247                         {
1248 //                              begin_critical_section(S_THREAD_LIST);
1249                                 CtdlThreadCreate("Worker Thread",
1250                                         CTDLTHREAD_BIGSTACK + CTDLTHREAD_WORKER,
1251                                         worker_thread,
1252                                         NULL
1253                                         );
1254 //                              end_critical_section(S_THREAD_LIST);
1255                         }
1256                 }
1257
1258                 CtdlThreadGC();
1259
1260                 if (CtdlThreadGetCount() <= 1) // Shutting down clean up the garbage collector
1261                 {
1262                         CtdlThreadGC();
1263                 }
1264
1265                 if (CtdlThreadGetCount())
1266                         CtdlThreadSleep(1);
1267         }
1268         /*
1269          * If the above loop exits we must be shutting down since we obviously have no threads
1270          */
1271         ctdl_thread_internal_cleanup();
1272 }