From 2b402dc292b6dcdbc7a3786a6c722e06e3be5ab5 Mon Sep 17 00:00:00 2001 From: Art Cancro Date: Sun, 20 Mar 2011 23:34:08 -0400 Subject: [PATCH] Began working on the 'clean shutdown' code for the new thread architecture. --- citadel/citserver.c | 4 +- citadel/housekeeping.c | 2 +- citadel/locate_host.c | 6 +-- citadel/modules/expire/serv_expire.c | 18 ++++----- citadel/modules/fulltext/serv_fulltext.c | 6 +-- citadel/modules/migrate/serv_migrate.c | 4 +- citadel/modules/network/serv_network.c | 20 +++++----- citadel/modules/pop3client/serv_pop3client.c | 32 ++++++++++----- citadel/modules/rssclient/serv_rssclient.c | 24 +++++------ citadel/modules/smtp/serv_smtp.c | 4 +- .../urldeshortener/serv_expand_shorter_urls.c | 2 +- citadel/sysdep.c | 28 +++++++------ citadel/threads.c | 40 ++++++------------- citadel/threads.h | 3 +- 14 files changed, 96 insertions(+), 97 deletions(-) diff --git a/citadel/citserver.c b/citadel/citserver.c index cff393ba0..2b70a2667 100644 --- a/citadel/citserver.c +++ b/citadel/citserver.c @@ -817,7 +817,7 @@ void cmd_down(char *argbuf) { cprintf(Reply, CIT_OK + SERVER_SHUTTING_DOWN); } CC->kill_me = KILLME_SERVER_SHUTTING_DOWN; - CtdlThreadStopAll(); + server_shutting_down = 1; } @@ -829,7 +829,7 @@ void cmd_halt(char *argbuf) { if (CtdlAccessCheck(ac_aide)) return; cprintf("%d Halting server. Goodbye.\n", CIT_OK); - CtdlThreadStopAll(); + server_shutting_down = 1; shutdown_and_halt = 1; } diff --git a/citadel/housekeeping.c b/citadel/housekeeping.c index 5773925dd..f4d930d64 100644 --- a/citadel/housekeeping.c +++ b/citadel/housekeeping.c @@ -63,7 +63,7 @@ void check_sched_shutdown(void) { if ((ScheduledShutdown == 1) && (ContextList == NULL)) { syslog(LOG_NOTICE, "Scheduled shutdown initiating.\n"); - CtdlThreadStopAll(); + server_shutting_down = 1; } } diff --git a/citadel/locate_host.c b/citadel/locate_host.c index 20704c927..c3a196369 100644 --- a/citadel/locate_host.c +++ b/citadel/locate_host.c @@ -105,7 +105,7 @@ int rblcheck_backend(char *domain, char *txtbuf, int txtbufsize) { /* Make our DNS query. */ answer = fixedans; - if (CtdlThreadCheckStop()) + if (server_shutting_down) { if (txtbuf != NULL) { snprintf(txtbuf, txtbufsize, "System shutting down"); @@ -136,7 +136,7 @@ int rblcheck_backend(char *domain, char *txtbuf, int txtbufsize) { return(1); } } - if (CtdlThreadCheckStop()) + if (server_shutting_down) { if (txtbuf != NULL) snprintf(txtbuf, txtbufsize, "System shutting down"); @@ -153,7 +153,7 @@ int rblcheck_backend(char *domain, char *txtbuf, int txtbufsize) { * nameserver we're using. */ len = res_query(domain, C_IN, T_TXT, answer, PACKETSZ); - if (CtdlThreadCheckStop()) + if (server_shutting_down) { if (txtbuf != NULL) { snprintf(txtbuf, txtbufsize, "System shutting down"); diff --git a/citadel/modules/expire/serv_expire.c b/citadel/modules/expire/serv_expire.c index 70ddb316c..c92ae6d80 100644 --- a/citadel/modules/expire/serv_expire.c +++ b/citadel/modules/expire/serv_expire.c @@ -876,55 +876,55 @@ void purge_databases(void) syslog(LOG_INFO, "Auto-purger: starting.\n"); - if (!CtdlThreadCheckStop()) + if (!server_shutting_down) { retval = PurgeUsers(); syslog(LOG_NOTICE, "Purged %d users.\n", retval); } - if (!CtdlThreadCheckStop()) + if (!server_shutting_down) { PurgeMessages(); syslog(LOG_NOTICE, "Expired %d messages.\n", messages_purged); } - if (!CtdlThreadCheckStop()) + if (!server_shutting_down) { retval = PurgeRooms(); syslog(LOG_NOTICE, "Expired %d rooms.\n", retval); } - if (!CtdlThreadCheckStop()) + if (!server_shutting_down) { retval = PurgeVisits(); syslog(LOG_NOTICE, "Purged %d visits.\n", retval); } - if (!CtdlThreadCheckStop()) + if (!server_shutting_down) { retval = PurgeUseTable(); syslog(LOG_NOTICE, "Purged %d entries from the use table.\n", retval); } - if (!CtdlThreadCheckStop()) + if (!server_shutting_down) { retval = PurgeEuidIndexTable(); syslog(LOG_NOTICE, "Purged %d entries from the EUID index.\n", retval); } - if (!CtdlThreadCheckStop()) + if (!server_shutting_down) { retval = PurgeStaleOpenIDassociations(); syslog(LOG_NOTICE, "Purged %d stale OpenID associations.\n", retval); } - if (!CtdlThreadCheckStop()) + if (!server_shutting_down) { retval = TDAP_ProcessAdjRefCountQueue(); syslog(LOG_NOTICE, "Processed %d message reference count adjustments.\n", retval); } - if (!CtdlThreadCheckStop()) + if (!server_shutting_down) { syslog(LOG_INFO, "Auto-purger: finished.\n"); last_purge = now; /* So we don't do it again soon */ diff --git a/citadel/modules/fulltext/serv_fulltext.c b/citadel/modules/fulltext/serv_fulltext.c index bac3e90b9..db0edbdd1 100644 --- a/citadel/modules/fulltext/serv_fulltext.c +++ b/citadel/modules/fulltext/serv_fulltext.c @@ -228,7 +228,7 @@ void ft_index_msg(long msgnum, void *userdata) { */ void ft_index_room(struct ctdlroom *qrbuf, void *data) { - if (CtdlThreadCheckStop()) + if (server_shutting_down) return; CtdlGetRoom(&CC->room, qrbuf->QRname); @@ -321,7 +321,7 @@ void do_fulltext_indexing(void) { ft_index_message(ft_newmsgs[i], 1); /* Check to see if we need to quit early */ - if (CtdlThreadCheckStop()) { + if (server_shutting_down) { syslog(LOG_DEBUG, "Indexer quitting early\n"); ft_newhighest = ft_newmsgs[i]; break; @@ -343,7 +343,7 @@ void do_fulltext_indexing(void) { } end_time = time(NULL); - if (CtdlThreadCheckStop()) { + if (server_shutting_down) { is_running = 0; return; } diff --git a/citadel/modules/migrate/serv_migrate.c b/citadel/modules/migrate/serv_migrate.c index 10404772b..091822c38 100644 --- a/citadel/modules/migrate/serv_migrate.c +++ b/citadel/modules/migrate/serv_migrate.c @@ -831,7 +831,7 @@ void migr_xml_end(void *data, const char *el) { msglist = NULL; msglist_alloc = 0; syslog(LOG_DEBUG, "Imported %d messages.\n", msgcount); - if (CtdlThreadCheckStop()) { + if (server_shutting_down) { return; } } @@ -932,7 +932,7 @@ void migr_do_import(void) { linelen = strlen(buf); strcpy(&buf[linelen++], "\n"); - if (CtdlThreadCheckStop()) + if (server_shutting_down) break; // Should we break or return? if (buf[0] == '\0') diff --git a/citadel/modules/network/serv_network.c b/citadel/modules/network/serv_network.c index 99d7f6a28..4518740bf 100644 --- a/citadel/modules/network/serv_network.c +++ b/citadel/modules/network/serv_network.c @@ -1986,7 +1986,7 @@ void receive_spool(int *sock, char *remote_nodename) { * If shutting down we can exit here and unlink the temp file. * this shouldn't loose us any messages. */ - if (CtdlThreadCheckStop()) + if (server_shutting_down) { fclose(fp); unlink(tempfilename); @@ -2028,7 +2028,7 @@ void receive_spool(int *sock, char *remote_nodename) { fclose(fp); /* Last chance for shutdown exit */ - if (CtdlThreadCheckStop()) + if (server_shutting_down) { unlink(tempfilename); return; @@ -2099,7 +2099,7 @@ void transmit_spool(int *sock, char *remote_nodename) bytes_to_write = plen; while (bytes_to_write > 0L) { /* Exit if shutting down */ - if (CtdlThreadCheckStop()) + if (server_shutting_down) { close(fd); return; @@ -2132,7 +2132,7 @@ ABORTUPL: close(fd); /* Last chance for shutdown exit */ - if(CtdlThreadCheckStop()) + if(server_shutting_down) return; if (sock_puts(sock, "UCLS 1") < 0) return; @@ -2205,9 +2205,9 @@ void network_poll_node(char *node, char *secret, char *host, char *port) { } /* At this point we are authenticated. */ - if (!CtdlThreadCheckStop()) + if (!server_shutting_down) receive_spool(&sock, node); - if (!CtdlThreadCheckStop()) + if (!server_shutting_down) transmit_spool(&sock, node); } @@ -2244,7 +2244,7 @@ void network_poll_other_citadel_nodes(int full_poll) { /* Use the string tokenizer to grab one line at a time */ for (i=0; iname, sizeof spoolroomname); begin_critical_section(S_RPLIST); @@ -2386,7 +2386,7 @@ void network_do_queue(void) { } /* If there is anything in the inbound queue, process it */ - if (!CtdlThreadCheckStop()) { + if (!server_shutting_down) { network_do_spoolin(); } diff --git a/citadel/modules/pop3client/serv_pop3client.c b/citadel/modules/pop3client/serv_pop3client.c index 83ddf64cc..6adaf1549 100644 --- a/citadel/modules/pop3client/serv_pop3client.c +++ b/citadel/modules/pop3client/serv_pop3client.c @@ -174,6 +174,7 @@ eNextState FailAggregationRun(AsyncIO *IO) return eAbort; } + #define POP3C_DBG_SEND() syslog(LOG_DEBUG, "POP3 client[%ld]: > %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.SendBuf.Buf)) #define POP3C_DBG_READ() syslog(LOG_DEBUG, "POP3 client[%ld]: < %s\n", RecvMsg->n, ChrPtr(RecvMsg->IO.IOBuf)) #define POP3C_OK (strncasecmp(ChrPtr(RecvMsg->IO.IOBuf), "+OK", 3) == 0) @@ -186,7 +187,6 @@ eNextState POP3C_ReadGreeting(pop3aggr *RecvMsg) else return eSendReply; } - eNextState POP3C_SendUser(pop3aggr *RecvMsg) { /* Identify ourselves. NOTE: we have to append a CR to each command. The LF will @@ -301,7 +301,9 @@ eNextState POP3_FetchNetworkUsetableEntry(AsyncIO *IO) if(GetNextHashPos(RecvMsg->MsgNumbers, RecvMsg->Pos, &HKLen, &HKey, &vData)) { struct UseTable ut; - + if (server_shutting_down) + return eAbort; + RecvMsg->CurrMsg = (FetchItem*) vData; syslog(LOG_DEBUG, "CHECKING: whether %s has already been seen: ", ChrPtr(RecvMsg->CurrMsg->MsgUID)); /* Find out if we've already seen this item */ @@ -890,13 +892,15 @@ void pop3client_scan_room(struct ctdlroom *qrbuf, void *data) qrbuf->QRnumber, qrbuf->QRname); pthread_mutex_unlock(&POP3QueueMutex); - return; } pthread_mutex_unlock(&POP3QueueMutex); + if (server_shutting_down) + return; + assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir); - if (CtdlThreadCheckStop()) + if (server_shutting_down) return; /* Only do net processing for rooms that have netconfigs */ @@ -905,14 +909,14 @@ void pop3client_scan_room(struct ctdlroom *qrbuf, void *data) //syslog(LOG_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname); return; } - if (CtdlThreadCheckStop()) + if (server_shutting_down) return; if (fstat(fd, &statbuf) == -1) { syslog(LOG_DEBUG, "ERROR: could not stat configfile '%s' - %s\n", filename, strerror(errno)); return; } - if (CtdlThreadCheckStop()) + if (server_shutting_down) return; CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1); if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) { @@ -923,7 +927,7 @@ void pop3client_scan_room(struct ctdlroom *qrbuf, void *data) return; } close(fd); - if (CtdlThreadCheckStop()) + if (server_shutting_down) return; CfgPtr = NULL; @@ -1062,15 +1066,25 @@ void pop3client_scan(void) { syslog(LOG_DEBUG, "pop3client started\n"); CtdlForEachRoom(pop3client_scan_room, NULL); - pthread_mutex_lock(&POP3QueueMutex); it = GetNewHashPos(POP3FetchUrls, 0); - while (GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) && + while (!server_shutting_down && + GetNextHashPos(POP3FetchUrls, it, &len, &Key, &vrptr) && (vrptr != NULL)) { cptr = (pop3aggr *)vrptr; if (cptr->RefCount == 0) if (!pop3_do_fetching(cptr)) DeletePOP3Aggregator(cptr);////TODO + +/* + if ((palist->interval && time(NULL) > (last_run + palist->interval)) + || (time(NULL) > last_run + config.c_pop3_fetch)) + pop3_do_fetching(palist->roomname, palist->pop3host, + palist->pop3user, palist->pop3pass, palist->keep); + pptr = palist; + palist = palist->next; + free(pptr); +*/ } DeleteHashPos(&it); pthread_mutex_unlock(&POP3QueueMutex); diff --git a/citadel/modules/rssclient/serv_rssclient.c b/citadel/modules/rssclient/serv_rssclient.c index 7b49ec5a6..67c325a47 100644 --- a/citadel/modules/rssclient/serv_rssclient.c +++ b/citadel/modules/rssclient/serv_rssclient.c @@ -137,6 +137,9 @@ void UnlinkRooms(rss_aggregator *Cfg) long *lData = (long*) vData; DeleteRoomReference(*lData); } +/* + if (server_shutting_down) + break; /* TODO */ DeleteHashPos(&At); } @@ -451,7 +454,6 @@ int rss_do_fetching(rss_aggregator *Cfg) } - void DeleteRssCfg(void *vptr) { rss_aggregator *rncptr = (rss_aggregator *)vptr; @@ -545,7 +547,7 @@ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data) assoc_file_name(filename, sizeof filename, qrbuf, ctdl_netcfg_dir); - if (CtdlThreadCheckStop()) + if (server_shutting_down) return; /* Only do net processing for rooms that have netconfigs */ @@ -554,15 +556,10 @@ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data) //syslog(LOG_DEBUG, "rssclient: %s no config.\n", qrbuf->QRname); return; } - if (CtdlThreadCheckStop()) - return; - if (fstat(fd, &statbuf) == -1) { - syslog(LOG_DEBUG, "ERROR: could not stat configfile '%s' - %s\n", - filename, strerror(errno)); - return; - } - if (CtdlThreadCheckStop()) - return; + + if (server_shutting_down) + return + CfgData = NewStrBufPlain(NULL, statbuf.st_size + 1); if (StrBufReadBLOB(CfgData, &fd, 1, statbuf.st_size, &Err) < 0) { close(fd); @@ -572,7 +569,7 @@ void rssclient_scan_room(struct ctdlroom *qrbuf, void *data) return; } close(fd); - if (CtdlThreadCheckStop()) + if (server_shutting_down) return; CfgPtr = NULL; @@ -686,7 +683,8 @@ void rssclient_scan(void) { pthread_mutex_lock(&RSSQueueMutex); it = GetNewHashPos(RSSFetchUrls, 0); - while (GetNextHashPos(RSSFetchUrls, it, &len, &Key, &vrptr) && + while (!server_shutting_down && + GetNextHashPos(RSSFetchUrls, it, &len, &Key, &vrptr) && (vrptr != NULL)) { rptr = (rss_aggregator *)vrptr; if (rptr->RefCount == 0) diff --git a/citadel/modules/smtp/serv_smtp.c b/citadel/modules/smtp/serv_smtp.c index e2d9766a6..885bf950f 100644 --- a/citadel/modules/smtp/serv_smtp.c +++ b/citadel/modules/smtp/serv_smtp.c @@ -131,7 +131,7 @@ void smtp_greeting(int is_msa) */ if ( (config.c_rbl_at_greeting) && (sSMTP->is_msa == 0) ) { if (rbl_check(message_to_spammer)) { - if (CtdlThreadCheckStop()) + if (server_shutting_down) cprintf("421 %s\r\n", message_to_spammer); else cprintf("550 %s\r\n", message_to_spammer); @@ -584,7 +584,7 @@ void smtp_rcpt(char *argbuf) { && (!sSMTP->is_lmtp) ) { /* Don't RBL LMTP clients */ if (config.c_rbl_at_greeting == 0) { /* Don't RBL again if we already did it */ if (rbl_check(message_to_spammer)) { - if (CtdlThreadCheckStop()) + if (server_shutting_down) cprintf("421 %s\r\n", message_to_spammer); else cprintf("550 %s\r\n", message_to_spammer); diff --git a/citadel/modules/urldeshortener/serv_expand_shorter_urls.c b/citadel/modules/urldeshortener/serv_expand_shorter_urls.c index 0bc442f57..6b038bec1 100644 --- a/citadel/modules/urldeshortener/serv_expand_shorter_urls.c +++ b/citadel/modules/urldeshortener/serv_expand_shorter_urls.c @@ -139,7 +139,7 @@ int LookupUrl(StrBuf *ShorterUrlStr) OPT(WRITEHEADER, ShorterUrlStr); - if (CtdlThreadCheckStop()) + if (server_shutting_down) goto shutdown ; evcurl_handle_start(IO); diff --git a/citadel/sysdep.c b/citadel/sysdep.c index ee75bb9cb..ea89f5ac5 100644 --- a/citadel/sysdep.c +++ b/citadel/sysdep.c @@ -101,7 +101,7 @@ volatile int running_as_daemon = 0; static RETSIGTYPE signal_cleanup(int signum) { syslog(LOG_DEBUG, "Caught signal %d; shutting down.", signum); exit_signal = signum; -/// server_shutting_down = 1; + server_shutting_down = 1; } static RETSIGTYPE signal_exit(int signum) { @@ -485,7 +485,7 @@ int client_write(const char *buf, int nbytes) syslog(LOG_DEBUG, "client_write(%d bytes) select() interrupted.", nbytes-bytes_written ); - if (CtdlThreadCheckStop()) { + if (server_shutting_down) { CC->kill_me = KILLME_SELECT_INTERRUPTED; return (-1); } else { @@ -1145,7 +1145,7 @@ void *worker_thread(void *blah) { ++num_workers; - while (!CtdlThreadCheckStop()) { + while (!server_shutting_down) { /* make doubly sure we're not holding any stale db handles * which might cause a deadlock. @@ -1194,7 +1194,7 @@ do_select: force_purge = 0; * ahead and get ready to select(). */ - if (!CtdlThreadCheckStop()) { + if (!server_shutting_down) { tv.tv_sec = 1; /* wake up every second if no input */ tv.tv_usec = 0; retval = select(highest + 1, &readfds, NULL, NULL, &tv); @@ -1213,18 +1213,18 @@ do_select: force_purge = 0; } if (errno != EINTR) { syslog(LOG_EMERG, "Exiting (%s)\n", strerror(errno)); - CtdlThreadStopAll(); + server_shutting_down = 1; continue; } else { #if 0 syslog(LOG_DEBUG, "Interrupted select()\n"); #endif - if (CtdlThreadCheckStop()) return(NULL); + if (server_shutting_down) return(NULL); goto do_select; } } else if (retval == 0) { - if (CtdlThreadCheckStop()) return(NULL); + if (server_shutting_down) return(NULL); } /* It must be a client socket. Find a context that has data @@ -1297,7 +1297,9 @@ SKIP_SELECT: do_housekeeping(); --active_workers; } - /* If control reaches this point, the server is shutting down */ + + /* If control reaches this point, the server is shutting down */ + --num_workers; return(NULL); } @@ -1325,7 +1327,7 @@ void *select_on_master(void *blah) CtdlFillSystemContext(&select_on_master_CC, "select_on_master"); pthread_setspecific(MyConKey, (void *)&select_on_master_CC); - while (!CtdlThreadCheckStop()) { + while (!server_shutting_down) { /* Initialize the fdset. */ FD_ZERO(&master_fds); highest = 0; @@ -1340,7 +1342,7 @@ void *select_on_master(void *blah) } } - if (!CtdlThreadCheckStop()) { + if (!server_shutting_down) { tv.tv_sec = 60; /* wake up every second if no input */ tv.tv_usec = 0; retval = select(highest + 1, &master_fds, NULL, NULL, &tv); @@ -1359,17 +1361,17 @@ void *select_on_master(void *blah) } if (errno != EINTR) { syslog(LOG_EMERG, "Exiting (%s)\n", strerror(errno)); - CtdlThreadStopAll(); + server_shutting_down = 1; } else { #if 0 syslog(LOG_DEBUG, "Interrupted CtdlThreadSelect.\n"); #endif - if (CtdlThreadCheckStop()) return(NULL); + if (server_shutting_down) return(NULL); continue; } } else if(retval == 0) { - if (CtdlThreadCheckStop()) return(NULL); + if (server_shutting_down) return(NULL); continue; } /* Next, check to see if it's a new client connecting diff --git a/citadel/threads.c b/citadel/threads.c index 0d3caed9f..965f866da 100644 --- a/citadel/threads.c +++ b/citadel/threads.c @@ -67,6 +67,7 @@ int active_workers = 0; /* Number of ACTIVE worker threads */ pthread_key_t ThreadKey; pthread_mutex_t Critters[MAX_SEMAPHORES]; /* Things needing locking */ struct thread_tsd masterTSD; +int server_shutting_down = 0; /* set to nonzero during shutdown */ @@ -128,27 +129,6 @@ void end_critical_section(int which_one) } -/* - * A function to tell all threads to exit - */ -void CtdlThreadStopAll(void) -{ - terminate_all_sessions(); /* close all client sockets */ - CtdlShutdownServiceHooks(); /* close all listener sockets to prevent new connections */ - PerformSessionHooks(EVT_SHUTDOWN); /* run any registered shutdown hooks */ -} - - -/* - * A function for a thread to check if it has been asked to stop - */ -int CtdlThreadCheckStop(void) -{ - - /* FIXME this needs to do something useful. moar code pls ! */ - - return 0; -} /* @@ -224,16 +204,22 @@ void go_threading(void) /* The supervisor thread monitors worker threads and spawns more of them if it finds that * they are all in use. FIXME make the 256 max threads a configurable value. */ - while(!CtdlThreadCheckStop()) { + while (!server_shutting_down) { if ((active_workers == num_workers) && (num_workers < 256)) { - syslog(LOG_DEBUG, "worker threads: %d, active: %d\n", num_workers, active_workers); CtdlThreadCreate(worker_thread); - syslog(LOG_DEBUG, "worker threads: %d, active: %d\n", num_workers, active_workers); } sleep(1); } - /* Shut down */ - CtdlThreadStopAll(); - exit(0); + /* When we get to this point we are getting ready to shut down our Citadel server */ + + terminate_all_sessions(); /* close all client sockets */ + CtdlShutdownServiceHooks(); /* close all listener sockets to prevent new connections */ + PerformSessionHooks(EVT_SHUTDOWN); /* run any registered shutdown hooks */ + + int countdown = 10; + while ( (num_workers > 0) && (countdown-- > 0)) { + syslog(LOG_DEBUG, "Waiting for %d worker threads to exit", num_workers); + sleep(1); + } } diff --git a/citadel/threads.h b/citadel/threads.h index f96ab77d5..2119e0239 100644 --- a/citadel/threads.h +++ b/citadel/threads.h @@ -35,14 +35,13 @@ extern struct thread_tsd masterTSD; extern int num_workers; extern int active_workers; +extern int server_shutting_down; struct thread_tsd *MyThread(void); int try_critical_section (int which_one); void begin_critical_section (int which_one); void end_critical_section (int which_one); void go_threading(void); -int CtdlThreadCheckStop(void); -void CtdlThreadStopAll(void); void InitializeMasterTSD(void); void CtdlThreadCreate(void *(*start_routine)(void*)); -- 2.30.2