* added message subject to all those tiny messages
[citadel.git] / citadel / sysdep.c
index 23acaa4bea0a71deee7c7135b56353f592b2913d..f4e20d72ebb3c6bda71acff9ecad6fce5d95f1e9 100644 (file)
  *
  */
 
-#ifdef DLL_EXPORT
-#define IN_LIBCIT
-#endif
-
 #include "sysdep.h"
 #include <stdlib.h>
 #include <unistd.h>
@@ -66,6 +62,7 @@
 #include "housekeeping.h"
 #include "tools.h"
 #include "serv_crypto.h"
+#include "serv_fulltext.h"
 
 #ifdef HAVE_SYS_SELECT_H
 #include <sys/select.h>
@@ -97,21 +94,22 @@ struct CitContext masterCC;
 time_t last_purge = 0;                         /* Last dead session purge */
 static int num_threads = 0;                    /* Current number of threads */
 int num_sessions = 0;                          /* Current number of sessions */
+pthread_t indexer_thread_tid;
+pthread_t checkpoint_thread_tid;
 
-int syslog_facility = (-1);
-
+int syslog_facility = LOG_DAEMON;
+int enable_syslog = 0;
+extern int running_as_daemon;
 
 /*
  * lprintf()  ...   Write logging information
  */
-extern int running_as_daemon;
-static int enable_syslog = 1;
 void lprintf(enum LogLevel loglevel, const char *format, ...) {   
        va_list arg_ptr;
 
        if (enable_syslog) {
                va_start(arg_ptr, format);
-                       vsyslog(loglevel, format, arg_ptr);
+                       vsyslog((syslog_facility | loglevel), format, arg_ptr);
                va_end(arg_ptr);
        }
 
@@ -156,6 +154,7 @@ void lprintf(enum LogLevel loglevel, const char *format, ...) {
  */
 
 volatile int time_to_die = 0;
+volatile int shutdown_and_halt = 0;
 
 static RETSIGTYPE signal_cleanup(int signum) {
        lprintf(CTDL_DEBUG, "Caught signal %d; shutting down.\n", signum);
@@ -199,7 +198,8 @@ void init_sysdep(void) {
         * session to which the calling thread is currently bound.
         */
        if (pthread_key_create(&MyConKey, NULL) != 0) {
-               lprintf(CTDL_CRIT, "Can't create TSD key!!  %s\n", strerror(errno));
+               lprintf(CTDL_CRIT, "Can't create TSD key: %s\n",
+                       strerror(errno));
        }
 
        /*
@@ -248,6 +248,7 @@ void begin_critical_section(int which_one)
 #ifdef DEBUG_MEMORY_LEAKS
                && (which_one != S_DEBUGMEMLEAKS)
 #endif
+               && (which_one != S_RPLIST)
        ) {
                cdb_check_handles();
        }
@@ -269,7 +270,7 @@ void end_critical_section(int which_one)
  * a TCP port.  The server shuts down if the bind fails.
  *
  */
-int ig_tcp_server(char *ip_addr, int port_number, int queue_len)
+int ig_tcp_server(char *ip_addr, int port_number, int queue_len, char **errormessage)
 {
        struct sockaddr_in sin;
        int s, i;
@@ -295,8 +296,11 @@ int ig_tcp_server(char *ip_addr, int port_number, int queue_len)
        s = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
 
        if (s < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't create a socket: %s\n",
-                       strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                                "citserver: Can't create a socket: %s",
+                                strerror(errno));
+               lprintf(CTDL_EMERG, "%s\n", *errormessage);
                return(-1);
        }
 
@@ -304,22 +308,32 @@ int ig_tcp_server(char *ip_addr, int port_number, int queue_len)
        setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &i, sizeof(i));
 
        if (bind(s, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't bind: %s\n",
-                       strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                                "citserver: Can't bind: %s",
+                                strerror(errno));
+               lprintf(CTDL_EMERG, "%s\n", *errormessage);
                close(s);
                return(-1);
        }
 
        /* set to nonblock - we need this for some obscure situations */
        if (fcntl(s, F_SETFL, O_NONBLOCK) < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't set socket to non-blocking: %s\n",
-                       strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                                "citserver: Can't set socket to non-blocking: %s",
+                                strerror(errno));
+               lprintf(CTDL_EMERG, "%s\n", *errormessage);
                close(s);
                return(-1);
        }
 
        if (listen(s, actual_queue_len) < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't listen: %s\n", strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                                "citserver: Can't listen: %s",
+                                strerror(errno));
+               lprintf(CTDL_EMERG, "%s\n", *errormessage);
                close(s);
                return(-1);
        }
@@ -332,7 +346,7 @@ int ig_tcp_server(char *ip_addr, int port_number, int queue_len)
 /*
  * Create a Unix domain socket and listen on it
  */
-int ig_uds_server(char *sockpath, int queue_len)
+int ig_uds_server(char *sockpath, int queue_len, char **errormessage)
 {
        struct sockaddr_un addr;
        int s;
@@ -344,8 +358,10 @@ int ig_uds_server(char *sockpath, int queue_len)
 
        i = unlink(sockpath);
        if (i != 0) if (errno != ENOENT) {
-               lprintf(CTDL_EMERG, "citserver: can't unlink %s: %s\n",
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, "citserver: can't unlink %s: %s",
                        sockpath, strerror(errno));
+               lprintf(CTDL_EMERG, "%s\n", *errormessage);
                return(-1);
        }
 
@@ -355,27 +371,40 @@ int ig_uds_server(char *sockpath, int queue_len)
 
        s = socket(AF_UNIX, SOCK_STREAM, 0);
        if (s < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't create a socket: %s\n",
-                       strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                        "citserver: Can't create a socket: %s",
+                        strerror(errno));
+               lprintf(CTDL_EMERG, "%s\n", *errormessage);
                return(-1);
        }
 
        if (bind(s, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't bind: %s\n",
-                       strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                        "citserver: Can't bind: %s",
+                        strerror(errno));
+               lprintf(CTDL_EMERG, "%s\n", *errormessage);
                return(-1);
        }
 
        /* set to nonblock - we need this for some obscure situations */
        if (fcntl(s, F_SETFL, O_NONBLOCK) < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't set socket to non-blocking: %s\n",
-                       strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                        "citserver: Can't set socket to non-blocking: %s",
+                        strerror(errno));
+               lprintf(CTDL_EMERG, "%s\n", *errormessage);
                close(s);
                return(-1);
        }
 
        if (listen(s, actual_queue_len) < 0) {
-               lprintf(CTDL_EMERG, "citserver: Can't listen: %s\n", strerror(errno));
+               *errormessage = (char*) malloc(SIZ + 1);
+               snprintf(*errormessage, SIZ, 
+                        "citserver: Can't listen: %s",
+                        strerror(errno));
+               lprintf(CTDL_EMERG, "%s\n", *errormessage);
                return(-1);
        }
 
@@ -406,8 +435,7 @@ struct CitContext *MyContext(void) {
  * Initialize a new context and place it in the list.  The session number
  * used to be the PID (which is why it's called cs_pid), but that was when we
  * had one process per session.  Now we just assign them sequentially, starting
- * at 1 (don't change it to 0 because masterCC uses 0) and re-using them when
- * sessions terminate.
+ * at 1 (don't change it to 0 because masterCC uses 0).
  */
 struct CitContext *CreateNewContext(void) {
        struct CitContext *me;
@@ -426,7 +454,6 @@ struct CitContext *CreateNewContext(void) {
         */
        me->state = CON_EXECUTING;
 
-
        /*
         * Generate a unique session number and insert this context into
         * the list.
@@ -532,11 +559,6 @@ void client_write(char *buf, int nbytes)
                return;
        }
 
-       if (CC->redirect_fp != NULL) {
-               fwrite(buf, (size_t)nbytes, (size_t)1, CC->redirect_fp);
-               return;
-       }
-
 #ifndef HAVE_TCP_BUFFERING
        /* If we're buffering for later, do that now. */
        if (CC->buffering) {
@@ -561,8 +583,10 @@ void client_write(char *buf, int nbytes)
                retval = write(CC->client_socket, &buf[bytes_written],
                        nbytes - bytes_written);
                if (retval < 1) {
-                       lprintf(CTDL_ERR, "client_write() failed: %s\n",
-                               strerror(errno));
+                       lprintf(CTDL_ERR,
+                               "client_write(%d bytes) failed: %s (%d)\n",
+                               nbytes - bytes_written,
+                               strerror(errno), errno);
                        CC->kill_me = 1;
                        return;
                }
@@ -578,7 +602,7 @@ void client_write(char *buf, int nbytes)
  */
 void cprintf(const char *format, ...) {   
        va_list arg_ptr;   
-       char buf[SIZ];   
+       char buf[1024];   
    
        va_start(arg_ptr, format);   
        if (vsnprintf(buf, sizeof buf, format, arg_ptr) == -1)
@@ -624,8 +648,7 @@ int client_read_to(char *buf, int bytes, int timeout)
 
                rlen = read(CC->client_socket, &buf[len], bytes-len);
                if (rlen<1) {
-                       lprintf(CTDL_ERR, "client_read() failed: %s\n",
-                               strerror(errno));
+                       /* The socket has been disconnected! */
                        CC->kill_me = 1;
                        return(-1);
                }
@@ -668,12 +691,13 @@ int client_getln(char *buf, int bufsize)
                while (buf[i] != '\n' && retval == 1)
                        retval = client_read(&buf[i], 1);
 
-       /* Strip the trailing newline and any trailing nonprintables (cr's)
+       /* Strip the trailing LF, and the trailing CR if present.
         */
        buf[i] = 0;
-       while ((strlen(buf)>0)&&(!isprint(buf[strlen(buf)-1])))
+       while ( (strlen(buf) > 0) && ((buf[strlen(buf)-1]==10) || (buf[strlen(buf)-1] == 13)) ) {
                buf[strlen(buf)-1] = 0;
-       if (retval < 0) strcpy(buf, "000");
+       }
+       if (retval < 0) safestrncpy(buf, "000", bufsize);
        return(retval);
 }
 
@@ -789,12 +813,13 @@ void create_worker(void) {
                return;
        }
 
-       /* Our per-thread stacks need to be bigger than the default size, otherwise
-        * the MIME parser crashes on FreeBSD, and the IMAP service crashes on
-        * 64-bit Linux.
+       /* Our per-thread stacks need to be bigger than the default size,
+        * otherwise the MIME parser crashes on FreeBSD, and the IMAP service
+        * crashes on 64-bit Linux.
         */
-       if ((ret = pthread_attr_setstacksize(&attr, 1024 * 1024))) {
-               lprintf(CTDL_EMERG, "pthread_attr_setstacksize: %s\n", strerror(ret));
+       if ((ret = pthread_attr_setstacksize(&attr, THREADSTACKSIZE))) {
+               lprintf(CTDL_EMERG, "pthread_attr_setstacksize: %s\n",
+                       strerror(ret));
                time_to_die = -1;
                pthread_attr_destroy(&attr);
                return;
@@ -813,6 +838,44 @@ void create_worker(void) {
 }
 
 
+/*
+ * Create the indexer thread and begin its operation.
+ * Then create the checkpoint thread and begin its operation.
+ */
+void create_maintenance_threads(void) {
+       int ret;
+       pthread_attr_t attr;
+
+       if ((ret = pthread_attr_init(&attr))) {
+               lprintf(CTDL_EMERG, "pthread_attr_init: %s\n", strerror(ret));
+               time_to_die = -1;
+               return;
+       }
+
+       /* Our per-thread stacks need to be bigger than the default size,
+        * otherwise the MIME parser crashes on FreeBSD, and the IMAP service
+        * crashes on 64-bit Linux.
+        */
+       if ((ret = pthread_attr_setstacksize(&attr, THREADSTACKSIZE))) {
+               lprintf(CTDL_EMERG, "pthread_attr_setstacksize: %s\n",
+                       strerror(ret));
+               time_to_die = -1;
+               pthread_attr_destroy(&attr);
+               return;
+       }
+
+       if ((ret = pthread_create(&indexer_thread_tid, &attr, indexer_thread, NULL) != 0)) {
+               lprintf(CTDL_ALERT, "Can't create thread: %s\n", strerror(ret));
+       }
+
+       if ((ret = pthread_create(&checkpoint_thread_tid, &attr, checkpoint_thread, NULL) != 0)) {
+               lprintf(CTDL_ALERT, "Can't create thread: %s\n", strerror(ret));
+       }
+
+       pthread_attr_destroy(&attr);
+}
+
+
 
 /*
  * Purge all sessions which have the 'kill_me' flag set.
@@ -826,7 +889,8 @@ void create_worker(void) {
  * if such an action is appropriate.
  */
 void dead_session_purge(int force) {
-       struct CitContext *ptr, *rem;
+       struct CitContext *ptr;         /* general-purpose utility pointer */
+       struct CitContext *rem = NULL;  /* list of sessions to be destroyed */
 
        if (force == 0) {
                if ( (time(NULL) - last_purge) < 5 ) {
@@ -835,28 +899,44 @@ void dead_session_purge(int force) {
        }
        time(&last_purge);
 
-       do {
-               rem = NULL;
-               begin_critical_section(S_SESSION_TABLE);
-               for (ptr = ContextList; ptr != NULL; ptr = ptr->next) {
-                       if ( (ptr->state == CON_IDLE) && (ptr->kill_me) ) {
-                               rem = ptr;
+       begin_critical_section(S_SESSION_TABLE);
+       for (ptr = ContextList; ptr != NULL; ptr = ptr->next) {
+               if ( (ptr->state == CON_IDLE) && (ptr->kill_me) ) {
+
+                       /* Remove the session from the active list */
+                       if (ptr->prev) {
+                               ptr->prev->next = ptr->next;
                        }
-               }
-               end_critical_section(S_SESSION_TABLE);
+                       else {
+                               ContextList = ptr->next;
+                       }
+                       if (ptr->next) {
+                               ptr->next->prev = ptr->prev;
+                       }
+
+                       --num_sessions;
+
+                       /* And put it on our to-be-destroyed list */
+                       ptr->next = rem;
+                       rem = ptr;
 
-               /* RemoveContext() enters its own S_SESSION_TABLE critical
-                * section, so we have to do it like this.
-                */     
-               if (rem != NULL) {
-                       lprintf(CTDL_DEBUG, "Purging session %d\n", rem->cs_pid);
-                       RemoveContext(rem);
                }
+       }
+       end_critical_section(S_SESSION_TABLE);
 
-       } while (rem != NULL);
+       /* Now that we no longer have the session list locked, we can take
+        * our time and destroy any sessions on the to-be-killed list, which
+        * is allocated privately on this thread's stack.
+        */
+       while (rem != NULL) {
+               lprintf(CTDL_DEBUG, "Purging session %d\n", rem->cs_pid);
+               RemoveContext(rem);
+               ptr = rem;
+               rem = rem->next;
+               free(ptr);
+       }
 
        /* Raise the size of the worker thread pool if necessary. */
-
        if ( (num_sessions > num_threads)
           && (num_threads < config.c_max_workers) ) {
                begin_critical_section(S_WORKER_LIST);
@@ -869,18 +949,6 @@ void dead_session_purge(int force) {
 
 
 
-/*
- * Redirect a session's output to a file.
- * This function may be called with a file handle.
- * Call with NULL to return output to its normal client socket.
- */
-void CtdlRedirectOutput(FILE *fp)
-{
-       if (fp != NULL) CC->redirect_fp = fp;
-       else CC->redirect_fp = NULL;
-}
-
-
 /*
  * masterCC is the context we use when not attached to a session.  This
  * function initializes it.
@@ -1004,29 +1072,34 @@ do_select:      force_purge = 0;
 
                        if (FD_ISSET(serviceptr->msock, &readfds)) {
                                ssock = accept(serviceptr->msock, NULL, 0);
-                               if (ssock < 0) {
-                                       lprintf(CTDL_CRIT,
-                                               "citserver: accept(): %s\n",
-                                               strerror(errno));
-                               }
-                               else {
+                               if (ssock >= 0) {
                                        lprintf(CTDL_DEBUG,
                                                "New client socket %d\n",
                                                ssock);
 
+                                       /* The master socket is non-blocking but the client
+                                        * sockets need to be blocking, otherwise certain
+                                        * operations barf on FreeBSD.  Not a fatal error.
+                                        */
+                                       if (fcntl(ssock, F_SETFL, 0) < 0) {
+                                               lprintf(CTDL_EMERG,
+                                                       "citserver: Can't set socket to blocking: %s\n",
+                                                       strerror(errno));
+                                       }
+
                                        /* New context will be created already
-                                       * set up in the CON_EXECUTING state.
-                                       */
+                                        * set up in the CON_EXECUTING state.
+                                        */
                                        con = CreateNewContext();
 
-                                       /* Assign new socket number to it. */
+                                       /* Assign our new socket number to it. */
                                        con->client_socket = ssock;
                                        con->h_command_function =
                                                serviceptr->h_command_function;
                                        con->h_async_function =
                                                serviceptr->h_async_function;
 
-                                       /* Determine whether local socket */
+                                       /* Determine whether it's a local socket */
                                        if (serviceptr->sockpath != NULL)
                                                con->is_local_socket = 1;