Bring in new dkim code
[citadel.git] / webcit / sysdep.c
index b4db58f9e1488ae5b2a96a9cb1d926759da76574..e5f928d521caf8b85b4bc56ead6a50beb6eec9dd 100644 (file)
@@ -1,16 +1,17 @@
 /*
- * $Id: sysdep.c 5691 2007-11-04 23:19:17Z dothebart $
+ * WebCit "system dependent" code.
  *
- * Citadel "system dependent" stuff.
- * See copyright.txt for copyright information.
+ * Copyright (c) 1996-2021 by the citadel.org team
  *
- * Here's where we (hopefully) have most parts of the Citadel server that
- * would need to be altered to run the server in a non-POSIX environment.
- * 
- * If we ever port to a different platform and either have multiple
- * variants of this file or simply load it up with #ifdefs.
+ * This program is open source software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 3.
  *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
  */
+
 #include "sysdep.h"
 #include <stdlib.h>
 #include <unistd.h>
 #include <sys/socket.h>
 #include <syslog.h>
 #include <sys/syslog.h>
-
-#if TIME_WITH_SYS_TIME
-# include <sys/time.h>
-# include <time.h>
-#else
-# if HAVE_SYS_TIME_H
-#  include <sys/time.h>
-# else
-#  include <time.h>
-# endif
-#endif
-
+#include <time.h>
 #include <limits.h>
 #include <sys/resource.h>
 #include <netinet/in.h>
 #include <sys/select.h>
 #endif
 
-#ifndef HAVE_SNPRINTF
-#include "snprintf.h"
-#endif
-#include "webserver.h"
+
 #include "modules_init.h"
 #if HAVE_BACKTRACE
 #include <execinfo.h>
@@ -72,27 +59,81 @@ pthread_key_t MyConKey;                             /* TSD key for MyContext() */
 pthread_key_t MyReq;                           /* TSD key for MyReq() */
 int msock;                     /* master listening socket */
 int time_to_die = 0;            /* Nonzero if server is shutting down */
-int verbosity = 9;             /* Logging level */
 
 extern void *context_loop(ParsedHttpHdrs *Hdr);
 extern void *housekeeping_loop(void);
+extern void do_housekeeping(void);
 
-char ctdl_key_dir[PATH_MAX]=SSL_DIR;
-char file_crpt_file_key[PATH_MAX]="";
-char file_crpt_file_csr[PATH_MAX]="";
-char file_crpt_file_cer[PATH_MAX]="";
+char file_etc_mimelist[PATH_MAX]="";
 
-const char editor_absolut_dir[PATH_MAX]=EDITORDIR;     /* nailed to what configure gives us. */
+char etc_dir[PATH_MAX];
 char static_dir[PATH_MAX];             /* calculated on startup */
 char static_local_dir[PATH_MAX];               /* calculated on startup */
 char static_icon_dir[PATH_MAX];          /* where should we find our mime icons? */
 char  *static_dirs[]={                         /* needs same sort order as the web mapping */
-       (char*)static_dir,                      /* our templates on disk */
-       (char*)static_local_dir,                /* user provided templates disk */
-       (char*)editor_absolut_dir,              /* the editor on disk */
-       (char*)static_icon_dir                  /* our icons... */
+       "./static",
+       "./static.local",
+       "./tiny_mce",
+       "./static/webcit_icons"
 };
 
+int ExitPipe[2];
+HashList *GZMimeBlackList = NULL; /* mimetypes which shouldn't be gzip compressed */
+
+void LoadMimeBlacklist(void)
+{
+       StrBuf *MimeBlackLine;
+       IOBuffer IOB;
+       eReadState state;
+       
+       memset(&IOB, 0, sizeof(IOBuffer));
+       IOB.fd = open(file_etc_mimelist, O_RDONLY);
+
+       IOB.Buf = NewStrBuf();
+       MimeBlackLine = NewStrBuf();
+       GZMimeBlackList = NewHash(1, NULL);
+
+       do
+       {
+               state = StrBufChunkSipLine(MimeBlackLine, &IOB);
+
+               switch (state)
+               {
+               case eMustReadMore:
+                       if (StrBuf_read_one_chunk_callback(IOB.fd, 0, &IOB) <= 0)
+                               state = eReadFail;
+                       break;
+               case eReadSuccess:
+                       if ((StrLength(MimeBlackLine) > 1) && 
+                           (*ChrPtr(MimeBlackLine) != '#'))
+                       {
+                               Put(GZMimeBlackList, SKEY(MimeBlackLine),
+                                   (void*) 1, reference_free_handler);
+                       }
+                       FlushStrBuf(MimeBlackLine);
+                       break;
+               case eReadFail:
+                       break;
+               case eBufferNotEmpty:
+                       break;
+               }
+       }
+       while (state != eReadFail);
+
+       close(IOB.fd);
+       
+       FreeStrBuf(&IOB.Buf);
+       FreeStrBuf(&MimeBlackLine);
+}
+
+void CheckGZipCompressionAllowed(const char *MimeType, long MLen)
+{
+       void *v;
+
+       if (WC->Hdr->HR.gzip_ok)
+           WC->Hdr->HR.gzip_ok = GetHash(GZMimeBlackList, MimeType, MLen, &v) == 0;
+}
+
 void InitialiseSemaphores(void)
 {
        int i;
@@ -101,6 +142,15 @@ void InitialiseSemaphores(void)
        for (i=0; i<MAX_SEMAPHORES; ++i) {
                pthread_mutex_init(&Critters[i], NULL);
        }
+
+       if (pipe(ExitPipe))
+       {
+               syslog(LOG_WARNING, "Failed to open exit pipe: %d [%s]\n", 
+                      errno, 
+                      strerror(errno));
+               
+               exit(-1);
+       }
 }
 
 /*
@@ -108,7 +158,6 @@ void InitialiseSemaphores(void)
  */
 void begin_critical_section(int which_one)
 {
-       /* lprintf(CTDL_DEBUG, "begin_critical_section(%d)\n", which_one); */
        pthread_mutex_lock(&Critters[which_one]);
 }
 
@@ -123,16 +172,13 @@ void end_critical_section(int which_one)
 
 void ShutDownWebcit(void)
 {
+
+       DeleteHash(&GZMimeBlackList);
        free_zone_directory ();
        icaltimezone_release_zone_tab ();
        icalmemory_free_ring ();
        ShutDownLibCitadel ();
        shutdown_modules ();
-#ifdef HAVE_OPENSSL
-       if (is_https) {
-               shutdown_ssl();
-       }
-#endif
 }
 
 /*
@@ -143,52 +189,39 @@ void worker_entry(void)
        int ssock;
        int i = 0;
        int fail_this_transaction = 0;
-       int ret;
-       struct timeval tv;
-       fd_set readset, tempset;
        ParsedHttpHdrs Hdr;
 
        memset(&Hdr, 0, sizeof(ParsedHttpHdrs));
        Hdr.HR.eReqType = eGET;
        http_new_modules(&Hdr); 
-       tv.tv_sec = 0;
-       tv.tv_usec = 10000;
-       FD_ZERO(&readset);
-       FD_SET(msock, &readset);
 
        do {
-               /* Only one thread can accept at a time */
+               /* Each worker thread blocks on accept() while waiting for something to do. */
                fail_this_transaction = 0;
                ssock = -1; 
                errno = EAGAIN;
                do {
-                       ret = -1; /* just one at once should select... */
-                       begin_critical_section(S_SELECT);
-
-                       FD_ZERO(&tempset);
-                       if (msock > 0) FD_SET(msock, &tempset);
-                       tv.tv_sec = 0;
-                       tv.tv_usec = 10000;
-                       if (msock > 0)  ret = select(msock+1, &tempset, NULL, NULL,  &tv);
-                       end_critical_section(S_SELECT);
-                       if ((ret < 0) && (errno != EINTR) && (errno != EAGAIN))
-                       {/* EINTR and EAGAIN are thrown but not of interest. */
-                               lprintf(2, "accept() failed:%d %s\n",
-                                       errno, strerror(errno));
-                       }
-                       else if ((ret > 0) && (msock > 0) && FD_ISSET(msock, &tempset))
-                       {/* Successfully selected, and still not shutting down? Accept! */
-                               ssock = accept(msock, NULL, 0);
-                       }
-                       
+                       fd_set wset;
+                       --num_threads_executing;
+                        FD_ZERO(&wset);
+                        FD_SET(msock, &wset);
+                        FD_SET(ExitPipe[1], &wset);
+
+                        select(msock + 1, NULL, &wset, NULL, NULL);
+                       if (time_to_die)
+                               break;
+
+                       ssock = accept(msock, NULL, 0);
+                       ++num_threads_executing;
+                       if (ssock < 0) fail_this_transaction = 1;
                } while ((msock > 0) && (ssock < 0)  && (time_to_die == 0));
 
                if ((msock == -1)||(time_to_die))
                {/* ok, we're going down. */
                        int shutdown = 0;
 
-                       /* the first to come here will have to do the cleanup.
-                        * make shure its realy just one.
+                       /* The first thread to get here will have to do the cleanup.
+                        * Make sure it's really just one.
                         */
                        begin_critical_section(S_SHUTDOWN);
                        if (msock == -1)
@@ -200,32 +233,35 @@ void worker_entry(void)
                        if (shutdown == 1)
                        {/* we're the one to cleanup the mess. */
                                http_destroy_modules(&Hdr);
-                               lprintf(2, "I'm master shutdown: tagging sessions to be killed.\n");
+                               syslog(LOG_DEBUG, "I'm master shutdown: tagging sessions to be killed.\n");
                                shutdown_sessions();
-                               lprintf(2, "master shutdown: waiting for others\n");
+                               syslog(LOG_DEBUG, "master shutdown: waiting for others\n");
                                sleeeeeeeeeep(1); /* wait so some others might finish... */
-                               lprintf(2, "master shutdown: cleaning up sessions\n");
+                               syslog(LOG_DEBUG, "master shutdown: cleaning up sessions\n");
                                do_housekeeping();
-                               lprintf(2, "master shutdown: cleaning up libical\n");
+                               syslog(LOG_DEBUG, "master shutdown: cleaning up libical\n");
 
                                ShutDownWebcit();
 
-                               lprintf(2, "master shutdown exiting!.\n");                              
+                               syslog(LOG_DEBUG, "master shutdown exiting.\n");                                
                                exit(0);
                        }
                        break;
                }
                if (ssock < 0 ) continue;
 
+               check_thread_pool_size();
+
+               /* Now do something. */
                if (msock < 0) {
                        if (ssock > 0) close (ssock);
-                       lprintf(2, "inbetween.");
+                       syslog(LOG_DEBUG, "in between.");
                        pthread_exit(NULL);
-               } else { /* Got it? do some real work! */
+               } else {
+                       /* Got it? do some real work! */
                        /* Set the SO_REUSEADDR socket option */
                        i = 1;
-                       setsockopt(ssock, SOL_SOCKET, SO_REUSEADDR,
-                                  &i, sizeof(i));
+                       setsockopt(ssock, SOL_SOCKET, SO_REUSEADDR, &i, sizeof(i));
 
                        /* If we are an HTTPS server, go crypto now. */
 #ifdef HAVE_OPENSSL
@@ -241,11 +277,11 @@ void worker_entry(void)
                                int fdflags; 
                                fdflags = fcntl(ssock, F_GETFL);
                                if (fdflags < 0)
-                                       lprintf(1, "unable to get server socket flags! %s \n",
+                                       syslog(LOG_WARNING, "unable to get server socket flags! %s \n",
                                                strerror(errno));
                                fdflags = fdflags | O_NONBLOCK;
                                if (fcntl(ssock, F_SETFL, fdflags) < 0)
-                                       lprintf(1, "unable to set server socket nonblocking flags! %s \n",
+                                       syslog(LOG_WARNING, "unable to set server socket nonblocking flags! %s \n",
                                                strerror(errno));
                        }
 
@@ -263,8 +299,9 @@ void worker_entry(void)
 #endif
 
                                /* ...and close the socket. */
-                               if (Hdr.http_sock > 0)
+                               if (Hdr.http_sock > 0) {
                                        lingering_close(ssock);
+                               }
                                http_detach_modules(&Hdr);
 
                        }
@@ -274,30 +311,10 @@ void worker_entry(void)
        } while (!time_to_die);
 
        http_destroy_modules(&Hdr);
-       lprintf (1, "bye\n");
+       syslog(LOG_DEBUG, "Thread exiting.\n");
        pthread_exit(NULL);
 }
 
-/*
- * print log messages 
- * logs to stderr if loglevel is lower than the verbosity set at startup
- *
- * loglevel    level of the message
- * format      the printf like format string
- * ...         the strings to put into format
- */
-int lprintf(int loglevel, const char *format, ...)
-{
-       va_list ap;
-
-       if (loglevel <= verbosity) {
-               va_start(ap, format);
-               vfprintf(stderr, format, ap);
-               va_end(ap);
-               fflush(stderr);
-       }
-       return 1;
-}
 
 /*
  * Shut us down the regular way.
@@ -305,15 +322,14 @@ int lprintf(int loglevel, const char *format, ...)
  */
 pid_t current_child;
 void graceful_shutdown_watcher(int signum) {
-       lprintf (1, "bye; shutting down watcher.");
+       syslog(LOG_INFO, "Watcher thread exiting.\n");
+       write(ExitPipe[0], HKEY("                              "));
        kill(current_child, signum);
        if (signum != SIGHUP)
                exit(0);
 }
 
 
-
-
 /*
  * Shut us down the regular way.
  * signum is the signal we want to forward
@@ -323,7 +339,7 @@ void graceful_shutdown(int signum) {
        FILE *FD;
        int fd;
 
-       lprintf (1, "WebCit is being shut down on signal %d.\n", signum);
+       syslog(LOG_INFO, "WebCit is being shut down on signal %d.\n", signum);
        fd = msock;
        msock = -1;
        time_to_die = 1;
@@ -331,6 +347,7 @@ void graceful_shutdown(int signum) {
        fflush (FD);
        fclose (FD);
        close(fd);
+       write(ExitPipe[0], HKEY("                              "));
 }
 
 
@@ -343,8 +360,6 @@ void start_daemon(char *pid_file)
        pid_t child = 0;
        FILE *fp;
        int do_restart = 0;
-       int rv;
-       FILE *rvfp = NULL;
 
        current_child = 0;
 
@@ -352,7 +367,7 @@ void start_daemon(char *pid_file)
         * We don't just call close() because we don't want these fd's
         * to be reused for other files.
         */
-       rv = chdir("/");
+       chdir("/");
 
        signal(SIGHUP, SIG_IGN);
        signal(SIGINT, SIG_IGN);
@@ -365,9 +380,9 @@ void start_daemon(char *pid_file)
 
        setsid();
        umask(0);
-       rvfp = freopen("/dev/null", "r", stdin);
-       rvfp = freopen("/dev/null", "w", stdout);
-       rvfp = freopen("/dev/null", "w", stderr);
+       freopen("/dev/null", "r", stdin);
+       freopen("/dev/null", "w", stdout);
+       freopen("/dev/null", "w", stderr);
        signal(SIGTERM, graceful_shutdown_watcher);
        signal(SIGHUP, graceful_shutdown_watcher);
 
@@ -432,6 +447,7 @@ void start_daemon(char *pid_file)
        exit(WEXITSTATUS(status));
 }
 
+
 /*
  * Spawn an additional worker thread into the pool.
  */
@@ -441,7 +457,8 @@ void spawn_another_worker_thread()
        pthread_attr_t attr;    /* Thread attributes */
        int ret;
 
-       lprintf(3, "Creating a new thread.  Pool size is now %d\n", ++num_threads);
+       ++num_threads_existing;
+       ++num_threads_executing;
 
        /* set attributes for the new thread */
        pthread_attr_init(&attr);
@@ -452,16 +469,13 @@ void spawn_another_worker_thread()
         * otherwise the MIME parser crashes on FreeBSD.
         */
        if ((ret = pthread_attr_setstacksize(&attr, 1024 * 1024))) {
-               lprintf(1, "pthread_attr_setstacksize: %s\n",
-                       strerror(ret));
+               syslog(LOG_WARNING, "pthread_attr_setstacksize: %s\n", strerror(ret));
                pthread_attr_destroy(&attr);
        }
 
        /* now create the thread */
-       if (pthread_create(&SessThread, &attr,
-                          (void *(*)(void *)) worker_entry, NULL)
-           != 0) {
-               lprintf(1, "Can't create thread: %s\n", strerror(errno));
+       if (pthread_create(&SessThread, &attr, (void *(*)(void *)) worker_entry, NULL) != 0) {
+               syslog(LOG_WARNING, "Can't create thread: %s\n", strerror(errno));
        }
 
        /* free up the attributes */
@@ -497,28 +511,27 @@ webcit_calc_dirs_n_files(int relh, const char *basedir, int home, char *webcitdi
        StripSlashes(static_icon_dir, 1);
        StripSlashes(static_local_dir, 1);
 
-       snprintf(file_crpt_file_key,
-                sizeof file_crpt_file_key, 
-                "%s/citadel.key",
-                ctdl_key_dir);
-       snprintf(file_crpt_file_csr,
-                sizeof file_crpt_file_csr, 
-                "%s/citadel.csr",
-                ctdl_key_dir);
-       snprintf(file_crpt_file_cer,
-                sizeof file_crpt_file_cer, 
-                "%s/citadel.cer",
-                ctdl_key_dir);
+       basedir=ETCDIR;
+       COMPUTE_DIRECTORY(etc_dir);
+       StripSlashes(etc_dir, 1);
+       snprintf(file_etc_mimelist,
+                sizeof file_etc_mimelist, 
+                "%s/nogz-mimetypes.txt",
+                etc_dir);
 
        /* we should go somewhere we can leave our coredump, if enabled... */
-       lprintf(9, "Changing directory to %s\n", socket_dir);
+       syslog(LOG_INFO, "Changing directory to %s\n", socket_dir);
        if (chdir(webcitdir) != 0) {
                perror("chdir");
        }
 }
+
 void drop_root(uid_t UID)
 {
        struct passwd pw, *pwp = NULL;
+#ifdef HAVE_GETPWUID_R
+       char pwbuf[SIZ];
+#endif
 
        /*
         * Now that we've bound the sockets, change to the Citadel user id and its
@@ -529,26 +542,26 @@ void drop_root(uid_t UID)
 #ifdef HAVE_GETPWUID_R
 #ifdef SOLARIS_GETPWUID
                pwp = getpwuid_r(UID, &pw, pwbuf, sizeof(pwbuf));
-#else // SOLARIS_GETPWUID
+#else /* SOLARIS_GETPWUID */
                getpwuid_r(UID, &pw, pwbuf, sizeof(pwbuf), &pwp);
-#endif // SOLARIS_GETPWUID
-#else // HAVE_GETPWUID_R
+#endif /* SOLARIS_GETPWUID */
+#else /* HAVE_GETPWUID_R */
                pwp = NULL;
-#endif // HAVE_GETPWUID_R
+#endif /* HAVE_GETPWUID_R */
 
                if (pwp == NULL)
-                       lprintf(CTDL_CRIT, "WARNING: getpwuid(%ld): %s\n"
+                       syslog(LOG_CRIT, "WARNING: getpwuid(%d): %s\n"
                                "Group IDs will be incorrect.\n", UID,
                                strerror(errno));
                else {
                        initgroups(pw.pw_name, pw.pw_gid);
                        if (setgid(pw.pw_gid))
-                               lprintf(CTDL_CRIT, "setgid(%ld): %s\n", (long)pw.pw_gid,
+                               syslog(LOG_CRIT, "setgid(%ld): %s\n", (long)pw.pw_gid,
                                        strerror(errno));
                }
-               lprintf(CTDL_INFO, "Changing uid to %ld\n", (long)UID);
+               syslog(LOG_INFO, "Changing uid to %ld\n", (long)UID);
                if (setuid(UID) != 0) {
-                       lprintf(CTDL_CRIT, "setuid() failed: %s\n", strerror(errno));
+                       syslog(LOG_CRIT, "setuid() failed: %s\n", strerror(errno));
                }
 #if defined (HAVE_SYS_PRCTL_H) && defined (PR_SET_DUMPABLE)
                prctl(PR_SET_DUMPABLE, 1);
@@ -560,7 +573,7 @@ void drop_root(uid_t UID)
 /*
  * print the actual stack frame.
  */
-void wc_backtrace(void)
+void wc_backtrace(long LogLevel)
 {
 #ifdef HAVE_BACKTRACE
        void *stack_frames[50];
@@ -572,9 +585,9 @@ void wc_backtrace(void)
        strings = backtrace_symbols(stack_frames, size);
        for (i = 0; i < size; i++) {
                if (strings != NULL)
-                       lprintf(1, "%s\n", strings[i]);
+                       syslog(LogLevel, "%s\n", strings[i]);
                else
-                       lprintf(1, "%p\n", stack_frames[i]);
+                       syslog(LogLevel, "%p\n", stack_frames[i]);
        }
        free(strings);
 #endif