Add /skip to robots.txt exclusion

[citadel.git] / webcit / static.c

diff --git a/webcit/static.c b/webcit/static.c
index 70f3cd36b4562213cdfd5b10a8f4daed29ad9771..235322131fa16c93bc117f9f809f85b90d4bc2ef 100644 (file)
--- a/webcit/static.c
+++ b/webcit/static.c
@@ -1,6 +1,4 @@
 /*
- * $Id: webcit.c 7459 2009-05-17 08:34:33Z dothebart $
- *
  * This is the main transaction loop of the web service.  It maintains a
  * persistent session to the Citadel server, handling HTTP WebCit requests as
  * they arrive and presenting a user interface.
@@ -23,7 +21,7 @@
 HashList *StaticFilemappings[4] = {NULL, NULL, NULL, NULL};
 /*
                {
-                       lprintf(9, "Suspicious request. Ignoring.");
+                       syslog(9, "Suspicious request. Ignoring.");
                        hprintf("HTTP/1.1 404 Security check failed\r\n");
                        hprintf("Content-Type: text/plain\r\n\r\n");
                        wc_printf("You have sent a malformed or invalid request.\r\n");
@@ -45,7 +43,7 @@ void output_static(const char *what)
 
        fd = open(what, O_RDONLY);
        if (fd <= 0) {
-               lprintf(9, "output_static('%s') [%s]  -- NOT FOUND --\n", what, ChrPtr(WC->Hdr->this_page));
+               syslog(9, "output_static('%s') [%s]  -- NOT FOUND --\n", what, ChrPtr(WC->Hdr->this_page));
                hprintf("HTTP/1.1 404 %s\r\n", strerror(errno));
                hprintf("Content-Type: text/plain\r\n");
                begin_burst();
@@ -56,12 +54,13 @@ void output_static(const char *what)
                content_type = GuessMimeByFilename(what, len);
 
                if (fstat(fd, &statbuf) == -1) {
-                       lprintf(9, "output_static('%s')  -- FSTAT FAILED --\n", what);
+                       syslog(9, "output_static('%s')  -- FSTAT FAILED --\n", what);
                        hprintf("HTTP/1.1 404 %s\r\n", strerror(errno));
                        hprintf("Content-Type: text/plain\r\n");
                        begin_burst();
                        wc_printf("Cannot fstat %s: %s\n", what, strerror(errno));
                        end_burst();
+                       if (fd > 0) close(fd);
                        return;
                }
 
@@ -71,7 +70,7 @@ void output_static(const char *what)
                if (StrBufReadBLOB(WC->WBuf, &fd, 1, bytes, &Err) < 0)
                {
                        if (fd > 0) close(fd);
-                       lprintf(9, "output_static('%s')  -- FREAD FAILED (%s) --\n", what, strerror(errno));
+                       syslog(9, "output_static('%s')  -- FREAD FAILED (%s) --\n", what, strerror(errno));
                                hprintf("HTTP/1.1 500 internal server error \r\n");
                                hprintf("Content-Type: text/plain\r\n");
                                end_burst();
@@ -80,9 +79,6 @@ void output_static(const char *what)
 
 
                close(fd);
-#ifndef TECH_PREVIEW
-               lprintf(9, "output_static('%s')  %s\n", what, content_type);
-#endif
                http_transmit_thing(content_type, 2);
        }
        if (yesbstr("force_close_session")) {
@@ -202,7 +198,7 @@ int LoadStaticDir(const char *DirName, HashList *DirList, const char *RelDir)
                        StrBufAppendBufPlain(OneWebName, filedir_entry->d_name, d_namelen, 0);
 
                        Put(DirList, SKEY(OneWebName), FileName, HFreeStrBuf);
-                       /* lprintf(9, "[%s | %s]\n", ChrPtr(OneWebName), ChrPtr(FileName)); */
+                       /* syslog(9, "[%s | %s]\n", ChrPtr(OneWebName), ChrPtr(FileName)); */
                        break;
                default:
                        break;
@@ -225,6 +221,8 @@ void output_flat_static(void)
        void *vFile;
        StrBuf *File;
 
+       if (WCC->Hdr->HR.Handler == NULL)
+               return;
        if (GetHash(StaticFilemappings[0], SKEY(WCC->Hdr->HR.Handler->Name), &vFile) &&
            (vFile != NULL))
        {
@@ -233,8 +231,6 @@ void output_flat_static(void)
        }
 }
 
-extern void do_404(void);
-
 void output_static_safe(HashList *DirList)
 {
        wcsession *WCC = WC;
@@ -248,7 +244,7 @@ void output_static_safe(HashList *DirList)
                output_static(ChrPtr(vFile));
        }
        else {
-               lprintf(1, "output_static_safe() file %s not found. \n", 
+               syslog(1, "output_static_safe() file %s not found. \n", 
                        ChrPtr(WCC->Hdr->HR.ReqLine));
 ///TODO: detect image & output blank image
                do_404();
@@ -271,6 +267,39 @@ void output_static_3(void)
        output_static_safe(StaticFilemappings[3]);
 }
 
+
+/*
+ * robots.txt
+ */
+void robots_txt(void) {
+       output_headers(0, 0, 0, 0, 0, 0);
+
+       hprintf("Content-type: text/plain\r\n"
+               "Server: %s\r\n"
+               "Connection: close\r\n",
+               PACKAGE_STRING);
+       begin_burst();
+
+       wc_printf("User-agent: *\r\n"
+               "Disallow: /printmsg\r\n"
+               "Disallow: /msgheaders\r\n"
+               "Disallow: /groupdav\r\n"
+               "Disallow: /do_template\r\n"
+               "Disallow: /static\r\n"
+               "Disallow: /display_page\r\n"
+               "Disallow: /readnew\r\n"
+               "Disallow: /display_enter\r\n"
+               "Disallow: /skip\r\n"
+               "Sitemap: %s/sitemap.xml\r\n"
+               "\r\n"
+               ,
+               ChrPtr(site_prefix)
+       );
+
+       wDumpContent(0);
+}
+
+
 void 
 ServerStartModule_STATIC
 (void)
@@ -300,7 +329,7 @@ InitModule_STATIC
        LoadStaticDir(static_dirs[2], StaticFilemappings[2], "");
        LoadStaticDir(static_dirs[3], StaticFilemappings[3], "");
 
-       WebcitAddUrlHandler(HKEY("robots.txt"), "", 0, output_flat_static, ANONYMOUS|COOKIEUNNEEDED|ISSTATIC|LOGCHATTY);
+       WebcitAddUrlHandler(HKEY("robots.txt"), "", 0, robots_txt, ANONYMOUS|COOKIEUNNEEDED|ISSTATIC|LOGCHATTY);
        WebcitAddUrlHandler(HKEY("favicon.ico"), "", 0, output_flat_static, ANONYMOUS|COOKIEUNNEEDED|ISSTATIC|LOGCHATTY);
        WebcitAddUrlHandler(HKEY("static"), "", 0, output_static_0, ANONYMOUS|COOKIEUNNEEDED|ISSTATIC|LOGCHATTY);
        WebcitAddUrlHandler(HKEY("static.local"), "", 0, output_static_1, ANONYMOUS|COOKIEUNNEEDED|ISSTATIC|LOGCHATTY);