Add /skip to robots.txt exclusion

[citadel.git] / webcit / static.c

diff --git a/webcit/static.c b/webcit/static.c
index 971def8905503f61557354f83cdf5274d9f7597c..235322131fa16c93bc117f9f809f85b90d4bc2ef 100644 (file)
--- a/webcit/static.c
+++ b/webcit/static.c
@@ -231,8 +231,6 @@ void output_flat_static(void)
        }
 }
 
-extern void do_404(void);
-
 void output_static_safe(HashList *DirList)
 {
        wcsession *WCC = WC;
@@ -283,7 +281,15 @@ void robots_txt(void) {
        begin_burst();
 
        wc_printf("User-agent: *\r\n"
-               "Disallow:\r\n"
+               "Disallow: /printmsg\r\n"
+               "Disallow: /msgheaders\r\n"
+               "Disallow: /groupdav\r\n"
+               "Disallow: /do_template\r\n"
+               "Disallow: /static\r\n"
+               "Disallow: /display_page\r\n"
+               "Disallow: /readnew\r\n"
+               "Disallow: /display_enter\r\n"
+               "Disallow: /skip\r\n"
                "Sitemap: %s/sitemap.xml\r\n"
                "\r\n"
                ,