From: Art Cancro <ajc@citadel.org>
Date: Tue, 21 Sep 2010 02:48:51 +0000 (-0400)
Subject: The URL to the sitemap is now published through robots.txt
X-Git-Tag: v8.01~702^2~2
X-Git-Url: https://code.citadel.org/?p=citadel.git;a=commitdiff_plain;h=e0e4f70a2f9d60f08382fe144f28e018fd7103da

The URL to the sitemap is now published through robots.txt
---

diff --git a/webcit/static.c b/webcit/static.c
index 46770bf12..11b868ad5 100644
--- a/webcit/static.c
+++ b/webcit/static.c
@@ -271,6 +271,31 @@ void output_static_3(void)
 	output_static_safe(StaticFilemappings[3]);
 }
 
+
+/*
+ * robots.txt
+ */
+void robots_txt(void) {
+	output_headers(0, 0, 0, 0, 0, 0);
+
+	hprintf("Content-type: text/plain\r\n"
+		"Server: %s\r\n"
+		"Connection: close\r\n",
+		PACKAGE_STRING);
+	begin_burst();
+
+	wc_printf("User-agent: *\r\n"
+		"Disallow:\r\n"
+		"Sitemap: %s/sitemap.xml\r\n"
+		"\r\n"
+		,
+		ChrPtr(site_prefix)
+	);
+
+	wDumpContent(0);
+}
+
+
 void 
 ServerStartModule_STATIC
 (void)
@@ -300,7 +325,7 @@ InitModule_STATIC
 	LoadStaticDir(static_dirs[2], StaticFilemappings[2], "");
 	LoadStaticDir(static_dirs[3], StaticFilemappings[3], "");
 
-	WebcitAddUrlHandler(HKEY("robots.txt"), "", 0, output_flat_static, ANONYMOUS|COOKIEUNNEEDED|ISSTATIC|LOGCHATTY);
+	WebcitAddUrlHandler(HKEY("robots.txt"), "", 0, robots_txt, ANONYMOUS|COOKIEUNNEEDED|ISSTATIC|LOGCHATTY);
 	WebcitAddUrlHandler(HKEY("favicon.ico"), "", 0, output_flat_static, ANONYMOUS|COOKIEUNNEEDED|ISSTATIC|LOGCHATTY);
 	WebcitAddUrlHandler(HKEY("static"), "", 0, output_static_0, ANONYMOUS|COOKIEUNNEEDED|ISSTATIC|LOGCHATTY);
 	WebcitAddUrlHandler(HKEY("static.local"), "", 0, output_static_1, ANONYMOUS|COOKIEUNNEEDED|ISSTATIC|LOGCHATTY);
diff --git a/webcit/static/robots.txt b/webcit/static/robots.txt
deleted file mode 100644
index eb0536286..000000000
--- a/webcit/static/robots.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-User-agent: *
-Disallow: