Revert "Return 404 for robots.txt"

This reverts commit 7632402d82d9d7f7a4c20c1a3f5c0a3ea13c38b0.

diff --git a/webcit/context_loop.c b/webcit/context_loop.c
index 1f140184c5f750bdaf958c8771e54f6e68ec4323..266b2f4c7d4a4ffb328bc1dc390236493bec0245 100644 (file)
--- a/webcit/context_loop.c
+++ b/webcit/context_loop.c
@@ -805,7 +805,6 @@ InitModule_CONTEXT
        RegisterNamespace("NONCE", 0, 0, tmplput_nonce, NULL, 0);
 
        WebcitAddUrlHandler(HKEY("404"), "", 0, do_404, ANONYMOUS|COOKIEUNNEEDED);
-       WebcitAddUrlHandler(HKEY("robots.txt"), "", 0, do_404, ANONYMOUS|COOKIEUNNEEDED);
 /*
  * Look for commonly-found probes of malware such as worms, viruses, trojans, and Microsoft Office.
  * Short-circuit these requests so we don't have to send them through the full processing loop.