Too many planets, no point in crawling them

diff --git a/apache-conf.conf b/apache-conf.conf
index 881105a7380417344bb9d1790d84c7c5bd783eb6..d28cbcac730b15003b81cf42ca16fa4f0620423c 100644 (file)
--- a/apache-conf.conf
+++ b/apache-conf.conf
@@ -60,6 +60,7 @@ NameVirtualHost *:443
 
        Alias /static /var/www/ndawn/root/static
        Alias /favicon.ico /var/www/ndawn/root/favicon.ico
+       Alias /robots.txt /var/www/ndawn/root/robots.txt
 
        FastCgiExternalServer /tmp/ndweb.fcgi -socket /tmp/ndweb.socket
        Alias / /tmp/ndweb.fcgi/

diff --git a/root/robots.txt b/root/robots.txt
new file mode 100644 (file)
index 0000000..2b48efa
--- /dev/null
+++ b/root/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /stats/planet/