slow down crawlers to 1/3 hit/s; nginx will 503 if more than 1/2 hit/s

2012-06-05 13:12:58 +01:00 · 2012-06-05 13:12:58 +01:00 · 16f557954d
commit 16f557954d
parent 8de1bd0c56
1 changed files with 3 additions and 0 deletions
--- a/lib/static/robots.txt
+++ b/lib/static/robots.txt
@ -6,3 +6,6 @@ Disallow: /comment/
 # to the same images that the index shows
 Disallow: /post/next/
 Disallow: /post/prev/
+# Site is rate limited to 1 request / sec,
+# returns 503 for more than that
+Crawl-delay: 3