From 16f557954d8bfe9a11c1856d7e93945ab8120704 Mon Sep 17 00:00:00 2001 From: Shish Date: Tue, 5 Jun 2012 13:12:58 +0100 Subject: [PATCH] slow down crawlers to 1/3 hit/s; nginx will 503 if more than 1/2 hit/s --- lib/static/robots.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/static/robots.txt b/lib/static/robots.txt index 9a42a35c..e9ae5fde 100644 --- a/lib/static/robots.txt +++ b/lib/static/robots.txt @@ -6,3 +6,6 @@ Disallow: /comment/ # to the same images that the index shows Disallow: /post/next/ Disallow: /post/prev/ +# Site is rate limited to 1 request / sec, +# returns 503 for more than that +Crawl-delay: 3