diff --git a/lib/static/robots.txt b/lib/static/robots.txt index 9a42a35c..e9ae5fde 100644 --- a/lib/static/robots.txt +++ b/lib/static/robots.txt @@ -6,3 +6,6 @@ Disallow: /comment/ # to the same images that the index shows Disallow: /post/next/ Disallow: /post/prev/ +# Site is rate limited to 1 request / sec, +# returns 503 for more than that +Crawl-delay: 3