2012-02-22 09:54:49 +00:00
|
|
|
User-agent: *
|
|
|
|
|
# comment lists change all the time, crawlers should
|
|
|
|
|
# index individual image's comments
|
|
|
|
|
Disallow: /comment/
|
|
|
|
|
# next and prev are just CPU-heavier ways of getting
|
|
|
|
|
# to the same images that the index shows
|
|
|
|
|
Disallow: /post/next/
|
|
|
|
|
Disallow: /post/prev/
|
2012-06-05 13:12:58 +01:00
|
|
|
# Site is rate limited to 1 request / sec,
|
|
|
|
|
# returns 503 for more than that
|
|
|
|
|
Crawl-delay: 3
|
