User-agent: *
Allow: /
Crawl-delay: 20
User-agent: amazonaws.com
Disallow: /
User-agent: http://www.80legs.com/webcrawler.html
Disallow: /