User-agent: Googlebot Disallow: /rss/ User-agent: MSNBot Disallow: /rss/ User-agent: Slurp Disallow: /rss/ User-agent: * Disallow: /write-review.html Disallow: /saved-software.html Disallow: /rd.html # Limit ScoutJet's crawl rate (example is to crawl no more than 1 page every 1 second) User-agent: ScoutJet Crawl-delay: 1