# Robots.txt
#
# All robots will spider the domain

User-agent: *
Disallow: /cgi-bin/ 

User-Agent: msnbot 
Disallow: /cgi-bin/ 
Crawl-Delay: 5