# robots.txt text file # 2006.08.23 Jim Weller # # allow the major crawlers but with a delay of 20 seconds between crawls #User-agent: Googlebot #Disallow: #User-agent: Googlebot-Image #Disallow: #User-agent: MSNBot #Disallow: #User-agent: Slurp #Disallow: #User-agent: ia_archiver #Disallow: #User-agent: * #Disallow: / #Crawl-delay: 20 # 6/1/2007 8:16AM Jim Weller User-agent: * Crawl-delay: 5