# Robots.txt file updated January 10, 2008 # All robots will spider the domain User-agent: * Disallow: # Disallow directory /cgi-bin/ User-agent: * Disallow: /cgi-bin/ # Disallow directory /custom-code/ User-agent: * Disallow: /custom-code/ # Disallow directory /database/ User-agent: * Disallow: /database/ # Disallow directory /js/ User-agent: * Disallow: /js/ # Disallow directory /shopping/ User-agent: * Disallow: /shopping/