# robots.txt file for NSLC Pacific web sites User-agent: ia_archiver Disallow: / User-agent: webcrawler Disallow:/ User-agent: * Disallow: /logs Disallow: /cgi-bin Disallow: /scripts