# Disallow all crawlers access to certain pages. User-agent: * Disallow: /AccountArea/ Disallow: /erp/ Disallow: /researchpapers/iframe/ Disallow: /researchpapers/price2.asp Disallow: /researchpapers/cap.asp # Disallow Following crawlers access to the site User-agent: ia_archiver-web.archive.org Disallow: / User-agent: ia_archiver Disallow: / User-agent: ia_archiver-web.archive.org Disallow: /robots.txt User-agent: ia_archiver-web.archive.org Disallow: robots.txt User-agent: ia_archiver Disallow: /robots.txt User-agent: ia_archiver Disallow: robots.txt User-agent: AboutUsBot Disallow: /