www.robotstxt.org/norobots-rfc.txt # banned User-agent: TurnitinBot Disallow: / User-agent: Zealbot Disallow: / # Yahoo Slurp: so they don't see cross-links User-agent: Slurp Crawl-delay: 5 Disallow: /about/ # all others User-agent: * Crawl-delay: 5 Disallow: /about/privacy-policy Disallow: /about/terms-of-use Disallow: /request-info/ Disallow: /site/common/js/ # Sitemap Sitemap: allcareerschools.com/sitemap-index.xml