User-agent: Teoma Crawl-delay: 10 User-agent: Slurp Crawl-delay: 10 User-agent: aipbot Disallow: / User-agent: BecomeBot Disallow: / User-agent: psbot Disallow: / # Disallow all crawlers access to certain pages. # This says to apply these settings to ALL search engine spiders/crawlers User-agent: * Disallow: /catalog/account.php Disallow: /catalog/advanced_search.php Disallow: /catalog/checkout_shipping.php Disallow: /catalog/create_account.php Disallow: /catalog/includes Disallow: /catalog/login.php Disallow: /catalog/password_forgotten.php Disallow: /catalog/popup_image.php Disallow: /catalog/shopping_cart.php Disallow: /cookie_usage.php Disallow: /tests/ Disallow: /trash/ Disallow: /noadmit/ Disallow: /cgi-bin/ Disallow: /images/ Disallow: /rmdirectory/banner_images/ Disallow: /rmdirectory/Net/ Disallow: /rmdirectory/admin.php Disallow: /rmdirectory/adminf.php Disallow: /rmdirectory/adminh.php Disallow: /rmdirectory/admintemplate.php Disallow: /rmdirectory/backup.txt Disallow: /rmdirectory/crawler_class.php Disallow: /rmdirectory/cronJob.php Disallow: /rmdirectory/cronjob_help.php Disallow: /rmdirectory/ez_sql.php Disallow: /rmdirectory/html_import.php Disallow: /rmdirectory/import_funcs.php Disallow: /rmdirectory/linksmanager_import.php Disallow: /rmdirectory/makepages.php Disallow: /rmdirectory/parselinks.php Disallow: /rmdirectory/PEAR.php Disallow: /rmdirectory/Read_First.html Disallow: /rmdirectory/rm.php Disallow: /rmdirectory/template.php Disallow: /rmdirectory/template_help.php Disallow: /rmdirectory/update_huge.php Disallow: /rmdirectory/url_funcs.php Disallow: /rmdirectory/validatelinks.php Disallow: /rmdirectory/zeus_import.php Disallow: /stats/ Disallow: /_private/