# Robots.txt # Disallow all User-agent: * # Disallow directories User-agent: * Disallow: /Merchant2/ User-agent: * Disallow: /graphics/ User-agent: * Disallow: /cgi-bin/ User-agent: * Disallow: /downloads/ User-agent: * Disallow: /subscribe2/ User-agent: * Disallow: /webstats/ User-agent: * Disallow: /ioncube/ User-agent: * Disallow: /contacts/ User-agent: * Disallow: /page/ User-agent: * Disallow: /ae_v8/ # Disallow indexing of specific file extensions User-agent: * Disallow: /*.zip$ User-agent: * Disallow: /*.doc$ User-agent: * Disallow: /*.txt$ # Disallow Collectors and Spam User-agent: atSpider Disallow: / User-agent: cherrypicker Disallow: / User-agent: DSurf Disallow: / User-agent: EliteSys Entry Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: Mail Sweeper Disallow: / User-agent: munky Disallow: / User-agent: Roverbot Disallow: / User-agent: WebEmailExtrac Disallow: / # Disallow Offline Browsers User-agent: xGet Disallow: / User-agent: wGet Disallow: / User-agent: WebWalk Disallow: / User-agent: webvac Disallow: / User-agent: WebReaper Disallow: / User-agent: WebMirror Disallow: / User-agent: WebFetcher Disallow: / User-agent: WebCopy Disallow: / User-agent: webcopier Disallow: / User-agent: WebCatcher Disallow: / User-agent: WebBandit Disallow: / User-agent: w3mir Disallow: / User-agent: vobsub Disallow: / User-agent: Templeton Disallow: / User-agent: ssearcher100 Disallow: / User-agent: SpiderBot Disallow: / User-agent: Shai'Hulud Disallow: / User-agent: PBWF Disallow: / User-agent: LightningDownload Disallow: / User-agent: KDD Exploror Disallow: / User-agent: Jeeves Disallow: / User-agent: Internet Explore Disallow: / User-agent: InfoSpiders Disallow: / User-agent: httrack Disallow: / User-agent: HavIndex Disallow: / User-agent: GetUrl Disallow: / User-agent: GetBot Disallow: / User-agent: ESIRover Disallow: / User-agent: Download Wonder Disallow: / User-agent: Collage Disallow: / User-agent: Mozilla/2.0 (compatible; MS FrontPage 4.0) Disallow: / User-agent: BizBot04 kirk.overleaf.com Disallow: / User-agent: HappyBot (gserver.kw.net) Disallow: / User-agent: CaliforniaBrownSpider Disallow: / User-agent: EI*Net/0.1 libwww/0.1 Disallow: / User-agent: Ibot/1.0 libwww-perl/0.40 Disallow: / User-agent: Merritt/1.0 Disallow: / User-agent: StatFetcher/1.0 Disallow: / User-agent: TeacherSoft/1.0 libwww/2.17 Disallow: / User-agent: WWW Collector Disallow: / User-agent: processor/0.0ALPHA libwww-perl/0.20 Disallow: / User-agent: wobot/1.0 from 206.214.202.45 Disallow: / User-agent: Libertech-Rover www.libertech.com? Disallow: / User-agent: WhoWhere Robot Disallow: / User-agent: ITI Spider Disallow: / User-agent: w3index Disallow: / User-agent: MyCNNSpider Disallow: / User-agent: SummyCrawler Disallow: / User-agent: OGspider Disallow: / User-agent: linklooker Disallow: / User-agent: CyberSpyder (amant@www.cyberspyder.com) Disallow: / User-agent: SlowBot Disallow: / User-agent: heraSpider Disallow: / User-agent: Surfbot Disallow: / User-agent: Bizbot003 Disallow: / User-agent: SandBot Disallow: / User-agent: EnigmaBot Disallow: / User-agent: spyder3.microsys.com Disallow: / User-agent: www.freeloader.com. Disallow: /