# deny these

User-agent: *
Disallow: /admin/
Disallow: /adserver/
Disallow: /inc/
Disallow: /xml/
Disallow: /cgi-bin/
Disallow: /scripts/

User-agent: newsexpress
Disallow: /

User-agent: x-Tractor 
Disallow: /

User-agent: sitecheck.internetseer.com 
Disallow: /

User-agent: mobileGate-Spider 
Disallow: /

User-agent: IncyWincy 
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: WebHTTrack
Disallow: /

User-agent: WSB
Disallow: /

User-agent: ia_archiver/1.6
Disallow: /

User-agent: Openbot
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: VoilaBot
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: MetaGer-LinkChecker
Disallow: /

User-agent: appie
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: grub
Disallow: /

User-agent: larbin
Disallow: /

User-agent: WebSauger
Disallow: /

User-agent: CoolBot
Disallow: /

User-agent: Heritrix 
Disallow: /

User-agent: HTTrack 3.0x
Disallow: /

User-agent: psbot/0.1
Disallow: /

User-agent: augurfind
Disallow: /

User-agent: ASPseek/1.2.11pre
Disallow: /

User-agent: PHP/4.2.3
Disallow: /

User-agent: zyborg/1.0
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: Teleport Pro/1.29
Disallow: /