# disallow all crawling for baddies
# please obey
User-agent: RPT HTTPClient
User-agent: sitecheck.internetseer.com
User-agent: Zealbot
User-agent: MSIECrawler
User-agent: SiteSnagger
User-agent: WebStripper
User-agent: WebCopier
User-agent: Fetch
User-agent: Offline Explorer
User-agent: Teleport
User-agent: TeleportPro
User-agent: WebZIP
User-agent: linko
User-agent: HTTrack
User-agent: Microsoft.URL.Control
User-agent: Xenu
User-agent: larbin
User-agent: libwww
User-agent: ZyBORG
User-agent: Download Ninja
Disallow: /

# disallow crawling of content for certain bots
User-agent: Slurp
User-agent: YahooSeeker
User-agent: Yahoo! Slurp
User-agent: Googlebot
User-agent: ArchitextSpider
User-agent: Lycos_Spider_(T-Rex)
User-agent: MSNBot
User-agent: MantraAgent
User-agent: Yahoo Slurp
User-agent: Architext spider
Disallow: /approved_template/
Disallow: /cgi-bin/
Disallow: /corpimage/
Disallow: /development/
Disallow: /images/
Disallow: /images_2001/
Disallow: /images_new/
Disallow: /issamples/
Disallow: /ljj/
Disallow: /mailtemplates/
Disallow: /members/ViewMembers.asp
Disallow: /mempubs/
Disallow: /network_template/
Disallow: /new_template_2/
Disallow: /regreform/
Disallow: /stats/
Disallow: /weblinx/admin/
Disallow: /weblinx/cgi-bin/
Disallow: /weblinx/development/
Disallow: /weblinx/diag/
Disallow: /weblinx/images/
Disallow: /weblinx/old/
Disallow: /weblinx/template/
Disallow: /weblinx/testproduct/
Disallow: /weblinx/testsurveys/
Disallow: /forgoogle.htm
Disallow: /default_old.htm
Disallow: /default_2.htm
Disallow: /default_3.htm
Disallow: /default_4.htm
Disallow: /finalversion-r.swf
Disallow: /finalversion-r_old.swf
Disallow: /index2.html
Disallow: /sniffer.swf
Disallow: /sniffer_new.swf
Disallow: /pricewaterhouse/
Disallow: /nosearch/
Disallow: /usurvey/
Disallow: /_private/
Disallow: /port-rail/
Disallow: /Renewable-Energy/
Disallow: /boardwiseii/LayoutPurchaseDocument.asp
Disallow: /boardwiseii/LayoutContact.asp
Disallow: /boardwiseii/LayoutSearchResults.asp
Disallow: /boardwiseii/Document.asp
Disallow: /boardwiseii/Logout.asp
Disallow: /documents/
Disallow: /order/
Disallow: /support/
Disallow: /documents_summary_EA.asp
Disallow: /documents_EA.asp
Disallow: /documents_summary_ea.asp
Disallow: /documents_ea.asp
Disallow: /document_summary_SP.asp
Disallow: /documents_summary_sp.asp
Disallow: /documents_summary_SP.asp
Disallow: /TopSecret/
Disallow: /public_nsdocs/
Disallow: /PUBLIC_NSDOCS/
Disallow: /networks/cem/members/
Disallow: /networks/chsdm/members/
Disallow: /networks/ccmts/members/
Disallow: /networks/skpf/members/
Disallow: /networks/bcs/members/
Disallow: /networks/hrdc/members/
Disallow: /networks/src/members/
Disallow: /networks/pegc/members/
Disallow: /networks/chsr/members/
Disallow: /networks/cli/members/
Disallow: /networks/psen/members/
Disallow: /networks/cemt/members/
Disallow: /networks/ccio/members/
Disallow: /networks/cfse/members/
Disallow: /networks/lpic/members/
Disallow: /files/
Disallow: /PUBLIC_NSDOCS/
Disallow: /globe/default.aspx
Disallow: /globe/ThankYou.aspx
Disallow: /Libraries/CONF_PRES_PUBLIC/
Disallow: /conf/presentations/
Disallow: /GAW/