# limit  msnbot  to 100 requests per day (86400 seconds)
User-agent: msnbot
Crawl-delay: 864 

User-agent: crawl
Crawl-delay: 30

User-agent: crawl-9c
Crawl-delay: 30


User-agent: Go!Zilla 3.5
User-agent: SmartDownload/1.2.77
User-agent: NetZip-Downloader/1.MS0.62
User-agent: GetRight/3.2
User-agent: DA 4.0
User-agent: DA 5.0
User-agent: Offline Explorer/1.2
User-agent: Teleport Pro/1.29
User-agent: Wget/1.5.3 
User-agent: Wget/1.7
User-agent: sitecheck.internetseer.com
User-agent: ^Missigua Locator
User-agent: ^PEAR HTTP_Request class
User-agent: ^Java/1.4.1
User-agent: ^Java/1.5.0
User-agent: ^libwww-perl
User-agent: ^Wells Search II
User-agent: ^psycheclone
User-agent: ^WEP Search 00
User-agent: b3090822.crawl.yahoo.net
User-agent: AbachoBOT
User-agent: abcdatos_botlink
User-agent: AESOP_com_SpiderMan
User-agent: ah-ha.com crawler
User-agent: Scooter
User-agent: Mercator
User-agent: Scooter2_Mercator_3-1.0
User-agent: roach.smo.av.com-1.0
User-agent: Tv<nn>_Merc_resh_26_1_D-1.0
User-agent: AltaVista-Intranet
User-agent: FAST-WebCrawler
User-agent: Wget
User-agent: Acoon Robot
User-agent: antibot
User-agent: Atomz
User-agent: AxmoRobot
User-agent: Buscaplus Robi
User-agent: CanSeek/
User-agent: ChristCRAWLER
User-agent: Clushbot
User-agent: Googlebot/2.*ooglebot@googlebot.com)
User-agent: Msn â Msnbot
User-agent: Lycos â Lycos_Spider_(T-Rex)/3.
User-agent: Teoma- Mozilla/2.0 (compatible; Ask Jeeves/Teoma)
User-agent: Googlebot
User-agent: Mozilla
User-agent: Inktomi slurp
User-agent: InternetLinkAgent 3.1
User-agent: Istarthere
User-agent: Java 1.4.0
User-agent: JoBo 1.3
User-agent: K2spider
User-agent: KMcrawler
User-agent: Knowledge 0.2
User-agent: Knowledge 0.3
User-agent: Knowledge Engine
User-agent: Kuloko-bot 0.2
User-agent: Larbin
User-agent: Larbin 2.6.2
User-agent: Larbin 2.63
User-agent: Larbin 2.6.3
User-agent: Libwww-MGET 1.0
User-agent: Libwww-perl
User-agent: Libwww-perl 5.3.2
User-agent: Libwww-perl 5.4.0
User-agent: Libwww-perl 5.48
User-agent: Libwww-perl 5.50
User-agent: Libwww-perl 5.51
User-agent: Libwww-perl 5.52
User-agent: Libwww-perl 5.53
User-agent: Libwww-perl 5.63
User-agent: Libwww-perl 5.64
User-agent: Libwww-perl 5.65
User-agent: Libwww-perl 5.66
User-agent: Libwww-perl 5.68
User-agent: Libwww-perl 5.69
User-agent: Libwww-perl 5.74
User-agent: Libwww-perl 5.75
User-agent: Libwww-perl 5.76
User-agent: Libwww-perl 5.800
User-agent: Libwww-perl 5.801
User-agent: Libwww-perl 5.802
User-agent: Libwww-perl 5.803
User-agent: LimeBot 1.0
User-agent: Linkbot 3.0
User-agent: LinkLint-checkonly 2.3.5
User-agent: Linknzbot
User-agent: Linknzbot 2004
User-agent: Links SQL
User-agent: Lite Bot 0616B
User-agent: LNSpiderguy
User-agent: Look
User-agent: Lwp-request 1.29
User-agent: Lwp-request 1.35
User-agent: Lwp-request 1.36
User-agent: Lwp-request 2.01
User-agent: Lwp-request 5.48
User-agent: Lwp-request 5.65
User-agent: Lycos_Spider
User-agent: Mediapartners-Google 2.1
User-agent: Mercator 2.0
User-agent: Metacarta
User-agent: MetaGer-LinkChecker
User-agent: Microsoft-ATL-Native 7.00
User-agent: MicrosoftPrototypeCrawler
User-agent: Microsoft URL Control
User-agent: Moget 1.0
User-agent: Moget 2.1
User-agent: MozDex 0.04
User-agent: MozDex 0.05
User-agent: MS Data Access
User-agent: MS FrontPage 4.0
User-agent: MS FrontPage 5.0
User-agent: MSIECrawler
User-agent: Msnbot 0.1
User-agent: Msnbot 0.11
User-agent: Msnbot 0.3
User-agent: Msnbot 1.0
User-agent: MSProxy 2.0
User-agent: MSRBot 0.1
User-agent: MysteryBot
User-agent: NaverBot 1.0
User-agent: NaverBot_dloader 1.5
User-agent: NaverRobot 1.0
User-agent: NaverRobot 1.5
User-agent: NetAnts 1.25
User-agent: NetNoseCrawler 1.0
User-agent: NetNoseCrawler 2.0
User-agent: NetResearchServer
User-agent: NetResearchServer 2.4
User-agent: NetResearchServer 2.5
User-agent: NetResearchServer 2.7
User-agent: NetResearchServer 2.8
User-agent: NetResearchServer 2.9
User-agent: NetResearchServer 3.4
User-agent: NextGenSearchBot 1
User-agent: NG 1.0
User-agent: None
User-agent: NPBot
User-agent: NuSearch Spider
User-agent: NutchBot 0.05
User-agent: NutchBot 0.06
User-agent: NutchCrawler
User-agent: NutchOrg 0.03
User-agent: Obot
User-agent: Ocelli 1.3
User-agent: OmniExplorer_Bot 1.07
User-agent: Openbot 3.0
User-agent: OrangeBot
User-agent: Organica 2.24
User-agent: Overture-WebCrawler 3.8
User-agent: OWR_Crawler 0.1
User-agent: Parabot
User-agent: Patwebbot
User-agent: Pavuk 0.9
User-agent: PEERbot
User-agent: PipeLiner 0.3
User-agent: Planethosting
User-agent: Polybot 1.0
User-agent: Pompos 1.1
User-agent: Pompos 1.2
User-agent: Pompos 1.3
User-agent: Portal Manager 0.7
User-agent: Potbot 1.0
User-agent: Program Shareware 1.0.3
User-agent: ProWebGuide Link Checker
User-agent: Psbot 0.1
User-agent: Pverify 1.2
User-agent: PWS.Kiosk
User-agent: QPCreep Test Rig
User-agent: QuepasaCreep
User-agent: QuepasaCreep 0.9.14
User-agent: QuepasaCreep v0.9.13
User-agent: Reifier
User-agent: Rico 0.1
User-agent: RixBot
User-agent: RoboPal
User-agent: RobotMidareru 0.7
User-agent: Robots.txt Validator
User-agent: Robozilla 1.0
User-agent: RPT-HTTPClient 0.3
User-agent: SafariBookmarkChecker 1.25
User-agent: SafariBookmarkChecker 1.26
User-agent: Scooter 1.0
User-agent: Scooter 1.1
User-agent: Scooter 3.0
User-agent: Scooter 3.2
User-agent: Scooter 3.3
User-agent: Scrubby 2.1
User-agent: Scrubby 2.2
User-agent: Search Agent 1.0
User-agent: SearchSpider 1.1
User-agent: Seekbot 1.0
User-agent: Semanticdiscovery 0.1
User-agent: Sensis.au Web Crawler
User-agent: Sherlock 1.3
User-agent: Sherlock_spider
User-agent: Sitecheck
User-agent: Sitescooper 3.1.2
User-agent: SiteXpert
User-agent: Slurp*
User-agent: *slurp*
User-agent: SlySearch 1.3
User-agent: Sohu-search
User-agent: SpeedySpider
User-agent: Speedy Spider
User-agent: Speedy_Spider
User-agent: SpiderKU 0.9
User-agent: SpiderMonkey 7.04
User-agent: SpiderMonkey 7.06
User-agent: SpurlBot 0.2
User-agent: Sqworm 2.9.85
User-agent: Star Downloader
User-agent: Steeler 1.3
User-agent: SuperCleaner 2.56
User-agent: SYCLIKControl LinkChecker
User-agent: Szukacz 1.5
User-agent: Tarantula
User-agent: TclSOAP 1.0
User-agent: TclSOAP 2.3
User-agent: Teradex_Crawler
User-agent: Teradex Mapper
User-agent: TheSuBot 0.1
User-agent: TheSuBot Beta
User-agent: Thumbshots-de-Bot
User-agent: TimboBot 0.9
User-agent: Tkensaku 0.9
User-agent: TranSGeniKBot
User-agent: TranSGeniKBot www.tsgk.net
User-agent: TulipChain 5.7
User-agent: TulipChain 5.94
User-agent: TulipChain 6.01
User-agent: TulipChain 6.02
User-agent: TulipChain 6.03
User-agent: TurnitinBot 1.4
User-agent: TurnitinBot 1.5
User-agent: TurnitinBot 2.0
User-agent: TutorGigBot 1.5
User-agent: Tutorial Crawler 1.4
User-agent: UdmSearch 3.1.20
User-agent: UIowaCrawler 1.0
User-agent: UIowaCrawler 2.0
User-agent: Unchaos_crawler 2.0.2
User-agent: Updated 0.1
User-agent: USyd-NLP-Spider
User-agent: Vagabondo 2.0
User-agent: Vagabondo 2.1
User-agent: Vivante Link Checker
User-agent: Void-bot 0.1
User-agent: VoilaBot 1.6
User-agent: VSE 1.0
User-agent: Vspider
User-agent: W3C-checklink 2.90
User-agent: W3C-checklink 3.6.2.3
User-agent: W3C-checklink 3.9.2
User-agent: W3C-checklink 4.0
User-agent: W3C-checklink 4.1
User-agent: W3C_Validator 1.183
User-agent: W3C_Validator 1.305.2.109
User-agent: W3C_Validator 1.305.2.12
User-agent: W3C_Validator 1.305.2.137
User-agent: W3C_Validator 1.305.2.148
User-agent: Webbot
User-agent: Webclipping
User-agent: Webcollage 1.102
User-agent: Webcollage 1.104
User-agent: Webcollage 1.87
User-agent: Webcollage 1.93
User-agent: Webcollage 1.94
User-agent: WebcraftBoot
User-agent: WebcraftBoot 1.4.1
User-agent: WebcraftBoot 1.4.2
User-agent: WebFilter Robot 1.0
User-agent: Webinator-indexer 2.56
User-agent: WebRACE 1.1
User-agent: WebSauger 1.20
User-agent: WebSearch
User-agent: WebSearch 2.0.1
User-agent: WebSearch 3.0.1
User-agent: WebSearch 3.1.3
User-agent: WebSearch 3.1.6
User-agent: Webwombat
User-agent: Webyield robot
User-agent: Wget 1.5.2
User-agent: Wget 1.5.3
User-agent: Wget 1.5.3.1
User-agent: Wget 1.6
User-agent: Wget 1.7
User-agent: Wget 1.8
User-agent: Wget 1.8.1
User-agent: Wget 1.8.2
User-agent: Wget 1.9
User-agent: Wget 1.9.1
User-agent: Willow 2.1
User-agent: Wotbox alpha0.5.1
User-agent: Www.ciml.co.uk
User-agent: WWWeasel 1.00
User-agent: Xenu Link Sleuth 1.1a
User-agent: Xenu Link Sleuth 1.2b
User-agent: Xenu Link Sleuth 1.2d
User-agent: Xenu Link Sleuth 1.2e
User-agent: Xenu Link Sleuth 1.2f
User-agent: Yahoo-MMCrawler
User-agent: Yahoo! Slurp
User-agent: Yandex
User-agent: YottaCars_Bot 4.12
User-agent: Zao 0.1
User-agent: Zao 0.2
User-agent: Zao-Crawler
User-agent: Zeus 3140
User-agent: Zeus 57657
User-agent: ZipppBot 0.11
User-agent: ZoomSpider
User-agent: ZyBorg 1.0
Crawl-delay: 15
Disallow: /MENUFILES
Disallow: /legacysearch.php
Disallow: /_notes
Disallow: /3Dwork
Disallow: /advertise
Disallow: /advertise-0
Disallow: /aftermath
Disallow: /analog
Disallow: /ANTIQUES
Disallow: /ARCHIVE
Disallow: /archives
Disallow: /ART
Disallow: /ASINSTALLED
Disallow: /AUDIO
Disallow: /BACKUPS
Disallow: /banners
Disallow: /BUSINESS
Disallow: /cgi-bin
Disallow: /DATABASE
Disallow: /donate
Disallow: /FAQ
Disallow: /files
Disallow: /graphics
Disallow: /halloween
Disallow: /IMAGES
Disallow: /includes
Disallow: /JavaMenu
Disallow: /misc
Disallow: /modules
Disallow: /osCommerce
Disallow: /profiles
Disallow: /PROTESTPAGES
Disallow: /quickindex
Disallow: /scripts
Disallow: /sites
Disallow: /static
Disallow: /swfobject
Disallow: /themes
Disallow: /VANILLA
Disallow: /WEB_ACCESSORIES
Disallow: /webpageimages


User-agent: Slurp
Crawl-delay: 15
Disallow: /MENUFILES
Disallow: /legacysearch.php
Disallow: /_notes
Disallow: /3Dwork
Disallow: /advertise
Disallow: /advertise-0
Disallow: /aftermath
Disallow: /analog
Disallow: /ANTIQUES
Disallow: /ARCHIVE
Disallow: /archives
Disallow: /ART
Disallow: /ASINSTALLED
Disallow: /AUDIO
Disallow: /BACKUPS
Disallow: /banners
Disallow: /BUSINESS
Disallow: /cgi-bin
Disallow: /DATABASE
Disallow: /donate
Disallow: /FAQ
Disallow: /files
Disallow: /graphics
Disallow: /halloween
Disallow: /IMAGES
Disallow: /includes
Disallow: /JavaMenu
Disallow: /misc
Disallow: /modules
Disallow: /osCommerce
Disallow: /profiles
Disallow: /PROTESTPAGES
Disallow: /quickindex
Disallow: /scripts
Disallow: /sites
Disallow: /static
Disallow: /swfobject
Disallow: /themes
Disallow: /VANILLA
Disallow: /WEB_ACCESSORIES
Disallow: /webpageimages



User-agent: ""
User-agent: ^Java
User-agent: ^Jakarta
User-agent: User-agent
User-agent: compatible ;
User-agent: "Mozilla"
User-agent: panscient.com
User-agent: libwww, lwp-trivial, curl, PHP/, urllib, GT::WWW, Snoopy, MFC_Tear_Sample, HTTP::Lite, PHPCrawl, URI::Fetch, Zend_Http_Client, http client, PECL::HTTP
User-agent: [A-Z][a-z]{3,} [a-z]{4,} [a-z]{4,}
User-agent: larbin
User-agent: heritrix
User-agent: WEP Search, Wells Search II, Missigua Locator, ISC Systems iRc Search 2.1, Microsoft URL Control, Indy Library
User-agent: Googlebot-Image
User-agent: Nutch
Disallow: /


User-agent: *
Disallow: /MENUFILES
Disallow: /_notes
Disallow: /3Dwork
Disallow: /aftermath
Disallow: /analog
Disallow: /ANTIQUES
Disallow: /ARCHIVE
Disallow: /archives
Disallow: /ASINSTALLED
Disallow: /AUDIO
Disallow: /BACKUPS
Disallow: /BUSINESS
Disallow: /DATABASE
Disallow: /donate
Disallow: /FAQ
Disallow: /files
Disallow: /JavaMenu