User-agent: facebookexternalhit
Crawl-delay: 0
Allow: /

User-agent: *
Disallow: /datastaging
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: EmailCollector
Disallow: /
User-agent: EmailSiphon
Disallow: /
User-agent: WebBandit
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: Web Downloader
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: Offline Explorer Pro
Disallow: /
User-agent: HTTrack Website Copier
Disallow: /
User-agent: Offline Commander
Disallow: /
User-agent: Leech
Disallow: /
User-agent: WebSnake
Disallow: /
User-agent: BlackWidow
Disallow: /
User-agent: HTTP Weazel
Disallow: /
User-agent: httrack
Disallow: /
User-agent: NetCaptor
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: SpiderKU
Disallow: /
User-agent: Steeler
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: webcrawler
Disallow: /
User-agent: Web Downloader
Disallow: /
User-agent: WebGather
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: Wget
Disallow: /
User-agent: Zao
Disallow: /
User-agent: Zeus
Disallow: /
User-agent: *
Disallow: /cgi-bin/

User-agent: *
crawl-delay: 10