# Set the crawl delay to 5 seconds - not all search engines will honour this
User-agent: *
Crawl-delay: 5

# Tell all user agents to ignore wp-admin
User-agent: *
Disallow: /wp-admin/

# Tell all user agents to ignore URLs with querystrings
# IMPORTANT: Review this rule. It blocks all URLs with a '?' including
# pagination, filters, and some search results that might be important to you.
User-agent: *
Disallow: /?

# NEW: Exclude /staff/ pages and all sub-pages
User-agent: *
Disallow: /staff/
Disallow: /careers/

# ----------------------------------------------------
# BLOCK SPECIFIC BOTS (AI, Scrapers, Malicious, etc.)
# IMPORTANT: Each bot must have its own "User-agent:" and "Disallow: /" lines.
# This list is comprehensive. Consider if all of these need full site blocking.
# ----------------------------------------------------

# AI Bots (originally commented out, now explicitly blocked if uncommented)
# If you want to block these, uncomment each block below.
# User-agent: AI2Bot
# Disallow: /
# User-agent: Ai2Bot-Dolma
# Disallow: /
# User-agent: aiHitBot
# Disallow: /
# User-agent: Amazonbot
# Disallow: /
# User-agent: Andibot
# Disallow: /
# User-agent: anthropic-ai
# Disallow: /
# User-agent: Applebot
# Disallow: /
# User-agent: Applebot-Extended
# Disallow: /
# User-agent: Brightbot 1.0
# Disallow: /
# User-agent: Bytespider
# Disallow: /
# User-agent: CCBot
# Disallow: /
# User-agent: ChatGPT-User
# Disallow: /
# User-agent: Claude-SearchBot
# Disallow: /
# User-agent: Claude-User
# Disallow: /
# User-agent: Claude-Web
# Disallow: /
# User-agent: ClaudeBot
# Disallow: /
# User-agent: cohere-ai
# Disallow: /
# User-agent: cohere-training-data-crawler
# Disallow: /
# User-agent: Cotoyogi
# Disallow: /
# User-agent: Crawlspace
# Disallow: /
# User-agent: Diffbot
# Disallow: /
# User-agent: DuckAssistBot
# Disallow: /
# User-agent: FacebookBot
# Disallow: /
# User-agent: Factset_spyderbot
# Disallow: /
# User-agent: FirecrawlAgent
# Disallow: /
# User-agent: FriendlyCrawler
# Disallow: /
# User-agent: Google-CloudVertexBot
# Disallow: /
# User-agent: Google-Extended
# Disallow: /
# User-agent: GoogleOther
# Disallow: /
# User-agent: GoogleOther-Image
# Disallow: /
# User-agent: GoogleOther-Video
# Disallow: /
# User-agent: GPTBot
# Disallow: /
# User-agent: iaskspider/2.0
# Disallow: /
# User-agent: ICC-Crawler
# Disallow: /
# User-agent: ImagesiftBot
# Disallow: /
# User-agent: img2dataset
# Disallow: /
# User-agent: ISSCyberRiskCrawler
# Disallow: /
# User-agent: Kangaroo Bot
# Disallow: /
# User-agent: meta-externalagent
# Disallow: /
# User-agent: Meta-ExternalAgent
# Disallow: /
# User-agent: meta-externalfetcher
# Disallow: /
# User-agent: Meta-ExternalFetcher
# Disallow: /
# User-agent: MistralAI-User/1.0
# Disallow: /
# User-agent: NovaAct
# Disallow: /
# User-agent: OAI-SearchBot
# Disallow: /
# User-agent: omgili
# Disallow: /
# User-agent: omgilibot
# Disallow: /
# User-agent: Operator
# Disallow: /
# User-agent: PanguBot
# Disallow: /
# User-agent: Panscient
# Disallow: /
# User-agent: panscient.com
# Disallow: /
# User-agent: Perplexity-User
# Disallow: /
# User-agent: PerplexityBot
# Disallow: /
# User-agent: PetalBot
# Disallow: /
# User-agent: PhindBot
# Disallow: /
# User-agent: QualifiedBot
# Disallow: /
# User-agent: QuillBot
# Disallow: /
# User-agent: quillbot.com
# Disallow: /
# User-agent: SBIntuitionsBot
# Disallow: /
# User-agent: Scrapy
# Disallow: /
# User-agent: Sidetrade indexer bot
# Disallow: /
# User-agent: TikTokSpider
# Disallow: /
# User-agent: Timpibot
# Disallow: /
# User-agent: VelenPublicWebCrawler
# Disallow: /
# User-agent: Webzio-Extended
# Disallow: /
# User-agent: wpbot
# Disallow: /
# User-agent: YandexAdditional
# Disallow: /
# User-agent: YandexAdditionalBot
# Disallow: /
# User-agent: YouBot
# Disallow: /


# Other specific bots to block from the entire site
User-agent: AITCSRoboti
Disallow: /
User-agent: Accoona
Disallow: /
User-agent: admantx
Disallow: /
User-agent: admantx-usaspb
Disallow: /
User-agent: adbeat_bot
Disallow: /
User-agent: aiHitBot
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: Arachnophilia
Disallow: /
User-agent: AspiegelBot
Disallow: /
User-agent: AwarioSmartBot
Disallow: /
User-agent: BackDoorBot
Disallow: /
User-agent: BackRub
Disallow: /
User-agent: Baidu
Disallow: /
User-agent: BLEXbot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: BecomeBot
Disallow: /
User-agent: BlowFishi
Disallow: /
User-agent: BomboraBot
Disallow: /
User-agent: CatchBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: CherryPicker
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Clickagy
Disallow: /
User-agent: Cliqzbot
Disallow: /
User-agent: coccocbot
Disallow: /
User-agent: ConveraCrawler
Disallow: /
User-agent: contxbot
Disallow: /
User-agent: CrowdTanglebot
Disallow: /
User-agent: CyberSpyder
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: EchoboxBot
Disallow: /
User-agent: EmailCollector
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: Eyeotabot
Disallow: /
User-agent: findlinks
Disallow: /
User-agent: Foobot
Disallow: /
User-agent: Genieo
Disallow: /
User-agent: GetURL
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: GrapeshotCrawler
Disallow: /
User-agent: GumGum
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: Huaweisymantecspider
Disallow: /
User-agent: IAScrawler
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: JikeSpider
Disallow: /
User-agent: Jobboerse
Disallow: /
User-agent: Java
Disallow: /
User-agent: Jyxobot
Disallow: /
User-agent: Leikibot
Disallow: /
User-agent: LinkScan
Disallow: /
User-agent: LinkisBot
Disallow: /
User-agent: linkdexbot
Disallow: /
User-agent: linkfluence.com
Disallow: /
User-agent: LivelapBot
Disallow: /
User-agent: Mail.RU_Bot
Disallow: /
User-agent: MauiBot
Disallow: /
User-agent: MAZBot
Disallow: /
User-agent: MBCrawler
Disallow: /
User-agent: MegaIndex.ru
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: MojeekBot
Disallow: /
User-agent: mtbot/1.1.0i
Disallow: /
User-agent: NerdyBot
Disallow: /
User-agent: Nimbostratus-Bot
Disallow: /
User-agent: NTENTbot
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: Onespot-ScraperBot
Disallow: /
User-agent: Openbot
Disallow: /
User-agent: OutclicksBot
Disallow: /
User-agent: PaperLiBot
Disallow: /
User-agent: perl
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: PlurkBot
Disallow: /
User-agent: proximic
Disallow: /
User-agent: Proximi
Disallow: /
User-agent: python
Disallow: /
User-agent: Quantcastboti
Disallow: /
User-agent: Qwantify
Disallow: /
User-agent: ScholarBot
Disallow: /
User-agent: Scrap
Disallow: /
User-agent: Screaming Frog SEO Spider
Disallow: /
User-agent: Semantici
Disallow: /
User-agent: SentiBot
Disallow: /
User-agent: SEOkicks
Disallow: /
User-agent: SEOkicks-Robot
Disallow: /
User-agent: SerendeputyBot
Disallow: /
User-agent: serpstatbot
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: SiteCheck-sitecrawl
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: Snooper
Disallow: /
User-agent: Sogou
Disallow: /
User-agent: Sosospider
Disallow: /
User-agent: SuperBot
Disallow: /
User-agent: Taboolabot
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: TkBot
Disallow: /
User-agent: TTD-Content
Disallow: /
User-agent: TweetmemeBot
Disallow: /
User-agent: URLSpiderPro
Disallow: /
User-agent: Vagabondo
Disallow: /
User-agent: VelenPublicWebCrawler
Disallow: /
User-agent: VoilaBot
Disallow: /
User-agent: VoluumDSP-content-bot
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: weborama-fetcher
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: Xaldon_WebSpider
Disallow: /
User-agent: YaK
Disallow: /
User-agent: Yandex
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: YandexImages
Disallow: /
User-agent: ZGrab
Disallow: /
User-agent: ZoominfoBot
Disallow: /
User-agent: Scrapy
Disallow: /
User-agent: Buck
Disallow: /
User-agent: TinyTestBot
Disallow: /
User-agent: SEMrushBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: MJ12Bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: MauiBot
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: WhatStuffWhereBot
Disallow: /
User-agent: Applebot
Disallow: /
User-agent: Sogou Pic Spider/3.0( http://www.sogou.com/docs/help/webmasters.htm#07)
Disallow: /
User-agent: Sogou head spider/3.0( http://www.sogou.com/docs/help/webmasters.htm#07)
Disallow: /
User-agent: Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)
Disallow: /
User-agent: User-agent: Sogou Orion spider/3.0( http://www.sogou.com/docs/help/webmasters.htm#07)
Disallow: /
User-agent: Sogou-Test-Spider/4.0 (compatible; MSIE 5.5; Windows 98)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)
Disallow: /
User-agent: Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)
Disallow: /
User-agent: Swiftbot
Disallow: /
User-agent: Slurp
Disallow: /
User-agent: CCBot/2.0 (https://commoncrawl.org/faq/)
Disallow: /
User-agent: CCBot/2.0
Disallow: /
User-agent: CCBot/2.0 (http://commoncrawl.org/faq/)
Disallow: /

# ----------------------------------------------------
# EXPLICITLY ALLOWED BOTS
# These bots will override any Disallow rules that precede them for their specific user-agent.
# An empty Disallow: line means all content is allowed for that user-agent.
# ----------------------------------------------------
User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

User-agent: DuckDuckBot
Disallow:

# ---------------------------------
# END BOOKSWARM ROBOTS.TXT TEMPLATE
#         _
#        [ ]
#       (   )
#        |>|
#    __/===\__
#   //| o=o |\\
# <]  | o=o |  [>
#     \=====/
#    / / | \ \
#   <_________>