# ===========================
# Robots.txt for GrownBrilliance.com
# Last updated: 2025-07-08
# ===========================

# GLOBAL RULES
User-agent: *
Crawl-delay: 20

# Allow Googlebot full access
User-agent: Googlebot
Disallow:

User-agent: Googlebot-Image
Disallow:

User-agent: AdsBot-Google
Disallow:


# OPTIONAL: Allow LLM Crawlers
User-agent: GPTBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /


# Site audit tools
User-agent: Sitebulb
Crawl-delay: 2

User-agent: Pinterestbot
Crawl-delay: 1


# BLOCKED BOTS (Bad/Irrelevant/Malicious)
User-agent: archive.org_bot
User-agent: Yahoo! Slurp
User-agent: aranhabot
User-agent: Baiduspider
User-agent: Yandex
User-agent: GrapeshotCrawler
User-agent: Gluten Free Crawler
User-agent: FlipboardProxy
User-agent: MJ12bot
User-agent: LexxeBot
User-agent: oBot
User-agent: SputnikBot
User-agent: Symfony Spider
User-agent: EasouSpider
User-agent: panscient.com
User-agent: NextGenSearchBot
User-agent: rogerbot
User-agent: SiteBot
User-agent: crystalsemanticsbot
User-agent: CrystalSemanticsBot
User-agent: NetSeer crawler
User-agent: trovitBot
User-agent: Sosospider
User-agent: DotBot
User-agent: Ezooms
User-agent: discobot
User-agent: Jyxobot
User-agent: sogou
User-agent: Sogou web spider
User-agent: sistrix
User-agent: heritrix
User-agent: GarlikCrawler
User-agent: NerdByNature.Bot
User-agent: psbot
User-agent: WBSearchBot
User-agent: AddThis
User-agent: ia_archiver
User-agent: proximic
User-agent: discoverybot
User-agent: bl.uk_lddc_bot
User-agent: IstellaBot
User-agent: seokicks
User-agent: UnisterBot
User-agent: Bender
User-agent: wotbox
User-agent: JikeSpider
User-agent: netEstate NE Crawler
User-agent: Exabot
User-agent: Pixray-Seeker
User-agent: Linguee
User-agent: integromedb
User-agent: SearchmetricsBot
User-agent: BLEXBot
User-agent: BDCbot
User-agent: WeSEE:Search
User-agent: TurnitinBot
User-agent: admantx
User-agent: spbot
User-agent: Mail.RU_Bot
User-agent: Dot-Bot
User-agent: BTT Agent SFOAWS
User-agent: ProductAdsBot
User-agent: python-requests
User-agent: viralvideochart
User-agent: Apache-HttpClient
User-agent: FatBot
User-agent: FeedBurner
User-agent: GWPImages
User-agent: XoviBot
User-agent: tbot-nutch
User-agent: Typhoeus
User-agent: Android
User-agent: NING
User-agent: Mediatoolkitbot
User-agent: stores.pl
User-agent: ChimpFeedr.com
User-agent: Genieo
User-agent: ravencrawler
User-agent: CRAZYWEBCRAWLER
User-agent: CommonCrawler
User-agent: magpie-crawler
User-agent: R6_CommentReader
User-agent: Kyoto-Tohoku-Crawler
User-agent: ICC-Crawler
User-agent: Y!J-ASR
User-agent: WorldClient.dll
User-agent: ltx71
User-agent: Alexabot
User-agent: PetalBot

Disallow: /


# BLOCKED FILE TYPES (e.g., PDFs)
User-agent: Powermapper
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: *?algoliaQueryId*

# BLOCKED PARAMETERS AND DIRECTORIES
User-agent: *
Disallow: /*?*
Disallow: /document
Disallow: /pma_28535
Disallow: /staging
Disallow: /rebranding
Disallow: /navy
Disallow: /gb-panel
Disallow: /cron_jobs
Disallow: /login
Disallow: /shoppingcart
Disallow: /checkout
Disallow: /diamond_search_result_ajax
Disallow: /diamond_search_result_ajax?*
Disallow: /sucuri-backups
Disallow: /blog/wp-admin
Disallow: /blog/wp-admin/*
Disallow: /boca-customer-contact
Disallow: /hou-customer-contact
Disallow: /kop-customer-contact
Disallow: /nyc-customer-contact
Disallow: /hj-event-sign-up-form
Disallow: /brilliant-upgrades



# SITEMAPS
Sitemap: https://www.grownbrilliance.com/sitemapindex.xml
Sitemap: https://www.grownbrilliance.com/sitemap_category_static_pages.xml
Sitemap: https://www.grownbrilliance.com/sitemap_product_pages_1.xml
Sitemap: https://www.grownbrilliance.com/sitemap_product_pages_2.xml
Sitemap: https://www.grownbrilliance.com/sitemap_product_pages_3.xml
Sitemap: https://www.grownbrilliance.com/sitemap_product_pages_4.xml
Sitemap: https://www.grownbrilliance.com/sitemap_product_video_1.xml
Sitemap: https://www.grownbrilliance.com/sitemap_dyo_gemstone_product.xml
Sitemap: https://www.grownbrilliance.com/blog/post-sitemap1.xml
Sitemap: https://www.grownbrilliance.com/blog/page-sitemap1.xml
Sitemap: https://www.grownbrilliance.com/blog/category-sitemap1.xml
Sitemap: https://www.grownbrilliance.com/blog/post_tag-sitemap1.xml


#    ____                           ____       _ _ _ _                      
#   / ___|_ __ _____      ___ __   | __ ) _ __(_) | (_) __ _ _ __   ___ ___ 
#  | |  _| '__/ _ \ \ /\ / / '_ \  |  _ \| '__| | | | |/ _` | '_ \ / __/ _ \
#  | |_| | | | (_) \ V  V /| | | | | |_) | |  | | | | | (_| | | | | (_|  __/
#   \____|_|  \___/ \_/\_/ |_| |_| |____/|_|  |_|_|_|_|\__,_|_| |_|\___\___|
#