# Crawl-delay for major search engines (in seconds between requests)
# Note: Googlebot ignores Crawl-delay, use Google Search Console instead

User-agent: Bingbot
Crawl-delay: 5

User-agent: msnbot
Crawl-delay: 5

User-agent: Yandex
Crawl-delay: 10

User-agent: Baiduspider
Crawl-delay: 10

User-agent: DuckDuckBot
Crawl-delay: 5

User-agent: Sogou
Crawl-delay: 10

User-agent: Exabot
Crawl-delay: 10

User-agent: MojeekBot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: BLEXBot
Crawl-delay: 10

User-agent: PetalBot
Crawl-delay: 10

User-agent: Applebot
Crawl-delay: 5

# AI search & discovery crawlers - ALLOWED with rate limits
# These power AI shopping assistants and search engines (ChatGPT, Perplexity, Gemini, etc.)
User-agent: GPTBot
Crawl-delay: 5

User-agent: ChatGPT-User
Crawl-delay: 5

User-agent: anthropic-ai
Crawl-delay: 5

User-agent: Claude-Web
Crawl-delay: 5

User-agent: ClaudeBot
Crawl-delay: 5

User-agent: Claude-SearchBot
Crawl-delay: 5

User-agent: Claude-User
Crawl-delay: 5

User-agent: OAI-SearchBot
Crawl-delay: 5

User-agent: Amazonbot
Crawl-delay: 5

User-agent: DuckAssistBot
Crawl-delay: 5

User-agent: MistralAI-User
Crawl-delay: 5

User-agent: Perplexity-User
Crawl-delay: 5

User-agent: Google-Extended
Crawl-delay: 5

User-agent: PerplexityBot
Crawl-delay: 5

User-agent: FacebookBot
Crawl-delay: 5

# Block training-only crawlers (no search/discovery value)
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Brightbot
Disallow: /

User-agent: *
Crawl-delay: 2

# Standard Magento directories
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/

# Standard Magento paths
Disallow: /catalog/product_compare/
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /customer/
Disallow: /newsletter/
Disallow: /review/
Disallow: /wishlist/

# Block all layered navigation filters and sorting
# This is the most important section to add
Disallow: /*?age=
Disallow: /*?framed=
Disallow: /*?main_colour=
Disallow: /*?medium=
Disallow: /*?orientation=
Disallow: /*?size=
Disallow: /*?state=
Disallow: /*?style=
Disallow: /*?subject=
Disallow: /*?product_list_dir=
Disallow: /*?product_list_limit=
Disallow: /*?product_list_mode=
Disallow: /*?product_list_order=
Disallow: /*?q=
Disallow: /*?*newsletter_num=

# Block AJAX endpoints and tracking URLs (not useful for search)
Disallow: /sorting/track/
Disallow: /shippingestimator/
Disallow: /wishlistpanel/
Disallow: /rest/
Disallow: /graphql

# Disallow URLs with more than one parameter (good catch-all)
Disallow: /*?*&

# Do not index session IDs
Disallow: /*?SID=

Sitemap: https://www.sulisfineart.com/media/sitemaps/sitemap_sulis_group.xml