User-agent: *
Allow: /
Disallow: /Identity/
Disallow: /Checkout/
Disallow: /Cart
Disallow: /cart
Disallow: /Proforma/
Disallow: /Parts/LoadDiagrams/
Disallow: /Parts/LoadModels
Crawl-delay: 10

# --- AI Bots (BLOCKED) ---
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Google-Extended
User-agent: PerplexityBot
User-agent: Bytespider
Disallow: /

# Meta / Facebook AI
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
Disallow: /

# Apple AI training
User-agent: Applebot-Extended
Disallow: /

# Common Crawl (feeds many AI models)
User-agent: CCBot
Disallow: /

# Diffbot (AI extraction)
User-agent: Diffbot
Disallow: /

# Huawei
User-agent: PetalBot
Disallow: /

# --- Googlebot ---
User-agent: googlebot
Allow: /
Disallow: /Identity/
Disallow: /Checkout/
Disallow: /Cart
Disallow: /cart
Disallow: /Proforma/
Disallow: /Parts/LoadDiagrams/
Disallow: /Parts/LoadModels

# --- Standard Search Bots ---
User-agent: Amazonbot
User-agent: discobot
User-agent: AhrefsBot
User-agent: dotbot
User-agent: rogerbot
Disallow: /

Sitemap: https://www.eurospares.co.uk/sitemap-gb.xml