# ============================================================================
# BING BOT - Full access to sitemaps and content
# ============================================================================
User-agent: bingbot
User-agent: msnbot
Allow: /
Allow: /fr/sitemap*.xml
Allow: /de/sitemap*.xml
Allow: /nl/sitemap*.xml
Allow: /sitemap*.xml
Disallow: /beta/wp-admin/
Disallow: /wp-admin/
Disallow: /wp-content/uploads/woo-import-export
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 1

# ============================================================================
# GOOGLE BOT - Full access to sitemaps and content
# ============================================================================
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Googlebot-News
Allow: /
Allow: /fr/sitemap*.xml
Allow: /de/sitemap*.xml
Allow: /nl/sitemap*.xml
Allow: /sitemap*.xml
Disallow: /beta/wp-admin/
Disallow: /wp-admin/
Disallow: /wp-content/uploads/woo-import-export
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 1

# ============================================================================
# AI CRAWLERS - Allow access to content (ChatGPT, Claude, Grok, etc.)
# ============================================================================
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Grok
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
Allow: /
Allow: /fr/sitemap*.xml
Allow: /de/sitemap*.xml
Allow: /nl/sitemap*.xml
Allow: /sitemap*.xml
Disallow: /beta/wp-admin/
Disallow: /wp-admin/
Disallow: /wp-content/uploads/woo-import-export
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 2

# ============================================================================
# BLOCK HEAVY TRANSLATED PAGINATION + FILTERS (for other bots)
# ============================================================================
User-agent: *
Disallow: /nl/page/
Disallow: /fr/page/
Disallow: /de/page/
Disallow: /*?filter_
Disallow: /*&filter_
Disallow: /*?dgwt_wcas=
Disallow: /*&dgwt_wcas=
Disallow: /wp-json/

# Block pagination site‑wide (EN + locales)
User-agent: *
Disallow: /page/
Disallow: /*/page/
Disallow: /*?paged=
Disallow: /*&paged=

# Slow or block known noisy bots
User-agent: PetalBot
Disallow: /
Crawl-delay: 10

User-agent: Site24x7
Crawl-delay: 5

User-agent: CookieHubScan
Crawl-delay: 5

# ============================================================================
# OTHER SEARCH ENGINES - Allow access with reasonable crawl-delay
# ============================================================================
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: Sogou
User-agent: Exabot
User-agent: facebot
User-agent: ia_archiver
Allow: /
Allow: /fr/sitemap*.xml
Allow: /de/sitemap*.xml
Allow: /nl/sitemap*.xml
Allow: /sitemap*.xml
Disallow: /beta/wp-admin/
Disallow: /wp-admin/
Disallow: /wp-content/uploads/woo-import-export
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 5

# ============================================================================
# STANDARD ROBOTS RULES (for all other bots)
# ============================================================================
User-agent: *
Disallow: /beta/wp-admin/
Disallow: /wp-admin/
Disallow: /wp-content/uploads/woo-import-export
Allow: /wp-admin/admin-ajax.php
Allow: /beta/wp-admin/admin-ajax.php

Crawl-delay: 30

Sitemap: https://www.12taste.com/sitemap_index.xml
Sitemap: https://www.12taste.com/fr/sitemap_index.xml
Sitemap: https://www.12taste.com/de/sitemap_index.xml
Sitemap: https://www.12taste.com/nl/sitemap_index.xml