# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow: /wp-admin/
# Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /downloads/
Disallow: /wp-json/
Disallow: /?rest_route=

Allow: /wp-admin/admin-ajax.php
# robots.txt for Whole30.com
# Purpose: Allow reputable AI LLM crawlers full access to public content

# Default rule — allow all bots unless otherwise specified
User-agent: *
Disallow:

# --- AI / LLM Crawlers ---
# OpenAI GPTBot
User-agent: GPTBot
Allow: /

# OpenAI ChatGPT-User
User-agent: ChatGPT-User
Allow: /

# Anthropic ClaudeBot
User-agent: ClaudeBot
Allow: /

# Anthropic Claude-User
User-agent: Claude-User
Allow: /

# Google Extended (Gemini / Bard training)
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# --- Optional: Rate-limit heavier crawlers ---
# These directives are advisory — actual rate control requires server config.
# Crawl-delay in seconds (note: not all bots honor this)
User-agent: GPTBot
Crawl-delay: 5

User-agent: ClaudeBot
Crawl-delay: 5

User-agent: Claude-User
Crawl-delay: 5

User-agent: PerplexityBot
Crawl-delay: 10

User-agent: CCBot
Crawl-delay: 10

Sitemap: https://whole30.com/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK