User-agent: *
Crawl-delay: 8
Allow: /

# Block WordPress REST API endpoints for most bots
# (this prevents heavy crawling of /wp-json/ but allows Googlebot to index content normally)
Disallow: /wp-json/
Disallow: /wp-json/wp/v2/
Disallow: /wp-json/oembed/
Disallow: /wp-json/wc/           # if you use WooCommerce
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /?s=                   # block search result pages
Disallow: /search/
Disallow: /*?*

# Allow major search engines to crawl the API if needed (they usually don't need it)
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
Allow: /wp-json/
Allow: /

# Social media bots (Facebook & Twitter) - allow but with delay
User-agent: facebookexternalhit
User-agent: Facebot
User-agent: FacebookBot
Crawl-delay: 10
Allow: /
Disallow: /wp-json/
Disallow: /wp-admin/

User-agent: Twitterbot
Crawl-delay: 10
Allow: /
Disallow: /wp-json/
Disallow: /wp-admin/

# Optional: extra slowdown for aggressive SEO & AI bots
User-agent: AhrefsBot
Crawl-delay: 15
Disallow: /wp-json/

User-agent: SemrushBot
Crawl-delay: 15
Disallow: /wp-json/

User-agent: MJ12bot
Crawl-delay: 20
Disallow: /wp-json/

User-agent: DotBot
Crawl-delay: 20
Disallow: /wp-json/

# Block most AI/training scrapers completely
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: anthropic-ai
User-agent: GrokBot
User-agent: xAI-Grok
Disallow: /