# Dr. Berg Nutritionals - AI-Optimized
# Updated: 2026-03-19

# Traditional search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# OpenAI (ChatGPT training + retrieval + search)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic (Claude training + search)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Google AI (Gemini training)
User-agent: Google-Extended
Allow: /

# Perplexity AI search
User-agent: PerplexityBot
Allow: /

# Amazon (Alexa/AI)
User-agent: Amazonbot
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

# Other AI platforms
User-agent: YouBot
Allow: /

User-agent: Cohere-ai
Allow: /

# Block aggressive scrapers with no search/AI value
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Crawl-delay: 10

# Standard WordPress protections
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /*?s=
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/

Sitemap: https://www.drberg.com/sitemap_index.xml