# robots.txt for Overview.ai
User-agent: *
Allow: /

# Disallow admin and api routes
Disallow: /api/
Disallow: /admin/

# Allow search engines to crawl all public pages
Allow: /products/
Allow: /industries/
Allow: /applications/
Allow: /blog/
Allow: /communication-protocols/
Allow: /industrial-protocols/

# Sitemap location
Sitemap: https://www.overview.ai/sitemap.xml

# Crawl-delay for bots (optional, helps with server load)
User-agent: *
Crawl-delay: 1

# Special rules for common bots
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

# OpenAI / ChatGPT bots
# OAI-SearchBot: required to appear in ChatGPT search answers
User-agent: OAI-SearchBot
Allow: /

# GPTBot: allows content to be used in OpenAI model training
User-agent: GPTBot
Allow: /

# ChatGPT-User: user-initiated browsing via ChatGPT
User-agent: ChatGPT-User
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Meta AI
User-agent: meta-externalagent
Allow: /

# Apple Intelligence
User-agent: Applebot
Allow: /

# Google Gemini / Vertex AI
User-agent: Google-Extended
Allow: /