# =========================================
# Intelligems robots.txt
# Last updated: 2026-01-22
# =========================================

# Allow all search engines
User-agent: *
Allow: /

# -----------------------------------------
# OpenAI Crawlers
# Docs: https://platform.openai.com/docs/bots
# -----------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Note: ChatGPT-User no longer follows robots.txt (Dec 2025)
User-agent: ChatGPT-User
Allow: /

# -----------------------------------------
# Anthropic (Claude) Crawlers
# Docs: https://support.claude.com/en/articles/8896518
# -----------------------------------------
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# -----------------------------------------
# Google AI
# -----------------------------------------
User-agent: Google-Extended
Allow: /

# -----------------------------------------
# Other AI Crawlers
# -----------------------------------------
User-agent: PerplexityBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

User-agent: CCBot
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot-Extended
Allow: /

# -----------------------------------------
# Block internal app routes (authenticated / non-public)
# -----------------------------------------
User-agent: *
Disallow: /app/
Disallow: /dashboard/
Disallow: /login/
Disallow: /signup/
Disallow: /auth/
Disallow: /settings/
Disallow: /admin/

# Block query parameter crawl waste
Disallow: /*?*

# Allow key public resources explicitly
Allow: /pricing
Allow: /products/
Allow: /resources/
Allow: /company/
Allow: /privacy-policy
Allow: /terms-of-service
Allow: /cookie-policy

# -----------------------------------------
# Sitemaps & LLMs.txt
# -----------------------------------------
Sitemap: https://www.intelligems.io/sitemap.xml

# For AI context: https://www.intelligems.io/llms.txt