# ——— DEFAULT: ALL BOTS ———
User-agent: *
Allow: /

# Private & system areas
Disallow: /admin/
Disallow: /dashboard/
Disallow: /login/
Disallow: /signup/

# Transaction pages (no SEO value)
Disallow: /thank-you/
Disallow: /confirmation/

# API and system endpoints
Disallow: /api/
Disallow: /search/
Disallow: /feed/

# Thin taxonomy pages
Disallow: /blogs/category/
Disallow: /blogs/tag/
# NOTE: /blogs/author/ is intentionally ALLOWED — supports E-E-A-T signals

# Tracking parameters (prevents duplicate URL crawling)
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: /*?gclid=
Disallow: /*?fbclid=

# Priority content sections
Allow: /blogs/
Allow: /blogs/author/
Allow: /glossary/
Allow: /case-studies/
Allow: /chatbot-integrations/


# ——— LLM & AI PLATFORMS (Explicitly allow for AI visibility) ———
# Ensures BotPenguin content appears in ChatGPT, Claude, Gemini, and Perplexity answers

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Diffbot
Allow: /


Disallow: /health-check
Sitemap: https://botpenguin.com/sitemap.xml