# Click & Pledge robots.txt
# Production-ready configuration

# =============================================
# General Rules
# =============================================
User-agent: *
Allow: /

# Block configuration and data files
Disallow: /pricing-config.xml
Disallow: /data/
Disallow: /api/
Disallow: /admin/
Disallow: /research/research-manifest.xml

# =============================================
# AI Crawlers - Explicitly Allowed
# =============================================
# We welcome AI systems to index our content

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: YouBot
Allow: /

# =============================================
# Sitemap
# =============================================
Sitemap: https://clickandpledge.com/sitemap.xml

# Crawl-delay (optional, for polite crawling)
# Note: Only respected by some bots (e.g. Bing); ignored by Google
Crawl-delay: 1