# LLM and AI Crawler Guidelines for Heretto
# This file provides additional rules for large language model crawlers.


# -------------------------
# Default AI Agent Rules
# -------------------------
User-agent: *
Allow: /
Disallow: /staging/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /editor/
Disallow: /api/
Disallow: /checkout/
Disallow: /accounts/
Crawl-Delay: 3


# -------------------------
# OpenAI
# -------------------------

# GPTBot (training)
User-agent: GPTBot
Disallow: /

# ChatGPT link preview bot
User-agent: ChatGPT-User
Allow: /


# -------------------------
# Anthropic
# -------------------------
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /


# -------------------------
# Perplexity
# -------------------------
User-agent: PerplexityBot
Allow: /


# -------------------------
# You.com
# -------------------------
User-agent: YouBot
Allow: /


# -------------------------
# Apple AI
# -------------------------
User-agent: Applebot-Extended
Allow: /


# -------------------------
# Cohere
# -------------------------
User-agent: CCBot
Allow: /


# -------------------------
# Google AI
# -------------------------
User-agent: Google-Extended
Allow: /


# -------------------------
# Notes
# -------------------------
# - These rules apply to AI crawlers only.
# - Content requiring login or secure access is not allowed for any crawler.
# - For questions about data usage permissions, contact the Heretto team.