User-agent: *
Disallow: /staging/


# -------------------------
# AI Crawlers and LLM Agents
# -------------------------

# OpenAI GPTBot (used for model training)
User-agent: GPTBot
Disallow: /

# ChatGPT link preview fetcher (safe to allow)
User-agent: ChatGPT-User
Allow: /

# Anthropic Claude AI (official user-agents)
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Apple generative AI
User-agent: Applebot-Extended
Allow: /

# Cohere
User-agent: CCBot
Allow: /

# Google’s AI training crawler
User-agent: Google-Extended
Allow: /


# -------------------------
# Sitemap
# -------------------------

Sitemap: https://www.heretto.com/sitemap.xml


Sitemap: https://www.heretto.com/sitemap.xml