# Controls how large language models may use your content

# --- General Rules ---
allow: /
disallow: /non_responsive_files/
disallow: /html/

# --- AI Models & Crawlers ---

# OpenAI (GPT, ChatGPT)
user-agent: GPTBot
policy: no-train
policy: allow-read

# Anthropic (Claude)
user-agent: ClaudeBot
policy: no-train
policy: allow-read

# Perplexity
user-agent: PerplexityBot
policy: no-train
policy: allow-read

# Google AI
user-agent: Google-Extended
policy: no-train
policy: allow-read

# Common Crawl
user-agent: CCBot
policy: no-train
policy: allow-read