# SAGES robots.txt

Sitemap: https://www.sages.org/sitemap_index.xml

User-agent: *
# See https://contentsignals.org/ for explanation
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /
Allow: /feed/podcast/

# Core WordPress restrictions
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php

# Search and duplicate URLs
Disallow: /?s=
Disallow: /search/
Disallow: /page/*/?s=

# Prevent crawl traps / junk parameters
Disallow: /*?replytocom=
Disallow: /*?share=
Disallow: /*?amp
Disallow: /*?print
Disallow: /*?fbclid
Disallow: /*?gclid
Disallow: /*?utm_
Disallow: /*?orderby=
Disallow: /*?filter=
Disallow: /*?source=

# Block advertising / unwanted crawlers
User-agent: AdsBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: CCBot
Disallow: /

# -----------------------------
# AI Crawlers
# Keep this simple. Use Cloudflare/WAF for actual enforcement.
# -----------------------------

# OpenAI training crawler
User-agent: GPTBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# OpenAI search index
User-agent: OAI-SearchBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# ChatGPT live browsing / user-triggered fetches
User-agent: ChatGPT-User
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# Anthropic
User-agent: ClaudeBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# Perplexity
User-agent: PerplexityBot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# Apple AI / Siri
User-agent: Applebot
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# Meta AI crawler
User-agent: Meta-ExternalAgent
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /