# Content Signals (contentsignals.org) — declared AI/content-use preferences
# ai-train=yes: allow use of content to train AI models
# search=yes:   allow use for search indexing and retrieval
# ai-input=yes: allow use as AI input for retrieval-augmented answers
Content-Signal: ai-train=yes, search=yes, ai-input=yes

User-agent: *
Allow: /

# NSFW content policy (Issue #1957): crawl allowed, indexing blocked via
# X-Robots-Tag: noindex header set in HandleInertiaRequests middleware.
# Disallow rules were removed 2026-04-24 — robots-blocked pages already
# indexed by Google stay indexed forever because Google never re-crawls
# to see noindex. Allowing crawl lets the noindex directive propagate.
# Ref: app/Http/Middleware/HandleInertiaRequests.php::shouldNoindex()
#      seo/phase-5-ctr-recovery.md

# Block API-only endpoints from crawling
Disallow: /mcp/
Disallow: /models-json

# Block tracking parameter URLs (Google Ads, UTM, referral, share)
Disallow: /*?*utm_source=
Disallow: /*?*utm_medium=
Disallow: /*?*utm_campaign=
Disallow: /*?*utm_term=
Disallow: /*?*utm_content=
Disallow: /*?*utm_referrer=
Disallow: /*?*gclid=
Disallow: /*?*gad_source=
Disallow: /*?*gad_campaignid=
Disallow: /*?*gbraid=
Disallow: /*?*wbraid=
Disallow: /*?*fbclid=
Disallow: /*?*msclkid=
Disallow: /*?*querystring=
Disallow: /*?*searchterm=
Disallow: /*?*audience_segment=
Disallow: /*?*matchtype=
Disallow: /*?*device=c
Disallow: /*?*ref=
Disallow: /*?*trk=
Disallow: /*?*cta_id=
Disallow: /*?*share=
Disallow: /*?*shareId=
Disallow: /*?*from=
Disallow: /*?*platform=
Disallow: /*?*_rsc=

# Block llms text files on model subpaths (keep root files accessible)
Disallow: /models/*/llms.txt
Disallow: /models/*/llms-full.txt

# Block Soft 404 pagination patterns
Disallow: /*?*model_id=*&page=

# Block internal API/AJAX endpoints
Disallow: /api/
Disallow: /_debugbar/

# AI Training and Documentation Resources
Allow: /llms.txt$
Allow: /llms-full.txt$
Allow: /readme.md
Allow: /schema.json
Allow: /api-catalog.json
Allow: /comparison.json

# RSS/Atom Feeds for Google Discover Follow Feature
Allow: /feed
Allow: /atom
Allow: /models/feed
Allow: /models/atom

# Explicit AI crawler rules — future-proofing against accidental blocking
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: FacebookBot
Allow: /

# AI agents: append .md to any URL for markdown (e.g., /pricing.md)
# or send Accept: text/markdown header

Sitemap: https://modelslab.com/sitemap.xml