# =============================================================================
# robots.txt for metis.io
# Visibility-Focused Strategy: Maximize discoverability across AI and search
# Last Updated: December 2025
# =============================================================================

# -----------------------------------------------------------------------------
# DEFAULT RULE: Allow all crawlers
# -----------------------------------------------------------------------------
User-agent: *
Allow: /
Crawl-delay: 1

# -----------------------------------------------------------------------------
# TRADITIONAL SEARCH ENGINES
# Priority access with no crawl delay
# -----------------------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

# -----------------------------------------------------------------------------
# AI TRAINING CRAWLERS
# Allow for model training - maximizes long-term AI visibility
# -----------------------------------------------------------------------------

# OpenAI - ChatGPT model training
User-agent: GPTBot
Allow: /

# Anthropic - Claude model training
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: claude-web
Allow: /

# Google - Gemini AI training (separate from search)
User-agent: Google-Extended
Allow: /

# Apple - Apple Intelligence training
User-agent: Applebot-Extended
Allow: /

# Common Crawl - Open dataset used by many AI models
User-agent: CCBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Meta/Facebook AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# -----------------------------------------------------------------------------
# AI SEARCH & RAG CRAWLERS
# Allow for real-time AI search visibility (ChatGPT search, Perplexity, etc.)
# -----------------------------------------------------------------------------

# OpenAI Search (appears in ChatGPT search results)
User-agent: OAI-SearchBot
Allow: /

# OpenAI User-triggered browsing
User-agent: ChatGPT-User
Allow: /

# Perplexity AI Search
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# You.com AI Search
User-agent: YouBot
Allow: /

# Phind (developer-focused AI search)
User-agent: PhindBot
Allow: /

# Amazon Alexa / AI services
User-agent: Amazonbot
Allow: /

# DuckDuckGo AI features
User-agent: DuckAssistBot
Allow: /

# -----------------------------------------------------------------------------
# AGGRESSIVE CRAWLERS
# Allow with rate limiting to protect server resources
# -----------------------------------------------------------------------------

# ByteDance/TikTok (known for aggressive crawling)
User-agent: Bytespider
Allow: /
Crawl-delay: 10

# SEO tools (secondary priority)
User-agent: AhrefsBot
Allow: /
Crawl-delay: 5

User-agent: SemrushBot
Allow: /
Crawl-delay: 5

User-agent: MJ12bot
Allow: /
Crawl-delay: 5

# -----------------------------------------------------------------------------
# PROTECTED PATHS
# Block access to admin, API, and internal endpoints
# -----------------------------------------------------------------------------
User-agent: *
Disallow: /api/
Disallow: /admin/
Disallow: /_next/
Disallow: /revision/
Disallow: /linktree/
Disallow: /directory/
Disallow: /thank-you
Disallow: /*.json$
Disallow: /*?preview=*
Disallow: /*?draft=*

# -----------------------------------------------------------------------------
# SITEMAP REFERENCE
# -----------------------------------------------------------------------------
Sitemap: https://www.metis.io/sitemap.xml

# -----------------------------------------------------------------------------
# NOTES FOR METIS.IO
# -----------------------------------------------------------------------------
# This configuration prioritizes:
# 1. Maximum visibility in AI-generated answers (ChatGPT, Claude, Perplexity)
# 2. Full indexing by traditional search engines
# 3. Inclusion in AI model training for long-term brand presence
# 4. Server protection via crawl-delay for aggressive bots
#
# Key content sections indexed:
# - /blog/*          : Technical articles, ecosystem updates
# - /developers      : Developer documentation
# - /academy/*       : Educational content (high AI citation value)
# - /governance      : CEG and governance documentation
# - /ecosystem       : Ecosystem information
# - /platform        : Technical platform details
# - /decentralized-sequencer : Core differentiator content
#
# Review quarterly as AI crawler landscape evolves rapidly.
# =============================================================================


Sitemap: https://www.metis.io/sitemap.xml