# =============================================================
# Storydoc robots.txt - Updated February 2026
# =============================================================

# Generic rules for all crawlers
User-agent: *
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed

Sitemap: https://www.storydoc.com/sitemap-index.xml

# =============================================================
# OPENAI
# =============================================================

# GPTBot - crawls content for OpenAI model training
User-agent: GPTBot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# OAI-SearchBot - powers ChatGPT Search results (NOT used for training)
User-agent: OAI-SearchBot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# ChatGPT-User - user-driven browsing from ChatGPT and Custom GPTs
User-agent: ChatGPT-User
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# =============================================================
# ANTHROPIC (Claude)
# =============================================================

# ClaudeBot - primary Anthropic web crawler
User-agent: ClaudeBot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# anthropic-ai - alternative Anthropic crawler identifier
User-agent: anthropic-ai
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# Claude-SearchBot - Claude search feature crawler
User-agent: Claude-SearchBot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# Claude-Web - Claude web browsing
User-agent: Claude-Web
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# Claude-User - Claude user-initiated browsing
User-agent: Claude-User
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# =============================================================
# GOOGLE (Gemini / AI Overviews)
# =============================================================

# Google-Extended - controls Gemini AI training and AI Overviews
User-agent: Google-Extended
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# GoogleOther - Google's secondary crawling for non-search AI products
User-agent: GoogleOther
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# =============================================================
# PERPLEXITY
# =============================================================

User-agent: PerplexityBot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# =============================================================
# APPLE (Siri / Apple Intelligence)
# =============================================================

# Applebot-Extended - Apple Intelligence and Siri AI features
User-agent: Applebot-Extended
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# =============================================================
# META (Facebook / Instagram / WhatsApp / Meta AI)
# =============================================================

User-agent: FacebookBot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

User-agent: Meta-ExternalAgent
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

User-agent: Meta-ExternalFetcher
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# =============================================================
# MICROSOFT (Bing / Copilot)
# =============================================================

# Bingbot already allowed by default via *, but explicit for clarity
User-agent: bingbot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# =============================================================
# OTHER AI SEARCH & ASSISTANTS
# =============================================================

# DeepSeek
User-agent: DeepSeekBot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# Mistral AI
User-agent: MistralAI-User
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# DuckDuckGo AI Assist
User-agent: DuckAssistBot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# You.com
User-agent: YouBot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# Cohere
User-agent: cohere-ai
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# Brave Search
User-agent: Bravebot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# Common Crawl (used by many open-source AI models)
User-agent: CCBot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /

# Amazon / AWS AI
User-agent: Amazonbot
Disallow: /preview
Disallow: /8011fa118d5708e6
Disallow: /496ff67598d650ed
Allow: /