# robots.txt for eclincher.com
# Last updated: 2026-04-09
# Strategy: Maximum AI + Search visibility (allow training + retrieval bots)

# ---------------------
# DEFAULT: All crawlers
# ---------------------
User-agent: *
Allow: /
Disallow: /?s
Disallow: /search
Disallow: /coming-soon
Disallow: /template-info/
Disallow: /401
Disallow: /404

# ---------------------
# GOOGLE (Search + Gemini AI)
# ---------------------
# Googlebot: Standard search indexing (never block this)
User-agent: Googlebot
Allow: /

# Google-Extended: Controls Gemini AI training and AI Overviews grounding
# Allowing = your content can appear in Gemini answers and AI Overviews
User-agent: Google-Extended
Allow: /

# Googlebot-Image: Image search indexing
User-agent: Googlebot-Image
Allow: /

# ---------------------
# OPENAI (ChatGPT Search + Training)
# ---------------------
# OAI-SearchBot: Powers ChatGPT Search citations and results
# CRITICAL — blocking this removes you from ChatGPT search answers
User-agent: OAI-SearchBot
Allow: /

# GPTBot: Model training + search retrieval
# Allowing = content used for training AND search (shared crawl with OAI-SearchBot)
User-agent: GPTBot
Allow: /

# ChatGPT-User: User-triggered browsing (note: no longer follows robots.txt per Dec 2025 update)
User-agent: ChatGPT-User
Allow: /

# ---------------------
# ANTHROPIC (Claude)
# ---------------------
# ClaudeBot: Citation fetch for Claude web search
User-agent: ClaudeBot
Allow: /

# claude-web: Web-focused crawl agent
User-agent: claude-web
Allow: /

# anthropic-ai: Model training crawler
User-agent: anthropic-ai
Allow: /

# ---------------------
# PERPLEXITY
# ---------------------
# PerplexityBot: Index builder for Perplexity AI search
User-agent: PerplexityBot
Allow: /

# Perplexity-User: Human-triggered visit from Perplexity
User-agent: Perplexity-User
Allow: /

# ---------------------
# MICROSOFT (Bing + Copilot)
# ---------------------
User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /

# ---------------------
# APPLE (Siri + Apple Intelligence)
# ---------------------
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ---------------------
# META (Meta AI)
# ---------------------
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# ---------------------
# OTHER AI SEARCH / RETRIEVAL
# ---------------------
# Cohere AI
User-agent: cohere-ai
Allow: /

# You.com
User-agent: YouBot
Allow: /

# ---------------------
# SOCIAL MEDIA PREVIEW BOTS (critical for eClincher as a social platform)
# ---------------------
# Facebook/Meta link previews (Open Graph)
User-agent: facebookexternalhit
Allow: /

# X/Twitter card previews
User-agent: Twitterbot
Allow: /

# LinkedIn link previews
User-agent: LinkedInBot
Allow: /

# Pinterest Rich Pins
User-agent: Pinterest
Allow: /

# Slack link unfurling
User-agent: Slackbot
Allow: /

# WhatsApp link previews
User-agent: WhatsApp
Allow: /

# Discord embed previews
User-agent: Discordbot
Allow: /

# Telegram link previews
User-agent: TelegramBot
Allow: /

# ---------------------
# BLOCK: Low-value scrapers (no retrieval or search value)
# ---------------------
# Common Crawl (bulk scraping for training datasets, no search product)
User-agent: CCBot
Disallow: /

# ByteDance/TikTok AI training scraper (aggressive, no search product)
User-agent: Bytespider
Disallow: /

# Diffbot (commercial scraping)
User-agent: Diffbot
Disallow: /

# AI2 research scraper
User-agent: ai2bot
Disallow: /

# Scrapy-based generic scrapers
User-agent: Scrapy
Disallow: /

# Note: SemrushBot and AhrefsBot are intentionally NOT blocked.
# Visibility in SEO tools helps marketers discover eClincher during competitive research.

# ---------------------
# SITEMAP
# ---------------------
Sitemap: https://www.eclincher.com/sitemap.xml


Sitemap: https://www.eclincher.com/sitemap.xml