# ===========================
# AI & LLM Crawler Access
# ===========================

User-agent: meta-webindexer
Disallow: /

# OpenAI
User-agent: OAI-Searchbot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: GPTBot
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google AI / Bard / Gemini
User-agent: Google-Extended
Allow: /
User-agent: GoogleOther
Allow: /
User-agent: Google-LLM
Allow: /

# Microsoft / Bing AI
User-agent: BingAI
Allow: /
User-agent: BingPreview
Allow: /

# Meta (LLM research & AI indexing)
User-agent: Meta-ExternalAgent
Disallow: /
User-agent: FacebookBot
Allow: /

# You.com AI
User-agent: YouBot
Allow: /

# Neeva / Snowflake AI
User-agent: NeevaAI
Allow: /

# DuckDuckGo AI / DuckAssist
User-agent: DuckAssistBot
Allow: /
User-agent: DuckDuckBot
Allow: /

# CCBot (Harvey AI / CoCounsel / legal AI)
User-agent: CCBot
Disallow: /

# Andi AI Search
User-agent: AndiBot
Allow: /

# Phind AI
User-agent: PhindBot
Allow: /

# Komo AI
User-agent: KomoBot
Allow: /

# Kagi (Kagi Search & AI)
User-agent: KagiBot
Allow: /

# Brave AI / Summarizer
User-agent: BraveBot
Allow: /

# General AI Research Crawlers
User-agent: AI2Bot
Allow: /
User-agent: AllenAI
Allow: /

# ===========================
# General Crawlers
# ===========================
User-agent: *
Disallow: /kenNew/*
Disallow: /admin/
Disallow: /sample-report/
Disallow: /book-a-discovery-call/
Disallow: /talk-to-us/
Disallow: /not-found
Disallow: /case-study-list?_rsc=*
Disallow: /marketing-strategy?_rsc=*
Disallow: /dossier360?_rsc=*
Disallow: /white-papers?_rsc=*
Disallow: /paid-press-release?_rsc=*
Disallow: /v1/market-research-report?_rsc=*
Disallow: /v1/equity-research-report?_rsc=*
Disallow: /v1/uploads/images/*
Disallow: /country-reports?_rsc=*
Disallow: /publishers?_rsc=*

# ===========================
# Sitemaps for SEO + AI
# ===========================
Sitemap: https://www.kenresearch.com/sitemap.xml
Sitemap: https://www.kenresearch.com/ai_sitemap.json