# ============================================
# Kurby AI - robots.txt
# ============================================

# Default: allow all crawlers
User-agent: *
Allow: /
Allow: /neighborhoods/
Allow: /api/llms-full
Disallow: /api/
Disallow: /d8f3k2/
Disallow: /settings/
Disallow: /embed/
Disallow: /sign-in/
Disallow: /sign-up/
Disallow: /geo-blocked
Disallow: /invalid-link

# Disallow tracking/query parameters
Disallow: /*?utm_source=
Disallow: /*&utm_source=
Disallow: /*?utm_medium=
Disallow: /*&utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*&utm_campaign=
Disallow: /*?via=
Disallow: /*&via=
Disallow: /*?ref=
Disallow: /*&ref=
Disallow: /*?fpr=
Disallow: /*&fpr=

# Disallow auto-generated search strings under /app/ containing Arabic or non-slug characters
Disallow: /app/*،*
Disallow: /app/*%D8%*

# Allow clean slugs under /app/
Allow: /app/[a-zA-Z0-9\-\_]*$

# ============================================
# AI Crawler Access - Explicitly Allowed
# ============================================

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /
Allow: /api/llms-full
Disallow: /api/
Disallow: /settings/
Disallow: /sign-in/
Disallow: /sign-up/

User-agent: ChatGPT-User
Allow: /

# Google AI (Gemini, AI Overviews)
User-agent: Google-Extended
Allow: /

# Anthropic / Claude
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Bing / Copilot
User-agent: Bingbot
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# Apple Intelligence / Siri
User-agent: Applebot-Extended
Allow: /

# Common AI research crawlers
User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

# ============================================
# Sitemap & LLM Content Index
# ============================================

Sitemap: https://kurby.ai/sitemap-index.xml

# RSS / Atom Feeds
# RSS (all content): https://kurby.ai/feed/index.xml
# Atom (all content): https://kurby.ai/feed/atom.xml
# Blog RSS: https://kurby.ai/feed/blog.xml
# Cities RSS: https://kurby.ai/feed/cities.xml

# LLM-specific content index (llmstxt.org standard)
# llms.txt: https://kurby.ai/llms.txt
# Full machine-readable data: https://kurby.ai/api/llms-full