# DEMODESK ROBOTS.TXT - OPTIMIZED FOR AI/LLM DISCOVERABILITY
# Last Updated: 2026-04-16
#
# Strategy: Explicitly allow all major AI crawlers for training, search,
# and user-initiated retrieval. Block only internal/sensitive paths.
# Aligned with marketplace.demodesk.ai configuration.

# ==========================================================================
# DEFAULT RULES - Block internal/sensitive paths
# ==========================================================================

User-Agent: *
Disallow: /legal/subprocessors
Disallow: /de/rechtliches/unterauftragsverarbeiter
Disallow: /admin
Disallow: /book/*
Disallow: /recordings/*
Disallow: /manage/*
Disallow: /manage-api/*
Disallow: /tools/plg-*
Disallow: /tools/test-*
Disallow: /search
Disallow: /signup

# ==========================================================================
# OPENAI CRAWLERS
# ==========================================================================
# GPTBot - model training & ChatGPT retrieval
# OAI-SearchBot - ChatGPT search results (real-time)
# ChatGPT-User - user-initiated page fetches

User-Agent: GPTBot
Allow: /

User-Agent: OAI-SearchBot
Allow: /

User-Agent: ChatGPT-User
Allow: /

# ==========================================================================
# ANTHROPIC CRAWLERS
# ==========================================================================
# ClaudeBot - model training data
# Claude-SearchBot - indexes for Claude search results
# Claude-User - fetches pages when users ask Claude

User-Agent: ClaudeBot
Allow: /

User-Agent: Claude-SearchBot
Allow: /

User-Agent: Claude-User
Allow: /

# ==========================================================================
# GOOGLE AI
# ==========================================================================
# Google-Extended - Gemini model training (separate from standard Googlebot)

User-Agent: Google-Extended
Allow: /

# ==========================================================================
# PERPLEXITY
# ==========================================================================
# Note: Perplexity has been documented using stealth crawlers that bypass
# robots.txt (Cloudflare, Aug 2025). These directives are for documentation
# purposes - enforcement is limited.

User-Agent: PerplexityBot
Allow: /

User-Agent: Perplexity-User
Allow: /

# ==========================================================================
# OTHER AI CRAWLERS
# ==========================================================================

User-Agent: Meta-ExternalAgent
Allow: /

User-Agent: Amazonbot
Allow: /

User-Agent: Applebot-Extended
Allow: /

User-Agent: CCBot
Allow: /

User-Agent: YouBot
Allow: /

User-Agent: Cohere-ai
Allow: /

User-Agent: Bytespider
Allow: /

# ==========================================================================
# SITEMAPS
# ==========================================================================

Sitemap: https://demodesk.com/sitemap.xml