User-agent: *
Disallow: /*/login
Disallow: /*/register
Disallow: /*/payments
Disallow: /*/support
Disallow: /*/admin/
Disallow: /*/api/v2/admin/
Disallow: /*/404
Disallow: /*/500
Disallow: /*/kb/tags/

# OpenAI — ChatGPT
User-agent: GPTBot
Allow: /

# OpenAI — search & browse
User-agent: ChatGPT-User
Allow: /

# OpenAI — SearchBot
User-agent: OAI-SearchBot
Allow: /

# Anthropic — Claude
User-agent: ClaudeBot
Allow: /

# Perplexity — AI-searchbot
User-agent: PerplexityBot
Allow: /

# training crawlers are blocked
# Google Gemini
User-agent: Google-Extended
Disallow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /

# Common Crawl
User-agent: CCBot
Disallow: /

# Cohere AI
User-agent: cohere-ai
Disallow: /

# Anthropic
User-agent: anthropic-ai
Disallow: /

# --- Sitemap ---
Sitemap: https://xserver.cloud/sitemap.xml