# ==================================================
# robots.txt für https://www.mercury.ai
# Kontakt: hi@mercury.ai
# ==================================================

# ---------------------------
# 1. Globale Regeln (Alle Bots)
# ---------------------------
User-agent: *

# -- Erlauben (Explizit) --
Allow: /
Allow: /llms.txt
Allow: /ai-info-page
Allow: /assets/
Allow: /static/
Allow: *.pdf
Allow: *.css
Allow: *.js
Allow: *.png
Allow: *.jpg
Allow: *.svg

# -- Blockieren (Sicherheit & App) --
Disallow: /admin/
Disallow: /login/
Disallow: /signin/
Disallow: /dashboard/
Disallow: /app/
Disallow: /account/
Disallow: /reset-password/
Disallow: /private/

# -- Blockieren (Technik & SEO-Hygiene) --
Disallow: /api/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /search/
Disallow: /preview/
Disallow: /staging/

# ---------------------------
# 2. KI-Crawler (Explizit)
# ---------------------------

# OpenAI – ChatGPT Crawler
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Disallow: /admin/
Disallow: /api/

# OpenAI – ChatGPT Browse (Browser-Plugin)
User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Disallow: /admin/
Disallow: /api/

# Anthropic – Claude Crawler
User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Disallow: /admin/

# Anthropic – Claude Web Browse
User-agent: Claude-Web
Allow: /
Allow: /llms.txt
Disallow: /admin/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt

# Google – Gemini / Vertex AI
User-agent: Google-Extended
Allow: /
Allow: /llms.txt

# Amazon – Alexa / AWS AI
User-agent: Amazonbot
Allow: /
Allow: /llms.txt

# CCBot – Common Crawl / AI Training Data
User-agent: CCBot
Allow: /
Allow: /llms.txt

# Meta – Facebook AI
User-agent: FacebookBot
Allow: /
Allow: /llms.txt

# ByteDance – TikTok AI
User-agent: Bytespider
Allow: /
Allow: /llms.txt

# ---------------------------
# 3. Sitemaps
# ---------------------------
Sitemap: https://mercury.ai/sitemap.xml