# robots.txt — finstral.com
# GEO-optimiert: AI-Crawler explizit erlaubt | Stand: März 2026

# Standard-Crawler
User-agent: *
Disallow: /pfengine/
Disallow: /tracking/
Disallow: /default/
Allow: /default/css/
Allow: /default/js/

# === AI-CRAWLER — EXPLIZIT ERLAUBT FÜR GEO ===

# GPTBot (OpenAI / ChatGPT)
User-agent: GPTBot
Allow: /

# ClaudeBot (Anthropic / Claude)
User-agent: ClaudeBot
Allow: /

# anthropic-ai
User-agent: anthropic-ai
Allow: /

# PerplexityBot
User-agent: PerplexityBot
Allow: /

# Google-Extended (Gemini / AI Overviews)
User-agent: Google-Extended
Allow: /

# CCBot (Common Crawl — LLM Training Data)
User-agent: CCBot
Allow: /

# Applebot (Apple Intelligence)
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /

# Meta-ExternalAgent (Meta AI)
User-agent: Meta-ExternalAgent
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Sitemap
Sitemap: https://www.finstral.com/sitemap.xml