# robots.txt for administrator.de
# Last Updated: 20260310

# ================================================
# LICENSING NOTICE - IMPORTANT
# ================================================
# The use of content from administrator.de for
# AI training or commercial AI applications is
# STRICTLY PROHIBITED without written license agreement.
#
# License inquiries: licence@administrator.de
# Terms of Service: https://administrator.de/help/80 (EN: https://administrator.de/help/80?lang=en)
# Privacy Policy: https://administrator.de/help/79 (EN: https://administrator.de/help/79?lang=en)
# AI Licensing Info: https://administrator.de/help/82 (EN: https://administrator.de/help/82?lang=en)
# ================================================

# ================================================
# AI CRAWLERS - COMPLETELY BLOCKED without license
# ================================================

# --- OpenAI ---
# GPTBot: Model training crawler
User-agent: GPTBot
Disallow: /

# OAI-SearchBot: ChatGPT search indexing
User-agent: OAI-SearchBot
Disallow: /

# ChatGPT-User: Real-time browsing from ChatGPT
User-agent: ChatGPT-User
Disallow: /

# OpenAI misc
User-agent: ChatGPT-Bot
User-agent: OpenAIBot
Disallow: /

# --- Anthropic (Claude) ---
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-Bot
Disallow: /

# --- Google AI ---
# Google-Extended: Gemini/Bard AI training (separate from Googlebot for search)
User-agent: Google-Extended
Disallow: /

# --- Microsoft/Bing AI ---
# CopilotBot: Microsoft Copilot crawler
User-agent: CopilotBot
User-agent: msnbot-media
Disallow: /

# --- Meta/Facebook AI ---
# Meta-ExternalAgent: AI training crawler (NOT the preview bot)
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
Disallow: /

# --- Perplexity ---
User-agent: PerplexityBot
User-agent: Perplexity-User
Disallow: /

# --- Apple AI ---
# Applebot-Extended: Apple AI training (separate from Applebot for Siri/Spotlight)
User-agent: Applebot-Extended
Disallow: /

# --- Amazon ---
User-agent: Amazonbot
Disallow: /

# --- Other AI Companies ---
User-agent: CCBot
User-agent: cohere-ai
User-agent: YouBot
User-agent: Diffbot
User-agent: DuckAssistBot
Disallow: /

# --- Chinese AI Crawlers ---
User-agent: Bytespider
User-agent: Bytedance
User-agent: Baiduspider
User-agent: 360Spider
User-agent: Sogou
User-agent: YisouSpider
User-agent: PetalBot
Disallow: /

# --- SEO/Data Mining Crawlers ---
# User-agent: SemrushBot
# User-agent: AhrefsBot
User-agent: DataForSeoBot
User-agent: MJ12bot
User-agent: Scrapy
User-agent: magpie-crawler
User-agent: Omgilibot
Disallow: /

# --- Archive/Research Crawlers (often used for AI training) ---
User-agent: CommonCrawl
User-agent: WebCrawler
User-agent: ImagesiftBot
Disallow: /

# ================================================
# SOCIAL MEDIA / PREVIEW BOTS - ALLOWED
# (These generate link previews when users share URLs)
# ================================================
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: TelegramBot
User-agent: facebookexternalhit
User-agent: BingPreview
Disallow: /settings/
Disallow: /api/
Disallow: /members/dashboard/
Disallow: /members/notifications/
Disallow: /moderators/reports/
Disallow: /connections/
Allow: /

# ================================================
# LEGITIMATE SEARCH ENGINES - ALLOWED
# ================================================
User-agent: Googlebot
User-agent: Bingbot
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Slurp
User-agent: Applebot
Disallow: /settings/
Disallow: /api/
Disallow: /members/dashboard/
Disallow: /members/notifications/
Disallow: /moderators/reports/
Disallow: /connections/
Disallow: /r/
Disallow: /forum/?search=
Disallow: /comments/?search=
Disallow: /members/?search=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?time=
Disallow: /*?lang=
Disallow: /*?action=
Disallow: /*?distance=
Disallow: /*?uri=
Disallow: /*?topic=
Disallow: /*?target=
Disallow: /*?target_id=
Allow: /


# ================================================
# DEFAULT RULES
# ================================================
User-agent: *
Disallow: /settings/
Disallow: /api/
Disallow: /members/dashboard/
Disallow: /members/notifications/
Disallow: /moderators/reports/
Disallow: /connections/
Disallow: /r/
Disallow: /forum/?search=
Disallow: /comments/?search=
Disallow: /members/?search=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*?time=
Disallow: /*?lang=
Disallow: /*?action=
Disallow: /*?distance=
Disallow: /*?uri=
Disallow: /*?topic=
Disallow: /*?target=
Disallow: /*?target_id=
Allow: /


# Sitemap
Sitemap: https://administrator.de/sitemap/sitemap_index.xml

# ================================================
# LEGAL NOTICE - EU DSM DIRECTIVE
# ================================================
# According to Article 4 of EU Directive 2019/790 (DSM Directive)
# and § 44b Abs. 3 UrhG (German Copyright Act),
# we explicitly opt-out from text and data mining (TDM)
# for commercial purposes without license agreement.
#
# TDM-Reservation: 1
# TDM-Policy: https://administrator.de/help/82 (EN: https://administrator.de/help/82?lang=en)
# Terms of Service: https://administrator.de/help/80 (EN: https://administrator.de/help/80?lang=en)
# Privacy Policy: https://administrator.de/help/79 (EN: https://administrator.de/help/79?lang=en)