# =============================================================================
# robots.txt for europe.republic.com
# Updated: 2025-10-27
# =============================================================================

# Disallow website archival (Wayback Machine)
User-agent: ia_archiver
Disallow: /

# Rules for all other crawlers
User-agent: *

# ------------------------
# Block Admin & Internal
# ------------------------
Disallow: /admin

# ------------------------
# Block API Endpoints
# ------------------------
Disallow: /api/

# ------------------------
# Block Authentication
# ------------------------
Disallow: /login?*
Disallow: /signup?*
Disallow: /logout
Disallow: /passwords

# ------------------------
# Block User Accounts
# ------------------------
Disallow: /your/
Disallow: /portfolio

# ------------------------
# Sitemap
# ------------------------
Sitemap: https://europe.republic.com/sitemaps/sitemap.xml.gz

# ------------------------
# AI-Optimized Content
# ------------------------
# Signal the location of AI-optimized summaries
LLMS: https://europe.republic.com/llms.txt

# Explicitly allow major AI crawlers to hit the file
User-agent: GPTBot
Allow: /llms.txt

User-agent: Claude-Web
Allow: /llms.txt