# =============================================================
# robots.txt — UNRIC.org
# Last updated: Phil 8 April 2026
# =============================================================

User-agent: *

# --- Admin & system files ---
Disallow: /wp-admin/
Allow:    /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /wp-cron.php
Disallow: /xmlrpc.php
Disallow: /cgi-bin/

# --- WP Rocket / RocketCDN cache ---
# On-disk cache folders are served by Cloudflare CDN;
# no crawling value, excluded to preserve crawl budget.
Disallow: /wp-content/cache/
Disallow: /wp-content/cache/min/
Disallow: /wp-content/cache/busting/

# --- REST API ---
# The public REST API endpoint has no indexing value
# and exposes user enumeration (/wp-json/wp/v2/users).
Disallow: /wp-json/

# --- Duplicate content & low-value URL patterns ---

Disallow: /?s=
Disallow: /search/
Disallow: /page/*/?s=
Disallow: */trackback/
Disallow: */comments/feed/
Disallow: */comment-page-*/
Disallow: /*?replytocom
Disallow: /*?doing_wp_cron
Disallow: /*?preview=true

# --- Assets: allow full access for Google rendering ---

Allow: /wp-includes/
Allow: /wp-content/plugins/
Allow: /*.css$
Allow: /*.js$
Allow: /*.woff$
Allow: /*.woff2$
Allow: /*.ttf$
Allow: /*.eot$
Allow: /*.svg$
Allow: /*.gif$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$

# --- Multisite: per-language admin directories ---

Disallow: /en/wp-admin/
Allow:    /en/wp-admin/admin-ajax.php
Disallow: /fr/wp-admin/
Allow:    /fr/wp-admin/admin-ajax.php
Disallow: /it/wp-admin/
Allow:    /it/wp-admin/admin-ajax.php
Disallow: /pt/wp-admin/
Allow:    /pt/wp-admin/admin-ajax.php
Disallow: /nl/wp-admin/
Allow:    /nl/wp-admin/admin-ajax.php
Disallow: /es/wp-admin/
Allow:    /es/wp-admin/admin-ajax.php
Disallow: /de/wp-admin/
Allow:    /de/wp-admin/admin-ajax.php
Disallow: /sv/wp-admin/
Allow:    /sv/wp-admin/admin-ajax.php
Disallow: /da/wp-admin/
Allow:    /da/wp-admin/admin-ajax.php
Disallow: /no/wp-admin/
Allow:    /no/wp-admin/admin-ajax.php
Disallow: /fi/wp-admin/
Allow:    /fi/wp-admin/admin-ajax.php
Disallow: /is/wp-admin/
Allow:    /is/wp-admin/admin-ajax.php
Disallow: /el/wp-admin/
Allow:    /el/wp-admin/admin-ajax.php

# =============================================================
# AI CRAWLERS — explicit Allow for GEO / AEO visibility
# -------------------------------------------------------------
# Cloudflare Pro blocks AI bots by default. See config @Cloudflare
# =============================================================

User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Diffbot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Bytespider
Allow: /

# --- Third-party SEO crawlers ---
# Rate-limit high-frequency audit bots to reduce server load.
# MJ12bot, DotBot and BLEXBot provide no operational value.

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

# =============================================================
# Sitemaps — Yoast SEO Premium (14 language subsites)
# -------------------------------------------------------------
# Submit each sitemap_index.xml individually in:
#   - Google Search Console (one property per language path)
#   - Bing Webmaster Tools (feeds ChatGPT real-time search)
# =============================================================
Sitemap: https://unric.org/sitemap_index.xml
Sitemap: https://unric.org/en/sitemap_index.xml
Sitemap: https://unric.org/fr/sitemap_index.xml
Sitemap: https://unric.org/it/sitemap_index.xml
Sitemap: https://unric.org/pt/sitemap_index.xml
Sitemap: https://unric.org/nl/sitemap_index.xml
Sitemap: https://unric.org/es/sitemap_index.xml
Sitemap: https://unric.org/de/sitemap_index.xml
Sitemap: https://unric.org/sv/sitemap_index.xml
Sitemap: https://unric.org/da/sitemap_index.xml
Sitemap: https://unric.org/no/sitemap_index.xml
Sitemap: https://unric.org/fi/sitemap_index.xml
Sitemap: https://unric.org/is/sitemap_index.xml
Sitemap: https://unric.org/el/sitemap_index.xml

# --- AI visibility file (GEO / AEO) ---
# llms.txt provides LLMs with a structured overview of UNRIC
# content, preferred citations and key page index.
# Deploy at: https://unric.org/llms.txt
Sitemap: https://unric.org/llms.txt