# ==========================================================
# robots.txt - LentePubblica.it
# Gestito da Yoast SEO - Francesco Vicari
# ==========================================================

# === Regole generali per tutti i crawler ===
User-agent: *
# Blocca aree amministrative e tecniche
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /cgi-bin/
Disallow: /trackback/
Disallow: /comments/
Disallow: /readme.html
Disallow: /license.txt
Disallow: /?s=
Disallow: /*?replytocom
Disallow: /*.php$
Disallow: /wp-json/

# Consenti l'accesso ai file necessari al rendering
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-content/cache/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/

# === Indicazione sitemap Yoast SEO ===
Sitemap: https://lentepubblica.it/sitemap_index.xml

# === Bot di ricerca AI / Scraper indesiderati ===
User-agent: GPTBot
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: meta-externalagent
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: OmgiliBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /

# === Bot SEO troppo aggressivi (limitazione crawl) ===
# User-agent: AhrefsBot
# Crawl-delay: 15
# User-agent: SemrushBot
# Crawl-delay: 15
# User-agent: MJ12bot
# Crawl-delay: 20
# User-agent: dotbot
# Crawl-delay: 10

# === Consentiti (principali motori) ===
User-agent: Googlebot
Disallow:
User-agent: Bingbot
Disallow:
User-agent: DuckDuckBot
Disallow:
User-agent: Baiduspider
Disallow:

# === Note ===
# Questo file rispetta pienamente le specifiche RFC9309
# Non contiene direttive non standard (come Content-signal)
# Verificato compatibile con Google Search Console e PageSpeed
# ==========================================================