# Robots.txt 

# Configuration spécifique par bot avec crawl-delay
User-agent: Slurp
Crawl-delay: 100
Disallow: /config/
Disallow: /handlers/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /models/
Disallow: /modules/
Disallow: /views/
# Bloquer admin mais autoriser ressources publiques
Disallow: /modules_app/admin/
Allow: /modules_app/admin/includes/css/
Allow: /modules_app/admin/includes/js/
# Autoriser ressources SEO importantes
Allow: /includes/images/
Allow: /includes/css/
Allow: /includes/js/
Allow: /includes/fonts/

User-agent: gsa-crawler-www 
Crawl-delay: 100
Disallow: /config/
Disallow: /handlers/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /models/
Disallow: /modules/
Disallow: /views/
Disallow: /modules_app/admin/
Allow: /modules_app/admin/includes/css/
Allow: /modules_app/admin/includes/js/
Allow: /includes/images/
Allow: /includes/css/
Allow: /includes/js/
Allow: /includes/fonts/

User-agent: Googlebot
Crawl-delay: 100
Disallow: /config/
Disallow: /handlers/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /models/
Disallow: /modules/
Disallow: /views/
Disallow: /modules_app/admin/
Allow: /modules_app/admin/includes/css/
Allow: /modules_app/admin/includes/js/
Allow: /includes/images/
Allow: /includes/css/
Allow: /includes/js/
Allow: /includes/fonts/

User-agent: Mediapartners-Google
Disallow: /config/
Disallow: /handlers/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /models/
Disallow: /modules/
Disallow: /views/
Disallow: /modules_app/admin/
Allow: /modules_app/admin/includes/css/
Allow: /modules_app/admin/includes/js/
Allow: /includes/images/
Allow: /includes/css/
Allow: /includes/js/
Allow: /includes/fonts/

User-agent: Yahoo-NewsCrawler
Disallow: /config/
Disallow: /handlers/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /models/
Disallow: /modules/
Disallow: /views/
Disallow: /modules_app/admin/
Allow: /includes/images/
Allow: /includes/css/
Allow: /includes/js/
Allow: /includes/fonts/

User-Agent: msnbot
Crawl-delay: 100
Disallow: /config/
Disallow: /handlers/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /models/
Disallow: /modules/
Disallow: /views/
Disallow: /modules_app/admin/
Allow: /modules_app/admin/includes/css/
Allow: /modules_app/admin/includes/js/
Allow: /includes/images/
Allow: /includes/css/
Allow: /includes/js/
Allow: /includes/fonts/

# Configuration pour tous les autres bots
User-Agent: *
# BLOQUER - Dossiers système sensibles (sécurité maintenue)
Disallow: /config/
Disallow: /handlers/
Disallow: /interceptors/
Disallow: /layouts/
Disallow: /logs/
Disallow: /models/
Disallow: /modules/
Disallow: /views/

# BLOQUER - Admin mais autoriser ressources techniques nécessaires
Disallow: /modules_app/admin/
Allow: /modules_app/admin/includes/css/
Allow: /modules_app/admin/includes/js/

# AUTORISER - Ressources SEO critiques (CORRECTION PRINCIPALE)
Allow: /includes/images/
Allow: /includes/css/
Allow: /includes/js/
Allow: /includes/fonts/

# BLOQUER - Fichiers sensibles spécifiques
Disallow: /includes/config/
Disallow: /includes/private/
Disallow: /includes/backup/
Disallow: /*.sql
Disallow: /*.log
Disallow: /*.bak

# AUTORISER - Pages importantes
Allow: /
Allow: /fr/
Allow: /en/
Allow: /forfaits/
Allow: /solution-numerique/

# BOTS D'IA - Blocage pour protection propriété intellectuelle et bande passante

# OpenAI (ChatGPT, GPT models)
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

# Google AI (Bard, Gemini training)
User-agent: Google-Extended
Disallow: /

# Anthropic (Claude training)
User-agent: ClaudeBot
Disallow: /

# Common Crawl (utilisé par de nombreux modèles d'IA)
User-agent: CCBot
Disallow: /

# Meta AI (LLaMA, etc.)
User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# ByteDance (TikTok AI)
User-agent: Bytespider
Disallow: /

# Autres bots d'entraînement d'IA
User-agent: PerplexityBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

# Bots académiques/recherche (peuvent être autorisés si souhaité)
User-agent: ia_archiver
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# Plans de site
https://www.byscuit.com/en/sitemap-xml

# LLMS
Fichier LLMs : https://www.byscuit.com/llms.txt