User-agent: Scrapy
Allow: /

User-agent: Scrapy
Allow: /

User-agent: Scrapy
Allow: /

User-agent: Scrapy
Allow: /

User-agent: *

Disallow: /*/1000$

# On empeche l'indexation des dossiers sensibles
Disallow: /wp-admin
Disallow: /trackback
Disallow: /feed
Disallow: /comments
Disallow: */trackback
Disallow: */feed
Disallow: */comments
Disallow: /munin

# On désindexe la page de connexion (contenu inutile)
Disallow: /wp-login.php
Disallow: /commander/

# On autorise l'indexation de
Allow: /wp-content
Allow: /wp-admin/admin-ajax.php
Allow: /wp-includes/js

User-agent: Googlebot
# On empêche l'indexation des fichiers sensibles
Disallow: /*.php$
Disallow: /*.inc$
Disallow: /*.gz$
Disallow: /*.pdf$

# Autoriser Google Image
User-agent: Googlebot-Image
Disallow:
Allow: /*

# Autoriser Google AdSense
User-agent: Mediapartners-Google*
Disallow:
Allow: /*

# On indique au spider le lien vers notre sitemap
Sitemap: https://www.espritdog.com/sitemap_index.xml