########################################################################################################################
# Website Sitemap
Sitemap: https://www.retif.eu/media/sitemap/fr_fr/secure_sitemap_index.xml

########################################################################################################################
# Main crawler rules
User-agent: *

# Allow important parameters and media
Allow: /*?p=
Allow: /*?page=
Allow: /media/catalog/product/

# Static assets
Disallow: /static/

# System directories
Disallow: /app/
Disallow: /bin/
Disallow: /cgi-bin/
Disallow: /dev/
Disallow: /downloader/
Disallow: /errors/
Disallow: /generated/
Disallow: /includes/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /scripts/
Disallow: /setup/
Disallow: /shell/
Disallow: /stats/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/

# Media - sensitive directories
Disallow: /media/captcha/
Disallow: /media/customer/
Disallow: /media/dhl/
Disallow: /media/downloadable/
Disallow: /media/import/
Disallow: /media/pdf/
Disallow: /media/sales/
Disallow: /media/tmp/
Disallow: /media/xmlconnect/

# Functional pages - not for indexing
Disallow: /index.php/
Disallow: /catalog/product_compare/
Disallow: /catalog/product/gallery/
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /cart/
Disallow: /customer/
Disallow: /newsletter/
Disallow: /invitation/  
Disallow: /poll/
Disallow: /giftregistry/ 
Disallow: /review/
Disallow: /rma/ 
Disallow: /reward/ 
Disallow: /sendfriend/
Disallow: /tag/
Disallow: /wishlist/
Disallow: /j2tajaxcheckout/
Disallow: /storepickup/
Disallow: /productupdates/
Disallow: /stores/
Disallow: /quotation/
Disallow: /rest/
Disallow: /graphql
Disallow: /company/ 
Disallow: /requisition_list/ 
Disallow: /negotiable-quote/ 
Disallow: /purchase-order/ 

# System files
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /get.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt

# URL parameters that create duplicate content
Disallow: /*?SID=
Disallow: /*PHPSESSID
Disallow: /*?___from_store=
Disallow: /*___from_store=
Disallow: /*?dir=
Disallow: /*&dir=
Disallow: /*.php
Disallow: /rss

# Filter parameters
Disallow: /*product_list_order=
Disallow: /*product_list_limit=
Disallow: /*limit=
Disallow: /*?mode=   
Disallow: /*order=
Disallow: /*?price=
Disallow: /*?display_currency= 
Disallow: /*?product_list_mode= 

# Marketing/tracking parameters
Disallow: /*?cmp=
Disallow: /*?camplink=
Disallow: /*?campname=

# Internal parameters
Disallow: /aitdownloadablefiles/
Disallow: /multishipping/
Disallow: /paypal/ 
Disallow: /sales/ 
Disallow: /*?all=
Disallow: /*?zipcode=
Disallow: /*?city=
Disallow: /*?radius=
Disallow: /*?news=
Disallow: /*?object=

########################################################################################################################
# LLM bots - allow full access
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

########################################################################################################################
# Block unwanted bots
User-agent: Yandex
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-agent: Baiduspider-image
Disallow: /