User-agent: *
Allow: /en/
Allow: /en/supplier/
Allow: /en/articles/
Allow: /en/ingredients/
Allow: /en/trends/
Allow: /en/innovation_Centers
Allow: /en/contact-us
Allow: /en/category/
Allow: /en/promo/
Disallow: /en/login
Disallow: /en/register
Disallow: /en/j_spring_security_check
Disallow: /en/siteMap
Disallow: /en/formulationGallery
Disallow: /*?q=*
Disallow: /*?searchText=*
Disallow: /*?searchProduct=*
Disallow: /*?searchSupplier=*
Disallow: /*?searchCAS=*
Disallow: /*?searchINCI=*
Disallow: /*?searchFunction=*
Disallow: /*?searchApplication=*
Disallow: /*?searchClaim=*
Disallow: /*?searchCertification=*

# Block PDF indexing
Disallow: /medias/*.pdf$

# Language variants login pages
Disallow: /fr/login
Disallow: /ja/login
Disallow: /ko/login
Disallow: /vi/login
Disallow: /it/login
Disallow: /my/login

# AI crawlers - allow full access to public content
User-agent: GPTBot
Allow: /en/
Allow: /en/supplier/
Allow: /en/articles/
Allow: /en/promo/
Disallow: /en/login
Disallow: /en/register
Disallow: /medias/*.pdf$

User-agent: PerplexityBot
Allow: /en/
Allow: /en/supplier/
Allow: /en/articles/
Allow: /en/promo/
Disallow: /en/login
Disallow: /en/register
Disallow: /medias/*.pdf$

User-agent: ClaudeBot
Allow: /en/
Allow: /en/supplier/
Allow: /en/articles/
Allow: /en/promo/
Disallow: /en/login
Disallow: /en/register
Disallow: /medias/*.pdf$

User-agent: Google-Extended
Allow: /en/
Allow: /en/supplier/
Allow: /en/articles/
Allow: /en/promo/
Disallow: /en/login
Disallow: /en/register
Disallow: /medias/*.pdf$

User-agent: Applebot-Extended
Allow: /en/
Disallow: /en/login
Disallow: /en/register

Sitemap: https://www.dkshdiscover.com/en/sitemap.xml