# Allow AI-specific crawlers
User-agent: CCBot
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: MistralAI-User
User-agent: Google-Extended
User-agent: PerplexityBot
# Block private/restricted areas (same as standard bots)
Disallow: /topic/
Disallow: /user/
Disallow: /api/
Disallow: /register/
Disallow: /shop/
Disallow: /es/tema/
Disallow: /fr/sujet/
Disallow: /it/argomento/
Disallow: /pt-BR/tópico/
Disallow: /landing/geosol
# Block non-public/gated paths (mirroring standard bots)
Disallow: /es/cursos/*query=*
Disallow: /es/cursos?query=*
Disallow: /pt-BR/cursos/*query=*
Disallow: /pt-BR/cursos?query=*
Disallow: /fr/les-cours/*query=*
Disallow: /fr/les-cours?query=*
Disallow: /it/corsi/*query=*
Disallow: /it/corsi?query=*
Disallow: */country-id-to-code/*
Disallow: /blog/feed*
Disallow: */images/publishers/*
Disallow: */busca/*
Disallow: */matière/*
Disallow: */occupations/*
Disallow: /*-old$
Disallow: /certification/*
Disallow: /certificación/*
Disallow: /certificazione/*
Disallow: */resources

User-agent: *
Disallow:
Disallow: /es/cursos/*query=*
Disallow: /es/cursos?query=*
Disallow: /pt-BR/cursos/*query=*
Disallow: /pt-BR/cursos?query=*
Disallow: /fr/les-cours/*query=*
Disallow: /fr/les-cours?query=*
Disallow: /it/corsi/*query=*
Disallow: /it/corsi?query=*
Disallow: /topic/*
Disallow: /es/tema/*
Disallow: /fr/sujet/*
Disallow: /it/argomento/*
Disallow: /pt-BR/tópico/*
Disallow: */country-id-to-code/*
Disallow: /blog/feed*
Disallow: /register/
Disallow: */api/*
Disallow: */images/publishers/*
Disallow: */busca/*
Disallow: */matière/*
Disallow: */occupations/*
# Adding this to remove all old courses, which are causing issues with our structured data
Disallow: /*-old$
Disallow: /certification/*
Disallow: /certificación/*
Disallow: /certificazione/*
Disallow: /user/*
Disallow: */resources
Disallow: /shop/*
Disallow: /landing/geosol