# ===========================================
# WGSN robots.txt
# Allow only the explicitly listed user-agents below.
# Block all other user-agents.
# ===========================================

# -------------------------------------------
# ALLOWED USER-AGENTS
# -------------------------------------------
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Google-InspectionTool
User-agent: Storebot-Google
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: Mediapartners-Google
User-agent: bingbot
User-agent: Baiduspider
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Slurp
User-agent: Applebot

User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Slackbot-LinkExpanding
User-agent: Slackbot
User-agent: Discordbot
User-agent: WhatsApp
User-agent: TelegramBot
User-agent: Pinterestbot

User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Claude-Web
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Youbot
User-agent: DuckAssistBot
User-agent: Google-CloudVertexBot
User-agent: Gemini-Deep-Research
User-agent: GoogleOther
User-agent: anthropic-ai
User-agent: MistralAI-User

# ---- Allowed public surfaces ----
Allow: /$
Allow: /robots.txt$
Allow: /llms.txt$
Allow: /auth/login$
Allow: /en$
Allow: /en/
Allow: /es$
Allow: /es/
Allow: /pt$
Allow: /pt/
Allow: /de$
Allow: /de/
Allow: /fr$
Allow: /fr/
Allow: /jp$
Allow: /jp/
Allow: /ko$
Allow: /ko/
Allow: /cs$
Allow: /cs/

# ---- Allow assets for rendering/link previews ----
Allow: /sites/default/files/js/*.js
Allow: /sites/default/files/css/*.css
Allow: /themes/custom/custom/images/

Disallow: /*/blog/type?cat=
Disallow: /

# -------------------------------------------
# AI TRAINING USER-AGENTS
# -------------------------------------------
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Facebookbot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: diffbot
User-agent: Omgili
User-agent: Omgilibot
User-agent: webzio-extended
User-agent: ImagesiftBot
User-agent: Bytespider
User-agent: TikTokSpider
User-agent: ToutiaoSpider
User-agent: ShenmaSpider
User-agent: JikeSpider
User-agent: Linespider
User-agent: Amazonbot
User-agent: SemrushBot-OCOB
User-agent: Petalbot
User-agent: TurnitinBot
User-agent: Timpibot
User-agent: ICC-Crawler
User-agent: AI2Bot
User-agent: AI2Bot-Dolma
User-agent: PanguBot
User-agent: Kangaroo Bot
User-agent: Meltwater
User-agent: Seekr
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Cotoyogi
User-agent: aiHitBot
User-agent: Factset_spyderbot
User-agent: HaoSouSpider
User-agent: 360spider
User-agent: YoudaoBot
User-agent: Sogou web spider
User-agent: Sogou inst spider
User-agent: Sogou spider2
User-agent: Sogou blog
User-agent: Sogou News Spider
User-agent: Sogou Orion spider
User-agent: ChinasoSpider
User-agent: Sosospider
User-agent: YisouSpider
User-agent: EasouSpider

# ---- Allowed public surfaces ----
Allow: /$
Allow: /robots.txt$
Allow: /llms.txt$

Allow: /en$
Allow: /en/what-we-do/
Allow: /en/products/
Allow: /en/teams/
Allow: /en/features/
Allow: /en/wgsn$
Allow: /en/wgsn/
Allow: /en/contact$
Allow: /en/careers$
Allow: /en/blog$
Allow: /en/events$
Allow: /en/podcasts$
Allow: /en/resources$
Allow: /en/resources/case-studies$
Allow: /en/resources/sample-forecasts$
Allow: /en/resources/white-papers$
Allow: /en/demo$

Allow: /es$
Allow: /es/lo-que-hacemos/
Allow: /es/productos/
Allow: /es/equipos/
Allow: /es/features/
Allow: /es/wgsn$
Allow: /es/wgsn/
Allow: /es/contacto$
Allow: /es/careers$
Allow: /es/blog$
Allow: /es/eventos$
Allow: /es/podcasts$
Allow: /es/recursos$
Allow: /es/recursos/casos-de-clientes$
Allow: /es/recursos/muestras-de-contenido$
Allow: /es/recursos/white-papers$
Allow: /es/demo$

Allow: /pt$
Allow: /pt/o-que-fazemos/
Allow: /pt/produtos/
Allow: /pt/times/
Allow: /pt/features/
Allow: /pt/wgsn$
Allow: /pt/wgsn/
Allow: /pt/contato$
Allow: /pt/careers$
Allow: /pt/blog$
Allow: /pt/eventos$
Allow: /pt/podcasts$
Allow: /pt/recursos$
Allow: /pt/recursos/cases-de-sucesso$
Allow: /pt/recursos/amostras-de-conteudo$
Allow: /pt/recursos/white-papers$
Allow: /pt/demonstracao$

Allow: /fr$
Allow: /fr/ce-que-nous-faisons/
Allow: /fr/produits/
Allow: /fr/equipes/
Allow: /fr/caracteristiques/
Allow: /fr/wgsn$
Allow: /fr/wgsn/
Allow: /fr/contact$
Allow: /fr/careers$
Allow: /fr/blog$
Allow: /fr/evenements$
Allow: /fr/podcasts$
Allow: /fr/ressources$
Allow: /fr/ressources/etudes-de-cas$
Allow: /fr/ressources/exemples-de-previsions$
Allow: /fr/ressources/livres-blancs$
Allow: /fr/demo$

Allow: /de$
Allow: /de/was-wir-tun/
Allow: /de/produkte/
Allow: /de/teams/
Allow: /de/merkmale/
Allow: /de/wgsn$
Allow: /de/wgsn/
Allow: /de/kontakt$
Allow: /de/careers$
Allow: /de/blog$
Allow: /de/veranstaltungen$
Allow: /de/podcasts$
Allow: /de/ressourcen$
Allow: /de/ressourcen/fallstudien$
Allow: /de/ressourcen/musterprognosen$
Allow: /de/ressourcen/weissbucher$
Allow: /de/demo$

Allow: /cs$
Allow: /cs/what-we-do/
Allow: /cs/products/
Allow: /cs/teams/
Allow: /cs/features/
Allow: /cs/wgsn$
Allow: /cs/wgsn/
Allow: /cs/contact$
Allow: /cs/careers$
Allow: /cs/blog$
Allow: /cs/events$
Allow: /cs/podcasts$
Allow: /cs/resources$
Allow: /cs/resources/case-studies$
Allow: /cs/resources/sample-forecasts$
Allow: /cs/resources/white-papers$
Allow: /cs/demo$

Allow: /jp$
Allow: /jp/what-we-do/
Allow: /jp/products/
Allow: /jp/teams/
Allow: /jp/features/
Allow: /jp/wgsn$
Allow: /jp/wgsn/
Allow: /jp/contact$
Allow: /jp/careers$
Allow: /jp/blog$
Allow: /jp/events$
Allow: /jp/podcasts$
Allow: /jp/resources$
Allow: /jp/resources/case-studies$
Allow: /jp/resources/sample-forecasts$
Allow: /jp/resources/white-papers$
Allow: /jp/demo$

Allow: /ko$
Allow: /ko/what-we-do/
Allow: /ko/products/
Allow: /ko/teams/
Allow: /ko/features/
Allow: /ko/wgsn$
Allow: /ko/wgsn/
Allow: /ko/contact$
Allow: /ko/careers$
Allow: /ko/blog$
Allow: /ko/events$
Allow: /ko/podcasts$
Allow: /ko/resources$
Allow: /ko/resources/case-studies$
Allow: /ko/resources/sample-forecasts$
Allow: /ko/resources/white-papers$
Allow: /ko/demo$

# ---- Allow assets for rendering/link previews ----
Allow: /sites/default/files/js/*.js
Allow: /sites/default/files/css/*.css
Allow: /themes/custom/custom/images/

Disallow: /

# -------------------------------------------
# DEFAULT (all other user-agents)
# -------------------------------------------
User-agent: *
Allow: /robots.txt$
Allow: /llms.txt$
Disallow: /
DisallowAITraining: /
Content-Usage: ai=n

Sitemap: https://www.wgsn.com/sitemap.xml