# AI Search Crawlers — explicit Allow (retrieval / citation)
# -----------------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# -----------------------------------------------------------
# Training-only crawlers — no retrieval benefit, block all
# -----------------------------------------------------------
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

# -----------------------------------------------------------
# All other crawlers
# -----------------------------------------------------------
User-agent: *
Disallow: */lp/
Disallow: /en/lp/thinkcreators/login
Disallow: /ae/lp/thinkcreators/login
Disallow: /za/lp/thinkcreators/login

# -----------------------------------------------------------
# Sitemap (update URL once sitemap is created)
# -----------------------------------------------------------

Sitemap: https://www.thinkmarkets.com/sitemap_index.xml