# General rules for all bots
User-agent: *
Disallow: /search
Disallow: /?*
Disallow: /sign-up
Disallow: /login

# Specific rules for AI Crawlers
User-agent: GPTBot
Allow: /blog/
Allow: /docs/

User-agent: CCBot
Disallow: /

# Host and Sitemap
Host: https://www.gladia.io
Sitemap: https://www.gladia.io/sitemap.xml
Sitemap: https://www.gladia.io/sitemap-vercel.xml