# Allow AI search and agent use
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: FirecrawlAgent
User-agent: AndiBot
User-agent: ExaBot
User-agent: PhindBot
User-agent: YouBot
Allow: /

# Disallow AI training data collection
User-agent: GPTBot
User-agent: CCBot
User-agent: Google-Extended
Disallow: /

# Allow traditional search indexing
User-agent: Googlebot
User-agent: Bingbot
Allow: /

# Allow Screaming Frog
User-agent: Screaming Frog SEO Spider
Allow: /

# General bot rules
User-agent: *
Disallow: /api/
Disallow: /*?filter=
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?ref=
Disallow: /*?sort=

Sitemap: https://www.opensense.com/sitemap.xml