# General crawling rules
User-agent: *
Allow: /
Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /_next/
Crawl-delay: 1

# AI training and generative AI - ALLOWING for maximum visibility
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

# AI answer engines - welcome
User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

# SEO crawlers with no value - BLOCKING
# mj12bot (Majestic SEO) - 23K requests/month, aggressive burst crawling
User-agent: MJ12bot
Disallow: /

# barkrowler (Babbar.tech) - 7.6K requests/month, aggressive SEO crawler
User-agent: Barkrowler
Disallow: /

# petalbot (Huawei/Aspiegel) - 9.6K requests/month
User-agent: PetalBot
Disallow: /

# yandexbot (Yandex) - 5.7K requests/month, no relevant audience
User-agent: Yandex
Disallow: /

User-agent: YandexBot
Disallow: /

# Data firehose bots - BLOCKING
User-agent: CCBot
Disallow: /

# Sitemaps
Sitemap: https://ravecapture.com/sitemap.xml

# Staging protection: beta.ravecapture.com serves X-Robots-Tag: noindex, nofollow
# via next.config.js conditional headers to prevent indexing