# Block aggressive SEO crawler bots entirely (no SEO value, just cost)
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MegaIndex
Disallow: /

# Allow good bots (Google, Bing) with reasonable crawl delays
User-agent: Googlebot
Crawl-delay: 2
Allow: /

User-agent: Bingbot
Crawl-delay: 2
Allow: /

# Default for other bots
User-agent: *
Allow: /

# Disallow API routes and admin pages
Disallow: /api/
Disallow: /admin/
Disallow: /auth/

# Allow crawling of artists and releases
Allow: /artists/
Allow: /releases/

# BLOCK individual track pages (50M pages) - prevents massive crawl costs
# Tracks are accessible to users but not crawled by bots
Disallow: /releases/*/tracks/

# Disallow video pages
Disallow: /videos/

# Disallow private user content
Disallow: /*/edit
Disallow: /*/settings
Disallow: /create-username/
Disallow: /*/queue
Disallow: /*/playlists/*/edit

# Main crawl delay to prevent excessive load
Crawl-delay: 5

# Sitemap location
Sitemap: https://raygum.com/sitemap.xml