# robots.txt for https://www.bookmakersreview.com/
# Updated: 2026-04-20

###############################################
# 1. GLOBAL ALLOW FOR COMPLIANT SEARCH / AI BOTS
###############################################

User-agent: *
Allow: /

# Global restricted areas
Disallow: /wp-admin/
Disallow: /wp-content/plugins/
Disallow: /wp-login.php
Disallow: /readme.html
Disallow: /refer/
Disallow: /affiliate/
Disallow: /visit/
Disallow: /forum/attachments/
Disallow: /forums/*/?direction
Disallow: /forums/*/?order
Disallow: /forum/help/
Disallow: /forum/login/
Disallow: /forum/lost-password/
Disallow: /forum/members/
Disallow: /forum/misc/
Disallow: /forum/online/
Disallow: /forum/register/
Disallow: /forum/search/
Disallow: /forum/user/
Disallow: /forum/*.php$
Disallow: /forum/*?replyto=*
Disallow: /forum/*?action=*
Disallow: /forum/*?sort=*
Disallow: /forum/*&order=*

# Allow essential rendering / public assets
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /*.css$
Allow: /*.js$

# Forum Rules
User-agent: *
Disallow: /forum/attachments/
Disallow: /forums/*/?direction
Disallow: /forums/*/?order
Disallow: /forum/help/
Disallow: /forum/login/
Disallow: /forum/lost-password/
Disallow: /forum/members/
Disallow: /forum/misc/
Disallow: /forum/online/
Disallow: /forum/register/
Disallow: /forum/search/
Disallow: /forum/user/
Disallow: /forum/*.php$
# Allow crawling of all other content
Disallow: /forum/*?replyto=*
Disallow: /forum/*?action=*
Disallow: /forum/*?sort=*
Disallow: /forum/*&order=*

###############################################
# 2. EXPLICIT AI / SEARCH CRAWLER ALLOW LIST
###############################################

# Google Search + Google AI controls
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: Google-Extended
Allow: /

# Microsoft / Bing ecosystem
User-agent: bingbot
Allow: /

# Apple Search / Siri / Spotlight / Safari
User-agent: Applebot
Allow: /

# Apple generative AI usage control
User-agent: Applebot-Extended
Allow: /

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Common Crawl / downstream AI dataset use
User-agent: CCBot
Allow: /

###############################################
# 3. SEO / AUDIT TOOL CRAWLERS
###############################################

# Semrush (allowed for SEO audits)
User-agent: SemrushBot
Allow: /

User-agent: SemrushBot-SA
Allow: /

# Other SEO crawlers blocked
User-agent: AhrefsBot
Disallow: /

User-agent: AhrefsSiteAudit
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

###############################################
# 4. SITEMAP
###############################################

Sitemap: https://www.bookmakersreview.com/sitemap_index.xml