# =========================
# Sitemap
# =========================
Sitemap: https://boatcrazy.com/sitemap
# llms.txt: https://boatcrazy.com/llms.txt

# =========================
# Global rules (default)
# =========================
User-agent: *
Allow: /
Disallow: /account/
Disallow: /messages/
Disallow: /favorites/
Disallow: /login/
Disallow: /register/
Disallow: /html/header
Disallow: /*sort-

# =========================
# Major search engines
# =========================
User-agent: Googlebot
Allow: /

User-agent: bingbot
Allow: /
Crawl-delay: 10

User-agent: YandexBot
Allow: /
Crawl-delay: 10

# =========================
# AI Crawlers (ALLOW)
# =========================

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Common Crawl (used by many AI models)
User-agent: CCBot
Allow: /

# Amazon AI / Alexa
User-agent: Amazonbot
Allow: /

# =========================
# SEO tools you actively use (ALLOW)
# =========================

# Semrush
User-agent: SemrushBot
Allow: /

# =========================
# Block unwanted scrapers
# =========================
User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Moz
Disallow: /

# =========================
# Unknown / abusive crawlers
# =========================
User-agent: ai-crawler
Disallow: /

User-agent: web-crawlers.io
Disallow: /