User-agent: *
Disallow: /_backup*/
Disallow: /cache/
Disallow: /pages/
Disallow: /php/
Disallow: /api.php
Disallow: /wp/wp-admin/
Allow: /pages/api/
Crawl-delay: 2

# AI discovery
Allow: /llms.txt
Allow: /llms-full.txt

Sitemap: https://scrabbledb.com/sitemap-index.xml

# Major search engines - generous but not unlimited
User-agent: Googlebot
Crawl-delay: 1

User-agent: Googlebot-Image
Crawl-delay: 2

User-agent: bingbot
Crawl-delay: 2

User-agent: Applebot
Crawl-delay: 2

User-agent: Yandex
Crawl-delay: 3

User-agent: DuckDuckBot
Crawl-delay: 2

User-agent: Baiduspider
Crawl-delay: 5

# AI crawlers - welcome to index, please be gentle
User-agent: ClaudeBot
Crawl-delay: 10

User-agent: GPTBot
Crawl-delay: 10

User-agent: Google-Extended
Crawl-delay: 5

User-agent: CCBot
Crawl-delay: 15

User-agent: anthropic-ai
Crawl-delay: 10

User-agent: ChatGPT-User
Crawl-delay: 10

User-agent: PerplexityBot
Crawl-delay: 10

# Facebook crawler - heavy hammerer, restrict to homepage only
User-agent: meta-externalagent
Crawl-delay: 30
Disallow: /word/
Disallow: /words-starting-with/
Disallow: /words-ending-with/
Disallow: /words-containing/
Disallow: /sk/
Disallow: /pt/
Disallow: /en/
Disallow: /de/
Disallow: /fr/

# Aggressive SEO/data scrapers - slow them way down
User-agent: AhrefsBot
Crawl-delay: 30

User-agent: SemrushBot
Crawl-delay: 30

User-agent: MJ12bot
Crawl-delay: 30

User-agent: DotBot
Crawl-delay: 30

User-agent: SeznamBot
Crawl-delay: 10

# Meta webindexer (missing from config - same restrictions as meta-externalagent)
User-agent: meta-webindexer
Crawl-delay: 30
Disallow: /word/
Disallow: /words-starting-with/
Disallow: /words-ending-with/
Disallow: /words-containing/
Disallow: /sk/
Disallow: /pt/
Disallow: /en/
Disallow: /de/
Disallow: /fr/

# Amazon crawler
User-agent: Amazonbot
Crawl-delay: 20

# Alibaba security scanner - not useful, block
User-agent: AliyunSecBot
Disallow: /