# AI Crawlers: OpenAI
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User

# AI Crawlers: Google AI
User-agent: Google-Extended

# AI Crawlers: Anthropic
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: claude-web

# AI Crawlers: Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User

# AI Crawlers: Amazon
User-agent: Amazonbot

# AI Crawlers: Apple
User-agent: Applebot
User-agent: Applebot-Extended

# AI Crawlers: ByteDance
User-agent: Bytespider

# AI Crawlers: Cohere
User-agent: cohere-ai

# AI Crawlers: Mistral
User-agent: MistralAI-User

# AI Crawlers: CommonCrawl
User-agent: CCBot

# AI Crawlers: Other popular AI
User-agent: AI2Bot
User-agent: Diffbot
User-agent: omgili
User-agent: YouBot

# Standard Crawlers (all others)
User-agent: *
Disallow: /articles/wp-admin
Disallow: /articles/wp-includes
Disallow: /articles/wp-content/plugins
Disallow: /articles/wp-feed
Disallow: /articles/wp-comments
Disallow: /articles/?s=
Disallow: /articles/?amp=
Disallow: *?attachment_id=
Disallow: *?utm_source=
Disallow: */feed
Disallow: */rss
Disallow: */embed
Disallow: */goto/
Disallow: *?q=
Disallow: /for-testing

Sitemap: https://betterme.world/sitemap.xml