User-agent: *
Allow: /
Allow: /*?page=*
Disallow: /blog-dev

# Allow Applebot (for Siri/Spotlight)
User-agent: Applebot
Allow: /

# Allow Amazonbot (for Alexa and Amazon search)
User-agent: Amazonbot
Allow: /

# Allow GPTBot (OpenAI’s crawler)
User-agent: GPTBot
Allow: /

# Allow OAI-SearchBot (another OpenAI crawler variant)
User-agent: OAI-SearchBot
Allow: /

# Allow PerplexityBot (for Perplexity AI)
User-agent: PerplexityBot
Allow: /

# Allow ByteDance's Bytespider
User-agent: Bytespider
Allow: /

# Allow Common Crawl's CCBot
User-agent: CCBot
Allow: /

# Allow ClaudeBot (Anthropic's crawler)
User-agent: ClaudeBot
Allow: /

# Allow Internet Archive's crawler
User-agent: archive.org_bot
Allow: /

# Allow OAI-SearchBot (OpenAI's crawler)
User-agent: OAI-SearchBot
Allow: /

# Allow ExaBot
User-agent: ExaBot
Allow: /

Sitemap: https://www.bagmasters.com/sitemap_www.bagmasters.com_Index.xml