# ------------------------------------------
# Retail Brew robots.txt
# Version: Allow-All for AI / LLMs
# Last updated: 2025-11-21
# ------------------------------------------

# ==== Standard site path rules ====
User-agent: *
Disallow: /account/*
Disallow: /*/welcome
Disallow: /*/welcome_back
Disallow: /*/confirmed
Disallow: /*/refer-a-friend
Disallow: /*/r
Disallow: /tmp/
Disallow: /io-terms-conditions
Allow: /

# ==== Sitemaps ====
Sitemap: https://www.retailbrew.com/sitemaps/news.xml
Sitemap: https://www.retailbrew.com/sitemaps/site.xml


# =========================================================
# LLM + AI AGENTS (BROWSING, SEARCH, AND TRAINING)
# All explicitly allowed to ensure compatibility.
# =========================================================

# --- OpenAI ---
User-agent: ChatGPT-User
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: GPTBot
Allow: /

# --- Anthropic (Claude) ---
User-agent: Claude-Web
Allow: /
User-agent: ClaudeBot
Allow: /

# --- Perplexity ---
User-agent: PerplexityBot
Allow: /

# --- Google ---
User-agent: Googlebot
Allow: /
User-agent: Google-Extended
Allow: /

# --- Microsoft / Copilot ---
User-agent: Bingbot
Allow: /

# --- Meta (Facebook / Instagram) ---
User-agent: Meta-ExternalAgent
Allow: /
User-agent: Meta-ExternalCrawler
Allow: /

# --- Apple ---
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /

# --- Common Crawl ---
User-agent: CCBot
Allow: /

# --- Amazon ---
User-agent: Amazonbot
Allow: /

# --- DuckDuckGo / You.com / Phind / Diffbot ---
User-agent: DuckDuckBot
Allow: /
User-agent: YouBot
Allow: /
User-agent: PhindBot
Allow: /
User-agent: Diffbot
Allow: /

# --- Other AI Aggregators / Research Agents ---
User-agent: CoHere-Search
Allow: /
User-agent: MistralBot
Allow: /
User-agent: JinaBot
Allow: /
User-agent: DataForSeoBot
Allow: /
User-agent: NeevaAI
Allow: /

# --- Catch-all: any other AI / LLM crawler not listed above ---
User-agent: *
Allow: /

# =========================================================
# Notes:
# - This file explicitly allows both AI browsing and training access.
# - Attribution and use must still comply with llms.txt and Terms of Service.
# - CDN/WAF must not challenge listed User-Agents.
# =========================================================