# robots.txt
# SEO + AI / LLM Friendly 

User-agent: *
Allow: /
Disallow: /cgi-bin/
Disallow: /vacation-results/
Disallow: /vacation-results/search/


# Block known non-value or scraping bots
User-agent: proximic
Disallow: /

User-agent: SemrushBot
Disallow: /

# --- Explicit AI / LLM Crawlers ---
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: AnthropicBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Bingbot
Allow: /

# Sitemap
Sitemap: https://www.redtag.ca/sitemap.xml