# Robots.txt for Referoo.com
# Last updated: March 2026

# LLMs and AI Crawlers Information
# For AI/LLM system information, please see: https://www.referoo.com/hubfs/llms.txt
# LLMs.txt provides comprehensive information about Referoo for AI systems


User-agent: *
Allow: /llms.txt
Allow: /hubfs/llms.txt
Disallow: /downloads/
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

# AI and LLM Crawlers - Explicit allow for context file

User-agent: GPTBot
Allow: /llms.txt
Allow: /hubfs/llms.txt

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: ChatGPT-User
Allow: /llms.txt
Allow: /hubfs/llms.txt

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: CCBot
Allow: /llms.txt
Allow: /hubfs/llms.txt

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: anthropic-ai
Allow: /llms.txt
Allow: /hubfs/llms.txt

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: Claude-Web
Allow: /llms.txt
Allow: /hubfs/llms.txt

Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

User-agent: magicsearchbot
Allow: /llms.txt
Allow: /hubfs/llms.txt
Disallow: /uploads/
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

# Sitemap Location
Sitemap: https://www.referoo.com/sitemap.xml

# Additional Information
# LLMs.txt: https://www.referoo.com/hubfs/llms.txt
# This file provides structured information about Referoo for AI/LLM systems