Top Sites>Theneo - Build Docs Developers Love

# Theneo robots.txt
# Default: let search engines and AI crawlers index public pages
User-agent: *
Allow: /

# Keep non-content/system paths out of the index
Disallow: /admin/
Disallow: /dashboard/
Disallow: /editor/
Disallow: /api/
Disallow: /cart/
Disallow: /checkout/
Disallow: /ajax/
Disallow: /_*
Disallow: /search?*
Disallow: /?*

# Be polite (optional)
Crawl-delay: 5

# Block bandwidth-heavy crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: PetalBot
Disallow: /

# Sitemaps (point to the canonical host)
Sitemap: https://www.theneo.io/sitemap.xml
# (Optional safety for legacy links hitting apex)
Sitemap: https://theneo.io/sitemap.xml

# Major AI crawlers (kept explicit for clarity; all are already allowed by the default group)
User-agent: GPTBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

# =========================
# Theneo — llms.txt (AI usage preferences)
# This file declares Theneo’s preferences for AI/LLM crawlers.
# It complements robots.txt; robots rules still apply.
# =========================

Site: https://www.theneo.io
Owner: Theneo
Contact: legal@theneo.io
Sitemap: https://www.theneo.io/sitemap.xml
Canonical: https://www.theneo.io

# ---- Data-use policy for public pages ----
# Allowed: crawl, index, cache, and use PUBLIC content for answer generation and model training.
# Not allowed: collect or store non-public, gated, or personal data; bypass authentication; reproduce full pages.
Policy: public-content-allowed; non-public-prohibited; attribution-preferred

# ---- Paths (mirror robots exclusions) ----
Allow: /
Disallow: /search
Disallow: /404
Disallow: /401
Disallow: /admin
Disallow: /editor
Disallow: /*?*edit
Disallow: /*?*preview
Disallow: /*?*nocache
Disallow: /*?*utm_*
Disallow: /*?*ref=*
Disallow: /*?*fbclid=*
Disallow: /*?*gclid=*
Disallow: /*?*msclkid=*
Disallow: /*?*_hsenc=*
Disallow: /*?*_hsmi=*

# ---- AI/LLM crawlers explicitly opted in ----
User-agent: GPTBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

# Google-Extended governs some generative uses by Google.
User-agent: Google-Extended
Allow: /

# ---- Crawl etiquette (non-binding hints) ----
Crawl-delay: 2
Fetch-Window: 06:00-22:00 UTC

# ---- Attribution preferences (non-binding) ----
Attribution: required
Attribution-Name: Theneo
Attribution-URL: https://www.theneo.io/

# ---- Legal references ----
Terms: https://www.theneo.io/terms
Privacy: https://www.theneo.io/privacy