# robots.txt for https://www.befreed.ai
# Updated: 2025-07-11

# =============================================================================
# Malicious/Unwanted Bot Blocking
# =============================================================================

# Content scrapers and downloaders
User-agent: HTTrack
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: Fetch
Disallow: /

# =============================================================================
# High-frequency/Misbehaving Crawlers (Complete Block)
# =============================================================================

User-agent: wget
User-agent: fast
User-agent: grub-client
User-agent: NPBot
User-agent: larbin
User-agent: libwww
User-agent: ZyBORG
User-agent: Zealbot
User-agent: UbiCrawler
User-agent: MSIECrawler
User-agent: Xenu
Disallow: /

# =============================================================================
# SEO Analysis Tools (Special Restrictions)
# =============================================================================

# SEO analysis tools - additional restrictions beyond general rules 
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: MJ12bot

# (lift for now)
# Disallow: /categories/ 
# Disallow: /collection/
# Disallow: /books/

# Higher crawl delay for analysis tools
Crawl-delay: 10

# =============================================================================
# All Other Bots (Including AI Search Engines & Traditional Search)
# =============================================================================

User-agent: *

# Allow important public content (explicit allow for SEO clarity)
Allow: /
Allow: /blog/
Allow: /collection/
Allow: /book/
Allow: /about
Allow: /faq
Allow: /privacy
Allow: /term
Allow: /categories/
Allow: /partnerships
Allow: /career
Allow: /download
Allow: /list/
Allow: /topics/
Allow: /directory/

# Authentication routes and proxies
Disallow: /auth/
Disallow: /finish_signin

# Premium content requiring subscription
Disallow: /book/*/deep
Disallow: /book/*/flash
Disallow: /book/*/heart
Disallow: /book/*/funs

# User-specific authenticated content
Disallow: /app/browse
Disallow: /app/journey
Disallow: /app/section
Disallow: /library/
Disallow: /memory
Disallow: /search
Disallow: /activity
Disallow: /playgorund_experimental

# API endpoints (protect backend)
Disallow: /api/
Disallow: /book/opengraph-image*

# Next.js internal routes (image optimization API)
Disallow: /_next/image
Disallow: /_next/static

# URL parameters that don't add SEO value
Disallow: /*?*utm_source=
Disallow: /*?*utm_medium=
Disallow: /*?*utm_campaign=
Disallow: /*?*utm_content=
Disallow: /*?*utm_term=
Disallow: /*?*showSignup=
Disallow: /*?*showLogin=
Disallow: /*?*modal=
Disallow: /*?*popup=
Disallow: /*?*auth=
Disallow: /*?*showPaywall=
Disallow: /*?*via=

# Dynamic route parameters that might cause duplicate content
Disallow: /*?*ref=
Disallow: /*?*share=
Disallow: /*?*sid=
Disallow: /*?*sessionid=
Disallow: /*?*token=

# Prevent indexing of common file types that shouldn't be searched
Disallow: /*.json$
Disallow: /*.log$

# Sitemap
Sitemap: https://www.befreed.ai/sitemap.xml

# This site provides an LLM access policy at: https://www.befreed.ai/llms.txt

# Crawl delay for all bots (prevent server overload)
Crawl-delay: 2