User-agent: *
Allow: /

# Block API endpoints
Disallow: /api/
Disallow: /api/*

# Block paginated blog pages (page 2+)
Disallow: /*?page=
Disallow: /*&page=

# Block tag pages with pagination
Disallow: /*/blog/tag/*?page=
Disallow: /*/blog/tag/*&page=

# Block query parameters that create duplicate content
Disallow: /*?tag=
Disallow: /*&tag=

# Block internal/technical pages
Disallow: /download/
Disallow: /r/

# Block sign-in/sign-up redirects (handled by dashboard)
Disallow: /*/sign-in
Disallow: /*/sign-up

# Crawl-delay for polite crawling
Crawl-delay: 1

# AI crawlers - allow access to llms.txt
User-agent: GPTBot
Allow: /llms.txt
Allow: /

User-agent: ChatGPT-User
Allow: /llms.txt
Allow: /

User-agent: Claude-Web
Allow: /llms.txt
Allow: /

User-agent: Anthropic-AI
Allow: /llms.txt
Allow: /

User-agent: PerplexityBot
Allow: /llms.txt
Allow: /

# Host directive
Host: https://bytelixir.com

Sitemap: https://bytelixir.com/sitemap-index.xml
Sitemap: https://bytelixir.com/sitemap-priority.xml
Sitemap: https://bytelixir.com/sitemap.xml