# Allow all crawlers
User-agent: *
Allow: /

# Disallow API routes
Disallow: /api/
Disallow: /_next/
Disallow: /static/

# Allow important assets
Allow: /images/
Allow: /assets/
Allow: /fonts/

# Disallow confirmation pages
Disallow: /contact/write-to-us/confirmation
Disallow: /ErrorBoundary
Disallow: /whistleblower-mechanism

# Crawl-delay for all bots
# Specifies how many seconds crawlers should wait between requests
# This helps prevent overloading the server with too many requests
Crawl-delay: 10

# Host
Host: https://www.atherenergy.com


# ========== LLM & AI CRAWLERS ==========

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Google AI / Gemini / AI Overviews
User-agent: Google-Extended
Allow: /

# Anthropic Claude
User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Common Crawl – Block open dataset crawlers
User-agent: CCBot
Disallow: /

# Sitemaps
Sitemap: https://www.atherenergy.com/sitemap.xml
Sitemap: https://www.atherenergy.com/images_sitemap.xml

# AI/LLM Content Manifest
# This helps AI agents find your curated Markdown content
Sitemap: https://www.atherenergy.com/llms.txt