# https://www.robotstxt.org/robotstxt.html
# OilPriceAPI.com Robots.txt Configuration
# Last Updated: December 20, 2025

# === SITEMAP REFERENCES ===
Sitemap: https://www.oilpriceapi.com/sitemap.xml
Sitemap: https://docs.oilpriceapi.com/sitemap.xml
llms.txt: https://www.oilpriceapi.com/llms.txt

# === LLM & AI BOTS - FULL ACCESS ===
# OpenAI (ChatGPT)
User-agent: GPTBot
Allow: /
Crawl-delay: 1

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: claude-web
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# === SEARCH ENGINES - SPECIAL RULES ===
# Google - Allow JS/CSS for proper rendering
User-agent: Googlebot
Allow: /
Allow: /_next/static/
Allow: /*.js$
Allow: /*.css$
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Disallow: /auth/reset-password
Disallow: /debug-tokens
Disallow: /test-sms
Disallow: /api-test
Disallow: /unsubscribe
Crawl-delay: 1

# Bing
User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Crawl-delay: 2

# Baidu
User-agent: Baiduspider
Allow: /
Disallow: /admin/
Disallow: /dashboard/
Crawl-delay: 3

# Yandex
User-agent: YandexBot
Allow: /
Disallow: /admin/
Disallow: /dashboard/
Crawl-delay: 2

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# === GENERAL ACCESS ===
User-agent: *
Allow: /

# === BLOCKED AREAS FOR ALL BOTS ===
User-agent: *
# Admin & Private Areas
Disallow: /admin/
Disallow: /dashboard/
Disallow: /account/
Disallow: /billing/portal
Disallow: /private/

# Authentication Pages (no SEO value)
Disallow: /auth/
Disallow: /reset-password
Disallow: /confirm
Disallow: /unsubscribe
Disallow: /api-key
Disallow: /billing
Disallow: /upgrade

# Static files that shouldn't be indexed
Disallow: /manifest.json
Disallow: /*.woff2$
Disallow: /api/og

# Internal API Endpoints
Disallow: /api/
Disallow: /api/internal/
Disallow: /api/admin/
Disallow: /api/debug/
Disallow: /api/test/

# Next.js Internal (allow static assets for rendering, block data routes)
Disallow: /_next/data/
Disallow: /_error
Disallow: /404

# Test & Debug Pages
Disallow: /debug-tokens
Disallow: /test-sms
Disallow: /api-test

# User-specific pages
Disallow: /users/confirm-email

# === ALLOW IMPORTANT STATIC RESOURCES ===
User-agent: *
Allow: /_next/static/css/
Allow: /_next/static/chunks/
Allow: /_next/static/media/
Allow: /favicon.ico
Allow: /logo.png
Allow: /robots.txt
Allow: /sitemap.xml

# === BLOCK QUERY PARAMETERS TO PREVENT DUPLICATE CONTENT ===
User-agent: *
Disallow: /*?source=
Disallow: /*?plan=
Disallow: /*?ref=
Disallow: /*?utm_
Disallow: /*?fbclid=
Disallow: /*?gclid=
Disallow: /*?session=
Disallow: /*?token=
Disallow: /*?404=
Disallow: /*?page=-
Disallow: /blog/*?404=
Disallow: /blog/*?page=-
Disallow: /blog/archive?*

# === CRAWL DELAYS ===
# Default crawl delay for responsible crawling
User-agent: *
Crawl-delay: 1

# === BLOCK BAD BOTS ===
User-agent: BadBot
Disallow: /

User-agent: ScraperBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Crawl-delay: 10

# === SPECIAL PAGES - ALLOW FOR SEO ===
User-agent: *
Allow: /pricing
Allow: /documentation
Allow: /docs
Allow: /oil-price-api
Allow: /bloomberg-alternative
Allow: /python-oil-api
Allow: /developers/
Allow: /live/
Allow: /commodities/
Allow: /vs/
Allow: /blog
Allow: /faq
Allow: /contact
Allow: /about

# === HOST DIRECTIVE (FOR YANDEX) ===
Host: https://www.oilpriceapi.com

# End of robots.txt