# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file

# ============================================
# AI SEARCH CRAWLERS - ALLOWED
# Updated: March 2026 for GEO optimization
# ============================================

# OpenAI ChatGPT Web Search
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic Claude
User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google Bard / Gemini
User-agent: Google-Extended
Allow: /

# ============================================
# AI TRAINING CRAWLERS - BLOCKED (Optional)
# Uncomment to prevent AI model training on your content
# ============================================

# Common Crawl (used for AI training datasets)
# User-agent: CCBot
# Disallow: /

# Anthropic AI Training
# User-agent: anthropic-ai
# Disallow: /

# Bytedance (TikTok AI)
# User-agent: Bytespider
# Disallow: /

# Cohere AI
# User-agent: cohere-ai
# Disallow: /

# ============================================
# STANDARD SEARCH ENGINE CRAWLERS - ALLOWED
# ============================================

# Google
User-agent: Googlebot
Allow: /

# Bing
User-agent: Bingbot
Allow: /

# Yahoo
User-agent: Slurp
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# ============================================
# ALL OTHER CRAWLERS - ALLOWED BY DEFAULT
# ============================================

User-agent: *
Allow: /

# ============================================
# DISALLOWED PATHS
# ============================================

# Admin and backend paths
Disallow: /admin/
Disallow: /panel/
Disallow: /users/
Disallow: /api/

# ============================================
# SITEMAP LOCATION
# ============================================

Sitemap: https://www.erzap.com/sitemap