# Dapta AI - robots.txt
# Updated: 2026-04-08
# Purpose: Allow AI crawlers, remove aggressive crawl-delay

# === Default rules for all crawlers ===
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# === AI Crawlers - Explicitly Allowed ===

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Google AI / Gemini
User-agent: GoogleOther
Allow: /

# Google Extended (Gemini training)
User-agent: Google-Extended
Allow: /

# Common Crawl (used by many AI models)
User-agent: CCBot
Allow: /

# Bing / Copilot
User-agent: Bingbot
Allow: /

# === Blocked Crawlers ===

# ByteDance / TikTok (aggressive crawler, no SEO value)
User-agent: Bytespider
Disallow: /

# === Sitemap ===
Sitemap: https://dapta.ai/sitemap_index.xml