# robots.txt for turitop.com
# Updated: March 2026 — AI Search Optimization

# Default rules for all crawlers
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /xmlrpc.php
Disallow: /wp-json/
Disallow: /?s=
Disallow: /search/

# AI Search Engine Crawlers — explicitly allowed
# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Google Gemini / AI Overviews
User-agent: Google-Extended
Allow: /

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# TikTok / ByteDance
User-agent: Bytespider
Allow: /

# Common Crawl (feeds many LLM training sets)
User-agent: CCBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Microsoft Copilot / Bing
User-agent: Bingbot
Allow: /

User-agent: msnbot
Allow: /

# Sitemaps
Sitemap: https://www.turitop.com/sitemap_index.xml