# ======================================================
#   AI-Optimized Robots.txt (2026)
#   Strategy: Allow reputable AI crawlers; block scrapers.
# ======================================================

# -------------------------
# Global Directives
# -------------------------
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Disallow: /readme.html
Disallow: /refer/


# Protect drafts, feeds, query-heavy endpoints
Disallow: /?s=
Disallow: /search/
Disallow: /feed/
Disallow: /cgi-bin/
Disallow: /*?amp
Disallow: /*?replytocom
Disallow: /wp-json/wp/v2/

# -------------------------
# Allowlist: AI Crawlers for Visibility
# -------------------------

# Google — Search + AI Overviews
User-agent: Googlebot
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Google-Extended
Allow: /

# OpenAI
User-agent: GPTBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
User-agent: Claude-Web
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Microsoft Bing (Powering Copilot)
User-agent: bingbot
Allow: /

# CommonCrawl (training data source)
User-agent: CCBot
Allow: /

# -------------------------
# Blocklist: Bandwidth Wasters / Abusive Scrapers
# -------------------------

# Over-aggressive SEO crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Allow: /

User-Agent: SearchAtlasBot
Allow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Amazon (unclear training usage)
User-agent: Amazonbot
Disallow: /

# Yandex (no relevance for U.S. audience)
User-agent: Yandex
Disallow: /

# Generic AI scrapers (known)
User-agent: IASpider
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Diffbot
Disallow: /

# -------------------------
# Sitemaps
# -------------------------
Sitemap: https://thnks.com/sitemap_index.xml