# robots.txt for https://easyitguys.com
# SEO + WordPress hardening + AI visibility (training and input allowed)

# ---- Global content signals (Cloudflare-compatible) ----
User-agent: *
Content-signal: search=yes,ai-input=yes,ai-train=yes
Allow: /
Allow: /wp-json/
Allow: /wp-content/uploads/
Allow: /*.json$
Allow: /*.xml$

# ---- WordPress noise / low-value URLs ----
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/
Disallow: /cgi-bin/
Disallow: /*?s=*
Disallow: /*?*replytocom
Disallow: /xmlrpc.php
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /?author=*
Disallow: *add-to-cart=*
Disallow: *product-page=*
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Allow: /wp-content/uploads/

# ---- Ensure Google can render assets ----
User-agent: Googlebot
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/

# ---- Allow major AI/search crawlers (ChatGPT, Claude, Perplexity, Apple, Meta, etc.) ----
User-agent: GPTBot
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Claude-SearchBot
Allow: /
User-agent: Claude-User
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /
User-agent: FacebookBot
Allow: /
User-agent: Meta-ExternalAgent
Allow: /
User-agent: Meta-ExternalFetcher
Allow: /

# ---- Emerging AI crawlers ----
User-agent: MistralAI-User
Allow: /
User-agent: Novellum AI Crawl
Allow: /
User-agent: ProRataInc
Allow: /
User-agent: Timpibot
Allow: /

# ---- Traditional search engines ----
User-agent: Googlebot
Allow: /
User-agent: BingBot
Allow: /
User-agent: DuckAssistBot
Allow: /
User-agent: PetalBot
Allow: /

# Help search engines ignore tracking parameters
Disallow: /*?utm_*
Disallow: /*?fbclid=*

# ---- Block aggressive / low-value scrapers ----
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

# (Optional) If you don’t need archive mirroring:
# User-agent: archive.org_bot
# Disallow: /

# --- Policy ---
# Public content may be used for AI input and model training.
# Administrative, private, or user-specific data is not authorized for collection.
# Violations may be considered unauthorized data scraping under U.S. and EU data protection law.

# ---- Sitemap ----
Sitemap: https://easyitguys.com/sitemap.xml