# robots.txt for clearpathgps.com
# Policy: Allow search + assistant previews; opt out of AI training
# Updated: 2025-09-08

########################################
# 1) Opt out of AI Training
# (Does NOT affect normal search indexing)
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Cohere-ai
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: omgilibot
Disallow: /

########################################
# 2) Allow assistant link previews (optional but recommended)
# These do not grant training rights; they fetch pages when users share links.
User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

########################################
# 3) Disallow aggressive SEO crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DotBot
Disallow: /

########################################
# 4) Default rules for all other bots
User-agent: *
# Block Admin & system
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-json.php
Disallow: /wp-json/

# Keep plugin/theme/cache internals out of the index
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/

# De-duplicate thin/utility pages typical to WordPress
Disallow: /search
Disallow: /?s=
Disallow: /tag/
Disallow: /category/uncategorized/
Disallow: /comments/feed/
Disallow: /feed/
Disallow: /page/
Disallow: /read/
Disallow: /trackback/
Disallow: /wp-json/oembed
Disallow: /amp/

# Conversion/utility pages (prefer meta noindex too)
Disallow: /thank-you
Disallow: /thankyou/

# Strip tracking-parameter variants
Disallow: /*?utm_*
Disallow: /*?fbclid=*
Disallow: /*?gclid=*
Disallow: /*?msclkid=*
Disallow: /*?dclid=*
Disallow: /*?ref=*
Disallow: /*?source=*
Disallow: /*?campaign=*
Disallow: /*?rest_route=*

# Extra rules for ClearPath GPS
Disallow: /sample-*
Disallow: /blog/sample-*
Disallow: /_hcms/preview/
Disallow: /hs/manage-preferences/
Disallow: /hs/preferences-center/
Disallow: /*?*hs_preview=*
Disallow: /*?*hsCacheBuster=*

########################################
# 5) Sitemaps
Sitemap: https://www.clearpathgps.com/sitemap_index.xml