# ==================================================
# STERLINGNEXT ROBOTS.TXT
# ==================================================

User-agent: *

# ----------------------------------
# BLOCK NON-SEO / INTERNAL PAGES
# ----------------------------------
Disallow: /admin/
Disallow: /login/
Disallow: /checkout/
Disallow: /cart/
Disallow: /callback/
Disallow: /webhook/
Disallow: /opstatus.php
Disallow: /form-success/
Disallow: /request-call/
Disallow: /request-demo/
Disallow: /get-quote/
Disallow: /reviews-gather/
Disallow: /schedules/

# ----------------------------------
# BLOCK SEARCH / FILTER / JUNK URLS
# (if applicable)
# ----------------------------------
Disallow: /search
Disallow: /*?q=
Disallow: /*?s=
Disallow: /*?sort=
Disallow: /*?filter=

# ----------------------------------
# BLOCK TRACKING PARAMETERS
# Saves crawl budget
# ----------------------------------
Disallow: /*utm_
Disallow: /*gclid=
Disallow: /*fbclid=
Disallow: /*msclkid=
Disallow: /*hsa_

# ----------------------------------
# ALLOW STATIC ASSETS
# Helps rendering
# ----------------------------------
Allow: /*.css$
Allow: /*.js$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.webp$
Allow: /*.svg$
Allow: /*.woff$
Allow: /*.woff2$

# ----------------------------------
# IMPORTANT:
# Keep course + city + schedules crawlable
# if they are valuable ranking pages
# ----------------------------------

# ----------------------------------
# AI TRAINING CRAWLERS
# ----------------------------------
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Allow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Amazonbot
Crawl-delay: 20
Disallow: /search
Disallow: /*?q=
Disallow: /*?s=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*utm_
Disallow: /*gclid=
Disallow: /*fbclid=

# ----------------------------------
# SITEMAP
# ----------------------------------
Sitemap: https://www.sterlingnext.com/sitemap.xml