#
# Agile1 - WTE
# Updated: 2026-04-07
#

User-agent: *
Disallow: /admin
Disallow: /cmsdesk
Disallow: /System/
Disallow: /NoTrack/
Disallow: /Tiny
Disallow: /wip/
Disallow: /client-portal
Disallow: /client-portal/
Disallow: /apps/
Disallow: /system/search-results
Disallow: /search
Disallow: /search/
Disallow: /*?q=
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?gclid=
Disallow: /*?fbclid=
Allow: /*

User-agent: Googlebot
Disallow: /admin
Disallow: /cmsdesk
Disallow: /System/
Disallow: /NoTrack/
Disallow: /wip/
Disallow: /Tiny
Disallow: /client-portal
Disallow: /client-portal/
Disallow: /apps/
Disallow: /system/search-results
Disallow: /search
Disallow: /search/
Disallow: /*?q=
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?gclid=
Disallow: /*?fbclid=
Allow: /*

# ------------------------------------------------
# AI / LLM Crawlers - Explicitly Allowed
# ------------------------------------------------

# OpenAI
User-agent: GPTBot
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Google – core + AI extension
User-agent: Googlebot
Allow: /
User-agent: Google-Extended
Allow: /

# Microsoft
User-agent: BingBot
Allow: /

# Common Crawl (used by many LLMs)
User-agent: CCBot
Allow: /

# Site verification token
tiktok-developers-site-verification=LaXzpg5qqbM68C2mhXfP4BD6iehy1342

# Sitemap
Sitemap: https://www.wte.net/sitemap2.xml