# --------------------------------------------
# Global rules
# --------------------------------------------
User-agent: *
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# De-duplicate/low-value URLs (query params & internal search)
# Note: Wildcards are Google/Bing-compatible and match querystrings.

Disallow: /*?*utm_=
Disallow: /*openstat=

# Block specific documents
Disallow: /wp-content/uploads/2025/04/240719-customer-dpa-.pdf
Disallow: /wp-content/uploads/2025/09/250905-doit-cloud-intelligence-terms-for-quote.pdf
Disallow: /wp-content/uploads/2025/09/250905-doit-cloud-intelligence-terms-for-quote-1.pdf
Disallow: /wp-content/uploads/2025/10/dci-online-terms-boston.pdf
Disallow: /wp-content/uploads/2025/10/dci-online-terms-distributor.pdf
Disallow: /wp-content/uploads/2026/03/AI-Driven-Engineering-March-2026-webinar-w-Sascha-Heyer.pdf

# Disallow paginated pages
Disallow: */page/

# Disallow explicit archive paths
Disallow: /archive/

# Disallow author archives
Disallow: /author/

# Disallow taxonomy/filter archives (Review if these are used as landing pages)
Disallow: /client_categories/
Disallow: /content_type/
Disallow: /topics/
Disallow: /use-cases/
Disallow: /case_study_industries/

# Disallow search result pages
Disallow: /?s=
Disallow: /search/

# Prevent indexing of tracking parameters
Disallow: /*?__hstc=
Disallow: /*?__hssc=
Disallow: /*?__hsfp=

# --------------------------------------------
# AI / LLM crawlers — explicit allow
# (redundant with the global group, but included for clarity)
# --------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Amazonbot
Allow: /

# --------------------------------------------
# Atlassian
# --------------------------------------------
User-agent: atlassian-bot
Allow: /

# --------------------------------------------
# Sitemap
# --------------------------------------------
Sitemap: https://www.doit.com/sitemap_index.xml