# we use Shopify as our ecommerce platform

# ==================================================
# Robots & Agent policy
# ==================================================
# Checkouts are for humans.
# * Automated scraping or full purchase flows without human review are not allowed.
# * Official integration: https://www.shopify.com/checkout-kit
# Terms: https://www.shopify.com/legal/terms

# ==================================================
# GLOBAL RULES
# ==================================================
User-agent: *

# ---- Shopify Pixel / WPM / Dynamic URL cleanup ----
Disallow: /wpm@
Disallow: */wpm@
Disallow: /*/wpm@

Disallow: /processing
Disallow: */processing
Disallow: /*/processing

Disallow: */sandbox/
Disallow: /*/sandbox/

# ---- Checkout protection (critical) ----
Disallow: /checkout
Disallow: /checkouts/
Disallow: */checkout
Disallow: */checkouts/
Disallow: /*/checkout
Disallow: /*/checkouts/

# ---- Core system paths ----
Disallow: /admin
Disallow: /cart
Disallow: /carts
Disallow: /orders
Disallow: /account

# ---- Shopify internal / tracking ----
Disallow: /cdn/wpm/*.js
Disallow: /.well-known/shopify/monorail
Disallow: /sf_private_access_tokens

# ---- Search & filters (avoid crawl waste) ----
Disallow: /search
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*

# ---- Collection & blog parameter pollution ----
Disallow: /collections/*sort_by*
Disallow: /*/collections/*sort_by*
Disallow: /collections/*+*
Disallow: /collections/*%2B*
Disallow: /collections/*%2b*
Disallow: /*/collections/*+*
Disallow: /*/collections/*%2B*
Disallow: /*/collections/*%2b*
Disallow: */collections/*filter*&*filter*

Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /*/blogs/*+*
Disallow: /*/blogs/*%2B*
Disallow: /*/blogs/*%2b*

# ---- Policy pages (optional indexing control) ----
Disallow: /policies/
Disallow: /*/policies/

# ---- Localization duplicates (optional) ----
Disallow: /en-ca/
Disallow: /en-fr/
Disallow: /en-au/
Disallow: /en-se/
Disallow: /en-ch/
Disallow: /en-gb/

# ---- Product recommendation endpoints ----
Disallow: /recommendations/products
Disallow: /*/recommendations/products

# ---- Remote product duplicates ----
Disallow: /products/*-[a-f0-9]{8}-remote
Disallow: /*/products/*-[a-f0-9]{8}-remote
Disallow: /collections/*/products/*-[a-f0-9]{8}-remote
Disallow: /*/collections/*/products/*-[a-f0-9]{8}-remote

# ==================================================
# GOOGLE ADS BOT (explicit rules required)
# ==================================================
User-agent: adsbot-google
Disallow: /checkout
Disallow: /checkouts/
Disallow: /cart
Disallow: /carts
Disallow: /orders
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /cdn/wpm/*.js
Disallow: /sf_private_access_tokens

# ==================================================
# AGGRESSIVE / HEAVY CRAWLERS CONTROL
# ==================================================
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: AhrefsSiteAudit
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

# ==================================================
# BLOCK BAD ACTORS
# ==================================================
User-agent: Nutch
Disallow: /

# ==================================================
# AI / SEARCH ASSISTANTS
# ==================================================
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

# ==================================================
# SITEMAP
# ==================================================
Sitemap: https://merachfit.com/sitemap.xml