# --- Default: allow major engines to render assets (important for CWV) ---
User-agent: *
Allow: /*.js$
Allow: /*.css$
Allow: /cdn/*
Allow: /assets/*
Allow: /files/*
Allow: /content/*
Allow: /s/files/*

# --- Disallow low-value system pages & account flows ---
Disallow: /cart
Disallow: /checkout
Disallow: /carts
Disallow: /orders
Disallow: /account
Disallow: /gift_cards
Disallow: /apps/

# --- Disallow internal search & filter/facet duplicates ---
Disallow: /search
# Faceted / duplicate variants & filtered collection URLs:
Disallow: /*?*sort_by=*
Disallow: /*?*filter=*
Disallow: /*?*filters=*
Disallow: /*?*constraint=*
Disallow: /*?*view=*
Disallow: /*?*variant=*
Disallow: /*?*utm_*
Disallow: /*?*ref=*
Disallow: /*?*gclid=*
Disallow: /*?*fbclid=*
# Allow normal pagination (use canonical on templates)
Allow: /*?page=*

# --- Shopify collection duplicates commonly created by “+” filter joins ---
# e.g., /collections/steelers+jerseys etc.
Disallow: /collections/*+*

# --- Policies (index the canonical policy pages only) ---
# Shopify auto-canonicals policies, but keep parameters out
Disallow: /policies/*?*

# --- Keep feeds and non-HTML endpoints out of the index ---
Disallow: /*.atom$
Disallow: /*.json$
Disallow: /*.rss$

# --- Good bot specifics (ensure full rendering) ---
User-agent: Googlebot
Allow: /*.js$
Allow: /*.css$
Allow: /cdn/*
Allow: /assets/*

User-agent: Bingbot
Allow: /*.js$
Allow: /*.css$
Allow: /cdn/*
Allow: /assets/*

# --- Throttle/deny known noisy or non-essential crawlers ---
# If you rely on these services, remove their blocks.
User-agent: Bytespider
Disallow: /

User-agent: BytedanceSpider
Disallow: /

User-agent: Bytedance
Disallow: /

# Common bandwidth-heavy SEO scrapers — optional:
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: SemrushBot
Disallow: /

# --- Sitemap(s) to guide discovery ---
Sitemap: https://www.tseshop.com/sitemap.xml