# we use Shopify as our ecommerce platform
#  ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
#  ┃  Robots & Agent policy                                               ┃
#  ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
#  ┃  Checkouts are for humans.                                           ┃
#  ┃  * Automated scraping, “buy-for-me” agents, or any end-to-end flow   ┃
#  ┃    that completes payment without a final human review step is not   ┃
#  ┃    permitted.                                                        ┃
#  ┃  * Legitimate integrators must use the official Checkout Kit:        ┃
#  ┃      https://www.shopify.com/checkout-kit                            ┃
#  ┃                                                                      ┃
#  ┃  Terms of Service: https://www.shopify.com/legal/terms               ┃
#  ┃  Contact: bots@shopify.com                                           ┃
#  ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
# Block unwanted bots completelyUser-agent: SeekportBot
    Disallow: /
    User-agent: MJ12bot
    Disallow: /
    User-agent: Nutch
    Disallow: /
    # Shopify core disallows
    User-agent: *
    Disallow: /a/downloads/-/*
    Disallow: /admin
    Disallow: /cart
    Disallow: /orders
    Disallow: /checkouts/
    Disallow: /checkout
    Disallow: /88653758791/checkouts
    Disallow: /88653758791/orders
    Disallow: /carts
    Disallow: /account
    Disallow: /collections/*sort_by*
    Disallow: /*/collections/*sort_by*
    Disallow: /collections/*+*
    Disallow: /collections/*%2B*
    Disallow: /collections/*%2b*
    Disallow: /*/collections/*+*
    Disallow: /*/collections/*%2B*
    Disallow: /*/collections/*%2b*
    Disallow: */collections/*filter*&*filter*
    Disallow: /blogs/*+*
    Disallow: /blogs/*%2B*
    Disallow: /blogs/*%2b*
    Disallow: /*/blogs/*+*
    Disallow: /*/blogs/*%2B*
    Disallow: /*/blogs/*%2b*
    Disallow: /*?*oseid=*
    Disallow: /*preview_theme_id*
    Disallow: /*preview_script_id*
    Disallow: /policies/
    Disallow: /*/policies/
    Disallow: /*/*?*ls=*&ls=*
    Disallow: /*/*?*ls%3D*%3Fls%3D*
    Disallow: /*/*?*ls%3d*%3fls%3d*
    Disallow: /search
    Disallow: /sf_private_access_tokens
    Disallow: /apple-app-site-association
    Disallow: /.well-known/shopify/monorail
    Disallow: /cdn/wpm/*.js
    Disallow: /recommendations/products
    Disallow: /*/recommendations/products
    Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
    Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
    Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
    Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
    # Google adsbot rules
    User-agent: adsbot-google
    Disallow: /checkouts/
    Disallow: /checkout
    Disallow: /carts
    Disallow: /orders
    Disallow: /88653758791/checkouts
    Disallow: /88653758791/orders
    Disallow: /*?*oseid=*
    Disallow: /*preview_theme_id*
    Disallow: /*preview_script_id*
    Disallow: /cdn/wpm/*.js
    Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
    Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
    Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
    Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote
    Disallow /pages/fabric-shops/*
    # Ahrefs throttling
    User-agent: AhrefsBot
    Crawl-delay: 10
    User-agent: AhrefsSiteAudit
    Crawl-delay: 10
    # Pinterest throttling
    User-agent: Pinterest
    Crawl-delay: 1
    # --- AI & SEARCH CRAWLERS EXPLICITLY ALLOWED ---
    User-agent: Googlebot
    Allow: /
    User-agent: Googlebot-Image
    Allow: /
    User-agent: Googlebot-News
    Allow: /
    User-agent: Googlebot-Video
    Allow: /
    User-agent: Google-Extended
    Allow: /
    User-agent: Googlebot/2.1
    Allow: /
    User-agent: bingbot
    Allow: /
    User-agent: Bingbot/2.0
    Allow: /
    User-agent: GPTBot
    Allow: /
    User-agent: GPTBot/1.1
    Allow: /
    User-agent: GPTBot/1.2
    Allow: /
    User-agent: Meta-ExternalFetcher
    Allow: /
    User-agent: Meta-ExternalAgent
    Allow: /
    User-agent: FacebookBot
    Allow: /
    User-agent: ClaudeBot
    Allow: /
    User-agent: Claude-Web
    Allow: /
    User-agent: anthropic-ai
    Allow: /
    User-agent: Claude-User
    Allow: /
    User-agent: Claude-SearchBot
    Allow: /
    User-agent: Perplexity-User
    Allow: /
    User-agent: PerplexityBot
    Allow: /
    # OpenAI / Agentic Search
    User-agent: ChatGPT-User
    Allow: /
    User-agent: OAI-SearchBot
    Allow: /
    User-agent: Applebot
    Allow: /
    User-agent: Applebot-Extended
    Allow: /
    User-agent: Amazonbot
    Allow: /
    User-agent: NovaAct
    Allow: /
    User-agent: Bytespider
    Allow: /
    User-agent: CCBot
    Allow: /
    User-agent: DuckAssistBot
    Allow: /
    User-agent: cohere-ai
    Allow: /
    User-agent: cohere-training-data-crawler
    Allow: /
    User-agent: AI2Bot
    Allow: /
    User-agent: YouBot
    Allow: /
    User-agent: MistralAI-User
    Allow: /
    User-agent: Kangaroo Bot
    Allow: /
    # Sitemap — update to new domain after launch
    Sitemap: https://www.justfabrics.co.uk/sitemap.xml