User-agent: * Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /carts Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /59741798600/checkouts Disallow: /59741798600/orders Disallow: /account # Collection crawl traps (sorting + filters + page=1) Disallow: /collections/sort_by Disallow: //collections/*sort_by Disallow: /collections/+ Disallow: /collections/%2B Disallow: /collections/%2b Disallow: //collections/+* Disallow: //collections/%2B* Disallow: //collections/%2b* Disallow: /collections/?*filter Disallow: //collections/?filter Disallow: /?*page=1 # Blog crawl traps Disallow: /blogs/+ Disallow: /blogs/%2B Disallow: /blogs/%2b Disallow: //blogs/+* Disallow: //blogs/%2B* Disallow: //blogs/%2b* Disallow: /*.atom # Tracking / preview / internal params Disallow: /?*oseid= Disallow: /preview_theme_id Disallow: /preview_script_id Disallow: //?ls=&ls=* Disallow: //?ls%3D%3Fls%3D* Disallow: //?ls%3d%3fls%3d* # Internal search (avoid index bloat) Disallow: /search # Shopify internal endpoints Disallow: /policies/ Disallow: /*/policies/ Disallow: /sf_private_access_tokens Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /recommendations/products Disallow: /*/recommendations/products # Web pixels / WPM sandbox URLs Disallow: /wpm@ Disallow: /web-pixels@ Disallow: /cdn/wpm/*.js # Biggest crawl waste in your GSC samples: pr_* recommendation parameters Disallow: /?*pr_ # Duplicate parameters frequently causing alternates/redirects Disallow: /?*variant= Disallow: /?*currency= # Shopify remote product URL pattern (keep Shopify default) Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: //products/-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections//products/-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: //collections//products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://www.beyondeast.com/sitemap.xml # Google adsbot ignores robots.txt unless specifically named User-agent: adsbot-google Disallow: /checkouts/ Disallow: /checkout Disallow: /carts Disallow: /orders Disallow: /59741798600/checkouts Disallow: /59741798600/orders Disallow: /?*oseid= Disallow: /preview_theme_id Disallow: /preview_script_id Disallow: /cdn/wpm/*.js Disallow: /wpm@ Disallow: /web-pixels@ Disallow: /?*pr_ Disallow: /?*variant= Disallow: /?*currency= Disallow: /sf_private_access_tokens Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: //products/-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections//products/-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: //collections//products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://www.beyondeast.com/sitemap.xml User-agent: Nutch Disallow: / # Optional: slow down common SEO crawlers (does not affect Google) User-agent: AhrefsBot Crawl-delay: 10 User-agent: AhrefsSiteAudit Crawl-delay: 10 User-agent: MJ12bot Crawl-delay: 10 User-agent: Pinterest Crawl-delay: 1