User-agent: * Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /8009842786/checkouts Disallow: /8009842786/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /recommendations/products Disallow: /*/recommendations/products Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /blogs/*/tagged Disallow: /collections/vendors*?*q= Sitemap: https://travelpro.com/sitemap.xml User-agent: adsbot-google Disallow: /checkouts/ Disallow: /checkout Disallow: /carts Disallow: /orders Disallow: /8009842786/checkouts Disallow: /8009842786/orders Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /cdn/wpm/*.js Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /services/login_with_shop User-agent: Nutch Disallow: / User-agent: AhrefsBot Crawl-delay: 10 Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /8009842786/checkouts Disallow: /8009842786/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://travelpro.com/sitemap.xml User-agent: AhrefsSiteAudit Crawl-delay: 10 Disallow: /a/downloads/-/* Disallow: /admin Disallow: /cart Disallow: /orders Disallow: /checkouts/ Disallow: /checkout Disallow: /8009842786/checkouts Disallow: /8009842786/orders Disallow: /carts Disallow: /account Disallow: /collections/*sort_by* Disallow: /*/collections/*sort_by* Disallow: /collections/*+* Disallow: /collections/*%2B* Disallow: /collections/*%2b* Disallow: /*/collections/*+* Disallow: /*/collections/*%2B* Disallow: /*/collections/*%2b* Disallow: */collections/*filter*&*filter* Disallow: /blogs/*+* Disallow: /blogs/*%2B* Disallow: /blogs/*%2b* Disallow: /*/blogs/*+* Disallow: /*/blogs/*%2B* Disallow: /*/blogs/*%2b* Disallow: /*?*oseid=* Disallow: /*preview_theme_id* Disallow: /*preview_script_id* Disallow: /policies/ Disallow: /*/policies/ Disallow: /*/*?*ls=*&ls=* Disallow: /*/*?*ls%3D*%3Fls%3D* Disallow: /*/*?*ls%3d*%3fls%3d* Disallow: /search Disallow: /apple-app-site-association Disallow: /.well-known/shopify/monorail Disallow: /cdn/wpm/*.js Disallow: /services/login_with_shop Disallow: /products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Disallow: /*/collections/*/products/*-[a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9][a-f0-9]-remote Sitemap: https://travelpro.com/sitemap.xml User-agent: MJ12bot Crawl-delay: 10 User-agent: Pinterest Crawl-delay: 1 # ============================================================ # AI CRAWLER ACCESS RULES # Explicitly allow AI search/citation bots to crawl public content # Last updated: 2026-02-23 # ============================================================ # --- OPENAI --- # Search results in ChatGPT (NOT used for training) User-agent: OAI-SearchBot Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account Disallow: /admin Disallow: /orders Disallow: /search # User-initiated browsing from ChatGPT User-agent: ChatGPT-User Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account Disallow: /admin # Model training (your brand info gets baked into GPT models) User-agent: GPTBot Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account Disallow: /admin Disallow: /orders # --- ANTHROPIC (Claude) --- User-agent: ClaudeBot Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account Disallow: /admin User-agent: Claude-SearchBot Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account User-agent: anthropic-ai Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account User-agent: Claude-User Allow: / # --- PERPLEXITY --- User-agent: PerplexityBot Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account User-agent: Perplexity-User Allow: / # --- GOOGLE (Gemini / AI Overviews) --- User-agent: Google-Extended Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account # --- APPLE (Siri / Apple Intelligence) --- User-agent: Applebot-Extended Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account # --- MICROSOFT (Bing Chat / Copilot) --- User-agent: Bingbot Allow: / # --- AMAZON --- User-agent: Amazonbot Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account # --- OTHER AI SEARCH --- User-agent: YouBot Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account User-agent: Bytespider Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account User-agent: PhindBot Allow: / User-agent: CCBot Allow: /products/ Allow: /collections/ Allow: /blogs/ Allow: /pages/ Disallow: /checkout Disallow: /cart Disallow: /account # ============================================================ # BLOCK ABUSIVE / DATA-RESELLING CRAWLERS # ============================================================ User-agent: Bytespider Disallow: / User-agent: PanguBot Disallow: / User-agent: Omgilibot Disallow: / User-agent: Omgili Disallow: / User-agent: Webzio-Extended Disallow: / User-agent: TikTokSpider Disallow: / # Sitemap Sitemap: /sitemap.xml