# robots.txt — Big Boy Toyz (bigboytoyz.com) # Last updated: 2026-03-15 # ============================================================================= # Default rules for all crawlers # ============================================================================= User-agent: * Allow: / # --- Internal / utility pages (no SEO value) --- Disallow: /api/ Disallow: /payment/ Disallow: /booking Disallow: /deeplinking Disallow: /search-results Disallow: /filter-results Disallow: /compare-cars Disallow: /test-page # --- Next.js internals --- Disallow: /_next/ Disallow: /api-test.js # --- Duplicate / legacy sell pages --- Disallow: /sell-used-luxury-car-2 Disallow: /sell-my-used-bmw-car # --- Crawl rate (optional, respected by well-behaved bots) --- Crawl-delay: 1 # ============================================================================= # Google (no crawl-delay support, uses Search Console settings instead) # ============================================================================= User-agent: Googlebot Allow: / Disallow: /api/ Disallow: /payment/ Disallow: /booking Disallow: /deeplinking Disallow: /search-results Disallow: /filter-results Disallow: /compare-cars Disallow: /test-page Disallow: /_next/ Disallow: /api-test.js Disallow: /sell-used-luxury-car-2 Disallow: /sell-my-used-bmw-car User-agent: Googlebot-Image Allow: /images/ Allow: /sitemaps/ # ============================================================================= # Bing # ============================================================================= User-agent: Bingbot Allow: / Disallow: /api/ Disallow: /payment/ Disallow: /booking Disallow: /deeplinking Disallow: /search-results Disallow: /filter-results Disallow: /compare-cars Disallow: /test-page Disallow: /_next/ Disallow: /api-test.js Disallow: /sell-used-luxury-car-2 Disallow: /sell-my-used-bmw-car Crawl-delay: 1 # ============================================================================= # AI Search Crawlers — Allow for AI Overview / citation visibility # ============================================================================= User-agent: GPTBot Allow: / Disallow: /api/ Disallow: /payment/ Disallow: /booking Disallow: /_next/ User-agent: ChatGPT-User Allow: / Disallow: /api/ Disallow: /payment/ Disallow: /booking Disallow: /_next/ User-agent: ClaudeBot Allow: / Disallow: /api/ Disallow: /payment/ Disallow: /booking Disallow: /_next/ User-agent: PerplexityBot Allow: / Disallow: /api/ Disallow: /payment/ Disallow: /booking Disallow: /_next/ User-agent: Applebot Allow: / Disallow: /api/ Disallow: /payment/ Disallow: /booking Disallow: /_next/ User-agent: cohere-ai Allow: / Disallow: /api/ Disallow: /payment/ Disallow: /booking Disallow: /_next/ # ============================================================================= # Social media crawlers — Allow for rich link previews # ============================================================================= User-agent: Twitterbot Allow: / User-agent: facebookexternalhit Allow: / User-agent: LinkedInBot Allow: / User-agent: WhatsApp Allow: / # ============================================================================= # Block known bad bots / scrapers # ============================================================================= User-agent: AhrefsBot Crawl-delay: 10 User-agent: SemrushBot Crawl-delay: 10 User-agent: DotBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: BLEXBot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: PetalBot Crawl-delay: 10 # ============================================================================= # Sitemaps # ============================================================================= Sitemap: https://www.bigboytoyz.com/sitemaps/sitemap-index.xml