# ========================================= # General rules for all bots # ========================================= User-Agent: * Disallow: /suggest/?* Disallow: /*add-to-cart=* Disallow: /?s= Disallow: /*?s=* Disallow: /*?s=*&* Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Allow: /wp-admin/nav-menus.php Sitemap: https://www.celebrity-cutouts.com/sitemap_index.xml # ========================================= # SeekportBot: slow it down # ========================================= User-agent: SeekportBot Crawl-delay: 8 # ========================================= # Meta’s AI training crawler: polite limits # ========================================= User-agent: meta-externalagent # Block search results and paginated archive pages Disallow: /page/ Disallow: /*?s= # Allow product pages & categories Allow: /product-category/ Allow: /product/ # Polite crawl delay suggestion (~3 sec between requests) Crawl-delay: 3 # ========================================= # Optional: block other heavy Meta crawlers if needed # User-agent: meta-externalads # Crawl-delay: 5 # Disallow: /page/