# EpicPxls robots.txt # AI Crawlers - Welcome with guidelines # See also: /llms.txt for detailed AI crawler guidance User-agent: GPTBot Allow: /items Allow: /free Allow: /alternatives Disallow: /admin Disallow: /users Disallow: */download Disallow: */carousel Crawl-delay: 1 User-agent: anthropic-web Allow: / Disallow: /admin Disallow: /users Disallow: */download Disallow: */carousel Crawl-delay: 1 User-agent: PerplexityBot Allow: / Disallow: /admin Disallow: /users Disallow: */download Crawl-delay: 1 User-agent: Google-Extended Allow: / Disallow: /admin Disallow: /users Crawl-delay: 1 User-agent: CCBot Allow: / Disallow: /admin Disallow: /users Crawl-delay: 5 # Default rules for all other crawlers User-agent: * # Admin area Disallow: /admin # Don't index carousel links Disallow: */carousel # Don't index download links Disallow: */download # Block pagination pages (crawl budget optimization) # Page 2+ are noindexed but blocking prevents wasted crawls Disallow: /items/*/page/* Disallow: */page/* # Block sort variations (same content, different order) Disallow: */sort/* # Filter pages (category, platform, type) are legitimate and indexable # Canonical tags handle deduplication; sort variations are blocked above # Block dialog format URLs (bare modal fragments, no SEO value) # These are internal AJAX endpoints that render without / tags Disallow: /*?*format=dialog Disallow: /*?*format=js # Block junk query parameters (crawl budget optimization) # Font preview state — creates 1000s of duplicate URL variations Disallow: /*?*preview_size= Disallow: /*?*preview_text= # Font style filter with preview creates spam-injected variations Disallow: /*?*style= # PageSpeed noscript artifacts Disallow: /*?*PageSpeed= # Referral tracking params Disallow: /*?*ref= # UTM tracking params (campaign tracking, not content) Disallow: /*?*utm_source= Disallow: /*?*utm_medium= Disallow: /*?*utm_campaign= # Internal search results (thin content, infinite variations) Disallow: /*?*q= Sitemap: https://www.epicpxls.com/sitemap.xml