# ════════════════════════════════════════════════════════
# robots.txt — esd.equipment
# Last updated: 2026-04-09
# ════════════════════════════════════════════════════════

# ────────────────────────────────────────────────────────
# AI CRAWLERS — ALLOWED (retrieval/search only)
# Training opt-out handled via X-Robots-Tag / ai.txt
# ────────────────────────────────────────────────────────
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: meta-externalagent
Allow: /

# ────────────────────────────────────────────────────────
# AI / SCRAPER CRAWLERS — BLOCKED
# Aggressive scrapers, training-focused, or low-value bots
# ────────────────────────────────────────────────────────
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: FacebookBot
Disallow: /

# ────────────────────────────────────────────────────────
# ALL CRAWLERS — DEFAULT RULES
# (Googlebot, Bingbot, YandexBot, DuckDuckBot, Applebot
#  and all other well-behaved bots land here)
# ────────────────────────────────────────────────────────
User-agent: *
Crawl-delay: 1

# ── Explicitly allowed resources ──
Allow: /sitemap.xml
Allow: /media/sitemap/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /media/catalog/product/
Allow: /media/catalog/category/
Allow: /static/

# ── Magento admin & auth areas ──
Disallow: /admin/
Disallow: /adminhtml/
Disallow: /customer/
Disallow: /checkout/
Disallow: /cart/
Disallow: /wishlist/
Disallow: /review/
Disallow: /compare/
Disallow: /tag/
Disallow: /sendfriend/
Disallow: /newsletter/
Disallow: /contact/
Disallow: /vault/
Disallow: /persistent/
Disallow: /multishipping/

# ── Search (duplicate content) ──
Disallow: /catalogsearch/
Disallow: /search/

# ── Magento API endpoints (not for crawlers) ──
Disallow: /rest/
Disallow: /graphql
Disallow: /soap/

# ── Payment callbacks ──
Disallow: /paypal/
Disallow: /stripe/
Disallow: /braintree/

# ── Magento system / infrastructure paths ──
Disallow: /cgi-bin/
Disallow: /var/
Disallow: /pub/
Disallow: /app/
Disallow: /lib/
Disallow: /setup/
Disallow: /update/
Disallow: /downloader/
Disallow: /errors/
Disallow: /shell/
Disallow: /report/
Disallow: /dev/
Disallow: /phpserver/
Disallow: /generated/

# ── Duplicate view URLs (canonical = clean SEF paths) ──
Disallow: /catalog/product/view/
Disallow: /catalog/category/view/
Disallow: /catalog/product/gallery/
Disallow: /index.php/
Disallow: /index.php

# ── Faceted navigation & crawl budget protection ──
# Sort / pagination / display mode
Disallow: /*?dir=
Disallow: /*?limit=
Disallow: /*?mode=
Disallow: /*?p=
Disallow: /*&p=
Disallow: /*?product_list_dir=
Disallow: /*?product_list_limit=
Disallow: /*?product_list_mode=
Disallow: /*?product_list_order=

# Common filter parameters (Amasty Layered Nav, native)
Disallow: /*?price=
Disallow: /*?cat=
Disallow: /*?color=
Disallow: /*?size=
Disallow: /*?manufacturer=
Disallow: /*?brand=
Disallow: /*?amnoroute=

# Catch-all: any URL with multiple filter params
Disallow: /*?*&*&

# Session / tracking parameters
Disallow: /*?SID=
Disallow: /*?___from_store=
Disallow: /*?___store=

Disallow: /*?manufacturer=
Disallow: /*&manufacturer=

# ────────────────────────────────────────────────────────
# SITEMAPS
# ────────────────────────────────────────────────────────
Sitemap: https://esd.equipment/sitemap.xml
Sitemap: https://esd.equipment/media/sitemap/sitemap_en.xml
Sitemap: https://esd.equipment/media/sitemap/sitemap_de.xml
Sitemap: https://esd.equipment/media/sitemap/sitemap_fr.xml