# ============================================================
# robots.txt — carparts-onlineshop.com (Magento 2, 5 talen)
# ============================================================

# 1. Google Ads bots — moeten alles kunnen crawlen
User-agent: AdsBot-Google
Disallow:

User-agent: AdsBot-Google-Mobile
Disallow:

# 2. Agressieve scrapers volledig blokkeren
User-agent: Amazonbot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: TikTokSpider
Disallow: /

User-agent: AwarioBot
Disallow: /

User-agent: GeedoProductSearch
Disallow: /

User-agent: FacebookExternalHit
Disallow: /

User-agent: facebookexternalhit
Disallow: /

# 3. Alle overige crawlers (Googlebot, Bingbot, GPTBot, etc.)
User-agent: *

# --- Magento layered navigation & gefilterde URLs ---
Disallow: /*?
Disallow: /*&

# --- Magento-specifieke verspilling ---
Disallow: /catalogsearch/
Disallow: /search/
Disallow: /checkout/
Disallow: /customer/
Disallow: /account/
Disallow: /wishlist/
Disallow: /review/product/
Disallow: /catalog/product_compare/
Disallow: /catalog/product/view/id/

# --- Systeem- en beveiligingsmappen ---
Disallow: /admin/
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/
Disallow: /setup/
Disallow: /update/
Disallow: /var/
Disallow: /vendor/

# --- Gevoelige bestanden ---
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /php.ini.sample

# Crawl-delay verlaagd (10s was te restrictief voor grote catalogi)
Crawl-delay: 2

# ============================================================
# Sitemaps
# ============================================================
Sitemap: https://www.carparts-onlineshop.com/sitemap.xml
Sitemap: https://www.carparts-onlineshop.com/sitemap_nl.xml
Sitemap: https://www.carparts-onlineshop.com/sitemap_de.xml
Sitemap: https://www.carparts-onlineshop.com/sitemap_fr.xml
Sitemap: https://www.carparts-onlineshop.com/sitemap_es.xml