# ============================================
#  GLOBAL RULES — FOR ALL BOTS (clean + SEO-safe)
# ============================================

User-agent: *
# Block unnecessary load
Disallow: /wp-admin/
Disallow: /readme.html
Disallow: /refer/
Disallow: /thanks/
Disallow: /wishlist/
Disallow: /profil/
Disallow: /track-your-order/
Disallow: /author/
Disallow: /tag/
Disallow: /feed/
Disallow: /*?add-to-cart=
Disallow: /*&add-to-cart=
Disallow: /*?products_per_page=
Disallow: /*&products_per_page=
Disallow: /*?products_per_row=
Disallow: /*&products_per_row=
Disallow: /*?stock_status=
Disallow: /*&stock_status=
Disallow: /*?s=
Disallow: /*&s=

# Block internal searches (heavy MySQL)
Disallow: /search/
Disallow: /?s=

# Block REST API scrapers
Disallow: /wp-json/

# Allow essential assets
Allow: /wp-content/uploads/
Allow: /wp-admin/admin-ajax.php

# Block noisy URL parameters
Disallow: /*?*xts-html-block=
Disallow: /*?add-to-cart=
Disallow: /*?orderby=
Disallow: /*&orderby=
Disallow: /*?filter_
Disallow: /*&filter_

Sitemap: https://drones.bg/sitemap_index.xml


# ============================================
#   META / FACEBOOK CRAWLER — RATE LIMITED
# ============================================
# This applies to the crawler seen as: meta-externalads/1.1
# We keep it allowed, but slow it down.

User-agent: meta-externalads
Crawl-delay: 5
Allow: /

User-agent: facebookexternalhit
Crawl-delay: 5
Allow: /


# ============================================
#   ALLOWED AI BOTS (optional)
# ============================================

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

User-agent: ZipchatBot
Disallow: /*?
Disallow: /cart/
Disallow: /checkout/
Disallow: /my-account/
Disallow: /wp-json/
Disallow: /wp-admin/


# ============================================
#   BLOCK AGGRESSIVE SEO + SCRAPERS
# ============================================

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: AspiegelBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: PetalBot
Disallow: /