User-agent: *

# Admin & private areas
Disallow: /login
Disallow: /register
Disallow: /registerresult
Disallow: /customer/

# Ecommerce transactional flows
Disallow: /checkout
Disallow: /cart
Disallow: /clearcomparelist
Disallow: /compareproducts
Disallow: /onepagecheckout
Disallow: /order/
Disallow: /passwordrecovery/
Disallow: /returnrequest/
Disallow: /rewardpoints/
Disallow: /wishlist
Disallow: /backinstocksubscriptions/
Disallow: /emailwishlist
Disallow: /newsletter/
Disallow: /recentlyviewedproducts
Disallow: /storeclosed

# Application / API internals
Disallow: /_next/image
Disallow: /RetrieveProductRibbons

# Tracking parameters
Disallow: /*?*utm_
Disallow: /*?*gclid=
Disallow: /*?*fbclid=
Disallow: /*?*returnUrl=

# Presentation-only parameters
Disallow: /*?*orderby=
Disallow: /*?*pagesize=
Disallow: /*?*viewmode=
Disallow: /search?

# Faceted navigation (avoid crawl explosion)
Disallow: /*?*spec=*&*spec=

# Default pagination state
Disallow: /*?*pagenumber=1


# -------------------------------------
# Google Ads bot
# -------------------------------------
User-agent: AdsBot-Google
Disallow: /checkout


# -------------------------------------
# Ahrefs bots
# -------------------------------------
User-agent: AhrefsBot

User-agent: AhrefsSiteAudit
Crawl-delay: 10
Disallow: /login
Disallow: /register
Disallow: /registerresult
Disallow: /checkout
Disallow: /_next/image
Disallow: /RetrieveProductRibbons


# -------------------------------------
# Aggressive / unwanted crawlers
# -------------------------------------
User-agent: Nutch
Disallow: /

User-agent: zauba.io
Disallow: /

User-agent: zgrab
Disallow: /


# -------------------------------------
# MJ12 (Majestic)
# -------------------------------------
User-agent: MJ12bot
Crawl-delay: 10


# -------------------------------------
# Pinterest
# -------------------------------------
User-agent: Pinterest
Crawl-delay: 1


# -------------------------------------
# Sitemap
# -------------------------------------
Sitemap: https://www.busesforsale.com/sitemap.xml