# ===========================================================================
# robots.txt — Gumtree.co.za
# Updated: 2026-04-14
#
# CHANGES FROM PREVIOUS VERSION
# ──────────────────────────────
# 1. Replaced three legacy sitemap references (vip_index, loccat_index,
#    loccatattr_index) with a single sitemap_index.xml that covers all page
#    types. The new index references sub-sitemaps for static pages,
#    categories, locations, category×location pages, and rolling monthly
#    listing batches. Submit the new URL to Google Search Console and
#    Bing Webmaster Tools.
# 2. Added AI training bot blocks (GPTBot, Google-Extended, ClaudeBot, etc.)
#    to prevent listing data being used in third-party model training without
#    a commercial agreement.
# 3. All existing Disallow rules are carried over unchanged.
# ===========================================================================


# ---------------------------------------------------------------------------
# SITEMAPS
# ---------------------------------------------------------------------------
# REMOVED (legacy):
#   Sitemap: https://www.gumtree.co.za/sitemaps/za/vip/vip_index.xml
#   Sitemap: https://www.gumtree.co.za/sitemaps/za/loccat/loccat_index.xml
#   Sitemap: https://www.gumtree.co.za/sitemaps/za/loccatattr/loccatattr_index.xml
#
# REPLACED WITH:
Sitemap: https://www.gumtree.co.za/sitemap_index.xml


# ===========================================================================
# ALL BOTS (catch-all)
# ===========================================================================
User-agent: *
#Sorting parameters
Disallow: *?*order=*
Disallow: *?*galleryView=*
Disallow: *?*sort=*
#Other comments:
Disallow: *?*q=*
Disallow: *?*sr=*
Disallow: /f-pbfvb/*
Disallow: *?*be=*
Disallow: *?*fe=*
Disallow: *sr~1$
Disallow: *sr~2$
Disallow: *sr~8$
Disallow: /s-*/page-*/v1*q0p*
Disallow: /306629115/*
Disallow: /bolt-2dot0-frontend$
Disallow: /7162/*
Disallow: *?*AdType=*
Disallow: *?*categoryId=*
Disallow: *?*isSearchForm=*
Disallow: *?*km=*
Disallow: *?*locationId=*
Disallow: *?*pr=*
Disallow: *?*redirect=*
Disallow: *?*redirectToParent*
Disallow: *?*skipAttrFilter=*
Disallow: *?status=*
Disallow: /forgotPassword.html?*
Disallow: /register.html?*
Disallow: /login.html?*
Disallow: /post.html?*
Disallow: *?*priceType=*
Disallow: /rui-api/*
Disallow: *.ugzy


# ===========================================================================
# BINGBOT
# ===========================================================================
User-agent: bingbot
#Sorting parameters
Disallow: *?*order=*
Disallow: *?*galleryView=*
Disallow: *?*sort=*
#Other comments:
Disallow: *?*q=*
Disallow: *?*sr=*
Disallow: /f-pbfvb/*
Disallow: *?*be=*
Disallow: *?*fe=*
Disallow: *sr~1$
Disallow: *sr~2$
Disallow: *sr~8$
Disallow: /s-*/page-*/v1*q0p*
Disallow: /306629115/*
Disallow: /bolt-2dot0-frontend$
Disallow: /7162/*
Disallow: *?*AdType=*
Disallow: *?*categoryId=*
Disallow: *?*isSearchForm=*
Disallow: *?*km=*
Disallow: *?*locationId=*
Disallow: *?*pr=*
Disallow: *?*redirect=*
Disallow: *?*redirectToParent*
Disallow: *?*skipAttrFilter=*
Disallow: *?status=*
Disallow: /forgotPassword.html?*
Disallow: /register.html?*
Disallow: /login.html?*
Disallow: /post.html?*
Disallow: *?*priceType=*
Disallow: /rui-api/*
Disallow: *.ugzy
Disallow: */undefined$


# ===========================================================================
# ADS BOT - GOOGLE
# ===========================================================================
User-agent: AdsBot-Google
#Sorting parameters
Disallow: *?*order=*
Disallow: *?*galleryView=*
Disallow: *?*sort=*
#Other comments:
Disallow: *?*q=*
Disallow: *?*sr=*
Disallow: /f-pbfvb/*
Disallow: *?*be=*
Disallow: *?*fe=*
Disallow: *sr~1$
Disallow: *sr~2$
Disallow: *sr~8$
Disallow: /s-*/page-*/v1*q0p*
Disallow: /306629115/*
Disallow: /bolt-2dot0-frontend$
Disallow: /7162/*
Disallow: *?*AdType=*
Disallow: *?*categoryId=*
Disallow: *?*isSearchForm=*
Disallow: *?*km=*
Disallow: *?*locationId=*
Disallow: *?*pr=*
Disallow: *?*redirect=*
Disallow: *?*redirectToParent*
Disallow: *?*skipAttrFilter=*
Disallow: *?status=*
Disallow: /forgotPassword.html?*
Disallow: /register.html?*
Disallow: /login.html?*
Disallow: /post.html?*
Disallow: *?*priceType=*
Disallow: /rui-api/*
Disallow: *.ugzy


# ===========================================================================
# MEDIAPARTNERS - GOOGLE
# ===========================================================================
User-agent: Mediapartners-Google
#Sorting parameters
Disallow: *?*order=*
Disallow: *?*galleryView=*
Disallow: *?*sort=*
#Other comments:
Disallow: *?*q=*
Disallow: *?*sr=*
Disallow: /f-pbfvb/*
Disallow: *?*be=*
Disallow: *?*fe=*
Disallow: *sr~1$
Disallow: *sr~2$
Disallow: *sr~8$
Disallow: /s-*/page-*/v1*q0p*
Disallow: /306629115/*
Disallow: /bolt-2dot0-frontend$
Disallow: /7162/*
Disallow: *?*AdType=*
Disallow: *?*categoryId=*
Disallow: *?*isSearchForm=*
Disallow: *?*km=*
Disallow: *?*locationId=*
Disallow: *?*pr=*
Disallow: *?*redirect=*
Disallow: *?*redirectToParent*
Disallow: *?*skipAttrFilter=*
Disallow: *?status=*
Disallow: /forgotPassword.html?*
Disallow: /register.html?*
Disallow: /login.html?*
Disallow: /post.html?*
Disallow: *?*priceType=*
Disallow: /rui-api/*
Disallow: *.ugzy


# ===========================================================================
# GOOGLEBOT
# ===========================================================================
User-agent: Googlebot
#Sorting parameters
Disallow: *?*order=*
Disallow: *?*galleryView=*
Disallow: *?*sort=*
#Other comments:
Disallow: *?*q=*
Disallow: *?*sr=*
Disallow: /f-pbfvb/*
Disallow: *?*be=*
Disallow: *?*fe=*
Disallow: *sr~1$
Disallow: *sr~2$
Disallow: *sr~8$
Disallow: /s-*/page-*/v1*q0p*
Disallow: /306629115/*
Disallow: /bolt-2dot0-frontend$
Disallow: /7162/*
Disallow: *?*AdType=*
Disallow: *?*categoryId=*
Disallow: *?*isSearchForm=*
Disallow: *?*km=*
Disallow: *?*locationId=*
Disallow: *?*pr=*
Disallow: *?*redirect=*
Disallow: *?*redirectToParent*
Disallow: *?*skipAttrFilter=*
Disallow: *?status=*
Disallow: /forgotPassword.html?*
Disallow: /register.html?*
Disallow: /login.html?*
Disallow: /post.html?*
Disallow: *?*priceType=*
Disallow: /rui-api/*
Disallow: *.ugzy


# ===========================================================================
# AI TRAINING CRAWLERS (NEW)
# Block known AI training bots. Gumtree listing data should not be used
# to train third-party AI models without a commercial agreement in place.
# Review this section if a data-licensing deal is signed with any provider.
# ===========================================================================
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /