###################################
# DEFAULT — ALLOW EVERYTHING
###################################
User-agent: *
Allow: /

###################################
# GOOGLE — Crawl Budget Optimized
###################################
User-agent: Googlebot
# Block parameter pollution
Disallow: /*?page=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*&page=
Disallow: /*&sort=
Disallow: /*&filter=
# Exception: member pages need params
Allow: /members/*?*
# Block thin/low-value content
Disallow: /ore-sorting
Disallow: /surface-miner-and-terrain-leveler-hire
Disallow: /concrete-placing-systems
Disallow: /belly-dumper-hire
Disallow: /wall-formwork
Disallow: /formwork-props
Disallow: /high-reach-excavator-hire
Disallow: /fall-arrest-system-hire
Disallow: /hoarding-signage
Disallow: /water-scraper-hire
Disallow: /column-formwork
Disallow: /fauna-spotter-catcher
Disallow: /rumble-grid
Disallow: /revegetation
Disallow: /underground-loader-hire
Disallow: /grout-pump-hire
Disallow: /environmental-management
Disallow: /drill-stem-testing
Disallow: /ptz-camera-inspections
Disallow: /tyre-handler-hire
Disallow: /grab-truck-hire
Disallow: /block-and-tackle-hire
Disallow: /anchor-drilling
Disallow: /amphibious-excavator-hire
Disallow: /pilot-vehicle-hire
Disallow: /spider-excavator-hire
Disallow: /beaver-tail-truck-hire
Disallow: /ice-machine-hire
Disallow: /hydraulic-jack-hire
Disallow: /duct-lift-hire

###################################
# BING — More Permissive for AI
###################################
User-agent: bingbot
# Only block genuinely thin pages
Disallow: /ore-sorting
Disallow: /surface-miner-and-terrain-leveler-hire
Disallow: /concrete-placing-systems
Disallow: /belly-dumper-hire
Disallow: /wall-formwork
Disallow: /formwork-props
Disallow: /high-reach-excavator-hire
Disallow: /fall-arrest-system-hire
Disallow: /hoarding-signage
Disallow: /water-scraper-hire
Disallow: /column-formwork
Disallow: /fauna-spotter-catcher
Disallow: /rumble-grid
Disallow: /revegetation
Disallow: /underground-loader-hire
Disallow: /grout-pump-hire
Disallow: /environmental-management
Disallow: /drill-stem-testing
Disallow: /ptz-camera-inspections
Disallow: /tyre-handler-hire
Disallow: /grab-truck-hire
Disallow: /block-and-tackle-hire
Disallow: /anchor-drilling
Disallow: /amphibious-excavator-hire
Disallow: /pilot-vehicle-hire
Disallow: /spider-excavator-hire
Disallow: /beaver-tail-truck-hire
Disallow: /ice-machine-hire
Disallow: /hydraulic-jack-hire
Disallow: /duct-lift-hire

# Let Bing crawl more for Copilot/ChatGPT

###################################
# AI / LLM CRAWLERS — FULL ACCESS
###################################
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Diffbot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Amazonbot
Allow: /

###################################
# SEO TOOLS — Controlled Access
###################################
User-agent: SemrushBot
User-agent: SemrushBot-SA
Crawl-delay: 2

User-agent: AhrefsBot
Crawl-delay: 2

User-agent: DotBot
Crawl-delay: 2

###################################
# BLOCK BAD / AGGRESSIVE BOTS
###################################
User-agent: MJ12Bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 5

###################################
# SITEMAP
###################################
Sitemap: https://www.iseekplant.com.au/sitemap.xml