# Explicit Allow Rules for Major AI and LLM Crawlers

# OpenAI bots
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
Allow: /

# Anthropic (Claude)
User-agent: AnthropicBot
Allow: /

#Perplexity AI
User-agent: PerplexityBot
Allow: /

# allow the google advertising bot full access
User-agent: Mediapartners-Google
Allow: /

User-agent: TalkTalk Virus Alerts Scanning Engine
Disallow: /jobs/

User-agent: JoobleBot
Crawl-delay: 3

User-agent: AhrefsBot
Crawl-delay: 3

# Unified rules for all bots to avoid duplication
# Explicitly including Googlebot and Bingbot to ensure they follow the same rules as other crawlers
User-agent: *
User-agent: bingbot
User-agent: Googlebot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
User-agent: AnthropicBot
User-agent: PerplexityBot
Disallow: /linkclicked.html
Disallow: /api/
Disallow: /handlers/
Disallow: /salary-checker

Disallow: /*?sourceInternal=
Disallow: /*?directjob
Disallow: /*buy?path

# Courses
Disallow: /courses/api/
Disallow: /courses/errorlogging
Disallow: /courses/logging/logcourseview
Disallow: /courses/*/enquiry/externalredirect
Disallow: /courses/externalredirect/*
Disallow: /courses/account/signin?returnUrl=
Disallow: /courses/*?*pricerangeoptions=
Disallow: /courses/*?*sortby=
Disallow: /courses/*?*pagesize=
Disallow: /courses/*?*location=
Disallow: /courses/*?*pricefrom=
Disallow: /courses/*?*priceto=
Disallow: /courses/*?*duration=
Disallow: /courses/buy-now/
Disallow: /courses/*/buy-now
Disallow: /courses/on-demand/
Disallow: /courses/*/on-demand
Disallow: /courses/checkout/createdigitalwallet

# Sitemap URL:
Sitemap: https://www.reed.co.uk/sitemap.xml
Sitemap: https://www.reed.co.uk/priority-jobseeker-sitemap.xml
Sitemap: https://www.reed.co.uk/courses/sitemap.xml

#Allow: WebSurge