# robots.txt


User-agent: Googlebot
Disallow: /search

User-agent: bingbot
Crawl-delay: 5
Disallow: /search
Disallow: /*?p=
Disallow: /*&p=

# ---- OpenAI ----
User-agent: OAI-SearchBot
Crawl-delay: 10
Disallow: /search
Disallow: /*?p=
Disallow: /*&p=

User-agent: GPTBot
Crawl-delay: 10
Disallow: /search
Disallow: /*?p=
Disallow: /*&p=

# ---- Anthropic ----
User-agent: Claude-SearchBot
Crawl-delay: 10
Disallow: /search
Disallow: /*?p=
Disallow: /*&p=

User-agent: ClaudeBot
Crawl-delay: 10
Disallow: /search
Disallow: /*?p=
Disallow: /*&p=

# ---- Perplexity ----
User-agent: PerplexityBot
Crawl-delay: 10
Disallow: /search
Disallow: /*?p=
Disallow: /*&p=

# ---- Common Crawl ----
User-agent: CCBot
Crawl-delay: 15
Disallow: /search
Disallow: /*?p=
Disallow: /*&p=

# ---- SEO / backlink crawlers (often “chatty”) ----
User-agent: AhrefsBot
Crawl-delay: 15
Disallow: /search
Disallow: /*?p=
Disallow: /*&p=

User-agent: SemrushBot
Crawl-delay: 15
Disallow: /search
Disallow: /*?p=
Disallow: /*&p=

User-agent: DotBot
Crawl-delay: 15
Disallow: /search
Disallow: /*?p=
Disallow: /*&p=

User-agent: MJ12bot
Crawl-delay: 15
Disallow: /search
Disallow: /*?p=
Disallow: /*&p=

# ---- Default for everything else ----
User-agent: *
Crawl-delay: 10
Disallow: /search
Disallow: /*?p=
Disallow: /*&p=