# DEFAULT RULES FOR UNNAMED / GENERIC BOTS
User-agent: *
Disallow: /wp*
Disallow: /*?*__hs*
Disallow: /blog/redpanda-zookeeper-april-fools
Disallow: /console-marketing-resources*
Allow: /llms.txt
Allow: /ai-info

# ESSENTIAL SEARCH ENGINE CRAWLERS - MUST ALLOW (NO RESTRICTIONS)
User-agent: Googlebot
Disallow:

User-agent: Bingbot
Disallow:

User-agent: Yandexbot
Disallow:

# AI REAL-TIME RETRIEVAL BOTS - ALLOW FOR CITATIONS (NO RESTRICTIONS)
User-agent: ChatGPT-User
Disallow:

User-agent: OAI-SearchBot
Disallow:

User-agent: ClaudeBot
Disallow:

User-agent: PerplexityBot
Disallow:

# PAGE PREVIEW & SHARING - ALLOW FOR SOCIAL/LINK SHARING (NO RESTRICTIONS)
User-agent: FacebookExternalHit
Disallow:

User-agent: Google-Image-Proxy
Disallow:

User-agent: PinterestBot
Disallow:

# ADVERTISING BOTS - ALLOW IF YOU RUN ADS (NO RESTRICTIONS)
User-agent: Google-AdsBot
Disallow:

User-agent: Meta-ExternalAds
Disallow:

# SEO ANALYSIS TOOLS - OPTIONAL (ALLOW SEO INSIGHTS) (NO RESTRICTIONS)
User-agent: AhrefsBot
Disallow:

User-agent: SemrushBot
Disallow:

# AI TRAINING DATA COLLECTION - ALLOW / TRAINING WELCOME (NO RESTRICTIONS)
User-agent: GPTBot
Disallow:

User-agent: anthropic-ai
Disallow:

User-agent: Google-Extended
Disallow:

User-agent: GoogleOther
Disallow:

User-agent: Meta-ExternalAgent
Disallow:

User-agent: Amazonbot
Disallow:

User-agent: PetalBot
Disallow:

Sitemap: https://www.redpanda.com/sitemap.xml

Sitemap: https://www.redpanda.com/sitemap.xml