# Leadpipe robots.txt
# Full marketing site, blog, and public documentation is open to all crawlers,
# search engines, and AI agents. No rate limiting, no path-level blocks.

User-agent: *
Allow: /

# -----------------------------------------------------------------------------
# Explicit AI crawler allowlist
# -----------------------------------------------------------------------------
# Declared individually so AI operators that honor per-agent rules get a clear
# green light rather than falling through to the wildcard. Covers training
# crawlers, search index crawlers, and on-request retrieval agents.

# OpenAI
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic
User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google (Gemini training + general retrieval separate from Googlebot)
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Apple
User-agent: Applebot-Extended
Allow: /

# ByteDance
User-agent: Bytespider
Allow: /

# Meta
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Common Crawl (feeds many open AI datasets)
User-agent: CCBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# DuckDuckGo Assist
User-agent: DuckAssistBot
Allow: /

# Amazon
User-agent: Amazonbot
Allow: /

# Timpi
User-agent: Timpibot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Diffbot
User-agent: Diffbot
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# xAI
User-agent: xAI-Agent
Allow: /

# -----------------------------------------------------------------------------
# Content usage signals (IETF draft: contentsignals.org)
# -----------------------------------------------------------------------------
# yes across the board: content may be used for model training, search
# indexing, and on-demand AI answers. Leadpipe prefers visibility over
# scarcity; every mention in an AI answer is a branded impression.

Content-Signal: ai-train=yes, search=yes, ai-input=yes

Sitemap: https://www.leadpipe.com/sitemap-index.xml