# BasedLabs AI Robots.txt
# Last updated: 2026-04-24

# Default rule for all bots
# Content-Signal: AI content usage preferences (contentsignals.org)
User-agent: *
Content-Signal: ai-train=yes, search=yes, ai-input=yes
Allow: /
Allow: /explore/
Allow: /images/
Allow: /videos/
Allow: /models
Allow: /articles
Allow: /threads
Allow: /tools
Allow: /tools/
Allow: /tools/*
Allow: /tools/*/*
Allow: /apps
Allow: /memes/

# Disallow access to private/internal pages
Disallow: /api/
Disallow: /dashboard/
Disallow: /signin
Disallow: /auth/
Disallow: /test/
Disallow: /admin/
Disallow: /_next/
Disallow: /static/

# Block infinite URL spaces (crawl budget optimization)
Disallow: /generate
Disallow: /generate?
Disallow: /view/
Disallow: /collections/
Disallow: /templates/
Disallow: /feed/
Disallow: /c/
Disallow: /profile/

# Allow access to API documentation
Allow: /ai-api
Allow: /ai-plugins

# Specific rules for major search engines
User-agent: Googlebot
Allow: /
Crawl-delay: 2

User-agent: Bingbot
Allow: /
Crawl-delay: 2

User-agent: Slurp
Allow: /
Crawl-delay: 5

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 5

# Facebook crawler - block explore to reduce load
User-agent: facebookexternalhit
Disallow: /explore/
Crawl-delay: 10

User-agent: meta-externalagent
Disallow: /explore/
Crawl-delay: 10

# AI/ML crawlers - allow high-value pages, block infinite/dynamic routes
# This gets us into LLM training data while avoiding ISR cost explosion
# AI crawlers - allow cached/static pages for AI citations, block infinite dynamic routes
User-agent: GPTBot
Allow: /$
Allow: /tools
Allow: /tools/
Allow: /apps
Allow: /apps/
Allow: /articles
Allow: /articles/
Allow: /features
Allow: /features/
Allow: /colors
Allow: /colors/
Allow: /memes
Allow: /memes/
Allow: /ai-api
Allow: /models
Disallow: /explore/
Disallow: /c/
Disallow: /view/
Disallow: /generate/
Disallow: /templates/
Disallow: /collections/
Disallow: /feed/
Disallow: /profile/
Disallow: /api/
Crawl-delay: 5

User-agent: ClaudeBot
Allow: /$
Allow: /tools
Allow: /tools/
Allow: /apps
Allow: /apps/
Allow: /articles
Allow: /articles/
Allow: /features
Allow: /features/
Allow: /colors
Allow: /colors/
Allow: /memes
Allow: /memes/
Allow: /ai-api
Allow: /models
Disallow: /explore/
Disallow: /c/
Disallow: /view/
Disallow: /generate/
Disallow: /templates/
Disallow: /collections/
Disallow: /feed/
Disallow: /profile/
Disallow: /api/
Crawl-delay: 5

User-agent: ChatGPT-User
Allow: /$
Allow: /tools
Allow: /tools/
Allow: /apps
Allow: /apps/
Allow: /articles
Allow: /articles/
Allow: /colors
Allow: /colors/
Allow: /memes
Allow: /memes/
Disallow: /explore/
Disallow: /c/
Disallow: /profile/
Disallow: /api/
Crawl-delay: 5

User-agent: anthropic-ai
Allow: /$
Allow: /tools
Allow: /tools/
Allow: /apps
Allow: /apps/
Allow: /articles
Allow: /articles/
Allow: /colors
Allow: /colors/
Allow: /memes
Allow: /memes/
Disallow: /explore/
Disallow: /c/
Disallow: /profile/
Disallow: /api/
Crawl-delay: 5

User-agent: Claude-SearchBot
Allow: /$
Allow: /tools
Allow: /tools/
Allow: /apps
Allow: /apps/
Allow: /articles
Allow: /articles/
Disallow: /explore/
Disallow: /c/
Disallow: /profile/
Disallow: /generate/
Disallow: /api/
Crawl-delay: 10

User-agent: PerplexityBot
Allow: /$
Allow: /tools
Allow: /tools/
Allow: /apps
Allow: /apps/
Allow: /articles
Allow: /articles/
Allow: /features
Allow: /features/
Allow: /colors
Allow: /colors/
Allow: /memes
Allow: /memes/
Disallow: /explore/
Disallow: /c/
Disallow: /profile/
Disallow: /api/
Crawl-delay: 5

# OAI-SearchBot - ChatGPT real-time browsing agent (separate from GPTBot training)
User-agent: OAI-SearchBot
Allow: /$
Allow: /tools
Allow: /tools/
Allow: /apps
Allow: /apps/
Allow: /articles
Allow: /articles/
Allow: /features
Allow: /features/
Disallow: /explore/
Disallow: /c/
Disallow: /profile/
Disallow: /generate/
Disallow: /api/
Crawl-delay: 5

# Google-Extended - Gemini training crawler (separate from Googlebot search)
User-agent: Google-Extended
Allow: /tools
Allow: /tools/
Allow: /apps
Allow: /apps/
Allow: /articles
Allow: /articles/
Allow: /features
Allow: /features/
Disallow: /explore/
Disallow: /c/
Disallow: /profile/
Disallow: /generate/
Disallow: /api/
Crawl-delay: 5

# Applebot - allow valuable pages but block infinite explore crawl space
User-agent: Applebot
Allow: /tools
Allow: /tools/
Allow: /apps
Allow: /apps/
Allow: /articles
Allow: /articles/
Allow: /memes
Allow: /memes/
Allow: /models
Disallow: /explore/
Disallow: /c/
Disallow: /view/
Disallow: /generate/
Disallow: /profile/
Disallow: /api/
Crawl-delay: 5

# Block aggressive/low-value AI crawlers entirely
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Image crawlers
User-agent: Googlebot-Image
Allow: /
Allow: /explore/

User-agent: bingbot-image
Allow: /
Allow: /explore/

# SEO tools - allow for analysis
User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: SemrushBot
Allow: /
Crawl-delay: 5

# Crawl-delay for respectful crawling (general bots)
Crawl-delay: 1

# Sitemaps
Sitemap: https://www.basedlabs.ai/sitemap.xml
Sitemap: https://www.basedlabs.ai/sitemaps/static.xml
Sitemap: https://www.basedlabs.ai/sitemaps/explore.xml
Sitemap: https://www.basedlabs.ai/sitemaps/pages.xml
Sitemap: https://www.basedlabs.ai/sitemaps/articles.xml
Sitemap: https://www.basedlabs.ai/sitemaps/memes.xml