# CamDiv Robots.txt - SEO Optimized for Global Reach
# Free Anonymous Video Chat with Strangers Worldwide | Best Omegle Alternative

# -----------------------------------------------------------------------------
# GROUP 1: Global Rules (Applies to all bots unless specified otherwise below)
# -----------------------------------------------------------------------------
User-agent: *
Allow: /
# Disallow system and dynamic paths
Disallow: /api/
Disallow: /node_modules/
Disallow: /.well-known/
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /socket.io/
Disallow: /*.json$
Disallow: /error
Disallow: /404
Disallow: /500
# Allow critical SEO files
Allow: /sitemap.xml
Allow: /manifest.json
Allow: /robots.txt
Allow: /brand.ico
Allow: /favicon.svg
Allow: /apple-touch-icon*.png
Allow: /og-image.jpg
Allow: /browserconfig.xml
Crawl-delay: 1

# -----------------------------------------------------------------------------
# GROUP 1a: AI & LLM Bots (Explicitly Welcomed for GEO)
# -----------------------------------------------------------------------------
# OpenAI (ChatGPT)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot

# Google (Gemini/Bard)
User-agent: Google-Extended

# Anthropic (Claude)
User-agent: ClaudeBot
User-agent: anthropic-ai

# Perplexity
User-agent: PerplexityBot

# Common Crawl (Training Data for many LLMs)
User-agent: CCBot
User-agent: CCBot/2.0

# Meta (Llama/Meta AI)
User-agent: FacebookBot
User-agent: Meta-ExternalAgent

# Apple (Apple Intelligence)
User-agent: Applebot
User-agent: Applebot-Extended

# Amazon (Titan/Q)
User-agent: Amazonbot

# ByteDance (Doubao/TikTok)
User-agent: Bytespider

# Other AI Scrapers
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: cohere-ai
User-agent: omug
User-agent: webz.io
# Prevent crawling of system/private paths to save budget
Disallow: /api/
Disallow: /node_modules/
Disallow: /.well-known/
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /socket.io/
Disallow: /*.json$
Disallow: /error
Disallow: /404
Disallow: /500
Allow: /sitemap.xml
Allow: /manifest.json
Allow: /
Crawl-delay: 1

# -----------------------------------------------------------------------------
# GROUP 2: Googlebot (Specific optimization)
# -----------------------------------------------------------------------------
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
Allow: /
# Disallows must be repeated because specific UA ignores '*'
Disallow: /api/
Disallow: /node_modules/
Disallow: /.well-known/
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /socket.io/
Disallow: /*.json$
Disallow: /error
Disallow: /404
Disallow: /500
Allow: /sitemap.xml
Allow: /manifest.json
Crawl-delay: 0

# -----------------------------------------------------------------------------
# GROUP 3: Bing (Specific optimization)
# -----------------------------------------------------------------------------
User-agent: Bingbot
Allow: /
# Disallows must be repeated
Disallow: /api/
Disallow: /node_modules/
Disallow: /.well-known/
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /socket.io/
Disallow: /*.json$
Disallow: /error
Disallow: /404
Disallow: /500
Allow: /sitemap.xml
Allow: /manifest.json
Crawl-delay: 1

# -----------------------------------------------------------------------------
# GROUP 4: Specific Bots (Advertising/Social/Tools)
# -----------------------------------------------------------------------------
# Most of these are fine falling back to '*' but if you want to be explicit:
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: facebookexternalhit
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: WhatsApp
User-agent: TelegramBot
User-agent: DiscordBot
User-agent: SlackBot
# They will inherit rules from here down if treated as a group, 
# BUT widely supported standard allows grouping User-agent lines.
Allow: /
# Shared Disallows
Disallow: /api/
Disallow: /node_modules/
Disallow: /.well-known/
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /socket.io/
Disallow: /*.json$
Disallow: /error
Disallow: /404
Disallow: /500
Allow: /sitemap.xml
Allow: /manifest.json


# -----------------------------------------------------------------------------
# Sitemap
# -----------------------------------------------------------------------------
Sitemap: https://camdiv.com/sitemap.xml