# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file

# ===========================================
# AI Crawlers - Explicitly Allowed
# ===========================================

# OpenAI (ChatGPT)
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /_next/
Disallow: /api/v

# OpenAI (SearchGPT)
User-agent: OAI-SearchBot
Allow: /

# Google (Gemini AI)
User-agent: Google-Extended
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /

# Anthropic (Claude User)
User-agent: anthropic-ai
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Apple (Siri/Spotlight AI)
User-agent: Applebot-Extended
Allow: /

# Common Crawl (used by many AI training sets)
User-agent: CCBot
Allow: /

# Microsoft (Copilot/Bing AI)
User-agent: Bingbot
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# ===========================================
# Social Media Crawlers - Full Access
# ===========================================

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

# ===========================================
# All Other Crawlers
# ===========================================

User-agent: *
Allow: /

# Disallow crawling of admin or private areas
Disallow: /admin/
Disallow: /private/
Disallow: /_next/
Disallow: /api/v

# Block known bad bots
User-agent: Bytespider
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

# Crawl-delay for polite crawling
Crawl-delay: 1

# Sitemaps
Sitemap: https://www.justdeliverit.net/sitemap.xml

# Host directive
Host: https://www.justdeliverit.net

# LLM-readable content
# See https://www.justdeliverit.net/llms.txt
# See https://www.justdeliverit.net/llms-full.txt