# iblnews.org - Robots Configuration

User-agent: *
Allow: /

# Allow all content pages
Allow: /story
Allow: /story/*
Allow: /category
Allow: /category/*
Allow: /blog
Allow: /top-news
Allow: /search
Allow: /aisearch
Allow: /summary
Allow: /followup/*
Allow: /video/*

# Allow static pages
Allow: /about
Allow: /advertise
Allow: /terms
Allow: /events
Allow: /platforms
Allow: /views

# Disallow admin and API routes
Disallow: /api/
Disallow: /login
Disallow: /_next/
Disallow: /admin/

# Disallow search query parameters to avoid duplicate content
Disallow: /*?*query=
Disallow: /*?*search=

# Crawl-delay (optional - adjust as needed)
# Crawl-delay: 1

# Sitemap location
Sitemap: https://iblnews.org/sitemap.xml

# Specific bot rules

# Allow OpenAI GPTBot
User-agent: GPTBot
Allow: /
Crawl-delay: 2

# Allow Google AI (Bard/Gemini)
User-agent: Google-Extended
Allow: /

# Allow Anthropic Claude
User-agent: anthropic-ai
Allow: /

# Allow Common Crawl
User-agent: CCBot
Allow: /