# ========================================
# robots.txt for icbt.lk
# Optimized for SEO + AI Visibility + Crawl Optimization
# Managed by Cloudflare
# ========================================

# --- Global rule ---
User-agent: *
Allow: /

# --- Content-Signals (for AI) ---
# AI can read content for answers but not train on it
Content-Signal: search=yes,ai-input=yes,ai-train=no

# --- Allow Major AI Crawlers ---
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: meta-externalagent
Allow: /

# --- Block low-value or irrelevant bots ---
User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

# --- Crawl Optimization ---
# Prevent indexing of unnecessary system files
Disallow: /cgi-bin/
Disallow: /wp-includes/
Disallow: /readme.html

# --- WordPress ---
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# --- Sitemap ---
Sitemap: https://icbt.lk/sitemaps.xml
Sitemap: https://icbt.lk/wp-content/uploads/icbt-courses-sitemap.xml