# ==========================================================
# KATALON ROBOTS.TXT (2026 Update)
# ==========================================================

# SECTION 1: ALLOW AI SEARCH & AGENTS (Traffic Drivers)
# These bots browse the web in real-time to answer user queries.
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: FirecrawlAgent
User-agent: AndiBot
User-agent: ExaBot
User-agent: PhindBot
User-agent: YouBot
User-agent: Gemini-Deep-Research
Allow: /
Allow: /llms.txt

# SECTION 2: DISALLOW AI TRAINING (Data Scraping)
# These bots ingest your content to train foundation models.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: CCBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
Disallow: /

# SECTION 3: TRADITIONAL SEARCH ENGINES
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
Allow: /

# SECTION 4: GLOBAL RULES
User-agent: *
Disallow: /admin/
Disallow: /login/
Disallow: /private/
Allow: /

Sitemap: https://katalon.com/sitemap.xml