# Rudaw Media Network - robots.txt

User-agent: *
Disallow: /bin/
Disallow: /admin/
Disallow: /SkwidAdministration/
Disallow: /Newsletter/
Disallow: /cache/
Disallow: /ErrorLog/

# AI Crawlers - Allowed
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Sitemaps
Sitemap: https://www.rudaw.net/sitemap.xml