# --------------------------------------------------
# CLOUDFLARE CONTENT SIGNALS (AI PROTECTIONS)
# --------------------------------------------------
User-agent: *
# Content-Signal: search=yes,ai-train=no
Allow: /

# --------------------------------------------------
# BLOCK AGGRESSIVE AI SCRAPERS
# --------------------------------------------------
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: GPTBot
User-agent: meta-externalagent
Disallow: /

# --------------------------------------------------
# GENERAL SEARCH ENGINE RULES
# --------------------------------------------------
User-agent: *

# Allow all content by default
Allow: /

# Disallow sensitive or low-value areas
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /login/
Disallow: /search/
Disallow: /wp-admin/
Disallow: /api/

# Ensure bots can see styling and scripts
Allow: /wp-content/themes/*.css
Allow: /wp-content/themes/*.js
Allow: /assets/*.css
Allow: /assets/*.js

# --------------------------------------------------
# SITEMAP LOCATION
# --------------------------------------------------
Sitemap: https://thesun.my/sitemap_index.xml