# SecNews.gr Robots.txt - Optimized for News + AI Search
# Last Updated: 2026-04-22

# Main crawlers
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /search/
Disallow: /author/
Disallow: /comments/
Disallow: /trackback/
Disallow: /*?preview=
Disallow: /*?replytocom=
Disallow: /cdn-cgi/
Allow: /wp-content/uploads/
Allow: /wp-json/wp/v2/
Allow: /wp-json/indexnow/
Allow: /wp-json/

# Google bots - full access for news
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

# Google AI - allow for AI Overviews
User-agent: Google-Extended
Allow: /

# Bing - powers Copilot AI too
User-agent: Bingbot
Allow: /

# Social media crawlers
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

# AI Search Bots - ALLOW (drive traffic with source links)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Amazonbot
Allow: /

# Parasitic crawlers - BLOCK
User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

# SEO Tools - Block
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Block
User-agent: YandexBot
Disallow: /

User-agent: YandexImages
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: SiteBot
Disallow: /

User-agent: EmailCollector
Disallow: /

# Sitemaps
Sitemap: https://www.secnews.gr/sitemap_index.xml
Sitemap: https://www.secnews.gr/news-sitemap.xml