# robots.txt – November 2025 edition
# Suvudu AI Enterprises – we welcome good bots, ban the parasites

User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /*?replytocom*
Disallow: /*?s=
Disallow: /checkout/
Disallow: /my-account/
Disallow: /thank-you/
Crawl-delay: 10
Sitemap: https://w341th.com/sitemap_index.xml

# Block the absolute worst offenders (2025 edition)
User-agent: Amazonbot              # trains AWS models aggressively
Disallow: /

User-agent: Meta-ExternalAgent     # Facebook/Instagram scraper
Disallow: /

User-agent: anthropic-ai           # Claude training crawler
Disallow: /

User-agent: Applebot-Extended      # Apple AI training
Disallow: /

User-agent: Bytespider             # ByteDance/TikTok, super aggressive
Disallow: /

User-agent: Claude-Web              # another Anthropic one
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Google-Extended        # Google’s explicit AI-training bot
Disallow: /

User-agent: GPTBot                 # OpenAI – still the biggest offender
Disallow: /

User-agent: OAI-SearchBot          # OpenAI again
Disallow: /

User-agent: omgili                 # omgili/omgili bot
Disallow: /

User-agent: PerplexityBot
Disallow: /                        # they respect llms.txt anyway, but double-block

# Explicitly allow the good/respectful ones (optional but nice)
User-agent: GrokBot
Allow: /
Crawl-delay: 5

User-agent: ClaudeBot
Allow: /
Crawl-delay: 5

User-agent: Googlebot              # normal search indexing is fine
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /