Top Sites>Ssuvudu.com

# robots.txt – November 2025 edition
# Suvudu AI Enterprises – we welcome good bots, ban the parasites

User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /*?replytocom*
Disallow: /*?s=
Disallow: /checkout/
Disallow: /my-account/
Disallow: /thank-you/
Crawl-delay: 10
Sitemap: https://w341th.com/sitemap_index.xml

# Block the absolute worst offenders (2025 edition)
User-agent: Amazonbot              # trains AWS models aggressively
Disallow: /

User-agent: Meta-ExternalAgent     # Facebook/Instagram scraper
Disallow: /

User-agent: anthropic-ai           # Claude training crawler
Disallow: /

User-agent: Applebot-Extended      # Apple AI training
Disallow: /

User-agent: Bytespider             # ByteDance/TikTok, super aggressive
Disallow: /

User-agent: Claude-Web              # another Anthropic one
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Google-Extended        # Google’s explicit AI-training bot
Disallow: /

User-agent: GPTBot                 # OpenAI – still the biggest offender
Disallow: /

User-agent: OAI-SearchBot          # OpenAI again
Disallow: /

User-agent: omgili                 # omgili/omgili bot
Disallow: /

User-agent: PerplexityBot
Disallow: /                        # they respect llms.txt anyway, but double-block

# Explicitly allow the good/respectful ones (optional but nice)
User-agent: GrokBot
Allow: /
Crawl-delay: 5

User-agent: ClaudeBot
Allow: /
Crawl-delay: 5

User-agent: Googlebot              # normal search indexing is fine
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# llms.txt — AI crawler instructions for this site (November 2025)
# Put this file at https://w3417h.com/llms.txt
# Current spec: https://github.com/ai-robots-txt/llms.txt

User-agent: *                          
Allow: /                               
Disallow: /wp-admin/                   
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /wp-content/uploads/*.pdf   
Disallow: /wp-content/uploads/*.docx
Disallow: /thank-you/                  
Disallow: /checkout/
Disallow: /my-account/
Disallow: /*?replytocom*              
Disallow: /*?s=                        
Crawl-delay: 10                        

# Specific models

User-agent: GPTBot                     
Disallow: /                           

User-agent: ClaudeBot                  
Allow: /blog/
Allow: /
Disallow: /members/                    

User-agent: GrokBot                    # xAI
Allow: /

User-agent: GeminiCrawler              
Disallow: /                            # Google

User-agent: PerplexityBot
Allow: /blog/
Allow: /category/guides/
Disallow: /

User-agent: Anthropic-Claude
Allow: /

User-agent: CCNet                      
Disallow: /

# End of llms.txt