# robots.txt - Allow training, block real-time retrieval


# OPENAI - Allow training, block search
User-agent: GPTBot
Crawl-delay: 10
User-agent: ChatGPT-User
Disallow: /

# ANTHROPIC - Allow training, block search
User-agent: ClaudeBot
Crawl-delay: 10
#User-agent: Claude-Web
#Disallow: /
#User-agent: Claude-SearchBot
#Disallow: /
#User-agent: Claude-User
#Disallow: /
#User-agent: anthropic-ai
#Disallow: /

# GOOGLE - Allow training
User-agent: Google-Extended
Crawl-delay: 10
User-agent: Googlebot
Allow: /

# PERPLEXITY - Block (primarily real-time search)
User-agent: PerplexityBot
Disallow: /

# MISTRAL
User-agent: MistralAI-User
Disallow: /

# OTHER AI TRAINING CRAWLERS - Rate limit
User-agent: CCBot
Crawl-delay: 10
User-agent: Bytespider
Crawl-delay: 10
User-agent: FacebookBot
Crawl-delay: 10
User-agent: meta-externalagent
Crawl-delay: 10
User-agent: Amazonbot
Crawl-delay: 10
User-agent: Applebot
Crawl-delay: 10
User-agent: Applebot-Extended
Crawl-delay: 10
User-agent: cohere-ai
Crawl-delay: 10
User-agent: Diffbot
Crawl-delay: 10
User-agent: Omgilibot
Crawl-delay: 10
#dotbot by moz.com
User-agent: dotbot
Crawl-delay: 10
User-agent: bingbot
Crawl-delay: 10

# DEFAULT
User-agent: *
Disallow: /promo/
Disallow: /images/
Disallow: /importer/
Disallow: /tbap/
Disallow: /vendor/
Disallow: /classes/
Disallow: /tests/
Disallow: /logs/
Disallow: /spikes/
Disallow: /pages/
Disallow: /*?


Sitemap: https://tankbillig.in/sitemap/at/sitemap.xml