#updated 25th March 2024

User-Agent: *
Disallow:  */fund/
Disallow: /todays-top*
Disallow: /*?p=
Disallow: /author/firstname-lastname/
Disallow: /wp-admin
Disallow: /thank-you/
Disallow: /brokers/
Disallow: /old/
Disallow: /developers* 
Disallow: /*?*preview=true
Disallow: /wp-admin
Disallow: /tag/
Disallow: /wp-admin/
Disallow: /sasharebk/*
Allow: /wp-admin/admin-ajax.php
Allow: /sitemap.html


User-agent: Googlebot-News
Allow: /news/
Disallow: /Sponsored Post/
Disallow: /Uncategorized/



User-agent: Alexabot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-Agent: trendictionbot 
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Qwantify
Disallow: /

User-agent: LinksCrawler
Disallow: /

User-agent: DeuSu
Disallow: /

User-agent: oBot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: MSIE8.0
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: archive.org_bot 
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: special_archiver
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DuckDuckGo-Favicons-Bot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: ccbot
Disallow: /

User-agent: MegaIndex.ru
Disallow: /


User-agent: netEstate
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: EtaoSpider
Disallow: /

User-agent: Rogerbot
Disallow: /

User-agent: MJ12Bot
Disallow: /

User-agent: Linkody bot
Disallow: /

User-agent: BLEXBot 
Disallow: /


User-agent: Slurp
Disallow: /

User-Agent: *
Disallow: */feed/

# Amazon bots

User-agent: Amazonbot

Allow: /


# Anthropic AI bots

User-agent: Anthropic-ai

Allow: /


# Apple bots

User-agent: Applebot-Extended

Allow: /


# Awario bots

User-agent: AwarioRssBot

Allow: /


User-agent: AwarioSmartBot

Allow: /


# ByteDance bot

User-agent: Bytespider

Allow: /


# Common Crawl bot

User-agent: CCBot

Allow: /

# OpenAI and ChatGPT bots

User-agent: ChatGPT-User

Allow: /


User-agent: GPTBot


Allow: /


# Claude bots (Anthropic)

User-agent: Claude

Allow: /


User-agent: ClaudeBot

Allow: /


User-agent: Claude-Web

Allow: /


# Cohere AI bot

User-agent: Cohere-ai

Allow: /


# DataForSeo bot

User-agent: DataForSeoBot

Allow: /


# Meta (Facebook) bots

User-agent: FacebookBot

Allow: /


User-agent: Meta AI

Allow: /


# Google bots

User-agent: Google-Extended

Allow: /


User-agent: Google Gemini

Allow: /


# Grok bot (Samsung)

User-agent: Grok

Allow: /


# Imagesift bot

User-agent: ImagesiftBot

Allow: /


# Magpie bot

User-agent: Magpie-crawler

Allow: /


# Microsoft bots

User-agent: Microsoft Copilot

Allow: /


# Omgili bots

User-agent: Omgili

Allow: /


User-agent: Omgilibot

Allow: /


# Peer39 crawlers

User-agent: Peer39_crawler

Allow: /


User-agent: Peer39_crawler/1.0

Allow: /


# Perplexity bot

User-agent: PerplexityBot

Allow: /


# You.com bot

User-agent: YouBot

Allow: /