Sitemap: https://www.exchange-rates.org/sitemap_index.xml


# Allow AI search and agent use
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: ChatGPT-User  
User-agent: PerplexityBot
User-agent: FirecrawlAgent
User-agent: AndiBot
User-agent: ExaBot
User-agent: PhindBot
User-agent: YouBot
Crawl-delay: 20
Allow: /

# Disallow AI training data collection  
User-agent: GPTBot  
User-agent: CCBot  
User-agent: Google-Extended  
Disallow: /

# Allow traditional search indexing  
User-agent: Googlebot  
User-agent: Bingbot  
Allow: /

User-agent: *
Disallow: /?layout=v2
Disallow: /?layout=1
Disallow: /GetCustomContent.aspx*
Disallow: /HistoryExchangeRatesReportDownload.aspx*
Disallow: /*.aspx*

User-agent: Amazonbot
Disallow: /

User-agent: SemrushBot-SA
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: Dotbot
User-agent: MJ12bot
User-agent: MauiBot
User-agent: MauiBot (crawler.feedback+wc@gmail.com) 
User-agent: MegaIndex.ru/2.0
User-agent: Seekport Crawler
User-agent: SputnikBot/2.3
User-agent: The Knowledge AI 
User-agent: TurnitinBot
User-agent: UCBrowser
User-agent: yacybot
User-agent: Yeti
User-agent: SeznamBot
User-agent: coccocbot-web
User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Baiduspider
Disallow: /
Allow: /zh
Allow: /zh-hant
Disallow: /zh/*.aspx$
Disallow: /zh/api*
Disallow: /zh-hant/*.aspx$
Disallow: /zh-hant/api*
Disallow: /*.aspx*

User-agent: Baiduspider-image
Disallow: /
Allow: /zh
Allow: /zh-hant
Disallow: /zh/*.aspx$
Disallow: /zh-hant/*.aspx$
Disallow: /*.aspx*

User-agent: YoudaoBot
Disallow: /
Allow: /zh
Allow: /zh-hant
Disallow: /zh/*.aspx$
Disallow: /zh-hant/*.aspx$
Disallow: /*.aspx*

User-agent: Sogou web spider
Crawl-Delay: 10
Disallow: /
Allow: /zh
Allow: /zh-hant
Disallow: /zh/*.aspx$
Disallow: /zh-hant/*.aspx$
Disallow: /*.aspx*

User-agent: Sogou inst spider
Crawl-Delay: 10
Disallow: /
Allow: /zh
Allow: /zh-hant
Disallow: /zh/*.aspx$
Disallow: /zh-hant/*.aspx$
Disallow: /*.aspx*

User-agent: Sogou spider2
Crawl-Delay: 10
Disallow: /
Allow: /zh
Allow: /zh-hant
Disallow: /zh/*.aspx$
Disallow: /zh-hant/*.aspx$
Disallow: /*.aspx*
User-agent: Sogou blog
Disallow: /

User-agent: Sogou News Spider
Crawl-Delay: 10
Disallow: /


User-agent: Sogou Orion spider
Crawl-Delay: 10
Disallow: /
Allow: /zh
Allow: /zh-hant
Disallow: /zh/*.aspx$
Disallow: /zh-hant/*.aspx$
Disallow: /*.aspx*

User-agent: ChinasoSpider
Disallow: /

User-agent: Sosospider
Disallow: /

User-agent: yisouspider
Disallow: /


User-agent: SeekportBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /


User-agent: grapeshot
Crawl-delay: 20

User-agent: ICC-Crawler
Crawl-delay: 20