User-agent: *
Allow: /
Crawl-Delay: 15

sitemap: https://www.contractscounsel.com/sitemap_index.xml

User-agent: SeekportBot
Crawl-delay: 10

User-agent: bingbot
Crawl-Delay: 10

# https://ahrefs.com/robot/
User-agent: AhrefsBot
Crawl-Delay: 20

# https://dataforseo.com/dataforseo-bot
User-agent: DataForSeoBot
Crawl-delay: 20

# http://webmeup-crawler.com/
User-agent: BLEXBot
Crawl-delay: 20

# https://www.semrush.com/bot/ - maximum value is 10
User-agent: SemrushBot
Crawl-delay: 10

User-agent: SiteAuditBot
Crawl-delay: 5

# https://mj12bot.com/
User-agent: MJ12bot
Crawl-delay: 20

# https://imagesift.com/
User-agent: ImagesiftBot
Crawl-delay: 20

# OpenAI
User-agent: GPTBot
Crawl-delay: 10

# ClaudeBot - many IP addresses and highly aggressive
User-agent: ClaudeBot
Crawl-delay: 10

# https://relemind.com/impressum/ - no indication if Crawl-delay is respected
User-agent: relemindbot
Crawl-delay: 20

# Amazon's user agent - does not respect Crawl-Delay, and heavy use 12/10/2023
# https://developer.amazon.com/support/amazonbot
User-agent: Amazonbot
Crawl-delay: 20


User-agent: dotbot # +https://opensiteexplorer.org/dotbot; help@moz.com
Crawl-delay: 10

User-agent: Applebot
Crawl-delay: 20