User-Agent: AdsBot-Google
Allow: /quotes/
Allow: /quote/

User-Agent: AhrefsBot
Crawl-Delay: 5

# AI Crawlers - Allow full access for AEO optimization
User-Agent: OAI-SearchBot
Allow: /

User-Agent: PerplexityBot
Allow: /

User-Agent: GPTBot
Allow: /

User-Agent: ClaudeBot
Allow: /

User-Agent: Google-Extended
Allow: /

User-Agent: CCBot
Allow: /

User-Agent: anthropic-ai
Allow: /

User-Agent: Claude-Web
Allow: /

User-Agent: ChatGPT-User
Allow: /

User-Agent: *
Disallow: /pro/*
Disallow: /pros/tools/*
Disallow: /quotes/*
Disallow: /quote/*
Disallow: /editorials/*
Disallow: /editorial/*
Disallow: /wp-admin/
Disallow: /trackback/
Disallow: /xmlrpc.php
Disallow: /page/
Disallow: /*/feed/
Disallow: /careers*gh_jid
Disallow: /my/*
Disallow: /tag/
Disallow: /projex-articles*
Disallow: /*?tag=*
Sitemap: https://modernize.com/sitemap_index.xml