User-agent: Nuclei
User-agent: WikiDo
User-agent: Riddler
User-agent: Zoominfobot
User-agent: Go-http-client
User-agent: Node/simplecrawler
User-agent: CazoodleBot
User-agent: dotbot/1.0
User-agent: Gigabot
User-agent: Barkrowler
User-agent: magpie-crawler
User-agent: PetalBot
User-agent: Amazonbot
User-agent: BLEXBot
User-agent: AhrefsBot
User-agent: SERankingBacklinksBot
Disallow: /
User-agent: *
Allow: /
# Explicit signals for AI crawlers (GPTBot, PerplexityBot, ClaudeBot, etc.)
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /resume
Disallow: /auth
Disallow: /carrier/unsubscribe
Disallow: /unsubscribe
Disallow: /magic-link
Sitemap: https://www.lanefinder.com/sitemap.xml
Host: https://www.lanefinder.com