# https://www.robotstxt.org/robotstxt.html
# LeaseEnd.com - Optimized for traditional crawlers and AI/LLM systems
# Last updated: 2026-01-19

# ============================================
# Traditional Search Engine Crawlers
# ============================================

# Google
User-agent: Googlebot
Allow: /
Disallow: /flow/
Disallow: /cj
Disallow: /cold-email
Disallow: /credit-union-lease-end-loan-rates
Disallow: /cv
Disallow: /lt
Disallow: /top-ten-car-loans
Crawl-delay: 1

# Bing
User-agent: Bingbot
Allow: /
Disallow: /flow/
Disallow: /cj
Disallow: /cold-email
Disallow: /credit-union-lease-end-loan-rates
Disallow: /cv
Disallow: /lt
Disallow: /top-ten-car-loans
Crawl-delay: 1

# ============================================
# AI / LLM Crawlers (Conversational Search)
# ============================================

# OpenAI ChatGPT
User-agent: GPTBot
Allow: /
Allow: /llm-data/
Allow: /llms.txt
Allow: /learn/
Allow: /faqs
Allow: /about-us
Disallow: /flow/
Disallow: /cj
Disallow: /cold-email
Disallow: /credit-union-lease-end-loan-rates
Disallow: /cv
Disallow: /lt
Disallow: /top-ten-car-loans
Crawl-delay: 5

User-agent: ChatGPT-User
Allow: /
Allow: /llm-data/
Allow: /llms.txt
Allow: /learn/
Allow: /faqs
Allow: /about-us
Disallow: /flow/
Disallow: /cj
Disallow: /cold-email
Disallow: /credit-union-lease-end-loan-rates
Disallow: /cv
Disallow: /lt
Disallow: /top-ten-car-loans
Crawl-delay: 5

# Anthropic Claude
User-agent: Claude-Web
Allow: /
Allow: /llm-data/
Allow: /llms.txt
Allow: /learn/
Allow: /faqs
Allow: /about-us
Disallow: /flow/
Disallow: /cj
Disallow: /cold-email
Disallow: /credit-union-lease-end-loan-rates
Disallow: /cv
Disallow: /lt
Disallow: /top-ten-car-loans
Crawl-delay: 5

User-agent: anthropic-ai
Allow: /
Allow: /llm-data/
Allow: /llms.txt
Allow: /learn/
Allow: /faqs
Allow: /about-us
Disallow: /flow/
Disallow: /cj
Disallow: /cold-email
Disallow: /credit-union-lease-end-loan-rates
Disallow: /cv
Disallow: /lt
Disallow: /top-ten-car-loans
Crawl-delay: 5

# Perplexity
User-agent: PerplexityBot
Allow: /
Allow: /llm-data/
Allow: /llms.txt
Allow: /learn/
Allow: /faqs
Allow: /about-us
Disallow: /flow/
Disallow: /cj
Disallow: /cold-email
Disallow: /credit-union-lease-end-loan-rates
Disallow: /cv
Disallow: /lt
Disallow: /top-ten-car-loans
Crawl-delay: 5

# Google Extended (Gemini AI training)
User-agent: Google-Extended
Allow: /
Allow: /llm-data/
Allow: /llms.txt
Allow: /learn/
Allow: /faqs
Allow: /about-us
Disallow: /flow/
Disallow: /cj
Disallow: /cold-email
Disallow: /credit-union-lease-end-loan-rates
Disallow: /cv
Disallow: /lt
Disallow: /top-ten-car-loans
Crawl-delay: 3

# Meta AI
User-agent: FacebookBot
Allow: /
Allow: /llm-data/
Allow: /llms.txt
Allow: /learn/
Allow: /faqs
Allow: /about-us
Disallow: /flow/
Disallow: /cj
Disallow: /cold-email
Disallow: /credit-union-lease-end-loan-rates
Disallow: /cv
Disallow: /lt
Disallow: /top-ten-car-loans
Crawl-delay: 5

# Cohere
User-agent: cohere-ai
Allow: /
Allow: /llm-data/
Allow: /llms.txt
Allow: /learn/
Allow: /faqs
Allow: /about-us
Disallow: /flow/
Disallow: /cj
Disallow: /cold-email
Disallow: /credit-union-lease-end-loan-rates
Disallow: /cv
Disallow: /lt
Disallow: /top-ten-car-loans
Crawl-delay: 5

# ============================================
# AI Training Data Crawlers (Optional)
# ============================================
# These crawlers collect data for AI model training
# Uncomment to BLOCK if you don't want content used for training

# Common Crawl
# User-agent: CCBot
# Disallow: /

# Omgili (used by various AI companies)
# User-agent: omgili
# Disallow: /

# ============================================
# All Other Crawlers (Fallback)
# ============================================

User-agent: *
Allow: /
Disallow: /flow/
Disallow: /cj
Disallow: /cold-email
Disallow: /credit-union-lease-end-loan-rates
Disallow: /cv
Disallow: /lt
Disallow: /top-ten-car-loans/

# ============================================
# Sitemaps
# ============================================

Sitemap: https://www.leaseend.com/sitemap.xml
Sitemap: https://www.leaseend.com/llm-data/sitemap.xml

# ============================================
# Rate Limiting (Server-side enforcement)
# ============================================
# Note: Crawl-delay is a suggestion, not enforced by all bots
# Consider implementing rate limiting at the server/CDN level for:
# - /llm-data/: Max 100 requests/hour per IP
# - /learn/: Standard rate limits
# - /faqs: Standard rate limits