# robots.txt for https://www.ezrankings.com
# Last updated: 2026-04-21

# ============================================================
# Main rules — applies to all search engine crawlers
# ============================================================
User-agent: *
Allow: /*.js
Allow: /*.css
Allow: /*.jpg
Allow: /*.png
Disallow: /cgi-bin/
Disallow: /lp/
Disallow: /blog/category/
Disallow: /checkout-transaction.html
Disallow: /checkout-transaction-in-inr.html
Disallow: /page/
Disallow: /blog/category/*/page/

# ============================================================
# AI crawlers — explicitly allowed
# Note: These enable AI training and/or AI search visibility.
# Remove any block below to disallow that specific crawler.
# ============================================================
User-agent: Google-Extended
Allow: /

User-agent: GPTBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# ============================================================
# Blocked scrapers and offline downloaders
# ============================================================
User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: Wget
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: NetCaptor
Disallow: /

User-agent: WebGather
Disallow: /

User-agent: Web Downloader
Disallow: /

User-agent: SpiderKU
Disallow: /

User-agent: Steeler
Disallow: /

User-agent: Zeus
Disallow: /

User-agent: Zao
Disallow: /

# ============================================================
# Sitemaps
# ============================================================
Sitemap: https://www.ezrankings.com/sitemap.xml
Sitemap: https://www.ezrankings.com/blog/sitemap_index.xml