# robots.txt for https://cryptojobslist.com
# CryptoJobsList.com — specialized crypto & web3 job board

User-agent: *
Allow: /

# High-value crawlable content
Allow: /jobs/
Allow: /companies/
Allow: /salary/
Allow: /talent/

# Block private / low-value / internal areas
Disallow: /api/
Allow: /api/og/
Disallow: /*/apply$
Disallow: /*/apply/
Disallow: /dashboard
Disallow: /admin
Disallow: /monitoring
Disallow: /undefined
Disallow: /edit
Disallow: /unpublish
Disallow: /settings/
Disallow: /login
Disallow: /hire
Disallow: /submit
Disallow: /review
Disallow: /newsletter
Disallow: /recommend/
Disallow: /payment-pending

# Tracking / referral params
Disallow: /*?ref=*
Disallow: /*?utm_*

# Sitemaps
Sitemap: https://cryptojobslist.com/sitemap.xml
Sitemap: https://cryptojobslist.com/sitemap-jobs.xml
Sitemap: https://cryptojobslist.com/sitemap-categories.xml
Sitemap: https://cryptojobslist.com/sitemap-companies.xml
Sitemap: https://cryptojobslist.com/sitemap-blog-posts.xml

# --- AI search crawlers (indexing allowed, training discouraged) ---
User-agent: OAI-SearchBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

# --- Social crawlers ---
User-agent: Twitterbot
Allow: /

# --- SEO scrapers blocked ---
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
Disallow: /

Crawl-delay: 1