# robots.txt — purrweb.com
# Strategy: allow search + AI answers, deny AI training
# Updated: 2026-04-24

User-agent: *
Disallow: /privacy-policy/
Disallow: /mvp-calculator/result/
Disallow: /mvp-calculator/result/ru/
Disallow: /services/ui-ux-clutch/
Disallow: /clutch/
Disallow: /ru/clutch/
Disallow: /ru/uslugi/ui-ux-clutch/
Disallow: /blog/*?*preview*
Disallow: /cdn-cgi/
Disallow: /*/feed/

User-agent: Yandex
Disallow: /privacy-policy/
Disallow: /mvp-calculator/result/
Disallow: /mvp-calculator/result/ru/
Disallow: /services/ui-ux-clutch/
Disallow: /clutch/
Disallow: /ru/clutch/
Disallow: /ru/uslugi/ui-ux-clutch/
Disallow: /blog/*?*preview*
Disallow: /cdn-cgi/
Disallow: /*/feed/
Clean-param: utm_source&utm_medium&utm_campaign&utm_content&utm_term&gclid&yclid&fbclid&preview /

# === AI training crawlers: DENY ===
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# === AI answer / search crawlers: ALLOW (not listed = allowed via * block) ===
# OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User,
# PerplexityBot, Perplexity-User, Applebot, DuckAssistBot

Sitemap: https://www.purrweb.com/sitemap.xml