# Astro - Premium Proxy Provider
# https://astroproxy.com

User-agent: *
Allow: /
Allow: /en/blog
Allow: /ru/blog

# Block admin and internal pages
Disallow: /admin/
Disallow: /api/
Disallow: /lk/
Disallow: /dashboard/
Disallow: /storage/

# Block old/duplicate URL patterns
Disallow: /confidentiality/
Disallow: /license/agreement
Disallow: /signin
Disallow: /signup
Disallow: /logout
Disallow: /setLocale/
Disallow: /use-v2
Disallow: /verify-email/
Disallow: /password/reset/
Disallow: /callback.php
Disallow: /redistest
Disallow: /free/

# Block index.php duplicates
Disallow: /index.php/
Disallow: /index.php

# Block technical pages
Disallow: /*/account-inactive
Disallow: /*/cookies-refusal
Disallow: /*/sign-up
Disallow: /*/sign-in
Disallow: /*/forgot-password

# Block unsupported languages (only en, ru, ua, am, cn supported)
Disallow: /es/
Disallow: /tr/
Disallow: /de/
Disallow: /fr/
Disallow: /uk/
Disallow: /fa/
Disallow: /it/
Disallow: /pt/
Disallow: /ja/
Disallow: /ko/

# Block blog without language prefix (middleware redirects to /en/blog)
Disallow: /blog$
Disallow: /blog/

# Block URL parameters that cause duplicates
Disallow: /*?*utm_
Disallow: /*?*fbclid
Disallow: /*?*gclid
Disallow: /*?*_ga
Disallow: /*?*mc_
Disallow: /*?*sort=
Disallow: /*?*order=
Disallow: /*?*filter=

# Crawl-delay for polite crawling (Googlebot ignores this, Bing respects it)
Crawl-delay: 1

# Sitemaps
Sitemap: https://astroproxy.com/sitemap.xml

# AI/LLM Crawlers - Welcome!
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Disallow:

User-agent: ChatGPT-User
Allow: /
Allow: /llms.txt
Disallow:

User-agent: Claude-Web
Allow: /
Allow: /llms.txt
Disallow:

User-agent: Anthropic-AI
Allow: /
Allow: /llms.txt
Disallow:

User-agent: Google-Extended
Allow: /
Disallow:

User-agent: Applebot-Extended
Allow: /
Disallow:

User-agent: CCBot
Allow: /
Disallow:

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Disallow:

User-agent: Cohere-AI
Allow: /
Allow: /llms.txt
Disallow:

# IndexNow
# API Key file: /8ec6eb76d58c28b9de78c6ce8ce78333.txt