# LinkTwin robots.txt

# AI crawlers — explicitly allowed on all public content
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: Google-Extended
User-agent: Gemini-Deep-Research
User-agent: Google-CloudVertexBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: meta-externalagent
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: CCBot
Allow: /
Disallow: /user/
Disallow: /admin/
Disallow: /login
Disallow: /register
Disallow: /forgot
Disallow: /api/
Disallow: /cron/
Disallow: /server/

User-agent: *
Allow: /

# Block authenticated/private areas
Disallow: /user/
Disallow: /admin/
Disallow: /login
Disallow: /register
Disallow: /forgot

# Block API endpoints (not for crawling)
Disallow: /api/

# Block utility/system paths
Disallow: /cron/
Disallow: /server/

# Allow Facebook crawler full access
User-agent: facebookexternalhit
Allow: /

# Sitemap
Sitemap: https://linktw.in/sitemap.xml