# Global robots.txt for uni.xyz
# This file should be served at https://uni.xyz/robots.txt by nginx
# Last updated: 2025-12-15

User-agent: *
Allow: /

# Public content sections - fully crawlable
Allow: /projects/
Allow: /journals/
Allow: /competitions/
Allow: /market/
Allow: /organizations/
Allow: /publications/
Allow: /inspirations/
Allow: /topic/
Allow: /about/
Allow: /profile/

# Disallow authentication, admin, and private routes
Disallow: /api/
Disallow: /admin/
Disallow: /accounts/
Disallow: /builder/
Disallow: /payment/
Disallow: /staff/
Disallow: /unify/

# Allow specific public API endpoints for rich results
Allow: /api/v2/neo_projects/
Allow: /api/v2/blogs/
Allow: /api/v2/bb_competitions/
Allow: /api/v2/market/products/
Allow: /api/v2/organizations/

# Prevent duplicate content from tracking parameters only
# Note: Pagination params (?page=) are now allowed since Round 21 added proper canonical handling
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: /*?source=
Disallow: /*?fbclid=
Disallow: /*?gclid=

# Google bots - prioritize crawling
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: AdsBot-Google
Allow: /

# Bingbot
User-agent: Bingbot
Allow: /
Crawl-delay: 0

# Baidu (China's primary search engine)
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# Yandex (Russia's primary search engine)
User-agent: Yandex
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /
Crawl-delay: 2

# Naver Yeti (Korea)
User-agent: Yeti
Allow: /
Crawl-delay: 2

# Sogou (China, secondary)
User-agent: Sogou
Allow: /
Crawl-delay: 2

# Social media crawlers - allow full access
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

# AI Crawlers - rate limited, block deep pagination
User-agent: GPTBot
Allow: /
Crawl-delay: 2
Disallow: /*?page=1*
Disallow: /*?page=2*
Disallow: /*?page=3*
Disallow: /*?page=4*
Disallow: /*?page=5*
Disallow: /*?page=6*
Disallow: /*?page=7*
Disallow: /*?page=8*
Disallow: /*?page=9*
Allow: /*?page=1
Allow: /*?page=2
Allow: /*?page=3
Allow: /*?page=4
Allow: /*?page=5
Allow: /*?page=6
Allow: /*?page=7
Allow: /*?page=8
Allow: /*?page=9

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 2
Disallow: /*?page=1*
Disallow: /*?page=2*
Disallow: /*?page=3*
Disallow: /*?page=4*
Disallow: /*?page=5*
Disallow: /*?page=6*
Disallow: /*?page=7*
Disallow: /*?page=8*
Disallow: /*?page=9*
Allow: /*?page=1
Allow: /*?page=2
Allow: /*?page=3
Allow: /*?page=4
Allow: /*?page=5
Allow: /*?page=6
Allow: /*?page=7
Allow: /*?page=8
Allow: /*?page=9

User-agent: ClaudeBot
Allow: /
Crawl-delay: 2

User-agent: PerplexityBot
Allow: /
Crawl-delay: 2

# Google AI Overviews / Gemini training
User-agent: Google-Extended
Allow: /
Crawl-delay: 2

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /
Crawl-delay: 2

# Amazon Alexa / AI
User-agent: Amazonbot
Allow: /
Crawl-delay: 5

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 2

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Allow: /
Crawl-delay: 5

# Block problematic SEO bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

# Sitemap location (served by Django backend)
Sitemap: https://uni.xyz/sitemap.xml