# ============================================
# Global Banking & Finance Review
# robots.txt - Optimized for SEO, GEO & AEO
# Last updated: 2026-03-03
# ============================================
# Per RFC 9309: the default User-agent: * group applies to all
# crawlers not listed below. Named agents override the default
# only when they need different Allow/Disallow rules.
# ============================================

User-agent: *

# ============================================
# Allow Rules
# ============================================

# Core content
Allow: /
Allow: /*.xml$

# Static assets
Allow: /_next/static/
Allow: /_next/image
Allow: /_next/data/
Allow: /fonts/
Allow: /images/

# Sitemaps (XML & HTML)
Allow: /sitemap.xml
Allow: /sitemaps/
Allow: /sitemaps/*
Allow: /sitemap-news.xml
Allow: /sitemap-news/
Allow: /sitemap-news/*
Allow: /sitemap-image.xml
Allow: /sitemap-image/
Allow: /sitemap-image/*
Allow: /html-sitemap/
Allow: /html-sitemap/*

# Feeds
Allow: /feed/
Allow: /rss.xml
Allow: /atom.xml

# Images (all formats)
Allow: /_next/image?*
Allow: /_next/image*
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.webp
Allow: /*.avif
Allow: /*.svg
Allow: /*.ico

# JSON & config files
Allow: /_next/static/*.json
Allow: /manifest.json
Allow: /*.jsonld

# Advertising & AI
Allow: /ads.txt
Allow: /llms.txt
Allow: /llms-full.txt

# ============================================
# Disallow Rules
# ============================================

# Admin & authentication
Disallow: /admin/
Disallow: /api/auth/

# Sensitive API endpoints
Disallow: /api/draft-mode/
Disallow: /api/revalidate/
Disallow: /api/google-indexing/
Disallow: /api/sendEmail/
Disallow: /api/test-email/
Disallow: /api/verify-captcha/
Disallow: /api/redeploy/
Disallow: /api/bulk-reindex/
Disallow: /api/regenerate-sitemaps/
Disallow: /api/upload-sitemap/
Disallow: /api/remove-urls/
Disallow: /api/sanity-stats/

# Internal data API endpoints (not for indexing)
Disallow: /api/categories/
Disallow: /api/tags/
Disallow: /api/navigation
Disallow: /api/company

# Google Search Console API (internal tools)
Disallow: /api/search-console/

# Search results pages: handled by noindex meta tag in layout.tsx
# Allowing crawlers to access /search lets them read the noindex directive.

# Sanity Studio
Disallow: /studio/

# Vercel internal paths (allow speed-insights for renderer compatibility)
Allow: /_vercel/speed-insights/
Allow: /_vercel/insights/
Disallow: /_vercel/

# Next.js error pages
Disallow: /_next/static/chunks/pages/_error
Disallow: /_next/static/chunks/pages/404
Disallow: /_next/static/chunks/pages/500

# Constant Contact (internal integration)
Disallow: /constant-contact/

# Proxy endpoint (internal)
Disallow: /proxy/

# ============================================
# AI Crawler Rules (2026 Selective Access)
# ============================================
# Citation crawlers: allowed (drive traffic via source links)
# Training-only crawlers: blocked (absorb content with no attribution)

# Citation / search crawlers — link back to source
User-agent: PerplexityBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Training-only crawlers — no attribution
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: CCBot
Disallow: /

# ============================================
# Sitemaps Declaration (news first for priority)
# ============================================

Sitemap: https://www.globalbankingandfinance.com/sitemap-news.xml
Sitemap: https://www.globalbankingandfinance.com/sitemap.xml
Sitemap: https://www.globalbankingandfinance.com/sitemap-image.xml

# ============================================
# LLMs.txt (AI-readable site information)
# ============================================
# For AI assistants and LLMs to understand site structure
# See: https://www.globalbankingandfinance.com/llms.txt
# See: https://www.globalbankingandfinance.com/llms-full.txt