# robots.txt for VPN.com
# Optimized for SEO and AI crawler visibility
# Last updated: 2025-12-29

# =====================================================
# Default crawl rules for all bots
# =====================================================
User-agent: *

# Allow AI crawlers (GPTBot, CCBot, etc.) - no restrictions
# We want maximum AI visibility for content authority

# =====================================================
# Block WordPress admin and technical directories
# =====================================================
# Prevents crawl waste on backend/system files
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-includes/
Disallow: /wp-content/plugins/

# Allow CSS/JS/images from themes (needed for rendering)
# Must come BEFORE theme directory block for proper precedence
Allow: /wp-content/themes/*.css
Allow: /wp-content/themes/*.js
Allow: /wp-content/themes/*.png
Allow: /wp-content/themes/*.jpg
Allow: /wp-content/themes/*.jpeg
Allow: /wp-content/themes/*.gif
Allow: /wp-content/themes/*.svg
Allow: /wp-content/themes/*.webp
Disallow: /wp-content/themes/

# =====================================================
# Block private data directory
# =====================================================
# Prevent crawling of JSON configuration files
Disallow: /wp-content/data/private/

# =====================================================
# Block WordPress API endpoints
# =====================================================
# REST API adds no SEO value and wastes crawl budget
Disallow: /wp-json/

# =====================================================
# Block duplicate/thin content
# =====================================================
# Search results pages (multiple formats)
Disallow: /?s=
Disallow: /search/

# Tag archives (often thin content)
Disallow: /tag/

# Author archives (not needed for product site)
Disallow: /author/

# Feed URLs (duplicate content)
Disallow: /feed/
Disallow: /*/feed/
Disallow: /comments/feed/

# Trackback endpoints
Disallow: /trackback/
Disallow: /*/trackback/

# =====================================================
# Block URL parameters that create duplicates
# =====================================================
# Removed: Disallow: /*?* — too aggressive, blocks legitimate params
# Instead: Use canonical tags and handle via Cloudflare Cache Key settings
# Specific query params blocked above (?s=, etc.)

# Pagination if needed (uncomment to block)
# Disallow: /page/

# =====================================================
# Block file types that shouldn't be indexed
# =====================================================
# Note: /*.php$ uses $ anchor (Google-supported but not universal)
# Safe for VPN.com since no content URLs end in .php
Disallow: /*.php$
Disallow: /*.inc$
Disallow: /*.gz$
Disallow: /*.cgi$

# =====================================================
# Sitemap location
# =====================================================
# Primary sitemap index for all content
Sitemap: https://www.vpn.com/sitemap_index.xml