# -----------------------------------------------------------------------------
# ROBOTS.TXT FOR ARYAKA.COM
# Optimized for SEO & LLM Discovery - Updated 2025
# -----------------------------------------------------------------------------

# 1. Global Rules for All Bots
User-agent: *
# Block WordPress Admin to save crawl budget
Disallow: /wp-admin/
# Block collaterals Dir
Disallow: /wp-content/uploads/collaterals/
# Block themes & plugins to prevent unnecessary crawling
Disallow: /wp-content/plugins/
# ALLOW the AJAX handler (Critical for some WP functionality)
Allow: /wp-admin/admin-ajax.php

# Block internal search results to prevent duplicate content/crawl waste
Disallow: /?s=
Disallow: /search/
Disallow: /page/
Disallow: /tag/

# Allow all other content (CSS/JS/Images must be crawlable for mobile indexing)
Allow: /wp-content/
Allow: /wp-includes/

# -----------------------------------------------------------------------------
# 2. AI & LLM Specific Permissions
# Explicitly welcome major AI bots to ensure your technical docs are ingested.
# While "User-agent: *" covers this, these explicit blocks prevent accidental
# blocking by security plugins and signal intent to AI crawlers.
# -----------------------------------------------------------------------------

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

# -----------------------------------------------------------------------------
# 3. Sitemaps & LLM Discovery
# -----------------------------------------------------------------------------

# Standard XML Sitemap (Index)
Sitemap: https://aryaka.com/sitemap_index.xml 

# [NEW] LLM Documentation Map
# Declaring this as a Sitemap ensures bots like GPTBot prioritize fetching it.
Sitemap: https://aryaka.com/llms.txt