# ============================================
# GLOBAL RULES
# ============================================

User-agent: *
# Allow critical rendering resources
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/themes/Divi/
Allow: /wp-content/themes/Divichildtheme/
Allow: /wp-content/plugins/*.css
Allow: /wp-content/plugins/*.js
Allow: /wp-content/plugins/*.png
Allow: /wp-content/plugins/*.jpg
Allow: /wp-content/plugins/*.jpeg
Allow: /wp-content/plugins/*.webp
Allow: /wp-content/plugins/*.gif
Allow: /wp-content/plugins/*.svg


# Block sensitive WordPress core areas
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /*?utm_
Disallow: /*?gclid=
Disallow: /*?fbclid=

# Block internal system folders
Disallow: /wp-includes/
Disallow: /wp-content/cache/
Disallow: /wp-content/backup/
Disallow: /wp-content/logs/
Disallow: /wp-content/tmp/

# Block unnecessary crawl paths
Disallow: /author/
Disallow: /tag/
Disallow: /comments/
Disallow: /feed/
Disallow: */feed/

# Block search and parameter URLs
Disallow: /*?s=
Disallow: /*?p=
Disallow: /*?preview=
Disallow: /*?replytocom=
Disallow: /*?orderby=
Disallow: /*?filter=


# Allow important APIs for modern indexing
Allow: /wp-admin/admin-ajax.php
Allow: /wp-json/


# -----------------------------------------
# Allow all LLM / AI engines to crawl site
# -----------------------------------------


# --- OpenAI (ChatGPT, GPTBot) ---
User-agent: GPTBot
Allow: /

# --- OpenAI (ChatGPT Search, Web Retrieval) ---
User-agent: ChatGPT-User
Allow: /

# --- Google (Gemini / Bard) ---
User-agent: Google-Extended
Allow: /

# --- Google (Googlebot) ---
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

# --- Google Other AI Agents ---
User-agent: GoogleOther
Allow: /

# --- Meta (Facebook / Instagram AI) ---
User-agent: Meta-ExternalAgent
Allow: /

# --- Anthropic (Claude.ai) ---
User-agent: ClaudeBot
Allow: /

# --- Perplexity AI ---
User-agent: PerplexityBot
Allow: /

# --- Apple (Apple Intelligence AI Crawler) ---
User-agent: Applebot-Extended
Allow: /

# --- Amazon (Alexa / Titan AI) ---
User-agent: Amazonbot
Allow: /

# --- Common AI Research Crawlers ---
User-agent: CCBot
Allow: /

Sitemap: https://unicommerce.com/sitemap.xml
Sitemap: https://unicommerce.com/sitemap-ai.xml