User-agent: *
# --- WordPress core ---
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /*preview=true*

# --- Search, filters, and crawl traps ---
Disallow: /?s=
Disallow: /search/
Disallow: /*?*
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/

# --- Low-value / duplicate archives ---
Disallow: /attachment/
Disallow: /date/
Disallow: /tag/

# --- Programmatic / internal content paths ---
Disallow: */grammarPost/
Disallow: */cultureNotePost/
Disallow: */landing-page-word/
Disallow: */ldp_screenshot/

# --- Files & internal tools ---
Disallow: /wp-content/uploads/wp-import-export-lite/
Disallow: /private_file.html
Disallow: /directory/

# --- Sitemap ---
Sitemap: https://ling-app.com/sitemap_index.xml

# ==============================
# Bot-specific handling
# ==============================

# Google
User-agent: Googlebot
Allow: /
Disallow: /*?*
Disallow: /search/
Disallow: /tag/
Disallow: /wp-admin/

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /
Disallow: /*?*
Disallow: /search/
Disallow: /tag/
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php

# Common Crawl (used by multiple AI systems)
User-agent: CCBot
Allow: /
Disallow: /*?*
Disallow: /search/
Disallow: /tag/
Disallow: /wp-admin/

# Anthropic
User-agent: anthropic-ai
Allow: /
Disallow: /*?*
Disallow: /search/
Disallow: /tag/
Disallow: /wp-admin/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /*?*
Disallow: /search/
Disallow: /tag/
Disallow: /wp-admin/

# Meta AI
User-agent: FacebookBot
Allow: /
Disallow: /*?*
Disallow: /search/
Disallow: /tag/
Disallow: /wp-admin/

# Social preview bots
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

# HubSpot
User-agent: HubSpot
Disallow: /

User-agent: HubSpot Crawler
Disallow: /

# Other bots
User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Bingbot
Allow: /