# https://www.robotstxt.org/robotstxt.html
# ================================
# Sitemap
# ================================
Sitemap: https://www.mindtickle.com/sitemap.xml

# Global rules – allow indexing
# ================================
User-agent: *
Allow: /
# ================================
# Block legacy WordPress paths
# ================================
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-json/
Disallow: /wp-json/wp/v2/users/
Disallow: /cgi-bin/
Disallow: /xmlrpc.php
# ================================
# Block internal / non-SEO pages
# ================================
Disallow: /search
Disallow: /?
Disallow: /&
Disallow: /utm_
Disallow: /ref=
Disallow: /preview=true
# ================================
# Block Strapi Admin & APIs
# ================================
Disallow: /admin
Disallow: /api
Disallow: /uploads/tmp/
Disallow: /i18n
Disallow: /content-manager
Disallow: /content-type-builder
Disallow: /email
Disallow: /users-permissions
Disallow: /graphql
# ============================
# AI / LLM Crawlers (Allowed)
# ============================

User-agent: GPTBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: bingbot
Allow: /

User-agent: BingPreview
Allow: /

User-agent: bingGPT
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Bard
Allow: /

User-agent: Gemini
Allow: /

User-agent: ClaudeBot

# Common Crawl (used by many AI models)
User-agent: CCBot
Allow: /
User-agent: CookieYesbot
Disallow: /
# ================================