# --------------------------------------------------------- # Global Rules (apply to all crawlers) # --------------------------------------------------------- User-agent: * # Block access to WordPress admin area Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php # Block feed URLs to prevent duplicate content indexing Disallow: */feed # Block search result pages and filtered content to avoid low-value indexing Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= # Sitemap location for better indexing efficiency Sitemap: https://www.realitymine.com/sitemap.xml # --------------------------------------------------------- # AI & LLM Bot Access (explicit, follows same global rules) # --------------------------------------------------------- # AI usage preferences: /llms.txt # OpenAI User-agent: GPTBot Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= User-agent: OAI-SearchBot Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= User-agent: ChatGPT-User Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= # Perplexity bots User-agent: PerplexityBot Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= User-agent: Perplexity-User Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= # Anthropic (Claude) bots User-agent: ClaudeBot Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= User-agent: Claude-Web Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= User-agent: anthropic-ai Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= User-agent: Claude-User Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= User-agent: Claude-SearchBot Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= # Meta (Facebook/Instagram) AI agents User-agent: meta-externalagent Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= # DeepSeek AI bots User-agent: DeepSeekBot Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= # Google Gemini AI bot User-agent: Google-Extended Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= # Potential new Gemini crawler (future-proof) User-agent: Gemini-Deep-Research Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= # Apple User-agent: Applebot Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= # Apple (Apple Intelligence / Siri) User-agent: Applebot-Extended Disallow: */feed Disallow: /*?from= Disallow: /*?utm_ Disallow: /*?trk= Disallow: /*?__hstc= Disallow: /*?__hssc= Disallow: /*?__hsfp= Sitemap: https://www.realitymine.com/sitemap.xml