# ============================================
# Mi Property Portal - robots.txt
# Last updated: March 2026
# ============================================

Sitemap: https://www.mipropertyportal.com/sitemap.xml
Sitemap: https://www.mipropertyportal.com/sitemap_index.xml

# ============================================
# MAIN CRAWL RULES
# ============================================

User-agent: *

# WordPress System Files
Disallow: /wp-login.php
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-json/
Disallow: /xmlrpc.php
Disallow: /wp-cron.php
Disallow: /wp-config.php

# Duplicate & Low-Value Content
Disallow: /?s=
Disallow: /search/
Disallow: /?p=
Disallow: /page/
Disallow: /comments/
Disallow: /trackback/
Disallow: /feed/

# URL Parameters (prevents duplicate content)
Disallow: /*?*replytocom=
Disallow: /*?*doing_wp_cron
Disallow: /*?*utm_
Disallow: /*&*

# System & Dev Paths
Disallow: /cgi-bin/
Disallow: /error
Disallow: /demoinformation
Disallow: /staging/
Disallow: /dev/

# Allow Critical Assets
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/

# Allow Important SEO Pages
Allow: /blog/
Allow: /category/
Allow: /tag/
Allow: /author/
Allow: /features/
Allow: /pricing/
Allow: /about/
Allow: /contact/

# Allow llms.txt for AI discovery
Allow: /llms.txt

# ============================================
# GOOGLE BOTS
# ============================================

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /wp-content/uploads/

User-agent: Googlebot-Mobile
Allow: /

# ============================================
# BING
# ============================================

User-agent: Bingbot
Allow: /

# ============================================
# AI SEARCH CRAWLERS - ALLOWED
# These bots power AI search features (ChatGPT search,
# Google AI Overviews, Perplexity answers, Claude).
# Allowing them = your site can appear in AI-generated answers.
# ============================================

User-agent: GPTBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# ============================================
# AI TRAINING CRAWLERS - BLOCKED
# These bots scrape for model training datasets,
# not for search. Safe to block.
# ============================================

User-agent: CCBot
Disallow: /

User-agent: Omgilibot
Disallow: /

# ============================================
# SOCIAL MEDIA BOTS - ALLOWED
# Needed for link previews on Facebook, LinkedIn, etc.
# ============================================

User-agent: FacebookBot
Allow: /

# ============================================
# SEO AUDIT BOTS
# ============================================

User-agent: SemrushBot
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MegaIndex
Disallow: /