# robots.txt for mashreqy.com
# Updated: 2026-02-21

User-agent: *
Allow: /

# Block non-public paths
Disallow: /api/
Disallow: /admin/
Disallow: /config/
Disallow: /*.php

# Block transactional pages (no SEO value)
Disallow: /checkout
Disallow: /my-orders
Disallow: /order-success
Disallow: /payment-result

# Block WhatsApp redirect links (no content, just redirect to wa.me)
Disallow: /wa/

# Dealer portal (internal use only)
Disallow: /dealers
Disallow: /wholesale
Disallow: /dealer-portal
Disallow: /api/v2/
Disallow: /api/internal/

# AI Search Crawlers (ALLOW - needed for AI search visibility)
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /checkout

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

# AI Training Crawlers (BLOCK - no search benefit, only training)
User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

# Sitemap location
Sitemap: https://mashreqy.com/sitemap.xml