# Robots.txt for betterhomes.com

# Allow all search engines to crawl the site
User-agent: *

# Block access to admin and private areas
Disallow: /admin/
Disallow: /api/
Disallow: /_next/
Disallow: /static/
Disallow: /tmp/
Disallow: /test/
Disallow: /debug/

# Block access to sensitive files and directories
Disallow: /*.json$

# Allow access to sitemaps (important for SEO)
Allow: /sitemap.xml
Allow: /sitemap/*.xml

# Block access to search and filter URLs with parameters to prevent duplicate content
Disallow: /*?*
Disallow: /*?*sort=
Disallow: /*?*filter=
Disallow: /*?*page=
Disallow: /*?*limit=

# Block access to development and staging paths
Disallow: /beta/

# Noindex pages (exclusive portfolio)
Disallow: /en/bh-exclusive
Disallow: /test-bot
Disallow: /ar/bh-exclusive

# Allow access to important static assets
Allow: /favicon.ico
Allow: /icon.svg
Allow: /manifest.json
Allow: /images/
Allow: /fonts/

# Allow access to all main content pages
Allow: /en/
Allow: /ar/

# Crawl delay to be respectful to server resources
Crawl-delay: 1

# Sitemap locations (update these URLs to match your actual domain)
Sitemap: https://www.bhomes.com/sitemap.xml
Sitemap: https://www.bhomes.com/sitemap/pages.xml
Sitemap: https://www.bhomes.com/sitemap/properties-for-sale.xml
Sitemap: https://www.bhomes.com/sitemap/properties-for-rent.xml
Sitemap: https://www.bhomes.com/sitemap/blogs.xml
Sitemap: https://www.bhomes.com/sitemap/developers.xml
Sitemap: https://www.bhomes.com/sitemap/Off-plan.xml
Sitemap: https://www.bhomes.com/sitemap/agents.xml
Sitemap: https://www.bhomes.com/sitemap/top-searches.xml


User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ChatGPT-Page
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: YouBot
Allow: /

User-agent: Cohere-ai
Allow: /