# robots.txt for deVere Group
# Optimized for SEO and Security
# Last Updated: 2025-10-20

# ========================================
# GLOBAL RULES - ALL BOTS
# ========================================

User-agent: *
Crawl-delay: 10

# ALLOW - Public Content (SEO Optimization)
Allow: /what-we-do/
Allow: /about-us/
Allow: /blog/
Allow: /insights/
Allow: /resources/
Allow: /wealth-management/
Allow: /investment-services/
Allow: /retirement-planning/
Allow: /news/

# DISALLOW - Security & Privacy Sensitive Areas
Disallow: /admin/
Disallow: /wp-admin/
Disallow: /wp-login/
Disallow: /user-account/
Disallow: /dashboard/
Disallow: /private/
Disallow: /internal/
Disallow: /api/
Disallow: /backend/

# DISALLOW - System Files & Duplicate Content
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /cgi-bin/
Disallow: /.git/
Disallow: /.env
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /*?*sort=
Disallow: /*?*filter=
Disallow: /*?*page=

# DISALLOW - Session & Temporary Files
Disallow: /tmp/
Disallow: /cache/
Disallow: /sessions/
Disallow: /*.pdf?
Disallow: /*.zip?

# DISALLOW - Client Personal Pages (Security)
Disallow: /client-portal/
Disallow: /my-account/
Disallow: /account-settings/
Disallow: /downloads/private/

# ========================================
# SPECIFIC BOT RULES
# ========================================

# Block Low-Quality/Aggressive Bots
User-agent: SemrushBot
User-agent: DotBot
User-agent: MJ12bot
Disallow: /


# ========================================
# CRAWL DELAY & RATE LIMITING
# ========================================

# Generous crawl delay for respectful indexing
Crawl-delay: 1
Request-rate: 1/1s

# ========================================
# SITEMAP LOCATION
# ========================================

Sitemap: https://www.devere-group.com/sitemap.xml
Sitemap: https://www.devere-group.com/sitemap-news.xml

# ========================================
# NOTES
# ========================================
# 1. Update sitemaps.xml locations if different
# 2. Add specific disallow rules for any new admin URLs
# 3. Review quarterly for new security threats
# 4. Monitor Google Search Console for crawl errors
# 5. Consider blocking email addresses from search results
# 6. Add captcha verification to forms to prevent scraping