User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Allow helpful crawlers

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /

User-agent: Baiduspider
Allow: /

# Allow major SEO tools and AI crawlers
User-agent: AhrefsBot
Allow: /

User-agent: SemrushBot
Allow: /

User-agent: DataForSeoBot
Allow: /

User-agent: PetalBot
Allow: /

# AI Bots

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

# Block abusive / scraping bots

User-agent: MJ12bot
Disallow: /

User-agent: YandexImages
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: SiteSucker
Disallow: /

User-agent: CCBot
Disallow: /

# Sitemap pointers (critical)
Sitemap: https://athensservices.com/post-sitemap.xml
Sitemap: https://athensservices.com/page-sitemap.xml
Sitemap: https://athensservices.com/commercial_services-sitemap.xml
Sitemap: https://athensservices.com/residential_services-sitemap.xml
Sitemap: https://athensservices.com/tribe_events-sitemap.xml
Sitemap: https://athensservices.com/local-sitemap.xml