# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /?*
Disallow: /programs/?*

# Block RSS feeds
Disallow: /feed/
Disallow: /*/feed/
Disallow: /comments/feed/
Disallow: /?feed=
Disallow: /*?feed=

Sitemap: https://ace.edu/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK

# ---------------------------------
# Explicitly allowed search crawlers
# ---------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp  # Yahoo
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# ---------------------------------
# Blocked search crawlers (Yandex & Baidu)
# ---------------------------------
User-agent: Yandex
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: YandexImages
Disallow: /

User-agent: YandexVideo
Disallow: /

User-agent: YandexMedia
Disallow: /

User-agent: YandexBlogs
Disallow: /

User-agent: YandexNews
Disallow: /

User-agent: YandexDirect
Disallow: /

User-agent: YandexMetrika
Disallow: /

User-agent: YandexMobileBot
Disallow: /

User-agent: YandexFavicons
Disallow: /

User-agent: YandexCalendar
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider-image
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-agent: Baiduspider-news
Disallow: /

User-agent: Baiduspider-favo
Disallow: /

User-agent: Baiduspider-cpro
Disallow: /

User-agent: Baiduspider-ads
Disallow: /

User-agent: Baiduspider-mobile
Disallow: /

# ---------------------------------
# Explicitly allowed ad crawlers
# ---------------------------------
User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: MSNBOT-Media  # Bing Ads
Allow: /

User-agent: Yahoo! Ad Bot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: Redditbot
Allow: /

# ---------------------------------
# Explicitly allowed AI crawlers
# ---------------------------------
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Add delay for Common Crawl
User-agent: CCBot
Crawl-delay: 5
Allow: /

# ---------------------------------
# Meta bots with crawl delay
# ---------------------------------
User-agent: FacebookExternalHit
Crawl-delay: 5
Allow: /

User-agent: facebot
Crawl-delay: 5
Allow: /

User-agent: Meta-ExternalAgent
Crawl-delay: 5
Allow: /

User-agent: Meta-WebIndexer
Crawl-delay: 5
Allow: /

# ---------------------------------
# Other AI crawlers
# ---------------------------------
User-agent: Applebot
Allow: /

User-agent: Amazonbot
Disallow: /

User-agent: AlexaMediaBot
Allow: /

User-agent: Gemini
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Copilot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: aibot
Allow: /

User-agent: Bytespider
Allow: /