# == GOOGLE ==
User-agent: Googlebot
Disallow:

User-agent: Googlebot-Image
Disallow:

User-agent: Googlebot-Video
Disallow:

User-agent: Googlebot-News
Disallow:

# Google-Extended: AI overview training (allow for visibility in AI search)
User-agent: Google-Extended
Disallow:

# == BING / MICROSOFT ==
User-agent: Bingbot
Disallow:
Crawl-delay: 5

User-agent: msnbot
Disallow:
Crawl-delay: 5

# == OTHER SEARCH ENGINES ==
User-agent: DuckDuckBot
Disallow:

User-agent: Yandex
Disallow:
Crawl-delay: 10

# == SOCIAL MEDIA (critical for article sharing) ==
User-agent: Twitterbot
Disallow:

User-agent: facebookexternalhit
Disallow:

User-agent: LinkedInBot
Disallow:

User-agent: WhatsApp
Disallow:

# == AI RETRIEVAL BOTS (allows IPS content in AI answers) ==
User-agent: PerplexityBot
Disallow:

User-agent: ClaudeBot
Disallow:

User-agent: Applebot-Extended
Disallow:

# == AI TRAINING BOTS (blocked - no value exchange) ==
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: cohere-ai
Disallow: /

# == ALL OTHER BOTS: block sensitive areas only ==
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-json/
Allow: /wp-admin/admin-ajax.php

# == SITEMAPS ==
Sitemap: https://ipsnews.net/sitemap_index.xml
Sitemap: https://ipsnews.net/news-sitemap.xml