# robots.txt for br7news.co.il
# Updated: 2025-12-10
# Canonical: https://br7news.co.il

# ========= Default rules for all crawlers =========
User-agent: *

# Admin and system directories - block
Disallow: /admin/
Disallow: /nihul/
Disallow: /cron/
Disallow: /login/
Disallow: /auth/
Disallow: /user/
Disallow: /dashboard/
Disallow: /inbox/
Disallow: /notification/
Disallow: /application/
Disallow: /system/
Disallow: /vendor/
Disallow: /cgi-bin/
Disallow: /webmail/
Disallow: /cpanel/
Disallow: /mail/
Disallow: /staging/
Disallow: /home/banner_counter
Disallow: /home/save_promo
Disallow: /noPage

# Block broken ad tracking URLs (garbage from ad macros)
Disallow: /*%%
Disallow: /*CLICK_URL

# AJAX endpoints - internal use only
Disallow: /article/list_view
Disallow: /article/get_quest
Disallow: /article/get_quiz
Disallow: /article/click_counter
Disallow: /article/contact_us
Disallow: /banner/get_inner_banner
Disallow: /business/register
Disallow: /business/save_message
Disallow: /business/search
Disallow: /business/list_view
Disallow: /coupon/list_view
Disallow: /coupon/search_location
Disallow: /coupon/update_used
Disallow: /event/list_view
Disallow: /tag/list_view

# Allow public assets (CSS, JS, images)
Allow: /public/uploads/
Allow: /public/assets/
Allow: /public/images/
Allow: /public/slick/
Allow: /public/css/
Allow: /public/js/

# ========= Googlebot =========
User-agent: Googlebot
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Image
Allow: /

# ========= Other search engines =========
User-agent: bingbot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: YandexNews
Allow: /

# ========= AI crawlers (allowed) =========
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ========= Sitemaps =========
Sitemap: https://br7news.co.il/sitemap.xml
Sitemap: https://br7news.co.il/sitemap-tags.xml
Sitemap: https://br7news.co.il/sitemap-authors.xml
Sitemap: https://br7news.co.il/news-sitemap.xml
Sitemap: https://br7news.co.il/news-sitemap-he.xml
Sitemap: https://br7news.co.il/news-sitemap-ru.xml
Sitemap: https://br7news.co.il/news-sitemap-ar.xml
Sitemap: https://br7news.co.il/sitemap-events.xml

# ========= Yandex News Feed (Russian) =========
Sitemap: https://br7news.co.il/yandex-feed.xml