# ==============================
# Kalaniketan.com robots.txt - Magento-compatible
# ==============================

# --- Googlebot (full access) ---
User-agent: Googlebot
Disallow:

# --- Bingbot (throttle crawl rate) ---
User-agent: Bingbot
Crawl-delay: 10
Disallow: /index.php/
Disallow: /*?
Disallow: /checkout/
Disallow: /app/
Disallow: /lib/
Disallow: /*.php$
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /customer/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /*SID=
Disallow: /terms-conditions
Disallow: /privacy-policy
Disallow: /returns-exchanges
Disallow: /shipping-delivery
Disallow: /add109f68a_admin/

# --- SEO crawlers with crawl limits ---
User-agent: SiteAuditBot
Crawl-delay: 10
Disallow: /index.php/
Disallow: /*?
Disallow: /checkout/
Disallow: /app/
Disallow: /lib/
Disallow: /*.php$
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /customer/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /*SID=
Disallow: /terms-conditions
Disallow: /privacy-policy
Disallow: /returns-exchanges
Disallow: /shipping-delivery
Disallow: /add109f68a_admin/

User-agent: AhrefsBot
Crawl-delay: 10
Disallow: same as above

User-agent: GPTBot
Crawl-delay: 10
Disallow: same as above

# --- Social / app crawlers ---
User-agent: meta-externalads
Disallow: /checkout/
Disallow: /admin_9b7kag1/
Disallow: /private/

User-agent: Instagram
Disallow: /checkout/
Disallow: /admin_9b7kag1/
Disallow: /private/

# --- Other known scrapers / bots ---
User-agent: Yandex
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: DuckDuckBot
Disallow: /

User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: AdsBot-Google-Mobile-Apps
Allow: /

# --- Default rules for all other bots ---
User-agent: *
Disallow: /index.php/
Disallow: /*?
Disallow: /checkout/
Disallow: /app/
Disallow: /lib/
Disallow: /*.php$
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /customer/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /*SID=
Disallow: /terms-conditions
Disallow: /privacy-policy
Disallow: /returns-exchanges
Disallow: /shipping-delivery
Disallow: /add109f68a_admin/
Crawl-delay: 10

# --- Sitemap ---
Sitemap: https://www.kalaniketan.com/media/sitemap/kncom_sitemap.xml