# robots.txt for bajajbroking.in
# Version: 1.0 | Updated: 2026-04-01 | Owner: SEO / AEO Team
# Purpose: SEO + AI-crawler friendly crawl control for AEM site

# === CORE CRAWL RULES ===
User-agent: *
Allow: /
Crawl-delay: 1

# Rendering assets
Allow: /content/dam/*.png
Allow: /content/dam/*.svg
Allow: /content/dam/*.ico
Allow: /content/dam/*.jpg
Allow: /content/dam/*.jpeg
Allow: /etc.clientlibs/bfsl/*.js
Allow: /etc.clientlibs/bfsl/*.css
Allow: /content/bfsl/*.woff2
Allow: /content/bfsl/*.woff
Allow: /content/bfsl/*.eot
Allow: /content/bfsl/*.ttf
Allow: /content/bfsl/*.otf

# Core blocks
Disallow: /corpwp.bajajfinservsecurities.in/
Disallow: /content/bfsl/
Disallow: /content/dam/
Disallow: /etc.clientlibs/bfsl/
Disallow: /etc/
Disallow: /libs/
Disallow: /samadhan/

# Ensure /llms.txt remains accessible to crawlers
Allow: /llms.txt

# === AI CRAWLER OVERRIDES ===
# Allow AI crawlers to index public content and llms.txt
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: CCBot
Allow: /

# === TRAINING CRAWLER BLOCKS ===
User-agent: GPTBot-Training
Disallow: /

User-agent: ClaudeBot-Training
Disallow: /

# === POLICY NOTE ===
# This file controls both search engines and AI crawlers.
# LLM-specific usage guidance (what they may train on, how to attribute, etc.)
# is provided in /llms.txt.

# === CONTACT & CHANGE LOG ===
# Questions? Contact: seo@bajajbroking.in
# Change log:
# 2026-04-01: Added AI-crawler blocks + llms.txt support

# === SITEMAPS ===
# Comprehensive sitemaps covering all major content pillars:
# Stocks, IPOs, Mutual Funds, Blog, News, Knowledge Center
Sitemap: https://www.bajajbroking.in/sitemap-index.xml
Sitemap: https://www.bajajbroking.in/sitemap.xml
Sitemap: https://www.bajajbroking.in/sitemap.ipo-sitemap.xml
Sitemap: https://www.bajajbroking.in/sitemap.knowledge-center-sitemap.xml
Sitemap: https://www.bajajbroking.in/sitemap.indices-sitemap.xml
Sitemap: https://www.bajajbroking.in/sitemap.blog-sitemap.xml
Sitemap: https://www.bajajbroking.in/sitemap.stock-sitemap.xml
Sitemap: https://www.bajajbroking.in/sitemap.web-stories-sitemap.xml
Sitemap: https://www.bajajbroking.in/sitemap.share-market-news-sitemap.xml
Sitemap: https://www.bajajbroking.in/sitemap.mutual-funds-sitemap.xml