#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html


# ------------------------------------------------
# ALLOW STATIC ASSETS
# ------------------------------------------------
User-agent: *
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg

Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg


# ------------------------------------------------
# BLOCK SYSTEM AREAS
# ------------------------------------------------
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /cgi-bin/


# ------------------------------------------------
# BLOCK TRACKING PARAMETERS
# ------------------------------------------------
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: /*?trk=
Disallow: /*?weekdayJdUid=
Disallow: /*?replytocom
Disallow: /?

# ------------------------------------------------
# BLOCK SEARCH PAGES
# ------------------------------------------------
Disallow: /search/
Disallow: /?s=


# ------------------------------------------------
# BLOCK FEEDS
# ------------------------------------------------
Disallow: /feed/
Disallow: /feed
Disallow: /*/feed/
Disallow: /comments/feed/


# ------------------------------------------------
# BLOCK CACHE
# ------------------------------------------------
Disallow: /wp-content/cache/


# ------------------------------------------------
# BLOCK DOCUMENT FILE TYPES
# ------------------------------------------------
Disallow: /*.pdf$
Disallow: /*.doc$
Disallow: /*.docx$
Disallow: /*.xls$
Disallow: /*.xlsx$
Disallow: /*.csv$
Disallow: /*.ppt$
Disallow: /*.pptx$
Disallow: /*.zip$
Disallow: /*.tar$
Disallow: /*.gz$
Disallow: /*.bak$
Disallow: /*.old$
Disallow: /*.backup$
Disallow: /*.sql$


# ------------------------------------------------
# BLOCK SENSITIVE FILES
# ------------------------------------------------
Disallow: /README.txt
Disallow: /web.config
Disallow: /readme.html
Disallow: /license.txt


# ------------------------------------------------
# BLOCK REST API
# ------------------------------------------------
Disallow: /wp-json/


# ------------------------------------------------
# Legacy Rules
# ------------------------------------------------
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/*
Disallow: /taxonomy/*
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout

Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout


# ------------------------------------------------
# EXCEL DIRECTORY BLOCK
# ------------------------------------------------
Disallow: /sites/default/files/documents/excel
Disallow: /*.xlsx$
Disallow: /*.xls$


# ------------------------------------------------
# AI BOTS
# ------------------------------------------------
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Gemini-Deep-Research
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Applebot-Extended
Allow: /


# ------------------------------------------------
# SEMRUSH RATE LIMIT
# ------------------------------------------------
User-agent: SemrushBot
Crawl-delay: 5

# ------------------------------------------------
# AHREFS RATE LIMIT
# ------------------------------------------------
User-agent: Ahrefs
Crawl-delay: 5


# ------------------------------------------------
# MOZ
# ------------------------------------------------
User-agent: rogerbot
Disallow:


# ------------------------------------------------
# SITEMAPS
# ------------------------------------------------
Sitemap: https://www.bajajamc.com/sitemap_index.xml
Sitemap: https://www.bajajamc.com/calculator-sitemap.xml
Sitemap: https://www.bajajamc.com/glossary-sitemap.xml
Sitemap: https://www.bajajamc.com/page-sitemap.xml
Sitemap: https://www.bajajamc.com/mutual-fund-sitemap.xml
Sitemap: https://www.bajajamc.com/web-story-sitemap.xml
Sitemap: https://www.bajajamc.com/knowledge-centre-sitemap.xml
Sitemap: https://www.bajajamc.com/leadership-corner-sitemap.xml
Sitemap: https://www.bajajamc.com/news-sitemap.xml
Sitemap: https://www.bajajamc.com/local-sitemap.xml