User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
Allow: /bonds/ 
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /Google-Ads.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/*
Disallow: /finance/search/*
Disallow: /hi/search/*
Disallow: /hi/finance/search/*
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /campaigns/gold-loan/generatelead.php
Disallow: /finance/media/press?_wrapper_format=html&page=2
Disallow: /finance/media/press?redirect=menu-bar&_wrapper_format=html&page=2
# Paths (no clean URLs) ..
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/
Disallow: /finance/taxonomy/term/*
Disallow: /finance/node/*
Disallow: /finance/taxonomy/term/1888
Disallow: /taxonomy/term/19?_wrapper_format=html&page=9
Disallow: /securities/business-partners/taxonomy/term/63
Disallow: /finance/node/*
Disallow: /finance/taxonomy/
Disallow: /taxonomy/
Disallow: */taxonomy/term/*
Disallow: /finance/taxonomy/*
Disallow: /taxonomy/*
Disallow: /node/*
Disallow: /hi/node/*
Disallow: /kn/node/*
Disallow: /gu/node/*
Disallow: /mr/node/*
Disallow: /or/node/*
Disallow: /ta/node/*
Disallow: /as/node/*
Disallow: /te/node/*
Disallow: /pa/node/*
Disallow: /bn/node/*
Disallow: /hi/finance/personal-loans/*
Disallow: /hi/finance/business-loans/*
Disallow: /finance/business-loans/*
Disallow: /finance/personal-loans/*
Disallow: /finance/iifl-finance-q3fy23-data-book
Disallow: /hi/finance/node/*
Disallow: /kn/finance/node/*
Disallow: /gu/finance/node/*
Disallow: /mr/finance/node/*
Disallow: /or/finance/node/*
Disallow: /ta/finance/node/*
Disallow: /as/finance/node/*
Disallow: /te/finance/node/*
Disallow: /pa/finance/node/*
Disallow: /bn/finance/node/*
Disallow: /sites/default/files/*
Disallow: /kn/search/*
Disallow: /gu/search/*
Disallow: /mr/search/*
Disallow: /or/search/*
Disallow: /ta/search/*
Disallow: /as/search/*
Disallow: /te/search/*
Disallow: /pa/search/*
Disallow: /bn/search/*
Disallow: /kn/finance/search/*
Disallow: /gu/finance/search/*
Disallow: /mr/finance/search/*
Disallow: /or/finance/search/*
Disallow: /ta/finance/search/*
Disallow: /as/finance/search/*
Disallow: /te/finance/search/*
Disallow: /pa/finance/search/*
Disallow: /bn/finance/search/*
Disallow: /bn/blogs/other/void(0);
Disallow: /KnowledgeCenter/11SuspiciousmovementInvestigate.aspx
Disallow: /KnowledgeCenter/Trustonlyexpertadviceespeciallyifyouareold.aspx
Disallow: /sites/default/files/newsroom/AGM_Bookclosure_Dispatch.pdf
Disallow: /sites/default/files/newsroom/IIFL_Securities_Q2FY20_Press_Release.pdf
Disallow: /sites/default/files/05JAN21_BuybackDailyReport.pdf
Disallow: /finance/page-not-found
Disallow: /hi/finance/page-not-found
Disallow: /kn/finance/page-not-found
Disallow: /gu/finance/page-not-found
Disallow: /mr/finance/page-not-found
Disallow: /or/finance/page-not-found
Disallow: /te/finance/page-not-found
Disallow: /pa/finance/page-not-found
Disallow: /bn/finance/page-not-found
Disallow: /ta/finance/page-not-found
Disallow: /as/finance/page-not-found
Disallow: /finance/csr-leadership-award-world-csr-day
Disallow: /finance/investorgrievancestatementprivateplacement30092020
Disallow: */likeit/like/*
Disallow: */like-dislike/dislike/*
Disallow: */like-dislike/like/*
Disallow: /campn/unlock-your-mahashakti/resend-otp
Disallow: /campn/unlock-your-mahashakti/verify-otp
Disallow: /campn/unlock-your-mahashakti/step1-form
Disallow: /*?amp=
Disallow: /*?redirect=
Disallow: /*void(0)
Disallow: /*/finance/media/


Sitemap: https://www.iifl.com/sitemap-index.xml
Sitemap: https://www.iifl.com/product-sitemap.xml  
Sitemap: https://www.iifl.com/blogs-sitemap.xml  
Sitemap: https://www.iifl.com/vernacular-sitemap.xml

# Training crawler — builds datasets for GPT model development
User-agent: GPTBot
Allow: /

# Search indexing crawler — powers ChatGPT Search citations
# WARNING: Blocking this removes your site from ChatGPT Search answers entirely
User-agent: OAI-SearchBot
Allow: /

# User-triggered fetcher — activated when a human pastes a URL into ChatGPT
# Note: OpenAI states this may not always respect robots.txt on user request
User-agent: ChatGPT-User
Allow: /

# Training crawler — used for Claude model training data collection
User-agent: ClaudeBot
Allow: /

# Search indexing crawler — powers Claude's web search and citation features
# WARNING: Anthropic states blocking this "may reduce" visibility in Claude answers
User-agent: Claude-SearchBot
Allow: /

# User-triggered fetcher — activated when a Claude user requests a specific page
User-agent: Claude-User
Allow: /

# Primary indexing crawler — builds Perplexity's AI search index
User-agent: PerplexityBot
Allow: /

# User-triggered retrieval — fetches pages when a Perplexity user clicks a citation
User-agent: Perplexity-User
Allow: /

# Controls Gemini model training and Vertex AI data collection
User-agent: Google-Extended
Allow: /

# Gemini-specific fetch for AI answer grounding
User-agent: Googlebot-Extended
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# ------------------------------------------------------------------------------
# CCBot — Common Crawl (open dataset used by many AI labs)
# Common Crawl data underlies training for GPT, LLaMA, Mistral, and many others.
# Allowing CCBot is a broad opt-in for open AI research datasets.
# ------------------------------------------------------------------------------

User-agent: CCBot
Allow: /