# robots.txt for InfosecTrain (infosectrain.com)
# Updated: April 2026

# -------------------------------------------------------
# GLOBAL RULES - ALL CRAWLERS
# -------------------------------------------------------

User-agent: *
Allow: /

# Public content - explicitly allowed
Allow: /courses
Allow: /blog
Allow: /webinars
Allow: /free-cybersecurity-training
Allow: /free-practice-tests
Allow: /category/interview-questions
Allow: /events
Allow: /fastrack-bootcamp
Allow: /membership
Allow: /corporate-training
Allow: /hire-a-trainer
Allow: /one-on-one-training
Allow: /career-mentorship-program
Allow: /cybersecurity-certification-training
Allow: /cloud-security-certification-training
Allow: /data-privacy
Allow: /information-security-certification-training-course
Allow: /offensive-security-training-courses
Allow: /vendor
Allow: /about-us
Allow: /instructors
Allow: /our-trainers
Allow: /leadership
Allow: /corporate-clients
Allow: /customers-review
Allow: /success-stories
Allow: /contact-us
Allow: /training-calendar

# LLM guidance files - must be crawlable
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai.txt

# Restricted - internal, payment, secure
Disallow: /secure/
Disallow: /admin-report/
Disallow: /payment/
Disallow: /online-payment/
Disallow: /non-disclosure/
Disallow: /non-disclosure-azp/
Disallow: /exam-non-disclosure/
Disallow: /api/
Disallow: /amp/
Disallow: /assessment/
Disallow: /dfir-assessment/
Disallow: /attendance/
Disallow: /my-admin/
Disallow: /internal-test/
Disallow: /enquiry/
Disallow: /emailerphp8/
Disallow: /phpmailer68/
Disallow: /sale-registration-form/
Disallow: /certificate_file_student/
Disallow: /certificate_eventfile/
Disallow: /secure-intelligence-ppt/
Disallow: /pages/
Disallow: /learning/
Disallow: /self-learning/
Disallow: /newsite/
Disallow: /hometest24/
Disallow: /home-2/
Disallow: /beta/
Disallow: /sh/
Disallow: /course/

# -------------------------------------------------------
# GOOGLE - SPECIFIC RULES
# -------------------------------------------------------

User-agent: Googlebot
Allow: /
Disallow: /secure/
Disallow: /admin-report/
Disallow: /payment/
Disallow: /api/

User-agent: Googlebot-Image
Allow: /

# Allow Google AI Overviews & Gemini
User-agent: Google-Extended
Allow: /
Disallow: /secure/
Disallow: /admin-report/
Disallow: /payment/

# -------------------------------------------------------
# AI SEARCH & LLM CRAWLERS
# -------------------------------------------------------

# OpenAI (ChatGPT, SearchGPT)
User-agent: GPTBot
Allow: /
Disallow: /secure/
Disallow: /admin-report/
Disallow: /payment/

User-agent: OAI-SearchBot
Allow: /
Disallow: /secure/
Disallow: /payment/

User-agent: ChatGPT-User
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
Allow: /
Disallow: /secure/
Disallow: /admin-report/
Disallow: /payment/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /secure/
Disallow: /payment/

# Microsoft Copilot / Bing
User-agent: Bingbot
Allow: /
Disallow: /secure/
Disallow: /payment/

# Other AI agents
User-agent: FirecrawlAgent
Allow: /

#Yahoo
User-agent: Slurp
allow: /

User-agent: PhindBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: AndiBot
Allow: /

User-agent: ExaBot
Allow: /

User-agent: Apple-Extended
Allow: /

User-agent: Baiduspider
Allow: /
Disallow: /secure/

User-agent: Yandex
Allow: /
Disallow: /secure/

# -------------------------------------------------------
# LLM POLICY REFERENCE
# -------------------------------------------------------

LLM-Policy: https://www.infosectrain.com/llms.txt
LLM-Full Policy: https://www.infosectrain.com/llms-full.txt
AI-Policy: https://www.infosectrain.com/ai.txt

# -------------------------------------------------------
# SITEMAP
# -------------------------------------------------------

Sitemap: https://www.infosectrain.com/blog-sitemap.xml
Sitemap: https://www.infosectrain.com/blog-sitemap2.xml
Sitemap: https://www.infosectrain.com/courses-sitemap.xml
Sitemap: https://www.infosectrain.com/events-sitemap.xml
Sitemap: https://www.infosectrain.com/training-sitemap.xml
Sitemap: https://www.infosectrain.com/training-sitemap1.xml
Sitemap: https://www.infosectrain.com/news-sitemap.xml