# ============================================================
# robots.txt — lsbr.uk
# Maximum crawlability for public content, zero waste on private routes
# ============================================================

# ── All well-behaved crawlers ────────────────────────────────
User-agent: *

# Public pages — explicitly welcomed
Allow: /
Allow: /course/
Allow: /courses
Allow: /courses/
Allow: /blog
Allow: /blog/
Allow: /topics/
Allow: /our-courses
Allow: /popular-online-courses
Allow: /doctorate-courses
Allow: /postgraduate-courses
Allow: /undergraduate-courses
Allow: /foundation-courses
Allow: /high-credit-programmes
Allow: /mba-degrees
Allow: /ba-bsc-degrees
Allow: /degree-programmes
Allow: /all-courses/
Allow: /contact-us
Allow: /partner-with-us
Allow: /terms-and-conditions
Allow: /privacy-policy
Allow: /refunds-policy

# Allow CSS/JS so search engines can render pages properly
Allow: /css/
Allow: /js/
Allow: /lib/

# Admin & back-office
Disallow: /Admin/
Disallow: /AdminDashboard/
Disallow: /PromoBanner/
Disallow: /LmsBanner/
Disallow: /CategoryBanner/
Disallow: /AlumniPromoBanner/

# Account, auth & user-specific
Disallow: /Account/
Disallow: /login
Disallow: /Dashboard/
Disallow: /User-Dashboard
Disallow: /Employee/

# LMS / enrolled-learner pages
Disallow: /Home/CoursePage/
Disallow: /Home/CourseStart/
Disallow: /Home/MyCourses
Disallow: /Home/UnitDetails/
Disallow: /Home/FreeEnroll/
Disallow: /Home/UserDashboard

# Payment & checkout
Disallow: /Payment/
Disallow: /Invoice/

# Assignments, quizzes, certificates
Disallow: /Assignment/
Disallow: /Quiz/
Disallow: /Certificate/
Disallow: /Support/

# APIs & webhooks
Disallow: /api/
Disallow: /ExternalApi/

# System, debug & preview routes
Disallow: /system/
Disallow: /TestProduction
Disallow: /TestDB
Disallow: /TestSave
Disallow: /CheckCourseContent/
Disallow: /Home/GeneratingPreview
Disallow: /Home/ForceUnitManagement
Disallow: /Home/GenerateUnitsBackground
Disallow: /Home/GenerateReviewsBackground
Disallow: /Home/GenerateSkillsAsync
Disallow: /Home/ForceRegenerateKeyFacts
Disallow: /Home/SkillsGainedDemo
Disallow: /Home/HeroInfoDemo
Disallow: /Home/FooterDemo
Disallow: /Home/PaymentSuccessDemo

# Uploaded user content & bulk assets (save crawl budget)
Disallow: /uploads/

# Prevent indexing search result pages with query strings (duplicate content)
Disallow: /Home/Courses?
Disallow: /search/

# Legacy WordPress URL structures (migrated — these 301/410 server-side too)
Disallow: /tag/
Disallow: /course-tag/
Disallow: /course-category-
Disallow: /instructor/
Disallow: /our-courses/course-category-
Disallow: /.well-known/sgcaptcha/
Disallow: /wp-admin/
Disallow: /wp-login.php

# ── Googlebot — no crawl-delay (Google ignores it and it only hurts you) ──
User-agent: Googlebot
Allow: /

# ── Googlebot-Image — let Google index course & blog images ──
User-agent: Googlebot-Image
Allow: /images/
Allow: /uploads/
Disallow: /Admin/

# ── Bingbot ───────────────────────────────────────────────────
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# ── AI search & answer-engine crawlers (drive referral traffic) ──
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ── Block aggressive scrapers that add no SEO or conversion value ──
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ── Sitemaps ──────────────────────────────────────────────────
Sitemap: https://www.lsbr.uk/sitemap.xml