Top Sites>The London School of Business and Research - Executive Education - LSBR

# ============================================================
# robots.txt — lsbr.uk
# Maximum crawlability for public content, zero waste on private routes
# ============================================================

# ── All well-behaved crawlers ────────────────────────────────
User-agent: *

# Public pages — explicitly welcomed
Allow: /
Allow: /course/
Allow: /courses
Allow: /courses/
Allow: /blog
Allow: /blog/
Allow: /topics/
Allow: /our-courses
Allow: /popular-online-courses
Allow: /doctorate-courses
Allow: /postgraduate-courses
Allow: /undergraduate-courses
Allow: /foundation-courses
Allow: /high-credit-programmes
Allow: /mba-degrees
Allow: /ba-bsc-degrees
Allow: /degree-programmes
Allow: /all-courses/
Allow: /contact-us
Allow: /partner-with-us
Allow: /terms-and-conditions
Allow: /privacy-policy
Allow: /refunds-policy

# Allow CSS/JS so search engines can render pages properly
Allow: /css/
Allow: /js/
Allow: /lib/

# Admin & back-office
Disallow: /Admin/
Disallow: /AdminDashboard/
Disallow: /PromoBanner/
Disallow: /LmsBanner/
Disallow: /CategoryBanner/
Disallow: /AlumniPromoBanner/

# Account, auth & user-specific
Disallow: /Account/
Disallow: /login
Disallow: /Dashboard/
Disallow: /User-Dashboard
Disallow: /Employee/

# LMS / enrolled-learner pages
Disallow: /Home/CoursePage/
Disallow: /Home/CourseStart/
Disallow: /Home/MyCourses
Disallow: /Home/UnitDetails/
Disallow: /Home/FreeEnroll/
Disallow: /Home/UserDashboard

# Payment & checkout
Disallow: /Payment/
Disallow: /Invoice/

# Assignments, quizzes, certificates
Disallow: /Assignment/
Disallow: /Quiz/
Disallow: /Certificate/
Disallow: /Support/

# APIs & webhooks
Disallow: /api/
Disallow: /ExternalApi/

# System, debug & preview routes
Disallow: /system/
Disallow: /TestProduction
Disallow: /TestDB
Disallow: /TestSave
Disallow: /CheckCourseContent/
Disallow: /Home/GeneratingPreview
Disallow: /Home/ForceUnitManagement
Disallow: /Home/GenerateUnitsBackground
Disallow: /Home/GenerateReviewsBackground
Disallow: /Home/GenerateSkillsAsync
Disallow: /Home/ForceRegenerateKeyFacts
Disallow: /Home/SkillsGainedDemo
Disallow: /Home/HeroInfoDemo
Disallow: /Home/FooterDemo
Disallow: /Home/PaymentSuccessDemo

# Uploaded user content & bulk assets (save crawl budget)
Disallow: /uploads/

# Prevent indexing search result pages with query strings (duplicate content)
Disallow: /Home/Courses?
Disallow: /search/

# Legacy WordPress URL structures (migrated — these 301/410 server-side too)
Disallow: /tag/
Disallow: /course-tag/
Disallow: /course-category-
Disallow: /instructor/
Disallow: /our-courses/course-category-
Disallow: /.well-known/sgcaptcha/
Disallow: /wp-admin/
Disallow: /wp-login.php

# ── Googlebot — no crawl-delay (Google ignores it and it only hurts you) ──
User-agent: Googlebot
Allow: /

# ── Googlebot-Image — let Google index course & blog images ──
User-agent: Googlebot-Image
Allow: /images/
Allow: /uploads/
Disallow: /Admin/

# ── Bingbot ───────────────────────────────────────────────────
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# ── AI search & answer-engine crawlers (drive referral traffic) ──
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ── Block aggressive scrapers that add no SEO or conversion value ──
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

# ── Sitemaps ──────────────────────────────────────────────────
Sitemap: https://www.lsbr.uk/sitemap.xml

# LLMS.TXT — Large Language Model Usage Policy
# Version: 1.0
# Last-Updated: 2025-08-11

site: https://www.lsbr.uk
owner: The London School of Business and Research (LSBR)
contact: https://www.lsbr.uk/Support/GetSupport

# Purpose
# This file communicates LSBR's policy for how Large Language Models (LLMs) and
# AI agents may interact with, use, or train on content hosted on this domain.

# General Policy
training: disallow           # Do not use any site content for model training
fine_tuning: disallow        # Do not use any site content for fine-tuning
embedding: disallow          # Do not store site content as vector embeddings
redistribution: disallow     # Do not redistribute or mirror the content
commercial_use: disallow     # Do not use content in paid/commercial AI services without a license

# Crawling / Access Guidance for AI agents (non-training use)
ai_crawling: allow           # Read/preview for search/assistant features is allowed subject to robots.txt
respect_robots: required     # You must respect robots.txt and standard HTTP caching rules
cache_ttl: 7d                # Do not cache content longer than 7 days without revalidation
crawl_rate_limit: reasonable # Avoid high-frequency or parallelized crawling that harms performance

# Sensitive Paths — must NEVER be accessed, processed or stored by AI agents
# (These paths mirror sensitive areas and augment robots.txt restrictions.)
disallow_paths:
  - /Account/
  - /Admin/
  - /AdminDashboard/
  - /Dashboard/
  - /Employee/
  - /Payment/
  - /Quiz/
  - /Support/

# Attribution and Licensing
copyright: © LSBR. All rights reserved.
license: No license is granted to use the content for AI training or dataset creation.

# Exceptions (none by default)
# If LSBR explicitly authorizes use for a specific agent, it will be listed here.
# exceptions:
#   - agent: example-ai-bot
#     scope: preview-only
#     expiry: 2025-12-31

# Compliance
# By accessing this site with an AI agent, you agree to these terms.
# For licensing inquiries, please contact us via the URL above.