# Allow search engines and AI crawlers but keep sensitive paths disallowed

User-agent: Googlebot
Disallow: /App_Browsers/
Disallow: /App_config/
Disallow: /App_Data/
Disallow: /sitecore
Disallow: /Sitecore
Disallow: /sitecore_files/
Disallow: /sitecore modules/
Disallow: /temp/
Disallow: /upload/
Disallow: /xsl/
Disallow: /-/media/

User-agent: Bingbot
Disallow: /App_Browsers/
Disallow: /App_config/
Disallow: /App_Data/
Disallow: /sitecore
Disallow: /Sitecore
Disallow: /sitecore_files/
Disallow: /sitecore modules/
Disallow: /temp/
Disallow: /upload/
Disallow: /xsl/
Disallow: /-/media/

User-agent: Applebot
Disallow: /App_Browsers/
Disallow: /App_config/
Disallow: /App_Data/
Disallow: /sitecore
Disallow: /Sitecore
Disallow: /sitecore_files/
Disallow: /sitecore modules/
Disallow: /temp/
Disallow: /upload/
Disallow: /xsl/
Disallow: /-/media/

User-agent: DuckDuckBot
Disallow: /App_Browsers/
Disallow: /App_config/
Disallow: /App_Data/
Disallow: /sitecore
Disallow: /Sitecore
Disallow: /sitecore_files/
Disallow: /sitecore modules/
Disallow: /temp/
Disallow: /upload/
Disallow: /xsl/
Disallow: /-/media/

User-agent: ChatGPT-User
Disallow: /App_Browsers/
Disallow: /App_config/
Disallow: /App_Data/
Disallow: /sitecore
Disallow: /Sitecore
Disallow: /sitecore_files/
Disallow: /sitecore modules/
Disallow: /temp/
Disallow: /upload/
Disallow: /xsl/
Disallow: /-/media/

User-agent: ClaudeBot
Disallow: /App_Browsers/
Disallow: /App_config/
Disallow: /App_Data/
Disallow: /sitecore
Disallow: /Sitecore
Disallow: /sitecore_files/
Disallow: /sitecore modules/
Disallow: /temp/
Disallow: /upload/
Disallow: /xsl/
Disallow: /-/media/

User-agent: PerplexityBot
Disallow: /App_Browsers/
Disallow: /App_config/
Disallow: /App_Data/
Disallow: /sitecore
Disallow: /Sitecore
Disallow: /sitecore_files/
Disallow: /sitecore modules/
Disallow: /temp/
Disallow: /upload/
Disallow: /xsl/
Disallow: /-/media/

User-agent: NeevaBot
Disallow: /App_Browsers/
Disallow: /App_config/
Disallow: /App_Data/
Disallow: /sitecore
Disallow: /Sitecore
Disallow: /sitecore_files/
Disallow: /sitecore modules/
Disallow: /temp/
Disallow: /upload/
Disallow: /xsl/
Disallow: /-/media/

User-agent: YouBot
Disallow: /App_Browsers/
Disallow: /App_config/
Disallow: /App_Data/
Disallow: /sitecore
Disallow: /Sitecore
Disallow: /sitecore_files/
Disallow: /sitecore modules/
Disallow: /temp/
Disallow: /upload/
Disallow: /xsl/
Disallow: /-/media/

User-agent: facebookexternalhit
Disallow: /App_Browsers/
Disallow: /App_config/
Disallow: /App_Data/
Disallow: /sitecore
Disallow: /Sitecore
Disallow: /sitecore_files/
Disallow: /sitecore modules/
Disallow: /temp/
Disallow: /upload/
Disallow: /xsl/
Disallow: /-/media/

User-agent: Facebot
Disallow: /App_Browsers/
Disallow: /App_config/
Disallow: /App_Data/
Disallow: /sitecore
Disallow: /Sitecore
Disallow: /sitecore_files/
Disallow: /sitecore modules/
Disallow: /temp/
Disallow: /upload/
Disallow: /xsl/
Disallow: /-/media/

# Block all other crawlers
User-agent: *
Disallow: /


Sitemap: https://ljmu.ac.uk/sitemap.xml