User-agent: *
# Directories
Disallow: /assets/
Disallow: /au/
Disallow: /cache/
Disallow: /iem/
Disallow: /images/
Disallow: /media/
Disallow: /nz/
Disallow: /phpMyAdmin/
Disallow: /sg/
Disallow: /support/
Disallow: /system/
Disallow: /themes/
Disallow: /uk/
Disallow: /webp/
Disallow: /forums/startTopic/
Disallow: /forums/discover/unread/
Disallow: /forums/markallread/
Disallow: /forums/staff/
Disallow: /forums/cookies/
Disallow: /forums/online/
Disallow: /forums/discover/
Disallow: /forums/leaderboard/
Disallow: /forums/search/
Disallow: /forums/tags/
Disallow: /forums/*?advancedSearchForm=
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/login/
Disallow: /forums/*currency=

# Block faceted pages and 301 redirect pages
Disallow: /forums/*?sortby=
Disallow: /forums/*?filter=
Disallow: /forums/*?tab=
Disallow: /forums/*?do=
Disallow: /forums/*ref=
Disallow: /forums/*?forumId*
Disallow: /forums/*?&controller=embed

# Block profile pages as these have little unique value, consume a lot of crawl time and contain hundreds of 301 links
Disallow: /forums/profile/

# Files
Disallow: /CHANGELOG.txt

Sitemap: https://www.stereonet.com/sitemap.xml
Sitemap: https://www.stereonet.com/news-sitemap.xml
Sitemap: https://www.stereonet.com/forums/sitemap.php

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

# =====================================================
# AI crawler policy
# Training / bulk-scraping crawlers: disallow.
# Live AI-search / assistant bots (cite sources, drive
# referral traffic): allow, with the same exclusions as
# the generic User-agent: * block above.
# =====================================================

# --- AI training & bulk scraping: blocked ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Kangaroo Bot
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

# --- AI search & assistant bots: allowed ---
# These fetch pages to answer user queries and cite sources back to StereoNET.

User-agent: OAI-SearchBot
Disallow: /assets/
Disallow: /cache/
Disallow: /images/
Disallow: /media/
Disallow: /system/
Disallow: /themes/
Disallow: /webp/
Disallow: /forums/login/
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/profile/
Disallow: /forums/search/
Disallow: /forums/staff/
Disallow: /CHANGELOG.txt
Allow: /

User-agent: ChatGPT-User
Disallow: /assets/
Disallow: /cache/
Disallow: /images/
Disallow: /media/
Disallow: /system/
Disallow: /themes/
Disallow: /webp/
Disallow: /forums/login/
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/profile/
Disallow: /forums/search/
Disallow: /forums/staff/
Disallow: /CHANGELOG.txt
Allow: /

User-agent: PerplexityBot
Disallow: /assets/
Disallow: /cache/
Disallow: /images/
Disallow: /media/
Disallow: /system/
Disallow: /themes/
Disallow: /webp/
Disallow: /forums/login/
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/profile/
Disallow: /forums/search/
Disallow: /forums/staff/
Disallow: /CHANGELOG.txt
Allow: /

User-agent: Perplexity-User
Disallow: /assets/
Disallow: /cache/
Disallow: /images/
Disallow: /media/
Disallow: /system/
Disallow: /themes/
Disallow: /webp/
Disallow: /forums/login/
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/profile/
Disallow: /forums/search/
Disallow: /forums/staff/
Disallow: /CHANGELOG.txt
Allow: /

User-agent: Claude-SearchBot
Disallow: /assets/
Disallow: /cache/
Disallow: /images/
Disallow: /media/
Disallow: /system/
Disallow: /themes/
Disallow: /webp/
Disallow: /forums/login/
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/profile/
Disallow: /forums/search/
Disallow: /forums/staff/
Disallow: /CHANGELOG.txt
Allow: /

User-agent: Claude-User
Disallow: /assets/
Disallow: /cache/
Disallow: /images/
Disallow: /media/
Disallow: /system/
Disallow: /themes/
Disallow: /webp/
Disallow: /forums/login/
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/profile/
Disallow: /forums/search/
Disallow: /forums/staff/
Disallow: /CHANGELOG.txt
Allow: /

User-agent: DuckAssistBot
Disallow: /assets/
Disallow: /cache/
Disallow: /images/
Disallow: /media/
Disallow: /system/
Disallow: /themes/
Disallow: /webp/
Disallow: /forums/login/
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/profile/
Disallow: /forums/search/
Disallow: /forums/staff/
Disallow: /CHANGELOG.txt
Allow: /

User-agent: Meta-ExternalFetcher
Disallow: /assets/
Disallow: /cache/
Disallow: /images/
Disallow: /media/
Disallow: /system/
Disallow: /themes/
Disallow: /webp/
Disallow: /forums/login/
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/profile/
Disallow: /forums/search/
Disallow: /forums/staff/
Disallow: /CHANGELOG.txt
Allow: /

User-agent: YouBot
Disallow: /assets/
Disallow: /cache/
Disallow: /images/
Disallow: /media/
Disallow: /system/
Disallow: /themes/
Disallow: /webp/
Disallow: /forums/login/
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/profile/
Disallow: /forums/search/
Disallow: /forums/staff/
Disallow: /CHANGELOG.txt
Allow: /

User-agent: Google-Extended
Disallow: /assets/
Disallow: /cache/
Disallow: /images/
Disallow: /media/
Disallow: /system/
Disallow: /themes/
Disallow: /webp/
Disallow: /forums/login/
Disallow: /forums/register/
Disallow: /forums/lostpassword/
Disallow: /forums/profile/
Disallow: /forums/search/
Disallow: /forums/staff/
Disallow: /CHANGELOG.txt
Allow: /