# Specific Permissions for AI Bots
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: ClaudeBot
Allow: /

# Crawlers Setup
User-agent: *
Crawl-delay: 5

# Allowable Index
Allow: /*?p=
Allow: /media/
Allow: /llms.txt

# Directories
Disallow: /app/
Disallow: /cgi/
Disallow: /cgi-bin/
Disallow: /dev/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /js/
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /Primer3input/
Disallow: /report/
Disallow: /skin/
Disallow: /shell/
Disallow: /stats/
Disallow: /SpryAssets/
Disallow: /TABS/
Disallow: /TMP/
Disallow: /var/
Disallow: /vendor/
Disallow: /.git/
Disallow: /.sucuriquarantine/
Disallow: /ACCORDION/
Disallow: /*?amp=1

# Paths (clean URLs)
Disallow: /*catalog/product_compare/
Disallow: /*catalogsearch/
Disallow: /*search/
Disallow: /*?SID=
Disallow: /*?PHPSESSID=
Disallow: /*checkout/
Disallow: /*control/
Disallow: /*contacts/
Disallow: /*customer/
Disallow: /*customize/
Disallow: /*newsletter/
Disallow: /*poll/
Disallow: /*review/
Disallow: /*sendfriend/
Disallow: /*tag/
Disallow: /*wishlist/
Disallow: /?keyword=
Disallow: /?videoID=
Disallow: /*productalert/
Disallow: /*?limit=
Disallow: /*&limit=
Disallow: /*&dir=
Disallow: /*?dir=
Disallow: /*?order=
Disallow: /*&order=
Disallow: /*stores/store/redirect/
Disallow: /*genes/index/index/

# Files
Disallow: /.htaccess
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /biocompare_feed_9d754.php
Disallow: /composer.json
Disallow: /google5c598a02a125a6fc.html
Disallow: /RELEASE_NOTES.txt
Disallow: /sucuri-70f9852742531c25f83ea14231fbc9dc.php
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt

# Paths (no clean URLs)
Disallow: /*.js$
Disallow: /*.css$
Disallow: /*.php$
Disallow: /*?p=*&
Disallow: /*?p=*$
Disallow: /*?SID=

# Disallow following robots
User-agent: coccocbot
User-agent: Sogou
User-agent: SemrushBot
User-agent: Mail.RU_Bot
User-agent: Mail.Ru
User-agent: Applebot
User-agent: Cliqzbot
User-agent: Lawinsiderbot
User-agent: SemanticScholarBot
User-agent: ZoominfoBot
User-agent: DotBot
User-agent: HubSpot
User-agent: AhrefsBot
User-agent: SEOkicks-Robot
User-agent: SISTRIX Crawler
User-agent: Ezooms Robot
Disallow: /


Sitemap: https://www.sciencellonline.com/sitemap.xml
Sitemap: https://sciencellonline.com/llms.txt