Top Sites>Bring structure to your research - protocols.io

# =========================================================
# robots.txt for https://www.protocols.io
# Purpose:
# - Allow discovery of public scientific content
# - Protect private, authenticated, and system areas
# - Provide explicit guidance to AI crawlers
# =========================================================

# -------------------------
# Default rule (all crawlers)
# -------------------------
User-agent: *
Disallow: /private/
Disallow: /blind/
Disallow: /api/
Disallow: /download
Disallow: /pubchase
Disallow: /spectro
Disallow: /neb
Disallow: /career/
Disallow: /essays
Disallow: /editorials
Disallow: /test
Disallow: /flux

# AI crawlers
User-agent: GPTBot
Disallow: /private/
Disallow: /api/

User-agent: anthropic-ai
Disallow: /private/
Disallow: /api/

User-agent: CCBot
Disallow: /private/
Disallow: /api/

# -------------------------
# Sitemap
# -------------------------
Sitemap: https://www.protocols.io/sitemaps/sitemap.xml

# llms.txt for https://www.protocols.io

# Public research content:
# Public protocols, documentation, and static resources are available for indexing,
# training, and inference. Users and AI agents may use this content for
# generative responses provided they respect the licensing terms.

allow: /view/
allow: /help/
allow: /about/
allow: /sitemaps/

# Provide structured content for better discoverability
preferred_canonical: https://www.protocols.io

# AI training guidance:
# Public protocols on this site are published under Creative Commons or equivalent
# open terms. AI systems training on this content should:
# - respect licenses and citations when generating responses
# - explicitly cite protocol titles and DOIs when feasible

# Disallowed content for LLM access:
# Private, authenticated, or user-specific content should not be accessed, indexed,
# or used for training or inference.

# Metadata and API guidance:
# If using the public API for content ingestion, only allow endpoints returning
# public protocols; respect API rate limits and authentication requirements.

# Attribution guidance:
# When outputs include information drawn from protocols.io content, include:
# "Source: protocols.io protocol [Title], DOI: dx.doi.org/..."