# robots.txt for u-boot.org
# The Elixir cross-reference site has millions of pages across
# hundreds of versions. Limit crawling to reduce server load.

User-agent: *

# Identifier cross-reference pages are a crawl trap
# (millions of unique URLs, not useful for search)
Disallow: /u-boot/*/C/ident/
Disallow: /u-boot/*/K/ident/
Disallow: /concept/*/C/ident/
Disallow: /concept/*/K/ident/

# Ancient versions (pre-2020) - no need to index
Disallow: /u-boot/LABEL_
Disallow: /u-boot/v200
Disallow: /u-boot/v201
Disallow: /concept/c200
Disallow: /concept/c201

# WordPress internals
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-json/
Disallow: /xmlrpc.php
Allow: /wp-admin/admin-ajax.php

# Mailing list archives - signup/auth pages
Disallow: /accounts/

Crawl-delay: 2