# https://www.robotstxt.org/robotstxt.html

User-agent: Googlebot
Disallow:

User-agent: Bingbot
Crawl-delay: 3

User-agent: Yandex
Crawl-delay: 5

# AI Crawlers - allowed, 1 request per 5 seconds
User-agent: GPTBot
Crawl-delay: 5
Allow: /

User-agent: ClaudeBot
Crawl-delay: 5
Allow: /

# All other bots - 1 request per 10 seconds
User-agent: *
Crawl-delay: 10

Allow: /

Disallow: /aj/
Disallow: /ajax/
Disallow: /account/*
Disallow: /images/
Disallow: /includes/
Disallow: /payment/
Disallow: /user/login/
Disallow: /user/register/
Disallow: /user/forgotten_password/
Disallow: /user/reset_password/
Disallow: /user/suspended/
Disallow: /review/view/
Disallow: /review/add/
Disallow: /review/confirm_add/
Disallow: /review/reports/
Disallow: /review/vid_popup/
Disallow: /review/invite/
Disallow: /business/edit/
Disallow: /business/add/
Disallow: /business/flag/
Disallow: /business/emailme/
Disallow: /business/emailanother/
Disallow: /business/reports/
Disallow: /business/vcard/
Disallow: /business/map/
Disallow: /business/claim/
Disallow: /dispute/
Disallow: /report/
Disallow: /businesses/
Disallow: */claim
Disallow: /business/*/*/claim
Disallow: /user-profile/*
Disallow: /countries/*
Disallow: /countries/*/categories
Disallow: /countries/*/states
Disallow: /countries/*/states/*
Disallow: /countries/*/states/*/cities
Disallow: /countries/*/states/*/cities/*
Disallow: /business/*/*/preview

Sitemap: https://api.brownbook.net/sitemap.xml