User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Crawl-delay: 10
Disallow: /page/*/?s=
Disallow: /search/
Disallow: /wp-json/

User-agent: Amazonbot
Crawl-delay: 5
Disallow: /parts*
Disallow: /products*
Disallow: /

User-agent: Rogerbot
Crawl-delay: 5
Disallow: /parts*
Disallow: /products*
Disallow: /

User-agent: Petalbot
Crawl-delay: 5
Disallow: /parts*
Disallow: /products*
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: Twitterbot
Crawl-delay: 300

User-agent: Googlebot
Crawl-delay: 10

User-agent: Bingbot
Crawl-delay: 180

User-agent: DotBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: meta-externalagent
Disallow: /

Sitemap: https://infrastructurereportcard.org/sitemap.xml