# robots.txt for https://www.chefscomplements.co.nz/

# Allow all crawlers, including AI agents
User-agent: *
Disallow: /wp-admin/
# Do NOT disallow /wp-includes/ or /wp-content/cache/ - crawlers need these
# to fetch JS/CSS (jQuery, theme scripts, minified assets) for rendering.
Disallow: /cgi-bin/
Disallow: /temp/
Disallow: /private/
Disallow: /backend/

# Allow uploads (images, videos, docs, etc.)
Allow: /wp-content/uploads/
# Allow cached/minified front-end assets (JS, CSS) so pages can be rendered
Allow: /wp-content/cache/
Allow: /wp-includes/
Allow: /wp-admin/admin-ajax.php

# Optional: block duplicate or thin content
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /category/*/*
Disallow: */trackback/
Disallow: */feed/
Disallow: */comments/
Disallow: /*?replytocom*

# Remove file extension disallows so CSS, JS, XML, and other resources remain crawlable

# Sitemap reference
Sitemap: https://www.chefscomplements.co.nz/sitemap_index.xml