# Website Robots.txt - Optimized & Clean

# Sitemap Declaration
Sitemap: https://www.truefacet.com/sitemap.xml
Sitemap: https://www.truefacet.com/index-sitemap.xml
Sitemap: https://www.truefacet.com/prod-sitemap.xml
Sitemap: https://www.truefacet.com/cat-sitemap.xml


# ================================
# Default Rules for All Crawlers
# ================================
User-agent: *
Crawl-delay: 1

# Allow Essential Files
Allow: /robots.txt
Allow: /llms.txt
Allow: /sitemap.xml
Allow: /css/
Allow: /js/
Allow: /images/
Allow: /fonts/

# Disallow Sensitive Directories
Disallow: /admin_tf/
Disallow: /admin/
Disallow: /api/
Disallow: /private/
Disallow: /tmp/
Disallow: /customer/
Disallow: /customer/account/login/
Disallow: /customer/account/create/
Disallow: /checkout/
Disallow: /cart/
Disallow: /checkout/cart/
Disallow: /wishlist/
Disallow: /dashboard/

# Disallow Duplicate Parameter URLs
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /*?price=
Disallow: /*&price=
Disallow: /*?filter=
Disallow: /*&filter/


# ================================
# AI Crawlers
# ================================
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: CCBot
Crawl-delay: 1


# ================================
# Block Problematic Bots
# ================================
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /