# -- Sitemaps --
Sitemap: https://www.kbauthority.com/sitemap.xml
Sitemap: https://www.kbauthority.com/news/sitemap_index.xml

# This directive will be effective for bots that support it, like CCBot.
User-agent: CCBot
Crawl-Delay: 10

# --- The following directives will likely be IGNORED by the bots ---
# --- Use server-side rate limiting or Google Search Console instead ---

# OpenAI's AI Bot (will ignore Crawl-Delay)
User-agent: GPTBot
Crawl-Delay: 10

# Google's AI Bot (will ignore Crawl-Delay)
User-agent: Google-Extended
Crawl-Delay: 10

# Anthropic's AI Bot (will ignore Crawl-Delay)
User-agent: ClaudeBot
Crawl-Delay: 10

# Perplexity's AI Bot (will ignore Crawl-Delay)
User-agent: PerplexityBot
Crawl-Delay: 10

# Block the Chinese Baidu spider, as its not of any usage in this region of the world
User-agent: YandexBot
User-agent: Amazonbot
Disallow: /

#Slow crawler rate based on importance
User-agent: *
Allow: *.js
Allow: *.css
Allow: *.jpg
Allow: *.jpeg
Allow: *.gif
Allow: *.png
Allow: *.pdf
Allow: *.html
Allow: *.xml
Disallow: /*printable=Y*
Disallow: /*js=*
Disallow: /*print_cat=*
Disallow: /*mode=add_vote*
Disallow: */get_block.php
Disallow: */get_info.php

# Directories
Disallow: /admin/
Disallow: /test_old/
Disallow: /test/
Disallow: /dev_store/
Disallow: /new_dev/
Disallow: /catalog/
Disallow: /customer/
Disallow: /include/
Disallow: /mail/
Disallow: /modules/
Disallow: /partner/
Disallow: /payment/
Disallow: /provider/
Disallow: /schemes/
Disallow: /shipping/
Disallow: /skin/
Disallow: /skin_backup/
Disallow: /sql/
Disallow: /upgrade/
Disallow: /var/
# Files

Disallow: /CHANGELOG
Disallow: /COPYRIGHT
Disallow: /INSTALL
Disallow: /NEW-4.5.0
Disallow: /README
Disallow: /VERSION
Disallow: /shop_closed_evaluation.html
Disallow: /TERMSOFSERVICE.CLOUD_SEARCH
Disallow: /VERSION.CLOUD_SEARCH

Disallow: /404.php
Disallow: /adaptive.php
Disallow: /address_book.php
Disallow: /adv_counter.php
Disallow: /amazon_checkout.php
Disallow: /antibot_image.php
Disallow: /auth.php
Disallow: /banner.php
Disallow: /bonuses.php
Disallow: /cart.php
Disallow: /change_password.php
Disallow: /check_requirements.php
Disallow: /choosing.php
Disallow: /cleanup.php
Disallow: /cloud_search_api.php
Disallow: /cmpi_popup.php
Disallow: /comparison.php
Disallow: /comparison_list.php
Disallow: /config.php
Disallow: /cron.php
Disallow: /dispatcher.php
Disallow: /download.php
Disallow: /error_message.php
Disallow: /featured_products.php
Disallow: /flash_container.swf
Disallow: /giftcert.php
Disallow: /giftreg_manage.php
Disallow: /giftregs.php
Disallow: /https.php
Disallow: /image.php
Disallow: /init.php
Disallow: /install.php
Disallow: /login.php
Disallow: /magnifier_xml.php
Disallow: /message.html
Disallow: /minicart.php
Disallow: /news.php
Disallow: /offers.php
Disallow: /order.php
Disallow: /orders.php
Disallow: /pages.php
Disallow: /pconf.php
Disallow: /permission_denied.html
Disallow: /popup_address.php
Disallow: /popup_ask.php
Disallow: /popup_cookie_settings.php
Disallow: /popup_edit_label.php
Disallow: /popup_estimate_shipping.php
Disallow: /popup_fc_products.php
Disallow: /popup_image.php
Disallow: /popup_info.php
Disallow: /popup_magnifier.php
Disallow: /popup_poptions.php
Disallow: /postauth.php
Disallow: /preauth.php
Disallow: /prepare.php
Disallow: /preview_image.gif
Disallow: /probe.php
Disallow: /process_order.php
Disallow: /products.php
Disallow: /products_map.php
Disallow: /recommends.php
Disallow: /referer.php
Disallow: /register.php
Disallow: /returns.php
Disallow: /saved_cards.php
Disallow: /search.php
Disallow: /send_to_friend.php
Disallow: /shop_closed.html
Disallow: /shop_registration.php
Disallow: /slabel.php
Disallow: /smarty.php
Disallow: /survey.php
Disallow: /top.inc.php
Disallow: /vote.php
Disallow: /wysiwyg.php
Disallow: /product_review.php

# Filters
# Paths (no clean URLs)
Disallow: /*?SID=
Disallow: /*?p=*
Disallow: /*?p=*
Disallow: /*?dir=*
Disallow: /*?order=*
Disallow: /search/
Disallow: /*?q=*