# ============================================================
# robots.txt for sohomod.com
# Last updated: April 11 2026
# ============================================================

# ============================================================
# SITEMAPS
# ============================================================

Sitemap: https://www.sohomod.com/main_cms.xml
Sitemap: https://www.sohomod.com/main_cat.xml
Sitemap: https://www.sohomod.com/main_prod.xml
Sitemap: https://www.sohomod.com/main_articles.xml
Sitemap: https://www.sohomod.com/feeds/sitemap_images_1.xml
Sitemap: https://www.sohomod.com/feeds/sitemap_images_2.xml
Sitemap: https://www.sohomod.com/feeds/sitemap_video_1.xml
Sitemap: https://www.sohomod.com/short_sitemap.xml

# ============================================================
# GLOBAL RULES
# ============================================================

User-agent: *

# System & server files
Disallow: /404/
Disallow: /app/
Disallow: /cgi-bin/
Disallow: /downloader/
Disallow: /includes/
Disallow: /lib/
Disallow: /pkginfo/
Disallow: /var/
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt

# Magento internal/technical paths
Disallow: /catalog/product_compare/
Disallow: /checkout/
Disallow: /control/
Disallow: /customer/
Disallow: /newsletter/
Disallow: /poll/
Disallow: /sendfriend/
Disallow: /wishlist/

# Ajax & dynamic UI paths
Disallow: /ajaxpage/
Disallow: /quickview/index/
Disallow: /fpc/
Disallow: /searchsphinx/

# Internal search (not for indexing)
Disallow: /catalogsearch/
Disallow: /search/

# Filtered/sorted URLs (parameter-based — surgical blocks only)
Disallow: /*?SID=
Disallow: /*?p=*&
Disallow: */sort-by/*
Disallow: */show/*

# Allow everything else (including /contacts/, /reviews/, /tag/)

# ============================================================
# GOOGLE MAIN CRAWLER
# ============================================================

User-agent: Googlebot
Disallow: /catalog/product_compare/
Disallow: /catalogsearch/
Disallow: /search/
Disallow: /ajaxpage/
Disallow: /quickview/index/
Disallow: /fpc/
Disallow: /searchsphinx/
Disallow: /checkout/
Disallow: /customer/
Disallow: /wishlist/
Disallow: */sort-by/*
Disallow: /*?SID=
Disallow: /*?p=*&

# ============================================================
# GOOGLE IMAGE CRAWLER
# ============================================================

User-agent: Googlebot-image
Disallow: /catalog/product_compare/
Disallow: /catalogsearch/
Disallow: /search/
Disallow: /ajaxpage/
Disallow: /quickview/index/
Disallow: /fpc/
Disallow: /searchsphinx/
Disallow: /checkout/
Disallow: /customer/
Disallow: /wishlist/
Disallow: */sort-by/*
Disallow: */show/*

# ============================================================
# GOOGLE ADS BOT
# ============================================================

User-agent: AdsBot-Google
Allow: /feeds/
Allow: /catalog/
Disallow: /ajaxpage/
Disallow: /checkout/
Disallow: /customer/

# ============================================================
# GOOGLE OTHER
# ============================================================

User-agent: Google-Other
Crawl-delay: 5
Disallow: /ajaxpage/
Disallow: /checkout/
Disallow: /customer/
Disallow: /fpc/

# ============================================================
# AI BOTS SETUP
# ============================================================

User-agent: GPTBot
Crawl-delay: 10
Disallow: /ajaxpage/
Disallow: /catalog/product_compare/
Disallow: /checkout/
Disallow: /customer/
Disallow: /fpc/

User-agent: OAI-SearchBot
Crawl-delay: 10
Disallow: /ajaxpage/
Disallow: /catalog/product_compare/
Disallow: /catalogsearch/
Disallow: /checkout/
Disallow: /customer/
Disallow: /fpc/
Disallow: /wishlist/
Disallow: */sort-by/*

User-agent: ClaudeBot
Crawl-delay: 10
Disallow: /ajaxpage/
Disallow: /catalog/product_compare/
Disallow: /checkout/
Disallow: /customer/
Disallow: /fpc/

User-agent: meta-externalagent
Crawl-delay: 10
Disallow: /ajaxpage/
Disallow: /catalog/product_compare/
Disallow: /checkout/
Disallow: /customer/
Disallow: /fpc/

User-agent: PerplexityBot
Crawl-delay: 10
Disallow: /ajaxpage/
Disallow: /catalog/product_compare/
Disallow: /checkout/
Disallow: /customer/
Disallow: /fpc/

User-agent: Applebot
Crawl-delay: 10
Disallow: /ajaxpage/
Disallow: /catalog/product_compare/
Disallow: /checkout/
Disallow: /customer/
Disallow: /fpc/

User-agent: YouBot
Crawl-delay: 10
Disallow: /ajaxpage/
Disallow: /checkout/
Disallow: /customer/
Disallow: /fpc/

User-agent: CCBot
Crawl-delay: 10
Disallow: /ajaxpage/
Disallow: /catalog/product_compare/
Disallow: /checkout/
Disallow: /customer/
Disallow: /fpc/

User-agent: Diffbot
Crawl-delay: 10
Disallow: /ajaxpage/
Disallow: /catalog/product_compare/
Disallow: /checkout/
Disallow: /customer/
Disallow: /fpc/

# ============================================================
# SEO BOTS SETUP
# ============================================================
User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /ajaxpage/
Disallow: /catalog/product_compare/
Disallow: /checkout/
Disallow: /customer/
Disallow: /fpc/

User-agent: SemrushBot
Crawl-delay: 10
Disallow: /ajaxpage/
Disallow: /catalog/product_compare/
Disallow: /checkout/
Disallow: /customer/
Disallow: /fpc/

# ============================================================
# BLOCK AGGRESSIVE SCRAPER BOTS
# ============================================================
User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /