# AI/bot instructions
User-agent: GPTBot
Allow: /

User-agent: OpenAI-SearchBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Google-Extended
Allow: /

# Default instructions:
User-agent: *

Disallow: /lib/
Disallow: /*.php$
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /catalog/
Disallow: /customer/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /*SID=

#If any FFD URIs have index.php within them - something is broken.
Disallow: /*index.php*

# Disallow common Magento files in the root directory:
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt

# Disallow User Account & Checkout URIs:
Disallow: /checkout/
Disallow: /onestepcheckout/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/

Disallow: /weltpixel_ga4/

# Disallow Catalog Search URIs:
Disallow: /catalogsearch/
Disallow: /catalog/product_compare/
Disallow: /catalog/category/view/
Disallow: /catalog/product/view/

# Disallow URL Filter Searches
Disallow: /*?*dir*
Disallow: /*?*dir=desc
Disallow: /*?*dir=asc
Disallow: /*?*limit=all
Disallow: /*?*mode*

# Block all multi-parameter URLs
Disallow: /*&*

# Block volatile / low-signal parameters
Disallow: /*?*dir*
Disallow: /*?*mode*
Disallow: /*?*limit*
Disallow: /*?*product_list*
Disallow: /*?*SID*
Disallow: /*?*utm_*
Disallow: /*?*gclid*
Disallow: /*?*fbclid*

# Allow pagination
Allow: /*?p=*

# Allow single-filter, high-value facets
Allow: /*?*filt_brand=*
Allow: /*?*width=*
Allow: /*?*height=*
Allow: /*?*depth=*
Allow: /*?*ffd_door_type=*
Allow: /*?*ffd_output=*
Allow: /*?*ffd_ice_type=*
Allow: /*?*ffd_warranty_cover=*
Allow: /*?*conf_inc_floor=*
Allow: /*?*special_price=*
Allow: /*?*category_ids=*

Allow: /amfeed/feed/*

# Disallow all filter urls as a base rule
Disallow: /*filterby/*

# Disallow CMS Directories:
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/

# Disallow Duplicate Content:
Disallow: /tag/
Disallow: /review/
Disallow: /*?*product_list_mode=
Disallow: /*?*product_list_order=
Disallow: /*?*product_list_limit=
Disallow: /*?*product_list_dir=

# Server Settings
# Disallow general technical directories and files on a server
Disallow: /cgi-bin/
Disallow: /cleanup.php
Disallow: /apc.php
Disallow: /memcache.php
Disallow: /phpinfo.php

# Disallow version control folders and others
Disallow: /*.git
Disallow: /*.CVS
Disallow: /*.Zip$
Disallow: /*.Svn$
Disallow: /*.Idea$
Disallow: /*.Sql$
Disallow: /*.Tgz$

#Disallow FFD files with these extensions
Disallow: /*index.html$
Disallow: /*.php$

#Disallows
User-agent: aranhabot
User-agent: Yandex
User-agent: YandexBot
User-agent: YandexBot/3.0
User-agent: Baidu
User-agent: Baiduspider
User-agent: baiduspider
User-agent: BaiduSpider
User-agent: Baiduspider/2.0
User-agent: Baiduspider+
User-agent: Baiduspider-video
User-agent: Baiduspider-image
User-agent: Soso
User-agent: Twengabot
User-agent: TwengaBot
User-agent: TwengaBot-2.0
User-agent: SemrushBot
User-agent: SemrushBot-SI
User-agent: DeuSu
User-agent: JamesBOT
User-agent: Riddler
User-agent: MJ12bot
User-agent: Vagabondo
User-agent: Vagabondo
User-agent: Vagabondo/4.0
User-agent: Qwantify
User-agent: Qwantify/2.2w
User-agent: Dataprovider
User-agent: Uptimebot
User-agent: UptimeRobot/2.0
User-agent: Exabot
User-agent: MegaIndex.ru/2.0
User-agent: sogou spider
User-agent: BLEXBot
User-agent: SEOkicks-Robot
User-agent: Ezooms Robot
User-agent: WiseGuys Robot
User-agent: Turnitin Robot
User-agent: Heritrix
User-agent: pimonster
User-agent: SearchmetricsBot
User-agent: YoudaoBot
User-agent: SafeDNSBot
User-agent: SeznamBot
User-Agent: CSS Certificate Spider 
User-agent: BUbiNG
User-agent: ltx71
User-agent: CCBot/2.0
User-agent: linkdexbot/2.0
User-agent: PrivateSearch/0.1.0
User-agent: Slackbot-LinkExpanding 1.0
User-agent: Slack-ImgProxy 0.19
User-agent: Slackbot 1.0
User-agent: Slackbot
User-agent: Nmap Scripting Engine
User-agent: imagecoccoc/1.0
User-agent: imagecoccoc
User-agent: Genieo/1.0
User-agent: Genieo
User-agent: CukBot
User-agent: grapeshot
User-agent: proximic
Disallow: / 

Sitemap: https://www.fridgefreezerdirect.co.uk/media/sitemap/ffd_sitemap.xml
Sitemap: https://www.fridgefreezerdirect.co.uk/ai-sitemap.xml

User-agent: SeekportBot
Crawl-delay: 2
Sitemap: https://www.fridgefreezerdirect.co.uk/media/sitemap/ffd_info_centre_sitemap.xml