Top Sites>Insight | Insight Enterprises | Insight

# Robots.txt for Insight.com

# Default rules
User-agent: *
Disallow: /*?*
Allow: /*.html
Disallow: /en_US/search*.html
Disallow: /insightweb/
Disallow: /flytrap/
Disallow: /content/dam/insight-web/*/solutions/service-provider/microsite/assets/
Disallow: /content/dam/insight-web/*/pdfs/
Disallow: /content/dam/insight/*/
Disallow: /content/dam/global/*/pdfs/
Allow: /insightweb/*.css$
Allow: /*?qtype=
Allow: /*?pq=
Allow: /*?identifier=shopping
Allow: /*?partnermessage

# Allowed AI crawlers
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: Google-Extended
User-agent: anthropic-ai
User-agent: Bingbot
User-agent: Googlebot
User-agent: PerplexityBot
User-agent: YouBot
Disallow:

# Blocked crawlers
User-agent: CCBot
User-agent: FacebookBot
User-agent: NeevaAI
Disallow: /


Sitemap: https://www.insight.com/sitemap.xml

# Allow OpenAI's GPT models (e.g. ChatGPT, GPT-4o) — used in enterprise procurement, integrations, summarization
User-Agent: gptbot
Allow: /

# Allow Google's Gemini (via Google-Extended) — used in Google AI Overviews, Workspace integrations
User-Agent: Google-Extended
Allow: /

# Allow Anthropic Claude (Sonnet/Haiku) — growing enterprise usage for AI summaries and safe content parsing
User-Agent: anthropic-ai
Allow: /

# Allow Meta's LLaMA (LLaMA 2/3/4) — top open-source model adopted by large organizations
User-Agent: meta-llama
Allow: /

# Allow Perplexity — AI-powered search engine increasingly used by IT and procurement managers
User-Agent: perplexitybot
Allow: /

# Allow Cohere — LLM provider focused on enterprise document embedding and retrieval
User-Agent: cohere-ai
Allow: /

# Allow AI21 Labs — known for enterprise use and structured text generation
User-Agent: ai21labs
Allow: /

# Allow IBM Granite — IBM's trusted enterprise-grade LLM, embedded into Watsonx for services/solutions
User-Agent: ibm-granite
Allow: /

# Allow Mistral — highly efficient open-source LLMs being adopted for hybrid infrastructure deployments
User-Agent: mistral
Allow: /

# Allow Hugging Face — serves Falcon, BLOOM, and many enterprise open LLMs
User-Agent: huggingface
Allow: /

# Allow Aleph Alpha — trusted in European enterprise AI deployments, good for multilingual contexts
User-Agent: aleph-alpha
Allow: /

# Allow Writer — used in enterprise product description generation and ecommerce copy
User-Agent: writer
Allow: /

# Allow xAI's Grok — growing influence due to live data integration (esp. in B2B social ecosystems)
User-Agent: xai-grok
Allow: /

# Allow You.com AI assistant — often used in product discovery and ecommerce comparisons
User-Agent: yousearch
Allow: /

# Allow Claude web-crawler — another Anthropic signal (variant)
User-Agent: claude-web
Allow: /

# Allow LlamaIndex — framework used to connect private datasets (e.g. product catalogs) to LLMs
User-Agent: llama-index
Allow: /

# Allow OpenRouter — serves multiple top LLMs like Mixtral, Claude, GPT via API gateway
User-Agent: openrouter
Allow: /

# ========== DISALLOWED BELOW ==========

# Block ModelScope — mostly experimental Alibaba research models with limited western enterprise adoption
User-Agent: modelscope
Disallow: /

# Block Semantic Kernel — Microsoft orchestration framework, not intended as a crawler
User-Agent: semantic-kernel
Disallow: /

# Block Stability AI — more focused on image/video generation than enterprise ecommerce LLM use
User-Agent: stabilityai
Disallow: /