# =====================================================
# robots.txt for AILabTools.com
# Version: 2.1
# Updated: 2025-11-20
# =====================================================

User-agent: *
Allow: /
Crawl-delay: 5

Disallow: /signin
Disallow: /*/signin
Disallow: /signup
Disallow: /*/signup
Disallow: /header
Disallow: /*/header
Disallow: /footer
Disallow: /*/footer
Disallow: /price
Disallow: /*/price
Disallow: /support
Disallow: /*/support
Disallow: /account/
Disallow: /*/account/
Disallow: /user/
Disallow: /*/user/
Disallow: /_next/
Disallow: /*/_next/
Disallow: /Iframe/
Disallow: /ai-editor-api
Disallow: /ai-editor-api/
Disallow: /image-editor
Disallow: /image-editor/
Disallow: /image-editor-mobile
Disallow: /image-editor-mobile/
Disallow: /developer-platform
Disallow: /developer-platform/
Disallow: /affiliate-marketing-platform
Disallow: /affiliate-marketing-platform/

Sitemap: https://www.ailabtools.com/sitemap.xml
Sitemap: https://www.ailabtools.com/_sitemap.xml
Sitemap: https://www.ailabtools.com/docs/sitemap.xml?v=1.0.1
Sitemap: https://www.ailabtools.com/features/sitemap.xml
Sitemap: https://www.ailabtools.com/blog/post-sitemap.xml
Sitemap: https://www.ailabtools.com/doc/sitemap.xml

# ------------------------------
# AI / LLM Crawlers
# ------------------------------

# OpenAI / ChatGPT
User-agent: GPTBot
User-agent: ChatGPT-User

# Anthropic Claude
User-agent: anthropic-ai
User-agent: Claude-Web

# Google AI / Gemini / Bard
User-agent: GoogleOther
User-agent: Google-Extended
User-agent: Google-LLM
User-agent: GoogleGenerativeAI
User-agent: Gemini-Bot

# Microsoft / Bing / Copilot
User-agent: Bingbot
User-agent: BingPreview
User-agent: MS-Copilot
User-agent: MSOfficeBot

# Apple Intelligence
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Apple-Intelligence

# Meta AI
User-agent: Meta-ExternalAgent
User-agent: MetaAI
User-agent: FacebookBot

# Amazon / Alexa AI
User-agent: Amazonbot
User-agent: AlexaBot

# ByteDance / TikTok / Douyin
User-agent: Bytespider
User-agent: ByteGPT
User-agent: TikTokBot

# Mistral AI
User-agent: MistralAI

# xAI Grok
User-agent: xAI-Grok

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-LLM

# You.com
User-agent: YouBot
User-agent: YouBot-LLM

# Neeva / Snowflake
User-agent: NeevaAI
User-agent: Snowflake-LLM

# Cohere
User-agent: cohere-ai

# Stability AI
User-agent: StabilityBot

# Replit AI
User-agent: Replit-AI

# Diffbot
User-agent: Diffbot

# HuggingFace
User-agent: HuggingFace-Bot
User-agent: HF-LLM

# Baidu ERNIE
User-agent: BaiduSpider
User-agent: ERNIE-Bot

# Alibaba Qwen
User-agent: AlibabaBot
User-agent: QwenBot

# Tencent Hunyuan
User-agent: TencentBot
User-agent: HunyuanBot

# 360 大模型
User-agent: 360Spider-LLM

# 知乎 / 小红书 / 搜索型AI
User-agent: Zhihu-Bot
User-agent: XiaohongshuBot
User-agent: KuaishouBot

# 其它常见 AI Data Crawlers
User-agent: CCBot
User-agent: DuckAssistBot
User-agent: PetalBot
User-agent: SeekrBot
User-agent: OpenSeekBot

# ------------------------------
# Allowed Paths
# ------------------------------
Allow: /
Allow: /blog/
Allow: /docs/
Allow: /llms.txt
Allow: /llms-full.txt

# ------------------------------
# Disallowed Paths
# ------------------------------
Disallow: /signin
Disallow: /*/signin
Disallow: /signup
Disallow: /*/signup
Disallow: /header
Disallow: /*/header
Disallow: /footer
Disallow: /*/footer
Disallow: /price
Disallow: /*/price
Disallow: /support
Disallow: /*/support
Disallow: /account/
Disallow: /*/account/
Disallow: /user/
Disallow: /*/user/
Disallow: /_next/
Disallow: /*/_next/
Disallow: /Iframe/
Disallow: /ai-editor-api
Disallow: /ai-editor-api/
Disallow: /image-editor
Disallow: /image-editor/
Disallow: /image-editor-mobile
Disallow: /image-editor-mobile/
Disallow: /developer-platform
Disallow: /developer-platform/
Disallow: /affiliate-marketing-platform
Disallow: /affiliate-marketing-platform/