#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    https://example.com/robots.txt
# Ignored: https://example.com/site/robots.txt
#
# ---------------------------------------------------------------------------
# AI ANSWER ENGINE & LLM CRAWLERS — Explicitly allowed
# ---------------------------------------------------------------------------
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: Amazonbot
Allow: /
User-agent: Applebot-Extended
Allow: /
User-agent: CCBot
Allow: /
User-agent: Bytespider
Disallow: /
# ---------------------------------------------------------------------------
# GENERAL CRAWLERS
# ---------------------------------------------------------------------------
User-agent: *
Allow: /
User-agent: Googlebot
User-agent: Googlebot-image
Disallow: /new-in/*
Disallow: /pre-launch
Disallow: /*/pre-launch
Disallow: /*/talents
Disallow: /*/success-story
Disallow: /*/office-furniture-bulk-orders
Disallow: /*/business-offer
Disallow: /*/reseller-program
Disallow: /*/influencer-review
Disallow: /*/press-feature
Disallow: /*/track-your-order
Disallow: /*/share-and-earn-money
Disallow: /*/QC-*
Disallow: /QC-*
Disallow: /anon/
Allow: /p/office-furniture-bulk-orders
# ---------------------------------------------------------------------------
# SPAM / SEO TOOL CRAWLERS — Blocked
# ---------------------------------------------------------------------------
User-agent: Yandex
Disallow: /
User-agent: ichiro
Disallow: /
User-agent: moget
Disallow: /
User-agent: NaverBot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: Sogou
Disallow: /
User-agent: YoudaoBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: SeznamBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: ltx71
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: Spiderbot
Disallow: /
User-agent: majestic12
Disallow: /
User-agent: scoutjet
Disallow: /
User-agent: changedetection
Disallow: /
User-agent: linkfluence
Disallow: /
User-agent: mj12bot
Disallow: /
User-agent: mojeek
Disallow: /
User-agent: aihitdata
Disallow: /
User-agent: wotbox
Disallow: /
User-agent: mediatoolkit
Disallow: /
User-agent: safedns
Disallow: /
User-agent: aboundex
Disallow: /
User-agent: webmeup-crawler
Disallow: /
User-agent: orangebot
Disallow: /
# ---------------------------------------------------------------------------
# INTERNAL / DEV PATHS — Blocked for all
# ---------------------------------------------------------------------------
User-agent: *
Disallow: /api
Disallow: /dev.
# ---------------------------------------------------------------------------
# SITEMAP
# ---------------------------------------------------------------------------
Sitemap: https://www.autonomous.ai/sitemap.xml