User-agent: *
Crawl-delay: 5
Sitemap: https://www.1001cosmetice.ro/sitemap.xml
# No need to index this stuff
Disallow: /feed/
Disallow: /html/
Disallow: /scripts/
Disallow: /invoice
# Avoid duplicate content as it may do more harm than good
Disallow: /*?f=*
Disallow: /*?fb_xd_fragment
Disallow: /*?qty=*
Disallow: /*?sort_by=
Disallow: /q/
Disallow: /*price[min]=
Disallow: /*price[max]=
# Disallow UTM BigBear params
Disallow: /*2pau
Disallow: /*2ptt
Disallow: /*2ptu
Disallow: /*2prp
Disallow: /*2pdlst
# Disallow some bots we do not care about
User-agent: 008
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: Baiduspider
Disallow: /
User-agent: EtaoSpider
Disallow: /
User-agent: NPBot
Disallow: /
User-agent: Synthesio
Disallow: /
# Allow agentic-AI users
User-agent: Claude-User
User-agent: ChatGPT-User
User-agent: Perplexity-User
User-agent: OAI-SearchBot
Disallow: