# sitemaps
Sitemap: https://www.datathistle.com/sitemaps/index.xml.gz


# all
User-agent: *
Disallow:
Disallow: /details/
Disallow: /events/*.xml$
Disallow: /places/*.xml$
Disallow: /*/show:*/
Disallow: /*/sort:*/
Disallow: /articles/*/what:*/
Disallow: /articles/what:*/page:*/
Disallow: /events/*/what:*/
Disallow: /events/what:*/page:*/
Disallow: /listings/*/what:*/
Disallow: /listings/what:*/page:*/
Disallow: /member/
# Disallow: /places/*/what:*/
Disallow: /places/what:*/page:*/
# Disallow: /*/where:*/
Disallow: /sign-in/
Disallow: /update/*/*/
Disallow: /js/
Disallow: /*/*/*/*/*/*/


# Apple
User-agent: Applebot
Disallow:
Disallow: /details/
Disallow: /events/*.xml$
Disallow: /places/*.xml$
Disallow: /*/what:*/
Disallow: /*/show:*/
Disallow: /*/sort:*/
Disallow: /*/page:*/
Disallow: /*/distance:any/
Disallow: /member/
Disallow: /sign-in/
Disallow: /update/
Disallow: /js/
Disallow: /*/*/*/*/*/


# stop Google News indexing non-news pages
User-agent: Googlebot-News
Disallow: /event/
Disallow: /listing/
Disallow: /place/
Disallow: /cinema/


# banned
User-agent: 008
User-agent: adsbot
User-agent: Abonti
User-agent: AhrefsBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot-Extended
User-agent: AwarioBot
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: Baiduspider
User-agent: barkrowler
User-agent: berlin-fu-cow
User-agent: BitSightBot
User-agent: BLEXBot
User-agent: Buck
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-Web
User-agent: coccocbot-web
User-agent: cohere-ai
User-agent: ConnexunBot
User-agent: CriteoBot/0.1
User-agent: CrystalSemanticsBot
User-agent: DataForSeoBot
User-agent: DeepSeek
User-agent: DeepSeekBot
User-agent: Diffbot
User-agent: DomainAppender
User-agent: domains-crawler
User-agent: dotbot
User-agent: eventseekerBot
User-agent: Exabot
User-agent: Ezooms
User-agent: FacebookBot
User-agent: Flamingo_SearchEngine
User-agent: GeedoBot
User-agent: GeedoProductSearch
User-agent: GenAI
User-agent: Genieo
User-agent: Google-Extended
User-agent: GPTBot
User-agent: grapeshot
User-agent: HawaiiBot
User-agent: HTTrack
User-agent: ImagesiftBot
User-agent: InfoTigerBot
User-agent: kalooga
User-agent: Kraken
User-agent: LCC
User-agent: magpie-crawler
User-agent: Mail.Ru
User-agent: MegaIndex.ru
User-agent: meta-webindexer
User-agent: MJ12bot
User-agent: moatbot
User-agent: MojeekBot
User-agent: nestReader
User-agent: netEstate NE Crawler
User-agent: netseer
User-agent: NewsNow
User-agent: Node/simplecrawler
User-agent: Nutch
User-agent: omgili
User-agent: omgilibot
User-agent: Owler
User-agent: panscient.com
User-agent: PaperLiBot
User-agent: Pcore-HTTP
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: PiplBot
User-agent: proximic
User-agent: psbot
User-agent: Punkspider
User-agent: Qwantify
User-agent: RavenCrawler
User-agent: Riddler
User-agent: R6_CommentReader
User-agent: ScooperBot
User-agent: Scrapy
User-agent: Screaming Frog SEO Spider
User-agent: SearchmetricsBot
User-agent: SeekportBot
User-agent: SemrushBot-BA
User-agent: SemrushBot-BM
User-agent: SemrushBot-COUB
User-agent: SemrushBot-CT
User-agent: SemrushBot-SI
User-agent: SemrushBot-SWA
User-agent: SEOkicks-Robot
User-agent: SERankingBacklinksBot
User-agent: SindiceBot
User-agent: SiteAuditBot
User-agent: SiteSucker
User-agent: Slurp
User-agent: sogou spider
User-agent: Sogou web spider
User-agent: spbot
User-agent: Spinn3r
User-agent: SplitSignalBot
User-agent: TikTokSpider
User-agent: Timpibot
User-agent: trendiction
User-agent: trendictionbot
User-agent: TTD-Content
User-agent: TurnitinBot
User-agent: Vocus
User-agent: WeSEE
User-agent: WikiDo
User-agent: YandexBot
User-agent: YioopBot
User-agent: YisouSpider
User-agent: ZoomBot
Disallow: /