# Below are the Crawl or Do Not Crawl instructions

User-agent: *
# Block everything by default
Disallow: /

# --------------------------- Allow start
# Allow ONLY the homepage root
Allow: /$

# Open up your specific Pages
Allow: /about/
Allow: /ads.txt
Allow: /privacy-policy/
Allow: /terms-of-service/
Allow: /tips/
Allow: /llms.txt
Allow: /robots.txt
Allow: /sitemap-posts.xml

# Open up your Posts directory
Allow: /regions/

# Allow WP assets
Allow: /wp-content/uploads/
Allow: /wp-content/themes/*.css
Allow: /wp-content/themes/*.js
Allow: /wp-includes/*.js
# --------------------------- Allow end

# Specific Block List (Note the spacing/newlines)
User-agent: AhrefsBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: CriteoBot/0.1
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: Ezooms
Disallow: /

User-agent: EzoicBot
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Nimbostratus-Bot
Disallow: /

User-agent: SBIntuitionsBot
Disallow: /

User-agent: SERankingBacklinksBot
Disallow: /

User-agent: SnapchatAdsBot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: ZoomBot
Disallow: /

Sitemap: https://travel.com/sitemap-posts.xml