# robots.txt for MediaThrive
# Last updated: 2026-02-27
# More info about MediaThriveBot: https://mediathrive.com/bot

# Allow MediaThriveBot (our own crawler) - full access
User-agent: MediaThriveBot
Allow: /
Crawl-delay: 0.5

# Major search engines - standard access
User-agent: Googlebot
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: facebookexternalhit

# Allow Next.js static assets first (specific allows before broader disallows)
Allow: /_next/static/
Allow: /images/
Allow: /videos/
Allow: /demo/
Allow: /grid.svg
Allow: /favicon.ico

# Allow marketing and public content
Allow: /blog/
Allow: /contact
Allow: /docs/
Allow: /faq
Allow: /pricing
Allow: /product/
Allow: /compare/
Allow: /solutions/
Allow: /technology
Allow: /scraper
Allow: /features
Allow: /roi-calculator
Allow: /about
Allow: /case-studies
Allow: /newsroom
Allow: /changelog
Allow: /integrations

# Allow legal pages
Allow: /bot
Allow: /cookie-policy
Allow: /privacy-policy
Allow: /terms-of-service

# Allow RSS feeds and sitemaps
Allow: /blog/feed.xml
Allow: /sitemap.xml

# Block admin and private areas
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /home/
Disallow: /onboarding/
Disallow: /join
Disallow: /update-password
Disallow: /healthcheck
Disallow: /version

# Block development and internal files
Disallow: /_next/
Disallow: /*.json$
Disallow: /.*

# Set crawl delay for well-behaved bots
Crawl-delay: 1

# Default rules for all other bots
User-agent: *
# Allow static assets
Allow: /_next/static/
Allow: /images/
Allow: /videos/
Allow: /demo/
Allow: /grid.svg
Allow: /favicon.ico

# Block private areas for all bots
Disallow: /admin/
Disallow: /api/
Disallow: /auth/
Disallow: /home/
Disallow: /onboarding/
Disallow: /join
Disallow: /update-password
Disallow: /healthcheck
Disallow: /version
Disallow: /_next/
Disallow: /*.json$
Disallow: /.*

# More restrictive crawl delay for unknown bots
Crawl-delay: 2

# Sitemap locations
Sitemap: https://mediathrive.com/sitemap.xml
Sitemap: https://mediathrive.com/blog/feed.xml

# Additional security directives
# Block access to development and temporary files
Disallow: /_next/
Disallow: /api/
Disallow: /*.json$
Disallow: /.*

# Allow specific assets
Allow: /images/
Allow: /videos/
Allow: /demo/
Allow: /grid.svg
Allow: /favicon.ico
Allow: /_next/static/