# This is the robots.txt file that controls how search engines and web crawlers can access our site

# Allow all search engines to crawl the site with a small delay between requests
User-agent: *
Allow: /
Crawl-delay: 0.25

# Special rules for Google Image bot - allow access with rate limiting
User-agent: Googlebot-Image 
Allow: /
Request-rate: 5/1s # Limit to 5 requests per second
Crawl-delay: 0.5

# Special rules for Google Mobile bot - allow access with rate limiting
User-agent: Googlebot-Mobile
Allow: /
Request-rate: 5/1s # Limit to 5 requests per second
Crawl-delay: 0.5

# Special rules for Google Inspection Tool - allow access with rate limiting
User-agent: Google-InspectionTool
Allow: /
Request-rate: 5/1s # Limit to 5 requests per second
Crawl-delay: 0.5

# Block access to known aggressive crawler bots that could overload our servers
User-agent: Baiduspider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DotBot
Disallow: /

# Blocking URLs with modal parameters
Disallow: /*?firstLessonFreeModal=
Disallow: /*?modal=

# Define locations of our XML sitemaps for search engines
Sitemap: https://teachme.to/sitemap.xml # Main sitemap
Sitemap: https://teachme.to/blog/sitemap.xml # Blog sitemap