User-agent: *
Allow: /
 
# Block non-indexable pages
Disallow: /404
Disallow: /thank-you
 
# Block API routes from being indexed as pages
Disallow: /api/
 
# Prevent crawling of tracking parameters
Disallow: /*?utm_
Disallow: /*?ref=
 
# Allow AI Crawlers
User-agent: GPTBot
Allow: /
 
User-agent: ChatGPT-User
Allow: /
 
User-agent: PerplexityBot
Allow: /
 
User-agent: ClaudeBot
Allow: /
 
User-agent: Google-Extended
Allow: /
 
# SEO Tools
User-agent: SemrushBot
Allow: /
 
# Sitemaps
Sitemap: https://www.debutinfotech.com/sitemap.xml
Sitemap: https://www.debutinfotech.com/server-sitemap.xml
Sitemap: https://www.debutinfotech.com/use-cases-sitemap.xml
Sitemap: https://www.debutinfotech.com/news-sitemap.xml