User-Agent: *
Allow: /
Disallow: /api/
Disallow: /admin/
Disallow: /settings/
Disallow: /auth/
# Cloudflare internal paths — never should be indexed
Disallow: /cdn-cgi/
# Legacy WordPress paths — all return 410 Gone
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /page/
Disallow: /index.php/

User-Agent: Googlebot
Allow: /

User-Agent: Googlebot-Image
Allow: /

User-Agent: Googlebot-News
Allow: /ai-news/
Allow: /feed
Disallow: /investment_rounds/
Disallow: /investment_funds/
Disallow: /investors/
Disallow: /startups/
Disallow: /people/
Disallow: /acquisitions/
Disallow: /exit-events/
Disallow: /c/
Disallow: /category/
Disallow: /countries/
Disallow: /mcp-servers/
Disallow: /lists/
Disallow: /rankings/
Disallow: /alternatives/
Disallow: /agent-readiness/
Disallow: /gift/

# Block bots that waste bandwidth and provide no SEO value
User-Agent: Turnitin
Disallow: /

User-Agent: TurnitinBot
Disallow: /

User-Agent: DuckAssistBot
Disallow: /

User-Agent: AhrefsBot
Disallow: /

User-Agent: SemrushBot
Disallow: /

User-Agent: MJ12bot
Disallow: /

User-Agent: DotBot
Disallow: /

User-Agent: BLEXBot
Disallow: /

User-Agent: DataForSeoBot
Disallow: /

User-Agent: PetalBot
Disallow: /

User-Agent: Bytespider
Disallow: /

Sitemap: https://www.startuphub.ai/sitemap.xml
Sitemap: https://www.startuphub.ai/sitemap-startups.xml
Sitemap: https://www.startuphub.ai/sitemap-investors.xml
Sitemap: https://www.startuphub.ai/sitemap-people.xml
Sitemap: https://www.startuphub.ai/sitemap-news.xml