User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

Disallow: /wp-login.php
Disallow: /cgi-bin/
Disallow: /comments/

# Allow all content URLs (important for news crawling)
Allow: /

# Allow important resources
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/

# Sitemap
Sitemap: https://www.nagpurtoday.in/sitemap_index.xml

# Crawl Delay (ignored by Google, fine for others)
Crawl-delay: 5

# Block specific bad bots
User-agent: BadBot
Disallow: /

User-agent: MJ12bot
Disallow: /