# Robots.txt for Siptalk Website
# https://www.siptalk.com.au/

# Allow all crawlers
User-agent: *
Allow: /

# Disallow internal directories
Disallow: /includes/
Disallow: /assets/js/
Disallow: /assets/css/

# Allow CSS and JS for rendering
Allow: /assets/css/*.css
Allow: /assets/js/*.js

# Specific crawler instructions
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Bingbot
Allow: /
Crawl-delay: 0

# Block bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Crawl-delay: 10

# Sitemap location
Sitemap: https://www.siptalk.com.au/sitemap.xml
Sitemap: https://www.siptalk.com.au/sitemap-articles.php