# robots.txt for Witness21.com
# Optimized for multilingual site with subdomain-specific crawling

# General rules for all bots
User-agent: *

Allow: /  # Allow everything unless explicitly blocked
Disallow: /wp-admin/
# Disallow: /wp-login.php*
Disallow: /wp-content/uploads/wp-import-export-lite/
Allow: /wp-admin/admin-ajax.php
Allow: /wp-json/jsw-ytimp/v1/csv-json*


# Queries Rules
Allow: /witnesshome-2/?*
Allow: /*.js*
Allow: /*.css*
Disallow: /*/page/*
# Allow: /?v_tag=
# Disallow: /*v_tag=*
# Disallow: /?*video=*
# Disallow: /?*list=*
# Disallow: /?*t=*

Disallow: /*?*

# Author Rules
Allow: /author
# Disallow: /author/*/videos/
Disallow: /author/*/home/
Disallow: /author/*/shorts/
Disallow: /author/*/post/
Disallow: /author/*/liked/
Disallow: /author/*/collections/
Disallow: /author/*/following/
Disallow: /author/*/followers/



### Block aggressive or irrelevant bots

# AhrefsBot: SEO backlink checker
User-agent: AhrefsBot
Disallow: /

# MJ12bot: Majestic SEO crawler
User-agent: MJ12bot
Disallow: /

# DotBot: SEO and web analytics bot
User-agent: DotBot
Disallow: /

# SemrushBot: SEO analytics and competitor research
User-agent: SemrushBot
Disallow: /

# MegaIndex.ru: Russian SEO bot
User-agent: MegaIndex.ru
Disallow: /

# Twiceler: Defunct crawler (Cuil search engine)
User-agent: Twiceler
Disallow: /

# YetiBot: Naver's (Korean) crawler, now obsolete
User-agent: YetiBot
Disallow: /

# Gigabot: GigaBlast crawler, low-value search engine
User-agent: Gigabot
Disallow: /

# HTTrack: Website mirroring/downloading tool
User-agent: HTTrack
Disallow: /

# WebCopier: Offline site downloader
User-agent: WebCopier
Disallow: /

# EmailSiphon: Email scraping bot
User-agent: EmailSiphon
Disallow: /

# EmailCollector: Another email scraper
User-agent: EmailCollector
Disallow: /

# Scrapy: Python-based web scraping framework
User-agent: Scrapy
Disallow: /

# BoardTracker: Tracks forums, irrelevant for non-forum sites
User-agent: BoardTracker
Disallow: /

# User-agent: OAI-SearchBot
# Disallow: /

User-agent: Applebot
Disallow: /


# Sitemap location
Sitemap: https://witness21.com/sitemaps/video-sitemap-index.xml
Sitemap: https://witness21.com/sitemap.xml
Sitemap: https://witness21.com/sitemap.rss