# =============================================================================
# NIUS.de robots.txt
# Strategy: Maximum visibility, internal endpoints protected
# Principle: Only define specific blocks when behavior differs from default
# Last updated: March 2026
# =============================================================================

User-agent: *
Allow: /

# Internal / sensitive routes
Disallow: /api/
Disallow: /profile
Disallow: /billing-options
Disallow: /credit-card
Disallow: /sepa
Disallow: /finalize-payment
Disallow: /transaction-history
Disallow: /search?
Disallow: /admin/
Disallow: /auth/

# Tracking parameters (duplicate content prevention)
Disallow: /*?utm_source=
Disallow: /*?utm_medium=
Disallow: /*?utm_campaign=
Disallow: /*?utm_content=
Disallow: /*?utm_term=
Disallow: /*?fbclid=
Disallow: /*?igsh=
Disallow: /*?xmt=
Disallow: /*?gclid=
Disallow: /*?mc_cid=
Disallow: /*?mc_eid=

# =============================================================================
# CRAWL-DELAY OVERRIDES (MUST repeat critical Disallows to avoid overrides)
# =============================================================================

User-agent: Bingbot
Crawl-delay: 1
Allow: /
Disallow: /search?
Disallow: /api/
Disallow: /admin/
Disallow: /auth/

User-agent: ScreamingFrogSEOSpider
Crawl-delay: 10
Allow: /
Disallow: /search?
Disallow: /api/
Disallow: /admin/
Disallow: /auth/

User-agent: SiteAuditBot
Crawl-delay: 10
Allow: /
Disallow: /search?
Disallow: /api/
Disallow: /admin/
Disallow: /auth/

# =============================================================================
# RESTRICTED ACCESS (TikTok: only feeds, no content)
# =============================================================================

User-agent: Bytespider
Allow: /rss
Allow: /sitemap.xml
Disallow: /

# =============================================================================
# BLOCKED BOTS
# =============================================================================

User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: MegaIndex
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: Rogerbot
Disallow: /
User-agent: MajesticSEO
Disallow: /
User-agent: SEOkicks
Disallow: /

User-agent: CCBot
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: archive.org_bot
Disallow: /

User-agent: Baiduspider
Disallow: /
User-agent: Sogou
Disallow: /
User-agent: SogouSpider
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: 360Spider
Disallow: /

User-agent: Yandex
Disallow: /
User-agent: YandexBot
Disallow: /
User-agent: YandexImages
Disallow: /
User-agent: YandexVideo
Disallow: /
User-agent: Mail.RU_Bot
Disallow: /

User-agent: Amazonbot
Disallow: /
User-agent: PiplBot
Disallow: /
User-agent: proximic
Disallow: /

User-agent: SeznamBot
Disallow: /
User-agent: Cliqzbot
Disallow: /
User-agent: MojeekBot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: Gigabot
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: BUbiNG
Disallow: /

# =============================================================================
# SITEMAPS
# =============================================================================
Sitemap: https://nius.de/sitemap.xml
Sitemap: https://nius.de/news-sitemap.xml
Sitemap: https://nius.de/video-sitemap.xml