# www.robotstxt.org/
# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449

# =============================================================================
# AI TRAINING BOTS - Block author content
# These bots scrape content to train AI models.
# Allow: platform pages (homepage, categories) so the site is known to exist
# Block: author content (works, profiles) to protect creative work
# =============================================================================

User-agent: GPTBot
User-agent: Google-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: anthropic-ai
User-agent: ClaudeBot
User-agent: Meta-ExternalAgent
User-agent: meta-externalagent
User-agent: Applebot-Extended
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: omgili
User-agent: omgilibot
User-agent: webzio-extended

# Block individual works (author content) - all categories
Disallow: /*/stihove/
Disallow: /*/poetry/
Disallow: /*/proza/
Disallow: /*/prose/
Disallow: /*/eseta/
Disallow: /*/essays/
Disallow: /*/jurnalistika/
Disallow: /*/journalism/
Disallow: /*/kartini/
Disallow: /*/pictures/
Disallow: /*/fotografia/
Disallow: /*/photography/
Disallow: /*/kolaji/
Disallow: /*/collages/
Disallow: /*/muzika/
Disallow: /*/music/
Disallow: /*/prevodi/
Disallow: /*/translations/
Disallow: /*/prilojni-izkustva/
Disallow: /*/applied-arts/
Disallow: /*/audio-prochiti/
Disallow: /*/audio-readings/
Disallow: /*/podcasts/
Disallow: /*/books/
Disallow: /*/cds/
Disallow: /*/video/

# Block author profiles
Disallow: /*/profile/
Disallow: /*/profiles/
Disallow: /*/author/
Disallow: /*/users/

# =============================================================================
# Regular Search Engines (Google, Bing, etc.)
# Full access to public content for traditional search indexing
# =============================================================================

User-agent: *

# Block creation/auth pages
Disallow: /*/work/add-new
Disallow: /*/work/new
Disallow: /*/users/register
Disallow: /*/users/login
Disallow: /*/oauth-redirect/
Disallow: /login_facebook
Disallow: /login_google

# Block API and admin
Disallow: /api/
Disallow: /api-docs
Disallow: /*/admin/

# Block callbacks and webhooks
Disallow: /callbacks/
Disallow: /sync-device-tokens
Disallow: /syncDeviceTokens

# Block action endpoints
Disallow: /*/message/
Disallow: /*/work/next
Disallow: /*/work/delete
Disallow: /*/work/undelete
Disallow: /*/work/moderate
Disallow: /*/work/publish
Disallow: /*/rating/

# Block vendor directories
Disallow: /vendor/
Disallow: /bundles/

# Block private user pages
Disallow: /*/profile/*/favourite-works
Disallow: /*/profile/*/watch-later-works
Disallow: /*/profile/*/view-history
Disallow: /*/profile/*/followed-authors-works
Disallow: /*/profile/*/my-works/
Disallow: /*/profile/*/comments
Disallow: /*/profile/*/membership/
Disallow: /*/author/*/my-works-comments
Disallow: /*/profiles/*/followers
Disallow: /*/profiles/*/following
Disallow: /*/profiles/*/notifications

# Sitemap
Sitemap: https://otkrovenia.com/sitemap.xml

# LLM Discovery: https://otkrovenia.com/llms.txt
# See: https://llmstxt.org