# ukrsmeta.ua robots.txt
# Main goal: allow search crawlers and AI search crawlers to access public content.
# Block only technical, duplicate, login and low-value WordPress URLs.

User-agent: Googlebot
Allow: /
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /template.html
Disallow: /archive/
Disallow: /myprofile/
Disallow: /message.php
Disallow: /politika-konfidencialnosti
Disallow: /otkaz-ot-otvetstvennosti
Disallow: /?s=
Disallow: /*?s=
Disallow: /*/trackback/
Disallow: /*/feed/
Disallow: /*/comments/

# Google AI features / Gemini / Vertex AI training and grounding
User-agent: Google-Extended
Allow: /

# ChatGPT Search
User-agent: OAI-SearchBot
Allow: /

# ChatGPT user browsing / retrieval
User-agent: ChatGPT-User
Allow: /

# OpenAI crawler
User-agent: GPTBot
Allow: /

# Semrush audit crawlers
User-agent: SemrushBot
Allow: /

User-agent: SiteAuditBot
Allow: /

# All other crawlers
User-agent: *
Allow: /
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /template.html
Disallow: /archive/
Disallow: /myprofile/
Disallow: /message.php
Disallow: /politika-konfidencialnosti
Disallow: /otkaz-ot-otvetstvennosti
Disallow: /?s=
Disallow: /*?s=
Disallow: /*/trackback/
Disallow: /*/feed/
Disallow: /*/comments/

Sitemap: https://ukrsmeta.ua/sitemap_index.xml
Sitemap: https://ukrsmeta.ua/sitemap-images.xml