# AI/LLM Crawlers - Explicit Allow
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Applebot-Extended
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: PerplexityBot
User-agent: Diffbot
Allow: /
Crawl-delay: 1

# AI Documentation
# LLMs.txt: https://www.ordio.com/llms.txt
# LLMs.txt (Full): https://www.ordio.com/llms-full.txt

# SEO Analysis Tools - Explicit Allow
# Semrush bots for SEO audits and analysis (pages may have noindex but should be crawlable for analysis)
User-agent: SemrushBot
User-agent: SiteAuditBot
User-agent: SemrushBot-BA
User-agent: SemrushBot-SI
User-agent: SemrushBot-SWA
User-agent: SemrushBot-OCOB
User-agent: SplitSignalBot
Allow: /
Crawl-delay: 1

# Standard Crawlers
User-agent: *
Disallow: /tag/
Disallow: /category/
Disallow: /new-lexikon/
Disallow: /feed/
Disallow: /comments/
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /author/
# Internal PHP component directories - never index (sections, base, components, helpers, config)
Disallow: /v2/sections/
Disallow: /v2/base/
Disallow: /v2/components/
Disallow: /v2/helpers/
Disallow: /v2/config/
# API endpoints - never index (internal APIs, not public content)
Disallow: /v2/api/
# Partner OAuth start URLs (302 to Google) — not indexable content
Disallow: /partner/oauth/
# Legacy HTML directory - never index (legacy files, downloads, templates)
# Allow favicon for Google Search – must be crawlable (Google Search Central)
Allow: /favicon.ico
Allow: /html/images/favicon.ico
Allow: /html/images/favicon-16x16.png
Allow: /html/images/favicon-32x32.png
Allow: /html/images/favicon-48x48.png
Allow: /html/images/apple-touch-icon.png
Allow: /html/images/android-chrome-192x192.png
Allow: /html/images/android-chrome-512x512.png
Allow: /html/site.webmanifest
Allow: /site.webmanifest
Disallow: /html/
# Demo page - internal use only, not for public indexing
Disallow: /demo
# English marketing preview (/en/*) — remove this disallow when include_in_public_search is true for `en` in v2/config/locale-config.php
Disallow: /en/
Allow: /

# Sitemaps (XML only; no sitemap.txt)
# Note: llms.txt and llms-full.txt are not valid Sitemap format (sitemaps.org expects XML).
# AI crawlers discover them via comments above and Allow: / crawl.
Sitemap: https://www.ordio.com/sitemap.xml
Sitemap: https://www.ordio.com/sitemap-produkt-updates.xml
Sitemap: https://www.ordio.com/sitemap-blog.xml