User-agent: *

Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Search pages (all folders)
Disallow: /*?s=
Disallow: /wp-json/
# REST API
Disallow: /*?rest_route=

# WordPress junk (root)
Disallow: /tag/
Disallow: /cgi-bin/
Disallow: /trackback/
Disallow: /feed/
Disallow: /comments/
Disallow: /*/trackback/
Disallow: /*/feed/
Disallow: /*/comments/

# WordPress junk (/service)
Disallow: /service/wp-admin/
Allow: /service/wp-admin/admin-ajax.php
Disallow: /service/tag/
Disallow: /service/cgi-bin/
Disallow: /service/trackback/
Disallow: /service/feed/
Disallow: /service/comments/

# Cloudflare internal paths
Disallow: /cdn-cgi/

Sitemap: https://shifton.com/sitemaps.xml
Sitemap: https://shifton.com/service/sitemaps.xml

# AI crawlers — allowed for GEO citations
Disallow: /service/wp-json/

Disallow: /service/*?rest_route=

User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: meta-externalagent
Allow: /

# Aggressive scrapers — blocked
User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Diffbot
Disallow: /
User-agent: OAI-SearchBot
Allow: /
Allow: /llms.txt