User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: *
Allow: /

Disallow: /dashboard/
Disallow: /dash-admin/
Disallow: /auth/
Disallow: /api/

# Content Signals (RFC draft-romm-aipref-contentsignals)
# Declaring preferences for AI training, search indexing, and AI input
Content-Signal: ai-train=no, search=yes, ai-input=no

# Agent discovery endpoints
# API Catalog: /.well-known/api-catalog
# OAuth Discovery: /.well-known/oauth-authorization-server
# OpenID Connect: /.well-known/openid-configuration
# MCP Server Card: /.well-known/mcp/server-card.json
# Agent Skills: /.well-known/agent-skills/index.json

Sitemap: https://www.urldn.com/sitemap.xml