# LinkCentre robots.txt
# Updated: 2025-11-27 - Added AI crawler directives

# ============================================
# DEFAULT RULES
# ============================================
User-agent: *
Allow: /
Disallow: /rate/
Disallow: /linkout/
Disallow: /search/?*page=
Disallow: /profile/websitesgb/
Disallow: /top.php
Disallow: /admin/
Disallow: /cache/
Disallow: /logs/
Disallow: /scripts/
Disallow: /backups/
Crawl-delay: 1

# ============================================
# SEO TOOLS - Rate Limited
# ============================================
User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

# ============================================
# AI CRAWLER PERMISSIONS
# ============================================

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /profile/
Allow: /review/
Allow: /dir/
Allow: /directory/
Allow: /charity/
Allow: /about-us/
Allow: /addurl/
Allow: /get-started/
Allow: /help-guide/
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin/
Disallow: /secure/
Crawl-delay: 2

User-agent: ChatGPT-User
Allow: /

# Anthropic / Claude
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /
Crawl-delay: 2

# Google AI / Bard / Gemini
User-agent: Google-Extended
Allow: /

# Microsoft / Bing AI
User-agent: Bingbot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: meta-externalagent
Allow: /

# Apple AI
User-agent: Applebot-Extended
Allow: /

# You.com
User-agent: YouBot
Allow: /

# ============================================
# SITEMAPS
# ============================================
Sitemap: https://www.linkcentre.com/sitemap.xml

# ============================================
# AI MANIFEST
# ============================================
# AI systems: See /llms.txt for structured site information
# Extended data: See /llms-full.txt for complete taxonomy