# ---- Default for all bots ----
User-agent: *
Crawl-delay: 5
Disallow: /tag/
Disallow: /page/
Disallow: /attachment/
Disallow: /?s=
Disallow: /search/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-json/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /category/uncategorized/
Disallow: /category/expired/

# ---- Google ----
User-agent: Googlebot
Disallow: /tag/
Disallow: /page/
Disallow: /attachment/
Disallow: /?s=
Disallow: /search/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-json/
Disallow: /feed/
Disallow: /category/uncategorized/
Disallow: /category/expired/

# ---- OpenAI / ChatGPT ----
User-agent: GPTBot
Allow: /
Crawl-delay: 2

# ---- Anthropic / Claude ----
User-agent: ClaudeBot
Allow: /
Crawl-delay: 2

# ---- Perplexity ----
User-agent: PerplexityBot
Allow: /
Crawl-delay: 2

# ---- Google AI Overviews ----
User-agent: Google-Extended
Allow: /

# ---- Common Crawl ----
User-agent: CCBot
Allow: /
Crawl-delay: 10

# ---- Microsoft Copilot / Bing ----
User-agent: Bingbot
Allow: /
Crawl-delay: 2

Sitemap: https://www.freestufftimes.com/sitemap_index.xml