User-agent: *
Allow: /

# Allow all search engines to crawl the site
Allow: /blogs/
Allow: /v/
Allow: /ft/
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /config/manifest.json
Allow: /site.webmanifest
Allow: /watch/
Allow: /playlist/
Allow: /live/
Allow: /trending.html
Allow: /live-tv.html
Allow: /404.html

# Disallow certain directories that don't need indexing
Disallow: /ad/
Disallow: /functions/
Disallow: /clean/
Disallow: /*.js$
Disallow: /*.css$
Disallow: /*.png$
Disallow: /*.jpg$
Disallow: /*.jpeg$
Disallow: /*.gif$
Disallow: /*.svg$
Disallow: /*.ico$
Disallow: /*.webp$
Disallow: /*.woff$
Disallow: /*.woff2$
Disallow: /*.ttf$
Disallow: /*.eot$

# Crawl delay for respectful crawling
Crawl-delay: 1

# Sitemap locations
Sitemap: https://skipcut.com/sitemap.xml
Sitemap: https://skipcut.com/news-sitemap.xml

# Additional directives for better SEO
# Allow Googlebot to access all content
User-agent: Googlebot
Allow: /

# Allow Bingbot to access all content
User-agent: Bingbot
Allow: /

# Allow all major search engines
User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

# Block AI training bots (optional - remove if you want AI training)
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow /

User-agent: CCBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: Applebot
Allow: /

# Block archive.org (optional - remove if you want archiving)
User-agent: ia_archiver
Disallow: /

# Block common scrapers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

# Allow legitimate SEO tools
User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Mediapartners-Google
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /