# ============================================================================= # Robots.txt for youstable.com # Optimized for Google Search and Other Search Engines # Last Updated: February 2026 # ============================================================================= # ----------------------------------------------------------------------------- # UNIVERSAL RULES # ----------------------------------------------------------------------------- User-agent: * # Allow essential assets Allow: /*.css$ Allow: /*.js$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.png$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.ico$ Allow: /*.woff$ Allow: /*.woff2$ # ----------------------------------------------------------------------------- # Explicit favicon access (critical for Google) Allow: /favicon.ico # ----------------------------------------------------------------------------- # Content Allow: /blog/ Allow: /blog/* Crawl-delay: 1 # ----------------------------------------------------------------------------- # GOOGLE # ----------------------------------------------------------------------------- User-agent: Googlebot Disallow: /*?fbclid=* Disallow: /*?gclid=* Disallow: /*?msclkid=* Disallow: /*?ref=* Disallow: /manage/ Disallow: /cgi/library/popup-script/pop_script.html Disallow: /cgi/library/form-validator/validator.html Disallow: /c1f8f92c0d4e00f3a7352ff9bb45a09a.php Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-Mobile Allow: / User-agent: Googlebot-News Allow: /blog/ User-agent: AdsBot-Google Allow: / User-agent: AdsBot-Google-Mobile Allow: / # ----------------------------------------------------------------------------- # OTHER SEARCH ENGINES # ----------------------------------------------------------------------------- User-agent: Bingbot Allow: / User-agent: DuckDuckBot Allow: / User-agent: Slurp Allow: / User-agent: YandexBot Allow: / Crawl-delay: 2 User-agent: Baiduspider Allow: / Crawl-delay: 2 User-agent: Brave-Search-Bot Allow: / # ----------------------------------------------------------------------------- # AI / LLM BOTS # ----------------------------------------------------------------------------- User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: ClaudeBot Allow: / User-agent: anthropic-ai Allow: / User-agent: Google-Extended Allow: / User-agent: CCBot Allow: / User-agent: PerplexityBot Allow: / User-agent: YouBot Allow: / User-agent: Amazonbot Allow: / User-agent: Applebot Allow: / User-agent: Applebot-Extended Allow: / # ----------------------------------------------------------------------------- # SEO TOOLS # ----------------------------------------------------------------------------- User-agent: AhrefsBot Allow: / Crawl-delay: 2 User-agent: SemrushBot Allow: / Crawl-delay: 2 User-agent: MJ12bot Allow: / Crawl-delay: 2 User-agent: rogerbot Allow: / # ----------------------------------------------------------------------------- # SOCIAL MEDIA PREVIEW BOTS # ----------------------------------------------------------------------------- User-agent: facebookexternalhit Allow: / User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / User-agent: Pinterestbot Allow: / User-agent: WhatsApp Allow: / User-agent: TelegramBot Allow: / User-agent: Slackbot Allow: / User-agent: Discordbot Allow: / # ----------------------------------------------------------------------------- # BAD BOTS (BLOCK) # ----------------------------------------------------------------------------- User-agent: HTTrack Disallow: / User-agent: WebCopier Disallow: / User-agent: Offline Explorer Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: EmailWolf Disallow: / User-agent: ExtractorPro Disallow: / User-agent: CherryPicker Disallow: / User-agent: ChinaClaw Disallow: / User-agent: Grabber Disallow: / User-agent: Teleport Disallow: / User-agent: Wget Disallow: / # ----------------------------------------------------------------------------- # CANONICAL HOST (for Yandex and other search engines) # ----------------------------------------------------------------------------- Host: https://www.youstable.com # ----------------------------------------------------------------------------- # CLEAN PARAMS (strip tracking parameters from URLs) # ----------------------------------------------------------------------------- Clean-param: utm_source&utm_medium&utm_campaign&utm_term&utm_content&ref&fbclid&gclid&msclkid # ----------------------------------------------------------------------------- # SITEMAPS & FEEDS # ----------------------------------------------------------------------------- Sitemap: https://www.youstable.com/sitemap.xml Sitemap: https://www.youstable.com/blog/sitemap_index.xml # llms Allow: /llms/ Allow: /llms/* # https://www.youstable.com/llms.txt # ============================================================================= # END # =============================================================================