Top Sites

# 常规搜索引擎规则 User-agent: * Allow: / # Disallow: /private/ # 网站地图 Sitemap: https://musicgeneratorai.com/sitemap.xml # AI爬虫特定规则 User-agent: GPTBot User-agent: Claude-Web User-agent:...

# Music Generator AI > Create Original Songs with AI Music Generator in Minutes Music Generator AI is a comprehensive AI-powered platform that helps you generate professional-qu...

# START nuxt-robots (indexable) User-agent: GPTBot User-agent: ChatGPT-User User-agent: Claude-bot User-agent: ClaudeWeb User-agent: PerplexityBot User-agent: YouBot User-agent:...

# Muzli - Design Inspiration Hub Designers' Secret Source: all the design inspiration you need. Served fresh. Muzli is the best shortcut to inspiration, aggregating the latest a...

User-Agent: * Allow: / Disallow: /api/ Disallow: /_nuxt/ Disallow: /xx/ # Prioritize fresh content for crawlers # Articles and tags are high-value pages # App pages are bulk con...

# MWM - AI App Builder & Mobile App Growth Platform > MWM powers two products: MWM AI (build apps with AI) and MWM Scale (grow apps to millions). Also provides real-time mobile...

User-agent: * Allow: / # Authentication & transient pages — no SEO value Disallow: /login Disallow: /signup Disallow: /signup-detail Disallow: /forgot-password Disallow: /create...

# Keeper Memorials > Keeper Memorials is an online memorial and obituary platform for families, funeral homes, > cemeteries, and hospice partners. Families create memorial pages...

User-agent: * #Allow: Allow: /*.jpg Allow: /*.jpeg Allow: /*.gif Allow: /*.png Allow: /*.webp #Disallow: Disallow: /_hcms/preview/ Disallow: /hs/manage-preferences/ Disallow: /h...

# www.dstny.fr > Améliorez vos communications d'entreprise interne et externe avec les solutions SaaS de Dstny, conçues pour dynamiser les entreprises. Covers 55 pages across 3...

User-agent: * Crawl-delay: 6 Disallow:

# llms.txt # Version: 1 # Last-Updated: 2026-01-24 Site: mypatta Domain: https://mypatta.in Purpose: Platform for verified land & property documents, state-wise document retriev...

User-agent: * Disallow: / User-agent: Googlebot User-agent: Yeti User-agent: Mediapartners-Google User-agent: DAUM User-agent: Bingbot User-agent: AhrefsSiteAudit User-agent: fa...

# Myrealtrip (마이리얼트립) > Myrealtrip is South Korea's leading travel platform offering flights, accommodations, tours, activities, transportation, and travel packages worldwide. #...

User-Agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-content/uploads/wc-logs/ Disallow: /wp-content/uploads/woocommerce_transient_files/ Disallow: /w...

# My Robot Doll (www.myrobotdoll.com) Canonical base URL: https://www.myrobotdoll.com/ Alternate: https://myrobotdoll.com/ (redirects to www) > High-signal references for LLM an...

User-agent: * Allow: / Disallow: /mio-account/ Disallow: /carrello/ Disallow: /checkout/ Disallow: /thank-you/ Disallow: /scarica-casi-studio/ Disallow: /1/ Sitemap: https://mys...

# MysteryClient.it - Mebius srl > MysteryClient.it is the main website of Mebius srl, an Italian market intelligence company specializing in mystery shopping, voice of customer,...

User-agent: Google-Extended Allow: / User-agent: GPTBot Allow: / User-agent: ClaudeBot Allow: / User-agent: meta-externalagent Allow: / User-agent: Applebot-Extended Allow: / Us...

# Mzad Qatar > Qatar's largest online marketplace for buying, selling, and renting products and services. Founded in Qatar, Mzad Qatar connects buyers and sellers across categor...

User-Agent: * Allow: / Disallow: /api/ Disallow: /app Disallow: /login Disallow: /profile Disallow: /request User-Agent: GPTBot User-Agent: ChatGPT-User User-Agent: ClaudeBot Us...

# Naasa Securities > Naasa Securities Company Limited is a licensed stock broker in Nepal offering brokerage, DEMAT-related services, calculators, and customer support. ## Prima...

User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-login.php Disallow: /cgi-bin/ Disallow: /comments/ # Allow all content URLs (important for news...

# Nagpur Today : Nagpur News: Nagpur India Daily News Website > Get the latest Nagpur news with fast, reliable coverage of crime, politics, civic issues, traffic, weather, and...

User-agent: * Allow: / Disallow: /login Disallow: /register Disallow: /forgot-password Disallow: /reset-password Disallow: /confirm-email Disallow: /account/ Disallow: /cart/ Di...

# name.am > Armenian domain registrar and marketplace. Register .AM domains and 200+ other extensions. name.am is a domain registrar platform based in Armenia offering domain re...

User-agent: * Disallow: /*?subject= Disallow: /api$ Disallow: /v4/ Disallow: /domain/renew Disallow: /leaving Disallow: /account/credentials/claim_account Disallow: */domain/sea...

# Name.com This site publishes machine-readable API discovery metadata. The discovered API is intended for credentialed reseller integrations, not as a complete interface for al...

user-agent: * allow: / disallow: /cdn-cgi/ disallow: /captcha disallow: /oauth2/ disallow: /sso/ disallow: /realms/ disallow: /checkout/ disallow: /inquiry/ disallow: /lease/ di...

# Domain Marketplace Buy, Sell, Rent, Lease & Trade Domains > Best domain marketplace for buying, renting, leasing, selling & trading domain names. Popular aftermarket for findi...

User-agent: * Disallow: /suspended/* Disallow: /p/ Disallow: /domain-offer/ User-agent: AdsBot-Google User-agent: AdsBot-Google-Mobile Allow: /p/ Allow: /domain-offer/

# llms.txt file for https://www.names.co.uk/ # This file provides guidance for AI crawlers and large language models (LLMs) # regarding the content on this website. # General Us...

User-agent: * Sitemap: https://www.namesilo.com/sitemap.xml Allow: /api-reference Disallow: /jslib Disallow: /stored_jobs Disallow: /popups Disallow: /api/* Disallow: /apibatch/...

# NameSilo | Domain Registrar & Web Hosting Domain registration, web hosting, and email services with free WHOIS privacy, competitive pricing, and a comprehensive API for develo...

User-agent: * Disallow: /*?*q= Disallow: /privacy-policy Disallow: /terms-of-service Sitemap: https://nanobanana.org/sitemap.xml User-agent: GPTBot User-agent: Claude-Web User-a...

# Nano Banana AI Creative Platform > Google's revolutionary AI model for advanced image generation and editing, featuring consistent character editing and scene preservation wit...

## Enable robots.txt rules for all crawlers User-agent: Googlebot Disallow: User-agent: Googlebot-image Disallow: User-agent: * Disallow: /readme.html Disallow: /cgi-bin/ Disall...

# NASH — llms.txt (Generated by Digitup LLMS Generator) language: - el - en business_name: NASH description_el: | Η συλλογή από βραδινά φορέματα της NĀSH έχει σχεδιαστεί με γνώμ...

User-agent: * Disallow: /wp-content/uploads/wc-logs/ Disallow: /wp-content/uploads/woocommerce_transient_files/ Disallow: /wp-content/uploads/woocommerce_uploads/ Disallow: /*?a...

# https://naturelliving.com/ llms.txt - [Naturel Living Recipes](https://naturelliving.com/): Discover delicious organic recipes and healthy living tips. - [Jo & Kin Recipes](ht...

User-agent: GPTBot Allow: /llms.txt Allow: /llms-full.txt User-agent: CCBot Allow: /llms.txt Allow: /llms-full.txt User-agent: Google-Extended Allow: /llms.txt Allow: /llms-full...

# nBit Soluzioni Binarie > nBit è un'azienda di consulenza e servizi informatici con sede ad Altamura (BA), specializzata in infrastrutture critiche, integrazione software e sol...

Crawl-delay: 10 # START YOAST BLOCK # --------------------------- User-agent: * Disallow: Sitemap: https://capacity.com/sitemap_index.xml Schemamap: https://capacity.com/wp-json...

[GENERAL INFO] Capacity is an AI-powered support automation platform founded in 2017 by David Karandish and Chris Sims. Headquartered in St. Louis and part of the Equity.com inc...

# As a condition of accessing this website, you agree to abide by the following # content signals: # (a) If a Content-Signal = yes, you may collect content for the corresponding...

# nemovideo.com > Documentation for nemovideo.com ## Documentation - [AI Caption Generator: Turn Subtitles into a Visual Power tool](https://www.nemovideo.com/features/ai-captio...

# Grup 1 User-agent: Googlebot Disallow: /nogooglebot/ # Grup 2 User-agent: * Allow: / Sitemap: https://www.nemutlu.net/sitemap.xml

<!doctype html> <html lang="tr"> <head> <meta charset="utf-8"> <!--[if IE]><meta http-equiv="X-UA-Compatible" content="IE=edge" /><![endif]--> <meta name="Nemutlu Yazılım - 404...

User-Agent: * Allow: / # Allow all current docs Allow: /docs/*/current # Allow docs that do not use /current Allow: /docs/aura Allow: /docs/browser Allow: /docs/genai Allow: /do...

# Neo4j Graph Database & Analytics > Neo4j is a native graph database. Nodes, relationships, and properties are stored directly connected -- traversals are O(1) per hop. Use it...

# https://www.netafy.com/robots.txt User-agent: * Allow: / Disallow: /dashboard Disallow: /auth/ Disallow: /client-portal-sign-in # AI Crawlers - explicitly allowed User-agent:...

# Netafy Broadband > Netafy Broadband is a wireless Internet service provider (WISP) serving residential and business customers in New Hampshire, Massachusetts, Vermont, and Mai...

User-agent: *

# Netlas: Comprehensive Internet-Wide Scanning & OSINT Platform > Netlas provides internet-wide scanning, OSINT, DNS & WHOIS tools, and eASM services for reconnaissance and secu...

User-agent: * Disallow:

<!DOCTYPE html> <html> <head> <title></title> <META http-equiv="Content-Type" Content="text/html; charset=UTF-8"> <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com...

User-agent: * Sitemap: https://netzfabrik.com/sitemap.xml

# NETZFABRIK > NETZFABRIK bietet professionelle Hostinglösungen und Infrastrukturen. Hostingpakete, Dedicated Server, Colocation, Managed Hosting und Solutions aus Köln und Fran...

# neue fische robots.txt — We tech you to the next level. # * User-agent: * Allow: / User-agent: GPTBot Allow: * User-agent: PerplexityBot Allow: * # Host Host: https://www.neue...

# llms.txt for [neuefische.de](http://neuefische.de/) ## About neue fische is a German-based coding bootcamp offering intensive training programs in various tech disciplines. Wi...