Top Sites>Aairtreks.com

# --- Primary search engines ---
User-agent: Googlebot
Allow: /


User-agent: Bingbot
Allow: /


# --- OpenAI crawlers ---
# OAI-SearchBot indexes content for answer surfaces with citation/attribution
User-agent: OAI-SearchBot
Allow: /


# GPTBot is OpenAI's general crawler often associated with model training
# We allow public pages but exclude /no-train/ for content you don't want used in training
User-agent: GPTBot
Disallow: /no-train/
Allow: /

User-agent: ChatGPT-User
Allow: /


# --- Perplexity crawlers ---
# Public guidance indicates these user-agents; keep WAF logs to validate behavior
User-agent: PerplexityBot
Allow: /


User-agent: Perplexity-User
Allow: /


# --- Other AI crawlers (examples) ---
User-agent: Claude-Web
Allow: /

User-agent: Google-Travel-Flights
Allow: /

# --- to add sitemaps ---

Not stored for this site.