# 1. Global Rule: Allow all helpful search engines
User-agent: *
Allow: /
Disallow: /_api/*
Disallow: /_partials*

# 2. Optimization for Gemini & Google Search
# 'Google-Extended' is the specific token for Gemini/Vertex AI training
User-agent: Googlebot
User-agent: Google-Extended
Allow: /
Disallow: *?lightbox=

# 3. Optimization for OpenAI (ChatGPT/SearchGPT)
User-agent: GPTBot
User-agent: ChatGPT-User
Allow: /

# 4. Optimization for Other Major AI & LLM Crawlers
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: anthropic-ai
User-agent: OAI-Search
User-agent: Grok
User-agent: BingAI
User-agent: DeepSeek
Allow: /

# 5. Optimization for Ads Performance
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google
Disallow: /_api/*
Disallow: /pro-gallery-webapp/v1/galleries/*

# 6. Block Aggressive/Non-Referral Bots
# These bots scrape your data but rarely send you traffic
User-agent: PetalBot
User-agent: CCBot
User-agent: Bytespider
Disallow: /

# 7. Respectful Crawl Delays for SEO Tools
User-agent: dotbot
User-agent: AhrefsBot
User-agent: SemrushBot
Crawl-delay: 10

# 8. Sitemaps
Sitemap: https://www.form5472.online/sitemap.xml
Sitemap: https://www.form5472.online/llms