# -----------------------------------------------
# General Crawlers
# -----------------------------------------------
User-agent: *
Allow: /
Allow: /wp-admin/admin-ajax.php
Allow: /*.js$
Allow: /*.css$
Allow: /*.jpg$
Allow: /*.png$
Allow: /*.jpeg$
Allow: /*.gif$
Disallow: /category/
Disallow: /author/
Disallow: /uncategorized/
Disallow: /search/
Disallow: */feed/
Disallow: /feed/$
Disallow: /*?d=
Disallow: /*?p=
Disallow: /*?replytocom=

# -----------------------------------------------
# Googlebot
# -----------------------------------------------
User-agent: Googlebot
Allow: /
Allow: /wp-admin/admin-ajax.php
Allow: /*.js$
Allow: /*.css$
Allow: /*.jpg$
Allow: /*.png$
Allow: /*.jpeg$
Allow: /*.gif$
Disallow: /category/
Disallow: /author/
Disallow: /uncategorized/
Disallow: /search/
Disallow: */feed/
Disallow: /feed/$
Disallow: /*?d=
Disallow: /*?p=
Disallow: /*?replytocom=

User-agent: Googlebot-Mobile
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

User-agent: Googlebot-Video
Allow: /

# -----------------------------------------------
# Bingbot / MSNBot
# -----------------------------------------------
User-agent: Bingbot
Crawl-delay: 15
Allow: /
Allow: /wp-admin/admin-ajax.php
Disallow: /category/
Disallow: /author/
Disallow: /uncategorized/
Disallow: /search/
Disallow: */feed/
Disallow: /feed/$
Disallow: /*?d=
Disallow: /*?p=
Disallow: /*?replytocom=

User-agent: MSNBot
Crawl-delay: 24
Allow: /
Disallow: /category/
Disallow: /author/
Disallow: /uncategorized/
Disallow: /search/
Disallow: */feed/
Disallow: /feed/$
Disallow: /*?d=
Disallow: /*?p=
Disallow: /*?replytocom=

# -----------------------------------------------
# Yandex
# -----------------------------------------------
User-agent: YandexBot
Crawl-delay: 28
Allow: /
Disallow: /category/
Disallow: /author/
Disallow: /uncategorized/
Disallow: /search/
Disallow: */feed/
Disallow: /feed/$
Disallow: /*?d=
Disallow: /*?p=
Disallow: /*?replytocom=

# -----------------------------------------------
# Baidu
# -----------------------------------------------
User-agent: Baiduspider
Crawl-delay: 30
Allow: /
Disallow: /category/
Disallow: /author/
Disallow: /uncategorized/
Disallow: /search/
Disallow: */feed/
Disallow: /feed/$
Disallow: /*?d=
Disallow: /*?p=
Disallow: /*?replytocom=

# -----------------------------------------------
# AI Bots
# -----------------------------------------------
User-agent: GPTBot
Crawl-delay: 10
Allow: /

User-agent: ChatGPT-User
Crawl-delay: 10
Allow: /

User-agent: OAI-SearchBot
Crawl-delay: 10
Allow: /

User-agent: Gemini
Crawl-delay: 10
Allow: /

# -----------------------------------------------
# Social Media Crawlers
# -----------------------------------------------
User-agent: facebookexternalhit
Crawl-delay: 30
Allow: /

User-agent: Twitterbot
Crawl-delay: 30
Allow: /

User-agent: LinkedInBot
Crawl-delay: 10
Allow: /

User-agent: Pinterestbot
Crawl-delay: 10
Allow: /

# -----------------------------------------------
# Apple / Alexa
# -----------------------------------------------
User-agent: Applebot
Crawl-delay: 17
Allow: /

User-agent: AlexaCrawler
Crawl-delay: 40
Allow: /

# -----------------------------------------------
Sitemap: https://drarchanaivf.com/sitemap_index.xml