User-agent: *
Allow: /

# Paths (clean URLs)
Disallow: /category/news/
Disallow: /demo_aus/
Disallow: /category/
Disallow: /category/seo/
Disallow: /category/magento/
Disallow: /home/flow-img-1/
Disallow: /tag/
Disallow: /thank-you/
Disallow: /services/ios2/
Disallow: /managment-icon/
Disallow: /blog/tag/
Disallow: /portfolio/racechip/
Disallow: /current_opening/
Disallow: /biocera/
Disallow: /coading-icon/
Disallow: /2017/
Disallow: /thank-you-audit-page/
Disallow: /2014/
Disallow: /services/ios4/
Disallow: /mehndicreations/
Disallow: /home/
Disallow: /osd_display/
Disallow: /portfolio/
Disallow: /awaning-html-v6a/
Disallow: /elite/videos
Disallow: /author/admin/
Disallow: /services/strong/
Disallow: /portfolio/featherdown/
Disallow: /home/featured_icon_1/
Disallow: /blog/category/
Disallow: /blog/2014
Disallow: /blog/2016
Disallow: /blog/2017
Disallow: /blog/2018
Disallow: /blog/author/admin
Disallow: /mobile-app
Disallow: /thank-you-ecommerce/
Disallow: /thank-you-mobile-app/
Disallow: /thank-you-digital-marketing
Disallow: /test-contact-form-7-email
Disallow: /blog/take-advantage-of-magneto-christmas-offer-for-2017/christmas-offer
Disallow: /blog/open-source-technology-trends-2016-2/open-source-technology-trends
Disallow: /blog/10-useful-angular-js-development-tools/angular-js-development-tools
Disallow: /estimated-mobile-apps-download
Disallow: /blog/author/juned
Disallow: /logo-white
Disallow: /portfolio_category
Disallow: /?attachment_id
Disallow: /?s={search_term}
Disallow: /.git/config
Disallow: */feed
Disallow: */feed/
Disallow: /feed/

# --- Standard Search Crawlers --- #
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: *
Allow: /

# --- AI/LLM CRAWLER --- #
User-agent: ChatGPT-User
Allow: /
User-agent: GPTBot
Allow: /
User-agent: OAI-Searchbot
Allow: /
User-agent: FacebookBot
Allow: /
User-agent: meta-externalagent
Allow: /
User-agent: YouBot
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Omgilibot
Allow: /
User-agent: omgili
Allow: /
User-agent: Applebot-Extended
Allow: /
User-agent: DuckAssistBot
Allow: /
User-agent: Anthropic-ai
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: CCBot
Allow: /
User-agent: Amazonbot
Allow: /
User-agent: cohere-ai
Allow: /
User-agent: FacebookExternalHit
Allow: /
User-agent: Gemini
Allow: /

# --- SEMrushBot Access --- #
User-agent: SemrushBot
Allow: /

# --- Sitemaps --- #
Sitemap: https://magnetoitsolutions.com/sitemap_index.xml
Sitemap: https://magnetoitsolutions.com/uk/sitemap_index.xml
Sitemap: https://magnetoitsolutions.com/au/sitemap_index.xml
Sitemap: https://magnetoitsolutions.com/sa/sitemap_index.xml
Sitemap: https://magnetoitsolutions.com/canada/sitemap_index.xml
Sitemap: https://magnetoitsolutions.com/kuwait/sitemap_index.xml
Sitemap: https://magnetoitsolutions.com/dubai/sitemap_index.xml

# --- LLMS File --- #
Allow: /llms.txt
Allow: /au/llms.txt
Allow: /uk/llms.txt
Allow: /sa/llms.txt