User-agent: *
Allow: /wp-admin/admin-ajax.php
Allow: /wp-includes/*.css
Allow: /wp-includes/*.js
Disallow: /wp-admin/
Disallow: /temp/
Disallow: /private/
Disallow: /xmlrpc.php
Disallow: /readme.html

# Allow AI crawlers to crawl but not train on content (optional, based on preference)
User-agent: GPTBot
Disallow: /private/
Disallow: /archive/

# Allow Google-Extended to parse content for AI snippets
User-agent: Google-Extended
Allow: /

# Sitemap Location
Sitemap: https://dredgebrokers.com/sitemap_index.xml
Sitemap: https://dredgebrokers.com/page-sitemap.xml
Sitemap: https://dredgebrokers.com/dredging-equipment-sitemap.xml
Sitemap: https://dredgebrokers.com/equipment-sitemap.xml

# Reference to the llms.txt file
llms-txt: https://dredgebrokers.com/llms.txt