# This site is very specific about who it allows crawling from. # Our default is to not allow crawling: User-agent: * Disallow: / # Below are the crawlers that are allowed to crawl this site. # Below that list, you'll find paths that are blocked, even for them, # and then paths within those blocked paths that are allowed. User-agent: AhrefsSiteAudit Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: AhrefsBot Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: Applebot Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: Bingbot Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: DuckDuckBot Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: Googlebot Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: Googlebot-Image Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: Googlebot-Video Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: GoogleOther Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: GoogleOther-Image Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: Google-InspectionTool Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ # Optional: Allows content access for Google AI systems outside Search User-agent: Google-Extended Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: AdsBot-Google Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: MediaPartners-Google Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: FacebookExternalHit Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: LinkedInBot Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: WhatsApp Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: Twitterbot Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: ia_archiver Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ User-agent: archive.org_bot Allow: / Disallow: /wp-json/ Disallow: /?rest_route= Disallow: /wp-admin/ Disallow: /wp-content/cache/ Disallow: /wp-content/plugins/ Disallow: /xmlrpc.php Disallow: /wp-includes/ Disallow: /wp-login.php Disallow: /wp-register.php Disallow: /search Disallow: /?s= Allow: /wp-includes/css/ Allow: /wp-includes/js/ Allow: /wp-admin/admin-ajax.php Allow: /wp-json/oembed/ Allow: /wp-content/uploads/ Allow: /*.css$ Allow: /*.js$ Allow: /*.png$ Allow: /*.jpg$ Allow: /*.jpeg$ Allow: /*.gif$ Allow: /*.svg$ Allow: /*.webp$ Allow: /*.avif$ Allow: /*.woff$ Allow: /*.woff2$ # XML Sitemap: Sitemap: https://www.shawu.edu/sitemap_index.xml