# Block specific unwanted bots
User-agent: CCBot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# You explicitly allow Scrapy (internal tooling, etc.)
User-agent: Scrapy
Allow: /

# Explicitly allow LinkedIn previews
User-agent: LinkedInBot
Disallow:

# Explicitly allow LoomlyBot
User-agent: LoomlyBot
Disallow:
# Might be related to loomly
User-agent: trendictionbot
Disallow: 

# Default policy for everyone else
User-agent: *
# Block a few specific files
Disallow: /wp-content/uploads/wpo-plugins-tables-list.json
Disallow: /wp-content/uploads/2023/07/GWTS23-Sponsorship-Brochure-2023-V2_compressed.pdf
Disallow: /wp-content/uploads/2023/07/GINSUSA23-Sponsorship-Brochure-v4_compressed.pdf
Disallow: /wp-content/uploads/2023/07/AI-in-Financial-Services-Forum-2023-Sponsorship-Brochure.pdf
Disallow: /wp-content/uploads/2023/07/Global-RegTech-Summit-USA-2023-Sponsorship-Brochure.pdf

# Keep admin & login out of good crawlers
Disallow: /wp-admin/
Disallow: /wp-login.php
Allow: /wp-admin/admin-ajax.php

# (Optional) throttle some crawlers (Google ignores this)
Crawl-delay: 10

# Hide the WP REST API from generic crawlers
Disallow: /wp-json/

# Sitemap for YOAST
Sitemap: https://fintech.global/sitemap_index.xml