# robots.txt for waikato.ac.nz 
# If you're a search engine or crawl operator and want to make contact with us - webmaster@waikato.ac.nz

# default disallow rules for crawlers not grouped below
# Rules are intentionally duplicated across User-agent groups below.
# Per RFC 9309, crawlers apply only rules from a matching User-agent group;
# The wildcard (*) group is used only if no other group matches.
# Duplication is therefore required to enforce consistent Disallow rules.
User-agent: *
Disallow: /admin
Disallow: /Security/
Disallow: /search/
Disallow: /int/search/
# discourage previous site paths
Disallow: /calculator.php
Disallow: /cgi-bin/
Disallow: /contact-us/feedback
Disallow: /__data/assets/excel_doc/
Disallow: /__data/assets/pdf_file/
Disallow: /__data/assets/powerpoint_doc/
Disallow: /__data/assets/thumbnail/
Disallow: /__data/assets/word_doc/
Disallow: /healthandsafety/
Disallow: /human-resources/
Disallow: /odw/
Disallow: /staff/
Disallow: /tehowhao/
# discourage past papers by year
Disallow: /study/papers/*/2025
Disallow: /int/study/papers/*/2025
Disallow: /study/papers/2025/
Disallow: /study/papers/*/2024
Disallow: /int/study/papers/*/2024
Disallow: /study/papers/2024/
Disallow: /study/papers/*/2023
Disallow: /int/study/papers/*/2023
Disallow: /study/papers/2023/
Disallow: /study/papers/*/2022
Disallow: /int/study/papers/*/2022
Disallow: /study/papers/2022/
Disallow: /study/papers/*/2021
Disallow: /int/study/papers/*/2021
Disallow: /study/papers/2021/
Disallow: /study/papers/*/2020
Disallow: /int/study/papers/*/2020
Disallow: /study/papers/2020/
Disallow: /study/papers/*/201*
Disallow: /int/study/papers/*/201*
Disallow: /study/papers/201*
# discourage requests such as "/about/faculties-schools/science/home/?q=warbirds+over+wanaka+2026"
Disallow: /home
Disallow: /about/faculties-schools/science/home

# crawldelay 5 - 17280 crawls per day
User-agent: Amzn-SearchBot
User-agent: anthropic-ai
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Baiduspider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: Diffbot
User-agent: GPTBot
User-agent: meta-externalagent
User-agent: meta-webindexer
User-agent: PerplexityBot
User-agent: SemanticScholarBot
User-agent: SemrushBot
User-agent: SemrushBot-BA
User-agent: SemrushBot-SI
User-agent: simplecrawler
User-agent: SiteAuditBot
User-agent: SnapchatAdsBot
User-agent: VelenPublicWebCrawler 
User-agent: YouBot
Crawl-delay: 5
Disallow: /admin
Disallow: /Security/
Disallow: /search/
Disallow: /int/search/
# discourage previous site paths
Disallow: /calculator.php
Disallow: /cgi-bin/
Disallow: /contact-us/feedback
Disallow: /__data/assets/excel_doc/
Disallow: /__data/assets/pdf_file/
Disallow: /__data/assets/powerpoint_doc/
Disallow: /__data/assets/thumbnail/
Disallow: /__data/assets/word_doc/
Disallow: /healthandsafety/
Disallow: /human-resources/
Disallow: /odw/
Disallow: /staff/
Disallow: /tehowhao/
# discourage past papers by year
Disallow: /study/papers/*/2025
Disallow: /int/study/papers/*/2025
Disallow: /study/papers/2025/
Disallow: /study/papers/*/2024
Disallow: /int/study/papers/*/2024
Disallow: /study/papers/2024/
Disallow: /study/papers/*/2023
Disallow: /int/study/papers/*/2023
Disallow: /study/papers/2023/
Disallow: /study/papers/*/2022
Disallow: /int/study/papers/*/2022
Disallow: /study/papers/2022/
Disallow: /study/papers/*/2021
Disallow: /int/study/papers/*/2021
Disallow: /study/papers/2021/
Disallow: /study/papers/*/2020
Disallow: /int/study/papers/*/2020
Disallow: /study/papers/2020/
Disallow: /study/papers/*/201*
Disallow: /int/study/papers/*/201*
Disallow: /study/papers/201*
# discourage requests such as "/about/faculties-schools/science/home/?q=warbirds+over+wanaka+2026"
Disallow: /home
Disallow: /about/faculties-schools/science/home

# crawldelay 10 - 8640 crawls per day
User-agent: AhrefsBot
User-agent: Bingbot
User-agent: Caliperbot
User-agent: DataForSeoBot 
User-agent: Lanai
User-agent: Monsidobot
User-agent: SleepBot
User-agent: SogouMSE
User-agent: SogouSpider
User-agent: Sogou Pic spider
User-agent: Sogou web spider
Crawl-delay: 10
Disallow: /admin
Disallow: /Security/
Disallow: /search/
Disallow: /int/search/
# discourage previous site paths
Disallow: /calculator.php
Disallow: /cgi-bin/
Disallow: /contact-us/feedback
Disallow: /__data/assets/excel_doc/
Disallow: /__data/assets/pdf_file/
Disallow: /__data/assets/powerpoint_doc/
Disallow: /__data/assets/thumbnail/
Disallow: /__data/assets/word_doc/
Disallow: /healthandsafety/
Disallow: /human-resources/
Disallow: /odw/
Disallow: /staff/
Disallow: /tehowhao/
# discourage past papers by year
Disallow: /study/papers/*/2025
Disallow: /int/study/papers/*/2025
Disallow: /study/papers/2025/
Disallow: /study/papers/*/2024
Disallow: /int/study/papers/*/2024
Disallow: /study/papers/2024/
Disallow: /study/papers/*/2023
Disallow: /int/study/papers/*/2023
Disallow: /study/papers/2023/
Disallow: /study/papers/*/2022
Disallow: /int/study/papers/*/2022
Disallow: /study/papers/2022/
Disallow: /study/papers/*/2021
Disallow: /int/study/papers/*/2021
Disallow: /study/papers/2021/
Disallow: /study/papers/*/2020
Disallow: /int/study/papers/*/2020
Disallow: /study/papers/2020/
Disallow: /study/papers/*/201*
Disallow: /int/study/papers/*/201*
Disallow: /study/papers/201*
# discourage requests such as "/about/faculties-schools/science/home/?q=warbirds+over+wanaka+2026"
Disallow: /home
Disallow: /about/faculties-schools/science/home

# crawldelay 30 - 2880 crawls per day
# Crawl-delay: 30
# crawldelay 60 - 1440 crawls per day
# Crawl-delay: 60

# disallowed agents
User-agent: Amazonbot
User-agent: BLEXBot
User-agent: Bytespider
User-agent: DotBot
User-agent: ImagesiftBot
User-agent: Linespider
User-agent: MegaIndex.ru
User-agent: MJ12bot
User-agent: omgili
User-agent: omgilibot
User-agent: PetalBot
User-agent: Seekr
User-agent: SEOkicks
User-agent: serpstatbot
User-agent: spbot
User-agent: TikTokSpider
User-agent: ZoominfoBot
Disallow: /

Sitemap: https://www.waikato.ac.nz/sitemap.xml