Top Sites>WWikiSpooks Home

# bots that ignore robots.txt

User-agent: *

Disallow: /here-be-gremlins

# advertising-related bots:

User-agent: Mediapartners-Google*

Disallow: /


# Wikipedia work bots:

User-agent: IsraBot

Disallow: /


User-agent: Orthogaffe

Disallow: /


# Crawlers that are kind enough to obey, but which we'd rather not have

# unless they're feeding search engines.

User-agent: UbiCrawler

Disallow: /


User-agent: DOC

Disallow: /


User-agent: Zao

Disallow: /


# Some bots are known to be trouble, particularly those designed to copy

# entire sites. Please obey robots.txt.

User-agent: sitecheck.internetseer.com

Disallow: /


User-agent: Zealbot

Disallow: /


User-agent: MSIECrawler

Disallow: /


User-agent: SiteSnagger

Disallow: /


User-agent: WebStripper

Disallow: /


User-agent: WebCopier

Disallow: /


User-agent: Fetch

Disallow: /


User-agent: Offline Explorer

Disallow: /


User-agent: Teleport

Disallow: /


User-agent: TeleportPro

Disallow: /


User-agent: WebZIP

Disallow: /


User-agent: linko

Disallow: /


User-agent: HTTrack

Disallow: /


User-agent: Microsoft.URL.Control

Disallow: /


User-agent: Xenu

Disallow: /


User-agent: larbin

Disallow: /


User-agent: libwww

Disallow: /


User-agent: ZyBORG

Disallow: /


User-agent: Download Ninja

Disallow: /


# Sorry, wget in its recursive mode is a frequent problem.

# Please read the man page and use it properly; there is a

# --wait option you can use to set the delay between hits,

# for instance.

#

User-agent: wget

Disallow: /


#

# The 'grub' distributed client has been *very* poorly behaved.

#

User-agent: grub-client

Disallow: /


#

# Doesn't follow robots.txt anyway, but...

#

User-agent: k2spider

Disallow: /


# Hits many times per second, not acceptable

# http://www.nameprotect.com/botinfo.html

User-agent: NPBot

Disallow: /


# A capture bot, downloads gazillions of pages with no public benefit

# http://www.webreaper.net/

User-agent: WebReaper

Disallow: /


### Allow internet archiver bot
User-agent: ia_archiver

Allow: /backups/	#Backups
Allow: /w/images/	#Images - for better archiving of pages
Allow: /w/		#Allow archiving of source code?


User-agent: *

Disallow: /secure/

Disallow: /backups/

Disallow: /piwik/

Disallow: /anon/

Disallow: /memcachedadmin/

Disallow: /Lux/

Disaalow: /ISGP/

Disallow: /ISGP-old/

Disaalow: /linux-dash/

Disallow: /thanks/

Disallow: /wikileaks/

Disallow: /wiki/Special:Search

Disallow: /wiki/Special:Random

Disallow: /MANUAL/

Disallow: /backup/

Disallow: /indices-1.1/

Disallow: /uloads/

disallow: /wiki/images/deleted/

disallow: /wiki/cache/


User-agent: googlebot

Disallow: /secure/

Disallow: /piwik/

Disallow: /anon/

Disallow: /w/

Disallow: /wiki/Special:Search

Disallow: /wiki/Special:Random

Disallow: /MANUAL/

Disallow: /backup/

Disallow: /indices-1.1/

Disallow: /uloads/

Disallow: /wiki/Special:Ask/

Disallow: /wiki/Special:Browse/

Disallow: /wiki/Special:SearchByProperty/

Disallow: /wiki/Special:ExportRDF/

Disallow: /wiki/Special:PageProperty/

Disallow: /wiki/Special:Properties/

Disallow: /wiki/Special:UnusedProperties/

Disallow: /wiki/Special:WantedProperties/

Disallow: /wiki/Special:SMWAdmin/

Disallow: /wiki/Special:Types/

Disallow: /wiki/Special:URIResolver/

Disallow: /wiki/Special:QueryCreator/

# Wikispooks

> An open source encylopedia of deep politics.

This is a creative commons licensed website containing deep political information such as lists of spooky conferences (e.g. Meetings of Le Cercle, the Bilderberg, Brusses Forum, Munich Security Conferences) and the people who attended them.

## Contents

- [About Wikispooks](https://wikispooks.com/wiki/Wikispooks:About): About the site

## Optional

- [FAQ](https://wikispooks.com/wiki/Wikispooks:FAQ)