User-agent: *
Disallow: /ref
Disallow: /to/
Disallow: /artist/list/search

User-Agent: Twitterbot
Allow: /ref

User-Agent: MJ12bot
Crawl-delay: 600