User-agent: *
Crawl-delay: 30

User-agent: BaiDuSpider
Disallow: / 

User-agent: Twiceler
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: *
Disallow: /img

User-agent: *
Disallow: /web/

User-agent: *
Disallow: /web

User-agent: *
Disallow: /search

User-Agent: Charlotte
Disallow: /