User-agent: * Disallow: /counter/ Disallow: /diensten/ Disallow: /gastenboek/ Disallow: /gemeente/ Disallow: /img/ Disallow: /inc/ Disallow: /links/ #Sitemap: http://www.biddenvoordeveenkolonien.nl/sitemap.xml # Hits many times per second, not acceptable # http://www.nameprotect.com/botinfo.html User-agent: NPBot Disallow: / # A capture bot, downloads gazillions of pages with no public benefit # http://www.webreaper.net/ User-agent: WebReaper Disallow: / # The 'grub' distributed client has been *very* poorly behaved. User-agent: grub-client Disallow: / # advertising-related bots: User-agent: Mediapartners-Google Disallow: / # Doesn't follow robots.txt anyway, but... User-agent: k2spider Disallow: / # Some bots are known to be trouble, particularly those designed to copy entire sites. Please obey robots.txt. User-agent: sitecheck.internetseer.com Disallow: /