User-agent: *    # applies to all robots
Disallow: /logs      # disallow indexing of pages
Disallow: /cgi-bin
Disallow: /css
Disallow: /error
Disallow: /test
Disallow: /_private
Disallow: /_vti_cnf
Disallow: /_vti_log
Disallow: /_vti_pvt
Disallow: /_vti_script
Disallow: /_vti_txt