#Disallow all crawlers access to certain dirs.

User-agent: *
Disallow: /pdf/
Disallow: /test/
Disallow: /csr/

User-Agent: Googlebot
Disallow: /pdf/
Disallow: /test/
Disallow: /csr/

#disallow specific file types

User-agent: *
Disallow: /*.gif$

User-agent: *
Disallow: /*.jpg$

User-agent: *
Disallow: /*.png$


User-agent: *
Disallow: /*.pdf$