User-agent: * # applies to all robots
Disallow: /cgi-bin/ # applies to cgi-bin
Disallow: /tmp/ # applies to tmp folder
Disallow: /test # applies to test folder and contents, /test disallows both /test.html and /test/index.html, whereas
Disallow: /test/ # would disallow /test/index.html but allow /test.html
Disallow: /private/ # would disallow /private/index.html but allow /private.html
Disallow: /client # applies to client folder and contents
Disallow: /dev # applies to dev folder and contents