# robots.txt for http://www.example.com/ User-agent: * Disallow: /assets/ Disallow: /calendar/ Disallow: /cms/ Disallow: /css/ Disallow: /flash/ Disallow: /i/ Disallow: /quotes/ Disallow: *.txt Disallow: /sphider/