# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: *
Disallow: /_notes
Disallow: /admin
Disallow: /banners
Disallow: /catalog
Disallow: /cookies
Disallow: /download
Disallow: /includes
Disallow: /logs
Disallow: /pub
Disallow: /templates
Disallow: /_notes/
Disallow: /admin/
Disallow: /banners/
Disallow: /catalog/
Disallow: /cookies/
Disallow: /download/
Disallow: /includes/
Disallow: /logs/
Disallow: /pub/
Disallow: /templates/