# robots.txt
# set a default 5s delay between hits
User-agent: *
Crawl-delay: 5
# don't let them hit some directories
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /administrator/
Disallow: /templates_c/
Disallow: /templates_admin_c/
Disallow: /cache/