#crawler google search
User-Agent: Googlebot
Crawl-delay: 30
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /css/
Disallow: /js/
Disallow: /photos/
Disallow: /scripts/