* update robots.txt

7 years ago · f809fe3bf5
1 changed files with 8 additions and 1 deletions
--- a/theme/site/robots.txt.ep
+++ b/theme/site/robots.txt.ep
@ -2,7 +2,14 @@ Host: linuxdv.org

 Sitemap: <%= $site->url('sitemap.xml') %>
 User-Agent: *
-Disallow:
+Disallow: /zerobin2/
+Disallow: /git/

 User-Agent: AhrefsBot
 Disallow: *
+
+User-Agent: MJ12bot
+Disallow: *
+
+User-agent: WebIndex
+Disallow: *