diff --git a/theme/site/robots.txt.ep b/theme/site/robots.txt.ep index 917de34..52c2953 100644 --- a/theme/site/robots.txt.ep +++ b/theme/site/robots.txt.ep @@ -2,7 +2,14 @@ Host: linuxdv.org Sitemap: <%= $site->url('sitemap.xml') %> User-Agent: * -Disallow: +Disallow: /zerobin2/ +Disallow: /git/ User-Agent: AhrefsBot Disallow: * + +User-Agent: MJ12bot +Disallow: * + +User-agent: WebIndex +Disallow: *