From aa98bb823e3a41b510e8fd757cc0e7fb93841657 Mon Sep 17 00:00:00 2001 From: Marcia Ramos Date: Thu, 28 Jun 2018 18:44:05 +0100 Subject: fix robots.txt to prevent bots from indexing versioned docs --- content/robots.txt.erb | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'content/robots.txt.erb') diff --git a/content/robots.txt.erb b/content/robots.txt.erb index e228e9f8..cb837894 100644 --- a/content/robots.txt.erb +++ b/content/robots.txt.erb @@ -6,6 +6,13 @@ is_hidden: true sitemap: https://docs.gitlab.com/sitemap.xml user-agent: * disallow: /ce/ +disallow: /11.0/ +disallow: /10.8/ +disallow: /10.7/ +disallow: /10.6/ +disallow: /10.5/ +disallow: /10.4/ +disallow: /10.3/ <% else %> # Ban all spiders from the entire site when not on master branch user-agent: * -- cgit v1.2.3