From ae8a0fcc6c59f67c84f5ccb3de6bc725561bf32e Mon Sep 17 00:00:00 2001
From: Achilleas Pipinellis <axil@gitlab.com>
Date: Fri, 29 Jun 2018 08:42:19 +0200
Subject: Use _data/versions.yml to loop through disallow pages

---
 content/robots.txt.erb | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

(limited to 'content/robots.txt.erb')

diff --git a/content/robots.txt.erb b/content/robots.txt.erb
index cb837894..afeacaad 100644
--- a/content/robots.txt.erb
+++ b/content/robots.txt.erb
@@ -6,13 +6,15 @@ is_hidden: true
 sitemap: https://docs.gitlab.com/sitemap.xml
 user-agent: *
 disallow: /ce/
-disallow: /11.0/
-disallow: /10.8/
-disallow: /10.7/
-disallow: /10.6/
-disallow: /10.5/
-disallow: /10.4/
-disallow: /10.3/
+<% @items['/_data/versions.yaml'][:current].each do |version| %>
+disallow: /<%= version %>/
+<% end %>
+<% @items['/_data/versions.yaml'][:previous].each do |version| %>
+disallow: /<%= version %>/
+<% end %>
+<% @items['/_data/versions.yaml'][:offline].each do |version| %>
+disallow: /<%= version %>/
+<% end %>
 <% else %>
 # Ban all spiders from the entire site when not on master branch
 user-agent: *
-- 
cgit v1.2.3