Add noidex tag for stable branches

We need to stop Google from indexing old docs, so we should make sure
the noindex tag is added to all archived docs:
https://developers.google.com/search/docs/advanced/crawling/block-indexing.

That way Google will stop serving up results for pages 12.10 versions of
pages, for example.

3 files changed, 9 insertions, 10 deletions

diff --git a/content/robots.txt.erb b/content/robots.txt.erb
index b368ca1e..f7bfd466 100644
--- a/content/robots.txt.erb
+++ b/content/robots.txt.erb
@@ -2,19 +2,18 @@
 is_hidden: true
 ---
 
-<% if ENV['CI_COMMIT_REF_NAME'] == ENV['CI_DEFAULT_BRANCH'] %>
+<% if production_and_default_branch? %>
+# If on default branch and production env
 sitemap: https://docs.gitlab.com/sitemap.xml
 user-agent: *
 disallow: /ce/
 disallow: /debug/
-<% @items['/_data/versions.yaml'][:online].each do |version| %>
-disallow: /<%= version %>/
-<% end %>
-<% @items['/_data/versions.yaml'][:offline].each do |version| %>
-disallow: /<%= version %>/
-<% end %>
+<% elsif stable_version?(current_branch) %>
+# If on stables branches
+sitemap: https://docs.gitlab.com/<%= current_branch %>/sitemap.xml
+user-agent: *
 <% else %>
-# Ban all spiders from the entire site when not on master branch
+# Ban all spiders from the entire site when not on default or stable branches
 user-agent: *
 disallow: /
 <% end %>
diff --git a/layouts/head.html b/layouts/head.html
index 5bf94d63..c88972d1 100644
--- a/layouts/head.html
+++ b/layouts/head.html
@@ -10,7 +10,7 @@
   <meta name="description" content="Documentation for GitLab Community Edition, GitLab Enterprise Edition, Omnibus GitLab, and GitLab Runner.">
 <% end %>
 <meta name="viewport" content="width=device-width, initial-scale=1, user-scalable=no">
-<% if @item[:noindex] %>
+<% if @item[:noindex] or !production_and_default_branch? %>
   <meta name="robots" content="noindex nofollow">
 <% end %>
 <!--https://community.algolia.com/docsearch/required-configuration.html#introduces-global-information-as-meta-tags-->
diff --git a/lib/helpers/generic.rb b/lib/helpers/generic.rb
index 264ba497..76399023 100644
--- a/lib/helpers/generic.rb
+++ b/lib/helpers/generic.rb
@@ -16,7 +16,7 @@ module Nanoc::Helpers
     # which they are considered production.
     #
     def production_and_default_branch?
-      ENV['NANOC_ENV'] == 'production' and ENV['CI_DEFAULT_BRANCH'] == ENV['CI_COMMIT_BRANCH']
+      ENV['NANOC_ENV'] == 'production' and ENV['CI_DEFAULT_BRANCH'] == ENV['CI_COMMIT_REF_NAME']
     end
 
     #