From ff882a4958ecd570f587099e27ac815aea895e65 Mon Sep 17 00:00:00 2001 From: Achilleas Pipinellis Date: Tue, 19 Jan 2021 17:30:52 +0000 Subject: Add noidex tag for stable branches We need to stop Google from indexing old docs, so we should make sure the noindex tag is added to all archived docs: https://developers.google.com/search/docs/advanced/crawling/block-indexing. That way Google will stop serving up results for pages 12.10 versions of pages, for example. --- content/robots.txt.erb | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'content/robots.txt.erb') diff --git a/content/robots.txt.erb b/content/robots.txt.erb index b368ca1e..f7bfd466 100644 --- a/content/robots.txt.erb +++ b/content/robots.txt.erb @@ -2,19 +2,18 @@ is_hidden: true --- -<% if ENV['CI_COMMIT_REF_NAME'] == ENV['CI_DEFAULT_BRANCH'] %> +<% if production_and_default_branch? %> +# If on default branch and production env sitemap: https://docs.gitlab.com/sitemap.xml user-agent: * disallow: /ce/ disallow: /debug/ -<% @items['/_data/versions.yaml'][:online].each do |version| %> -disallow: /<%= version %>/ -<% end %> -<% @items['/_data/versions.yaml'][:offline].each do |version| %> -disallow: /<%= version %>/ -<% end %> +<% elsif stable_version?(current_branch) %> +# If on stables branches +sitemap: https://docs.gitlab.com/<%= current_branch %>/sitemap.xml +user-agent: * <% else %> -# Ban all spiders from the entire site when not on master branch +# Ban all spiders from the entire site when not on default or stable branches user-agent: * disallow: / <% end %> -- cgit v1.2.3