diff options
Diffstat (limited to 'content/robots.txt.erb')
| -rw-r--r-- | content/robots.txt.erb | 8 |
1 files changed, 5 insertions, 3 deletions
diff --git a/content/robots.txt.erb b/content/robots.txt.erb index b4a4b64f..e228e9f8 100644 --- a/content/robots.txt.erb +++ b/content/robots.txt.erb @@ -3,9 +3,11 @@ is_hidden: true --- <% if ENV['CI_COMMIT_REF_NAME'] == 'master' %> -Sitemap: <%= @items['/sitemap.*'].path %> +sitemap: https://docs.gitlab.com/sitemap.xml +user-agent: * +disallow: /ce/ <% else %> # Ban all spiders from the entire site when not on master branch -User-Agent: * -Disallow: / +user-agent: * +disallow: / <% end %> |