Add latest changes from gitlab-org/gitlab@master

15 files changed, 234 insertions, 33 deletions

diff --git a/app/views/admin/application_settings/ci_cd.html.haml b/app/views/admin/application_settings/ci_cd.html.haml
index a2aa1687f80..81f06926188 100644
--- a/app/views/admin/application_settings/ci_cd.html.haml
+++ b/app/views/admin/application_settings/ci_cd.html.haml
@@ -15,6 +15,8 @@
 
 = render_if_exists 'admin/application_settings/required_instance_ci_setting', expanded: expanded_by_default?
 
+= render_if_exists 'admin/application_settings/package_registry', expanded: expanded_by_default?
+
 - if Gitlab.config.registry.enabled
   %section.settings.as-registry.no-animate#js-registry-settings{ class: ('expanded' if expanded_by_default?) }
     .settings-header
diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml
index 18bdbd42d0d..4c20ac84b24 100644
--- a/app/views/projects/ci/builds/_build.html.haml
+++ b/app/views/projects/ci/builds/_build.html.haml
@@ -45,7 +45,7 @@
             = tag
       - if job.try(:trigger_request)
         %span.badge.badge-info= _('triggered')
-      - if job.try(:allow_failure)
+      - if job.try(:allow_failure) && !job.success?
         %span.badge.badge-warning= _('allowed to fail')
       - if job.schedulable?
         %span.badge.badge-info= s_('DelayedJobs|delayed')
diff --git a/changelogs/unreleased/198052-in-the-jobs-page-the-allowed-to-fail-badge-looks-like-an-error.yml b/changelogs/unreleased/198052-in-the-jobs-page-the-allowed-to-fail-badge-looks-like-an-error.yml
new file mode 100644
index 00000000000..c69fb956219
--- /dev/null
+++ b/changelogs/unreleased/198052-in-the-jobs-page-the-allowed-to-fail-badge-looks-like-an-error.yml
@@ -0,0 +1,5 @@
+---
+title: Hides the "Allowed to fail" tag on jobs that are successful
+merge_request: 25458
+author:
+type: changed
diff --git a/doc/administration/static_objects_external_storage.md b/doc/administration/static_objects_external_storage.md
index ab4abb7716a..373e4366a1f 100644
--- a/doc/administration/static_objects_external_storage.md
+++ b/doc/administration/static_objects_external_storage.md
@@ -2,8 +2,8 @@
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/31025) in GitLab 12.3.
 
-GitLab can be configured to serve repository static objects (for example, archives) from an external
-storage, such as a CDN.
+GitLab can be configured to serve repository static objects (for example, archives or raw blobs) from an external
+storage, such as a Content Delivery Network (CDN).
 
 ## Configuring
 
@@ -11,7 +11,8 @@ To configure external storage for static objects:
 
 1. Navigate to **Admin Area > Settings > Repository**.
 1. Expand the **Repository static objects** section.
-1. Enter the base URL and an arbitrary token.
+1. Enter the base URL and an arbitrary token. When you [set up external storage](#set-up-external-storage),
+you'll use a script that uses these values as `ORIGIN_HOSTNAME` and `STORAGE_TOKEN`.
 
 The token is required to distinguish requests coming from the external storage, so users don't
 circumvent the external storage and go for the application directly. The token is expected to be
@@ -48,3 +49,173 @@ sequenceDiagram
       CDN->>User: master.zip
     end
 ```
+
+## Set up external storage
+
+While this procedure uses [CloudFlare Workers](https://workers.cloudflare.com) for external storage,
+other CDNs or Function as a Service (FaaS) systems should work using the same principles.
+
+1. Choose a CloudFlare Worker domain if you haven't done so already.
+1. In the following script, set the following values for the first two constants:
+
+   - `ORIGIN_HOSTNAME`: the hostname of your GitLab installation.
+   - `STORAGE_TOKEN`: any arbitrary secure token (e.g. you can get one by running
+     `pwgen -cn1 64` on a UNIX machine). Save this token for the admin panel, as
+     described in the [configuring](#configuring) section.
+
+    ```js
+    const ORIGIN_HOSTNAME = 'gitlab.installation.com' // FIXME: SET CORRECT VALUE
+    const STORAGE_TOKEN = 'very-secure-token' // FIXME: SET CORRECT VALUE
+    const CACHE_PRIVATE_OBJECTS = false
+
+    const CORS_HEADERS = {
+      'Access-Control-Allow-Origin': '*',
+      'Access-Control-Allow-Methods': 'GET, HEAD, OPTIONS',
+      'Access-Control-Allow-Headers': 'X-Csrf-Token, X-Requested-With',
+    }
+
+    self.addEventListener('fetch', event => event.respondWith(handle(event)))
+
+    async function handle(event) {
+      try {
+        let response = await verifyAndHandle(event);
+
+        // responses returned from cache are immutable, so we recreate them
+        // to set CORS headers
+        response = new Response(response.body, response)
+        response.headers.set('Access-Control-Allow-Origin', '*')
+
+        return response
+      } catch (e) {
+        return new Response('An error occurred!', {status: e.statusCode || 500})
+      }
+    }
+
+    async function verifyAndHandle(event) {
+      if (!validRequest(event.request)) {
+        return new Response(null, {status: 400})
+      }
+
+      if (event.request.method === 'OPTIONS') {
+        return handleOptions(event.request)
+      }
+
+      return handleRequest(event)
+    }
+
+    function handleOptions(request) {
+      // Make sure the necessary headers are present
+      // for this to be a valid pre-flight request
+      if (
+        request.headers.get('Origin') !== null &&
+        request.headers.get('Access-Control-Request-Method') !== null &&
+        request.headers.get('Access-Control-Request-Headers') !== null
+      ) {
+        // Handle CORS pre-flight request
+        return new Response(null, {
+          headers: CORS_HEADERS,
+        })
+      } else {
+        // Handle standard OPTIONS request
+        return new Response(null, {
+          headers: {
+            Allow: 'GET, HEAD, OPTIONS',
+          },
+        })
+      }
+    }
+
+    async function handleRequest(event) {
+      let cache = caches.default
+      let url = new URL(event.request.url)
+      let static_object_token = url.searchParams.get('token')
+      let headers = new Headers(event.request.headers)
+
+      url.host = ORIGIN_HOSTNAME
+      url = normalizeQuery(url)
+
+      headers.set('X-Gitlab-External-Storage-Token', STORAGE_TOKEN)
+      if (static_object_token !== null) {
+        headers.set('X-Gitlab-Static-Object-Token', static_object_token)
+      }
+
+      let request = new Request(url, { headers: headers })
+      let cached_response = await cache.match(request)
+      let is_conditional_header_set = headers.has('If-None-Match')
+
+      if (cached_response) {
+        return cached_response
+      }
+
+      // We don't want to override If-None-Match that is set on the original request
+      if (cached_response && !is_conditional_header_set) {
+        headers.set('If-None-Match', cached_response.headers.get('ETag'))
+      }
+
+      let response = await fetch(request, {
+        headers: headers,
+        redirect: 'manual'
+      })
+
+      if (response.status == 304) {
+        if (is_conditional_header_set) {
+          return response
+        } else {
+          return cached_response
+        }
+      } else if (response.ok) {
+        response = new Response(response.body, response)
+
+        // cache.put will never cache any response with a Set-Cookie header
+        response.headers.delete('Set-Cookie')
+
+        if (CACHE_PRIVATE_OBJECTS) {
+          response.headers.delete('Cache-Control')
+        }
+
+        event.waitUntil(cache.put(request, response.clone()))
+      }
+
+      return response
+    }
+
+    function normalizeQuery(url) {
+      let searchParams = url.searchParams
+      url = new URL(url.toString().split('?')[0])
+
+      if (url.pathname.includes('/raw/')) {
+        let inline = searchParams.get('inline')
+
+        if (inline == 'false' || inline == 'true') {
+          url.searchParams.set('inline', inline)
+        }
+      } else if (url.pathname.includes('/-/archive/')) {
+        let append_sha = searchParams.get('append_sha')
+        let path = searchParams.get('path')
+
+        if (append_sha == 'false' || append_sha == 'true') {
+          url.searchParams.set('append_sha', append_sha)
+        }
+        if (path) {
+          url.searchParams.set('path', path)
+        }
+      }
+
+      return url
+    }
+
+    function validRequest(request) {
+      let url = new URL(request.url)
+      let path = url.pathname
+
+      if (/^(.+)(\/raw\/|\/-\/archive\/)/.test(path)) {
+        return true
+      }
+
+      return false
+    }
+    ```
+
+1. Create a new worker with this script.
+1. Copy your values for `ORIGIN_HOSTNAME` and `STORAGE_TOKEN`.
+   Use those values [to configure external storage for static objects](#configuring).
diff --git a/doc/api/groups.md b/doc/api/groups.md
index 14e04b9dd17..4c287386c59 100644
--- a/doc/api/groups.md
+++ b/doc/api/groups.md
@@ -498,7 +498,7 @@ Parameters:
 
 ## Transfer project to group
 
-Transfer a project to the Group namespace. Available only to instance administrators. Transferring projects may fail when tagged packages exist in the project's repository.
+Transfer a project to the Group namespace. Available only to instance administrators, although an [alternative API endpoint](projects.md#transfer-a-project-to-a-new-namespace) is available which does not require instance administrator access. Transferring projects may fail when tagged packages exist in the project's repository.
 
 ```
 POST  /groups/:id/projects/:project_id
diff --git a/doc/api/users.md b/doc/api/users.md
index d9fcaafc30f..ecd3c306d81 100644
--- a/doc/api/users.md
+++ b/doc/api/users.md
@@ -1435,14 +1435,14 @@ Please note that `last_activity_at` is deprecated, please use `last_activity_on`
 
 ## User memberships (admin only)
 
-> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22518) in GitLab 12.8.
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/20532) in GitLab 12.8.
 
 Lists all projects and groups a user is a member of. This endpoint is available for admins only.
 It returns the `source_id`, `source_name`, `source_type` and `access_level` of a membership.
-Source can be of type `Namespace` (representing a group) or `Project`. The response represents only direct memberships. Inherited memberships, for example in subgroups, will not be included.
-Access levels will be represented by an integer value. Read more about the meaning of access level values [here](access_requests.md#valid-access-levels).
+Source can be of type `Namespace` (representing a group) or `Project`. The response represents only direct memberships. Inherited memberships, for example in subgroups, are not included.
+Access levels are represented by an integer value. For more details, read about the meaning of [access level values](access_requests.md#valid-access-levels).
 
-```
+```plaintext
 GET /users/:id/memberships
 ```
 
@@ -1456,7 +1456,7 @@ Parameters:
 Returns:
 
 - `200 OK` on success.
-- `404 User Not Found` if user cannot be found.
+- `404 User Not Found` if user can't be found.
 - `403 Forbidden` when not requested by an admin.
 - `400 Bad Request` when requested type is not supported.
 
diff --git a/doc/integration/bitbucket.md b/doc/integration/bitbucket.md
index 7a1bdf0d65c..1ce361761ff 100644
--- a/doc/integration/bitbucket.md
+++ b/doc/integration/bitbucket.md
@@ -1,7 +1,8 @@
 # Integrate your GitLab server with Bitbucket Cloud
 
 NOTE: **Note:**
-You need to [enable OmniAuth](omniauth.md) in order to use this.
+Starting from GitLab 11.4, OmniAuth is enabled by default. If you're using an
+earlier version, you'll need to explicitly enable it.
 
 Import projects from Bitbucket.org and login to your GitLab instance with your
 Bitbucket.org account.
diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md
index 3f286ca1057..cffd261e757 100644
--- a/doc/integration/elasticsearch.md
+++ b/doc/integration/elasticsearch.md
@@ -47,24 +47,26 @@ updated automatically.
 
 For indexing Git repository data, GitLab uses an [indexer written in Go](https://gitlab.com/gitlab-org/gitlab-elasticsearch-indexer).
 
+The way you install the Go indexer depends on your version of GitLab:
+
+- For GitLab Omnibus 11.8 and above, see [GitLab Omnibus](#gitlab-omnibus).
+- For older versions of GitLab, install the indexer [From Source](#from-source).
+
+### GitLab Omnibus
+
 The Go indexer was included in Omnibus GitLab 11.8 as an optional replacement to a
 Ruby-based indexer. [Since GitLab v12.3](https://gitlab.com/gitlab-org/gitlab/issues/6481),
-all indexing is done by the Go indexer, and the Ruby indexer is removed.
 
-If you would like to use the Elasticsearch Go indexer with a source installation or an older version of GitLab, please follow the instructions below.
-
-### Installation
+### From source
 
 First, we need to install some dependencies, then we'll build and install
 the indexer itself.
 
-#### Dependencies
-
 This project relies on [ICU](http://site.icu-project.org/) for text encoding,
 therefore we need to ensure the development packages for your platform are
 installed before running `make`.
 
-##### Debian / Ubuntu
+#### Debian / Ubuntu
 
 To install on Debian or Ubuntu, run:
 
@@ -72,7 +74,7 @@ To install on Debian or Ubuntu, run:
 sudo apt install libicu-dev
 ```
 
-##### CentOS / RHEL
+#### CentOS / RHEL
 
 To install on CentOS or RHEL, run:
 
@@ -89,7 +91,7 @@ brew install icu4c
 export PKG_CONFIG_PATH="/usr/local/opt/icu4c/lib/pkgconfig:$PKG_CONFIG_PATH"
 ```
 
-#### Building and installing
+### Building and installing
 
 To build and install the indexer, run:
 
@@ -622,7 +624,7 @@ Here are some common pitfalls and how to overcome them:
    ```
 
    You probably have not used either `http://` or `https://` as part of your value in the **"URL"** field of the Elasticseach Integration Menu. Please make sure you are using either `http://` or `https://` in this field as the [Elasticsearch client for Go](https://github.com/olivere/elastic) that we are using [needs the prefix for the URL to be acceped as valid](https://github.com/olivere/elastic/commit/a80af35aa41856dc2c986204e2b64eab81ccac3a).
-   Once you have corrected the formatting of the URL please delete the index (via the [dedicated rake task](#gitlab-elasticsearch-rake-tasks)) and [index the content of your intance](#adding-gitlabs-data-to-the-elasticsearch-index) once more.
+   Once you have corrected the formatting of the URL, delete the index (via the [dedicated rake task](#gitlab-elasticsearch-rake-tasks)) and [reindex the content of your instance](#adding-gitlabs-data-to-the-elasticsearch-index).
 
 ### Reverting to basic search
 
diff --git a/doc/integration/saml.md b/doc/integration/saml.md
index 30ba390f5cd..31d41433e6b 100644
--- a/doc/integration/saml.md
+++ b/doc/integration/saml.md
@@ -1,9 +1,11 @@
 # SAML OmniAuth Provider
 
-> This topic is for SAML on self-managed GitLab instances. For SAML on GitLab.com, see [SAML SSO for GitLab.com Groups](../user/group/saml_sso/index.md).
+Note that:
 
-NOTE: **Note:**
-You need to [enable OmniAuth](omniauth.md) in order to use this.
+- SAML OmniAuth Provider is for SAML on self-managed GitLab instances. For SAML on
+  GitLab.com, see [SAML SSO for GitLab.com Groups](../user/group/saml_sso/index.md).
+- Starting from GitLab 11.4, OmniAuth is enabled by default. If you're using an
+  earlier version, you'll need to explicitly enable it.
 
 GitLab can be configured to act as a SAML 2.0 Service Provider (SP). This allows
 GitLab to consume assertions from a SAML 2.0 Identity Provider (IdP) such as
diff --git a/doc/integration/shibboleth.md b/doc/integration/shibboleth.md
index 71ac6012ce6..4cc686cc0b6 100644
--- a/doc/integration/shibboleth.md
+++ b/doc/integration/shibboleth.md
@@ -16,7 +16,7 @@ The following changes are needed to enable Shibboleth:
 
 1. Protect OmniAuth Shibboleth callback URL:
 
-   ```
+   ```apache
    <Location /users/auth/shibboleth/callback>
      AuthType shibboleth
      ShibRequestSetting requireSession 1
@@ -36,7 +36,7 @@ The following changes are needed to enable Shibboleth:
 
 1. Exclude Shibboleth URLs from rewriting. Add `RewriteCond %{REQUEST_URI} !/Shibboleth.sso` and `RewriteCond %{REQUEST_URI} !/shibboleth-sp`. Config should look like this:
 
-   ```
+   ```apache
    # Apache equivalent of Nginx try files
    RewriteEngine on
    RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f
@@ -46,8 +46,12 @@ The following changes are needed to enable Shibboleth:
    RequestHeader set X_FORWARDED_PROTO 'https'
    ```
 
-1. Edit `/etc/gitlab/gitlab.rb` configuration file to enable OmniAuth and add
-   Shibboleth as an OmniAuth provider. User attributes will be sent from the
+   **NOTE:**
+   Starting from GitLab 11.4, OmniAuth is enabled by default. If you're using an
+   earlier version, you'll need to explicitly enable it in `/etc/gitlab/gitlab.rb`.
+
+1. In addition, add Shibboleth to `/etc/gitlab/gitlab.rb` as an OmniAuth provider.
+   User attributes will be sent from the
    Apache reverse proxy to GitLab as headers with the names from the Shibboleth
    attribute mapping. Therefore the values of the `args` hash
    should be in the form of `"HTTP_ATTRIBUTE"`. The keys in the hash are arguments
diff --git a/doc/update/patch_versions.md b/doc/update/patch_versions.md
index ac7aee779c0..02792730d2b 100644
--- a/doc/update/patch_versions.md
+++ b/doc/update/patch_versions.md
@@ -96,7 +96,7 @@ sudo -u git -H make
 
 ### 8. Install/Update `gitlab-elasticsearch-indexer` **(STARTER ONLY)**
 
-Please follow the [install instruction](../integration/elasticsearch.md#installation).
+Please follow the [install instruction](../integration/elasticsearch.md#installing-elasticsearch).
 
 ### 9. Start application
 
diff --git a/doc/update/upgrading_from_ce_to_ee.md b/doc/update/upgrading_from_ce_to_ee.md
index 5c395ddba70..28d5fe7aa5f 100644
--- a/doc/update/upgrading_from_ce_to_ee.md
+++ b/doc/update/upgrading_from_ce_to_ee.md
@@ -79,7 +79,7 @@ sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production
 
 ### 4. Install `gitlab-elasticsearch-indexer` **(STARTER ONLY)**
 
-Please follow the [install instruction](../integration/elasticsearch.md#installation).
+Please follow the [install instruction](../integration/elasticsearch.md#installing-elasticsearch).
 
 ### 5. Start application
 
diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb
index 85e89059dbb..b53c6c7d9fc 100644
--- a/lib/quality/test_level.rb
+++ b/lib/quality/test_level.rb
@@ -8,6 +8,7 @@ module Quality
       migration: %w[
         migrations
         lib/gitlab/background_migration
+        lib/ee/gitlab/background_migration
       ],
       unit: %w[
         bin
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index 1f6e1c9f441..ac8fdcedd79 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -13481,6 +13481,9 @@ msgstr ""
 msgid "Owner"
 msgstr ""
 
+msgid "Package Registry"
+msgstr ""
+
 msgid "Package deleted successfully"
 msgstr ""
 
@@ -17629,6 +17632,9 @@ msgstr ""
 msgid "Settings"
 msgstr ""
 
+msgid "Settings related to the use and experience of using GitLab's Package Registry."
+msgstr ""
+
 msgid "Settings to prevent self-approval across all projects in the instance. Only an administrator can modify these settings."
 msgstr ""
 
@@ -21946,6 +21952,9 @@ msgstr ""
 msgid "When enabled, any user visiting %{host} will be able to create an account."
 msgstr ""
 
+msgid "When enabled, if an NPM package isn't found in the GitLab Registry, we will attempt to pull from the global NPM registry."
+msgstr ""
+
 msgid "When enabled, users cannot use GitLab until the terms have been accepted."
 msgstr ""
 
diff --git a/spec/lib/quality/test_level_spec.rb b/spec/lib/quality/test_level_spec.rb
index 757a003946b..34163c6cfbc 100644
--- a/spec/lib/quality/test_level_spec.rb
+++ b/spec/lib/quality/test_level_spec.rb
@@ -28,7 +28,7 @@ RSpec.describe Quality::TestLevel do
     context 'when level is migration' do
       it 'returns a pattern' do
         expect(subject.pattern(:migration))
-          .to eq("spec/{migrations,lib/gitlab/background_migration}{,/**/}*_spec.rb")
+          .to eq("spec/{migrations,lib/gitlab/background_migration,lib/ee/gitlab/background_migration}{,/**/}*_spec.rb")
       end
     end
 
@@ -89,7 +89,7 @@ RSpec.describe Quality::TestLevel do
     context 'when level is migration' do
       it 'returns a regexp' do
         expect(subject.regexp(:migration))
-          .to eq(%r{spec/(migrations|lib/gitlab/background_migration)})
+          .to eq(%r{spec/(migrations|lib/gitlab/background_migration|lib/ee/gitlab/background_migration)})
       end
     end
 
@@ -134,7 +134,7 @@ RSpec.describe Quality::TestLevel do
       expect(subject.level_for('spec/migrations/add_default_and_free_plans_spec.rb')).to eq(:migration)
     end
 
-    it 'returns the correct level for a background_migration test' do
+    it 'returns the correct level for a background migration test' do
       expect(subject.level_for('spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb')).to eq(:migration)
     end
 
@@ -142,6 +142,10 @@ RSpec.describe Quality::TestLevel do
       expect(described_class.new('ee/').level_for('ee/spec/migrations/geo/migrate_ci_job_artifacts_to_separate_registry_spec.rb')).to eq(:migration)
     end
 
+    it 'returns the correct level for a EE-namespaced background migration test' do
+      expect(described_class.new('ee/').level_for('ee/spec/lib/ee/gitlab/background_migration/prune_orphaned_geo_events_spec.rb')).to eq(:migration)
+    end
+
     it 'returns the correct level for an integration test' do
       expect(subject.level_for('spec/mailers/abuse_report_mailer_spec.rb')).to eq(:integration)
     end