Add latest changes from gitlab-org/gitlab@master

10 files changed, 202 insertions, 92 deletions

diff --git a/app/finders/group_members_finder.rb b/app/finders/group_members_finder.rb
index fff17098c7b..4213a3f1965 100644
--- a/app/finders/group_members_finder.rb
+++ b/app/finders/group_members_finder.rb
@@ -60,6 +60,8 @@ class GroupMembersFinder < UnionFinder
       members = members.filter_by_2fa(params[:two_factor])
     end
 
+    members = apply_additional_filters(members)
+
     by_created_at(members)
   end
 
@@ -84,6 +86,11 @@ class GroupMembersFinder < UnionFinder
       raise ArgumentError, "#{(include_relations - RELATIONS).first} #{INVALID_RELATION_TYPE_ERROR_MSG}"
     end
   end
+
+  def apply_additional_filters(members)
+    # overridden in EE to include additional filtering conditions.
+    members
+  end
 end
 
 GroupMembersFinder.prepend_mod_with('GroupMembersFinder')
diff --git a/app/models/application_record.rb b/app/models/application_record.rb
index 75e1bab294f..198a3653cd3 100644
--- a/app/models/application_record.rb
+++ b/app/models/application_record.rb
@@ -102,6 +102,10 @@ class ApplicationRecord < ActiveRecord::Base
     where('EXISTS (?)', query.select(1))
   end
 
+  def self.where_not_exists(query)
+    where('NOT EXISTS (?)', query.select(1))
+  end
+
   def self.declarative_enum(enum_mod)
     enum(enum_mod.key => enum_mod.values)
   end
diff --git a/doc/administration/clusters/kas.md b/doc/administration/clusters/kas.md
index af31523e875..abc3ffa539e 100644
--- a/doc/administration/clusters/kas.md
+++ b/doc/administration/clusters/kas.md
@@ -6,32 +6,31 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 
 # Install the GitLab agent server for Kubernetes (KAS) **(FREE SELF)**
 
-> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3834) in GitLab 13.10, the GitLab agent server (KAS) became available on GitLab.com under `wss://kas.gitlab.com`.
+> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3834) in GitLab 13.10, the GitLab agent server (KAS) became available on GitLab.com at `wss://kas.gitlab.com`.
 > - [Moved](https://gitlab.com/groups/gitlab-org/-/epics/6290) from GitLab Premium to GitLab Free in 14.5.
 
-The GitLab agent server for Kubernetes is a GitLab backend service dedicated to
-managing the [GitLab agent for Kubernetes](../../user/clusters/agent/index.md).
+The GitLab agent server for Kubernetes (KAS) is a service that
+manages the [GitLab agent for Kubernetes](../../user/clusters/agent/index.md).
 
-The KAS acronym refers to the former name, Kubernetes agent server.
+The KAS acronym refers to the former name, `Kubernetes agent server`.
 
-The KAS is already installed and available in GitLab.com under `wss://kas.gitlab.com`.
-This document describes how to install a KAS for GitLab self-managed instances.
+The agent server for Kubernetes is installed and available on GitLab.com at `wss://kas.gitlab.com`.
+If you use self-managed GitLab, you must install an agent server or specify an external installation.
 
 ## Installation options
 
-As a GitLab administrator of self-managed instances, you can install KAS according to your GitLab
-installation method:
+As a GitLab administrator, you can install the agent server:
 
-- For [Omnibus installations](#install-kas-with-omnibus).
-- For [GitLab Helm Chart installations](#install-kas-with-the-gitlab-helm-chart).
+- For [Omnibus installations](#for-omnibus).
+- For [GitLab Helm Chart installations](#for-gitlab-helm-chart).
 
-You can also opt to use an [external KAS](#use-an-external-kas-installation).
+Or, you can [use an external agent server](#use-an-external-installation).
 
-### Install KAS with Omnibus
+### For Omnibus
 
 For [Omnibus](https://docs.gitlab.com/omnibus/) package installations:
 
-1. Edit `/etc/gitlab/gitlab.rb` to enable the agent server:
+1. To enable the agent server, edit `/etc/gitlab/gitlab.rb`:
 
    ```ruby
    gitlab_kas['enable'] = true
@@ -39,49 +38,49 @@ For [Omnibus](https://docs.gitlab.com/omnibus/) package installations:
 
 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure).
 
-To configure any additional options related to your KAS,
-refer to the **Enable GitLab KAS** section of the
+For additional configuration options, see the **Enable GitLab KAS** section of the
 [`gitlab.rb.template`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/files/gitlab-config-template/gitlab.rb.template).
 
-### Install KAS with the GitLab Helm Chart
+### For GitLab Helm Chart
 
-For GitLab [Helm Chart](https://docs.gitlab.com/charts/)
-installations, you must set `global.kas.enabled` to `true`.
-For example, in a shell with `helm` and `kubectl`
-installed, run:
+For GitLab [Helm Chart](https://docs.gitlab.com/charts/) installations:
 
-```shell
-helm repo add gitlab https://charts.gitlab.io/
-helm repo update
-helm upgrade --install gitlab gitlab/gitlab \
-  --timeout 600s \
-  --set global.hosts.domain=<YOUR_DOMAIN> \
-  --set global.hosts.externalIP=<YOUR_IP> \
-  --set certmanager-issuer.email=<YOUR_EMAIL> \
-  --set global.kas.enabled=true # <-- without this, KAS will not be installed
-```
+1. Set `global.kas.enabled` to `true`. For example, in a shell with `helm` and `kubectl`
+   installed, run:
 
-To configure KAS, use a `gitlab.kas` sub-section in your `values.yaml` file:
+   ```shell
+   helm repo add gitlab https://charts.gitlab.io/
+   helm repo update
+   helm upgrade --install gitlab gitlab/gitlab \
+     --timeout 600s \
+     --set global.hosts.domain=<YOUR_DOMAIN> \
+     --set global.hosts.externalIP=<YOUR_IP> \
+     --set certmanager-issuer.email=<YOUR_EMAIL> \
+     --set global.kas.enabled=true # <-- without this setting, the agent server will not be installed
+   ```
 
-```yaml
-gitlab:
-  kas:
-    # put your KAS custom options here
-```
+1. To configure the agent server, use a `gitlab.kas` sub-section in your `values.yaml` file:
+
+   ```yaml
+   gitlab:
+     kas:
+       # put your custom options here
+   ```
 
 For details, see [how to use the GitLab-KAS chart](https://docs.gitlab.com/charts/charts/gitlab/kas/).
 
-### Use an external KAS installation
+### Use an external installation
 
 > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/299850) in GitLab 13.10.
 
-Besides installing KAS with GitLab, you can opt to configure GitLab to use an external KAS.
+Instead of installing the agent server, you can configure GitLab to use an external agent server.
 
-For GitLab instances installed through the GitLab Helm Chart, see [how to configure your external KAS](https://docs.gitlab.com/charts/charts/globals.html#external-kas).
+If you used the GitLab Helm Chart to install GitLab, see
+[how to configure your external agent server](https://docs.gitlab.com/charts/charts/globals.html#external-kas).
 
-For GitLab instances installed through Omnibus packages:
+If you used the Omnibus packages:
 
-1. Edit `/etc/gitlab/gitlab.rb` adding the paths to your external KAS:
+1. Edit `/etc/gitlab/gitlab.rb` and add the paths to your external agent server:
 
    ```ruby
    gitlab_kas['enable'] = false
@@ -96,7 +95,7 @@ For GitLab instances installed through Omnibus packages:
 
 ## Troubleshooting
 
-If you have issues while using the GitLab agent server for Kubernetes, view the
+If you have issues while using the agent server for Kubernetes, view the
 service logs by running the following command:
 
 ```shell
@@ -107,7 +106,7 @@ In Omnibus GitLab, find the logs in `/var/log/gitlab/gitlab-kas/`.
 
 You can also [troubleshoot issues with individual agents](../../user/clusters/agent/troubleshooting.md).
 
-### KAS logs - GitOps: failed to get project information
+### GitOps: failed to get project information
 
 If you get the following error message:
 
@@ -115,11 +114,11 @@ If you get the following error message:
 {"level":"warn","time":"2020-10-30T08:37:26.123Z","msg":"GitOps: failed to get project info","agent_id":4,"project_id":"root/kas-manifest001","error":"error kind: 0; status: 404"}
 ```
 
-It means that the specified manifest project `root/kas-manifest001`
-doesn't exist or the manifest project is private. To fix it, make sure the project path is correct
-and its visibility is [set to public](../../public_access/public_access.md).
+The project specified by the manifest (`root/kas-manifest001`)
+doesn't exist or the project where the manifest is kept is private. To fix this issue,
+ensure the project path is correct and that the project's visibility is [set to public](../../public_access/public_access.md).
 
-### KAS logs - Configuration file not found
+### Configuration file not found
 
 If you get the following error message:
 
@@ -127,29 +126,29 @@ If you get the following error message:
 time="2020-10-29T04:44:14Z" level=warning msg="Config: failed to fetch" agent_id=2 error="configuration file not found: \".gitlab/agents/test-agent/config.yaml\
 ```
 
-It means that the path to the configuration project is incorrect,
-or the path to `config.yaml` inside the project is not valid.
+The path is incorrect for either:
+
+- The repository where the agent was registered.
+- The agent configuration file.
 
-To fix this, ensure that the paths to the configuration repository and to the `config.yaml` file
-are correct.
+To fix this issue, ensure that the paths are correct.
 
-### KAS logs - `dial tcp <GITLAB_INTERNAL_IP>:443: connect: connection refused`
+### `dial tcp <GITLAB_INTERNAL_IP>:443: connect: connection refused`
 
-If you are running a self-managed GitLab instance and:
+If you are running self-managed GitLab and:
 
 - The instance isn't running behind an SSL-terminating proxy.
 - The instance doesn't have HTTPS configured on the GitLab instance itself.
 - The instance's hostname resolves locally to its internal IP address.
 
-You may see the following error when the KAS tries to connect to the GitLab API:
+When the agent server tries to connect to the GitLab API, the following error might occur:
 
 ```json
 {"level":"error","time":"2021-08-16T14:56:47.289Z","msg":"GetAgentInfo()","correlation_id":"01FD7QE35RXXXX8R47WZFBAXTN","grpc_service":"gitlab.agent.reverse_tunnel.rpc.ReverseTunnel","grpc_method":"Connect","error":"Get \"https://gitlab.example.com/api/v4/internal/kubernetes/agent_info\": dial tcp 172.17.0.4:443: connect: connection refused"}
 ```
 
-To fix this for [Omnibus](https://docs.gitlab.com/omnibus/) package installations,
-set the following parameter in `/etc/gitlab/gitlab.rb`
-(replacing `gitlab.example.com` with your GitLab instance's hostname):
+To fix this issue for [Omnibus](https://docs.gitlab.com/omnibus/) package installations,
+set the following parameter in `/etc/gitlab/gitlab.rb`. Replace `gitlab.example.com` with your GitLab instance's hostname:
 
 ```ruby
 gitlab_kas['gitlab_address'] = 'http://gitlab.example.com'
diff --git a/doc/user/clusters/agent/install/index.md b/doc/user/clusters/agent/install/index.md
index 869c9e33d13..81171431bc5 100644
--- a/doc/user/clusters/agent/install/index.md
+++ b/doc/user/clusters/agent/install/index.md
@@ -21,6 +21,7 @@ Before you can install the agent in your cluster, you need:
   - [Amazon Elastic Kubernetes Service (EKS)](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html)
   - [Digital Ocean](https://docs.digitalocean.com/products/kubernetes/quickstart/)
 - On self-managed GitLab instances, a GitLab administrator must set up the [agent server](../../../../administration/clusters/kas.md).
+  On GitLab.com, the agent server is available at `wss://kas.gitlab.com`.
 
 ## Installation steps
 
@@ -122,7 +123,7 @@ Flags:
       --agent-token string     Access token registered for agent
       --agent-version string   Version of the agentk image to use (default "v14.8.1")
   -h, --help                   help for generate
-      --kas-address string     GitLab Kubernetes Agent Server address
+      --kas-address string     GitLab agent server for Kubernetes address
       --name-prefix string     The prefix to use for names of Kubernetes objects
       --namespace string       Kubernetes namespace to create resources in (default "gitlab-agent")
       --no-rbac                Do not include corresponding Roles and RoleBindings for the agent service account
diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb
index b8aa2cc8ea0..d38516e5288 100644
--- a/lib/atlassian/jira_connect/client.rb
+++ b/lib/atlassian/jira_connect/client.rb
@@ -14,14 +14,14 @@ module Atlassian
 
       def send_info(project:, update_sequence_id: nil, **args)
         common = { project: project, update_sequence_id: update_sequence_id }
-        dev_info = args.slice(:commits, :branches, :merge_requests)
+        dev_info = DevInfo.new(**common.merge(args.slice(:commits, :branches, :merge_requests)))
         build_info = args.slice(:pipelines)
         deploy_info = args.slice(:deployments)
         ff_info = args.slice(:feature_flags)
 
         responses = []
 
-        responses << store_dev_info(**common, **dev_info) if dev_info.present?
+        responses << store_dev_info(dev_info) if dev_info.present?
         responses << store_build_info(**common, **build_info) if build_info.present?
         responses << store_deploy_info(**common, **deploy_info) if deploy_info.present?
         responses << store_ff_info(**common, **ff_info) if ff_info.present?
@@ -93,17 +93,8 @@ module Atlassian
         handle_response(r, 'builds') { |data| errors(data, 'rejectedBuilds') }
       end
 
-      def store_dev_info(project:, commits: nil, branches: nil, merge_requests: nil, update_sequence_id: nil)
-        repo = ::Atlassian::JiraConnect::Serializers::RepositoryEntity.represent(
-          project,
-          commits: commits,
-          branches: branches,
-          merge_requests: merge_requests,
-          user_notes_count: user_notes_count(merge_requests),
-          update_sequence_id: update_sequence_id
-        )
-
-        post('/rest/devinfo/0.10/bulk', { repositories: [repo] })
+      def store_dev_info(dev_info)
+        post(dev_info.url, dev_info.body)
       end
 
       def post(path, payload)
@@ -157,14 +148,6 @@ module Atlassian
         { 'errorMessages' => messages }
       end
 
-      def user_notes_count(merge_requests)
-        return unless merge_requests
-
-        Note.count_for_collection(merge_requests.map(&:id), 'MergeRequest').to_h do |count_group|
-          [count_group.noteable_id, count_group.count]
-        end
-      end
-
       def jwt_token(http_method, uri)
         claims = Atlassian::Jwt.build_claims(
           Atlassian::JiraConnect.app_key,
diff --git a/lib/atlassian/jira_connect/dev_info.rb b/lib/atlassian/jira_connect/dev_info.rb
new file mode 100644
index 00000000000..90ccc1939d2
--- /dev/null
+++ b/lib/atlassian/jira_connect/dev_info.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Atlassian
+  module JiraConnect
+    class DevInfo
+      URL = '/rest/devinfo/0.10/bulk'
+
+      def initialize(project:, commits: nil, branches: nil, merge_requests: nil, update_sequence_id: nil)
+        @project = project
+        @commits = commits
+        @branches = branches
+        @merge_requests = merge_requests
+        @update_sequence_id = update_sequence_id
+      end
+
+      def url
+        URL
+      end
+
+      def body
+        repo = ::Atlassian::JiraConnect::Serializers::RepositoryEntity.represent(
+          @project,
+          commits: @commits,
+          branches: @branches,
+          merge_requests: @merge_requests,
+          user_notes_count: user_notes_count,
+          update_sequence_id: @update_sequence_id
+        )
+
+        { repositories: [repo] }
+      end
+
+      def present?
+        [@commits, @branches, @merge_requests].any?(&:present?)
+      end
+
+      private
+
+      def user_notes_count
+        return unless @merge_requests
+
+        Note.count_for_collection(@merge_requests.map(&:id), 'MergeRequest').to_h do |count_group|
+          [count_group.noteable_id, count_group.count]
+        end
+      end
+    end
+  end
+end
diff --git a/spec/fixtures/api/schemas/jira_connect/dev_info.json b/spec/fixtures/api/schemas/jira_connect/dev_info.json
new file mode 100644
index 00000000000..98437353fde
--- /dev/null
+++ b/spec/fixtures/api/schemas/jira_connect/dev_info.json
@@ -0,0 +1,8 @@
+{
+  "repositories": {
+    "type": "array",
+    "items": {
+      "$ref": "./repository.json"
+    }
+  }
+}
diff --git a/spec/lib/atlassian/jira_connect/client_spec.rb b/spec/lib/atlassian/jira_connect/client_spec.rb
index dd3130c78bf..1857a1431df 100644
--- a/spec/lib/atlassian/jira_connect/client_spec.rb
+++ b/spec/lib/atlassian/jira_connect/client_spec.rb
@@ -58,12 +58,16 @@ RSpec.describe Atlassian::JiraConnect::Client do
         deployments: :q
       ).and_return(:deploys_stored)
 
-      expect(subject).to receive(:store_dev_info).with(
+      expect(Atlassian::JiraConnect::DevInfo).to receive(:new).with(
         project: project,
         update_sequence_id: :x,
         commits: :a,
         branches: :b,
         merge_requests: :c
+      ).and_call_original
+
+      expect(subject).to receive(:store_dev_info).with(
+        instance_of(Atlassian::JiraConnect::DevInfo)
       ).and_return(:dev_stored)
 
       args = {
@@ -83,9 +87,7 @@ RSpec.describe Atlassian::JiraConnect::Client do
 
     it 'only calls methods that we need to call' do
       expect(subject).to receive(:store_dev_info).with(
-        project: project,
-        update_sequence_id: :x,
-        commits: :a
+        instance_of(Atlassian::JiraConnect::DevInfo)
       ).and_return(:dev_stored)
 
       args = {
@@ -402,15 +404,7 @@ RSpec.describe Atlassian::JiraConnect::Client do
     end
 
     it "calls the API with auth headers" do
-      subject.send(:store_dev_info, project: project)
-    end
-
-    it 'avoids N+1 database queries' do
-      control_count = ActiveRecord::QueryRecorder.new { subject.send(:store_dev_info, project: project, merge_requests: merge_requests) }.count
-
-      merge_requests << create(:merge_request, :unique_branches)
-
-      expect { subject.send(:store_dev_info, project: project, merge_requests: merge_requests) }.not_to exceed_query_limit(control_count)
+      subject.send(:store_dev_info, Atlassian::JiraConnect::DevInfo.new(project: project))
     end
   end
 end
diff --git a/spec/lib/atlassian/jira_connect/dev_info_spec.rb b/spec/lib/atlassian/jira_connect/dev_info_spec.rb
new file mode 100644
index 00000000000..357168a94b9
--- /dev/null
+++ b/spec/lib/atlassian/jira_connect/dev_info_spec.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Atlassian::JiraConnect::DevInfo do
+  let_it_be(:project) { create_default(:project, :repository).freeze }
+
+  let(:update_sequence_id) { '123' }
+
+  describe '#url' do
+    subject { described_class.new(project: project).url }
+
+    it { is_expected.to eq('/rest/devinfo/0.10/bulk') }
+  end
+
+  describe '#body' do
+    let_it_be(:merge_request) { create(:merge_request, :unique_branches, title: 'TEST-123') }
+    let_it_be(:note) { create(:note, noteable: merge_request, project: merge_request.project) }
+    let_it_be(:branches) do
+      project.repository.create_branch('TEST-123', project.default_branch_or_main)
+      [project.repository.find_branch('TEST-123')]
+    end
+
+    let(:merge_requests) { [merge_request] }
+
+    subject(:body) { described_class.new(project: project, branches: branches, merge_requests: merge_requests, update_sequence_id: update_sequence_id).body.to_json }
+
+    it 'matches the schema' do
+      expect(body).to match_schema('jira_connect/dev_info')
+    end
+
+    it 'avoids N+1 database queries' do
+      control_count = ActiveRecord::QueryRecorder.new { subject }.count
+
+      merge_requests << create(:merge_request, :unique_branches)
+
+      expect { subject }.not_to exceed_query_limit(control_count)
+    end
+  end
+
+  describe '#present?' do
+    let(:arguments) { { commits: nil, branches: nil, merge_requests: nil } }
+
+    subject { described_class.new(**{ project: project, update_sequence_id: update_sequence_id }.merge(arguments)).present? }
+
+    it { is_expected.to eq(false) }
+
+    context 'with commits, branches or merge requests' do
+      let(:arguments) { { commits: anything, branches: anything, merge_requests: anything } }
+
+      it { is_expected.to eq(true) }
+    end
+  end
+end
diff --git a/spec/models/application_record_spec.rb b/spec/models/application_record_spec.rb
index 9c9a048999c..c1cd44e9007 100644
--- a/spec/models/application_record_spec.rb
+++ b/spec/models/application_record_spec.rb
@@ -104,6 +104,18 @@ RSpec.describe ApplicationRecord do
     end
   end
 
+  describe '.where_not_exists' do
+    it 'produces a WHERE NOT EXISTS query' do
+      create(:user, :two_factor_via_u2f)
+      user_2 = create(:user)
+
+      expect(
+        User.where_not_exists(
+          U2fRegistration.where(U2fRegistration.arel_table[:user_id].eq(User.arel_table[:id])))
+      ).to match_array([user_2])
+    end
+  end
+
   describe '.transaction', :delete do
     it 'opens a new transaction' do
       expect(described_class.connection.transaction_open?).to be false