Add latest changes from gitlab-org/gitlab@master

13 files changed, 227 insertions, 55 deletions

diff --git a/spec/lib/api/entities/diff_spec.rb b/spec/lib/api/entities/diff_spec.rb
index 27d9ed44c98..535567d4d8d 100644
--- a/spec/lib/api/entities/diff_spec.rb
+++ b/spec/lib/api/entities/diff_spec.rb
@@ -23,7 +23,8 @@ RSpec.describe ::API::Entities::Diff, feature_category: :source_code_management
         b_mode: diff.b_mode,
         new_file: diff.new_file?,
         renamed_file: diff.renamed_file?,
-        deleted_file: diff.deleted_file?
+        deleted_file: diff.deleted_file?,
+        generated_file: diff.generated?
       }
     )
   end
diff --git a/spec/lib/gitlab/ci/yaml_processor/test_cases/include_spec.rb b/spec/lib/gitlab/ci/yaml_processor/test_cases/include_spec.rb
new file mode 100644
index 00000000000..d8f8a58edf3
--- /dev/null
+++ b/spec/lib/gitlab/ci/yaml_processor/test_cases/include_spec.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+module Gitlab
+  module Ci
+    RSpec.describe YamlProcessor, feature_category: :pipeline_composition do
+      include StubRequests
+
+      subject(:processor) do
+        described_class.new(config, project: project, user: project.first_owner, logger: logger)
+      end
+
+      let_it_be(:project) { create(:project, :repository) }
+
+      let(:logger) { Gitlab::Ci::Pipeline::Logger.new(project: project) }
+      let(:result) { processor.execute }
+      let(:builds) { result.builds }
+
+      context 'with include:remote' do
+        let(:config) do
+          <<~YAML
+            include:
+              - remote: http://my.domain.com/config1.yml
+              - remote: http://my.domain.com/config2.yml
+          YAML
+        end
+
+        before do
+          stub_full_request('http://my.domain.com/config1.yml')
+            .to_return(body: 'build1: { script: echo Hello World }')
+
+          stub_full_request('http://my.domain.com/config2.yml')
+            .to_return(body: 'build2: { script: echo Hello World }')
+        end
+
+        it 'returns builds from included files' do
+          expect(builds.pluck(:name)).to eq %w[build1 build2]
+        end
+
+        it 'stores instrumentation logs' do
+          result
+
+          expect(logger.observations_hash['config_mapper_process_duration_s']['count']).to eq(1)
+        end
+
+        # Remove with the FF ci_parallel_remote_includes
+        it 'does not store log with config_file_fetch_remote_content' do
+          result
+
+          expect(logger.observations_hash).not_to have_key('config_file_fetch_remote_content_duration_s')
+        end
+
+        context 'when the FF ci_parallel_remote_includes is disabled' do
+          before do
+            stub_feature_flags(ci_parallel_remote_includes: false)
+          end
+
+          it 'stores log with config_file_fetch_remote_content' do
+            result
+
+            expect(logger.observations_hash['config_file_fetch_remote_content_duration_s']['count']).to eq(2)
+          end
+
+          context 'when the FF is specifically enabled for the project' do
+            before do
+              stub_feature_flags(ci_parallel_remote_includes: [project])
+            end
+
+            it 'does not store log with config_file_fetch_remote_content' do
+              result
+
+              expect(logger.observations_hash).not_to have_key('config_file_fetch_remote_content_duration_s')
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/ci/yaml_processor/test_cases/interruptible_spec.rb b/spec/lib/gitlab/ci/yaml_processor/test_cases/interruptible_spec.rb
index 03ff7077969..297872f4cf3 100644
--- a/spec/lib/gitlab/ci/yaml_processor/test_cases/interruptible_spec.rb
+++ b/spec/lib/gitlab/ci/yaml_processor/test_cases/interruptible_spec.rb
@@ -5,9 +5,10 @@ require 'spec_helper'
 module Gitlab
   module Ci
     RSpec.describe YamlProcessor, feature_category: :pipeline_composition do
-      subject(:processor) { described_class.new(config, user: nil).execute }
+      subject(:processor) { described_class.new(config, user: nil) }
 
-      let(:builds) { processor.builds }
+      let(:result) { processor.execute }
+      let(:builds) { result.builds }
 
       context 'with interruptible' do
         let(:default_config) { nil }
diff --git a/spec/lib/gitlab/diff/file_collection/paginated_merge_request_diff_spec.rb b/spec/lib/gitlab/diff/file_collection/paginated_merge_request_diff_spec.rb
index 891336658ce..6c9a5341695 100644
--- a/spec/lib/gitlab/diff/file_collection/paginated_merge_request_diff_spec.rb
+++ b/spec/lib/gitlab/diff/file_collection/paginated_merge_request_diff_spec.rb
@@ -9,9 +9,10 @@ RSpec.describe Gitlab::Diff::FileCollection::PaginatedMergeRequestDiff, feature_
   let(:diffable) { merge_request.merge_request_diff }
   let(:diff_files_relation) { diffable.merge_request_diff_files }
   let(:diff_files) { subject.diff_files }
+  let(:diff_options) { {} }
 
   subject do
-    described_class.new(diffable, page, per_page)
+    described_class.new(diffable, page, per_page, diff_options)
   end
 
   describe '#diff_files' do
@@ -77,18 +78,32 @@ RSpec.describe Gitlab::Diff::FileCollection::PaginatedMergeRequestDiff, feature_
     context 'when last page' do
       it 'returns correct diff files' do
         last_page = diff_files_relation.count - per_page
-        collection = described_class.new(diffable, last_page, per_page)
+        collection = described_class.new(diffable, last_page, per_page, diff_options)
 
         expected_batch_files = diff_files_relation.page(last_page).per(per_page).map(&:new_path)
 
         expect(collection.diff_files.map(&:new_path)).to eq(expected_batch_files)
       end
     end
+
+    context 'when collapse_generated is given' do
+      let(:diff_options) { { collapse_generated: true } }
+
+      it 'returns generated value' do
+        expect(diff_files.first.generated?).not_to be_nil
+      end
+    end
+
+    context 'when collapse_generated is not given' do
+      it 'returns nil' do
+        expect(diff_files.first.generated?).to be_nil
+      end
+    end
   end
 
   it_behaves_like 'unfoldable diff' do
     subject do
-      described_class.new(merge_request.merge_request_diff, page, per_page)
+      described_class.new(merge_request.merge_request_diff, page, per_page, diff_options)
     end
   end
 
@@ -100,7 +115,7 @@ RSpec.describe Gitlab::Diff::FileCollection::PaginatedMergeRequestDiff, feature_
     let(:diffable) { merge_request.merge_request_diff }
 
     subject do
-      described_class.new(merge_request.merge_request_diff, page, per_page)
+      described_class.new(merge_request.merge_request_diff, page, per_page, diff_options)
     end
   end
 end
diff --git a/spec/lib/gitlab/event_store/event_spec.rb b/spec/lib/gitlab/event_store/event_spec.rb
index edcb0e5dd1a..367b3334020 100644
--- a/spec/lib/gitlab/event_store/event_spec.rb
+++ b/spec/lib/gitlab/event_store/event_spec.rb
@@ -93,6 +93,13 @@ RSpec.describe Gitlab::EventStore::Event, feature_category: :shared do
 
           expect(event_class.json_schema_valid).to eq(false)
         end
+
+        it 'does not store JSON schema on subclass' do
+          expect { event }.to raise_error(Gitlab::EventStore::InvalidEvent)
+
+          expect(event_class.instance_variables).not_to include(:@json_schema)
+          expect(described_class.instance_variables).to include(:@json_schema)
+        end
       end
     end
   end
diff --git a/spec/lib/gitlab/tracking/event_definition_spec.rb b/spec/lib/gitlab/tracking/event_definition_spec.rb
index 5e41c691da8..7c5047dc0c6 100644
--- a/spec/lib/gitlab/tracking/event_definition_spec.rb
+++ b/spec/lib/gitlab/tracking/event_definition_spec.rb
@@ -31,10 +31,6 @@ RSpec.describe Gitlab::Tracking::EventDefinition do
     File.write(path, content)
   end
 
-  it 'has all definitions valid' do
-    expect { described_class.definitions }.not_to raise_error
-  end
-
   it 'has no duplicated actions in InternalEventTracking events', :aggregate_failures do
     definitions_by_action = described_class.definitions
                                            .select { |d| d.category == 'InternalEventTracking' }
@@ -85,10 +81,8 @@ RSpec.describe Gitlab::Tracking::EventDefinition do
         attributes[attribute] = value
       end
 
-      it 'raise exception' do
-        expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Tracking::InvalidEventError))
-
-        described_class.new(path, attributes).validate!
+      it 'has validation errors' do
+        expect(described_class.new(path, attributes).validation_errors).not_to be_empty
       end
     end
   end
diff --git a/spec/lib/gitlab/tracking/event_definition_validate_all_spec.rb b/spec/lib/gitlab/tracking/event_definition_validate_all_spec.rb
new file mode 100644
index 00000000000..cc2ccc511bb
--- /dev/null
+++ b/spec/lib/gitlab/tracking/event_definition_validate_all_spec.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Tracking::EventDefinition, feature_category: :product_analytics_data_management do
+  it 'only has valid event definitions', :aggregate_failures do
+    described_class.definitions.each do |definition|
+      validation_errors = definition.validation_errors
+      expect(validation_errors).to be_empty, validation_errors.join
+    end
+  end
+end
diff --git a/spec/lib/gitlab/usage/metric_definition_spec.rb b/spec/lib/gitlab/usage/metric_definition_spec.rb
index fb46d48c1bb..caa114cb00f 100644
--- a/spec/lib/gitlab/usage/metric_definition_spec.rb
+++ b/spec/lib/gitlab/usage/metric_definition_spec.rb
@@ -33,6 +33,14 @@ RSpec.describe Gitlab::Usage::MetricDefinition, feature_category: :service_ping
     described_class.instance_variable_set(:@definitions, nil)
   end
 
+  def expect_validation_errors
+    expect(described_class.new(path, attributes).validation_errors).not_to be_empty
+  end
+
+  def expect_no_validation_errors
+    expect(described_class.new(path, attributes).validation_errors).to be_empty
+  end
+
   def write_metric(metric, path, content)
     path = File.join(metric, path)
     dir = File.dirname(path)
@@ -40,12 +48,6 @@ RSpec.describe Gitlab::Usage::MetricDefinition, feature_category: :service_ping
     File.write(path, content)
   end
 
-  it 'has only valid definitions' do
-    described_class.all.each do |definition|
-      expect { definition.validate! }.not_to raise_error
-    end
-  end
-
   describe '.instrumentation_class' do
     context 'for non internal events' do
       let(:attributes) { { key_path: 'metric1', instrumentation_class: 'RedisHLLMetric', data_source: 'redis_hll' } }
@@ -197,10 +199,8 @@ RSpec.describe Gitlab::Usage::MetricDefinition, feature_category: :service_ping
         attributes[attribute] = value
       end
 
-      it 'raise exception' do
-        expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError))
-
-        described_class.new(path, attributes).validate!
+      it 'has validation errors' do
+        expect_validation_errors
       end
     end
 
@@ -210,9 +210,7 @@ RSpec.describe Gitlab::Usage::MetricDefinition, feature_category: :service_ping
           attributes[:status] = 'broken'
           attributes.delete(:repair_issue_url)
 
-          expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError))
-
-          described_class.new(path, attributes).validate!
+          expect_validation_errors
         end
       end
 
@@ -221,20 +219,16 @@ RSpec.describe Gitlab::Usage::MetricDefinition, feature_category: :service_ping
           attributes[:status] = 'removed'
         end
 
-        it 'raise dev exception when removed_by_url is not provided' do
+        it 'has validation errors when removed_by_url is not provided' do
           attributes.delete(:removed_by_url)
 
-          expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError))
-
-          described_class.new(path, attributes).validate!
+          expect_validation_errors
         end
 
-        it 'raises dev exception when milestone_removed is not provided' do
+        it 'has validation errors when milestone_removed is not provided' do
           attributes.delete(:milestone_removed)
 
-          expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError))
-
-          described_class.new(path, attributes).validate!
+          expect_validation_errors
         end
       end
 
@@ -251,18 +245,16 @@ RSpec.describe Gitlab::Usage::MetricDefinition, feature_category: :service_ping
         end
 
         with_them do
-          it 'raises dev exception when invalid' do
+          it 'has validation errors when invalid' do
             attributes[:instrumentation_class] = instrumentation_class if instrumentation_class
             attributes[:options] = options if options
             attributes[:events] = events if events
 
             if is_valid
-              expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception)
+              expect_no_validation_errors
             else
-              expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError))
+              expect_validation_errors
             end
-
-            described_class.new(path, attributes).validate!
           end
         end
       end
@@ -294,12 +286,10 @@ RSpec.describe Gitlab::Usage::MetricDefinition, feature_category: :service_ping
             attributes[:options] = options
 
             if is_valid
-              expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception)
+              expect_no_validation_errors
             else
-              expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError))
+              expect_validation_errors
             end
-
-            described_class.new(path, attributes).validate!
           end
         end
       end
@@ -340,12 +330,10 @@ RSpec.describe Gitlab::Usage::MetricDefinition, feature_category: :service_ping
             attributes[:options] = options
 
             if is_valid
-              expect(Gitlab::ErrorTracking).not_to receive(:track_and_raise_for_dev_exception)
+              expect_no_validation_errors
             else
-              expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError))
+              expect_validation_errors
             end
-
-            described_class.new(path, attributes).validate!
           end
         end
       end
diff --git a/spec/lib/gitlab/usage/metric_definition_validate_all_spec.rb b/spec/lib/gitlab/usage/metric_definition_validate_all_spec.rb
new file mode 100644
index 00000000000..d6255b54068
--- /dev/null
+++ b/spec/lib/gitlab/usage/metric_definition_validate_all_spec.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Usage::MetricDefinition, feature_category: :product_analytics_data_management do
+  # rubocop:disable Rails/FindEach -- The all method invoked here is unrelated to the ActiveRecord scope all
+  it 'only has valid metric definitions', :aggregate_failures do
+    described_class.all.each do |definition|
+      validation_errors = definition.validation_errors
+      expect(validation_errors).to be_empty, validation_errors.join
+    end
+  end
+  # rubocop:enable Rails/FindEach
+end
diff --git a/spec/models/merge_request_diff_spec.rb b/spec/models/merge_request_diff_spec.rb
index 910b32cc1f1..e31a4399646 100644
--- a/spec/models/merge_request_diff_spec.rb
+++ b/spec/models/merge_request_diff_spec.rb
@@ -608,11 +608,35 @@ RSpec.describe MergeRequestDiff, feature_category: :code_review_workflow do
     end
 
     describe '#paginated_diffs' do
+      shared_examples 'diffs with generated files check' do
+        context 'when collapse_generated_diff_files FF is enabled' do
+          it 'checks generated files' do
+            diffs = diff_with_commits.paginated_diffs(1, 10)
+
+            expect(diffs.diff_files.first.generated?).not_to be_nil
+          end
+        end
+
+        context 'when collapse_generated_diff_files FF is disabled' do
+          before do
+            stub_feature_flags(collapse_generated_diff_files: false)
+          end
+
+          it 'does not check generated files' do
+            diffs = diff_with_commits.paginated_diffs(1, 10)
+
+            expect(diffs.diff_files.first.generated?).to be_nil
+          end
+        end
+      end
+
       context 'when no persisted files available' do
         before do
           diff_with_commits.clean!
         end
 
+        it_behaves_like 'diffs with generated files check'
+
         it 'returns a Gitlab::Diff::FileCollection::Compare' do
           diffs = diff_with_commits.paginated_diffs(1, 10)
 
@@ -622,6 +646,8 @@ RSpec.describe MergeRequestDiff, feature_category: :code_review_workflow do
       end
 
       context 'when persisted files available' do
+        it_behaves_like 'diffs with generated files check'
+
         it 'returns paginated diffs' do
           diffs = diff_with_commits.paginated_diffs(1, 10)
 
diff --git a/spec/requests/api/import_github_spec.rb b/spec/requests/api/import_github_spec.rb
index f555f39ff74..532492c9c2c 100644
--- a/spec/requests/api/import_github_spec.rb
+++ b/spec/requests/api/import_github_spec.rb
@@ -20,11 +20,18 @@ RSpec.describe API::ImportGithub, feature_category: :importers do
     }
   end
 
+  let(:headers) do
+    {
+      'x-oauth-scopes' => 'read:org'
+    }
+  end
+
   let(:client) { double('client', user: provider_user, repository: provider_repo) }
 
   before do
     Grape::Endpoint.before_each do |endpoint|
       allow(endpoint).to receive(:client).and_return(client)
+      allow(client).to receive_message_chain(:octokit, :last_response, :headers).and_return(headers)
     end
   end
 
diff --git a/spec/services/import/github_service_spec.rb b/spec/services/import/github_service_spec.rb
index 06ce00260ed..6fe17a31f3e 100644
--- a/spec/services/import/github_service_spec.rb
+++ b/spec/services/import/github_service_spec.rb
@@ -21,12 +21,19 @@ RSpec.describe Import::GithubService, feature_category: :importers do
     }
   end
 
+  let(:headers) do
+    {
+      'x-oauth-scopes' => 'read:org'
+    }
+  end
+
   let(:client) { Gitlab::GithubImport::Client.new(token) }
   let(:project_double) { instance_double(Project, persisted?: true) }
 
   subject(:github_importer) { described_class.new(client, user, params) }
 
   before do
+    allow(client).to receive_message_chain(:octokit, :last_response, :headers).and_return(headers)
     allow(Gitlab::GithubImport::Settings).to receive(:new).with(project_double).and_return(settings)
     allow(settings)
       .to receive(:write)
@@ -195,6 +202,24 @@ RSpec.describe Import::GithubService, feature_category: :importers do
       end
     end
 
+    context 'validates scopes when collaborator import is true' do
+      let(:optional_stages) do
+        {
+          collaborators_import: true
+        }
+      end
+
+      let(:headers) do
+        {
+          'x-oauth-scopes' => 'read:user'
+        }
+      end
+
+      it 'returns error when scope is not adequate' do
+        expect(subject.execute(access_params, :github)).to include(scope_error)
+      end
+    end
+
     context 'when timeout strategy param is present' do
       let(:timeout_strategy) { 'pessimistic' }
 
@@ -330,6 +355,14 @@ RSpec.describe Import::GithubService, feature_category: :importers do
     }
   end
 
+  def scope_error
+    {
+      status: :error,
+      http_status: :unprocessable_entity,
+      message: 'Your GitHub access token does not have the correct scope to import collaborators.'
+    }
+  end
+
   def blocked_url_error(url)
     {
       status: :error,
diff --git a/spec/support/rspec_order_todo.yml b/spec/support/rspec_order_todo.yml
index 6aac031be6f..2dc9546ab14 100644
--- a/spec/support/rspec_order_todo.yml
+++ b/spec/support/rspec_order_todo.yml
@@ -94,7 +94,6 @@
 - './ee/spec/controllers/groups/security/merge_commit_reports_controller_spec.rb'
 - './ee/spec/controllers/groups/security/policies_controller_spec.rb'
 - './ee/spec/controllers/groups/security/vulnerabilities_controller_spec.rb'
-- './ee/spec/controllers/groups/sso_controller_spec.rb'
 - './ee/spec/controllers/groups/todos_controller_spec.rb'
 - './ee/spec/controllers/groups/wikis_controller_spec.rb'
 - './ee/spec/controllers/ldap/omniauth_callbacks_controller_spec.rb'
@@ -284,7 +283,6 @@
 - './ee/spec/features/groups/settings/user_configures_insights_spec.rb'
 - './ee/spec/features/groups/settings/user_searches_in_settings_spec.rb'
 - './ee/spec/features/groups_spec.rb'
-- './ee/spec/features/groups/sso_spec.rb'
 - './ee/spec/features/groups/wikis_spec.rb'
 - './ee/spec/features/groups/wiki/user_views_wiki_empty_spec.rb'
 - './ee/spec/features/ide/user_opens_ide_spec.rb'
@@ -918,7 +916,6 @@
 - './ee/spec/helpers/gitlab_subscriptions/upcoming_reconciliation_helper_spec.rb'
 - './ee/spec/helpers/groups/ldap_sync_helper_spec.rb'
 - './ee/spec/helpers/groups/security_features_helper_spec.rb'
-- './ee/spec/helpers/groups/sso_helper_spec.rb'
 - './ee/spec/helpers/kerberos_helper_spec.rb'
 - './ee/spec/helpers/license_helper_spec.rb'
 - './ee/spec/helpers/license_monitoring_helper_spec.rb'
@@ -2619,14 +2616,11 @@
 - './ee/spec/services/gitlab_subscriptions/preview_billable_user_change_service_spec.rb'
 - './ee/spec/services/gitlab_subscriptions/reconciliations/calculate_seat_count_data_service_spec.rb'
 - './ee/spec/services/gitlab_subscriptions/reconciliations/check_seat_usage_alerts_eligibility_service_spec.rb'
-- './ee/spec/services/group_saml/group_managed_accounts/clean_up_members_service_spec.rb'
-- './ee/spec/services/group_saml/group_managed_accounts/transfer_membership_service_spec.rb'
 - './ee/spec/services/group_saml/identity/destroy_service_spec.rb'
 - './ee/spec/services/group_saml/saml_group_links/create_service_spec.rb'
 - './ee/spec/services/group_saml/saml_group_links/destroy_service_spec.rb'
 - './ee/spec/services/group_saml/saml_provider/create_service_spec.rb'
 - './ee/spec/services/group_saml/saml_provider/update_service_spec.rb'
-- './ee/spec/services/group_saml/sign_up_service_spec.rb'
 - './ee/spec/services/groups/create_service_spec.rb'
 - './ee/spec/services/groups/destroy_service_spec.rb'
 - './ee/spec/services/groups/epics_count_service_spec.rb'