Add latest changes from gitlab-org/gitlab@13-6-stable-eev13.6.0-rc42

14 files changed, 270 insertions, 69 deletions

diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml
index 5ee7fb2adbf..38fe2781331 100644
--- a/spec/lib/gitlab/import_export/all_models.yml
+++ b/spec/lib/gitlab/import_export/all_models.yml
@@ -179,6 +179,7 @@ merge_requests:
 - user_mentions
 - system_note_metadata
 - note_authors
+- cleanup_schedule
 external_pull_requests:
 - project
 merge_request_diff:
@@ -195,6 +196,8 @@ merge_request_diff_files:
 merge_request_context_commits:
 - merge_request
 - diff_files
+cleanup_schedule:
+- merge_request
 ci_pipelines:
 - project
 - user
@@ -240,6 +243,7 @@ ci_pipelines:
 - vulnerability_findings
 - pipeline_config
 - security_scans
+- security_findings
 - daily_build_group_report_results
 - latest_builds
 - daily_report_results
@@ -317,6 +321,7 @@ push_access_levels:
 - protected_branch
 - user
 - group
+- deploy_key
 create_access_levels:
 - user
 - protected_tag
@@ -652,6 +657,7 @@ milestone_releases:
 evidences:
 - release
 design: &design
+- authors
 - issue
 - actions
 - versions
diff --git a/spec/lib/gitlab/import_export/attributes_finder_spec.rb b/spec/lib/gitlab/import_export/attributes_finder_spec.rb
index 7f6ebf577af..428d8d605ee 100644
--- a/spec/lib/gitlab/import_export/attributes_finder_spec.rb
+++ b/spec/lib/gitlab/import_export/attributes_finder_spec.rb
@@ -59,7 +59,7 @@ RSpec.describe Gitlab::ImportExport::AttributesFinder do
     end
 
     before do
-      allow_any_instance_of(Gitlab::ImportExport).to receive(:config_file).and_return(test_config)
+      allow(Gitlab::ImportExport).to receive(:config_file).and_return(test_config)
     end
 
     it 'generates hash from project tree config' do
diff --git a/spec/lib/gitlab/import_export/group/legacy_tree_saver_spec.rb b/spec/lib/gitlab/import_export/group/legacy_tree_saver_spec.rb
index 6b324b952dc..9e1571ae3d8 100644
--- a/spec/lib/gitlab/import_export/group/legacy_tree_saver_spec.rb
+++ b/spec/lib/gitlab/import_export/group/legacy_tree_saver_spec.rb
@@ -12,7 +12,7 @@ RSpec.describe Gitlab::ImportExport::Group::LegacyTreeSaver do
 
     before do
       group.add_maintainer(user)
-      allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+      allow(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
     end
 
     after do
diff --git a/spec/lib/gitlab/import_export/importer_spec.rb b/spec/lib/gitlab/import_export/importer_spec.rb
index dc44296321c..0db038785d3 100644
--- a/spec/lib/gitlab/import_export/importer_spec.rb
+++ b/spec/lib/gitlab/import_export/importer_spec.rb
@@ -12,7 +12,7 @@ RSpec.describe Gitlab::ImportExport::Importer do
   subject(:importer) { described_class.new(project) }
 
   before do
-    allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
+    allow(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
     allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file)
     stub_uploads_object_storage(FileUploader)
 
@@ -65,10 +65,22 @@ RSpec.describe Gitlab::ImportExport::Importer do
         end
       end
 
-      it 'restores the ProjectTree' do
-        expect(Gitlab::ImportExport::Project::TreeRestorer).to receive(:new).and_call_original
+      context 'with sample_data_template' do
+        it 'initializes the Sample::TreeRestorer' do
+          project.create_or_update_import_data(data: { sample_data: true })
 
-        importer.execute
+          expect(Gitlab::ImportExport::Project::Sample::TreeRestorer).to receive(:new).and_call_original
+
+          importer.execute
+        end
+      end
+
+      context 'without sample_data_template' do
+        it 'initializes the ProjectTree' do
+          expect(Gitlab::ImportExport::Project::TreeRestorer).to receive(:new).and_call_original
+
+          importer.execute
+        end
       end
 
       it 'removes the import file' do
diff --git a/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb b/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb
index e208a1c383c..b477ac45577 100644
--- a/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb
+++ b/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb
@@ -67,6 +67,14 @@ RSpec.describe Gitlab::ImportExport::JSON::NdjsonReader do
       it 'yields nothing to the Enumerator' do
         expect(subject.to_a).to eq([])
       end
+
+      context 'with mark_as_consumed: false' do
+        subject { ndjson_reader.consume_relation(importable_path, key, mark_as_consumed: false) }
+
+        it 'yields every relation value to the Enumerator' do
+          expect(subject.count).to eq(1)
+        end
+      end
     end
 
     context 'key has not been consumed' do
@@ -102,14 +110,4 @@ RSpec.describe Gitlab::ImportExport::JSON::NdjsonReader do
       end
     end
   end
-
-  describe '#clear_consumed_relations' do
-    let(:dir_path) { fixture }
-
-    subject { ndjson_reader.clear_consumed_relations }
-
-    it 'returns empty set' do
-      expect(subject).to be_empty
-    end
-  end
 end
diff --git a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb
index 949cfb5a34d..762687beedb 100644
--- a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb
+++ b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb
@@ -19,6 +19,7 @@ RSpec.describe Gitlab::ImportExport::JSON::StreamingSerializer do
       group: group,
       approvals_before_merge: 1)
   end
+
   let_it_be(:issue) do
     create(:issue,
       assignees: [user],
diff --git a/spec/lib/gitlab/import_export/lfs_restorer_spec.rb b/spec/lib/gitlab/import_export/lfs_restorer_spec.rb
index a9f7fb72612..c8887b0ded1 100644
--- a/spec/lib/gitlab/import_export/lfs_restorer_spec.rb
+++ b/spec/lib/gitlab/import_export/lfs_restorer_spec.rb
@@ -13,7 +13,7 @@ RSpec.describe Gitlab::ImportExport::LfsRestorer do
   subject(:restorer) { described_class.new(project: project, shared: shared) }
 
   before do
-    allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+    allow(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
     FileUtils.mkdir_p(shared.export_path)
   end
 
diff --git a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb
index 50bc6a30044..56ba730e893 100644
--- a/spec/lib/gitlab/import_export/project/relation_factory_spec.rb
+++ b/spec/lib/gitlab/import_export/project/relation_factory_spec.rb
@@ -61,6 +61,7 @@ RSpec.describe Gitlab::ImportExport::Project::RelationFactory do
         'enable_ssl_verification' => true,
         'job_events' => false,
         'wiki_page_events' => true,
+        'releases_events' => false,
         'token' => token
       }
     end
diff --git a/spec/lib/gitlab/import_export/project/sample/date_calculator_spec.rb b/spec/lib/gitlab/import_export/project/sample/date_calculator_spec.rb
index 82f59245519..645242c6f05 100644
--- a/spec/lib/gitlab/import_export/project/sample/date_calculator_spec.rb
+++ b/spec/lib/gitlab/import_export/project/sample/date_calculator_spec.rb
@@ -13,7 +13,7 @@ RSpec.describe Gitlab::ImportExport::Project::Sample::DateCalculator do
     end
 
     context 'when dates are not empty' do
-      let(:dates) { [[nil, '2020-01-01 00:00:00 +0000'], [nil, '2021-01-01 00:00:00 +0000'], [nil, '2022-01-01 23:59:59 +0000']] }
+      let(:dates) { [nil, '2020-01-01 00:00:00 +0000', '2021-01-01 00:00:00 +0000', nil, '2022-01-01 23:59:59 +0000'] }
 
       it { is_expected.to eq(Time.zone.parse('2021-01-01 00:00:00 +0000')) }
     end
diff --git a/spec/lib/gitlab/import_export/project/sample/relation_factory_spec.rb b/spec/lib/gitlab/import_export/project/sample/relation_factory_spec.rb
new file mode 100644
index 00000000000..86d5f2402f8
--- /dev/null
+++ b/spec/lib/gitlab/import_export/project/sample/relation_factory_spec.rb
@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::ImportExport::Project::Sample::RelationFactory do
+  let(:group) { create(:group) }
+  let(:project) { create(:project, :repository, group: group) }
+  let(:members_mapper) { double('members_mapper').as_null_object }
+  let(:admin) { create(:admin) }
+  let(:importer_user) { admin }
+  let(:excluded_keys) { [] }
+  let(:date_calculator) { instance_double(Gitlab::ImportExport::Project::Sample::DateCalculator) }
+  let(:original_project_id) { 8 }
+  let(:start_date) { Time.current - 30.days }
+  let(:due_date) { Time.current - 20.days }
+  let(:created_object) do
+    described_class.create( # rubocop:disable Rails/SaveBang
+      relation_sym: relation_sym,
+      relation_hash: relation_hash,
+      object_builder: Gitlab::ImportExport::Project::ObjectBuilder,
+      members_mapper: members_mapper,
+      user: importer_user,
+      importable: project,
+      excluded_keys: excluded_keys,
+      date_calculator: date_calculator
+    )
+  end
+
+  context 'issue object' do
+    let(:relation_sym) { :issues }
+    let(:id) { 999 }
+
+    let(:relation_hash) do
+      {
+        'id' => id,
+        'title' => 'Necessitatibus magnam qui at velit consequatur perspiciatis.',
+        'project_id' => original_project_id,
+        'created_at' => '2016-08-12T09:41:03.462Z',
+        'updated_at' => '2016-08-12T09:41:03.462Z',
+        'description' => 'Molestiae corporis magnam et fugit aliquid nulla quia.',
+        'state' => 'closed',
+        'position' => 0,
+        'confidential' => false,
+        'due_date' => due_date
+      }
+    end
+
+    before do
+      allow(date_calculator).to receive(:closest_date_to_average) { Time.current - 10.days }
+      allow(date_calculator).to receive(:calculate_by_closest_date_to_average)
+    end
+
+    it 'correctly updated due date', :aggregate_failures do
+      expect(date_calculator).to receive(:calculate_by_closest_date_to_average)
+        .with(relation_hash['due_date']).and_return(due_date - 10.days)
+
+      expect(created_object.due_date).to eq((due_date - 10.days).to_date)
+    end
+  end
+
+  context 'milestone object' do
+    let(:relation_sym) { :milestones }
+    let(:id) { 1001 }
+
+    let(:relation_hash) do
+      {
+        'id' => id,
+        'title' => 'v3.0',
+        'project_id' => original_project_id,
+        'created_at' => '2016-08-12T09:41:03.462Z',
+        'updated_at' => '2016-08-12T09:41:03.462Z',
+        'description' => 'Rerum at autem exercitationem ea voluptates harum quam placeat.',
+        'state' => 'closed',
+        'start_date' => start_date,
+        'due_date' => due_date
+      }
+    end
+
+    before do
+      allow(date_calculator).to receive(:closest_date_to_average).twice { Time.current - 10.days }
+      allow(date_calculator).to receive(:calculate_by_closest_date_to_average).twice
+    end
+
+    it 'correctly updated due date', :aggregate_failures do
+      expect(date_calculator).to receive(:calculate_by_closest_date_to_average)
+        .with(relation_hash['due_date']).and_return(due_date - 10.days)
+
+      expect(created_object.due_date).to eq((due_date - 10.days).to_date)
+    end
+
+    it 'correctly updated start date', :aggregate_failures do
+      expect(date_calculator).to receive(:calculate_by_closest_date_to_average)
+        .with(relation_hash['start_date']).and_return(start_date - 20.days)
+
+      expect(created_object.start_date).to eq((start_date - 20.days).to_date)
+    end
+  end
+
+  context 'milestone object' do
+    let(:relation_sym) { :milestones }
+    let(:id) { 1001 }
+
+    let(:relation_hash) do
+      {
+        'id' => id,
+        'title' => 'v3.0',
+        'project_id' => original_project_id,
+        'created_at' => '2016-08-12T09:41:03.462Z',
+        'updated_at' => '2016-08-12T09:41:03.462Z',
+        'description' => 'Rerum at autem exercitationem ea voluptates harum quam placeat.',
+        'state' => 'closed',
+        'start_date' => start_date,
+        'due_date' => due_date
+      }
+    end
+
+    before do
+      allow(date_calculator).to receive(:closest_date_to_average).twice { Time.current - 10.days }
+      allow(date_calculator).to receive(:calculate_by_closest_date_to_average).twice
+    end
+
+    it 'correctly updated due date', :aggregate_failures do
+      expect(date_calculator).to receive(:calculate_by_closest_date_to_average)
+        .with(relation_hash['due_date']).and_return(due_date - 10.days)
+
+      expect(created_object.due_date).to eq((due_date - 10.days).to_date)
+    end
+
+    it 'correctly updated start date', :aggregate_failures do
+      expect(date_calculator).to receive(:calculate_by_closest_date_to_average)
+        .with(relation_hash['start_date']).and_return(start_date - 20.days)
+
+      expect(created_object.start_date).to eq((start_date - 20.days).to_date)
+    end
+  end
+
+  context 'hook object' do
+    let(:relation_sym) { :hooks }
+    let(:id) { 999 }
+    let(:service_id) { 99 }
+    let(:token) { 'secret' }
+
+    let(:relation_hash) do
+      {
+        'id' => id,
+        'url' => 'https://example.json',
+        'project_id' => original_project_id,
+        'created_at' => '2016-08-12T09:41:03.462Z',
+        'updated_at' => '2016-08-12T09:41:03.462Z',
+        'service_id' => service_id,
+        'push_events' => true,
+        'issues_events' => false,
+        'confidential_issues_events' => false,
+        'merge_requests_events' => true,
+        'tag_push_events' => false,
+        'note_events' => true,
+        'enable_ssl_verification' => true,
+        'job_events' => false,
+        'wiki_page_events' => true,
+        'token' => token
+      }
+    end
+
+    it 'does not calculate the closest date to average' do
+      expect(date_calculator).not_to receive(:calculate_by_closest_date_to_average)
+    end
+  end
+end
diff --git a/spec/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/sample/relation_tree_restorer_spec.rb
index f173345a4c6..f87f79d4462 100644
--- a/spec/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer_spec.rb
+++ b/spec/lib/gitlab/import_export/project/sample/relation_tree_restorer_spec.rb
@@ -9,7 +9,7 @@
 
 require 'spec_helper'
 
-RSpec.describe Gitlab::ImportExport::Project::Sample::SampleDataRelationTreeRestorer do
+RSpec.describe Gitlab::ImportExport::Project::Sample::RelationTreeRestorer do
   include_context 'relation tree restorer shared context'
 
   let(:sample_data_relation_tree_restorer) do
@@ -74,13 +74,26 @@ RSpec.describe Gitlab::ImportExport::Project::Sample::SampleDataRelationTreeRest
     let(:importable_name) { 'project' }
     let(:importable_path) { 'project' }
     let(:object_builder) { Gitlab::ImportExport::Project::ObjectBuilder }
-    let(:relation_factory) { Gitlab::ImportExport::Project::RelationFactory }
+    let(:relation_factory) { Gitlab::ImportExport::Project::Sample::RelationFactory }
     let(:reader) { Gitlab::ImportExport::Reader.new(shared: shared) }
+    let(:path) { 'spec/fixtures/lib/gitlab/import_export/sample_data/tree' }
+    let(:relation_reader) { Gitlab::ImportExport::JSON::NdjsonReader.new(path) }
 
-    context 'using ndjson reader' do
-      let(:path) { 'spec/fixtures/lib/gitlab/import_export/sample_data/tree' }
-      let(:relation_reader) { Gitlab::ImportExport::JSON::NdjsonReader.new(path) }
+    it 'initializes relation_factory with date_calculator as parameter' do
+      expect(Gitlab::ImportExport::Project::Sample::RelationFactory).to receive(:create).with(hash_including(:date_calculator)).at_least(:once).times
+
+      subject
+    end
+
+    context 'when relation tree restorer is initialized' do
+      it 'initializes date calculator with due dates' do
+        expect(Gitlab::ImportExport::Project::Sample::DateCalculator).to receive(:new).with(Array)
 
+        sample_data_relation_tree_restorer
+      end
+    end
+
+    context 'using ndjson reader' do
       it_behaves_like 'import project successfully'
     end
   end
diff --git a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb
index c05968c9a85..fd3b71deb37 100644
--- a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb
+++ b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb
@@ -681,13 +681,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do
         end
 
         it 'overrides project feature access levels' do
-          access_level_keys = project.project_feature.attributes.keys.select { |a| a =~ /_access_level/ }
-
-          # `pages_access_level` is not included, since it is not available in the public API
-          # and has a dependency on project's visibility level
-          # see ProjectFeature model
-          access_level_keys.delete('pages_access_level')
-
+          access_level_keys = ProjectFeature.available_features.map { |feature| ProjectFeature.access_level_attribute(feature) }
           disabled_access_levels = Hash[access_level_keys.collect { |item| [item, 'disabled'] }]
 
           project.create_import_data(data: { override_params: disabled_access_levels })
@@ -979,6 +973,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do
         create(:project, :builds_disabled, :issues_disabled,
                { name: 'project', path: 'project' })
       end
+
       let(:shared) { project.import_export_shared }
       let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) }
 
@@ -1040,41 +1035,6 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do
     it_behaves_like 'project tree restorer work properly', :legacy_reader, true
 
     it_behaves_like 'project tree restorer work properly', :ndjson_reader, true
-
-    context 'Sample Data JSON' do
-      let(:user) { create(:user) }
-      let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') }
-      let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) }
-
-      before do
-        setup_import_export_config('sample_data')
-        setup_reader(:ndjson_reader)
-      end
-
-      context 'with sample_data_template' do
-        before do
-          allow(project).to receive_message_chain(:import_data, :data, :dig).with('sample_data') { true }
-        end
-
-        it 'initialize SampleDataRelationTreeRestorer' do
-          expect_next_instance_of(Gitlab::ImportExport::Project::Sample::SampleDataRelationTreeRestorer) do |restorer|
-            expect(restorer).to receive(:restore).and_return(true)
-          end
-
-          expect(project_tree_restorer.restore).to eq(true)
-        end
-      end
-
-      context 'without sample_data_template' do
-        it 'initialize RelationTreeRestorer' do
-          expect_next_instance_of(Gitlab::ImportExport::RelationTreeRestorer) do |restorer|
-            expect(restorer).to receive(:restore).and_return(true)
-          end
-
-          expect(project_tree_restorer.restore).to eq(true)
-        end
-      end
-    end
   end
 
   context 'disable ndjson import' do
diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml
index e3d1f2c9368..b33462b4096 100644
--- a/spec/lib/gitlab/import_export/safe_model_attributes.yml
+++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml
@@ -489,6 +489,7 @@ ProjectHook:
 - confidential_issues_events
 - confidential_note_events
 - repository_update_events
+- releases_events
 ProtectedBranch:
 - id
 - project_id
@@ -575,6 +576,7 @@ ProjectFeature:
 - repository_access_level
 - pages_access_level
 - metrics_dashboard_access_level
+- requirements_access_level
 - created_at
 - updated_at
 ProtectedBranch::MergeAccessLevel:
@@ -683,6 +685,7 @@ ProjectCiCdSetting:
 ProjectSetting:
 - allow_merge_on_skipped_pipeline
 - has_confluence
+- has_vulnerabilities
 ProtectedEnvironment:
 - id
 - project_id
@@ -771,6 +774,7 @@ ExternalPullRequest:
 - target_sha
 DesignManagement::Design:
 - id
+- iid
 - project_id
 - filename
 - relative_position
@@ -858,3 +862,25 @@ ProjectSecuritySetting:
 IssuableSla:
   - issue_id
   - due_at
+PushRule:
+  - force_push_regex
+  - delete_branch_regex
+  - commit_message_regex
+  - author_email_regex
+  - file_name_regex
+  - branch_name_regex
+  - commit_message_negative_regex
+  - max_file_size
+  - deny_delete_tag
+  - member_check
+  - is_sample
+  - prevent_secrets
+  - reject_unsigned_commits
+  - commit_committer_check
+  - regexp_uses_re2
+MergeRequest::CleanupSchedule:
+- id
+- scheduled_at
+- completed_at
+- created_at
+- updated_at
diff --git a/spec/lib/gitlab/import_export/uploads_manager_spec.rb b/spec/lib/gitlab/import_export/uploads_manager_spec.rb
index 33ad0e12c37..8282ad9a070 100644
--- a/spec/lib/gitlab/import_export/uploads_manager_spec.rb
+++ b/spec/lib/gitlab/import_export/uploads_manager_spec.rb
@@ -23,13 +23,13 @@ RSpec.describe Gitlab::ImportExport::UploadsManager do
   end
 
   describe '#save' do
+    before do
+      project.uploads << upload
+    end
+
     context 'when the project has uploads locally stored' do
       let(:upload) { create(:upload, :issuable_upload, :with_file, model: project) }
 
-      before do
-        project.uploads << upload
-      end
-
       it 'does not cause errors' do
         manager.save
 
@@ -74,6 +74,22 @@ RSpec.describe Gitlab::ImportExport::UploadsManager do
         end
       end
     end
+
+    context 'when upload is in object storage' do
+      before do
+        stub_uploads_object_storage(FileUploader)
+        allow(manager).to receive(:download_or_copy_upload).and_raise(Errno::ENAMETOOLONG)
+      end
+
+      it 'ignores problematic upload and logs exception' do
+        expect(Gitlab::ErrorTracking).to receive(:log_exception).with(instance_of(Errno::ENAMETOOLONG), project_id: project.id)
+
+        manager.save
+
+        expect(shared.errors).to be_empty
+        expect(File).not_to exist(exported_file_path)
+      end
+    end
   end
 
   describe '#restore' do