From 9f46488805e86b1bc341ea1620b866016c2ce5ed Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 20 May 2020 14:34:42 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-0-stable-ee --- spec/lib/gitlab/import_export/all_models.yml | 22 ++- .../import_export/attribute_configuration_spec.rb | 3 - .../import_export/design_repo_restorer_spec.rb | 42 +++++ .../gitlab/import_export/design_repo_saver_spec.rb | 37 +++++ .../import_export/fast_hash_serializer_spec.rb | 2 +- .../group/legacy_tree_restorer_spec.rb | 2 +- .../import_export/group/tree_restorer_spec.rb | 184 +++++++++++++++++++++ .../gitlab/import_export/group/tree_saver_spec.rb | 140 ++++++++++++++++ .../import_export_equivalence_spec.rb | 4 +- .../import_export/import_test_coverage_spec.rb | 13 +- spec/lib/gitlab/import_export/importer_spec.rb | 79 ++++++--- .../import_export/json/legacy_reader/file_spec.rb | 2 +- .../import_export/json/legacy_reader/hash_spec.rb | 4 +- .../import_export/json/ndjson_reader_spec.rb | 14 +- spec/lib/gitlab/import_export/lfs_saver_spec.rb | 2 +- .../import_export/project/export_task_spec.rb | 43 ++++- .../import_export/project/import_task_spec.rb | 49 +----- .../import_export/project/tree_restorer_spec.rb | 108 +++++++----- .../import_export/project/tree_saver_spec.rb | 25 +++ .../import_export/relation_tree_restorer_spec.rb | 14 +- .../gitlab/import_export/safe_model_attributes.yml | 2 + 21 files changed, 632 insertions(+), 159 deletions(-) create mode 100644 spec/lib/gitlab/import_export/design_repo_restorer_spec.rb create mode 100644 spec/lib/gitlab/import_export/design_repo_saver_spec.rb create mode 100644 spec/lib/gitlab/import_export/group/tree_restorer_spec.rb create mode 100644 spec/lib/gitlab/import_export/group/tree_saver_spec.rb (limited to 'spec/lib/gitlab/import_export') diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 5d5e2fe2a33..c78b4501310 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -6,10 +6,12 @@ issues: - assignees - updated_by - milestone +- iteration - notes - resource_label_events - resource_weight_events - resource_milestone_events +- resource_state_events - sent_notifications - sentry_issue - label_links @@ -18,6 +20,7 @@ issues: - todos - user_agent_detail - moved_to +- moved_from - duplicated_to - promoted_to_epic - events @@ -39,6 +42,8 @@ issues: - related_vulnerabilities - user_mentions - system_note_metadata +- alert_management_alert +- status_page_published_incident events: - author - project @@ -111,9 +116,11 @@ merge_requests: - assignee - updated_by - milestone +- iteration - notes - resource_label_events - resource_milestone_events +- resource_state_events - label_links - labels - last_edited_by @@ -212,7 +219,7 @@ ci_pipelines: - vulnerability_findings - pipeline_config - security_scans -- daily_report_results +- daily_build_group_report_results pipeline_variables: - pipeline stages: @@ -222,6 +229,7 @@ stages: - processables - builds - bridges +- latest_statuses statuses: - project - pipeline @@ -343,6 +351,7 @@ project: - labels - events - milestones +- iterations - notes - snippets - hooks @@ -420,7 +429,6 @@ project: - mirror_user - push_rule - jenkins_service -- jenkins_deprecated_service - index_status - feature_usage - approval_rules @@ -443,6 +451,7 @@ project: - vulnerability_scanners - operations_feature_flags - operations_feature_flags_client +- operations_feature_flags_user_lists - prometheus_alerts - prometheus_alert_events - self_managed_prometheus_alert_events @@ -477,9 +486,14 @@ project: - status_page_setting - requirements - export_jobs -- daily_report_results +- daily_build_group_report_results - jira_imports - compliance_framework_setting +- metrics_users_starred_dashboards +- alert_management_alerts +- repository_storage_moves +- freeze_periods +- webex_teams_service award_emoji: - awardable - user @@ -631,3 +645,5 @@ epic_issue: system_note_metadata: - note - description_version +status_page_published_incident: +- issue diff --git a/spec/lib/gitlab/import_export/attribute_configuration_spec.rb b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb index 58da25bbedb..f97dafc6bf9 100644 --- a/spec/lib/gitlab/import_export/attribute_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb @@ -43,7 +43,4 @@ describe 'Import/Export attribute configuration' do IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} MSG end - - class Author < User - end end diff --git a/spec/lib/gitlab/import_export/design_repo_restorer_spec.rb b/spec/lib/gitlab/import_export/design_repo_restorer_spec.rb new file mode 100644 index 00000000000..5662b8af280 --- /dev/null +++ b/spec/lib/gitlab/import_export/design_repo_restorer_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::DesignRepoRestorer do + include GitHelpers + + describe 'bundle a design Git repo' do + let(:user) { create(:user) } + let!(:project_with_design_repo) { create(:project, :design_repo) } + let!(:project) { create(:project) } + let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } + let(:shared) { project.import_export_shared } + let(:bundler) { Gitlab::ImportExport::DesignRepoSaver.new(project: project_with_design_repo, shared: shared) } + let(:bundle_path) { File.join(shared.export_path, Gitlab::ImportExport.design_repo_bundle_filename) } + let(:restorer) do + described_class.new(path_to_bundle: bundle_path, + shared: shared, + project: project) + end + + before do + allow_next_instance_of(Gitlab::ImportExport) do |instance| + allow(instance).to receive(:storage_path).and_return(export_path) + end + + bundler.save + end + + after do + FileUtils.rm_rf(export_path) + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + FileUtils.rm_rf(project_with_design_repo.design_repository.path_to_repo) + FileUtils.rm_rf(project.design_repository.path_to_repo) + end + end + + it 'restores the repo successfully' do + expect(restorer.restore).to eq(true) + end + end +end diff --git a/spec/lib/gitlab/import_export/design_repo_saver_spec.rb b/spec/lib/gitlab/import_export/design_repo_saver_spec.rb new file mode 100644 index 00000000000..bff48e8b52a --- /dev/null +++ b/spec/lib/gitlab/import_export/design_repo_saver_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::DesignRepoSaver do + describe 'bundle a design Git repo' do + let_it_be(:user) { create(:user) } + let_it_be(:design) { create(:design, :with_file, versions_count: 1) } + let!(:project) { create(:project, :design_repo) } + let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } + let(:shared) { project.import_export_shared } + let(:design_bundler) { described_class.new(project: project, shared: shared) } + + before do + project.add_maintainer(user) + allow_next_instance_of(Gitlab::ImportExport) do |instance| + allow(instance).to receive(:storage_path).and_return(export_path) + end + end + + after do + FileUtils.rm_rf(export_path) + end + + it 'bundles the repo successfully' do + expect(design_bundler.save).to be true + end + + context 'when the repo is empty' do + let!(:project) { create(:project) } + + it 'bundles the repo successfully' do + expect(design_bundler.save).to be true + end + end + end +end diff --git a/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb b/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb index 15058684229..916ed692a05 100644 --- a/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb @@ -8,7 +8,7 @@ describe Gitlab::ImportExport::FastHashSerializer do # Wrapping the result into JSON generating/parsing is for making # the testing more convenient. Doing this, we can check that # all items are properly serialized while traversing the simple hash. - subject { JSON.parse(JSON.generate(described_class.new(project, tree).execute)) } + subject { Gitlab::Json.parse(Gitlab::Json.generate(described_class.new(project, tree).execute)) } let!(:project) { setup_project } let(:user) { create(:user) } diff --git a/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb index 3030cdf4cf8..4c926da1436 100644 --- a/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/legacy_tree_restorer_spec.rb @@ -141,7 +141,7 @@ describe Gitlab::ImportExport::Group::LegacyTreeRestorer do let(:filepath) { "group_exports/visibility_levels/#{visibility_level}" } it "imports all subgroups as #{visibility_level}" do - expect(group.children.map(&:visibility_level)).to eq(expected_visibilities) + expect(group.children.map(&:visibility_level)).to match_array(expected_visibilities) end end end diff --git a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb new file mode 100644 index 00000000000..327f36c664e --- /dev/null +++ b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb @@ -0,0 +1,184 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Group::TreeRestorer do + include ImportExport::CommonUtil + + describe 'restore group tree' do + before_all do + # Using an admin for import, so we can check assignment of existing members + user = create(:admin, email: 'root@gitlabexample.com') + create(:user, email: 'adriene.mcclure@gitlabexample.com') + create(:user, email: 'gwendolyn_robel@gitlabexample.com') + + RSpec::Mocks.with_temporary_scope do + @group = create(:group, name: 'group', path: 'group') + @shared = Gitlab::ImportExport::Shared.new(@group) + + setup_import_export_config('group_exports/complex') + + group_tree_restorer = described_class.new(user: user, shared: @shared, group: @group) + + expect(group_tree_restorer.restore).to be_truthy + end + end + + it 'has the group description' do + expect(Group.find_by_path('group').description).to eq('Group Description') + end + + it 'has group labels' do + expect(@group.labels.count).to eq(10) + end + + context 'issue boards' do + it 'has issue boards' do + expect(@group.boards.count).to eq(1) + end + + it 'has board label lists' do + lists = @group.boards.find_by(name: 'first board').lists + + expect(lists.count).to eq(3) + expect(lists.first.label.title).to eq('TSL') + expect(lists.second.label.title).to eq('Sosync') + end + end + + it 'has badges' do + expect(@group.badges.count).to eq(1) + end + + it 'has milestones' do + expect(@group.milestones.count).to eq(5) + end + + it 'has group children' do + expect(@group.children.count).to eq(2) + end + + it 'has group members' do + expect(@group.members.map(&:user).map(&:email)).to contain_exactly( + 'root@gitlabexample.com', + 'adriene.mcclure@gitlabexample.com', + 'gwendolyn_robel@gitlabexample.com' + ) + end + end + + context 'child with no parent' do + let(:user) { create(:user) } + let(:group) { create(:group) } + let(:shared) { Gitlab::ImportExport::Shared.new(group) } + let(:group_tree_restorer) { described_class.new(user: user, shared: shared, group: group) } + + before do + setup_import_export_config('group_exports/child_with_no_parent') + + expect(group_tree_restorer.restore).to be_falsey + end + + it 'fails when a child group does not have a valid parent_id' do + expect(shared.errors).to include('Parent group not found') + end + end + + context 'excluded attributes' do + let!(:source_user) { create(:user, id: 123) } + let!(:importer_user) { create(:user) } + let(:group) { create(:group, name: 'user-inputed-name', path: 'user-inputed-path') } + let(:shared) { Gitlab::ImportExport::Shared.new(group) } + let(:group_tree_restorer) { described_class.new(user: importer_user, shared: shared, group: group) } + let(:exported_file) { File.join(shared.export_path, 'tree/groups/4352.json') } + let(:group_json) { ActiveSupport::JSON.decode(IO.read(exported_file)) } + + shared_examples 'excluded attributes' do + excluded_attributes = %w[ + id + parent_id + owner_id + created_at + updated_at + runners_token + runners_token_encrypted + saml_discovery_token + ] + + before do + group.add_owner(importer_user) + + setup_import_export_config('group_exports/complex') + + expect(File.exist?(exported_file)).to be_truthy + + group_tree_restorer.restore + group.reload + end + + it 'does not import root group name' do + expect(group.name).to eq('user-inputed-name') + end + + it 'does not import root group path' do + expect(group.path).to eq('user-inputed-path') + end + + excluded_attributes.each do |excluded_attribute| + it 'does not allow override of excluded attributes' do + unless group.public_send(excluded_attribute).nil? + expect(group_json[excluded_attribute]).not_to eq(group.public_send(excluded_attribute)) + end + end + end + end + + include_examples 'excluded attributes' + end + + context 'group.json file access check' do + let(:user) { create(:user) } + let!(:group) { create(:group, name: 'group2', path: 'group2') } + let(:shared) { Gitlab::ImportExport::Shared.new(group) } + let(:group_tree_restorer) { described_class.new(user: user, shared: shared, group: group) } + + it 'does not read a symlink' do + Dir.mktmpdir do |tmpdir| + FileUtils.mkdir_p(File.join(tmpdir, 'tree', 'groups')) + setup_symlink(tmpdir, 'tree/groups/_all.ndjson') + + allow(shared).to receive(:export_path).and_return(tmpdir) + + expect(group_tree_restorer.restore).to eq(false) + expect(shared.errors).to include('Incorrect JSON format') + end + end + end + + context 'group visibility levels' do + let(:user) { create(:user) } + let(:shared) { Gitlab::ImportExport::Shared.new(group) } + let(:group_tree_restorer) { described_class.new(user: user, shared: shared, group: group) } + + before do + setup_import_export_config(filepath) + + group_tree_restorer.restore + end + + shared_examples 'with visibility level' do |visibility_level, expected_visibilities| + context "when visibility level is #{visibility_level}" do + let(:group) { create(:group, visibility_level) } + let(:filepath) { "group_exports/visibility_levels/#{visibility_level}" } + + it "imports all subgroups as #{visibility_level}" do + expect(group.children.map(&:visibility_level)).to eq(expected_visibilities) + end + end + end + + include_examples 'with visibility level', :public, [20, 10, 0] + include_examples 'with visibility level', :private, [0, 0, 0] + include_examples 'with visibility level', :internal, [10, 10, 0] + end +end diff --git a/spec/lib/gitlab/import_export/group/tree_saver_spec.rb b/spec/lib/gitlab/import_export/group/tree_saver_spec.rb new file mode 100644 index 00000000000..06e8484a3cb --- /dev/null +++ b/spec/lib/gitlab/import_export/group/tree_saver_spec.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::Group::TreeSaver do + describe 'saves the group tree into a json object' do + let_it_be(:user) { create(:user) } + let_it_be(:group) { setup_groups } + + let(:shared) { Gitlab::ImportExport::Shared.new(group) } + let(:export_path) { "#{Dir.tmpdir}/group_tree_saver_spec" } + + subject(:group_tree_saver) { described_class.new(group: group, current_user: user, shared: shared) } + + before_all do + group.add_maintainer(user) + end + + before do + allow_next_instance_of(Gitlab::ImportExport) do |import_export| + allow(import_export).to receive(:storage_path).and_return(export_path) + end + end + + after do + FileUtils.rm_rf(export_path) + end + + it 'saves the group successfully' do + expect(group_tree_saver.save).to be true + end + + it 'fails to export a group' do + allow_next_instance_of(Gitlab::ImportExport::JSON::NdjsonWriter) do |ndjson_writer| + allow(ndjson_writer).to receive(:write_relation_array).and_raise(RuntimeError, 'exception') + end + + expect(shared).to receive(:error).with(RuntimeError).and_call_original + + expect(group_tree_saver.save).to be false + end + + context 'exported files' do + before do + group_tree_saver.save + end + + it 'has one group per line' do + groups_catalog = + File.readlines(exported_path_for('_all.ndjson')) + .map { |line| Integer(line) } + + expect(groups_catalog.size).to eq(3) + expect(groups_catalog).to eq([ + group.id, + group.descendants.first.id, + group.descendants.first.descendants.first.id + ]) + end + + it 'has a file per group' do + group.self_and_descendants.pluck(:id).each do |id| + group_attributes_file = exported_path_for("#{id}.json") + + expect(File.exist?(group_attributes_file)).to be(true) + end + end + + context 'group attributes file' do + let(:group_attributes_file) { exported_path_for("#{group.id}.json") } + let(:group_attributes) { ::JSON.parse(File.read(group_attributes_file)) } + + it 'has a file for each group with its attributes' do + expect(group_attributes['description']).to eq(group.description) + expect(group_attributes['parent_id']).to eq(group.parent_id) + end + + shared_examples 'excluded attributes' do + excluded_attributes = %w[ + owner_id + created_at + updated_at + runners_token + runners_token_encrypted + saml_discovery_token + ] + + excluded_attributes.each do |excluded_attribute| + it 'does not contain excluded attribute' do + expect(group_attributes).not_to include(excluded_attribute => group.public_send(excluded_attribute)) + end + end + end + + include_examples 'excluded attributes' + end + + it 'has a file for each group association' do + group.self_and_descendants do |g| + %w[ + badges + boards + epics + labels + members + milestones + ].each do |association| + path = exported_path_for("#{g.id}", "#{association}.ndjson") + expect(File.exist?(path)).to eq(true), "#{path} does not exist" + end + end + end + end + end + + def exported_path_for(*file) + File.join(group_tree_saver.full_path, 'groups', *file) + end + + def setup_groups + root = setup_group + subgroup = setup_group(parent: root) + setup_group(parent: subgroup) + + root + end + + def setup_group(parent: nil) + group = create(:group, description: 'description', parent: parent) + create(:milestone, group: group) + create(:group_badge, group: group) + group_label = create(:group_label, group: group) + board = create(:board, group: group, milestone_id: Milestone::Upcoming.id) + create(:list, board: board, label: group_label) + create(:group_badge, group: group) + create(:label_priority, label: group_label, priority: 1) + + group + end +end diff --git a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb index 707975f20b6..95df9cd0e6e 100644 --- a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb +++ b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb @@ -46,8 +46,8 @@ describe Gitlab::ImportExport do export_path: test_tmp_path) ).to be true - imported_json = JSON.parse(File.read("#{test_fixture_path}/project.json")) - exported_json = JSON.parse(File.read("#{test_tmp_path}/project.json")) + imported_json = Gitlab::Json.parse(File.read("#{test_fixture_path}/project.json")) + exported_json = Gitlab::Json.parse(File.read("#{test_tmp_path}/project.json")) assert_relations_match(imported_json, exported_json) end diff --git a/spec/lib/gitlab/import_export/import_test_coverage_spec.rb b/spec/lib/gitlab/import_export/import_test_coverage_spec.rb index 335b0031147..038b95809b4 100644 --- a/spec/lib/gitlab/import_export/import_test_coverage_spec.rb +++ b/spec/lib/gitlab/import_export/import_test_coverage_spec.rb @@ -53,22 +53,15 @@ describe 'Test coverage of the Project Import' do ].freeze # A list of JSON fixture files we use to test Import. - # Note that we use separate fixture to test ee-only features. # Most of the relations are present in `complex/project.json` # which is our main fixture. - PROJECT_JSON_FIXTURES_EE = - if Gitlab.ee? - ['ee/spec/fixtures/lib/gitlab/import_export/designs/project.json'].freeze - else - [] - end - PROJECT_JSON_FIXTURES = [ 'spec/fixtures/lib/gitlab/import_export/complex/project.json', 'spec/fixtures/lib/gitlab/import_export/group/project.json', 'spec/fixtures/lib/gitlab/import_export/light/project.json', - 'spec/fixtures/lib/gitlab/import_export/milestone-iid/project.json' - ].freeze + PROJECT_JSON_FIXTURES_EE + 'spec/fixtures/lib/gitlab/import_export/milestone-iid/project.json', + 'spec/fixtures/lib/gitlab/import_export/designs/project.json' + ].freeze it 'ensures that all imported/exported relations are present in test JSONs' do not_tested_relations = (relations_from_config - tested_relations) - MUTED_RELATIONS diff --git a/spec/lib/gitlab/import_export/importer_spec.rb b/spec/lib/gitlab/import_export/importer_spec.rb index e03c95525df..60179146416 100644 --- a/spec/lib/gitlab/import_export/importer_spec.rb +++ b/spec/lib/gitlab/import_export/importer_spec.rb @@ -51,7 +51,8 @@ describe Gitlab::ImportExport::Importer do Gitlab::ImportExport::UploadsRestorer, Gitlab::ImportExport::LfsRestorer, Gitlab::ImportExport::StatisticsRestorer, - Gitlab::ImportExport::SnippetsRepoRestorer + Gitlab::ImportExport::SnippetsRepoRestorer, + Gitlab::ImportExport::DesignRepoRestorer ].each do |restorer| it "calls the #{restorer}" do fake_restorer = double(restorer.to_s) @@ -89,36 +90,74 @@ describe Gitlab::ImportExport::Importer do end context 'when project successfully restored' do - let!(:existing_project) { create(:project, namespace: user.namespace) } - let(:project) { create(:project, namespace: user.namespace, name: 'whatever', path: 'whatever') } + context "with a project in a user's namespace" do + let!(:existing_project) { create(:project, namespace: user.namespace) } + let(:project) { create(:project, namespace: user.namespace, name: 'whatever', path: 'whatever') } - before do - restorers = double(:restorers, all?: true) + before do + restorers = double(:restorers, all?: true) - allow(subject).to receive(:import_file).and_return(true) - allow(subject).to receive(:check_version!).and_return(true) - allow(subject).to receive(:restorers).and_return(restorers) - allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path })) + allow(subject).to receive(:import_file).and_return(true) + allow(subject).to receive(:check_version!).and_return(true) + allow(subject).to receive(:restorers).and_return(restorers) + allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path })) + end + + context 'when import_data' do + context 'has original_path' do + it 'overwrites existing project' do + expect_next_instance_of(::Projects::OverwriteProjectService) do |service| + expect(service).to receive(:execute).with(existing_project) + end + + subject.execute + end + end + + context 'has not original_path' do + before do + allow(project).to receive(:import_data).and_return(double(data: {})) + end + + it 'does not call the overwrite service' do + expect(::Projects::OverwriteProjectService).not_to receive(:new) + + subject.execute + end + end + end end - context 'when import_data' do + context "with a project in a group namespace" do + let(:group) { create(:group) } + let!(:existing_project) { create(:project, group: group) } + let(:project) { create(:project, creator: user, group: group, name: 'whatever', path: 'whatever') } + + before do + restorers = double(:restorers, all?: true) + + allow(subject).to receive(:import_file).and_return(true) + allow(subject).to receive(:check_version!).and_return(true) + allow(subject).to receive(:restorers).and_return(restorers) + allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path })) + end + context 'has original_path' do it 'overwrites existing project' do - expect_any_instance_of(::Projects::OverwriteProjectService).to receive(:execute).with(existing_project) + group.add_owner(user) - subject.execute - end - end + expect_next_instance_of(::Projects::OverwriteProjectService) do |service| + expect(service).to receive(:execute).with(existing_project) + end - context 'has not original_path' do - before do - allow(project).to receive(:import_data).and_return(double(data: {})) + subject.execute end - it 'does not call the overwrite service' do - expect_any_instance_of(::Projects::OverwriteProjectService).not_to receive(:execute).with(existing_project) + it 'does not allow user to overwrite existing project' do + expect(::Projects::OverwriteProjectService).not_to receive(:new) - subject.execute + expect { subject.execute }.to raise_error(Projects::ImportService::Error, + "User #{user.username} (#{user.id}) cannot overwrite a project in #{group.path}") end end end diff --git a/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb b/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb index 1021ce3cd50..99932404fd9 100644 --- a/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb +++ b/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb @@ -7,7 +7,7 @@ describe Gitlab::ImportExport::JSON::LegacyReader::File do it_behaves_like 'import/export json legacy reader' do let(:valid_path) { 'spec/fixtures/lib/gitlab/import_export/light/project.json' } let(:data) { valid_path } - let(:json_data) { JSON.parse(File.read(valid_path)) } + let(:json_data) { Gitlab::Json.parse(File.read(valid_path)) } end describe '#exist?' do diff --git a/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb b/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb index 8c4dfd2f356..e793dc7339d 100644 --- a/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb +++ b/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb @@ -9,8 +9,8 @@ describe Gitlab::ImportExport::JSON::LegacyReader::Hash do # the hash is modified by the `LegacyReader` # we need to deep-dup it - let(:json_data) { JSON.parse(File.read(path)) } - let(:data) { JSON.parse(File.read(path)) } + let(:json_data) { Gitlab::Json.parse(File.read(path)) } + let(:data) { Gitlab::Json.parse(File.read(path)) } end describe '#exist?' do diff --git a/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb b/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb index 40b784fdb87..34e8b1ddd59 100644 --- a/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb +++ b/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb @@ -6,18 +6,10 @@ describe Gitlab::ImportExport::JSON::NdjsonReader do include ImportExport::CommonUtil let(:fixture) { 'spec/fixtures/lib/gitlab/import_export/light/tree' } - let(:root_tree) { JSON.parse(File.read(File.join(fixture, 'project.json'))) } + let(:root_tree) { Gitlab::Json.parse(File.read(File.join(fixture, 'project.json'))) } let(:ndjson_reader) { described_class.new(dir_path) } let(:importable_path) { 'project' } - before :all do - extract_archive('spec/fixtures/lib/gitlab/import_export/light', 'tree.tar.gz') - end - - after :all do - cleanup_artifacts_from_extract_archive('light') - end - describe '#exist?' do subject { ndjson_reader.exist? } @@ -101,8 +93,8 @@ describe Gitlab::ImportExport::JSON::NdjsonReader do context 'relation file contains multiple lines' do let(:key) { 'custom_attributes' } - let(:attr_1) { JSON.parse('{"id":201,"project_id":5,"created_at":"2016-06-14T15:01:51.315Z","updated_at":"2016-06-14T15:01:51.315Z","key":"color","value":"red"}') } - let(:attr_2) { JSON.parse('{"id":202,"project_id":5,"created_at":"2016-06-14T15:01:51.315Z","updated_at":"2016-06-14T15:01:51.315Z","key":"size","value":"small"}') } + let(:attr_1) { Gitlab::Json.parse('{"id":201,"project_id":5,"created_at":"2016-06-14T15:01:51.315Z","updated_at":"2016-06-14T15:01:51.315Z","key":"color","value":"red"}') } + let(:attr_2) { Gitlab::Json.parse('{"id":202,"project_id":5,"created_at":"2016-06-14T15:01:51.315Z","updated_at":"2016-06-14T15:01:51.315Z","key":"size","value":"small"}') } it 'yields every relation value to the Enumerator' do expect(subject.to_a).to eq([[attr_1, 0], [attr_2, 1]]) diff --git a/spec/lib/gitlab/import_export/lfs_saver_spec.rb b/spec/lib/gitlab/import_export/lfs_saver_spec.rb index a8ff7867410..e9d06573e70 100644 --- a/spec/lib/gitlab/import_export/lfs_saver_spec.rb +++ b/spec/lib/gitlab/import_export/lfs_saver_spec.rb @@ -26,7 +26,7 @@ describe Gitlab::ImportExport::LfsSaver do let(:lfs_json_file) { File.join(shared.export_path, Gitlab::ImportExport.lfs_objects_filename) } def lfs_json - JSON.parse(IO.read(lfs_json_file)) + Gitlab::Json.parse(IO.read(lfs_json_file)) end before do diff --git a/spec/lib/gitlab/import_export/project/export_task_spec.rb b/spec/lib/gitlab/import_export/project/export_task_spec.rb index cf11a1df33c..dc8eb54dc14 100644 --- a/spec/lib/gitlab/import_export/project/export_task_spec.rb +++ b/spec/lib/gitlab/import_export/project/export_task_spec.rb @@ -3,13 +3,14 @@ require 'rake_helper' describe Gitlab::ImportExport::Project::ExportTask do - let(:username) { 'root' } + let_it_be(:username) { 'root' } let(:namespace_path) { username } - let!(:user) { create(:user, username: username) } + let_it_be(:user) { create(:user, username: username) } let(:measurement_enabled) { false } let(:file_path) { 'spec/fixtures/gitlab/import_export/test_project_export.tar.gz' } let(:project) { create(:project, creator: user, namespace: user.namespace) } let(:project_name) { project.name } + let(:rake_task) { described_class.new(task_params) } let(:task_params) do { @@ -21,7 +22,7 @@ describe Gitlab::ImportExport::Project::ExportTask do } end - subject { described_class.new(task_params).export } + subject { rake_task.export } context 'when project is found' do let(:project) { create(:project, creator: user, namespace: user.namespace) } @@ -29,9 +30,13 @@ describe Gitlab::ImportExport::Project::ExportTask do around do |example| example.run ensure - File.delete(file_path) + File.delete(file_path) if File.exist?(file_path) end + include_context 'rake task object storage shared context' + + it_behaves_like 'rake task with disabled object_storage', ::Projects::ImportExport::ExportService, :success + it 'performs project export successfully' do expect { subject }.to output(/Done!/).to_stdout @@ -39,8 +44,6 @@ describe Gitlab::ImportExport::Project::ExportTask do expect(File).to exist(file_path) end - - it_behaves_like 'measurable' end context 'when project is not found' do @@ -66,4 +69,32 @@ describe Gitlab::ImportExport::Project::ExportTask do expect(subject).to eq(false) end end + + context 'when after export strategy fails' do + before do + allow_next_instance_of(Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy) do |after_export_strategy| + allow(after_export_strategy).to receive(:strategy_execute).and_raise(Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy::StrategyError) + end + end + + it 'error is logged' do + expect(rake_task).to receive(:error).and_call_original + + expect(subject).to eq(false) + end + end + + context 'when saving services fail' do + before do + allow_next_instance_of(::Projects::ImportExport::ExportService) do |service| + allow(service).to receive(:execute).and_raise(Gitlab::ImportExport::Error) + end + end + + it 'error is logged' do + expect(rake_task).to receive(:error).and_call_original + + expect(subject).to eq(false) + end + end end diff --git a/spec/lib/gitlab/import_export/project/import_task_spec.rb b/spec/lib/gitlab/import_export/project/import_task_spec.rb index 4f4fcd3ad8a..7c11161aaa7 100644 --- a/spec/lib/gitlab/import_export/project/import_task_spec.rb +++ b/spec/lib/gitlab/import_export/project/import_task_spec.rb @@ -8,7 +8,7 @@ describe Gitlab::ImportExport::Project::ImportTask, :request_store do let!(:user) { create(:user, username: username) } let(:measurement_enabled) { false } let(:project) { Project.find_by_full_path("#{namespace_path}/#{project_name}") } - let(:import_task) { described_class.new(task_params) } + let(:rake_task) { described_class.new(task_params) } let(:task_params) do { username: username, @@ -19,29 +19,16 @@ describe Gitlab::ImportExport::Project::ImportTask, :request_store do } end - before do - allow(Settings.uploads.object_store).to receive(:[]=).and_call_original - end - - around do |example| - old_direct_upload_setting = Settings.uploads.object_store['direct_upload'] - old_background_upload_setting = Settings.uploads.object_store['background_upload'] - - Settings.uploads.object_store['direct_upload'] = true - Settings.uploads.object_store['background_upload'] = true - - example.run - - Settings.uploads.object_store['direct_upload'] = old_direct_upload_setting - Settings.uploads.object_store['background_upload'] = old_background_upload_setting - end - - subject { import_task.import } + subject { rake_task.import } context 'when project import is valid' do let(:project_name) { 'import_rake_test_project' } let(:file_path) { 'spec/fixtures/gitlab/import_export/lightweight_project_export.tar.gz' } + include_context 'rake task object storage shared context' + + it_behaves_like 'rake task with disabled object_storage', ::Projects::GitlabProjectsImportService, :execute_sidekiq_job + it 'performs project import successfully' do expect { subject }.to output(/Done!/).to_stdout expect { subject }.not_to raise_error @@ -52,30 +39,6 @@ describe Gitlab::ImportExport::Project::ImportTask, :request_store do expect(project.milestones.count).to be > 0 expect(project.import_state.status).to eq('finished') end - - it 'disables direct & background upload only during project creation' do - expect_next_instance_of(Projects::GitlabProjectsImportService) do |service| - expect(service).to receive(:execute).and_wrap_original do |m| - expect(Settings.uploads.object_store['background_upload']).to eq(false) - expect(Settings.uploads.object_store['direct_upload']).to eq(false) - - m.call - end - end - - expect(import_task).to receive(:execute_sidekiq_job).and_wrap_original do |m| - expect(Settings.uploads.object_store['background_upload']).to eq(true) - expect(Settings.uploads.object_store['direct_upload']).to eq(true) - expect(Settings.uploads.object_store).not_to receive(:[]=).with('backgroud_upload', false) - expect(Settings.uploads.object_store).not_to receive(:[]=).with('direct_upload', false) - - m.call - end - - subject - end - - it_behaves_like 'measurable' end context 'when project import is invalid' do diff --git a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb index 04e8bd05666..58589a7bbbe 100644 --- a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb @@ -8,6 +8,7 @@ end describe Gitlab::ImportExport::Project::TreeRestorer do include ImportExport::CommonUtil + using RSpec::Parameterized::TableSyntax let(:shared) { project.import_export_shared } @@ -44,10 +45,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do end end - after(:context) do - cleanup_artifacts_from_extract_archive('complex') - end - context 'JSON' do it 'restores models based on JSON' do expect(@restored_project_json).to be_truthy @@ -536,10 +533,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do expect(restored_project_json).to eq(true) end - after do - cleanup_artifacts_from_extract_archive('light') - end - it 'issue system note metadata restored successfully' do note_content = 'created merge request !1 to address this issue' note = project.issues.first.notes.select { |n| n.note.match(/#{note_content}/)}.first @@ -586,10 +579,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do expect(restored_project_json).to eq(true) end - after do - cleanup_artifacts_from_extract_archive('multi_pipeline_ref_one_external_pr') - end - it_behaves_like 'restores project successfully', issues: 0, labels: 0, @@ -620,10 +609,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do .and_raise(exception) end - after do - cleanup_artifacts_from_extract_archive('light') - end - it 'report post import error' do expect(restored_project_json).to eq(false) expect(shared.errors).to include('post_import_error') @@ -646,10 +631,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do expect(restored_project_json).to eq(true) end - after do - cleanup_artifacts_from_extract_archive('light') - end - it_behaves_like 'restores project successfully', issues: 1, labels: 2, @@ -678,10 +659,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do setup_reader(reader) end - after do - cleanup_artifacts_from_extract_archive('light') - end - it 'handles string versions of visibility_level' do # Project needs to be in a group for visibility level comparison # to happen @@ -747,10 +724,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do expect(restored_project_json).to eq(true) end - after do - cleanup_artifacts_from_extract_archive('group') - end - it_behaves_like 'restores project successfully', issues: 3, labels: 2, @@ -784,10 +757,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do setup_reader(reader) end - after do - cleanup_artifacts_from_extract_archive('light') - end - it 'does not import any templated services' do expect(restored_project_json).to eq(true) @@ -835,10 +804,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do setup_reader(reader) end - after do - cleanup_artifacts_from_extract_archive('milestone-iid') - end - it 'preserves the project milestone IID' do expect_any_instance_of(Gitlab::ImportExport::Shared).not_to receive(:error) @@ -855,10 +820,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do setup_reader(reader) end - after do - cleanup_artifacts_from_extract_archive('light') - end - it 'converts empty external classification authorization labels to nil' do project.create_import_data(data: { override_params: { external_authorization_classification_label: "" } }) @@ -1004,10 +965,6 @@ describe Gitlab::ImportExport::Project::TreeRestorer do subject end - after do - cleanup_artifacts_from_extract_archive('with_invalid_records') - end - context 'when failures occur because a relation fails to be processed' do it_behaves_like 'restores project successfully', issues: 0, @@ -1031,6 +988,69 @@ describe Gitlab::ImportExport::Project::TreeRestorer do end end end + + context 'JSON with design management data' do + let_it_be(:user) { create(:admin, email: 'user_1@gitlabexample.com') } + let_it_be(:second_user) { create(:user, email: 'user_2@gitlabexample.com') } + let_it_be(:project) do + create(:project, :builds_disabled, :issues_disabled, + { name: 'project', path: 'project' }) + end + let(:shared) { project.import_export_shared } + let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) } + + subject(:restored_project_json) { project_tree_restorer.restore } + + before do + setup_import_export_config('designs') + restored_project_json + end + + it_behaves_like 'restores project successfully', issues: 2 + + it 'restores project associations correctly' do + expect(project.designs.size).to eq(7) + end + + describe 'restores issue associations correctly' do + let(:issue) { project.issues.offset(index).first } + + where(:index, :design_filenames, :version_shas, :events, :author_emails) do + 0 | %w[chirrido3.jpg jonathan_richman.jpg mariavontrap.jpeg] | %w[27702d08f5ee021ae938737f84e8fe7c38599e85 9358d1bac8ff300d3d2597adaa2572a20f7f8703 e1a4a501bcb42f291f84e5d04c8f927821542fb6] | %w[creation creation creation modification modification deletion] | %w[user_1@gitlabexample.com user_1@gitlabexample.com user_2@gitlabexample.com] + 1 | ['1 (1).jpeg', '2099743.jpg', 'a screenshot (1).jpg', 'chirrido3.jpg'] | %w[73f871b4c8c1d65c62c460635e023179fb53abc4 8587e78ab6bda3bc820a9f014c3be4a21ad4fcc8 c9b5f067f3e892122a4b12b0a25a8089192f3ac8] | %w[creation creation creation creation modification] | %w[user_1@gitlabexample.com user_2@gitlabexample.com user_2@gitlabexample.com] + end + + with_them do + it do + expect(issue.designs.pluck(:filename)).to contain_exactly(*design_filenames) + expect(issue.design_versions.pluck(:sha)).to contain_exactly(*version_shas) + expect(issue.design_versions.flat_map(&:actions).map(&:event)).to contain_exactly(*events) + expect(issue.design_versions.map(&:author).map(&:email)).to contain_exactly(*author_emails) + end + end + end + + describe 'restores design version associations correctly' do + let(:project_designs) { project.designs.reorder(:filename, :issue_id) } + let(:design) { project_designs.offset(index).first } + + where(:index, :version_shas) do + 0 | %w[73f871b4c8c1d65c62c460635e023179fb53abc4 c9b5f067f3e892122a4b12b0a25a8089192f3ac8] + 1 | %w[73f871b4c8c1d65c62c460635e023179fb53abc4] + 2 | %w[c9b5f067f3e892122a4b12b0a25a8089192f3ac8] + 3 | %w[27702d08f5ee021ae938737f84e8fe7c38599e85 9358d1bac8ff300d3d2597adaa2572a20f7f8703 e1a4a501bcb42f291f84e5d04c8f927821542fb6] + 4 | %w[8587e78ab6bda3bc820a9f014c3be4a21ad4fcc8] + 5 | %w[27702d08f5ee021ae938737f84e8fe7c38599e85 e1a4a501bcb42f291f84e5d04c8f927821542fb6] + 6 | %w[27702d08f5ee021ae938737f84e8fe7c38599e85] + end + + with_them do + it do + expect(design.versions.pluck(:sha)).to contain_exactly(*version_shas) + end + end + end + end end context 'enable ndjson import' do diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index 8adc360026d..b9bfe253f10 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -168,6 +168,28 @@ describe Gitlab::ImportExport::Project::TreeSaver do it 'has issue resource label events' do expect(subject.first['resource_label_events']).not_to be_empty end + + it 'saves the issue designs correctly' do + expect(subject.first['designs'].size).to eq(1) + end + + it 'saves the issue design notes correctly' do + expect(subject.first['designs'].first['notes']).not_to be_empty + end + + it 'saves the issue design versions correctly' do + issue_json = subject.first + actions = issue_json['design_versions'].flat_map { |v| v['actions'] } + + expect(issue_json['design_versions'].size).to eq(2) + issue_json['design_versions'].each do |version| + expect(version['author_id']).to be_kind_of(Integer) + end + expect(actions.size).to eq(2) + actions.each do |action| + expect(action['design']).to be_present + end + end end context 'with ci_pipelines' do @@ -442,6 +464,9 @@ describe Gitlab::ImportExport::Project::TreeSaver do board = create(:board, project: project, name: 'TestBoard') create(:list, board: board, position: 0, label: project_label) + design = create(:design, :with_file, versions_count: 2, issue: issue) + create(:diff_note_on_design, noteable: design, project: project, author: user) + project end end diff --git a/spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb index 0b58a75220d..8fe419da450 100644 --- a/spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/relation_tree_restorer_spec.rb @@ -64,7 +64,7 @@ describe Gitlab::ImportExport::RelationTreeRestorer do shared_examples 'logging of relations creation' do context 'when log_import_export_relation_creation feature flag is enabled' do before do - stub_feature_flags(log_import_export_relation_creation: { enabled: true, thing: group }) + stub_feature_flags(log_import_export_relation_creation: group) end it 'logs top-level relation creation' do @@ -79,7 +79,7 @@ describe Gitlab::ImportExport::RelationTreeRestorer do context 'when log_import_export_relation_creation feature flag is disabled' do before do - stub_feature_flags(log_import_export_relation_creation: { enabled: false, thing: group }) + stub_feature_flags(log_import_export_relation_creation: false) end it 'does not log top-level relation creation' do @@ -126,14 +126,6 @@ describe Gitlab::ImportExport::RelationTreeRestorer do let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/tree' } let(:relation_reader) { Gitlab::ImportExport::JSON::NdjsonReader.new(path) } - before :all do - extract_archive('spec/fixtures/lib/gitlab/import_export/complex', 'tree.tar.gz') - end - - after :all do - cleanup_artifacts_from_extract_archive('complex') - end - it_behaves_like 'import project successfully' end end @@ -156,7 +148,7 @@ describe Gitlab::ImportExport::RelationTreeRestorer do let(:reader) do Gitlab::ImportExport::Reader.new( shared: shared, - config: Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.group_config_file).to_h + config: Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.legacy_group_config_file).to_h ) end diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 88d7fdaef36..c29a85ce624 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -480,6 +480,7 @@ Service: - pipeline_events - job_events - comment_on_event_enabled +- comment_detail - category - default - wiki_page_events @@ -487,6 +488,7 @@ Service: - confidential_note_events - deployment_events - description +- inherit_from_id ProjectHook: - id - url -- cgit v1.2.3