diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
commit | 43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch) | |
tree | dceebdc68925362117480a5d672bcff122fb625b /spec/lib/gitlab/import_export | |
parent | 20c84b99005abd1c82101dfeff264ac50d2df211 (diff) |
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc42
Diffstat (limited to 'spec/lib/gitlab/import_export')
29 files changed, 661 insertions, 673 deletions
diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 0c2c3ffc664..34f9948b9dc 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -14,6 +14,7 @@ issues: - resource_milestone_events - resource_state_events - resource_iteration_events +- assignment_events - sent_notifications - sentry_issue - issuable_severity @@ -92,6 +93,25 @@ notes: - suggestions - diff_note_positions - review +- note_metadata +note_metadata: + - note + - email_participant +commit_notes: +- award_emoji +- noteable +- author +- updated_by +- last_edited_by +- resolved_by +- todos +- events +- system_note_metadata +- note_diff_file +- suggestions +- diff_note_positions +- review +- note_metadata label_links: - target - label @@ -166,6 +186,7 @@ merge_requests: - resource_milestone_events - resource_state_events - resource_iteration_events +- assignment_events - label_links - labels - last_edited_by @@ -202,7 +223,7 @@ merge_requests: - approver_groups - approved_by_users - draft_notes -- merge_train +- merge_train_car - blocks_as_blocker - blocks_as_blockee - blocking_merge_requests @@ -246,6 +267,11 @@ ci_pipelines: - statuses - statuses_order_id_desc - latest_statuses_ordered_by_stage +- latest_statuses +- all_jobs +- current_jobs +- all_processable_jobs +- current_processable_jobs - builds - bridges - processables @@ -283,6 +309,7 @@ ci_pipelines: - job_artifacts - vulnerabilities_finding_pipelines - vulnerability_findings +- vulnerability_state_transitions - pipeline_config - security_scans - security_findings @@ -293,7 +320,6 @@ ci_pipelines: - latest_builds_report_results - messages - pipeline_artifacts -- latest_statuses - dast_profile - dast_profiles_pipeline - dast_site_profile @@ -317,6 +343,7 @@ stages: - processables - builds - bridges +- generic_commit_statuses - latest_statuses - retried_statuses statuses: @@ -327,6 +354,92 @@ statuses: - auto_canceled_by - needs - ci_stage +builds: +- user +- auto_canceled_by +- ci_stage +- needs +- resource +- pipeline +- sourced_pipeline +- resource_group +- metadata +- runner +- trigger_request +- erased_by +- deployment +- pending_state +- queuing_entry +- runtime_metadata +- trace_chunks +- report_results +- namespace +- job_artifacts +- job_variables +- sourced_pipelines +- pages_deployments +- job_artifacts_archive +- job_artifacts_metadata +- job_artifacts_trace +- job_artifacts_junit +- job_artifacts_sast +- job_artifacts_dependency_scanning +- job_artifacts_container_scanning +- job_artifacts_dast +- job_artifacts_codequality +- job_artifacts_license_scanning +- job_artifacts_performance +- job_artifacts_metrics +- job_artifacts_metrics_referee +- job_artifacts_network_referee +- job_artifacts_lsif +- job_artifacts_dotenv +- job_artifacts_cobertura +- job_artifacts_terraform +- job_artifacts_accessibility +- job_artifacts_cluster_applications +- job_artifacts_secret_detection +- job_artifacts_requirements +- job_artifacts_coverage_fuzzing +- job_artifacts_browser_performance +- job_artifacts_load_performance +- job_artifacts_api_fuzzing +- job_artifacts_cluster_image_scanning +- job_artifacts_cyclonedx +- job_artifacts_requirements_v2 +- runner_manager +- runner_manager_build +- runner_session +- trace_metadata +- terraform_state_versions +- taggings +- base_tags +- tag_taggings +- tags +- security_scans +- dast_site_profiles_build +- dast_site_profile +- dast_scanner_profiles_build +- dast_scanner_profile +bridges: +- user +- pipeline +- auto_canceled_by +- ci_stage +- needs +- resource +- sourced_pipeline +- resource_group +- metadata +- trigger_request +- downstream_pipeline +- upstream_pipeline +generic_commit_statuses: +- user +- pipeline +- auto_canceled_by +- ci_stage +- needs variables: - project triggers: @@ -391,6 +504,7 @@ container_repositories: - project - name project: +- catalog_resource - external_status_checks - base_tags - project_topics @@ -399,7 +513,9 @@ project: - cluster - clusters - cluster_agents +- ci_access_project_authorizations - cluster_project +- workspaces - creator - cycle_analytics_stages - value_streams @@ -408,6 +524,7 @@ project: - project_namespace - management_clusters - boards +- application_setting - last_event - integrations - push_hooks_integrations @@ -432,6 +549,7 @@ project: - discord_integration - drone_ci_integration - emails_on_push_integration +- google_play_integration - pipelines_email_integration - mattermost_slash_commands_integration - shimo_integration @@ -466,12 +584,14 @@ project: - external_wiki_integration - mock_ci_integration - mock_monitoring_integration +- squash_tm_integration - forked_to_members - forked_from_project - forks - merge_requests - fork_merge_requests - issues +- work_items - labels - events - milestones @@ -527,6 +647,7 @@ project: - redirect_routes - statistics - container_repositories +- container_registry_data_repair_detail - uploads - file_uploads - import_state @@ -600,14 +721,15 @@ project: - project_registry - packages - package_files -- repository_files +- rpm_repository_files +- npm_metadata_caches - packages_cleanup_policy - alerting_setting - project_setting - webide_pipelines - reviews - incident_management_setting -- merge_trains +- merge_train_cars - designs - project_aliases - external_pull_requests @@ -618,6 +740,8 @@ project: - upstream_project_subscriptions - downstream_project_subscriptions - service_desk_setting +- service_desk_custom_email_verification +- service_desk_custom_email_credential - security_setting - import_failures - container_expiration_policy @@ -673,6 +797,7 @@ project: - sbom_occurrences - analytics_dashboards_configuration_project - analytics_dashboards_pointer +- design_management_repository award_emoji: - awardable - user @@ -759,6 +884,7 @@ incident_management_setting: - project merge_trains: - project +merge_train_cars: - merge_request boards: - group @@ -859,6 +985,8 @@ bulk_import_export: - group service_desk_setting: - file_template_project +service_desk_custom_email_verification: + - triggerer approvals: - user - merge_request @@ -890,3 +1018,22 @@ resource_iteration_events: iterations_cadence: - group - iterations +catalog_resource: + - project +approval_rules: + - users + - groups + - group_users + - security_orchestration_policy_configuration + - protected_branches + - approval_merge_request_rule_sources + - approval_merge_request_rules + - approval_project_rules_users + - approval_project_rules_protected_branches + - scan_result_policy_read +approval_project_rules_users: + - user + - approval_project_rule +approval_project_rules_protected_branches: + - protected_branch + - approval_project_rule diff --git a/spec/lib/gitlab/import_export/attribute_configuration_spec.rb b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb index 572f809e43b..1d84cba3825 100644 --- a/spec/lib/gitlab/import_export/attribute_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/attribute_configuration_spec.rb @@ -9,7 +9,7 @@ require 'spec_helper' # to be included as part of the export, or blacklist them using the import_export.yml configuration file. # Likewise, new models added to import_export.yml, will need to be added with their correspondent attributes # to this spec. -RSpec.describe 'Import/Export attribute configuration' do +RSpec.describe 'Import/Export attribute configuration', feature_category: :importers do include ConfigurationHelper let(:safe_attributes_file) { 'spec/lib/gitlab/import_export/safe_model_attributes.yml' } diff --git a/spec/lib/gitlab/import_export/attributes_finder_spec.rb b/spec/lib/gitlab/import_export/attributes_finder_spec.rb index 6536b895b2f..f12cbe4f82f 100644 --- a/spec/lib/gitlab/import_export/attributes_finder_spec.rb +++ b/spec/lib/gitlab/import_export/attributes_finder_spec.rb @@ -2,7 +2,7 @@ require 'fast_spec_helper' -RSpec.describe Gitlab::ImportExport::AttributesFinder do +RSpec.describe Gitlab::ImportExport::AttributesFinder, feature_category: :importers do describe '#find_root' do subject { described_class.new(config: config).find_root(model_key) } @@ -177,7 +177,8 @@ RSpec.describe Gitlab::ImportExport::AttributesFinder do end def setup_yaml(hash) - allow(YAML).to receive(:load_file).with(test_config).and_return(hash) + allow(YAML).to receive(:safe_load_file) + .with(test_config, aliases: true, permitted_classes: [Symbol]).and_return(hash) end end end @@ -207,6 +208,19 @@ RSpec.describe Gitlab::ImportExport::AttributesFinder do it { is_expected.to be_nil } end + + context 'when include_import_only_tree is true' do + subject { described_class.new(config: config).find_relations_tree(model_key, include_import_only_tree: true) } + + let(:config) do + { + tree: { project: { ci_pipelines: { stages: { builds: nil } } } }, + import_only_tree: { project: { ci_pipelines: { stages: { statuses: nil } } } } + } + end + + it { is_expected.to eq({ ci_pipelines: { stages: { builds: nil, statuses: nil } } }) } + end end describe '#find_excluded_keys' do diff --git a/spec/lib/gitlab/import_export/attributes_permitter_spec.rb b/spec/lib/gitlab/import_export/attributes_permitter_spec.rb index c748f966463..8089b40cae8 100644 --- a/spec/lib/gitlab/import_export/attributes_permitter_spec.rb +++ b/spec/lib/gitlab/import_export/attributes_permitter_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::AttributesPermitter do +RSpec.describe Gitlab::ImportExport::AttributesPermitter, feature_category: :importers do let(:yml_config) do <<-EOF tree: @@ -12,6 +12,15 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do - milestones: - events: - :push_event_payload + - ci_pipelines: + - stages: + - :builds + + import_only_tree: + project: + - ci_pipelines: + - stages: + - :statuses included_attributes: labels: @@ -43,12 +52,16 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do it 'builds permitted attributes hash' do expect(subject.permitted_attributes).to match( a_hash_including( - project: [:labels, :milestones], + project: [:labels, :milestones, :ci_pipelines], labels: [:priorities, :title, :description, :type], events: [:push_event_payload], milestones: [:events], priorities: [], - push_event_payload: [] + push_event_payload: [], + ci_pipelines: [:stages], + stages: [:builds, :statuses], + statuses: [], + builds: [] ) ) end @@ -129,6 +142,9 @@ RSpec.describe Gitlab::ImportExport::AttributesPermitter do :external_pull_request | true :external_pull_requests | true :statuses | true + :builds | true + :generic_commit_statuses | true + :bridges | true :ci_pipelines | true :stages | true :actions | true diff --git a/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb b/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb index a8b4b9a6f05..e42a1d0ff8b 100644 --- a/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb +++ b/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb @@ -82,24 +82,13 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver, feature_category it 'saves valid subrelations and logs invalid subrelation' do expect(relation_object.notes).to receive(:<<).twice.and_call_original expect(relation_object).to receive(:save).and_call_original - expect(Gitlab::Import::Logger) - .to receive(:info) - .with( - message: '[Project/Group Import] Invalid subrelation', - project_id: project.id, - relation_key: 'issues', - error_messages: "Project does not match noteable project" - ) saver.execute issue = project.issues.last - import_failure = project.import_failures.last expect(invalid_note.persisted?).to eq(false) expect(issue.notes.count).to eq(5) - expect(import_failure.source).to eq('RelationObjectSaver#save!') - expect(import_failure.exception_message).to eq('Project does not match noteable project') end context 'when invalid subrelation can still be persisted' do @@ -111,7 +100,6 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver, feature_category it 'saves the subrelation' do expect(approval_1.valid?).to eq(false) - expect(Gitlab::Import::Logger).not_to receive(:info) saver.execute @@ -128,24 +116,10 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver, feature_category let(:invalid_priority) { build(:label_priority, priority: -1) } let(:relation_object) { build(:group_label, group: importable, title: 'test', priorities: valid_priorities + [invalid_priority]) } - it 'logs invalid subrelation for a group' do - expect(Gitlab::Import::Logger) - .to receive(:info) - .with( - message: '[Project/Group Import] Invalid subrelation', - group_id: importable.id, - relation_key: 'labels', - error_messages: 'Priority must be greater than or equal to 0' - ) - + it 'saves relation without invalid subrelations' do saver.execute - label = importable.labels.last - import_failure = importable.import_failures.last - - expect(label.priorities.count).to eq(5) - expect(import_failure.source).to eq('RelationObjectSaver#save!') - expect(import_failure.exception_message).to eq('Priority must be greater than or equal to 0') + expect(importable.labels.last.priorities.count).to eq(5) end end end diff --git a/spec/lib/gitlab/import_export/command_line_util_spec.rb b/spec/lib/gitlab/import_export/command_line_util_spec.rb index f47f1ab58a8..91cfab1688a 100644 --- a/spec/lib/gitlab/import_export/command_line_util_spec.rb +++ b/spec/lib/gitlab/import_export/command_line_util_spec.rb @@ -2,13 +2,14 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::CommandLineUtil do +RSpec.describe Gitlab::ImportExport::CommandLineUtil, feature_category: :importers do include ExportFileHelper let(:path) { "#{Dir.tmpdir}/symlink_test" } let(:archive) { 'spec/fixtures/symlink_export.tar.gz' } let(:shared) { Gitlab::ImportExport::Shared.new(nil) } let(:tmpdir) { Dir.mktmpdir } + let(:archive_dir) { Dir.mktmpdir } subject do Class.new do @@ -25,20 +26,38 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do before do FileUtils.mkdir_p(path) - subject.untar_zxf(archive: archive, dir: path) end after do FileUtils.rm_rf(path) + FileUtils.rm_rf(archive_dir) FileUtils.remove_entry(tmpdir) end - it 'has the right mask for project.json' do - expect(file_permissions("#{path}/project.json")).to eq(0755) # originally 777 - end - - it 'has the right mask for uploads' do - expect(file_permissions("#{path}/uploads")).to eq(0755) # originally 555 + shared_examples 'deletes symlinks' do |compression, decompression| + it 'deletes the symlinks', :aggregate_failures do + Dir.mkdir("#{tmpdir}/.git") + Dir.mkdir("#{tmpdir}/folder") + FileUtils.touch("#{tmpdir}/file.txt") + FileUtils.touch("#{tmpdir}/folder/file.txt") + FileUtils.touch("#{tmpdir}/.gitignore") + FileUtils.touch("#{tmpdir}/.git/config") + File.symlink('file.txt', "#{tmpdir}/.symlink") + File.symlink('file.txt', "#{tmpdir}/.git/.symlink") + File.symlink('file.txt', "#{tmpdir}/folder/.symlink") + archive = File.join(archive_dir, 'archive') + subject.public_send(compression, archive: archive, dir: tmpdir) + + subject.public_send(decompression, archive: archive, dir: archive_dir) + + expect(File.exist?("#{archive_dir}/file.txt")).to eq(true) + expect(File.exist?("#{archive_dir}/folder/file.txt")).to eq(true) + expect(File.exist?("#{archive_dir}/.gitignore")).to eq(true) + expect(File.exist?("#{archive_dir}/.git/config")).to eq(true) + expect(File.exist?("#{archive_dir}/.symlink")).to eq(false) + expect(File.exist?("#{archive_dir}/.git/.symlink")).to eq(false) + expect(File.exist?("#{archive_dir}/folder/.symlink")).to eq(false) + end end describe '#download_or_copy_upload' do @@ -228,12 +247,6 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do end describe '#tar_cf' do - let(:archive_dir) { Dir.mktmpdir } - - after do - FileUtils.remove_entry(archive_dir) - end - it 'archives a folder without compression' do archive_file = File.join(archive_dir, 'archive.tar') @@ -256,12 +269,24 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do end end - describe '#untar_xf' do - let(:archive_dir) { Dir.mktmpdir } + describe '#untar_zxf' do + it_behaves_like 'deletes symlinks', :tar_czf, :untar_zxf - after do - FileUtils.remove_entry(archive_dir) + it 'has the right mask for project.json' do + subject.untar_zxf(archive: archive, dir: path) + + expect(file_permissions("#{path}/project.json")).to eq(0755) # originally 777 + end + + it 'has the right mask for uploads' do + subject.untar_zxf(archive: archive, dir: path) + + expect(file_permissions("#{path}/uploads")).to eq(0755) # originally 555 end + end + + describe '#untar_xf' do + it_behaves_like 'deletes symlinks', :tar_cf, :untar_xf it 'extracts archive without decompression' do filename = 'archive.tar.gz' diff --git a/spec/lib/gitlab/import_export/config_spec.rb b/spec/lib/gitlab/import_export/config_spec.rb index 8f848af8bd3..2a52a0a2ff2 100644 --- a/spec/lib/gitlab/import_export/config_spec.rb +++ b/spec/lib/gitlab/import_export/config_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Config do +RSpec.describe Gitlab::ImportExport::Config, feature_category: :importers do let(:yaml_file) { described_class.new } describe '#to_h' do @@ -21,7 +21,9 @@ RSpec.describe Gitlab::ImportExport::Config do end it 'parses default config' do - expected_keys = [:tree, :excluded_attributes, :included_attributes, :methods, :preloads, :export_reorders] + expected_keys = [ + :tree, :import_only_tree, :excluded_attributes, :included_attributes, :methods, :preloads, :export_reorders + ] expected_keys << :include_if_exportable if ee expect { subject }.not_to raise_error @@ -82,7 +84,7 @@ RSpec.describe Gitlab::ImportExport::Config do EOF end - let(:config_hash) { YAML.safe_load(config, [Symbol]) } + let(:config_hash) { YAML.safe_load(config, permitted_classes: [Symbol]) } before do allow_any_instance_of(described_class).to receive(:parse_yaml) do @@ -110,6 +112,7 @@ RSpec.describe Gitlab::ImportExport::Config do } } }, + import_only_tree: {}, included_attributes: { user: [:id] }, @@ -153,6 +156,7 @@ RSpec.describe Gitlab::ImportExport::Config do } } }, + import_only_tree: {}, included_attributes: { user: [:id, :name_ee] }, diff --git a/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb b/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb index f18d9e64f52..02419267f0e 100644 --- a/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license do +RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license, feature_category: :importers do # FastHashSerializer#execute generates the hash which is not easily accessible # and includes `JSONBatchRelation` items which are serialized at this point. # Wrapping the result into JSON generating/parsing is for making @@ -125,13 +125,13 @@ RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license do expect(subject.dig('ci_pipelines', 0, 'stages')).not_to be_empty end - it 'has pipeline statuses' do - expect(subject.dig('ci_pipelines', 0, 'stages', 0, 'statuses')).not_to be_empty + it 'has pipeline builds' do + expect(subject.dig('ci_pipelines', 0, 'stages', 0, 'builds')).not_to be_empty end it 'has pipeline builds' do builds_count = subject - .dig('ci_pipelines', 0, 'stages', 0, 'statuses') + .dig('ci_pipelines', 0, 'stages', 0, 'builds') .count { |hash| hash['type'] == 'Ci::Build' } expect(builds_count).to eq(1) @@ -141,8 +141,8 @@ RSpec.describe Gitlab::ImportExport::FastHashSerializer, :with_license do expect(subject['ci_pipelines']).not_to be_empty end - it 'has ci pipeline notes' do - expect(subject['ci_pipelines'].first['notes']).not_to be_empty + it 'has commit notes' do + expect(subject['commit_notes']).not_to be_empty end it 'has labels with no associations' do diff --git a/spec/lib/gitlab/import_export/fork_spec.rb b/spec/lib/gitlab/import_export/fork_spec.rb deleted file mode 100644 index 9d766eb3af1..00000000000 --- a/spec/lib/gitlab/import_export/fork_spec.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe 'forked project import' do - include ProjectForksHelper - - let(:user) { create(:user) } - let!(:project_with_repo) { create(:project, :repository, name: 'test-repo-restorer', path: 'test-repo-restorer') } - let!(:project) { create(:project, name: 'test-repo-restorer-no-repo', path: 'test-repo-restorer-no-repo') } - let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } - let(:shared) { project.import_export_shared } - let(:forked_from_project) { create(:project, :repository) } - let(:forked_project) { fork_project(project_with_repo, nil, repository: true) } - let(:repo_saver) { Gitlab::ImportExport::RepoSaver.new(exportable: project_with_repo, shared: shared) } - let(:bundle_path) { File.join(shared.export_path, Gitlab::ImportExport.project_bundle_filename) } - - let(:repo_restorer) do - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: bundle_path, shared: shared, importable: project) - end - - let!(:merge_request) do - create(:merge_request, source_project: forked_project, target_project: project_with_repo) - end - - let(:saver) do - Gitlab::ImportExport::Project::TreeSaver.new(project: project_with_repo, current_user: user, shared: shared) - end - - let(:restorer) do - Gitlab::ImportExport::Project::TreeRestorer.new(user: user, shared: shared, project: project) - end - - before do - stub_feature_flags(project_export_as_ndjson: false) - - allow_next_instance_of(Gitlab::ImportExport) do |instance| - allow(instance).to receive(:storage_path).and_return(export_path) - end - - saver.save # rubocop:disable Rails/SaveBang - repo_saver.save # rubocop:disable Rails/SaveBang - - repo_restorer.restore - restorer.restore - end - - after do - FileUtils.rm_rf(export_path) - project_with_repo.repository.remove - project.repository.remove - end - - it 'can access the MR', :sidekiq_might_not_need_inline do - project.merge_requests.first.fetch_ref! - - expect(project.repository.ref_exists?('refs/merge-requests/1/head')).to be_truthy - end -end diff --git a/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb index 5e84284a060..495cefa002a 100644 --- a/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/relation_tree_restorer_spec.rb @@ -9,25 +9,31 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer do +RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer, feature_category: :importers do let(:group) { create(:group).tap { |g| g.add_owner(user) } } let(:importable) { create(:group, parent: group) } include_context 'relation tree restorer shared context' do - let(:importable_name) { nil } + let(:importable_name) { 'groups/4353' } end - let(:path) { 'spec/fixtures/lib/gitlab/import_export/group_exports/no_children/group.json' } + let(:path) { Rails.root.join('spec/fixtures/lib/gitlab/import_export/group_exports/no_children/tree') } let(:relation_reader) do - Gitlab::ImportExport::Json::LegacyReader::File.new( - path, - relation_names: reader.group_relation_names) + Gitlab::ImportExport::Json::NdjsonReader.new(path) end let(:reader) do Gitlab::ImportExport::Reader.new( shared: shared, - config: Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.legacy_group_config_file).to_h + config: Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.group_config_file).to_h + ) + end + + let(:members_mapper) do + Gitlab::ImportExport::MembersMapper.new( + exported_members: relation_reader.consume_relation(importable_name, 'members').map(&:first), + user: user, + importable: importable ) end @@ -41,7 +47,7 @@ RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer do relation_factory: Gitlab::ImportExport::Group::RelationFactory, reader: reader, importable: importable, - importable_path: nil, + importable_path: importable_name, importable_attributes: attributes ) end @@ -60,4 +66,74 @@ RSpec.describe Gitlab::ImportExport::Group::RelationTreeRestorer do subject end + + describe 'relation object saving' do + before do + allow(shared.logger).to receive(:info).and_call_original + allow(relation_reader).to receive(:consume_relation).and_call_original + + allow(relation_reader) + .to receive(:consume_relation) + .with(importable_name, 'labels') + .and_return([[label, 0]]) + end + + context 'when relation object is new' do + context 'when relation object has invalid subrelations' do + let(:label) do + { + 'title' => 'test', + 'priorities' => [LabelPriority.new, LabelPriority.new], + 'type' => 'GroupLabel' + } + end + + it 'logs invalid subrelations' do + expect(shared.logger) + .to receive(:info) + .with( + message: '[Project/Group Import] Invalid subrelation', + group_id: importable.id, + relation_key: 'labels', + error_messages: "Project can't be blank, Priority can't be blank, and Priority is not a number" + ) + + subject + + label = importable.labels.first + failure = importable.import_failures.first + + expect(importable.import_failures.count).to eq(2) + expect(label.title).to eq('test') + expect(failure.exception_class).to eq('ActiveRecord::RecordInvalid') + expect(failure.source).to eq('RelationTreeRestorer#save_relation_object') + expect(failure.exception_message) + .to eq("Project can't be blank, Priority can't be blank, and Priority is not a number") + end + end + end + + context 'when relation object is persisted' do + context 'when relation object is invalid' do + let(:label) { create(:group_label, group: group, title: 'test') } + + it 'saves import failure with nested errors' do + label.priorities << [LabelPriority.new, LabelPriority.new] + + subject + + failure = importable.import_failures.first + + expect(importable.labels.count).to eq(0) + expect(importable.import_failures.count).to eq(1) + expect(failure.exception_class).to eq('ActiveRecord::RecordInvalid') + expect(failure.source).to eq('process_relation_item!') + expect(failure.exception_message) + .to eq("Validation failed: Priorities is invalid, Project can't be blank, Priority can't be blank, " \ + "Priority is not a number, Project can't be blank, Priority can't be blank, " \ + "Priority is not a number") + end + end + end + end end diff --git a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb index aa30e24296e..a6afd0a36ec 100644 --- a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Group::TreeRestorer, feature: :subgroups do +RSpec.describe Gitlab::ImportExport::Group::TreeRestorer, feature: :subgroups, feature_category: :importers do include ImportExport::CommonUtil shared_examples 'group restoration' do @@ -171,7 +171,7 @@ RSpec.describe Gitlab::ImportExport::Group::TreeRestorer, feature: :subgroups do allow(shared).to receive(:export_path).and_return(tmpdir) expect(group_tree_restorer.restore).to eq(false) - expect(shared.errors).to include('Incorrect JSON format') + expect(shared.errors).to include('Invalid file') end end end diff --git a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb deleted file mode 100644 index 6c997dc1361..00000000000 --- a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -# Verifies that given an exported project meta-data tree, when importing this -# tree and then exporting it again, we should obtain the initial tree. -# -# This equivalence only works up to a certain extent, for instance we need -# to ignore: -# -# - row IDs and foreign key IDs -# - some timestamps -# - randomly generated fields like tokens -# -# as these are expected to change between import/export cycles. -RSpec.describe Gitlab::ImportExport, feature_category: :importers do - include ImportExport::CommonUtil - include ConfigurationHelper - include ImportExport::ProjectTreeExpectations - - let(:json_fixture) { 'complex' } - - before do - stub_feature_flags(project_export_as_ndjson: false) - end - - it 'yields the initial tree when importing and exporting it again' do - project = create(:project) - user = create(:user, :admin) - - # We first generate a test fixture dynamically from a seed-fixture, so as to - # account for any fields in the initial fixture that are missing and set to - # defaults during import (ideally we should have realistic test fixtures - # that "honestly" represent exports) - expect( - restore_then_save_project( - project, - user, - import_path: seed_fixture_path, - export_path: test_fixture_path) - ).to be true - # Import, then export again from the generated fixture. Any residual changes - # in the JSON will count towards comparison i.e. test failures. - expect( - restore_then_save_project( - project, - user, - import_path: test_fixture_path, - export_path: test_tmp_path) - ).to be true - - imported_json = Gitlab::Json.parse(File.read("#{test_fixture_path}/project.json")) - exported_json = Gitlab::Json.parse(File.read("#{test_tmp_path}/project.json")) - - assert_relations_match(imported_json, exported_json) - end - - private - - def seed_fixture_path - "#{fixtures_path}/#{json_fixture}" - end - - def test_fixture_path - "#{test_tmp_path}/#{json_fixture}" - end -end diff --git a/spec/lib/gitlab/import_export/import_failure_service_spec.rb b/spec/lib/gitlab/import_export/import_failure_service_spec.rb index 51f1fc9c6a2..30d16347828 100644 --- a/spec/lib/gitlab/import_export/import_failure_service_spec.rb +++ b/spec/lib/gitlab/import_export/import_failure_service_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::ImportFailureService do +RSpec.describe Gitlab::ImportExport::ImportFailureService, feature_category: :importers do let(:importable) { create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project') } let(:label) { create(:label) } let(:subject) { described_class.new(importable) } diff --git a/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb b/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb deleted file mode 100644 index 793b3ebfb9e..00000000000 --- a/spec/lib/gitlab/import_export/json/legacy_reader/file_spec.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_relative 'shared_example' - -RSpec.describe Gitlab::ImportExport::Json::LegacyReader::File do - it_behaves_like 'import/export json legacy reader' do - let(:valid_path) { 'spec/fixtures/lib/gitlab/import_export/light/project.json' } - let(:data) { valid_path } - let(:json_data) { Gitlab::Json.parse(File.read(valid_path)) } - end - - describe '#exist?' do - let(:legacy_reader) do - described_class.new(path, relation_names: []) - end - - subject { legacy_reader.exist? } - - context 'given valid path' do - let(:path) { 'spec/fixtures/lib/gitlab/import_export/light/project.json' } - - it { is_expected.to be true } - end - - context 'given invalid path' do - let(:path) { 'spec/non-existing-folder/do-not-create-this-file.json' } - - it { is_expected.to be false } - end - end -end diff --git a/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb b/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb deleted file mode 100644 index 57d66dc0f50..00000000000 --- a/spec/lib/gitlab/import_export/json/legacy_reader/hash_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_relative 'shared_example' - -RSpec.describe Gitlab::ImportExport::Json::LegacyReader::Hash do - it_behaves_like 'import/export json legacy reader' do - let(:path) { 'spec/fixtures/lib/gitlab/import_export/light/project.json' } - - # the hash is modified by the `LegacyReader` - # we need to deep-dup it - let(:json_data) { Gitlab::Json.parse(File.read(path)) } - let(:data) { Gitlab::Json.parse(File.read(path)) } - end - - describe '#exist?' do - let(:legacy_reader) do - described_class.new(tree_hash, relation_names: []) - end - - subject { legacy_reader.exist? } - - context 'tree_hash is nil' do - let(:tree_hash) { nil } - - it { is_expected.to be_falsey } - end - - context 'tree_hash presents' do - let(:tree_hash) { { "issues": [] } } - - it { is_expected.to be_truthy } - end - end -end diff --git a/spec/lib/gitlab/import_export/json/legacy_reader/shared_example.rb b/spec/lib/gitlab/import_export/json/legacy_reader/shared_example.rb deleted file mode 100644 index 3e9bd3fe741..00000000000 --- a/spec/lib/gitlab/import_export/json/legacy_reader/shared_example.rb +++ /dev/null @@ -1,102 +0,0 @@ -# frozen_string_literal: true - -RSpec.shared_examples 'import/export json legacy reader' do - let(:relation_names) { [] } - - let(:legacy_reader) do - described_class.new( - data, - relation_names: relation_names, - allowed_path: "project") - end - - describe '#consume_attributes' do - context 'when valid path is passed' do - subject { legacy_reader.consume_attributes("project") } - - context 'no excluded attributes' do - let(:relation_names) { [] } - - it 'returns the whole tree from parsed JSON' do - expect(subject).to eq(json_data) - end - end - - context 'some attributes are excluded' do - let(:relation_names) { %w[milestones labels] } - - it 'returns hash without excluded attributes and relations' do - expect(subject).not_to include('milestones', 'labels') - end - end - end - - context 'when invalid path is passed' do - it 'raises an exception' do - expect { legacy_reader.consume_attributes("invalid-path") } - .to raise_error(ArgumentError) - end - end - end - - describe '#consume_relation' do - context 'when valid path is passed' do - let(:key) { 'labels' } - - subject { legacy_reader.consume_relation("project", key) } - - context 'key has not been consumed' do - it 'returns an Enumerator' do - expect(subject).to be_an_instance_of(Enumerator) - end - - context 'value is nil' do - before do - expect(legacy_reader).to receive(:relations).and_return({ key => nil }) - end - - it 'yields nothing to the Enumerator' do - expect(subject.to_a).to eq([]) - end - end - - context 'value is an array' do - before do - expect(legacy_reader).to receive(:relations).and_return({ key => %w[label1 label2] }) - end - - it 'yields every relation value to the Enumerator' do - expect(subject.to_a).to eq([['label1', 0], ['label2', 1]]) - end - end - - context 'value is not array' do - before do - expect(legacy_reader).to receive(:relations).and_return({ key => 'non-array value' }) - end - - it 'yields the value with index 0 to the Enumerator' do - expect(subject.to_a).to eq([['non-array value', 0]]) - end - end - end - - context 'key has been consumed' do - before do - legacy_reader.consume_relation("project", key).first - end - - it 'yields nothing to the Enumerator' do - expect(subject.to_a).to eq([]) - end - end - end - - context 'when invalid path is passed' do - it 'raises an exception' do - expect { legacy_reader.consume_relation("invalid") } - .to raise_error(ArgumentError) - end - end - end -end diff --git a/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb b/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb deleted file mode 100644 index e8ecd98b1e1..00000000000 --- a/spec/lib/gitlab/import_export/json/legacy_writer_spec.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true - -require 'fast_spec_helper' - -RSpec.describe Gitlab::ImportExport::Json::LegacyWriter do - let(:path) { "#{Dir.tmpdir}/legacy_writer_spec/test.json" } - - subject do - described_class.new(path, allowed_path: "project") - end - - after do - FileUtils.rm_rf(path) - end - - describe "#write_attributes" do - it "writes correct json" do - expected_hash = { "key" => "value_1", "key_1" => "value_2" } - subject.write_attributes("project", expected_hash) - - expect(subject_json).to eq(expected_hash) - end - - context 'when invalid path is used' do - it 'raises an exception' do - expect { subject.write_attributes("invalid", { "key" => "value" }) } - .to raise_error(ArgumentError) - end - end - end - - describe "#write_relation" do - context "when key is already written" do - it "raises exception" do - subject.write_relation("project", "key", "old value") - - expect { subject.write_relation("project", "key", "new value") } - .to raise_exception("key 'key' already written") - end - end - - context "when key is not already written" do - context "when multiple key value pairs are stored" do - it "writes correct json" do - expected_hash = { "key" => "value_1", "key_1" => "value_2" } - expected_hash.each do |key, value| - subject.write_relation("project", key, value) - end - - expect(subject_json).to eq(expected_hash) - end - end - end - - context 'when invalid path is used' do - it 'raises an exception' do - expect { subject.write_relation("invalid", "key", "value") } - .to raise_error(ArgumentError) - end - end - end - - describe "#write_relation_array" do - context 'when array is used' do - it 'writes correct json' do - subject.write_relation_array("project", "key", ["value"]) - - expect(subject_json).to eq({ "key" => ["value"] }) - end - end - - context 'when enumerable is used' do - it 'writes correct json' do - values = %w(value1 value2) - - enumerator = Enumerator.new do |items| - values.each { |value| items << value } - end - - subject.write_relation_array("project", "key", enumerator) - - expect(subject_json).to eq({ "key" => values }) - end - end - - context "when key is already written" do - it "raises an exception" do - subject.write_relation_array("project", "key", %w(old_value)) - - expect { subject.write_relation_array("project", "key", %w(new_value)) } - .to raise_error(ArgumentError) - end - end - end - - def subject_json - subject.close - - ::JSON.parse(File.read(subject.path)) - end -end diff --git a/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb b/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb index 0ca4c4ccc87..98afe01c08b 100644 --- a/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb +++ b/spec/lib/gitlab/import_export/json/ndjson_reader_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Json::NdjsonReader do +RSpec.describe Gitlab::ImportExport::Json::NdjsonReader, feature_category: :importers do include ImportExport::CommonUtil let(:fixture) { 'spec/fixtures/lib/gitlab/import_export/light/tree' } @@ -26,14 +26,6 @@ RSpec.describe Gitlab::ImportExport::Json::NdjsonReader do end end - describe '#legacy?' do - let(:dir_path) { fixture } - - subject { ndjson_reader.legacy? } - - it { is_expected.to be false } - end - describe '#consume_attributes' do let(:dir_path) { fixture } @@ -42,6 +34,20 @@ RSpec.describe Gitlab::ImportExport::Json::NdjsonReader do it 'returns the whole root tree from parsed JSON' do expect(subject).to eq(root_tree) end + + context 'when project.json is symlink' do + it 'raises error an error' do + Dir.mktmpdir do |tmpdir| + FileUtils.touch(File.join(tmpdir, 'passwd')) + File.symlink(File.join(tmpdir, 'passwd'), File.join(tmpdir, 'project.json')) + + ndjson_reader = described_class.new(tmpdir) + + expect { ndjson_reader.consume_attributes(importable_path) } + .to raise_error(Gitlab::ImportExport::Error, 'Invalid file') + end + end + end end describe '#consume_relation' do @@ -91,6 +97,22 @@ RSpec.describe Gitlab::ImportExport::Json::NdjsonReader do end end + context 'when relation file is a symlink' do + it 'yields nothing to the Enumerator' do + Dir.mktmpdir do |tmpdir| + Dir.mkdir(File.join(tmpdir, 'project')) + File.write(File.join(tmpdir, 'passwd'), "{}\n{}") + File.symlink(File.join(tmpdir, 'passwd'), File.join(tmpdir, 'project', 'issues.ndjson')) + + ndjson_reader = described_class.new(tmpdir) + + result = ndjson_reader.consume_relation(importable_path, 'issues') + + expect(result.to_a).to eq([]) + end + end + end + context 'relation file is empty' do let(:key) { 'empty' } diff --git a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb index 103d3512e8b..f4c9189030b 100644 --- a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb @@ -28,7 +28,7 @@ RSpec.describe Gitlab::ImportExport::Json::StreamingSerializer, feature_category let(:exportable_path) { 'project' } let(:logger) { Gitlab::Export::Logger.build } - let(:json_writer) { instance_double('Gitlab::ImportExport::Json::LegacyWriter') } + let(:json_writer) { instance_double('Gitlab::ImportExport::Json::NdjsonWriter') } let(:hash) { { name: exportable.name, description: exportable.description }.stringify_keys } let(:include) { [] } let(:custom_orderer) { nil } diff --git a/spec/lib/gitlab/import_export/model_configuration_spec.rb b/spec/lib/gitlab/import_export/model_configuration_spec.rb index 4f01f470ce7..8e5fe96f3b4 100644 --- a/spec/lib/gitlab/import_export/model_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/model_configuration_spec.rb @@ -5,11 +5,11 @@ require 'spec_helper' # Part of the test security suite for the Import/Export feature # Finds if a new model has been added that can potentially be part of the Import/Export # If it finds a new model, it will show a +failure_message+ with the options available. -RSpec.describe 'Import/Export model configuration' do +RSpec.describe 'Import/Export model configuration', feature_category: :importers do include ConfigurationHelper let(:all_models_yml) { 'spec/lib/gitlab/import_export/all_models.yml' } - let(:all_models_hash) { YAML.load_file(all_models_yml) } + let(:all_models_hash) { YAML.safe_load_file(all_models_yml, aliases: true) } let(:current_models) { setup_models } let(:model_names) { relation_names_for(:project) } diff --git a/spec/lib/gitlab/import_export/project/export_task_spec.rb b/spec/lib/gitlab/import_export/project/export_task_spec.rb index 3dd1e9257cc..95971d08175 100644 --- a/spec/lib/gitlab/import_export/project/export_task_spec.rb +++ b/spec/lib/gitlab/import_export/project/export_task_spec.rb @@ -10,14 +10,14 @@ RSpec.describe Gitlab::ImportExport::Project::ExportTask, :silence_stdout do let(:measurement_enabled) { false } let(:file_path) { 'spec/fixtures/gitlab/import_export/test_project_export.tar.gz' } let(:project) { create(:project, creator: user, namespace: user.namespace) } - let(:project_name) { project.name } + let(:project_path) { project.path } let(:rake_task) { described_class.new(task_params) } let(:task_params) do { username: username, namespace_path: namespace_path, - project_path: project_name, + project_path: project_path, file_path: file_path, measurement_enabled: measurement_enabled } @@ -48,10 +48,10 @@ RSpec.describe Gitlab::ImportExport::Project::ExportTask, :silence_stdout do end context 'when project is not found' do - let(:project_name) { 'invalid project name' } + let(:project_path) { 'invalid project path' } it 'logs an error' do - expect { subject }.to output(/Project with path: #{project_name} was not found. Please provide correct project path/).to_stdout + expect { subject }.to output(/Project with path: #{project_path} was not found. Please provide correct project path/).to_stdout end it 'returns false' do diff --git a/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb b/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb index d70e89c6856..f8018e75879 100644 --- a/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb +++ b/spec/lib/gitlab/import_export/project/exported_relations_merger_spec.rb @@ -64,8 +64,8 @@ RSpec.describe Gitlab::ImportExport::Project::ExportedRelationsMerger do expect(result).to eq(false) expect(shared.errors).to match_array( [ - "undefined method `export_file' for nil:NilClass", - "undefined method `export_file' for nil:NilClass" + /^undefined method `export_file' for nil:NilClass/, + /^undefined method `export_file' for nil:NilClass/ ] ) end diff --git a/spec/lib/gitlab/import_export/project/import_task_spec.rb b/spec/lib/gitlab/import_export/project/import_task_spec.rb index c847224cb9b..693f1984ce8 100644 --- a/spec/lib/gitlab/import_export/project/import_task_spec.rb +++ b/spec/lib/gitlab/import_export/project/import_task_spec.rb @@ -2,7 +2,7 @@ require 'rake_helper' -RSpec.describe Gitlab::ImportExport::Project::ImportTask, :request_store, :silence_stdout do +RSpec.describe Gitlab::ImportExport::Project::ImportTask, :request_store, :silence_stdout, feature_category: :importers do let(:username) { 'root' } let(:namespace_path) { username } let!(:user) { create(:user, username: username) } diff --git a/spec/lib/gitlab/import_export/project/object_builder_spec.rb b/spec/lib/gitlab/import_export/project/object_builder_spec.rb index 189b798c2e8..5fa8590e8fd 100644 --- a/spec/lib/gitlab/import_export/project/object_builder_spec.rb +++ b/spec/lib/gitlab/import_export/project/object_builder_spec.rb @@ -86,13 +86,16 @@ RSpec.describe Gitlab::ImportExport::Project::ObjectBuilder do 'group' => group)).to eq(group_label) end - it 'creates a new label' do + it 'creates a new project label' do label = described_class.build(Label, 'title' => 'group label', 'project' => project, - 'group' => project.group) + 'group' => project.group, + 'group_id' => project.group.id) expect(label.persisted?).to be true + expect(label).to be_an_instance_of(ProjectLabel) + expect(label.group_id).to be_nil end end diff --git a/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb index 6053df8ba97..180a6b6ff0a 100644 --- a/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/relation_tree_restorer_spec.rb @@ -50,58 +50,24 @@ RSpec.describe Gitlab::ImportExport::Project::RelationTreeRestorer, feature_cate expect(project.custom_attributes.count).to eq(2) expect(project.project_badges.count).to eq(2) expect(project.snippets.count).to eq(1) + expect(project.commit_notes.count).to eq(3) end end end - context 'with legacy reader' do - let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/project.json' } - let(:relation_reader) do - Gitlab::ImportExport::Json::LegacyReader::File.new( - path, - relation_names: reader.project_relation_names, - allowed_path: 'project' - ) - end - - let(:attributes) { relation_reader.consume_attributes('project') } - - it_behaves_like 'import project successfully' - - context 'with logging of relations creation' do - let_it_be(:group) { create(:group).tap { |g| g.add_maintainer(user) } } - let_it_be(:importable) do - create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project', group: group) - end - - it 'logs top-level relation creation' do - expect(shared.logger) - .to receive(:info) - .with(hash_including(message: '[Project/Group Import] Created new object relation')) - .at_least(:once) - - subject - end - end - end - - context 'with ndjson reader' do + context 'when inside a group' do let(:path) { 'spec/fixtures/lib/gitlab/import_export/complex/tree' } let(:relation_reader) { Gitlab::ImportExport::Json::NdjsonReader.new(path) } - it_behaves_like 'import project successfully' - - context 'when inside a group' do - let_it_be(:group) do - create(:group, :disabled_and_unoverridable).tap { |g| g.add_maintainer(user) } - end - - before do - importable.update!(shared_runners_enabled: false, group: group) - end + let_it_be(:group) do + create(:group, :disabled_and_unoverridable).tap { |g| g.add_maintainer(user) } + end - it_behaves_like 'import project successfully' + before do + importable.update!(shared_runners_enabled: false, group: group) end + + it_behaves_like 'import project successfully' end context 'with invalid relations' do diff --git a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb index 125d1736b9b..5aa16f9508d 100644 --- a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb @@ -12,7 +12,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i let(:shared) { project.import_export_shared } - RSpec.shared_examples 'project tree restorer work properly' do |reader, ndjson_enabled| + RSpec.shared_examples 'project tree restorer work properly' do describe 'restore project tree' do before_all do # Using an admin for import, so we can check assignment of existing members @@ -27,10 +27,9 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i @shared = @project.import_export_shared stub_all_feature_flags - stub_feature_flags(project_import_ndjson: ndjson_enabled) setup_import_export_config('complex') - setup_reader(reader) + setup_reader allow_any_instance_of(Repository).to receive(:fetch_source_branch!).and_return(true) allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false) @@ -295,6 +294,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i it 'has project labels' do expect(ProjectLabel.count).to eq(3) + expect(ProjectLabel.pluck(:group_id).compact).to be_empty end it 'has merge request approvals' do @@ -528,7 +528,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i it 'has the correct number of pipelines and statuses' do expect(@project.ci_pipelines.size).to eq(7) - @project.ci_pipelines.order(:id).zip([2, 0, 2, 2, 2, 2, 0]) + @project.ci_pipelines.order(:id).zip([2, 0, 2, 3, 2, 2, 0]) .each do |(pipeline, expected_status_size)| expect(pipeline.statuses.size).to eq(expected_status_size) end @@ -548,8 +548,16 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i expect(Ci::Stage.all).to all(have_attributes(pipeline_id: a_value > 0)) end - it 'restores statuses' do - expect(CommitStatus.all.count).to be 10 + it 'restores builds' do + expect(Ci::Build.all.count).to be 7 + end + + it 'restores bridges' do + expect(Ci::Bridge.all.count).to be 1 + end + + it 'restores generic commit statuses' do + expect(GenericCommitStatus.all.count).to be 1 end it 'correctly restores association between a stage and a job' do @@ -574,6 +582,10 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i expect(@project.import_failures.size).to eq 0 end end + + it 'restores commit notes' do + expect(@project.commit_notes.count).to eq(3) + end end end @@ -593,23 +605,15 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i end end - context 'project.json file access check' do + context 'when expect tree structure is not present in the export path' do let(:user) { create(:user) } - let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } - let(:project_tree_restorer) do - described_class.new(user: user, shared: shared, project: project) - end - - let(:restored_project_json) { project_tree_restorer.restore } + let_it_be(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') } - it 'does not read a symlink' do - Dir.mktmpdir do |tmpdir| - setup_symlink(tmpdir, 'project.json') - allow(shared).to receive(:export_path).and_call_original + it 'fails to restore the project' do + result = described_class.new(user: user, shared: shared, project: project).restore - expect(project_tree_restorer.restore).to eq(false) - expect(shared.errors).to include('invalid import format') - end + expect(result).to eq(false) + expect(shared.errors).to include('invalid import format') end end @@ -622,7 +626,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i context 'with a simple project' do before do setup_import_export_config('light') - setup_reader(reader) + setup_reader expect(restored_project_json).to eq(true) end @@ -657,7 +661,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i context 'multiple pipelines reference the same external pull request' do before do setup_import_export_config('multi_pipeline_ref_one_external_pr') - setup_reader(reader) + setup_reader expect(restored_project_json).to eq(true) end @@ -685,7 +689,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('light') - setup_reader(reader) + setup_reader expect(project).to receive(:merge_requests).and_call_original expect(project).to receive(:merge_requests).and_raise(exception) @@ -702,7 +706,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('light') - setup_reader(reader) + setup_reader expect(project).to receive(:merge_requests).and_call_original expect(project).to receive(:merge_requests).and_raise(exception) @@ -734,7 +738,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i context 'when the project has overridden params in import data' do before do setup_import_export_config('light') - setup_reader(reader) + setup_reader end it 'handles string versions of visibility_level' do @@ -800,7 +804,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('group') - setup_reader(reader) + setup_reader expect(restored_project_json).to eq(true) end @@ -836,7 +840,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('light') - setup_reader(reader) + setup_reader end it 'imports labels' do @@ -872,7 +876,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('milestone-iid') - setup_reader(reader) + setup_reader end it 'preserves the project milestone IID' do @@ -888,7 +892,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i context 'with external authorization classification labels' do before do setup_import_export_config('light') - setup_reader(reader) + setup_reader end it 'converts empty external classification authorization labels to nil' do @@ -915,76 +919,80 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i described_class.new(user: user, shared: shared, project: project) end - before do - allow_any_instance_of(Gitlab::ImportExport::Json::LegacyReader::File).to receive(:exist?).and_return(true) - allow_any_instance_of(Gitlab::ImportExport::Json::NdjsonReader).to receive(:exist?).and_return(false) - allow_any_instance_of(Gitlab::ImportExport::Json::LegacyReader::File).to receive(:tree_hash) { tree_hash } - end - - context 'no group visibility' do - let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } + describe 'visibility level' do + before do + setup_import_export_config('light') - it 'uses the project visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(visibility) + allow_next_instance_of(Gitlab::ImportExport::Json::NdjsonReader) do |relation_reader| + allow(relation_reader).to receive(:consume_attributes).and_return(tree_hash) + end end - end - context 'with restricted internal visibility' do - describe 'internal project' do - let(:visibility) { Gitlab::VisibilityLevel::INTERNAL } - - it 'uses private visibility' do - stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + context 'no group visibility' do + let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } + it 'uses the project visibility' do expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + expect(restorer.project.visibility_level).to eq(visibility) end end - end - context 'with group visibility' do - before do - group = create(:group, visibility_level: group_visibility) - group.add_members([user], GroupMember::MAINTAINER) - project.update!(group: group) - end + context 'with restricted internal visibility' do + describe 'internal project' do + let(:visibility) { Gitlab::VisibilityLevel::INTERNAL } - context 'private group visibility' do - let(:group_visibility) { Gitlab::VisibilityLevel::PRIVATE } - let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + it 'uses private visibility' do + stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) - it 'uses the group visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(group_visibility) + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end end end - context 'public group visibility' do - let(:group_visibility) { Gitlab::VisibilityLevel::PUBLIC } - let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } + context 'with group visibility' do + before do + group = create(:group, visibility_level: group_visibility) + group.add_members([user], GroupMember::MAINTAINER) + project.update!(group: group) + end - it 'uses the project visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(visibility) + context 'private group visibility' do + let(:group_visibility) { Gitlab::VisibilityLevel::PRIVATE } + let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + + it 'uses the group visibility' do + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(group_visibility) + end end - end - context 'internal group visibility' do - let(:group_visibility) { Gitlab::VisibilityLevel::INTERNAL } - let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + context 'public group visibility' do + let(:group_visibility) { Gitlab::VisibilityLevel::PUBLIC } + let(:visibility) { Gitlab::VisibilityLevel::PRIVATE } - it 'uses the group visibility' do - expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(group_visibility) + it 'uses the project visibility' do + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(visibility) + end end - context 'with restricted internal visibility' do - it 'sets private visibility' do - stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + context 'internal group visibility' do + let(:group_visibility) { Gitlab::VisibilityLevel::INTERNAL } + let(:visibility) { Gitlab::VisibilityLevel::PUBLIC } + it 'uses the group visibility' do expect(restorer.restore).to eq(true) - expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + expect(restorer.project.visibility_level).to eq(group_visibility) + end + + context 'with restricted internal visibility' do + it 'sets private visibility' do + stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + + expect(restorer.restore).to eq(true) + expect(restorer.project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end end end end @@ -995,24 +1003,35 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i let(:user2) { create(:user) } let(:project_members) do [ - { - "id" => 2, - "access_level" => 40, - "source_type" => "Project", - "notification_level" => 3, - "user" => { - "id" => user2.id, - "email" => user2.email, - "username" => 'test' - } - } + [ + { + "id" => 2, + "access_level" => 40, + "source_type" => "Project", + "notification_level" => 3, + "user" => { + "id" => user2.id, + "email" => user2.email, + "username" => 'test' + } + }, + 0 + ] ] end - let(:tree_hash) { { 'project_members' => project_members } } - before do project.add_maintainer(user) + + setup_import_export_config('light') + + allow_next_instance_of(Gitlab::ImportExport::Json::NdjsonReader) do |relation_reader| + allow(relation_reader).to receive(:consume_relation).and_call_original + + allow(relation_reader).to receive(:consume_relation) + .with('project', 'project_members') + .and_return(project_members) + end end it 'restores project members' do @@ -1032,7 +1051,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i before do setup_import_export_config('with_invalid_records') - setup_reader(reader) + setup_reader subject end @@ -1125,13 +1144,5 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer, feature_category: :i end end - context 'enable ndjson import' do - it_behaves_like 'project tree restorer work properly', :legacy_reader, true - - it_behaves_like 'project tree restorer work properly', :ndjson_reader, true - end - - context 'disable ndjson import' do - it_behaves_like 'project tree restorer work properly', :legacy_reader, false - end + it_behaves_like 'project tree restorer work properly' end diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index 74b6e039601..4166eba4e8e 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -2,35 +2,28 @@ require 'spec_helper' -RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do +RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license, feature_category: :importers do let_it_be(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } let_it_be(:exportable_path) { 'project' } let_it_be(:user) { create(:user) } let_it_be(:group) { create(:group) } let_it_be(:project) { setup_project } - shared_examples 'saves project tree successfully' do |ndjson_enabled| + shared_examples 'saves project tree successfully' do include ImportExport::CommonUtil - subject { get_json(full_path, exportable_path, relation_name, ndjson_enabled) } + subject { get_json(full_path, exportable_path, relation_name) } describe 'saves project tree attributes' do let_it_be(:shared) { project.import_export_shared } let(:relation_name) { :projects } - let_it_be(:full_path) do - if ndjson_enabled - File.join(shared.export_path, 'tree') - else - File.join(shared.export_path, Gitlab::ImportExport.project_filename) - end - end + let_it_be(:full_path) { File.join(shared.export_path, 'tree') } before_all do RSpec::Mocks.with_temporary_scope do stub_all_feature_flags - stub_feature_flags(project_export_as_ndjson: ndjson_enabled) project.add_maintainer(user) @@ -223,22 +216,31 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do expect(subject.dig(0, 'stages')).not_to be_empty end - it 'has pipeline statuses' do - expect(subject.dig(0, 'stages', 0, 'statuses')).not_to be_empty + it 'has pipeline builds' do + count = subject.dig(0, 'stages', 0, 'builds').count + + expect(count).to eq(1) end - it 'has pipeline builds' do - builds_count = subject.dig(0, 'stages', 0, 'statuses') - .count { |hash| hash['type'] == 'Ci::Build' } + it 'has pipeline generic_commit_statuses' do + count = subject.dig(0, 'stages', 0, 'generic_commit_statuses').count - expect(builds_count).to eq(1) + expect(count).to eq(1) end - it 'has ci pipeline notes' do - expect(subject.first['notes']).not_to be_empty + it 'has pipeline bridges' do + count = subject.dig(0, 'stages', 0, 'bridges').count + + expect(count).to eq(1) end end + context 'with commit_notes' do + let(:relation_name) { :commit_notes } + + it { is_expected.not_to be_empty } + end + context 'with labels' do let(:relation_name) { :labels } @@ -291,13 +293,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do let_it_be(:group) { create(:group) } let(:project) { setup_project } - let(:full_path) do - if ndjson_enabled - File.join(shared.export_path, 'tree') - else - File.join(shared.export_path, Gitlab::ImportExport.project_filename) - end - end + let(:full_path) { File.join(shared.export_path, 'tree') } let(:shared) { project.import_export_shared } let(:params) { {} } @@ -305,7 +301,6 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do let(:project_tree_saver ) { described_class.new(project: project, current_user: user, shared: shared, params: params) } before do - stub_feature_flags(project_export_as_ndjson: ndjson_enabled) project.add_maintainer(user) FileUtils.rm_rf(export_path) @@ -416,13 +411,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do end end - context 'with JSON' do - it_behaves_like "saves project tree successfully", false - end - - context 'with NDJSON' do - it_behaves_like "saves project tree successfully", true - end + it_behaves_like "saves project tree successfully" context 'when streaming has to retry', :aggregate_failures do let(:shared) { double('shared', export_path: exportable_path) } @@ -468,6 +457,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do end end + # rubocop: disable Metrics/AbcSize def setup_project release = create(:release) @@ -496,6 +486,8 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do ci_build = create(:ci_build, project: project, when: nil) ci_build.pipeline.update!(project: project) create(:commit_status, project: project, pipeline: ci_build.pipeline) + create(:generic_commit_status, pipeline: ci_build.pipeline, ci_stage: ci_build.ci_stage, project: project) + create(:ci_bridge, pipeline: ci_build.pipeline, ci_stage: ci_build.ci_stage, project: project) create(:milestone, project: project) discussion_note = create(:discussion_note, noteable: issue, project: project) @@ -528,4 +520,5 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver, :with_license do project end + # rubocop: enable Metrics/AbcSize end diff --git a/spec/lib/gitlab/import_export/references_configuration_spec.rb b/spec/lib/gitlab/import_export/references_configuration_spec.rb index ad165790b77..84c5b564cb1 100644 --- a/spec/lib/gitlab/import_export/references_configuration_spec.rb +++ b/spec/lib/gitlab/import_export/references_configuration_spec.rb @@ -9,7 +9,7 @@ require 'spec_helper' # or to be blacklisted by using the import_export.yml configuration file. # Likewise, new models added to import_export.yml, will need to be added with their correspondent relations # to this spec. -RSpec.describe 'Import/Export Project configuration' do +RSpec.describe 'Import/Export Project configuration', feature_category: :importers do include ConfigurationHelper where(:relation_path, :relation_name) do diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index e14e929faf3..faf345e8f78 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -86,6 +86,12 @@ Note: - original_discussion_id - confidential - last_edited_at +- internal +Notes::NoteMetadata: +- note_id +- email_participant +- created_at +- updated_at LabelLink: - id - target_type @@ -347,7 +353,111 @@ Ci::Stage: - pipeline_id - created_at - updated_at -CommitStatus: +Ci::Build: +- id +- project_id +- status +- finished_at +- trace +- created_at +- updated_at +- started_at +- runner_id +- coverage +- commit_id +- commands +- job_id +- name +- deploy +- options +- allow_failure +- stage +- trigger_request_id +- stage_idx +- stage_id +- tag +- ref +- user_id +- type +- target_url +- description +- artifacts_file +- artifacts_file_store +- artifacts_metadata +- artifacts_metadata_store +- erased_by_id +- erased_at +- artifacts_expire_at +- environment +- artifacts_size +- when +- yaml_variables +- queued_at +- token +- lock_version +- coverage_regex +- auto_canceled_by_id +- retried +- protected +- failure_reason +- scheduled_at +- upstream_pipeline_id +- interruptible +- processed +- scheduling_type +Ci::Bridge: +- id +- project_id +- status +- finished_at +- trace +- created_at +- updated_at +- started_at +- runner_id +- coverage +- commit_id +- commands +- job_id +- name +- deploy +- options +- allow_failure +- stage +- trigger_request_id +- stage_idx +- stage_id +- tag +- ref +- user_id +- type +- target_url +- description +- artifacts_file +- artifacts_file_store +- artifacts_metadata +- artifacts_metadata_store +- erased_by_id +- erased_at +- artifacts_expire_at +- environment +- artifacts_size +- when +- yaml_variables +- queued_at +- token +- lock_version +- coverage_regex +- auto_canceled_by_id +- retried +- protected +- failure_reason +- scheduled_at +- upstream_pipeline_id +- interruptible +- processed +- scheduling_type +GenericCommitStatus: - id - project_id - status @@ -822,6 +932,11 @@ DesignManagement::Version: - created_at - sha - author_id +DesignManagement::Repository: +- id +- project_id +- created_at +- updated_at ZoomMeeting: - id - project_id @@ -955,3 +1070,21 @@ ResourceIterationEvent: - action Iterations::Cadence: - title +ApprovalProjectRule: + - approvals_required + - name + - rule_type + - scanners + - vulnerabilities_allowed + - severity_levels + - report_type + - vulnerability_states + - orchestration_policy_idx + - applies_to_all_protected_branches +ApprovalProjectRulesUser: + - user_id + - approval_project_rule_id +ApprovalProjectRulesProtectedBranch: + - protected_branch_id + - approval_project_rule_id + - branch_name |