diff options
Diffstat (limited to 'spec/lib/gitlab/import_export')
17 files changed, 727 insertions, 68 deletions
diff --git a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb index 451fd6c6f46..42cf9c54798 100644 --- a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb +++ b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb @@ -9,12 +9,21 @@ RSpec.describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do allow_next_instance_of(ProjectExportWorker) do |job| allow(job).to receive(:jid).and_return(SecureRandom.hex(8)) end + + stub_feature_flags(import_export_web_upload_stream: false) + stub_uploads_object_storage(FileUploader, enabled: false) end let(:example_url) { 'http://www.example.com' } let(:strategy) { subject.new(url: example_url, http_method: 'post') } - let!(:project) { create(:project, :with_export) } - let!(:user) { build(:user) } + let(:user) { build(:user) } + let(:project) { import_export_upload.project } + let(:import_export_upload) do + create( + :import_export_upload, + export_file: fixture_file_upload('spec/fixtures/gitlab/import_export/lightweight_project_export.tar.gz') + ) + end subject { described_class } @@ -36,20 +45,42 @@ RSpec.describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do describe '#execute' do context 'when upload succeeds' do before do - allow(strategy).to receive(:send_file) - allow(strategy).to receive(:handle_response_error) + stub_full_request(example_url, method: :post).to_return(status: 200) end - it 'does not remove the exported project file after the upload' do + it 'does not remove the exported project file after the upload', :aggregate_failures do expect(project).not_to receive(:remove_exports) - strategy.execute(user, project) + expect { strategy.execute(user, project) }.not_to change(project, :export_status) + + expect(project.export_status).to eq(:finished) end - it 'has finished export status' do - strategy.execute(user, project) + it 'logs when upload starts and finishes' do + export_size = import_export_upload.export_file.size + + expect_next_instance_of(Gitlab::Export::Logger) do |logger| + expect(logger).to receive(:info).ordered.with( + { + message: "Started uploading project", + project_id: project.id, + project_name: project.name, + export_size: export_size + } + ) + + expect(logger).to receive(:info).ordered.with( + { + message: "Finished uploading project", + project_id: project.id, + project_name: project.name, + export_size: export_size, + upload_duration: anything + } + ) + end - expect(project.export_status).to eq(:finished) + strategy.execute(user, project) end end @@ -64,5 +95,124 @@ RSpec.describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do expect(errors.first).to eq "Error uploading the project. Code 404: Page not found" end end + + context 'when object store is disabled' do + it 'reads file from disk and uploads to external url' do + stub_request(:post, example_url).to_return(status: 200) + expect(Gitlab::ImportExport::RemoteStreamUpload).not_to receive(:new) + expect(Gitlab::HttpIO).not_to receive(:new) + + strategy.execute(user, project) + + expect(a_request(:post, example_url)).to have_been_made + end + end + + context 'when object store is enabled' do + before do + object_store_url = 'http://object-storage/project.tar.gz' + stub_uploads_object_storage(FileUploader) + stub_request(:get, object_store_url) + stub_request(:post, example_url) + allow(import_export_upload.export_file).to receive(:url).and_return(object_store_url) + allow(import_export_upload.export_file).to receive(:file_storage?).and_return(false) + end + + it 'reads file using Gitlab::HttpIO and uploads to external url' do + expect_next_instance_of(Gitlab::HttpIO) do |http_io| + expect(http_io).to receive(:read).and_call_original + end + expect(Gitlab::ImportExport::RemoteStreamUpload).not_to receive(:new) + + strategy.execute(user, project) + + expect(a_request(:post, example_url)).to have_been_made + end + end + + context 'when `import_export_web_upload_stream` feature is enabled' do + before do + stub_feature_flags(import_export_web_upload_stream: true) + end + + context 'when remote object store is disabled' do + it 'reads file from disk and uploads to external url' do + stub_request(:post, example_url).to_return(status: 200) + expect(Gitlab::ImportExport::RemoteStreamUpload).not_to receive(:new) + expect(Gitlab::HttpIO).not_to receive(:new) + + strategy.execute(user, project) + + expect(a_request(:post, example_url)).to have_been_made + end + end + + context 'when object store is enabled' do + let(:object_store_url) { 'http://object-storage/project.tar.gz' } + + before do + stub_uploads_object_storage(FileUploader) + + allow(import_export_upload.export_file).to receive(:url).and_return(object_store_url) + allow(import_export_upload.export_file).to receive(:file_storage?).and_return(false) + end + + it 'uploads file as a remote stream' do + arguments = { + download_url: object_store_url, + upload_url: example_url, + options: { + upload_method: :post, + upload_content_type: 'application/gzip' + } + } + + expect_next_instance_of(Gitlab::ImportExport::RemoteStreamUpload, arguments) do |remote_stream_upload| + expect(remote_stream_upload).to receive(:execute) + end + expect(Gitlab::HttpIO).not_to receive(:new) + + strategy.execute(user, project) + end + + context 'when upload as remote stream raises an exception' do + before do + allow_next_instance_of(Gitlab::ImportExport::RemoteStreamUpload) do |remote_stream_upload| + allow(remote_stream_upload).to receive(:execute).and_raise( + Gitlab::ImportExport::RemoteStreamUpload::StreamError.new('Exception error message', 'Response body') + ) + end + end + + it 'logs the exception and stores the error message' do + expect_next_instance_of(Gitlab::Export::Logger) do |logger| + expect(logger).to receive(:error).ordered.with( + { + project_id: project.id, + project_name: project.name, + message: 'Exception error message', + response_body: 'Response body' + } + ) + + expect(logger).to receive(:error).ordered.with( + { + project_id: project.id, + project_name: project.name, + message: 'After export strategy failed', + 'exception.class' => 'Gitlab::ImportExport::RemoteStreamUpload::StreamError', + 'exception.message' => 'Exception error message', + 'exception.backtrace' => anything + } + ) + end + + strategy.execute(user, project) + + expect(project.import_export_shared.errors.first).to eq('Exception error message') + end + end + end + end end end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 8c1e60e78b0..9aec3271913 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -140,6 +140,12 @@ project_members: - project - member_task - member_namespace +- member_role +member_roles: +- members +- namespace +- base_access_level +- download_code merge_requests: - status_check_responses - subscriptions @@ -591,6 +597,7 @@ project: - alert_management_alerts - repository_storage_moves - freeze_periods +- pumble_integration - webex_teams_integration - build_report_results - vulnerability_statistic @@ -621,6 +628,7 @@ project: - security_trainings - vulnerability_reads - build_artifacts_size_refresh +- project_callouts award_emoji: - awardable - user @@ -646,6 +654,11 @@ search_data: merge_request_assignees: - merge_request - assignee +- updated_state_by +merge_request_reviewers: +- merge_request +- reviewer +- updated_state_by lfs_file_locks: - user project_badges: @@ -805,3 +818,6 @@ bulk_import_export: - group service_desk_setting: - file_template_project +approvals: + - user + - merge_request diff --git a/spec/lib/gitlab/import_export/base/relation_factory_spec.rb b/spec/lib/gitlab/import_export/base/relation_factory_spec.rb index b8999f608b1..4ef8f4b5d76 100644 --- a/spec/lib/gitlab/import_export/base/relation_factory_spec.rb +++ b/spec/lib/gitlab/import_export/base/relation_factory_spec.rb @@ -139,6 +139,30 @@ RSpec.describe Gitlab::ImportExport::Base::RelationFactory do expect(subject.value).to be_nil end end + + context 'with duplicate assignees' do + let(:relation_sym) { :issues } + let(:relation_hash) do + { "title" => "title", "state" => "opened" }.merge(issue_assignees) + end + + context 'when duplicate assignees are present' do + let(:issue_assignees) do + { + "issue_assignees" => [ + IssueAssignee.new(user_id: 1), + IssueAssignee.new(user_id: 2), + IssueAssignee.new(user_id: 1), + { user_id: 3 } + ] + } + end + + it 'removes duplicate assignees' do + expect(subject.issue_assignees.map(&:user_id)).to contain_exactly(1, 2) + end + end + end end end diff --git a/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb b/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb index 7c84b9604a6..9f1b15aa049 100644 --- a/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb +++ b/spec/lib/gitlab/import_export/base/relation_object_saver_spec.rb @@ -58,8 +58,8 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver do end context 'when subrelation collection count is small' do - let(:notes) { build_list(:note, 2, project: project, importing: true) } - let(:relation_object) { build(:issue, project: project, notes: notes) } + let(:note) { build(:note, project: project, importing: true) } + let(:relation_object) { build(:issue, project: project, notes: [note]) } let(:relation_definition) { { 'notes' => {} } } it 'saves subrelation as part of the relation object itself' do @@ -68,7 +68,7 @@ RSpec.describe Gitlab::ImportExport::Base::RelationObjectSaver do saver.execute issue = project.issues.last - expect(issue.notes.count).to eq(2) + expect(issue.notes.count).to eq(1) end end diff --git a/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb b/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb index dea584e5019..9af72cc0dea 100644 --- a/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb +++ b/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb @@ -51,10 +51,11 @@ RSpec.describe Gitlab::ImportExport::DecompressedArchiveSizeValidator do shared_examples 'logs raised exception and terminates validator process group' do let(:std) { double(:std, close: nil, value: nil) } let(:wait_thr) { double } + let(:wait_threads) { [wait_thr, wait_thr] } before do allow(Process).to receive(:getpgid).and_return(2) - allow(Open3).to receive(:popen3).and_return([std, std, std, wait_thr]) + allow(Open3).to receive(:pipeline_r).and_return([std, wait_threads]) allow(wait_thr).to receive(:[]).with(:pid).and_return(1) allow(wait_thr).to receive(:value).and_raise(exception) end @@ -67,7 +68,7 @@ RSpec.describe Gitlab::ImportExport::DecompressedArchiveSizeValidator do import_upload_archive_size: File.size(filepath), message: error_message ) - expect(Process).to receive(:kill).with(-1, 2) + expect(Process).to receive(:kill).with(-1, 2).twice expect(subject.valid?).to eq(false) end end diff --git a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb index 9b01005c2e9..89ae869ae86 100644 --- a/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/group/tree_restorer_spec.rb @@ -204,19 +204,5 @@ RSpec.describe Gitlab::ImportExport::Group::TreeRestorer do end end - context 'when import_relation_object_persistence feature flag is enabled' do - before do - stub_feature_flags(import_relation_object_persistence: true) - end - - include_examples 'group restoration' - end - - context 'when import_relation_object_persistence feature flag is disabled' do - before do - stub_feature_flags(import_relation_object_persistence: false) - end - - include_examples 'group restoration' - end + include_examples 'group restoration' end diff --git a/spec/lib/gitlab/import_export/import_test_coverage_spec.rb b/spec/lib/gitlab/import_export/import_test_coverage_spec.rb index 90966cb4915..51c0008b2b4 100644 --- a/spec/lib/gitlab/import_export/import_test_coverage_spec.rb +++ b/spec/lib/gitlab/import_export/import_test_coverage_spec.rb @@ -88,8 +88,8 @@ RSpec.describe 'Test coverage of the Project Import' do def relations_from_json(json_file) json = Gitlab::Json.parse(IO.read(json_file)) - [].tap {|res| gather_relations({ project: json }, res, [])} - .map {|relation_names| relation_names.join('.')} + [].tap { |res| gather_relations({ project: json }, res, []) } + .map { |relation_names| relation_names.join('.') } end def gather_relations(item, res, path) @@ -103,7 +103,7 @@ RSpec.describe 'Test coverage of the Project Import' do end end when Array - item.each {|i| gather_relations(i, res, path)} + item.each { |i| gather_relations(i, res, path) } end end diff --git a/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb b/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb index 9be95591ae9..452d63d548e 100644 --- a/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb +++ b/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb @@ -41,7 +41,7 @@ RSpec.describe Gitlab::ImportExport::Json::NdjsonWriter do file_path = File.join(path, exportable_path, "#{relation}.ndjson") subject.write_relation(exportable_path, relation, values[0]) - expect {subject.write_relation(exportable_path, relation, values[1])}.to raise_exception("The #{file_path} already exist") + expect { subject.write_relation(exportable_path, relation, values[1]) }.to raise_exception("The #{file_path} already exist") end end end diff --git a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb index 3f73a730744..3088129a732 100644 --- a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb @@ -27,6 +27,7 @@ RSpec.describe Gitlab::ImportExport::Json::StreamingSerializer do end let(:exportable_path) { 'project' } + let(:logger) { Gitlab::Export::Logger.build } let(:json_writer) { instance_double('Gitlab::ImportExport::Json::LegacyWriter') } let(:hash) { { name: exportable.name, description: exportable.description }.stringify_keys } let(:include) { [] } @@ -42,7 +43,7 @@ RSpec.describe Gitlab::ImportExport::Json::StreamingSerializer do end subject do - described_class.new(exportable, relations_schema, json_writer, exportable_path: exportable_path) + described_class.new(exportable, relations_schema, json_writer, exportable_path: exportable_path, logger: logger) end describe '#execute' do @@ -73,6 +74,21 @@ RSpec.describe Gitlab::ImportExport::Json::StreamingSerializer do subject.execute end + it 'logs the relation name and the number of records to export' do + allow(json_writer).to receive(:write_relation_array) + allow(logger).to receive(:info) + + subject.execute + + expect(logger).to have_received(:info).with( + importer: 'Import/Export', + message: "Exporting issues relation. Number of records to export: 16", + project_id: exportable.id, + project_name: exportable.name, + project_path: exportable.full_path + ) + end + context 'default relation ordering' do it 'orders exported issues by primary key(:id)' do expected_issues = exportable.issues.reorder(:id).map(&:to_json) @@ -138,6 +154,21 @@ RSpec.describe Gitlab::ImportExport::Json::StreamingSerializer do subject.execute end + + it 'logs the relation name' do + allow(json_writer).to receive(:write_relation) + allow(logger).to receive(:info) + + subject.execute + + expect(logger).to have_received(:info).with( + importer: 'Import/Export', + message: 'Exporting group relation', + project_id: exportable.id, + project_name: exportable.name, + project_path: exportable.full_path + ) + end end context 'with array relation' do @@ -155,6 +186,21 @@ RSpec.describe Gitlab::ImportExport::Json::StreamingSerializer do subject.execute end + + it 'logs the relation name and the number of records to export' do + allow(json_writer).to receive(:write_relation_array) + allow(logger).to receive(:info) + + subject.execute + + expect(logger).to have_received(:info).with( + importer: 'Import/Export', + message: 'Exporting project_members relation. Number of records to export: 1', + project_id: exportable.id, + project_name: exportable.name, + project_path: exportable.full_path + ) + end end describe 'load balancing' do diff --git a/spec/lib/gitlab/import_export/log_util_spec.rb b/spec/lib/gitlab/import_export/log_util_spec.rb new file mode 100644 index 00000000000..2b1a4b7bb61 --- /dev/null +++ b/spec/lib/gitlab/import_export/log_util_spec.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::ImportExport::LogUtil do + describe '.exportable_to_log_payload' do + subject { described_class.exportable_to_log_payload(exportable) } + + context 'when exportable is a group' do + let(:exportable) { build_stubbed(:group) } + + it 'returns hash with group keys' do + expect(subject).to be_a(Hash) + expect(subject.keys).to eq(%i[group_id group_name group_path]) + end + end + + context 'when exportable is a project' do + let(:exportable) { build_stubbed(:project) } + + it 'returns hash with project keys' do + expect(subject).to be_a(Hash) + expect(subject.keys).to eq(%i[project_id project_name project_path]) + end + end + + context 'when exportable is a new record' do + let(:exportable) { Project.new } + + it 'returns empty hash' do + expect(subject).to eq({}) + end + end + + context 'when exportable is an unexpected type' do + let(:exportable) { build_stubbed(:issue) } + + it 'returns empty hash' do + expect(subject).to eq({}) + end + end + end +end diff --git a/spec/lib/gitlab/import_export/project/relation_saver_spec.rb b/spec/lib/gitlab/import_export/project/relation_saver_spec.rb new file mode 100644 index 00000000000..dec51b3afd1 --- /dev/null +++ b/spec/lib/gitlab/import_export/project/relation_saver_spec.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::ImportExport::Project::RelationSaver do + include ImportExport::CommonUtil + + subject(:relation_saver) { described_class.new(project: project, shared: shared, relation: relation) } + + let_it_be(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } + let_it_be(:project) { setup_project } + + let(:relation) { Projects::ImportExport::RelationExport::ROOT_RELATION } + let(:shared) do + shared = project.import_export_shared + allow(shared).to receive(:export_path).and_return(export_path) + shared + end + + after do + FileUtils.rm_rf(export_path) + end + + describe '#save' do + context 'when relation is the root node' do + let(:relation) { Projects::ImportExport::RelationExport::ROOT_RELATION } + + it 'serializes the root node as a json file in the export path' do + relation_saver.save # rubocop:disable Rails/SaveBang + + json = read_json(File.join(shared.export_path, 'project.json')) + expect(json).to include({ 'description' => 'Project description' }) + end + + it 'serializes only allowed attributes' do + relation_saver.save # rubocop:disable Rails/SaveBang + + json = read_json(File.join(shared.export_path, 'project.json')) + expect(json).to include({ 'description' => 'Project description' }) + expect(json.keys).not_to include('name') + end + + it 'successfuly serializes without errors' do + result = relation_saver.save # rubocop:disable Rails/SaveBang + + expect(result).to eq(true) + expect(shared.errors).to be_empty + end + end + + context 'when relation is a child node' do + let(:relation) { 'labels' } + + it 'serializes the child node as a ndjson file in the export path inside the project folder' do + relation_saver.save # rubocop:disable Rails/SaveBang + + ndjson = read_ndjson(File.join(shared.export_path, 'project', "#{relation}.ndjson")) + expect(ndjson.first).to include({ 'title' => 'Label 1' }) + expect(ndjson.second).to include({ 'title' => 'Label 2' }) + end + + it 'serializes only allowed attributes' do + relation_saver.save # rubocop:disable Rails/SaveBang + + ndjson = read_ndjson(File.join(shared.export_path, 'project', "#{relation}.ndjson")) + expect(ndjson.first.keys).not_to include('description_html') + end + + it 'successfuly serializes without errors' do + result = relation_saver.save # rubocop:disable Rails/SaveBang + + expect(result).to eq(true) + expect(shared.errors).to be_empty + end + end + + context 'when relation name is not supported' do + let(:relation) { 'unknown' } + + it 'returns false and register the error' do + result = relation_saver.save # rubocop:disable Rails/SaveBang + + expect(result).to eq(false) + expect(shared.errors).to be_present + end + end + + context 'when an exception occurs during serialization' do + it 'returns false and register the exception error message' do + allow_next_instance_of(Gitlab::ImportExport::Json::StreamingSerializer) do |serializer| + allow(serializer).to receive(:serialize_root).and_raise('Error!') + end + + result = relation_saver.save # rubocop:disable Rails/SaveBang + + expect(result).to eq(false) + expect(shared.errors).to include('Error!') + end + end + end + + def setup_project + project = create(:project, + description: 'Project description' + ) + + create(:label, project: project, title: 'Label 1') + create(:label, project: project, title: 'Label 2') + + project + end + + def read_json(path) + Gitlab::Json.parse(IO.read(path)) + end + + def read_ndjson(path) + relations = [] + File.foreach(path) do |line| + json = Gitlab::Json.parse(line) + relations << json + end + relations + end +end diff --git a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb index 157cd408da9..47d7555c8f4 100644 --- a/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_restorer_spec.rb @@ -254,6 +254,16 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do end end + it 'has multiple merge request assignees' do + expect(MergeRequest.find_by(title: 'MR1').assignees).to contain_exactly(@user, *@existing_members) + expect(MergeRequest.find_by(title: 'MR2').assignees).to be_empty + end + + it 'has multiple merge request reviewers' do + expect(MergeRequest.find_by(title: 'MR1').reviewers).to contain_exactly(@user, *@existing_members) + expect(MergeRequest.find_by(title: 'MR2').reviewers).to be_empty + end + it 'has labels associated to label links, associated to issues' do expect(Label.first.label_links.first.target).not_to be_nil end @@ -262,6 +272,11 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do expect(ProjectLabel.count).to eq(3) end + it 'has merge request approvals' do + expect(MergeRequest.find_by(title: 'MR1').approvals.pluck(:user_id)).to contain_exactly(@user.id, *@existing_members.map(&:id)) + expect(MergeRequest.find_by(title: 'MR2').approvals).to be_empty + end + it 'has no group labels' do expect(GroupLabel.count).to eq(0) end @@ -589,7 +604,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do it 'issue system note metadata restored successfully' do note_content = 'created merge request !1 to address this issue' - note = project.issues.first.notes.find { |n| n.note.match(/#{note_content}/)} + note = project.issues.first.notes.find { |n| n.note.match(/#{note_content}/) } expect(note.noteable_type).to eq('Issue') expect(note.system).to eq(true) @@ -1085,35 +1100,13 @@ RSpec.describe Gitlab::ImportExport::Project::TreeRestorer do end end - context 'when import_relation_object_persistence feature flag is enabled' do - before do - stub_feature_flags(import_relation_object_persistence: true) - end - - context 'enable ndjson import' do - it_behaves_like 'project tree restorer work properly', :legacy_reader, true + context 'enable ndjson import' do + it_behaves_like 'project tree restorer work properly', :legacy_reader, true - it_behaves_like 'project tree restorer work properly', :ndjson_reader, true - end - - context 'disable ndjson import' do - it_behaves_like 'project tree restorer work properly', :legacy_reader, false - end + it_behaves_like 'project tree restorer work properly', :ndjson_reader, true end - context 'when import_relation_object_persistence feature flag is disabled' do - before do - stub_feature_flags(import_relation_object_persistence: false) - end - - context 'enable ndjson import' do - it_behaves_like 'project tree restorer work properly', :legacy_reader, true - - it_behaves_like 'project tree restorer work properly', :ndjson_reader, true - end - - context 'disable ndjson import' do - it_behaves_like 'project tree restorer work properly', :legacy_reader, false - end + context 'disable ndjson import' do + it_behaves_like 'project tree restorer work properly', :legacy_reader, false end end diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index ba781ae78b7..15108d28bf2 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -68,6 +68,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver do it 'has merge request\'s milestones' do expect(subject.first['milestone']).not_to be_empty end + it 'has merge request\'s source branch SHA' do expect(subject.first['source_branch_sha']).to eq('b83d6e391c22777fca1ed3012fce84f633d7fed0') end @@ -100,9 +101,30 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver do expect(subject.first['notes'].first['author']).not_to be_empty end + it 'has merge request approvals' do + approval = subject.first['approvals'].first + + expect(approval).not_to be_nil + expect(approval['user_id']).to eq(user.id) + end + it 'has merge request resource label events' do expect(subject.first['resource_label_events']).not_to be_empty end + + it 'has merge request assignees' do + reviewer = subject.first['merge_request_assignees'].first + + expect(reviewer).not_to be_nil + expect(reviewer['user_id']).to eq(user.id) + end + + it 'has merge request reviewers' do + reviewer = subject.first['merge_request_reviewers'].first + + expect(reviewer).not_to be_nil + expect(reviewer['user_id']).to eq(user.id) + end end context 'with snippets' do @@ -404,7 +426,7 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver do context 'when streaming has to retry', :aggregate_failures do let(:shared) { double('shared', export_path: exportable_path) } - let(:logger) { Gitlab::Import::Logger.build } + let(:logger) { Gitlab::Export::Logger.build } let(:serializer) { double('serializer') } let(:error_class) { Net::OpenTimeout } let(:info_params) do @@ -468,7 +490,8 @@ RSpec.describe Gitlab::ImportExport::Project::TreeSaver do create(:label_link, label: group_label, target: issue) create(:label_priority, label: group_label, priority: 1) milestone = create(:milestone, project: project) - merge_request = create(:merge_request, source_project: project, milestone: milestone) + merge_request = create(:merge_request, source_project: project, milestone: milestone, assignees: [user], reviewers: [user]) + create(:approval, merge_request: merge_request, user: user) ci_build = create(:ci_build, project: project, when: nil) ci_build.pipeline.update!(project: project) diff --git a/spec/lib/gitlab/import_export/remote_stream_upload_spec.rb b/spec/lib/gitlab/import_export/remote_stream_upload_spec.rb new file mode 100644 index 00000000000..b1bc6b7eeaf --- /dev/null +++ b/spec/lib/gitlab/import_export/remote_stream_upload_spec.rb @@ -0,0 +1,232 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::ImportExport::RemoteStreamUpload do + include StubRequests + + subject do + described_class.new( + download_url: download_url, + upload_url: upload_url, + options: { + upload_method: upload_method, + upload_content_type: upload_content_type + } + ) + end + + let(:download_url) { 'http://object-storage/file.txt' } + let(:upload_url) { 'http://example.com/file.txt' } + let(:upload_method) { :post } + let(:upload_content_type) { 'text/plain' } + + describe '#execute' do + context 'when download request and upload request return 200' do + it 'uploads the downloaded content' do + stub_request(:get, download_url).to_return(status: 200, body: 'ABC', headers: { 'Content-Length' => 3 }) + stub_request(:post, upload_url) + + subject.execute + + expect( + a_request(:post, upload_url).with( + body: 'ABC', headers: { 'Content-Length' => 3, 'Content-Type' => 'text/plain' } + ) + ).to have_been_made + end + end + + context 'when upload method is put' do + let(:upload_method) { :put } + + it 'uploads using the put method' do + stub_request(:get, download_url).to_return(status: 200, body: 'ABC', headers: { 'Content-Length' => 3 }) + stub_request(:put, upload_url) + + subject.execute + + expect( + a_request(:put, upload_url).with( + body: 'ABC', headers: { 'Content-Length' => 3, 'Content-Type' => 'text/plain' } + ) + ).to have_been_made + end + end + + context 'when download request does not return 200' do + it do + stub_request(:get, download_url).to_return(status: 404) + + expect { subject.execute }.to raise_error( + Gitlab::ImportExport::RemoteStreamUpload::StreamError, + "Invalid response code while downloading file. Code: 404" + ) + end + end + + context 'when upload request does not returns 200' do + it do + stub_request(:get, download_url).to_return(status: 200, body: 'ABC', headers: { 'Content-Length' => 3 }) + stub_request(:post, upload_url).to_return(status: 403) + + expect { subject.execute }.to raise_error( + Gitlab::ImportExport::RemoteStreamUpload::StreamError, + "Invalid response code while uploading file. Code: 403" + ) + end + end + + context 'when download URL is a local address' do + let(:download_url) { 'http://127.0.0.1/file.txt' } + + before do + stub_request(:get, download_url) + stub_request(:post, upload_url) + end + + it 'raises error' do + expect { subject.execute }.to raise_error( + Gitlab::HTTP::BlockedUrlError, + "URL 'http://127.0.0.1/file.txt' is blocked: Requests to localhost are not allowed" + ) + end + + context 'when local requests are allowed' do + before do + stub_application_setting(allow_local_requests_from_web_hooks_and_services: true) + end + + it 'raises does not error' do + expect { subject.execute }.not_to raise_error + end + end + end + + context 'when download URL is a local network' do + let(:download_url) { 'http://172.16.0.0/file.txt' } + + before do + stub_request(:get, download_url) + stub_request(:post, upload_url) + end + + it 'raises error' do + expect { subject.execute }.to raise_error( + Gitlab::HTTP::BlockedUrlError, + "URL 'http://172.16.0.0/file.txt' is blocked: Requests to the local network are not allowed" + ) + end + + context 'when local network requests are allowed' do + before do + stub_application_setting(allow_local_requests_from_web_hooks_and_services: true) + end + + it 'raises does not error' do + expect { subject.execute }.not_to raise_error + end + end + end + + context 'when upload URL is a local address' do + let(:upload_url) { 'http://127.0.0.1/file.txt' } + + before do + stub_request(:get, download_url) + stub_request(:post, upload_url) + end + + it 'raises error' do + stub_request(:get, download_url) + + expect { subject.execute }.to raise_error( + Gitlab::HTTP::BlockedUrlError, + "URL 'http://127.0.0.1/file.txt' is blocked: Requests to localhost are not allowed" + ) + end + + context 'when local requests are allowed' do + before do + stub_application_setting(allow_local_requests_from_web_hooks_and_services: true) + end + + it 'raises does not error' do + expect { subject.execute }.not_to raise_error + end + end + end + + context 'when upload URL it is a request to local network' do + let(:upload_url) { 'http://172.16.0.0/file.txt' } + + before do + stub_request(:get, download_url) + stub_request(:post, upload_url) + end + + it 'raises error' do + expect { subject.execute }.to raise_error( + Gitlab::HTTP::BlockedUrlError, + "URL 'http://172.16.0.0/file.txt' is blocked: Requests to the local network are not allowed" + ) + end + + context 'when local network requests are allowed' do + before do + stub_application_setting(allow_local_requests_from_web_hooks_and_services: true) + end + + it 'raises does not error' do + expect { subject.execute }.not_to raise_error + end + end + end + + context 'when upload URL resolves to a local address' do + let(:upload_url) { 'http://example.com/file.txt' } + + it 'raises error' do + stub_request(:get, download_url) + stub_full_request(upload_url, ip_address: '127.0.0.1', method: upload_method) + + expect { subject.execute }.to raise_error( + Gitlab::HTTP::BlockedUrlError, + "URL 'http://example.com/file.txt' is blocked: Requests to localhost are not allowed" + ) + end + end + end + + describe Gitlab::ImportExport::RemoteStreamUpload::ChunkStream do + describe 'StringIO#copy_stream compatibility' do + it 'copies all chunks' do + chunks = %w[ABC EFD].to_enum + chunk_stream = described_class.new(chunks) + new_stream = StringIO.new + + IO.copy_stream(chunk_stream, new_stream) + new_stream.rewind + + expect(new_stream.read).to eq('ABCEFD') + end + + context 'with chunks smaller and bigger than buffer size' do + before do + stub_const('Gitlab::ImportExport::RemoteStreamUpload::ChunkStream::DEFAULT_BUFFER_SIZE', 4) + end + + it 'copies all chunks' do + chunks = %w[A BC DEF GHIJ KLMNOPQ RSTUVWXYZ].to_enum + chunk_stream = described_class.new(chunks) + new_stream = StringIO.new + + IO.copy_stream(chunk_stream, new_stream) + new_stream.rewind + + expect(new_stream.read).to eq('ABCDEFGHIJKLMNOPQRSTUVWXYZ') + end + end + end + end +end diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index bd60bb53d49..6cfc24a8996 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -521,7 +521,6 @@ Project: - star_count - ci_id - shared_runners_enabled -- build_coverage_regex - build_allow_git_fetchs - build_timeout - pending_delete @@ -584,6 +583,9 @@ ProjectFeature: - security_and_compliance_access_level - container_registry_access_level - package_registry_access_level +- environments_access_level +- feature_flags_access_level +- releases_access_level - created_at - updated_at ProtectedBranch::MergeAccessLevel: @@ -741,6 +743,14 @@ MergeRequestAssignee: - id - user_id - merge_request_id +- created_at +- state +MergeRequestReviewer: +- id +- user_id +- merge_request_id +- created_at +- state ProjectMetricsSetting: - project_id - external_dashboard_url @@ -903,3 +913,7 @@ MergeRequest::CleanupSchedule: - completed_at - created_at - updated_at +Approval: + - user_id + - created_at + - updated_at diff --git a/spec/lib/gitlab/import_export/shared_spec.rb b/spec/lib/gitlab/import_export/shared_spec.rb index 1945156ca59..408ed3a2176 100644 --- a/spec/lib/gitlab/import_export/shared_spec.rb +++ b/spec/lib/gitlab/import_export/shared_spec.rb @@ -68,12 +68,18 @@ RSpec.describe Gitlab::ImportExport::Shared do expect(subject.errors).to eq(['Error importing into [FILTERED] Permission denied @ unlink_internal - [FILTERED]']) end - it 'updates the import JID' do + it 'tracks exception' do import_state = create(:import_state, project: project, jid: 'jid-test') expect(Gitlab::ErrorTracking) .to receive(:track_exception) - .with(error, hash_including(import_jid: import_state.jid)) + .with(error, hash_including( + importer: 'Import/Export', + project_id: project.id, + project_name: project.name, + project_path: project.full_path, + import_jid: import_state.jid + )) subject.error(error) end diff --git a/spec/lib/gitlab/import_export/version_checker_spec.rb b/spec/lib/gitlab/import_export/version_checker_spec.rb index 9e69e04b17c..14c62edb786 100644 --- a/spec/lib/gitlab/import_export/version_checker_spec.rb +++ b/spec/lib/gitlab/import_export/version_checker_spec.rb @@ -30,7 +30,7 @@ RSpec.describe Gitlab::ImportExport::VersionChecker do end context 'newer version' do - let(:version) { '900.0'} + let(:version) { '900.0' } it 'returns false if export version is newer' do expect(described_class.check!(shared: shared)).to be false |