diff options
44 files changed, 99 insertions, 893 deletions
diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 0eaf9f94e37..646d8329aff 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -191,10 +191,8 @@ class ProjectsController < Projects::ApplicationController end def download_export - if export_project_object_storage? + if @project.export_project_object_exists? send_upload(@project.import_export_upload.export_file) - elsif export_project_path - send_file export_project_path, disposition: 'attachment' else redirect_to( edit_project_path(@project, anchor: 'js-export-project'), @@ -425,12 +423,4 @@ class ProjectsController < Projects::ApplicationController def whitelist_query_limiting Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42440') end - - def export_project_path - @export_project_path ||= @project.export_project_path - end - - def export_project_object_storage? - @project.export_project_object_exists? - end end diff --git a/app/models/concerns/storage/legacy_namespace.rb b/app/models/concerns/storage/legacy_namespace.rb index 3b745657a9e..9785011720a 100644 --- a/app/models/concerns/storage/legacy_namespace.rb +++ b/app/models/concerns/storage/legacy_namespace.rb @@ -25,8 +25,6 @@ module Storage Gitlab::PagesTransfer.new.rename_namespace(full_path_was, full_path) end - remove_exports! - # If repositories moved successfully we need to # send update instructions to users. # However we cannot allow rollback since we moved namespace dir @@ -101,8 +99,6 @@ module Storage end end end - - remove_exports! end def remove_legacy_exports! diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 0deb44d7916..76920c3c039 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -253,18 +253,6 @@ class Namespace < ActiveRecord::Base end end - # Exports belonging to projects with legacy storage are placed in a common - # subdirectory of the namespace, so a simple `rm -rf` is sufficient to remove - # them. - # - # Exports of projects using hashed storage are placed in a location defined - # only by the project ID, so each must be removed individually. - def remove_exports! - remove_legacy_exports! - - all_projects.with_storage_feature(:repository).find_each(&:remove_exports) - end - def refresh_project_authorizations owner.refresh_authorized_projects end diff --git a/app/models/project.rb b/app/models/project.rb index 97d9fa355ef..35c93aec20a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1733,16 +1733,12 @@ class Project < ActiveRecord::Base import_export_shared.archive_path end - def export_project_path - Dir.glob("#{export_path}/*export.tar.gz").max_by { |f| File.ctime(f) } - end - def export_status if export_in_progress? :started elsif after_export_in_progress? :after_export_action - elsif export_project_path || export_project_object_exists? + elsif export_project_object_exists? :finished else :none @@ -1757,21 +1753,15 @@ class Project < ActiveRecord::Base import_export_shared.after_export_in_progress? end - def remove_exports(path = export_path) - if path.present? - FileUtils.rm_rf(path) - elsif export_project_object_exists? - import_export_upload.remove_export_file! - import_export_upload.save - end - end + def remove_exports + return unless export_project_object_exists? - def remove_exported_project_file - remove_exports(export_project_path) + import_export_upload.remove_export_file! + import_export_upload.save end def export_project_object_exists? - Gitlab::ImportExport.object_storage? && import_export_upload&.export_file&.file + import_export_upload&.export_file&.file end def full_path_slug diff --git a/app/uploaders/avatar_uploader.rb b/app/uploaders/avatar_uploader.rb index b29ef57b071..8526bc16390 100644 --- a/app/uploaders/avatar_uploader.rb +++ b/app/uploaders/avatar_uploader.rb @@ -18,6 +18,10 @@ class AvatarUploader < GitlabUploader false end + def absolute_path + self.class.absolute_path(model.avatar) + end + private def dynamic_segment diff --git a/changelogs/unreleased/48778-remove-old-storage-logic-from-import-export.yml b/changelogs/unreleased/48778-remove-old-storage-logic-from-import-export.yml new file mode 100644 index 00000000000..4ddbc118e86 --- /dev/null +++ b/changelogs/unreleased/48778-remove-old-storage-logic-from-import-export.yml @@ -0,0 +1,5 @@ +--- +title: Update Import/Export to only use new storage uploaders logic +merge_request: 21409 +author: +type: added diff --git a/doc/administration/raketasks/project_import_export.md b/doc/administration/raketasks/project_import_export.md index 7bd765a35e0..f43bba0a7a7 100644 --- a/doc/administration/raketasks/project_import_export.md +++ b/doc/administration/raketasks/project_import_export.md @@ -9,6 +9,7 @@ > application settings (`/admin/application_settings`) under 'Import sources'. > - The exports are stored in a temporary [shared directory][tmp] and are deleted > every 24 hours by a specific worker. +> - ImportExport can use object storage automatically starting from GitLab 11.3 The GitLab Import/Export version can be checked by using: @@ -30,12 +31,6 @@ sudo gitlab-rake gitlab:import_export:data bundle exec rake gitlab:import_export:data RAILS_ENV=production ``` -In order to enable Object Storage on the Export, you can use the [feature flag][feature-flags]: - -``` -import_export_object_storage -``` - [ce-3050]: https://gitlab.com/gitlab-org/gitlab-ce/issues/3050 [feature-flags]: https://docs.gitlab.com/ee/api/features.html [tmp]: ../../development/shared_files.md diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 15c57a2fc02..8954a0c0a4c 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -21,11 +21,7 @@ module API detail 'This feature was introduced in GitLab 10.6.' end get ':id/export/download' do - path = user_project.export_project_path - - if path - present_disk_file!(path, File.basename(path), 'application/gzip') - elsif user_project.export_project_object_exists? + if user_project.export_project_object_exists? present_carrierwave_file!(user_project.import_export_upload.export_file) else render_api_error!('404 Not found or has expired', 404) diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index be3710c5b7f..53fe2f8e436 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -40,10 +40,6 @@ module Gitlab "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" end - def object_storage? - Feature.enabled?(:import_export_object_storage) - end - def version VERSION end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index dce8f89c0ab..064092681f8 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -23,7 +23,7 @@ module Gitlab def strategy_execute handle_response_error(send_file) - project.remove_exported_project_file + project.remove_exports end def handle_response_error(response) @@ -40,15 +40,11 @@ module Gitlab def send_file Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend ensure - export_file.close if export_file && !object_storage? + export_file.close if export_file end def export_file - if object_storage? - project.import_export_upload.export_file.file.open - else - File.open(project.export_project_path) - end + project.import_export_upload.export_file.open end def send_file_options @@ -63,11 +59,7 @@ module Gitlab end def export_size - if object_storage? - project.import_export_upload.export_file.file.size - else - File.size(project.export_project_path) - end + project.import_export_upload.export_file.file.size end end end diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb index cfa595629f4..ded05f73cf8 100644 --- a/lib/gitlab/import_export/avatar_restorer.rb +++ b/lib/gitlab/import_export/avatar_restorer.rb @@ -19,7 +19,7 @@ module Gitlab private def avatar_export_file - @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first + @avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].first end def avatar_export_path diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 31ef0490cb3..6ffebf83dd2 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -1,8 +1,6 @@ module Gitlab module ImportExport class AvatarSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared @@ -14,19 +12,12 @@ module Gitlab Gitlab::ImportExport::UploadsManager.new( project: @project, shared: @shared, - relative_export_path: 'avatar', - from: avatar_path + relative_export_path: 'avatar' ).save rescue => e @shared.error(e) false end - - private - - def avatar_path - @project.avatar.path - end end end end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 4e179f63d8c..72d5b9b830c 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -92,8 +92,6 @@ module Gitlab end def remove_import_file - return unless Gitlab::ImportExport.object_storage? - upload = @project.import_export_upload return unless upload&.import_file&.file diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index f4106e03a57..00ea4b833e2 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -199,7 +199,7 @@ module Gitlab end def excluded_keys_for_relation(relation) - @reader.attributes_finder.find_excluded_keys(relation) + reader.attributes_finder.find_excluded_keys(relation) end end end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 3cd153a4fd2..a66f32f91d2 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -18,7 +18,7 @@ module Gitlab Rails.logger.info("Saved project export #{archive_file}") - save_on_object_storage if use_object_storage? + save_on_object_storage else @shared.error(Gitlab::ImportExport::Error.new(error_message)) false @@ -27,10 +27,8 @@ module Gitlab @shared.error(e) false ensure - if use_object_storage? - remove_archive - remove_export_path - end + remove_archive + remove_export_path end private @@ -59,12 +57,8 @@ module Gitlab upload.save! end - def use_object_storage? - Gitlab::ImportExport.object_storage? - end - def error_message - "Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}" + "Unable to save #{archive_file} into #{@shared.export_path}." end end end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index e0d4235e65b..8511319cb1c 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -5,18 +5,13 @@ module Gitlab UPLOADS_BATCH_SIZE = 100 - def initialize(project:, shared:, relative_export_path: 'uploads', from: nil) + def initialize(project:, shared:, relative_export_path: 'uploads') @project = project @shared = shared @relative_export_path = relative_export_path - @from = from || default_uploads_path end def save - if File.file?(@from) && @relative_export_path == 'avatar' - copy_files(@from, File.join(uploads_export_path, @project.avatar.filename)) - end - copy_project_uploads true @@ -55,17 +50,11 @@ module Gitlab copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path)) else - next unless Gitlab::ImportExport.object_storage? - download_and_copy(uploader) end end end - def default_uploads_path - FileUploader.absolute_base_dir(@project) - end - def uploads_export_path @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path) end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index 25f85936227..b4313ff4cb4 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -2,30 +2,14 @@ module Gitlab module ImportExport class UploadsRestorer < UploadsSaver def restore - if Gitlab::ImportExport.object_storage? - Gitlab::ImportExport::UploadsManager.new( - project: @project, - shared: @shared - ).restore - elsif File.directory?(uploads_export_path) - copy_files(uploads_export_path, uploads_path) - - true - else - true # Proceed without uploads - end + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).restore rescue => e @shared.error(e) false end - - def uploads_path - FileUploader.absolute_base_dir(@project) - end - - def uploads_export_path - @uploads_export_path ||= File.join(@shared.export_path, 'uploads') - end end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index b3f17af5661..0275f686c5e 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -1,8 +1,6 @@ module Gitlab module ImportExport class UploadsSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb index f24a01e6cf5..fc498dde723 100644 --- a/lib/gitlab/template_helper.rb +++ b/lib/gitlab/template_helper.rb @@ -1,24 +1,9 @@ module Gitlab module TemplateHelper - include Gitlab::Utils::StrongMemoize - def prepare_template_environment(file) return unless file - if Gitlab::ImportExport.object_storage? - params[:import_export_upload] = ImportExportUpload.new(import_file: file) - else - FileUtils.mkdir_p(File.dirname(import_upload_path)) - FileUtils.copy_entry(file.path, import_upload_path) - - params[:import_source] = import_upload_path - end - end - - def import_upload_path - strong_memoize(:import_upload_path) do - Gitlab::ImportExport.import_upload_path(filename: tmp_filename) - end + params[:import_export_upload] = ImportExportUpload.new(import_file: file) end def tmp_filename diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index a25f7ce59c7..3c873ba3a20 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -6,6 +6,8 @@ namespace :gitlab do desc "GitLab | Update project templates" task :update_project_templates do + include Gitlab::ImportExport::CommandLineUtil + if Rails.env.production? puts "This rake task is not meant fo production instances".red exit(1) @@ -52,7 +54,7 @@ namespace :gitlab do end Projects::ImportExport::ExportService.new(project, admin).execute - FileUtils.cp(project.export_project_path, template.archive_path) + download_or_copy_upload(project.import_export_upload.export_file, template.archive_path) Projects::DestroyService.new(admin, project).execute puts "Exported #{template.name}".green end diff --git a/spec/controllers/projects_controller_spec.rb b/spec/controllers/projects_controller_spec.rb index c3a66477b6a..3bc9cbe64c5 100644 --- a/spec/controllers/projects_controller_spec.rb +++ b/spec/controllers/projects_controller_spec.rb @@ -803,37 +803,7 @@ describe ProjectsController do project.add_maintainer(user) end - context 'object storage disabled' do - before do - stub_feature_flags(import_export_object_storage: false) - end - - context 'when project export is enabled' do - it 'returns 302' do - get :download_export, namespace_id: project.namespace, id: project - - expect(response).to have_gitlab_http_status(302) - end - end - - context 'when project export is disabled' do - before do - stub_application_setting(project_export_enabled?: false) - end - - it 'returns 404' do - get :download_export, namespace_id: project.namespace, id: project - - expect(response).to have_gitlab_http_status(404) - end - end - end - context 'object storage enabled' do - before do - stub_feature_flags(import_export_object_storage: true) - end - context 'when project export is enabled' do it 'returns 302' do get :download_export, namespace_id: project.namespace, id: project diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb index 17e457c04a5..c0d819498e1 100644 --- a/spec/factories/projects.rb +++ b/spec/factories/projects.rb @@ -103,27 +103,11 @@ FactoryBot.define do end trait :with_export do - before(:create) do |_project, _evaluator| - allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { false } - allow(Feature).to receive(:enabled?).with('import_export_object_storage') { false } - end - after(:create) do |project, _evaluator| ProjectExportWorker.new.perform(project.creator.id, project.id) end end - trait :with_object_export do - before(:create) do |_project, _evaluator| - allow(Feature).to receive(:enabled?).with(:import_export_object_storage) { true } - allow(Feature).to receive(:enabled?).with('import_export_object_storage') { true } - end - - after(:create) do |project, evaluator| - ProjectExportWorker.new.perform(project.creator.id, project.id) - end - end - trait :broken_storage do after(:create) do |project| project.update_column(:repository_storage, 'broken') diff --git a/spec/features/projects/import_export/export_file_spec.rb b/spec/features/projects/import_export/export_file_spec.rb index eb281cd2122..8a418356541 100644 --- a/spec/features/projects/import_export/export_file_spec.rb +++ b/spec/features/projects/import_export/export_file_spec.rb @@ -25,7 +25,6 @@ describe 'Import/Export - project export integration test', :js do before do allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - stub_feature_flags(import_export_object_storage: false) end after do diff --git a/spec/features/projects/import_export/import_file_object_storage_spec.rb b/spec/features/projects/import_export/import_file_object_storage_spec.rb deleted file mode 100644 index 0d364543916..00000000000 --- a/spec/features/projects/import_export/import_file_object_storage_spec.rb +++ /dev/null @@ -1,103 +0,0 @@ -require 'spec_helper' - -describe 'Import/Export - project import integration test', :js do - include Select2Helper - - let(:user) { create(:user) } - let(:file) { File.join(Rails.root, 'spec', 'features', 'projects', 'import_export', 'test_project_export.tar.gz') } - let(:export_path) { "#{Dir.tmpdir}/import_file_spec" } - - before do - stub_feature_flags(import_export_object_storage: true) - stub_uploads_object_storage(FileUploader) - allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - gitlab_sign_in(user) - end - - after do - FileUtils.rm_rf(export_path, secure: true) - end - - context 'when selecting the namespace' do - let(:user) { create(:admin) } - let!(:namespace) { user.namespace } - let(:project_path) { 'test-project-path' + SecureRandom.hex } - - context 'prefilled the path' do - it 'user imports an exported project successfully' do - visit new_project_path - - select2(namespace.id, from: '#project_namespace_id') - fill_in :project_path, with: project_path, visible: true - click_import_project_tab - click_link 'GitLab export' - - expect(page).to have_content('Import an exported GitLab project') - expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}") - - attach_file('file', file) - click_on 'Import project' - - expect(Project.count).to eq(1) - - project = Project.last - expect(project).not_to be_nil - expect(project.description).to eq("Foo Bar") - expect(project.issues).not_to be_empty - expect(project.merge_requests).not_to be_empty - expect(project_hook_exists?(project)).to be true - expect(wiki_exists?(project)).to be true - expect(project.import_state.status).to eq('finished') - end - end - - context 'path is not prefilled' do - it 'user imports an exported project successfully' do - visit new_project_path - click_import_project_tab - click_link 'GitLab export' - - fill_in :path, with: 'test-project-path', visible: true - attach_file('file', file) - - expect { click_on 'Import project' }.to change { Project.count }.by(1) - - project = Project.last - expect(project).not_to be_nil - expect(page).to have_content("Project 'test-project-path' is being imported") - end - end - end - - it 'invalid project' do - project = create(:project, namespace: user.namespace) - - visit new_project_path - - select2(user.namespace.id, from: '#project_namespace_id') - fill_in :project_path, with: project.name, visible: true - click_import_project_tab - click_link 'GitLab export' - attach_file('file', file) - click_on 'Import project' - - page.within('.flash-container') do - expect(page).to have_content('Project could not be imported') - end - end - - def wiki_exists?(project) - wiki = ProjectWiki.new(project) - wiki.repository.exists? && !wiki.repository.empty? - end - - def project_hook_exists?(project) - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - Gitlab::Git::Hook.new('post-receive', project.repository.raw_repository).exists? - end - end - - def click_import_project_tab - find('#import-project-tab').click - end -end diff --git a/spec/features/projects/import_export/import_file_spec.rb b/spec/features/projects/import_export/import_file_spec.rb index 2d86115de12..2936482a1f7 100644 --- a/spec/features/projects/import_export/import_file_spec.rb +++ b/spec/features/projects/import_export/import_file_spec.rb @@ -8,7 +8,7 @@ describe 'Import/Export - project import integration test', :js do let(:export_path) { "#{Dir.tmpdir}/import_file_spec" } before do - stub_feature_flags(import_export_object_storage: false) + stub_uploads_object_storage(FileUploader) allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) gitlab_sign_in(user) end @@ -33,7 +33,6 @@ describe 'Import/Export - project import integration test', :js do expect(page).to have_content('Import an exported GitLab project') expect(URI.parse(current_url).query).to eq("namespace_id=#{namespace.id}&path=#{project_path}") - expect(Gitlab::ImportExport).to receive(:import_upload_path).with(filename: /\A\h{32}\z/).and_call_original attach_file('file', file) click_on 'Import project' diff --git a/spec/features/projects/import_export/namespace_export_file_spec.rb b/spec/features/projects/import_export/namespace_export_file_spec.rb deleted file mode 100644 index 9bb8a2063b5..00000000000 --- a/spec/features/projects/import_export/namespace_export_file_spec.rb +++ /dev/null @@ -1,68 +0,0 @@ -require 'spec_helper' - -describe 'Import/Export - Namespace export file cleanup', :js do - let(:export_path) { Dir.mktmpdir('namespace_export_file_spec') } - - before do - allow(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - stub_feature_flags(import_export_object_storage: false) - end - - after do - FileUtils.rm_rf(export_path, secure: true) - end - - shared_examples_for 'handling project exports on namespace change' do - let!(:old_export_path) { project.export_path } - - before do - sign_in(create(:admin)) - - setup_export_project - end - - context 'moving the namespace' do - it 'removes the export file' do - expect(File).to exist(old_export_path) - - project.namespace.update!(path: build(:namespace).path) - - expect(File).not_to exist(old_export_path) - end - end - - context 'deleting the namespace' do - it 'removes the export file' do - expect(File).to exist(old_export_path) - - project.namespace.destroy - - expect(File).not_to exist(old_export_path) - end - end - end - - describe 'legacy storage' do - let(:project) { create(:project, :legacy_storage) } - - it_behaves_like 'handling project exports on namespace change' - end - - describe 'hashed storage' do - let(:project) { create(:project) } - - it_behaves_like 'handling project exports on namespace change' - end - - def setup_export_project - visit edit_project_path(project) - - expect(page).to have_content('Export project') - - find(:link, 'Export project').send_keys(:return) - - visit edit_project_path(project) - - expect(page).to have_content('Download export') - end -end diff --git a/spec/lib/gitlab/cleanup/project_uploads_spec.rb b/spec/lib/gitlab/cleanup/project_uploads_spec.rb index 11e605eece6..bf130b8fabd 100644 --- a/spec/lib/gitlab/cleanup/project_uploads_spec.rb +++ b/spec/lib/gitlab/cleanup/project_uploads_spec.rb @@ -132,7 +132,6 @@ describe Gitlab::Cleanup::ProjectUploads do let!(:path) { File.join(FileUploader.root, orphaned.model.full_path, orphaned.path) } before do - stub_feature_flags(import_export_object_storage: true) stub_uploads_object_storage(FileUploader) FileUtils.mkdir_p(File.dirname(path)) @@ -156,7 +155,6 @@ describe Gitlab::Cleanup::ProjectUploads do let!(:new_path) { File.join(FileUploader.root, '-', 'project-lost-found', 'wrong', orphaned.path) } before do - stub_feature_flags(import_export_object_storage: true) stub_uploads_object_storage(FileUploader) FileUtils.mkdir_p(File.dirname(path)) diff --git a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb deleted file mode 100644 index 5059d68e54b..00000000000 --- a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_object_storage_spec.rb +++ /dev/null @@ -1,105 +0,0 @@ -require 'spec_helper' - -describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do - let!(:service) { described_class.new } - let!(:project) { create(:project, :with_object_export) } - let(:shared) { project.import_export_shared } - let!(:user) { create(:user) } - - describe '#execute' do - before do - allow(service).to receive(:strategy_execute) - stub_feature_flags(import_export_object_storage: true) - end - - it 'returns if project exported file is not found' do - allow(project).to receive(:export_project_object_exists?).and_return(false) - - expect(service).not_to receive(:strategy_execute) - - service.execute(user, project) - end - - it 'creates a lock file in the export dir' do - allow(service).to receive(:delete_after_export_lock) - - service.execute(user, project) - - expect(lock_path_exist?).to be_truthy - end - - context 'when the method succeeds' do - it 'removes the lock file' do - service.execute(user, project) - - expect(lock_path_exist?).to be_falsey - end - end - - context 'when the method fails' do - before do - allow(service).to receive(:strategy_execute).and_call_original - end - - context 'when validation fails' do - before do - allow(service).to receive(:invalid?).and_return(true) - end - - it 'does not create the lock file' do - expect(service).not_to receive(:create_or_update_after_export_lock) - - service.execute(user, project) - end - - it 'does not execute main logic' do - expect(service).not_to receive(:strategy_execute) - - service.execute(user, project) - end - - it 'logs validation errors in shared context' do - expect(service).to receive(:log_validation_errors) - - service.execute(user, project) - end - end - - context 'when an exception is raised' do - it 'removes the lock' do - expect { service.execute(user, project) }.to raise_error(NotImplementedError) - - expect(lock_path_exist?).to be_falsey - end - end - end - end - - describe '#log_validation_errors' do - it 'add the message to the shared context' do - errors = %w(test_message test_message2) - - allow(service).to receive(:invalid?).and_return(true) - allow(service.errors).to receive(:full_messages).and_return(errors) - - expect(shared).to receive(:add_error_message).twice.and_call_original - - service.execute(user, project) - - expect(shared.errors).to eq errors - end - end - - describe '#to_json' do - it 'adds the current strategy class to the serialized attributes' do - params = { param1: 1 } - result = params.merge(klass: described_class.to_s).to_json - - expect(described_class.new(params).to_json).to eq result - end - end - - def lock_path_exist? - File.exist?(described_class.lock_file_path(project)) - end -end diff --git a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb index 566b7f46c87..94a12221333 100644 --- a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb +++ b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb @@ -9,11 +9,10 @@ describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do describe '#execute' do before do allow(service).to receive(:strategy_execute) - stub_feature_flags(import_export_object_storage: false) end it 'returns if project exported file is not found' do - allow(project).to receive(:export_project_path).and_return(nil) + allow(project).to receive(:export_project_object_exists?).and_return(false) expect(service).not_to receive(:strategy_execute) diff --git a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb index 7f2e0a4ee2c..ec17ad8541f 100644 --- a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb +++ b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb @@ -24,34 +24,13 @@ describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do end describe '#execute' do - context 'without object storage' do - before do - stub_feature_flags(import_export_object_storage: false) - end - - it 'removes the exported project file after the upload' do - allow(strategy).to receive(:send_file) - allow(strategy).to receive(:handle_response_error) - - expect(project).to receive(:remove_exported_project_file) - - strategy.execute(user, project) - end - end - - context 'with object storage' do - before do - stub_feature_flags(import_export_object_storage: true) - end + it 'removes the exported project file after the upload' do + allow(strategy).to receive(:send_file) + allow(strategy).to receive(:handle_response_error) - it 'removes the exported project file after the upload' do - allow(strategy).to receive(:send_file) - allow(strategy).to receive(:handle_response_error) + expect(project).to receive(:remove_exports) - expect(project).to receive(:remove_exported_project_file) - - strategy.execute(user, project) - end + strategy.execute(user, project) end end end diff --git a/spec/lib/gitlab/import_export/avatar_saver_spec.rb b/spec/lib/gitlab/import_export/avatar_saver_spec.rb index 90e6d653d34..2bd1b9924c6 100644 --- a/spec/lib/gitlab/import_export/avatar_saver_spec.rb +++ b/spec/lib/gitlab/import_export/avatar_saver_spec.rb @@ -8,8 +8,7 @@ describe Gitlab::ImportExport::AvatarSaver do before do FileUtils.mkdir_p("#{shared.export_path}/avatar/") - allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - stub_feature_flags(import_export_object_storage: false) + allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:export_path).and_return(export_path) end after do @@ -19,7 +18,7 @@ describe Gitlab::ImportExport::AvatarSaver do it 'saves a project avatar' do described_class.new(project: project_with_avatar, shared: shared).save - expect(File).to exist("#{shared.export_path}/avatar/dk.png") + expect(File).to exist(Dir["#{shared.export_path}/avatar/**/dk.png"].first) end it 'is fine not to have an avatar' do diff --git a/spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb b/spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb deleted file mode 100644 index 287745eb40e..00000000000 --- a/spec/lib/gitlab/import_export/file_importer_object_storage_spec.rb +++ /dev/null @@ -1,89 +0,0 @@ -require 'spec_helper' - -describe Gitlab::ImportExport::FileImporter do - let(:shared) { Gitlab::ImportExport::Shared.new(nil) } - let(:storage_path) { "#{Dir.tmpdir}/file_importer_spec" } - let(:valid_file) { "#{shared.export_path}/valid.json" } - let(:symlink_file) { "#{shared.export_path}/invalid.json" } - let(:hidden_symlink_file) { "#{shared.export_path}/.hidden" } - let(:subfolder_symlink_file) { "#{shared.export_path}/subfolder/invalid.json" } - let(:evil_symlink_file) { "#{shared.export_path}/.\nevil" } - - before do - stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0) - stub_feature_flags(import_export_object_storage: true) - stub_uploads_object_storage(FileUploader) - - allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(storage_path) - allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true) - allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test') - allow(SecureRandom).to receive(:hex).and_return('abcd') - setup_files - end - - after do - FileUtils.rm_rf(storage_path) - end - - context 'normal run' do - before do - described_class.import(project: build(:project), archive_file: '', shared: shared) - end - - it 'removes symlinks in root folder' do - expect(File.exist?(symlink_file)).to be false - end - - it 'removes hidden symlinks in root folder' do - expect(File.exist?(hidden_symlink_file)).to be false - end - - it 'removes evil symlinks in root folder' do - expect(File.exist?(evil_symlink_file)).to be false - end - - it 'removes symlinks in subfolders' do - expect(File.exist?(subfolder_symlink_file)).to be false - end - - it 'does not remove a valid file' do - expect(File.exist?(valid_file)).to be true - end - - it 'creates the file in the right subfolder' do - expect(shared.export_path).to include('test/abcd') - end - end - - context 'error' do - before do - allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError) - described_class.import(project: build(:project), archive_file: '', shared: shared) - end - - it 'removes symlinks in root folder' do - expect(File.exist?(symlink_file)).to be false - end - - it 'removes hidden symlinks in root folder' do - expect(File.exist?(hidden_symlink_file)).to be false - end - - it 'removes symlinks in subfolders' do - expect(File.exist?(subfolder_symlink_file)).to be false - end - - it 'does not remove a valid file' do - expect(File.exist?(valid_file)).to be true - end - end - - def setup_files - FileUtils.mkdir_p("#{shared.export_path}/subfolder/") - FileUtils.touch(valid_file) - FileUtils.ln_s(valid_file, symlink_file) - FileUtils.ln_s(valid_file, subfolder_symlink_file) - FileUtils.ln_s(valid_file, hidden_symlink_file) - FileUtils.ln_s(valid_file, evil_symlink_file) - end -end diff --git a/spec/lib/gitlab/import_export/file_importer_spec.rb b/spec/lib/gitlab/import_export/file_importer_spec.rb index 78fccdf1dfc..bf34cefe18f 100644 --- a/spec/lib/gitlab/import_export/file_importer_spec.rb +++ b/spec/lib/gitlab/import_export/file_importer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' describe Gitlab::ImportExport::FileImporter do let(:shared) { Gitlab::ImportExport::Shared.new(nil) } - let(:export_path) { "#{Dir.tmpdir}/file_importer_spec" } + let(:storage_path) { "#{Dir.tmpdir}/file_importer_spec" } let(:valid_file) { "#{shared.export_path}/valid.json" } let(:symlink_file) { "#{shared.export_path}/invalid.json" } let(:hidden_symlink_file) { "#{shared.export_path}/.hidden" } @@ -11,7 +11,9 @@ describe Gitlab::ImportExport::FileImporter do before do stub_const('Gitlab::ImportExport::FileImporter::MAX_RETRIES', 0) - allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) + stub_uploads_object_storage(FileUploader) + + allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(storage_path) allow_any_instance_of(Gitlab::ImportExport::CommandLineUtil).to receive(:untar_zxf).and_return(true) allow_any_instance_of(Gitlab::ImportExport::Shared).to receive(:relative_archive_path).and_return('test') allow(SecureRandom).to receive(:hex).and_return('abcd') @@ -19,12 +21,12 @@ describe Gitlab::ImportExport::FileImporter do end after do - FileUtils.rm_rf(export_path) + FileUtils.rm_rf(storage_path) end context 'normal run' do before do - described_class.import(project: nil, archive_file: '', shared: shared) + described_class.import(project: build(:project), archive_file: '', shared: shared) end it 'removes symlinks in root folder' do @@ -55,7 +57,7 @@ describe Gitlab::ImportExport::FileImporter do context 'error' do before do allow_any_instance_of(described_class).to receive(:wait_for_archived_file).and_raise(StandardError) - described_class.import(project: nil, archive_file: '', shared: shared) + described_class.import(project: build(:project), archive_file: '', shared: shared) end it 'removes symlinks in root folder' do diff --git a/spec/lib/gitlab/import_export/importer_object_storage_spec.rb b/spec/lib/gitlab/import_export/importer_object_storage_spec.rb deleted file mode 100644 index 24a994b3611..00000000000 --- a/spec/lib/gitlab/import_export/importer_object_storage_spec.rb +++ /dev/null @@ -1,115 +0,0 @@ -require 'spec_helper' - -describe Gitlab::ImportExport::Importer do - let(:user) { create(:user) } - let(:test_path) { "#{Dir.tmpdir}/importer_spec" } - let(:shared) { project.import_export_shared } - let(:project) { create(:project) } - let(:import_file) { fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz') } - - subject(:importer) { described_class.new(project) } - - before do - allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path) - allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file) - stub_feature_flags(import_export_object_storage: true) - stub_uploads_object_storage(FileUploader) - - FileUtils.mkdir_p(shared.export_path) - ImportExportUpload.create(project: project, import_file: import_file) - end - - after do - FileUtils.rm_rf(test_path) - end - - describe '#execute' do - it 'succeeds' do - importer.execute - - expect(shared.errors).to be_empty - end - - it 'extracts the archive' do - expect(Gitlab::ImportExport::FileImporter).to receive(:import).and_call_original - - importer.execute - end - - it 'checks the version' do - expect(Gitlab::ImportExport::VersionChecker).to receive(:check!).and_call_original - - importer.execute - end - - context 'all restores are executed' do - [ - Gitlab::ImportExport::AvatarRestorer, - Gitlab::ImportExport::RepoRestorer, - Gitlab::ImportExport::WikiRestorer, - Gitlab::ImportExport::UploadsRestorer, - Gitlab::ImportExport::LfsRestorer, - Gitlab::ImportExport::StatisticsRestorer - ].each do |restorer| - it "calls the #{restorer}" do - fake_restorer = double(restorer.to_s) - - expect(fake_restorer).to receive(:restore).and_return(true).at_least(1) - expect(restorer).to receive(:new).and_return(fake_restorer).at_least(1) - - importer.execute - end - end - - it 'restores the ProjectTree' do - expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original - - importer.execute - end - - it 'removes the import file' do - expect(importer).to receive(:remove_import_file).and_call_original - - importer.execute - - expect(project.import_export_upload.import_file&.file).to be_nil - end - end - - context 'when project successfully restored' do - let!(:existing_project) { create(:project, namespace: user.namespace) } - let(:project) { create(:project, namespace: user.namespace, name: 'whatever', path: 'whatever') } - - before do - restorers = double(:restorers, all?: true) - - allow(subject).to receive(:import_file).and_return(true) - allow(subject).to receive(:check_version!).and_return(true) - allow(subject).to receive(:restorers).and_return(restorers) - allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path })) - end - - context 'when import_data' do - context 'has original_path' do - it 'overwrites existing project' do - expect_any_instance_of(::Projects::OverwriteProjectService).to receive(:execute).with(existing_project) - - subject.execute - end - end - - context 'has not original_path' do - before do - allow(project).to receive(:import_data).and_return(double(data: {})) - end - - it 'does not call the overwrite service' do - expect_any_instance_of(::Projects::OverwriteProjectService).not_to receive(:execute).with(existing_project) - - subject.execute - end - end - end - end - end -end diff --git a/spec/lib/gitlab/import_export/importer_spec.rb b/spec/lib/gitlab/import_export/importer_spec.rb index 8053c48ad6c..11f98d782b1 100644 --- a/spec/lib/gitlab/import_export/importer_spec.rb +++ b/spec/lib/gitlab/import_export/importer_spec.rb @@ -4,16 +4,18 @@ describe Gitlab::ImportExport::Importer do let(:user) { create(:user) } let(:test_path) { "#{Dir.tmpdir}/importer_spec" } let(:shared) { project.import_export_shared } - let(:project) { create(:project, import_source: File.join(test_path, 'test_project_export.tar.gz')) } + let(:project) { create(:project) } + let(:import_file) { fixture_file_upload('spec/features/projects/import_export/test_project_export.tar.gz') } subject(:importer) { described_class.new(project) } before do allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path) allow_any_instance_of(Gitlab::ImportExport::FileImporter).to receive(:remove_import_file) + stub_uploads_object_storage(FileUploader) FileUtils.mkdir_p(shared.export_path) - FileUtils.cp(Rails.root.join('spec/features/projects/import_export/test_project_export.tar.gz'), test_path) + ImportExportUpload.create(project: project, import_file: import_file) end after do @@ -64,6 +66,14 @@ describe Gitlab::ImportExport::Importer do importer.execute end + it 'removes the import file' do + expect(importer).to receive(:remove_import_file).and_call_original + + importer.execute + + expect(project.import_export_upload.import_file&.file).to be_nil + end + it 'sets the correct visibility_level when visibility level is a string' do project.create_or_update_import_data( data: { override_params: { visibility_level: Gitlab::VisibilityLevel::PRIVATE.to_s } } @@ -85,7 +95,6 @@ describe Gitlab::ImportExport::Importer do allow(subject).to receive(:import_file).and_return(true) allow(subject).to receive(:check_version!).and_return(true) allow(subject).to receive(:restorers).and_return(restorers) - allow(restorers).to receive(:all?).and_return(true) allow(project).to receive(:import_data).and_return(double(data: { 'original_path' => existing_project.path })) end diff --git a/spec/lib/gitlab/import_export/saver_spec.rb b/spec/lib/gitlab/import_export/saver_spec.rb index 02f1a4b81aa..d185ff2dfcc 100644 --- a/spec/lib/gitlab/import_export/saver_spec.rb +++ b/spec/lib/gitlab/import_export/saver_spec.rb @@ -18,26 +18,12 @@ describe Gitlab::ImportExport::Saver do FileUtils.rm_rf(export_path) end - context 'local archive' do - it 'saves the repo to disk' do - stub_feature_flags(import_export_object_storage: false) + it 'saves the repo using object storage' do + stub_uploads_object_storage(ImportExportUploader) - subject.save + subject.save - expect(shared.errors).to be_empty - expect(Dir.empty?(shared.archive_path)).to be false - end - end - - context 'object storage' do - it 'saves the repo using object storage' do - stub_feature_flags(import_export_object_storage: true) - stub_uploads_object_storage(ImportExportUploader) - - subject.save - - expect(ImportExportUpload.find_by(project: project).export_file.url) - .to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*]) - end + expect(ImportExportUpload.find_by(project: project).export_file.url) + .to match(%r[\/uploads\/-\/system\/import_export_upload\/export_file.*]) end end diff --git a/spec/lib/gitlab/import_export/uploads_manager_spec.rb b/spec/lib/gitlab/import_export/uploads_manager_spec.rb index f799de18cd0..792117e1df1 100644 --- a/spec/lib/gitlab/import_export/uploads_manager_spec.rb +++ b/spec/lib/gitlab/import_export/uploads_manager_spec.rb @@ -4,6 +4,7 @@ describe Gitlab::ImportExport::UploadsManager do let(:shared) { project.import_export_shared } let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } let(:project) { create(:project) } + let(:upload) { create(:upload, :issuable_upload, :object_storage, model: project) } let(:exported_file_path) { "#{shared.export_path}/uploads/#{upload.secret}/#{File.basename(upload.path)}" } subject(:manager) { described_class.new(project: project, shared: shared) } @@ -69,44 +70,20 @@ describe Gitlab::ImportExport::UploadsManager do end end end + end - context 'using object storage' do - let!(:upload) { create(:upload, :issuable_upload, :object_storage, model: project) } - - before do - stub_feature_flags(import_export_object_storage: true) - stub_uploads_object_storage(FileUploader) - end - - it 'saves the file' do - fake_uri = double - - expect(fake_uri).to receive(:open).and_return(StringIO.new('File content')) - expect(URI).to receive(:parse).and_return(fake_uri) - - manager.save + describe '#restore' do + before do + stub_uploads_object_storage(FileUploader) - expect(File.read(exported_file_path)).to eq('File content') - end + FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038')) + FileUtils.touch(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038', "dummy.txt")) end - describe '#restore' do - context 'using object storage' do - before do - stub_feature_flags(import_export_object_storage: true) - stub_uploads_object_storage(FileUploader) - - FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038')) - FileUtils.touch(File.join(shared.export_path, 'uploads/72a497a02fe3ee09edae2ed06d390038', "dummy.txt")) - end + it 'restores the file' do + manager.restore - it 'restores the file' do - manager.restore - - expect(project.uploads.size).to eq(1) - expect(project.uploads.first.build_uploader.filename).to eq('dummy.txt') - end - end + expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt') end end end diff --git a/spec/lib/gitlab/import_export/uploads_restorer_spec.rb b/spec/lib/gitlab/import_export/uploads_restorer_spec.rb index acef97459b8..6072f18b8c7 100644 --- a/spec/lib/gitlab/import_export/uploads_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/uploads_restorer_spec.rb @@ -8,7 +8,7 @@ describe Gitlab::ImportExport::UploadsRestorer do before do allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) FileUtils.mkdir_p(File.join(shared.export_path, 'uploads/random')) - FileUtils.touch(File.join(shared.export_path, 'uploads/random', "dummy.txt")) + FileUtils.touch(File.join(shared.export_path, 'uploads/random', 'dummy.txt')) end after do @@ -27,9 +27,7 @@ describe Gitlab::ImportExport::UploadsRestorer do it 'copies the uploads to the project path' do subject.restore - uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) } - - expect(uploads).to include('dummy.txt') + expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt') end end @@ -45,9 +43,7 @@ describe Gitlab::ImportExport::UploadsRestorer do it 'copies the uploads to the project path' do subject.restore - uploads = Dir.glob(File.join(subject.uploads_path, '**/*')).map { |file| File.basename(file) } - - expect(uploads).to include('dummy.txt') + expect(project.uploads.map { |u| u.build_uploader.filename }).to include('dummy.txt') end end end diff --git a/spec/lib/gitlab/import_export/uploads_saver_spec.rb b/spec/lib/gitlab/import_export/uploads_saver_spec.rb index c716edd9397..24993460e51 100644 --- a/spec/lib/gitlab/import_export/uploads_saver_spec.rb +++ b/spec/lib/gitlab/import_export/uploads_saver_spec.rb @@ -7,7 +7,6 @@ describe Gitlab::ImportExport::UploadsSaver do let(:shared) { project.import_export_shared } before do - stub_feature_flags(import_export_object_storage: false) allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 9b7f932ec3a..3649990670b 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -394,12 +394,6 @@ describe Namespace do child.destroy end end - - it 'removes the exports folder' do - expect(namespace).to receive(:remove_exports!) - - namespace.destroy - end end context 'hashed storage' do @@ -414,12 +408,6 @@ describe Namespace do expect(File.exist?(deleted_path_in_dir)).to be(false) end - - it 'removes the exports folder' do - expect(namespace).to receive(:remove_exports!) - - namespace.destroy - end end end @@ -706,26 +694,6 @@ describe Namespace do end end - describe '#remove_exports' do - let(:legacy_project) { create(:project, :with_export, :legacy_storage, namespace: namespace) } - let(:hashed_project) { create(:project, :with_export, namespace: namespace) } - let(:export_path) { Dir.mktmpdir('namespace_remove_exports_spec') } - let(:legacy_export) { legacy_project.export_project_path } - let(:hashed_export) { hashed_project.export_project_path } - - it 'removes exports for legacy and hashed projects' do - allow(Gitlab::ImportExport).to receive(:storage_path) { export_path } - - expect(File.exist?(legacy_export)).to be_truthy - expect(File.exist?(hashed_export)).to be_truthy - - namespace.remove_exports! - - expect(File.exist?(legacy_export)).to be_falsy - expect(File.exist?(hashed_export)).to be_falsy - end - end - describe '#full_path_was' do context 'when the group has no parent' do it 'should return the path was' do diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 264632dba4b..809a21e80ee 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2854,73 +2854,12 @@ describe Project do end describe '#remove_export' do - let(:legacy_project) { create(:project, :legacy_storage, :with_export) } let(:project) { create(:project, :with_export) } - before do - stub_feature_flags(import_export_object_storage: false) - end - - it 'removes the exports directory for the project' do - expect(File.exist?(project.export_path)).to be_truthy - - allow(FileUtils).to receive(:rm_rf).and_call_original - expect(FileUtils).to receive(:rm_rf).with(project.export_path).and_call_original + it 'removes the export' do project.remove_exports - expect(File.exist?(project.export_path)).to be_falsy - end - - it 'is a no-op on legacy projects when there is no namespace' do - export_path = legacy_project.export_path - - legacy_project.namespace.delete - legacy_project.reload - - expect(FileUtils).not_to receive(:rm_rf).with(export_path) - - legacy_project.remove_exports - - expect(File.exist?(export_path)).to be_truthy - end - - it 'runs on hashed storage projects when there is no namespace' do - export_path = project.export_path - - project.namespace.delete - legacy_project.reload - - allow(FileUtils).to receive(:rm_rf).and_call_original - expect(FileUtils).to receive(:rm_rf).with(export_path).and_call_original - - project.remove_exports - - expect(File.exist?(export_path)).to be_falsy - end - - it 'is run when the project is destroyed' do - expect(project).to receive(:remove_exports).and_call_original - - project.destroy - end - end - - describe '#remove_exported_project_file' do - let(:project) { create(:project, :with_export) } - - it 'removes the exported project file' do - stub_feature_flags(import_export_object_storage: false) - - exported_file = project.export_project_path - - expect(File.exist?(exported_file)).to be_truthy - - allow(FileUtils).to receive(:rm_rf).and_call_original - expect(FileUtils).to receive(:rm_rf).with(exported_file).and_call_original - - project.remove_exported_project_file - - expect(File.exist?(exported_file)).to be_falsy + expect(project.export_project_object_exists?).to be_falsey end end diff --git a/spec/requests/api/project_export_spec.rb b/spec/requests/api/project_export_spec.rb index 45e4e35d773..0d4854a83b9 100644 --- a/spec/requests/api/project_export_spec.rb +++ b/spec/requests/api/project_export_spec.rb @@ -4,8 +4,8 @@ describe API::ProjectExport do set(:project) { create(:project) } set(:project_none) { create(:project) } set(:project_started) { create(:project) } - set(:project_finished) { create(:project) } - set(:project_after_export) { create(:project) } + let(:project_finished) { create(:project, :with_export) } + let(:project_after_export) { create(:project, :with_export) } set(:user) { create(:user) } set(:admin) { create(:admin) } @@ -29,13 +29,7 @@ describe API::ProjectExport do # simulate exporting work directory FileUtils.mkdir_p File.join(project_started.export_path, 'securerandom-hex') - # simulate exported - FileUtils.mkdir_p project_finished.export_path - FileUtils.touch File.join(project_finished.export_path, '_export.tar.gz') - # simulate in after export action - FileUtils.mkdir_p project_after_export.export_path - FileUtils.touch File.join(project_after_export.export_path, '_export.tar.gz') FileUtils.touch Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project_after_export) end @@ -191,14 +185,11 @@ describe API::ProjectExport do context 'when upload complete' do before do - FileUtils.rm_rf(project_after_export.export_path) - - if project_after_export.export_project_object_exists? - upload = project_after_export.import_export_upload + project_after_export.remove_exports + end - upload.remove_export_file! - upload.save - end + it 'has removed the export' do + expect(project_after_export.export_project_object_exists?).to be_falsey end it_behaves_like '404 response' do @@ -273,13 +264,13 @@ describe API::ProjectExport do before do stub_uploads_object_storage(ImportExportUploader) - [project, project_finished, project_after_export].each do |p| - p.add_maintainer(user) + project.add_maintainer(user) + project_finished.add_maintainer(user) + project_after_export.add_maintainer(user) - upload = ImportExportUpload.new(project: p) - upload.export_file = fixture_file_upload('spec/fixtures/project_export.tar.gz', "`/tar.gz") - upload.save! - end + upload = ImportExportUpload.new(project: project) + upload.export_file = fixture_file_upload('spec/fixtures/project_export.tar.gz', "`/tar.gz") + upload.save! end it_behaves_like 'get project download by strategy' diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index bc06f3c3732..c8fa4754810 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -7,7 +7,6 @@ describe API::ProjectImport do let(:namespace) { create(:group) } before do allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - stub_feature_flags(import_export_object_storage: true) stub_uploads_object_storage(FileUploader) namespace.add_owner(user) diff --git a/spec/support/import_export/export_file_helper.rb b/spec/support/import_export/export_file_helper.rb index 4d925ac77f4..cbe238e20ea 100644 --- a/spec/support/import_export/export_file_helper.rb +++ b/spec/support/import_export/export_file_helper.rb @@ -52,7 +52,7 @@ module ExportFileHelper # Expands the compressed file for an exported project into +tmpdir+ def in_directory_with_expanded_export(project) Dir.mktmpdir do |tmpdir| - export_file = project.export_project_path + export_file = project.import_export_upload.export_file.path _output, exit_status = Gitlab::Popen.popen(%W{tar -zxf #{export_file} -C #{tmpdir}}) yield(exit_status, tmpdir) |