diff options
author | Douwe Maan <douwe@gitlab.com> | 2018-03-30 18:46:00 +0300 |
---|---|---|
committer | Douwe Maan <douwe@gitlab.com> | 2018-03-30 18:46:00 +0300 |
commit | 96cd05e65ef6b3b98734f041a43fc8cc81dba9b5 (patch) | |
tree | e2e1cff25e9e4ab67252b0402cb5df95fdc98d25 | |
parent | 7c36e8561c60882e6b0b47c563f7d19f3d6b02a6 (diff) | |
parent | 22b05a1ff74d4f64490f93995259602b3d07c3cf (diff) |
Merge branch 'fj-42685-extend-project-export-endpoint' into 'master'
Extend API for exporting a project with direct upload URL
Closes #42685
See merge request gitlab-org/gitlab-ce!17686
19 files changed, 671 insertions, 25 deletions
diff --git a/app/models/project.rb b/app/models/project.rb index da03080f440..b343786d2c9 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1544,8 +1544,8 @@ class Project < ActiveRecord::Base @errors = original_errors end - def add_export_job(current_user:, params: {}) - job_id = ProjectExportWorker.perform_async(current_user.id, self.id, params) + def add_export_job(current_user:, after_export_strategy: nil, params: {}) + job_id = ProjectExportWorker.perform_async(current_user.id, self.id, after_export_strategy, params) if job_id Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}" @@ -1571,6 +1571,8 @@ class Project < ActiveRecord::Base def export_status if export_in_progress? :started + elsif after_export_in_progress? + :after_export_action elsif export_project_path :finished else @@ -1582,12 +1584,22 @@ class Project < ActiveRecord::Base import_export_shared.active_export_count > 0 end + def after_export_in_progress? + import_export_shared.after_export_in_progress? + end + def remove_exports return nil unless export_path.present? FileUtils.rm_rf(export_path) end + def remove_exported_project_file + return unless export_project_path.present? + + FileUtils.rm_f(export_project_path) + end + def full_path_slug Gitlab::Utils.slugify(full_path.to_s) end diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index d16aa3de639..402cddd3ec1 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -1,22 +1,36 @@ module Projects module ImportExport class ExportService < BaseService - def execute(_options = {}) + def execute(after_export_strategy = nil, options = {}) @shared = project.import_export_shared - save_all + + save_all! + execute_after_export_action(after_export_strategy) end private - def save_all - if [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save) + def execute_after_export_action(after_export_strategy) + return unless after_export_strategy + + unless after_export_strategy.execute(current_user, project) + cleanup_and_notify_error + end + end + + def save_all! + if save_services Gitlab::ImportExport::Saver.save(project: project, shared: @shared) notify_success else - cleanup_and_notify + cleanup_and_notify_error! end end + def save_services + [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save) + end + def version_saver Gitlab::ImportExport::VersionSaver.new(shared: @shared) end @@ -41,19 +55,22 @@ module Projects Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared) end - def cleanup_and_notify + def cleanup_and_notify_error Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}") FileUtils.rm_rf(@shared.export_path) notify_error + end + + def cleanup_and_notify_error! + cleanup_and_notify_error + raise Gitlab::ImportExport::Error.new(@shared.errors.join(', ')) end def notify_success Rails.logger.info("Import/Export - Project #{project.name} with ID: #{project.id} successfully exported") - - notification_service.project_exported(@project, @current_user) end def notify_error diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index 0b502143e5d..c3d84bb0b93 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -4,11 +4,19 @@ class ProjectExportWorker sidekiq_options retry: 3 - def perform(current_user_id, project_id, params = {}) - params = params.with_indifferent_access + def perform(current_user_id, project_id, after_export_strategy = {}, params = {}) current_user = User.find(current_user_id) project = Project.find(project_id) + after_export = build!(after_export_strategy) - ::Projects::ImportExport::ExportService.new(project, current_user, params).execute + ::Projects::ImportExport::ExportService.new(project, current_user, params).execute(after_export) + end + + private + + def build!(after_export_strategy) + strategy_klass = after_export_strategy&.delete('klass') + + Gitlab::ImportExport::AfterExportStrategyBuilder.build!(strategy_klass, after_export_strategy) end end diff --git a/changelogs/unreleased/fj-42685-extend-project-export-endpoint.yml b/changelogs/unreleased/fj-42685-extend-project-export-endpoint.yml new file mode 100644 index 00000000000..a06499d821a --- /dev/null +++ b/changelogs/unreleased/fj-42685-extend-project-export-endpoint.yml @@ -0,0 +1,5 @@ +--- +title: Extend API for exporting a project with direct upload URL +merge_request: 17686 +author: +type: added diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index de5207fc5e4..5467187788a 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -8,6 +8,14 @@ Start a new export. +The endpoint also accepts an `upload` param. This param is a hash that contains +all the necessary information to upload the exported project to a web server or +to any S3-compatible platform. At the moment we only support binary +data file uploads to the final server. + +If the `upload` params is present, `upload[url]` param is required. + (**Note:** This feature was introduced in GitLab 10.7) + ```http POST /projects/:id/export ``` @@ -16,9 +24,12 @@ POST /projects/:id/export | --------- | -------------- | -------- | ---------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | | `description` | string | no | Overrides the project description | +| `upload` | hash | no | Hash that contains the information to upload the exported project to a web server | +| `upload[url]` | string | yes | The URL to upload the project | +| `upload[http_method]` | string | no | The HTTP method to upload the exported project. Only `PUT` and `POST` methods allowed. Default is `PUT` | ```console -curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form "description=Foo Bar" https://gitlab.example.com/api/v4/projects/1/export +curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/export --data "description=FooBar&upload[http_method]=PUT&upload[url]=https://example-bucket.s3.eu-west-3.amazonaws.com/backup?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIMBJHN2O62W8IELQ%2F20180312%2Feu-west-3%2Fs3%2Faws4_request&X-Amz-Date=20180312T110328Z&X-Amz-Expires=900&X-Amz-SignedHeaders=host&X-Amz-Signature=8413facb20ff33a49a147a0b4abcff4c8487cc33ee1f7e450c46e8f695569dbd" ``` ```json @@ -43,7 +54,11 @@ GET /projects/:id/export curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/export ``` -Status can be one of `none`, `started`, or `finished`. +Status can be one of `none`, `started`, `after_export_action` or `finished`. The +`after_export_action` state represents that the export process has been completed successfully and +the platform is performing some actions on the resulted file. For example, sending +an email notifying the user to download the file, uploading the exported file +to a web server, etc. `_links` are only present when export has finished. diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index efc4a33ae1b..5ef4e9d530c 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -33,11 +33,28 @@ module API end params do optional :description, type: String, desc: 'Override the project description' + optional :upload, type: Hash do + optional :url, type: String, desc: 'The URL to upload the project' + optional :http_method, type: String, default: 'PUT', desc: 'HTTP method to upload the exported project' + end end post ':id/export' do project_export_params = declared_params(include_missing: false) + after_export_params = project_export_params.delete(:upload) || {} - user_project.add_export_job(current_user: current_user, params: project_export_params) + export_strategy = if after_export_params[:url].present? + params = after_export_params.slice(:url, :http_method).symbolize_keys + + Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(params) + end + + if export_strategy&.invalid? + render_validation_error!(export_strategy) + else + user_project.add_export_job(current_user: current_user, + after_export_strategy: export_strategy, + params: project_export_params) + end accepted! end diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb new file mode 100644 index 00000000000..aef371d81eb --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -0,0 +1,83 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class BaseAfterExportStrategy + include ActiveModel::Validations + extend Forwardable + + StrategyError = Class.new(StandardError) + + AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action'.freeze + + private + + attr_reader :project, :current_user + + public + + def initialize(attributes = {}) + @options = OpenStruct.new(attributes) + + self.class.instance_eval do + def_delegators :@options, *attributes.keys + end + end + + def execute(current_user, project) + return unless project&.export_project_path + + @project = project + @current_user = current_user + + if invalid? + log_validation_errors + + return + end + + create_or_update_after_export_lock + strategy_execute + + true + rescue => e + project.import_export_shared.error(e) + false + ensure + delete_after_export_lock + end + + def to_json(options = {}) + @options.to_h.merge!(klass: self.class.name).to_json + end + + def self.lock_file_path(project) + return unless project&.export_path + + File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME) + end + + protected + + def strategy_execute + raise NotImplementedError + end + + private + + def create_or_update_after_export_lock + FileUtils.touch(self.class.lock_file_path(project)) + end + + def delete_after_export_lock + lock_file = self.class.lock_file_path(project) + + FileUtils.rm(lock_file) if lock_file.present? && File.exist?(lock_file) + end + + def log_validation_errors + errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb new file mode 100644 index 00000000000..4371a7eff56 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb @@ -0,0 +1,17 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class DownloadNotificationStrategy < BaseAfterExportStrategy + private + + def strategy_execute + notification_service.project_exported(project, current_user) + end + + def notification_service + @notification_service ||= NotificationService.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb new file mode 100644 index 00000000000..938664a95a1 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -0,0 +1,61 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class WebUploadStrategy < BaseAfterExportStrategy + PUT_METHOD = 'PUT'.freeze + POST_METHOD = 'POST'.freeze + INVALID_HTTP_METHOD = 'invalid. Only PUT and POST methods allowed.'.freeze + + validates :url, url: true + + validate do + unless [PUT_METHOD, POST_METHOD].include?(http_method.upcase) + errors.add(:http_method, INVALID_HTTP_METHOD) + end + end + + def initialize(url:, http_method: PUT_METHOD) + super + end + + protected + + def strategy_execute + handle_response_error(send_file) + + project.remove_exported_project_file + end + + def handle_response_error(response) + unless response.success? + error_code = response.dig('Error', 'Code') || response.code + error_message = response.dig('Error', 'Message') || response.message + + raise StrategyError.new("Error uploading the project. Code #{error_code}: #{error_message}") + end + end + + private + + def send_file + export_file = File.open(project.export_project_path) + + Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend + ensure + export_file.close if export_file + end + + def send_file_options(export_file) + { + body_stream: export_file, + headers: headers + } + end + + def headers + { 'Content-Length' => File.size(project.export_project_path).to_s } + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategy_builder.rb b/lib/gitlab/import_export/after_export_strategy_builder.rb new file mode 100644 index 00000000000..7eabcae2380 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategy_builder.rb @@ -0,0 +1,24 @@ +module Gitlab + module ImportExport + class AfterExportStrategyBuilder + StrategyNotFoundError = Class.new(StandardError) + + def self.build!(strategy_klass, attributes = {}) + return default_strategy.new unless strategy_klass + + attributes ||= {} + klass = strategy_klass.constantize rescue nil + + unless klass && klass < AfterExportStrategies::BaseAfterExportStrategy + raise StrategyNotFoundError.new("Strategy #{strategy_klass} not found") + end + + klass.new(**attributes.symbolize_keys) + end + + def self.default_strategy + AfterExportStrategies::DownloadNotificationStrategy + end + end + end +end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 3d3d998a6a3..6d7c36ce38b 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -22,7 +22,7 @@ module Gitlab def error(error) error_out(error.message, caller[0].dup) - @errors << error.message + add_error_message(error.message) # Debug: if error.backtrace @@ -32,6 +32,14 @@ module Gitlab end end + def add_error_message(error_message) + @errors << error_message + end + + def after_export_in_progress? + File.exist?(after_export_lock_file) + end + private def relative_path @@ -45,6 +53,10 @@ module Gitlab def error_out(message, caller) Rails.logger.error("Import/Export error raised on #{caller}: #{message}") end + + def after_export_lock_file + AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project) + end end end end diff --git a/spec/fixtures/api/schemas/public_api/v4/project/export_status.json b/spec/fixtures/api/schemas/public_api/v4/project/export_status.json index d24a6f93f4b..81c8815caf6 100644 --- a/spec/fixtures/api/schemas/public_api/v4/project/export_status.json +++ b/spec/fixtures/api/schemas/public_api/v4/project/export_status.json @@ -1,7 +1,9 @@ { "type": "object", "allOf": [ - { "$ref": "identity.json" }, + { + "$ref": "identity.json" + }, { "required": [ "export_status" @@ -9,7 +11,12 @@ "properties": { "export_status": { "type": "string", - "enum": ["none", "started", "finished"] + "enum": [ + "none", + "started", + "finished", + "after_export_action" + ] } } } diff --git a/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb new file mode 100644 index 00000000000..ed54d87de4a --- /dev/null +++ b/spec/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy_spec.rb @@ -0,0 +1,104 @@ +require 'spec_helper' + +describe Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy do + let!(:service) { described_class.new } + let!(:project) { create(:project, :with_export) } + let(:shared) { project.import_export_shared } + let!(:user) { create(:user) } + + describe '#execute' do + before do + allow(service).to receive(:strategy_execute) + end + + it 'returns if project exported file is not found' do + allow(project).to receive(:export_project_path).and_return(nil) + + expect(service).not_to receive(:strategy_execute) + + service.execute(user, project) + end + + it 'creates a lock file in the export dir' do + allow(service).to receive(:delete_after_export_lock) + + service.execute(user, project) + + expect(lock_path_exist?).to be_truthy + end + + context 'when the method succeeds' do + it 'removes the lock file' do + service.execute(user, project) + + expect(lock_path_exist?).to be_falsey + end + end + + context 'when the method fails' do + before do + allow(service).to receive(:strategy_execute).and_call_original + end + + context 'when validation fails' do + before do + allow(service).to receive(:invalid?).and_return(true) + end + + it 'does not create the lock file' do + expect(service).not_to receive(:create_or_update_after_export_lock) + + service.execute(user, project) + end + + it 'does not execute main logic' do + expect(service).not_to receive(:strategy_execute) + + service.execute(user, project) + end + + it 'logs validation errors in shared context' do + expect(service).to receive(:log_validation_errors) + + service.execute(user, project) + end + end + + context 'when an exception is raised' do + it 'removes the lock' do + expect { service.execute(user, project) }.to raise_error(NotImplementedError) + + expect(lock_path_exist?).to be_falsey + end + end + end + end + + describe '#log_validation_errors' do + it 'add the message to the shared context' do + errors = %w(test_message test_message2) + + allow(service).to receive(:invalid?).and_return(true) + allow(service.errors).to receive(:full_messages).and_return(errors) + + expect(shared).to receive(:add_error_message).twice.and_call_original + + service.execute(user, project) + + expect(shared.errors).to eq errors + end + end + + describe '#to_json' do + it 'adds the current strategy class to the serialized attributes' do + params = { param1: 1 } + result = params.merge(klass: described_class.to_s).to_json + + expect(described_class.new(params).to_json).to eq result + end + end + + def lock_path_exist? + File.exist?(described_class.lock_file_path(project)) + end +end diff --git a/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb new file mode 100644 index 00000000000..5fe57d9987b --- /dev/null +++ b/spec/lib/gitlab/import_export/after_export_strategies/web_upload_strategy_spec.rb @@ -0,0 +1,36 @@ +require 'spec_helper' + +describe Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy do + let(:example_url) { 'http://www.example.com' } + let(:strategy) { subject.new(url: example_url, http_method: 'post') } + let!(:project) { create(:project, :with_export) } + let!(:user) { build(:user) } + + subject { described_class } + + describe 'validations' do + it 'only POST and PUT method allowed' do + %w(POST post PUT put).each do |method| + expect(subject.new(url: example_url, http_method: method)).to be_valid + end + + expect(subject.new(url: example_url, http_method: 'whatever')).not_to be_valid + end + + it 'onyl allow urls as upload urls' do + expect(subject.new(url: example_url)).to be_valid + expect(subject.new(url: 'whatever')).not_to be_valid + end + end + + describe '#execute' do + it 'removes the exported project file after the upload' do + allow(strategy).to receive(:send_file) + allow(strategy).to receive(:handle_response_error) + + expect(project).to receive(:remove_exported_project_file) + + strategy.execute(user, project) + end + end +end diff --git a/spec/lib/gitlab/import_export/after_export_strategy_builder_spec.rb b/spec/lib/gitlab/import_export/after_export_strategy_builder_spec.rb new file mode 100644 index 00000000000..bf727285a9f --- /dev/null +++ b/spec/lib/gitlab/import_export/after_export_strategy_builder_spec.rb @@ -0,0 +1,29 @@ +require 'spec_helper' + +describe Gitlab::ImportExport::AfterExportStrategyBuilder do + let!(:strategies_namespace) { 'Gitlab::ImportExport::AfterExportStrategies' } + + describe '.build!' do + context 'when klass param is' do + it 'null it returns the default strategy' do + expect(described_class.build!(nil).class).to eq described_class.default_strategy + end + + it 'not a valid class it raises StrategyNotFoundError exception' do + expect { described_class.build!('Whatever') }.to raise_error(described_class::StrategyNotFoundError) + end + + it 'not a descendant of AfterExportStrategy' do + expect { described_class.build!('User') }.to raise_error(described_class::StrategyNotFoundError) + end + end + + it 'initializes strategy with attributes param' do + params = { param1: 1, param2: 2, param3: 3 } + + strategy = described_class.build!("#{strategies_namespace}::DownloadNotificationStrategy", params) + + params.each { |k, v| expect(strategy.public_send(k)).to eq v } + end + end +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 05014222623..96adf64bcec 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -2560,7 +2560,7 @@ describe Project do end end - describe '#remove_exports' do + describe '#remove_export' do let(:legacy_project) { create(:project, :legacy_storage, :with_export) } let(:project) { create(:project, :with_export) } @@ -2608,6 +2608,23 @@ describe Project do end end + describe '#remove_exported_project_file' do + let(:project) { create(:project, :with_export) } + + it 'removes the exported project file' do + exported_file = project.export_project_path + + expect(File.exist?(exported_file)).to be_truthy + + allow(FileUtils).to receive(:rm_f).and_call_original + expect(FileUtils).to receive(:rm_f).with(exported_file).and_call_original + + project.remove_exported_project_file + + expect(File.exist?(exported_file)).to be_falsy + end + end + describe '#forks_count' do it 'returns the number of forks' do project = build(:project) diff --git a/spec/requests/api/project_export_spec.rb b/spec/requests/api/project_export_spec.rb index 12583109b59..3834d27d0a9 100644 --- a/spec/requests/api/project_export_spec.rb +++ b/spec/requests/api/project_export_spec.rb @@ -5,6 +5,7 @@ describe API::ProjectExport do set(:project_none) { create(:project) } set(:project_started) { create(:project) } set(:project_finished) { create(:project) } + set(:project_after_export) { create(:project) } set(:user) { create(:user) } set(:admin) { create(:admin) } @@ -12,11 +13,13 @@ describe API::ProjectExport do let(:path_none) { "/projects/#{project_none.id}/export" } let(:path_started) { "/projects/#{project_started.id}/export" } let(:path_finished) { "/projects/#{project_finished.id}/export" } + let(:path_after_export) { "/projects/#{project_after_export.id}/export" } let(:download_path) { "/projects/#{project.id}/export/download" } let(:download_path_none) { "/projects/#{project_none.id}/export/download" } let(:download_path_started) { "/projects/#{project_started.id}/export/download" } let(:download_path_finished) { "/projects/#{project_finished.id}/export/download" } + let(:download_path_export_action) { "/projects/#{project_after_export.id}/export/download" } let(:export_path) { "#{Dir.tmpdir}/project_export_spec" } @@ -29,6 +32,11 @@ describe API::ProjectExport do # simulate exported FileUtils.mkdir_p project_finished.export_path FileUtils.touch File.join(project_finished.export_path, '_export.tar.gz') + + # simulate in after export action + FileUtils.mkdir_p project_after_export.export_path + FileUtils.touch File.join(project_after_export.export_path, '_export.tar.gz') + FileUtils.touch Gitlab::ImportExport::AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project_after_export) end after do @@ -73,6 +81,14 @@ describe API::ProjectExport do expect(json_response['export_status']).to eq('started') end + it 'is after_export' do + get api(path_after_export, user) + + expect(response).to have_gitlab_http_status(200) + expect(response).to match_response_schema('public_api/v4/project/export_status') + expect(json_response['export_status']).to eq('after_export_action') + end + it 'is finished' do get api(path_finished, user) @@ -99,6 +115,7 @@ describe API::ProjectExport do project_none.add_master(user) project_started.add_master(user) project_finished.add_master(user) + project_after_export.add_master(user) end it_behaves_like 'get project export status ok' @@ -163,6 +180,36 @@ describe API::ProjectExport do end end + shared_examples_for 'get project export upload after action' do + context 'and is uploading' do + it 'downloads' do + get api(download_path_export_action, user) + + expect(response).to have_gitlab_http_status(200) + end + end + + context 'when upload complete' do + before do + FileUtils.rm_rf(project_after_export.export_path) + end + + it_behaves_like '404 response' do + let(:request) { get api(download_path_export_action, user) } + end + end + end + + shared_examples_for 'get project download by strategy' do + context 'when upload strategy set' do + it_behaves_like 'get project export upload after action' + end + + context 'when download strategy set' do + it_behaves_like 'get project export download' + end + end + it_behaves_like 'when project export is disabled' do let(:request) { get api(download_path, admin) } end @@ -171,7 +218,7 @@ describe API::ProjectExport do context 'when user is an admin' do let(:user) { admin } - it_behaves_like 'get project export download' + it_behaves_like 'get project download by strategy' end context 'when user is a master' do @@ -180,9 +227,10 @@ describe API::ProjectExport do project_none.add_master(user) project_started.add_master(user) project_finished.add_master(user) + project_after_export.add_master(user) end - it_behaves_like 'get project export download' + it_behaves_like 'get project download by strategy' end context 'when user is a developer' do @@ -229,10 +277,30 @@ describe API::ProjectExport do end shared_examples_for 'post project export start' do - it 'starts' do - post api(path, user) + context 'with upload strategy' do + context 'when params invalid' do + it_behaves_like '400 response' do + let(:request) { post(api(path, user), 'upload[url]' => 'whatever') } + end + end + + it 'starts' do + allow_any_instance_of(Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy).to receive(:send_file) + + post(api(path, user), 'upload[url]' => 'http://gitlab.com') - expect(response).to have_gitlab_http_status(202) + expect(response).to have_gitlab_http_status(202) + end + end + + context 'with download strategy' do + it 'starts' do + expect_any_instance_of(Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy).not_to receive(:send_file) + + post api(path, user) + + expect(response).to have_gitlab_http_status(202) + end end end @@ -253,6 +321,7 @@ describe API::ProjectExport do project_none.add_master(user) project_started.add_master(user) project_finished.add_master(user) + project_after_export.add_master(user) end it_behaves_like 'post project export start' diff --git a/spec/services/projects/import_export/export_service_spec.rb b/spec/services/projects/import_export/export_service_spec.rb new file mode 100644 index 00000000000..51491c7d529 --- /dev/null +++ b/spec/services/projects/import_export/export_service_spec.rb @@ -0,0 +1,85 @@ +require 'spec_helper' + +describe Projects::ImportExport::ExportService do + describe '#execute' do + let!(:user) { create(:user) } + let(:project) { create(:project) } + let(:shared) { project.import_export_shared } + let(:service) { described_class.new(project, user) } + let!(:after_export_strategy) { Gitlab::ImportExport::AfterExportStrategies::DownloadNotificationStrategy.new } + + context 'when all saver services succeed' do + before do + allow(service).to receive(:save_services).and_return(true) + end + + it 'saves the project in the file system' do + expect(Gitlab::ImportExport::Saver).to receive(:save).with(project: project, shared: shared) + + service.execute + end + + it 'calls the after export strategy' do + expect(after_export_strategy).to receive(:execute) + + service.execute(after_export_strategy) + end + + context 'when after export strategy fails' do + before do + allow(after_export_strategy).to receive(:execute).and_return(false) + end + + after do + service.execute(after_export_strategy) + end + + it 'removes the remaining exported data' do + allow(shared).to receive(:export_path).and_return('whatever') + allow(FileUtils).to receive(:rm_rf) + + expect(FileUtils).to receive(:rm_rf).with(shared.export_path) + end + + it 'notifies the user' do + expect_any_instance_of(NotificationService).to receive(:project_not_exported) + end + + it 'notifies logger' do + allow(Rails.logger).to receive(:error) + + expect(Rails.logger).to receive(:error) + end + end + end + + context 'when saver services fail' do + before do + allow(service).to receive(:save_services).and_return(false) + end + + after do + expect { service.execute }.to raise_error(Gitlab::ImportExport::Error) + end + + it 'removes the remaining exported data' do + allow(shared).to receive(:export_path).and_return('whatever') + allow(FileUtils).to receive(:rm_rf) + + expect(FileUtils).to receive(:rm_rf).with(shared.export_path) + end + + it 'notifies the user' do + expect_any_instance_of(NotificationService).to receive(:project_not_exported) + end + + it 'notifies logger' do + expect(Rails.logger).to receive(:error) + end + + it 'the after export strategy is not called' do + expect(service).not_to receive(:execute_after_export_action) + end + end + end +end diff --git a/spec/workers/project_export_worker_spec.rb b/spec/workers/project_export_worker_spec.rb new file mode 100644 index 00000000000..8899969c178 --- /dev/null +++ b/spec/workers/project_export_worker_spec.rb @@ -0,0 +1,28 @@ +require 'spec_helper' + +describe ProjectExportWorker do + let!(:user) { create(:user) } + let!(:project) { create(:project) } + + subject { described_class.new } + + describe '#perform' do + context 'when it succeeds' do + it 'calls the ExportService' do + expect_any_instance_of(::Projects::ImportExport::ExportService).to receive(:execute) + + subject.perform(user.id, project.id, { 'klass' => 'Gitlab::ImportExport::AfterExportStrategies::DownloadNotificationStrategy' }) + end + end + + context 'when it fails' do + it 'raises an exception when params are invalid' do + expect_any_instance_of(::Projects::ImportExport::ExportService).not_to receive(:execute) + + expect { subject.perform(1234, project.id, {}) }.to raise_exception(ActiveRecord::RecordNotFound) + expect { subject.perform(user.id, 1234, {}) }.to raise_exception(ActiveRecord::RecordNotFound) + expect { subject.perform(user.id, project.id, { 'klass' => 'Whatever' }) }.to raise_exception(Gitlab::ImportExport::AfterExportStrategyBuilder::StrategyNotFoundError) + end + end + end +end |