diff options
34 files changed, 648 insertions, 69 deletions
diff --git a/app/finders/todos_finder.rb b/app/finders/todos_finder.rb index 82acb5e9d45..cc71ea8e916 100644 --- a/app/finders/todos_finder.rb +++ b/app/finders/todos_finder.rb @@ -23,10 +23,16 @@ class TodosFinder NONE = '0' - TODO_TYPES = Set.new(%w(Issue MergeRequest Epic)).freeze + TODO_TYPES = Set.new(%w(Issue MergeRequest)).freeze attr_accessor :current_user, :params + class << self + def todo_types + TODO_TYPES + end + end + def initialize(current_user, params = {}) @current_user = current_user @params = params @@ -124,7 +130,7 @@ class TodosFinder end def type? - type.present? && TODO_TYPES.include?(type) + type.present? && self.class.todo_types.include?(type) end def type @@ -201,3 +207,5 @@ class TodosFinder end end end + +TodosFinder.prepend_if_ee('EE::TodosFinder') diff --git a/app/models/group.rb b/app/models/group.rb index 7760a3c69ce..71d81289bf5 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -55,6 +55,8 @@ class Group < Namespace has_many :todos + has_one :import_export_upload + accepts_nested_attributes_for :variables, allow_destroy: true validate :visibility_level_allowed_by_projects diff --git a/app/models/import_export_upload.rb b/app/models/import_export_upload.rb index 60f5491849a..7d73fd281f1 100644 --- a/app/models/import_export_upload.rb +++ b/app/models/import_export_upload.rb @@ -5,6 +5,7 @@ class ImportExportUpload < ApplicationRecord include ObjectStorage::BackgroundMove belongs_to :project + belongs_to :group # These hold the project Import/Export archives (.tar.gz files) mount_uploader :import_file, ImportExportUploader diff --git a/app/services/groups/import_export/export_service.rb b/app/services/groups/import_export/export_service.rb new file mode 100644 index 00000000000..26886fc67dc --- /dev/null +++ b/app/services/groups/import_export/export_service.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Groups + module ImportExport + class ExportService + def initialize(group:, user:, params: {}) + @group = group + @current_user = user + @params = params + @shared = @params[:shared] || Gitlab::ImportExport::Shared.new(@group) + end + + def execute + save! + end + + private + + attr_accessor :shared + + def save! + if savers.all?(&:save) + notify_success + else + cleanup_and_notify_error! + end + end + + def savers + [tree_exporter, file_saver] + end + + def tree_exporter + Gitlab::ImportExport::GroupTreeSaver.new(group: @group, current_user: @current_user, shared: @shared, params: @params) + end + + def file_saver + Gitlab::ImportExport::Saver.new(exportable: @group, shared: @shared) + end + + def cleanup_and_notify_error + FileUtils.rm_rf(shared.export_path) + + notify_error + end + + def cleanup_and_notify_error! + cleanup_and_notify_error + + raise Gitlab::ImportExport::Error.new(shared.errors.to_sentence) + end + + def notify_success + @shared.logger.info( + group_id: @group.id, + group_name: @group.name, + message: 'Group Import/Export: Export succeeded' + ) + end + + def notify_error + @shared.logger.error( + group_id: @group.id, + group_name: @group.name, + error: @shared.errors.join(', '), + message: 'Group Import/Export: Export failed' + ) + end + end + end +end diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index d3638c57552..8344397f67d 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -24,7 +24,7 @@ module Projects def save_all! if save_exporters - Gitlab::ImportExport::Saver.save(project: project, shared: shared) + Gitlab::ImportExport::Saver.save(exportable: project, shared: shared) notify_success else cleanup_and_notify_error! diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 10081840305..66b5214cfcb 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -179,3 +179,4 @@ - import_issues_csv - project_daily_statistics - create_evidence +- group_export diff --git a/app/workers/group_export_worker.rb b/app/workers/group_export_worker.rb new file mode 100644 index 00000000000..51dbdc95661 --- /dev/null +++ b/app/workers/group_export_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class GroupExportWorker + include ApplicationWorker + include ExceptionBacktrace + + feature_category :source_code_management + + def perform(current_user_id, group_id, params = {}) + current_user = User.find(current_user_id) + group = Group.find(group_id) + + ::Groups::ImportExport::ExportService.new(group: group, user: current_user, params: params).execute + end +end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 7f8ba35bf41..b4be61d8a3d 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -98,6 +98,7 @@ - [update_namespace_statistics, 1] - [chaos, 2] - [create_evidence, 2] + - [group_export, 1] # EE-specific queues - [analytics, 1] diff --git a/db/migrate/20191111115229_add_group_id_to_import_export_uploads.rb b/db/migrate/20191111115229_add_group_id_to_import_export_uploads.rb new file mode 100644 index 00000000000..74ef0f27b3e --- /dev/null +++ b/db/migrate/20191111115229_add_group_id_to_import_export_uploads.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddGroupIdToImportExportUploads < ActiveRecord::Migration[5.2] + DOWNTIME = false + + def change + add_column :import_export_uploads, :group_id, :bigint + end +end diff --git a/db/migrate/20191111115431_add_group_fk_to_import_export_uploads.rb b/db/migrate/20191111115431_add_group_fk_to_import_export_uploads.rb new file mode 100644 index 00000000000..403de3f33ed --- /dev/null +++ b/db/migrate/20191111115431_add_group_fk_to_import_export_uploads.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddGroupFkToImportExportUploads < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :import_export_uploads, :namespaces, column: :group_id, on_delete: :cascade + add_concurrent_index :import_export_uploads, :group_id, unique: true, where: 'group_id IS NOT NULL' + end + + def down + remove_foreign_key_without_error(:import_export_uploads, column: :group_id) + remove_concurrent_index(:import_export_uploads, :group_id) + end +end diff --git a/db/schema.rb b/db/schema.rb index c11c1059fd2..b5991904302 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_11_05_140942) do +ActiveRecord::Schema.define(version: 2019_11_11_115431) do # These are extensions that must be enabled in order to support this database enable_extension "pg_trgm" @@ -1874,6 +1874,8 @@ ActiveRecord::Schema.define(version: 2019_11_05_140942) do t.integer "project_id" t.text "import_file" t.text "export_file" + t.bigint "group_id" + t.index ["group_id"], name: "index_import_export_uploads_on_group_id", unique: true, where: "(group_id IS NOT NULL)" t.index ["project_id"], name: "index_import_export_uploads_on_project_id" t.index ["updated_at"], name: "index_import_export_uploads_on_updated_at" end @@ -4288,6 +4290,7 @@ ActiveRecord::Schema.define(version: 2019_11_05_140942) do add_foreign_key "group_group_links", "namespaces", column: "shared_group_id", on_delete: :cascade add_foreign_key "group_group_links", "namespaces", column: "shared_with_group_id", on_delete: :cascade add_foreign_key "identities", "saml_providers", name: "fk_aade90f0fc", on_delete: :cascade + add_foreign_key "import_export_uploads", "namespaces", column: "group_id", name: "fk_83319d9721", on_delete: :cascade add_foreign_key "import_export_uploads", "projects", on_delete: :cascade add_foreign_key "index_statuses", "projects", name: "fk_74b2492545", on_delete: :cascade add_foreign_key "insights", "namespaces", on_delete: :cascade diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index b2ac60fe825..516e7f54a6e 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -15,7 +15,7 @@ module Gitlab end def storage_path - File.join(Settings.shared['path'], 'tmp/project_exports') + File.join(Settings.shared['path'], 'tmp/gitlab_exports') end def import_upload_path(filename:) @@ -50,8 +50,8 @@ module Gitlab 'VERSION' end - def export_filename(project:) - basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.full_path.tr('/', '_')}" + def export_filename(exportable:) + basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{exportable.full_path.tr('/', '_')}" "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" end @@ -63,6 +63,14 @@ module Gitlab def reset_tokens? true end + + def group_filename + 'group.json' + end + + def group_config_file + Rails.root.join('lib/gitlab/import_export/group_import_export.yml') + end end end diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb index 6f4919ead4e..83c4bc47349 100644 --- a/lib/gitlab/import_export/config.rb +++ b/lib/gitlab/import_export/config.rb @@ -3,7 +3,8 @@ module Gitlab module ImportExport class Config - def initialize + def initialize(config: Gitlab::ImportExport.config_file) + @config = config @hash = parse_yaml @hash.deep_symbolize_keys! @ee_hash = @hash.delete(:ee) || {} @@ -50,7 +51,7 @@ module Gitlab end def parse_yaml - YAML.load_file(Gitlab::ImportExport.config_file) + YAML.load_file(@config) end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 05432f433e7..2fd12e3aa78 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -60,7 +60,7 @@ module Gitlab def copy_archive return if @archive_file - @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @project)) download_or_copy_upload(@project.import_export_upload.import_file, @archive_file) end diff --git a/lib/gitlab/import_export/group_import_export.yml b/lib/gitlab/import_export/group_import_export.yml new file mode 100644 index 00000000000..c1900350c86 --- /dev/null +++ b/lib/gitlab/import_export/group_import_export.yml @@ -0,0 +1,36 @@ +# Model relationships to be included in the group import/export +# +# This list _must_ only contain relationships that are available to both FOSS and +# Enterprise editions. EE specific relationships must be defined in the `ee` section further +# down below. +tree: + group: + - :milestones + - :badges + - labels: + - :priorities + - :boards + - members: + - :user + +included_attributes: + +excluded_attributes: + group: + - :runners_token + - :runners_token_encrypted + +methods: + labels: + - :type + badges: + - :type + +preloads: + +# EE specific relationships and settings to include. All of this will be merged +# into the previous structures if EE is used. +ee: + tree: + group: + - :epics diff --git a/lib/gitlab/import_export/group_tree_saver.rb b/lib/gitlab/import_export/group_tree_saver.rb new file mode 100644 index 00000000000..1d42bc8d3f3 --- /dev/null +++ b/lib/gitlab/import_export/group_tree_saver.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class GroupTreeSaver + attr_reader :full_path + + def initialize(group:, current_user:, shared:, params: {}) + @params = params + @current_user = current_user + @shared = shared + @group = group + @full_path = File.join(@shared.export_path, ImportExport.group_filename) + end + + def save + group_tree = serialize(@group, reader.group_tree) + tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) + + true + rescue => e + @shared.error(e) + false + end + + private + + def serialize(group, relations_tree) + group_tree = tree_saver.serialize(group, relations_tree) + + group.descendants.each do |descendant| + group_tree['descendants'] = [] unless group_tree['descendants'] + group_tree['descendants'] << serialize(descendant, relations_tree) + end + + group_tree + rescue => e + @shared.error(e) + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new( + shared: @shared, + config: Gitlab::ImportExport::Config.new( + config: Gitlab::ImportExport.group_config_file + ).to_h + ) + end + + def tree_saver + @tree_saver ||= RelationTreeSaver.new + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 63c71105efe..386a4cfdfc6 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -3,25 +3,20 @@ module Gitlab module ImportExport class ProjectTreeSaver - include Gitlab::ImportExport::CommandLineUtil - attr_reader :full_path def initialize(project:, current_user:, shared:, params: {}) - @params = params - @project = project + @params = params + @project = project @current_user = current_user - @shared = shared - @full_path = File.join(@shared.export_path, ImportExport.project_filename) + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) end def save - mkdir_p(@shared.export_path) - - project_tree = serialize_project_tree + project_tree = tree_saver.serialize(@project, reader.project_tree) fix_project_tree(project_tree) - project_tree_json = JSON.generate(project_tree) - File.write(full_path, project_tree_json) + tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) true rescue => e @@ -43,16 +38,6 @@ module Gitlab RelationRenameService.add_new_associations(project_tree) end - def serialize_project_tree - if Feature.enabled?(:export_fast_serialize, default_enabled: true) - Gitlab::ImportExport::FastHashSerializer - .new(@project, reader.project_tree) - .execute - else - @project.as_json(reader.project_tree) - end - end - def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end @@ -74,6 +59,10 @@ module Gitlab GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) end + + def tree_saver + @tree_saver ||= RelationTreeSaver.new + end end end end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 9e81c6a3d07..1390770acef 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -5,24 +5,31 @@ module Gitlab class Reader attr_reader :tree, :attributes_finder - def initialize(shared:) - @shared = shared - - @attributes_finder = Gitlab::ImportExport::AttributesFinder.new( - config: ImportExport::Config.new.to_h) + def initialize(shared:, config: ImportExport::Config.new.to_h) + @shared = shared + @config = config + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config) end # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html # for outputting a project in JSON format, including its relations and sub relations. def project_tree - attributes_finder.find_root(:project) - rescue => e - @shared.error(e) - false + tree_by_key(:project) + end + + def group_tree + tree_by_key(:group) end def group_members_tree - attributes_finder.find_root(:group_members) + tree_by_key(:group_members) + end + + def tree_by_key(key) + attributes_finder.find_root(key) + rescue => e + @shared.error(e) + false end end end diff --git a/lib/gitlab/import_export/relation_rename_service.rb b/lib/gitlab/import_export/relation_rename_service.rb index 179bde5e21e..03aaa6aefc3 100644 --- a/lib/gitlab/import_export/relation_rename_service.rb +++ b/lib/gitlab/import_export/relation_rename_service.rb @@ -8,7 +8,7 @@ # The behavior of these renamed relationships should be transient and it should # only last one release until you completely remove the renaming from the list. # -# When importing, this class will check the project hash and: +# When importing, this class will check the hash and: # - if only the old relationship name is found, it will rename it with the new one # - if only the new relationship name is found, it will do nothing # - if it finds both, it will use the new relationship data diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/relation_tree_saver.rb new file mode 100644 index 00000000000..a0452071ccf --- /dev/null +++ b/lib/gitlab/import_export/relation_tree_saver.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + class RelationTreeSaver + include Gitlab::ImportExport::CommandLineUtil + + def serialize(exportable, relations_tree) + if Feature.enabled?(:export_fast_serialize, default_enabled: true) + Gitlab::ImportExport::FastHashSerializer + .new(exportable, relations_tree) + .execute + else + exportable.as_json(relations_tree) + end + end + + def save(tree, dir_path, filename) + mkdir_p(dir_path) + + tree_json = JSON.generate(tree) + + File.write(File.join(dir_path, filename), tree_json) + end + end + end +end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index bea7a7cce65..ae82c380755 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -9,16 +9,16 @@ module Gitlab new(*args).save end - def initialize(project:, shared:) - @project = project - @shared = shared + def initialize(exportable:, shared:) + @exportable = exportable + @shared = shared end def save if compress_and_save remove_export_path - Rails.logger.info("Saved project export #{archive_file}") # rubocop:disable Gitlab/RailsLogger + Rails.logger.info("Saved #{@exportable.class} export #{archive_file}") # rubocop:disable Gitlab/RailsLogger save_upload else @@ -48,11 +48,11 @@ module Gitlab end def archive_file - @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) + @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @exportable)) end def save_upload - upload = ImportExportUpload.find_or_initialize_by(project: @project) + upload = initialize_upload File.open(archive_file) { |file| upload.export_file = file } @@ -62,6 +62,12 @@ module Gitlab def error_message "Unable to save #{archive_file} into #{@shared.export_path}." end + + def initialize_upload + exportable_kind = @exportable.class.name.downcase + + ImportExportUpload.find_or_initialize_by(Hash[exportable_kind, @exportable]) + end end end end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 02d46a1f498..2539a6828c3 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -23,21 +23,21 @@ module Gitlab module ImportExport class Shared - attr_reader :errors, :project + attr_reader :errors, :exportable, :logger LOCKS_DIRECTORY = 'locks' - def initialize(project) - @project = project - @errors = [] - @logger = Gitlab::Import::Logger.build + def initialize(exportable) + @exportable = exportable + @errors = [] + @logger = Gitlab::Import::Logger.build end def active_export_count Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) } end - # The path where the project metadata and repository bundle is saved + # The path where the exportable metadata and repository bundle (in case of project) is saved def export_path @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path) end @@ -84,11 +84,18 @@ module Gitlab end def relative_archive_path - @relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex) + @relative_archive_path ||= File.join(relative_base_path, SecureRandom.hex) end def relative_base_path - @project.disk_path + case exportable_type + when 'Project' + @exportable.disk_path + when 'Group' + @exportable.full_path + else + raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}") + end end def log_error(details) @@ -100,17 +107,24 @@ module Gitlab end def log_base_data - { - importer: 'Import/Export', - import_jid: @project&.import_state&.jid, - project_id: @project&.id, - project_path: @project&.full_path + log = { + importer: 'Import/Export', + exportable_id: @exportable&.id, + exportable_path: @exportable&.full_path } + + log[:import_jid] = @exportable&.import_state&.jid if exportable_type == 'Project' + + log end def filtered_error_message(message) Projects::ImportErrorFilter.filter_message(message) end + + def exportable_type + @exportable.class.name + end end end end diff --git a/spec/finders/todos_finder_spec.rb b/spec/finders/todos_finder_spec.rb index 6234d596745..75aa10c66a2 100644 --- a/spec/finders/todos_finder_spec.rb +++ b/spec/finders/todos_finder_spec.rb @@ -234,6 +234,19 @@ describe TodosFinder do end end + describe '.todo_types' do + it 'returns the expected types' do + expected_result = + if Gitlab.ee? + %w[Epic Issue MergeRequest] + else + %w[Issue MergeRequest] + end + + expect(described_class.todo_types).to contain_exactly(*expected_result) + end + end + describe '#any_for_target?' do it 'returns true if there are any todos for the given target' do todo = create(:todo, :pending) diff --git a/spec/fixtures/group_export.tar.gz b/spec/fixtures/group_export.tar.gz Binary files differnew file mode 100644 index 00000000000..83e360d7cc2 --- /dev/null +++ b/spec/fixtures/group_export.tar.gz diff --git a/spec/lib/gitlab/import_export/group_tree_saver_spec.rb b/spec/lib/gitlab/import_export/group_tree_saver_spec.rb new file mode 100644 index 00000000000..c752c557d99 --- /dev/null +++ b/spec/lib/gitlab/import_export/group_tree_saver_spec.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::GroupTreeSaver do + describe 'saves the group tree into a json object' do + let(:shared) { Gitlab::ImportExport::Shared.new(group) } + let(:group_tree_saver) { described_class.new(group: group, current_user: user, shared: shared) } + let(:export_path) { "#{Dir.tmpdir}/group_tree_saver_spec" } + let(:user) { create(:user) } + let!(:group) { setup_group } + + before do + group.add_maintainer(user) + allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) + end + + after do + FileUtils.rm_rf(export_path) + end + + it 'saves group successfully' do + expect(group_tree_saver.save).to be true + end + + context ':export_fast_serialize feature flag checks' do + before do + expect(Gitlab::ImportExport::Reader).to receive(:new).with(shared: shared, config: group_config).and_return(reader) + expect(reader).to receive(:group_tree).and_return(group_tree) + end + + let(:reader) { instance_double('Gitlab::ImportExport::Reader') } + let(:group_config) { Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.group_config_file).to_h } + let(:group_tree) do + { + include: [{ milestones: { include: [] } }], + preload: { milestones: nil } + } + end + + context 'when :export_fast_serialize feature is enabled' do + before do + stub_feature_flags(export_fast_serialize: true) + end + + it 'uses FastHashSerializer' do + expect_any_instance_of(Gitlab::ImportExport::FastHashSerializer).to receive(:execute).and_call_original + + group_tree_saver.save + end + end + + context 'when :export_fast_serialize feature is disabled' do + before do + stub_feature_flags(export_fast_serialize: false) + end + + it 'is serialized via built-in `as_json`' do + expect(group).to receive(:as_json).with(group_tree).and_call_original + + group_tree_saver.save + end + end + end + + # It is mostly duplicated in + # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` + # except: + # context 'with description override' do + # context 'group members' do + # ^ These are specific for the groupTreeSaver + context 'JSON' do + let(:saved_group_json) do + group_tree_saver.save + group_json(group_tree_saver.full_path) + end + + it 'saves the correct json' do + expect(saved_group_json).to include({ 'description' => 'description', 'visibility_level' => 20 }) + end + + it 'has milestones' do + expect(saved_group_json['milestones']).not_to be_empty + end + + it 'has labels' do + expect(saved_group_json['labels']).not_to be_empty + end + + it 'has boards' do + expect(saved_group_json['boards']).not_to be_empty + end + + it 'has group members' do + expect(saved_group_json['members']).not_to be_empty + end + + it 'has priorities associated to labels' do + expect(saved_group_json['labels'].first['priorities']).not_to be_empty + end + + it 'has badges' do + expect(saved_group_json['badges']).not_to be_empty + end + + context 'group members' do + let(:user2) { create(:user, email: 'group@member.com') } + let(:member_emails) do + saved_group_json['members'].map do |pm| + pm['user']['email'] + end + end + + before do + group.add_developer(user2) + end + + it 'exports group members as group owner' do + group.add_owner(user) + + expect(member_emails).to include('group@member.com') + end + + context 'as admin' do + let(:user) { create(:admin) } + + it 'exports group members as admin' do + expect(member_emails).to include('group@member.com') + end + + it 'exports group members' do + member_types = saved_group_json['members'].map { |pm| pm['source_type'] } + + expect(member_types).to all(eq('Namespace')) + end + end + end + + context 'group attributes' do + it 'does not contain the runners token' do + expect(saved_group_json).not_to include("runners_token" => 'token') + end + end + end + end + + def setup_group + group = create(:group, description: 'description') + create(:milestone, group: group) + create(:group_badge, group: group) + group_label = create(:group_label, group: group) + create(:label_priority, label: group_label, priority: 1) + create(:board, group: group) + create(:group_badge, group: group) + + group + end + + def group_json(filename) + JSON.parse(IO.read(filename)) + end +end diff --git a/spec/lib/gitlab/import_export/import_export_spec.rb b/spec/lib/gitlab/import_export/import_export_spec.rb index 40a5f2294a2..a6b0dc758cd 100644 --- a/spec/lib/gitlab/import_export/import_export_spec.rb +++ b/spec/lib/gitlab/import_export/import_export_spec.rb @@ -6,17 +6,17 @@ describe Gitlab::ImportExport do let(:project) { create(:project, :public, path: 'project-path', namespace: group) } it 'contains the project path' do - expect(described_class.export_filename(project: project)).to include(project.path) + expect(described_class.export_filename(exportable: project)).to include(project.path) end it 'contains the namespace path' do - expect(described_class.export_filename(project: project)).to include(project.namespace.full_path.tr('/', '_')) + expect(described_class.export_filename(exportable: project)).to include(project.namespace.full_path.tr('/', '_')) end it 'does not go over a certain length' do project.path = 'a' * 100 - expect(described_class.export_filename(project: project).length).to be < 70 + expect(described_class.export_filename(exportable: project).length).to be < 70 end end end diff --git a/spec/lib/gitlab/import_export/relation_rename_service_spec.rb b/spec/lib/gitlab/import_export/relation_rename_service_spec.rb index 843de27df1a..d62f5725f9e 100644 --- a/spec/lib/gitlab/import_export/relation_rename_service_spec.rb +++ b/spec/lib/gitlab/import_export/relation_rename_service_spec.rb @@ -96,15 +96,20 @@ describe Gitlab::ImportExport::RelationRenameService do let(:export_content_path) { project_tree_saver.full_path } let(:export_content_hash) { ActiveSupport::JSON.decode(File.read(export_content_path)) } let(:injected_hash) { renames.values.product([{}]).to_h } + let(:relation_tree_saver) { Gitlab::ImportExport::RelationTreeSaver.new } let(:project_tree_saver) do Gitlab::ImportExport::ProjectTreeSaver.new( project: project, current_user: user, shared: shared) end + before do + allow(project_tree_saver).to receive(:tree_saver).and_return(relation_tree_saver) + end + it 'adds old relationships to the exported file' do # we inject relations with new names that should be rewritten - expect(project_tree_saver).to receive(:serialize_project_tree).and_wrap_original do |method, *args| + expect(relation_tree_saver).to receive(:serialize).and_wrap_original do |method, *args| method.call(*args).merge(injected_hash) end diff --git a/spec/lib/gitlab/import_export/relation_tree_saver_spec.rb b/spec/lib/gitlab/import_export/relation_tree_saver_spec.rb new file mode 100644 index 00000000000..2fc26c0e3d4 --- /dev/null +++ b/spec/lib/gitlab/import_export/relation_tree_saver_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::ImportExport::RelationTreeSaver do + let(:exportable) { create(:group) } + let(:relation_tree_saver) { described_class.new } + let(:tree) { {} } + + describe '#serialize' do + context 'when :export_fast_serialize feature is enabled' do + let(:serializer) { instance_double(Gitlab::ImportExport::FastHashSerializer) } + + before do + stub_feature_flags(export_fast_serialize: true) + end + + it 'uses FastHashSerializer' do + expect(Gitlab::ImportExport::FastHashSerializer) + .to receive(:new) + .with(exportable, tree) + .and_return(serializer) + + expect(serializer).to receive(:execute) + + relation_tree_saver.serialize(exportable, tree) + end + end + + context 'when :export_fast_serialize feature is disabled' do + before do + stub_feature_flags(export_fast_serialize: false) + end + + it 'is serialized via built-in `as_json`' do + expect(exportable).to receive(:as_json).with(tree) + + relation_tree_saver.serialize(exportable, tree) + end + end + end +end diff --git a/spec/lib/gitlab/import_export/saver_spec.rb b/spec/lib/gitlab/import_export/saver_spec.rb index d185ff2dfcc..aca63953677 100644 --- a/spec/lib/gitlab/import_export/saver_spec.rb +++ b/spec/lib/gitlab/import_export/saver_spec.rb @@ -5,7 +5,7 @@ describe Gitlab::ImportExport::Saver do let!(:project) { create(:project, :public, name: 'project') } let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } let(:shared) { project.import_export_shared } - subject { described_class.new(project: project, shared: shared) } + subject { described_class.new(exportable: project, shared: shared) } before do allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) diff --git a/spec/lib/gitlab/import_export/shared_spec.rb b/spec/lib/gitlab/import_export/shared_spec.rb index 62669836973..fc011f7e1be 100644 --- a/spec/lib/gitlab/import_export/shared_spec.rb +++ b/spec/lib/gitlab/import_export/shared_spec.rb @@ -7,7 +7,7 @@ describe Gitlab::ImportExport::Shared do context 'with a repository on disk' do let(:project) { create(:project, :repository) } - let(:base_path) { %(/tmp/project_exports/#{project.disk_path}/) } + let(:base_path) { %(/tmp/gitlab_exports/#{project.disk_path}/) } describe '#archive_path' do it 'uses a random hash to avoid conflicts' do diff --git a/spec/services/groups/import_export/export_service_spec.rb b/spec/services/groups/import_export/export_service_spec.rb new file mode 100644 index 00000000000..2024e1ed457 --- /dev/null +++ b/spec/services/groups/import_export/export_service_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Groups::ImportExport::ExportService do + describe '#execute' do + let!(:user) { create(:user) } + let(:group) { create(:group) } + let(:shared) { Gitlab::ImportExport::Shared.new(group) } + let(:export_path) { shared.export_path } + let(:service) { described_class.new(group: group, user: user, params: { shared: shared }) } + + after do + FileUtils.rm_rf(export_path) + end + + it 'saves the models' do + expect(Gitlab::ImportExport::GroupTreeSaver).to receive(:new).and_call_original + + service.execute + end + + context 'when saver succeeds' do + it 'saves the group in the file system' do + service.execute + + expect(group.import_export_upload.export_file.file).not_to be_nil + expect(File.directory?(export_path)).to eq(false) + expect(File.exist?(shared.archive_path)).to eq(false) + end + end + + context 'when saving services fail' do + before do + allow(service).to receive_message_chain(:tree_exporter, :save).and_return(false) + end + + it 'removes the remaining exported data' do + allow_any_instance_of(Gitlab::ImportExport::Saver).to receive(:compress_and_save).and_return(false) + + expect { service.execute }.to raise_error(Gitlab::ImportExport::Error) + + expect(group.import_export_upload).to be_nil + expect(File.directory?(export_path)).to eq(false) + expect(File.exist?(shared.archive_path)).to eq(false) + end + + it 'notifies logger' do + expect_any_instance_of(Gitlab::Import::Logger).to receive(:error) + + expect { service.execute }.to raise_error(Gitlab::ImportExport::Error) + end + end + end +end diff --git a/spec/services/import_export_clean_up_service_spec.rb b/spec/services/import_export_clean_up_service_spec.rb index 51720e786dc..9f811f56f50 100644 --- a/spec/services/import_export_clean_up_service_spec.rb +++ b/spec/services/import_export_clean_up_service_spec.rb @@ -6,7 +6,7 @@ describe ImportExportCleanUpService do describe '#execute' do let(:service) { described_class.new } - let(:tmp_import_export_folder) { 'tmp/project_exports' } + let(:tmp_import_export_folder) { 'tmp/gitlab_exports' } context 'when the import/export directory does not exist' do it 'does not remove any archives' do diff --git a/spec/services/projects/import_export/export_service_spec.rb b/spec/services/projects/import_export/export_service_spec.rb index 146d656c909..a557e61da78 100644 --- a/spec/services/projects/import_export/export_service_spec.rb +++ b/spec/services/projects/import_export/export_service_spec.rb @@ -66,7 +66,7 @@ describe Projects::ImportExport::ExportService do end it 'saves the project in the file system' do - expect(Gitlab::ImportExport::Saver).to receive(:save).with(project: project, shared: shared) + expect(Gitlab::ImportExport::Saver).to receive(:save).with(exportable: project, shared: shared) service.execute end diff --git a/spec/workers/group_export_worker_spec.rb b/spec/workers/group_export_worker_spec.rb new file mode 100644 index 00000000000..4aa85d2b381 --- /dev/null +++ b/spec/workers/group_export_worker_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe GroupExportWorker do + let!(:user) { create(:user) } + let!(:group) { create(:group) } + + subject { described_class.new } + + describe '#perform' do + context 'when it succeeds' do + it 'calls the ExportService' do + expect_any_instance_of(::Groups::ImportExport::ExportService).to receive(:execute) + + subject.perform(user.id, group.id, {}) + end + end + + context 'when it fails' do + it 'raises an exception when params are invalid' do + expect_any_instance_of(::Groups::ImportExport::ExportService).not_to receive(:execute) + + expect { subject.perform(1234, group.id, {}) }.to raise_exception(ActiveRecord::RecordNotFound) + expect { subject.perform(user.id, 1234, {}) }.to raise_exception(ActiveRecord::RecordNotFound) + end + end + end +end |