Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2019-11-12 00:06:20 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2019-11-12 00:06:20 +0300
commit0c3f12149372a79b825d265a6c28dc547e4a1afc (patch)
tree43bdaa20afb0b061d09c56d8507efd51d28601be
parentff67e3ed08355fb2d6f6e69d4ed06cd09052e573 (diff)
Add latest changes from gitlab-org/gitlab@master
-rw-r--r--app/finders/todos_finder.rb12
-rw-r--r--app/models/group.rb2
-rw-r--r--app/models/import_export_upload.rb1
-rw-r--r--app/services/groups/import_export/export_service.rb71
-rw-r--r--app/services/projects/import_export/export_service.rb2
-rw-r--r--app/workers/all_queues.yml1
-rw-r--r--app/workers/group_export_worker.rb15
-rw-r--r--config/sidekiq_queues.yml1
-rw-r--r--db/migrate/20191111115229_add_group_id_to_import_export_uploads.rb9
-rw-r--r--db/migrate/20191111115431_add_group_fk_to_import_export_uploads.rb19
-rw-r--r--db/schema.rb5
-rw-r--r--lib/gitlab/import_export.rb14
-rw-r--r--lib/gitlab/import_export/config.rb5
-rw-r--r--lib/gitlab/import_export/file_importer.rb2
-rw-r--r--lib/gitlab/import_export/group_import_export.yml36
-rw-r--r--lib/gitlab/import_export/group_tree_saver.rb55
-rw-r--r--lib/gitlab/import_export/project_tree_saver.rb31
-rw-r--r--lib/gitlab/import_export/reader.rb27
-rw-r--r--lib/gitlab/import_export/relation_rename_service.rb2
-rw-r--r--lib/gitlab/import_export/relation_tree_saver.rb27
-rw-r--r--lib/gitlab/import_export/saver.rb18
-rw-r--r--lib/gitlab/import_export/shared.rb40
-rw-r--r--spec/finders/todos_finder_spec.rb13
-rw-r--r--spec/fixtures/group_export.tar.gzbin0 -> 4551 bytes
-rw-r--r--spec/lib/gitlab/import_export/group_tree_saver_spec.rb162
-rw-r--r--spec/lib/gitlab/import_export/import_export_spec.rb6
-rw-r--r--spec/lib/gitlab/import_export/relation_rename_service_spec.rb7
-rw-r--r--spec/lib/gitlab/import_export/relation_tree_saver_spec.rb42
-rw-r--r--spec/lib/gitlab/import_export/saver_spec.rb2
-rw-r--r--spec/lib/gitlab/import_export/shared_spec.rb2
-rw-r--r--spec/services/groups/import_export/export_service_spec.rb55
-rw-r--r--spec/services/import_export_clean_up_service_spec.rb2
-rw-r--r--spec/services/projects/import_export/export_service_spec.rb2
-rw-r--r--spec/workers/group_export_worker_spec.rb29
34 files changed, 648 insertions, 69 deletions
diff --git a/app/finders/todos_finder.rb b/app/finders/todos_finder.rb
index 82acb5e9d45..cc71ea8e916 100644
--- a/app/finders/todos_finder.rb
+++ b/app/finders/todos_finder.rb
@@ -23,10 +23,16 @@ class TodosFinder
NONE = '0'
- TODO_TYPES = Set.new(%w(Issue MergeRequest Epic)).freeze
+ TODO_TYPES = Set.new(%w(Issue MergeRequest)).freeze
attr_accessor :current_user, :params
+ class << self
+ def todo_types
+ TODO_TYPES
+ end
+ end
+
def initialize(current_user, params = {})
@current_user = current_user
@params = params
@@ -124,7 +130,7 @@ class TodosFinder
end
def type?
- type.present? && TODO_TYPES.include?(type)
+ type.present? && self.class.todo_types.include?(type)
end
def type
@@ -201,3 +207,5 @@ class TodosFinder
end
end
end
+
+TodosFinder.prepend_if_ee('EE::TodosFinder')
diff --git a/app/models/group.rb b/app/models/group.rb
index 7760a3c69ce..71d81289bf5 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -55,6 +55,8 @@ class Group < Namespace
has_many :todos
+ has_one :import_export_upload
+
accepts_nested_attributes_for :variables, allow_destroy: true
validate :visibility_level_allowed_by_projects
diff --git a/app/models/import_export_upload.rb b/app/models/import_export_upload.rb
index 60f5491849a..7d73fd281f1 100644
--- a/app/models/import_export_upload.rb
+++ b/app/models/import_export_upload.rb
@@ -5,6 +5,7 @@ class ImportExportUpload < ApplicationRecord
include ObjectStorage::BackgroundMove
belongs_to :project
+ belongs_to :group
# These hold the project Import/Export archives (.tar.gz files)
mount_uploader :import_file, ImportExportUploader
diff --git a/app/services/groups/import_export/export_service.rb b/app/services/groups/import_export/export_service.rb
new file mode 100644
index 00000000000..26886fc67dc
--- /dev/null
+++ b/app/services/groups/import_export/export_service.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Groups
+ module ImportExport
+ class ExportService
+ def initialize(group:, user:, params: {})
+ @group = group
+ @current_user = user
+ @params = params
+ @shared = @params[:shared] || Gitlab::ImportExport::Shared.new(@group)
+ end
+
+ def execute
+ save!
+ end
+
+ private
+
+ attr_accessor :shared
+
+ def save!
+ if savers.all?(&:save)
+ notify_success
+ else
+ cleanup_and_notify_error!
+ end
+ end
+
+ def savers
+ [tree_exporter, file_saver]
+ end
+
+ def tree_exporter
+ Gitlab::ImportExport::GroupTreeSaver.new(group: @group, current_user: @current_user, shared: @shared, params: @params)
+ end
+
+ def file_saver
+ Gitlab::ImportExport::Saver.new(exportable: @group, shared: @shared)
+ end
+
+ def cleanup_and_notify_error
+ FileUtils.rm_rf(shared.export_path)
+
+ notify_error
+ end
+
+ def cleanup_and_notify_error!
+ cleanup_and_notify_error
+
+ raise Gitlab::ImportExport::Error.new(shared.errors.to_sentence)
+ end
+
+ def notify_success
+ @shared.logger.info(
+ group_id: @group.id,
+ group_name: @group.name,
+ message: 'Group Import/Export: Export succeeded'
+ )
+ end
+
+ def notify_error
+ @shared.logger.error(
+ group_id: @group.id,
+ group_name: @group.name,
+ error: @shared.errors.join(', '),
+ message: 'Group Import/Export: Export failed'
+ )
+ end
+ end
+ end
+end
diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb
index d3638c57552..8344397f67d 100644
--- a/app/services/projects/import_export/export_service.rb
+++ b/app/services/projects/import_export/export_service.rb
@@ -24,7 +24,7 @@ module Projects
def save_all!
if save_exporters
- Gitlab::ImportExport::Saver.save(project: project, shared: shared)
+ Gitlab::ImportExport::Saver.save(exportable: project, shared: shared)
notify_success
else
cleanup_and_notify_error!
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index 10081840305..66b5214cfcb 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -179,3 +179,4 @@
- import_issues_csv
- project_daily_statistics
- create_evidence
+- group_export
diff --git a/app/workers/group_export_worker.rb b/app/workers/group_export_worker.rb
new file mode 100644
index 00000000000..51dbdc95661
--- /dev/null
+++ b/app/workers/group_export_worker.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class GroupExportWorker
+ include ApplicationWorker
+ include ExceptionBacktrace
+
+ feature_category :source_code_management
+
+ def perform(current_user_id, group_id, params = {})
+ current_user = User.find(current_user_id)
+ group = Group.find(group_id)
+
+ ::Groups::ImportExport::ExportService.new(group: group, user: current_user, params: params).execute
+ end
+end
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 7f8ba35bf41..b4be61d8a3d 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -98,6 +98,7 @@
- [update_namespace_statistics, 1]
- [chaos, 2]
- [create_evidence, 2]
+ - [group_export, 1]
# EE-specific queues
- [analytics, 1]
diff --git a/db/migrate/20191111115229_add_group_id_to_import_export_uploads.rb b/db/migrate/20191111115229_add_group_id_to_import_export_uploads.rb
new file mode 100644
index 00000000000..74ef0f27b3e
--- /dev/null
+++ b/db/migrate/20191111115229_add_group_id_to_import_export_uploads.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddGroupIdToImportExportUploads < ActiveRecord::Migration[5.2]
+ DOWNTIME = false
+
+ def change
+ add_column :import_export_uploads, :group_id, :bigint
+ end
+end
diff --git a/db/migrate/20191111115431_add_group_fk_to_import_export_uploads.rb b/db/migrate/20191111115431_add_group_fk_to_import_export_uploads.rb
new file mode 100644
index 00000000000..403de3f33ed
--- /dev/null
+++ b/db/migrate/20191111115431_add_group_fk_to_import_export_uploads.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class AddGroupFkToImportExportUploads < ActiveRecord::Migration[5.2]
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ def up
+ add_concurrent_foreign_key :import_export_uploads, :namespaces, column: :group_id, on_delete: :cascade
+ add_concurrent_index :import_export_uploads, :group_id, unique: true, where: 'group_id IS NOT NULL'
+ end
+
+ def down
+ remove_foreign_key_without_error(:import_export_uploads, column: :group_id)
+ remove_concurrent_index(:import_export_uploads, :group_id)
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index c11c1059fd2..b5991904302 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 2019_11_05_140942) do
+ActiveRecord::Schema.define(version: 2019_11_11_115431) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_trgm"
@@ -1874,6 +1874,8 @@ ActiveRecord::Schema.define(version: 2019_11_05_140942) do
t.integer "project_id"
t.text "import_file"
t.text "export_file"
+ t.bigint "group_id"
+ t.index ["group_id"], name: "index_import_export_uploads_on_group_id", unique: true, where: "(group_id IS NOT NULL)"
t.index ["project_id"], name: "index_import_export_uploads_on_project_id"
t.index ["updated_at"], name: "index_import_export_uploads_on_updated_at"
end
@@ -4288,6 +4290,7 @@ ActiveRecord::Schema.define(version: 2019_11_05_140942) do
add_foreign_key "group_group_links", "namespaces", column: "shared_group_id", on_delete: :cascade
add_foreign_key "group_group_links", "namespaces", column: "shared_with_group_id", on_delete: :cascade
add_foreign_key "identities", "saml_providers", name: "fk_aade90f0fc", on_delete: :cascade
+ add_foreign_key "import_export_uploads", "namespaces", column: "group_id", name: "fk_83319d9721", on_delete: :cascade
add_foreign_key "import_export_uploads", "projects", on_delete: :cascade
add_foreign_key "index_statuses", "projects", name: "fk_74b2492545", on_delete: :cascade
add_foreign_key "insights", "namespaces", on_delete: :cascade
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index b2ac60fe825..516e7f54a6e 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -15,7 +15,7 @@ module Gitlab
end
def storage_path
- File.join(Settings.shared['path'], 'tmp/project_exports')
+ File.join(Settings.shared['path'], 'tmp/gitlab_exports')
end
def import_upload_path(filename:)
@@ -50,8 +50,8 @@ module Gitlab
'VERSION'
end
- def export_filename(project:)
- basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.full_path.tr('/', '_')}"
+ def export_filename(exportable:)
+ basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{exportable.full_path.tr('/', '_')}"
"#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
end
@@ -63,6 +63,14 @@ module Gitlab
def reset_tokens?
true
end
+
+ def group_filename
+ 'group.json'
+ end
+
+ def group_config_file
+ Rails.root.join('lib/gitlab/import_export/group_import_export.yml')
+ end
end
end
diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb
index 6f4919ead4e..83c4bc47349 100644
--- a/lib/gitlab/import_export/config.rb
+++ b/lib/gitlab/import_export/config.rb
@@ -3,7 +3,8 @@
module Gitlab
module ImportExport
class Config
- def initialize
+ def initialize(config: Gitlab::ImportExport.config_file)
+ @config = config
@hash = parse_yaml
@hash.deep_symbolize_keys!
@ee_hash = @hash.delete(:ee) || {}
@@ -50,7 +51,7 @@ module Gitlab
end
def parse_yaml
- YAML.load_file(Gitlab::ImportExport.config_file)
+ YAML.load_file(@config)
end
end
end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 05432f433e7..2fd12e3aa78 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -60,7 +60,7 @@ module Gitlab
def copy_archive
return if @archive_file
- @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
+ @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @project))
download_or_copy_upload(@project.import_export_upload.import_file, @archive_file)
end
diff --git a/lib/gitlab/import_export/group_import_export.yml b/lib/gitlab/import_export/group_import_export.yml
new file mode 100644
index 00000000000..c1900350c86
--- /dev/null
+++ b/lib/gitlab/import_export/group_import_export.yml
@@ -0,0 +1,36 @@
+# Model relationships to be included in the group import/export
+#
+# This list _must_ only contain relationships that are available to both FOSS and
+# Enterprise editions. EE specific relationships must be defined in the `ee` section further
+# down below.
+tree:
+ group:
+ - :milestones
+ - :badges
+ - labels:
+ - :priorities
+ - :boards
+ - members:
+ - :user
+
+included_attributes:
+
+excluded_attributes:
+ group:
+ - :runners_token
+ - :runners_token_encrypted
+
+methods:
+ labels:
+ - :type
+ badges:
+ - :type
+
+preloads:
+
+# EE specific relationships and settings to include. All of this will be merged
+# into the previous structures if EE is used.
+ee:
+ tree:
+ group:
+ - :epics
diff --git a/lib/gitlab/import_export/group_tree_saver.rb b/lib/gitlab/import_export/group_tree_saver.rb
new file mode 100644
index 00000000000..1d42bc8d3f3
--- /dev/null
+++ b/lib/gitlab/import_export/group_tree_saver.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ class GroupTreeSaver
+ attr_reader :full_path
+
+ def initialize(group:, current_user:, shared:, params: {})
+ @params = params
+ @current_user = current_user
+ @shared = shared
+ @group = group
+ @full_path = File.join(@shared.export_path, ImportExport.group_filename)
+ end
+
+ def save
+ group_tree = serialize(@group, reader.group_tree)
+ tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename)
+
+ true
+ rescue => e
+ @shared.error(e)
+ false
+ end
+
+ private
+
+ def serialize(group, relations_tree)
+ group_tree = tree_saver.serialize(group, relations_tree)
+
+ group.descendants.each do |descendant|
+ group_tree['descendants'] = [] unless group_tree['descendants']
+ group_tree['descendants'] << serialize(descendant, relations_tree)
+ end
+
+ group_tree
+ rescue => e
+ @shared.error(e)
+ end
+
+ def reader
+ @reader ||= Gitlab::ImportExport::Reader.new(
+ shared: @shared,
+ config: Gitlab::ImportExport::Config.new(
+ config: Gitlab::ImportExport.group_config_file
+ ).to_h
+ )
+ end
+
+ def tree_saver
+ @tree_saver ||= RelationTreeSaver.new
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb
index 63c71105efe..386a4cfdfc6 100644
--- a/lib/gitlab/import_export/project_tree_saver.rb
+++ b/lib/gitlab/import_export/project_tree_saver.rb
@@ -3,25 +3,20 @@
module Gitlab
module ImportExport
class ProjectTreeSaver
- include Gitlab::ImportExport::CommandLineUtil
-
attr_reader :full_path
def initialize(project:, current_user:, shared:, params: {})
- @params = params
- @project = project
+ @params = params
+ @project = project
@current_user = current_user
- @shared = shared
- @full_path = File.join(@shared.export_path, ImportExport.project_filename)
+ @shared = shared
+ @full_path = File.join(@shared.export_path, ImportExport.project_filename)
end
def save
- mkdir_p(@shared.export_path)
-
- project_tree = serialize_project_tree
+ project_tree = tree_saver.serialize(@project, reader.project_tree)
fix_project_tree(project_tree)
- project_tree_json = JSON.generate(project_tree)
- File.write(full_path, project_tree_json)
+ tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
true
rescue => e
@@ -43,16 +38,6 @@ module Gitlab
RelationRenameService.add_new_associations(project_tree)
end
- def serialize_project_tree
- if Feature.enabled?(:export_fast_serialize, default_enabled: true)
- Gitlab::ImportExport::FastHashSerializer
- .new(@project, reader.project_tree)
- .execute
- else
- @project.as_json(reader.project_tree)
- end
- end
-
def reader
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
end
@@ -74,6 +59,10 @@ module Gitlab
GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
end
+
+ def tree_saver
+ @tree_saver ||= RelationTreeSaver.new
+ end
end
end
end
diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
index 9e81c6a3d07..1390770acef 100644
--- a/lib/gitlab/import_export/reader.rb
+++ b/lib/gitlab/import_export/reader.rb
@@ -5,24 +5,31 @@ module Gitlab
class Reader
attr_reader :tree, :attributes_finder
- def initialize(shared:)
- @shared = shared
-
- @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(
- config: ImportExport::Config.new.to_h)
+ def initialize(shared:, config: ImportExport::Config.new.to_h)
+ @shared = shared
+ @config = config
+ @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config)
end
# Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
# for outputting a project in JSON format, including its relations and sub relations.
def project_tree
- attributes_finder.find_root(:project)
- rescue => e
- @shared.error(e)
- false
+ tree_by_key(:project)
+ end
+
+ def group_tree
+ tree_by_key(:group)
end
def group_members_tree
- attributes_finder.find_root(:group_members)
+ tree_by_key(:group_members)
+ end
+
+ def tree_by_key(key)
+ attributes_finder.find_root(key)
+ rescue => e
+ @shared.error(e)
+ false
end
end
end
diff --git a/lib/gitlab/import_export/relation_rename_service.rb b/lib/gitlab/import_export/relation_rename_service.rb
index 179bde5e21e..03aaa6aefc3 100644
--- a/lib/gitlab/import_export/relation_rename_service.rb
+++ b/lib/gitlab/import_export/relation_rename_service.rb
@@ -8,7 +8,7 @@
# The behavior of these renamed relationships should be transient and it should
# only last one release until you completely remove the renaming from the list.
#
-# When importing, this class will check the project hash and:
+# When importing, this class will check the hash and:
# - if only the old relationship name is found, it will rename it with the new one
# - if only the new relationship name is found, it will do nothing
# - if it finds both, it will use the new relationship data
diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/relation_tree_saver.rb
new file mode 100644
index 00000000000..a0452071ccf
--- /dev/null
+++ b/lib/gitlab/import_export/relation_tree_saver.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ class RelationTreeSaver
+ include Gitlab::ImportExport::CommandLineUtil
+
+ def serialize(exportable, relations_tree)
+ if Feature.enabled?(:export_fast_serialize, default_enabled: true)
+ Gitlab::ImportExport::FastHashSerializer
+ .new(exportable, relations_tree)
+ .execute
+ else
+ exportable.as_json(relations_tree)
+ end
+ end
+
+ def save(tree, dir_path, filename)
+ mkdir_p(dir_path)
+
+ tree_json = JSON.generate(tree)
+
+ File.write(File.join(dir_path, filename), tree_json)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index bea7a7cce65..ae82c380755 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -9,16 +9,16 @@ module Gitlab
new(*args).save
end
- def initialize(project:, shared:)
- @project = project
- @shared = shared
+ def initialize(exportable:, shared:)
+ @exportable = exportable
+ @shared = shared
end
def save
if compress_and_save
remove_export_path
- Rails.logger.info("Saved project export #{archive_file}") # rubocop:disable Gitlab/RailsLogger
+ Rails.logger.info("Saved #{@exportable.class} export #{archive_file}") # rubocop:disable Gitlab/RailsLogger
save_upload
else
@@ -48,11 +48,11 @@ module Gitlab
end
def archive_file
- @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
+ @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @exportable))
end
def save_upload
- upload = ImportExportUpload.find_or_initialize_by(project: @project)
+ upload = initialize_upload
File.open(archive_file) { |file| upload.export_file = file }
@@ -62,6 +62,12 @@ module Gitlab
def error_message
"Unable to save #{archive_file} into #{@shared.export_path}."
end
+
+ def initialize_upload
+ exportable_kind = @exportable.class.name.downcase
+
+ ImportExportUpload.find_or_initialize_by(Hash[exportable_kind, @exportable])
+ end
end
end
end
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
index 02d46a1f498..2539a6828c3 100644
--- a/lib/gitlab/import_export/shared.rb
+++ b/lib/gitlab/import_export/shared.rb
@@ -23,21 +23,21 @@
module Gitlab
module ImportExport
class Shared
- attr_reader :errors, :project
+ attr_reader :errors, :exportable, :logger
LOCKS_DIRECTORY = 'locks'
- def initialize(project)
- @project = project
- @errors = []
- @logger = Gitlab::Import::Logger.build
+ def initialize(exportable)
+ @exportable = exportable
+ @errors = []
+ @logger = Gitlab::Import::Logger.build
end
def active_export_count
Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) }
end
- # The path where the project metadata and repository bundle is saved
+ # The path where the exportable metadata and repository bundle (in case of project) is saved
def export_path
@export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path)
end
@@ -84,11 +84,18 @@ module Gitlab
end
def relative_archive_path
- @relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex)
+ @relative_archive_path ||= File.join(relative_base_path, SecureRandom.hex)
end
def relative_base_path
- @project.disk_path
+ case exportable_type
+ when 'Project'
+ @exportable.disk_path
+ when 'Group'
+ @exportable.full_path
+ else
+ raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}")
+ end
end
def log_error(details)
@@ -100,17 +107,24 @@ module Gitlab
end
def log_base_data
- {
- importer: 'Import/Export',
- import_jid: @project&.import_state&.jid,
- project_id: @project&.id,
- project_path: @project&.full_path
+ log = {
+ importer: 'Import/Export',
+ exportable_id: @exportable&.id,
+ exportable_path: @exportable&.full_path
}
+
+ log[:import_jid] = @exportable&.import_state&.jid if exportable_type == 'Project'
+
+ log
end
def filtered_error_message(message)
Projects::ImportErrorFilter.filter_message(message)
end
+
+ def exportable_type
+ @exportable.class.name
+ end
end
end
end
diff --git a/spec/finders/todos_finder_spec.rb b/spec/finders/todos_finder_spec.rb
index 6234d596745..75aa10c66a2 100644
--- a/spec/finders/todos_finder_spec.rb
+++ b/spec/finders/todos_finder_spec.rb
@@ -234,6 +234,19 @@ describe TodosFinder do
end
end
+ describe '.todo_types' do
+ it 'returns the expected types' do
+ expected_result =
+ if Gitlab.ee?
+ %w[Epic Issue MergeRequest]
+ else
+ %w[Issue MergeRequest]
+ end
+
+ expect(described_class.todo_types).to contain_exactly(*expected_result)
+ end
+ end
+
describe '#any_for_target?' do
it 'returns true if there are any todos for the given target' do
todo = create(:todo, :pending)
diff --git a/spec/fixtures/group_export.tar.gz b/spec/fixtures/group_export.tar.gz
new file mode 100644
index 00000000000..83e360d7cc2
--- /dev/null
+++ b/spec/fixtures/group_export.tar.gz
Binary files differ
diff --git a/spec/lib/gitlab/import_export/group_tree_saver_spec.rb b/spec/lib/gitlab/import_export/group_tree_saver_spec.rb
new file mode 100644
index 00000000000..c752c557d99
--- /dev/null
+++ b/spec/lib/gitlab/import_export/group_tree_saver_spec.rb
@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::GroupTreeSaver do
+ describe 'saves the group tree into a json object' do
+ let(:shared) { Gitlab::ImportExport::Shared.new(group) }
+ let(:group_tree_saver) { described_class.new(group: group, current_user: user, shared: shared) }
+ let(:export_path) { "#{Dir.tmpdir}/group_tree_saver_spec" }
+ let(:user) { create(:user) }
+ let!(:group) { setup_group }
+
+ before do
+ group.add_maintainer(user)
+ allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+ end
+
+ after do
+ FileUtils.rm_rf(export_path)
+ end
+
+ it 'saves group successfully' do
+ expect(group_tree_saver.save).to be true
+ end
+
+ context ':export_fast_serialize feature flag checks' do
+ before do
+ expect(Gitlab::ImportExport::Reader).to receive(:new).with(shared: shared, config: group_config).and_return(reader)
+ expect(reader).to receive(:group_tree).and_return(group_tree)
+ end
+
+ let(:reader) { instance_double('Gitlab::ImportExport::Reader') }
+ let(:group_config) { Gitlab::ImportExport::Config.new(config: Gitlab::ImportExport.group_config_file).to_h }
+ let(:group_tree) do
+ {
+ include: [{ milestones: { include: [] } }],
+ preload: { milestones: nil }
+ }
+ end
+
+ context 'when :export_fast_serialize feature is enabled' do
+ before do
+ stub_feature_flags(export_fast_serialize: true)
+ end
+
+ it 'uses FastHashSerializer' do
+ expect_any_instance_of(Gitlab::ImportExport::FastHashSerializer).to receive(:execute).and_call_original
+
+ group_tree_saver.save
+ end
+ end
+
+ context 'when :export_fast_serialize feature is disabled' do
+ before do
+ stub_feature_flags(export_fast_serialize: false)
+ end
+
+ it 'is serialized via built-in `as_json`' do
+ expect(group).to receive(:as_json).with(group_tree).and_call_original
+
+ group_tree_saver.save
+ end
+ end
+ end
+
+ # It is mostly duplicated in
+ # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb`
+ # except:
+ # context 'with description override' do
+ # context 'group members' do
+ # ^ These are specific for the groupTreeSaver
+ context 'JSON' do
+ let(:saved_group_json) do
+ group_tree_saver.save
+ group_json(group_tree_saver.full_path)
+ end
+
+ it 'saves the correct json' do
+ expect(saved_group_json).to include({ 'description' => 'description', 'visibility_level' => 20 })
+ end
+
+ it 'has milestones' do
+ expect(saved_group_json['milestones']).not_to be_empty
+ end
+
+ it 'has labels' do
+ expect(saved_group_json['labels']).not_to be_empty
+ end
+
+ it 'has boards' do
+ expect(saved_group_json['boards']).not_to be_empty
+ end
+
+ it 'has group members' do
+ expect(saved_group_json['members']).not_to be_empty
+ end
+
+ it 'has priorities associated to labels' do
+ expect(saved_group_json['labels'].first['priorities']).not_to be_empty
+ end
+
+ it 'has badges' do
+ expect(saved_group_json['badges']).not_to be_empty
+ end
+
+ context 'group members' do
+ let(:user2) { create(:user, email: 'group@member.com') }
+ let(:member_emails) do
+ saved_group_json['members'].map do |pm|
+ pm['user']['email']
+ end
+ end
+
+ before do
+ group.add_developer(user2)
+ end
+
+ it 'exports group members as group owner' do
+ group.add_owner(user)
+
+ expect(member_emails).to include('group@member.com')
+ end
+
+ context 'as admin' do
+ let(:user) { create(:admin) }
+
+ it 'exports group members as admin' do
+ expect(member_emails).to include('group@member.com')
+ end
+
+ it 'exports group members' do
+ member_types = saved_group_json['members'].map { |pm| pm['source_type'] }
+
+ expect(member_types).to all(eq('Namespace'))
+ end
+ end
+ end
+
+ context 'group attributes' do
+ it 'does not contain the runners token' do
+ expect(saved_group_json).not_to include("runners_token" => 'token')
+ end
+ end
+ end
+ end
+
+ def setup_group
+ group = create(:group, description: 'description')
+ create(:milestone, group: group)
+ create(:group_badge, group: group)
+ group_label = create(:group_label, group: group)
+ create(:label_priority, label: group_label, priority: 1)
+ create(:board, group: group)
+ create(:group_badge, group: group)
+
+ group
+ end
+
+ def group_json(filename)
+ JSON.parse(IO.read(filename))
+ end
+end
diff --git a/spec/lib/gitlab/import_export/import_export_spec.rb b/spec/lib/gitlab/import_export/import_export_spec.rb
index 40a5f2294a2..a6b0dc758cd 100644
--- a/spec/lib/gitlab/import_export/import_export_spec.rb
+++ b/spec/lib/gitlab/import_export/import_export_spec.rb
@@ -6,17 +6,17 @@ describe Gitlab::ImportExport do
let(:project) { create(:project, :public, path: 'project-path', namespace: group) }
it 'contains the project path' do
- expect(described_class.export_filename(project: project)).to include(project.path)
+ expect(described_class.export_filename(exportable: project)).to include(project.path)
end
it 'contains the namespace path' do
- expect(described_class.export_filename(project: project)).to include(project.namespace.full_path.tr('/', '_'))
+ expect(described_class.export_filename(exportable: project)).to include(project.namespace.full_path.tr('/', '_'))
end
it 'does not go over a certain length' do
project.path = 'a' * 100
- expect(described_class.export_filename(project: project).length).to be < 70
+ expect(described_class.export_filename(exportable: project).length).to be < 70
end
end
end
diff --git a/spec/lib/gitlab/import_export/relation_rename_service_spec.rb b/spec/lib/gitlab/import_export/relation_rename_service_spec.rb
index 843de27df1a..d62f5725f9e 100644
--- a/spec/lib/gitlab/import_export/relation_rename_service_spec.rb
+++ b/spec/lib/gitlab/import_export/relation_rename_service_spec.rb
@@ -96,15 +96,20 @@ describe Gitlab::ImportExport::RelationRenameService do
let(:export_content_path) { project_tree_saver.full_path }
let(:export_content_hash) { ActiveSupport::JSON.decode(File.read(export_content_path)) }
let(:injected_hash) { renames.values.product([{}]).to_h }
+ let(:relation_tree_saver) { Gitlab::ImportExport::RelationTreeSaver.new }
let(:project_tree_saver) do
Gitlab::ImportExport::ProjectTreeSaver.new(
project: project, current_user: user, shared: shared)
end
+ before do
+ allow(project_tree_saver).to receive(:tree_saver).and_return(relation_tree_saver)
+ end
+
it 'adds old relationships to the exported file' do
# we inject relations with new names that should be rewritten
- expect(project_tree_saver).to receive(:serialize_project_tree).and_wrap_original do |method, *args|
+ expect(relation_tree_saver).to receive(:serialize).and_wrap_original do |method, *args|
method.call(*args).merge(injected_hash)
end
diff --git a/spec/lib/gitlab/import_export/relation_tree_saver_spec.rb b/spec/lib/gitlab/import_export/relation_tree_saver_spec.rb
new file mode 100644
index 00000000000..2fc26c0e3d4
--- /dev/null
+++ b/spec/lib/gitlab/import_export/relation_tree_saver_spec.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::ImportExport::RelationTreeSaver do
+ let(:exportable) { create(:group) }
+ let(:relation_tree_saver) { described_class.new }
+ let(:tree) { {} }
+
+ describe '#serialize' do
+ context 'when :export_fast_serialize feature is enabled' do
+ let(:serializer) { instance_double(Gitlab::ImportExport::FastHashSerializer) }
+
+ before do
+ stub_feature_flags(export_fast_serialize: true)
+ end
+
+ it 'uses FastHashSerializer' do
+ expect(Gitlab::ImportExport::FastHashSerializer)
+ .to receive(:new)
+ .with(exportable, tree)
+ .and_return(serializer)
+
+ expect(serializer).to receive(:execute)
+
+ relation_tree_saver.serialize(exportable, tree)
+ end
+ end
+
+ context 'when :export_fast_serialize feature is disabled' do
+ before do
+ stub_feature_flags(export_fast_serialize: false)
+ end
+
+ it 'is serialized via built-in `as_json`' do
+ expect(exportable).to receive(:as_json).with(tree)
+
+ relation_tree_saver.serialize(exportable, tree)
+ end
+ end
+ end
+end
diff --git a/spec/lib/gitlab/import_export/saver_spec.rb b/spec/lib/gitlab/import_export/saver_spec.rb
index d185ff2dfcc..aca63953677 100644
--- a/spec/lib/gitlab/import_export/saver_spec.rb
+++ b/spec/lib/gitlab/import_export/saver_spec.rb
@@ -5,7 +5,7 @@ describe Gitlab::ImportExport::Saver do
let!(:project) { create(:project, :public, name: 'project') }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:shared) { project.import_export_shared }
- subject { described_class.new(project: project, shared: shared) }
+ subject { described_class.new(exportable: project, shared: shared) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
diff --git a/spec/lib/gitlab/import_export/shared_spec.rb b/spec/lib/gitlab/import_export/shared_spec.rb
index 62669836973..fc011f7e1be 100644
--- a/spec/lib/gitlab/import_export/shared_spec.rb
+++ b/spec/lib/gitlab/import_export/shared_spec.rb
@@ -7,7 +7,7 @@ describe Gitlab::ImportExport::Shared do
context 'with a repository on disk' do
let(:project) { create(:project, :repository) }
- let(:base_path) { %(/tmp/project_exports/#{project.disk_path}/) }
+ let(:base_path) { %(/tmp/gitlab_exports/#{project.disk_path}/) }
describe '#archive_path' do
it 'uses a random hash to avoid conflicts' do
diff --git a/spec/services/groups/import_export/export_service_spec.rb b/spec/services/groups/import_export/export_service_spec.rb
new file mode 100644
index 00000000000..2024e1ed457
--- /dev/null
+++ b/spec/services/groups/import_export/export_service_spec.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Groups::ImportExport::ExportService do
+ describe '#execute' do
+ let!(:user) { create(:user) }
+ let(:group) { create(:group) }
+ let(:shared) { Gitlab::ImportExport::Shared.new(group) }
+ let(:export_path) { shared.export_path }
+ let(:service) { described_class.new(group: group, user: user, params: { shared: shared }) }
+
+ after do
+ FileUtils.rm_rf(export_path)
+ end
+
+ it 'saves the models' do
+ expect(Gitlab::ImportExport::GroupTreeSaver).to receive(:new).and_call_original
+
+ service.execute
+ end
+
+ context 'when saver succeeds' do
+ it 'saves the group in the file system' do
+ service.execute
+
+ expect(group.import_export_upload.export_file.file).not_to be_nil
+ expect(File.directory?(export_path)).to eq(false)
+ expect(File.exist?(shared.archive_path)).to eq(false)
+ end
+ end
+
+ context 'when saving services fail' do
+ before do
+ allow(service).to receive_message_chain(:tree_exporter, :save).and_return(false)
+ end
+
+ it 'removes the remaining exported data' do
+ allow_any_instance_of(Gitlab::ImportExport::Saver).to receive(:compress_and_save).and_return(false)
+
+ expect { service.execute }.to raise_error(Gitlab::ImportExport::Error)
+
+ expect(group.import_export_upload).to be_nil
+ expect(File.directory?(export_path)).to eq(false)
+ expect(File.exist?(shared.archive_path)).to eq(false)
+ end
+
+ it 'notifies logger' do
+ expect_any_instance_of(Gitlab::Import::Logger).to receive(:error)
+
+ expect { service.execute }.to raise_error(Gitlab::ImportExport::Error)
+ end
+ end
+ end
+end
diff --git a/spec/services/import_export_clean_up_service_spec.rb b/spec/services/import_export_clean_up_service_spec.rb
index 51720e786dc..9f811f56f50 100644
--- a/spec/services/import_export_clean_up_service_spec.rb
+++ b/spec/services/import_export_clean_up_service_spec.rb
@@ -6,7 +6,7 @@ describe ImportExportCleanUpService do
describe '#execute' do
let(:service) { described_class.new }
- let(:tmp_import_export_folder) { 'tmp/project_exports' }
+ let(:tmp_import_export_folder) { 'tmp/gitlab_exports' }
context 'when the import/export directory does not exist' do
it 'does not remove any archives' do
diff --git a/spec/services/projects/import_export/export_service_spec.rb b/spec/services/projects/import_export/export_service_spec.rb
index 146d656c909..a557e61da78 100644
--- a/spec/services/projects/import_export/export_service_spec.rb
+++ b/spec/services/projects/import_export/export_service_spec.rb
@@ -66,7 +66,7 @@ describe Projects::ImportExport::ExportService do
end
it 'saves the project in the file system' do
- expect(Gitlab::ImportExport::Saver).to receive(:save).with(project: project, shared: shared)
+ expect(Gitlab::ImportExport::Saver).to receive(:save).with(exportable: project, shared: shared)
service.execute
end
diff --git a/spec/workers/group_export_worker_spec.rb b/spec/workers/group_export_worker_spec.rb
new file mode 100644
index 00000000000..4aa85d2b381
--- /dev/null
+++ b/spec/workers/group_export_worker_spec.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe GroupExportWorker do
+ let!(:user) { create(:user) }
+ let!(:group) { create(:group) }
+
+ subject { described_class.new }
+
+ describe '#perform' do
+ context 'when it succeeds' do
+ it 'calls the ExportService' do
+ expect_any_instance_of(::Groups::ImportExport::ExportService).to receive(:execute)
+
+ subject.perform(user.id, group.id, {})
+ end
+ end
+
+ context 'when it fails' do
+ it 'raises an exception when params are invalid' do
+ expect_any_instance_of(::Groups::ImportExport::ExportService).not_to receive(:execute)
+
+ expect { subject.perform(1234, group.id, {}) }.to raise_exception(ActiveRecord::RecordNotFound)
+ expect { subject.perform(user.id, 1234, {}) }.to raise_exception(ActiveRecord::RecordNotFound)
+ end
+ end
+ end
+end