Add latest changes from gitlab-org/gitlab@master

10 files changed, 189 insertions, 54 deletions

diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb
index 6f4919ead4e..83c4bc47349 100644
--- a/lib/gitlab/import_export/config.rb
+++ b/lib/gitlab/import_export/config.rb
@@ -3,7 +3,8 @@
 module Gitlab
   module ImportExport
     class Config
-      def initialize
+      def initialize(config: Gitlab::ImportExport.config_file)
+        @config = config
         @hash = parse_yaml
         @hash.deep_symbolize_keys!
         @ee_hash = @hash.delete(:ee) || {}
@@ -50,7 +51,7 @@ module Gitlab
       end
 
       def parse_yaml
-        YAML.load_file(Gitlab::ImportExport.config_file)
+        YAML.load_file(@config)
       end
     end
   end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 05432f433e7..2fd12e3aa78 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -60,7 +60,7 @@ module Gitlab
       def copy_archive
         return if @archive_file
 
-        @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
+        @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @project))
 
         download_or_copy_upload(@project.import_export_upload.import_file, @archive_file)
       end
diff --git a/lib/gitlab/import_export/group_import_export.yml b/lib/gitlab/import_export/group_import_export.yml
new file mode 100644
index 00000000000..c1900350c86
--- /dev/null
+++ b/lib/gitlab/import_export/group_import_export.yml
@@ -0,0 +1,36 @@
+# Model relationships to be included in the group import/export
+#
+# This list _must_ only contain relationships that are available to both FOSS and
+# Enterprise editions. EE specific relationships must be defined in the `ee` section further
+# down below.
+tree:
+  group:
+    - :milestones
+    - :badges
+    - labels:
+        - :priorities
+    - :boards
+    - members:
+        - :user
+
+included_attributes:
+
+excluded_attributes:
+  group:
+    - :runners_token
+    - :runners_token_encrypted
+
+methods:
+  labels:
+    - :type
+  badges:
+    - :type
+
+preloads:
+
+# EE specific relationships and settings to include. All of this will be merged
+# into the previous structures if EE is used.
+ee:
+  tree:
+    group:
+      - :epics
diff --git a/lib/gitlab/import_export/group_tree_saver.rb b/lib/gitlab/import_export/group_tree_saver.rb
new file mode 100644
index 00000000000..1d42bc8d3f3
--- /dev/null
+++ b/lib/gitlab/import_export/group_tree_saver.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    class GroupTreeSaver
+      attr_reader :full_path
+
+      def initialize(group:, current_user:, shared:, params: {})
+        @params       = params
+        @current_user = current_user
+        @shared       = shared
+        @group        = group
+        @full_path    = File.join(@shared.export_path, ImportExport.group_filename)
+      end
+
+      def save
+        group_tree = serialize(@group, reader.group_tree)
+        tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename)
+
+        true
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def serialize(group, relations_tree)
+        group_tree = tree_saver.serialize(group, relations_tree)
+
+        group.descendants.each do |descendant|
+          group_tree['descendants'] = [] unless group_tree['descendants']
+          group_tree['descendants'] << serialize(descendant, relations_tree)
+        end
+
+        group_tree
+      rescue => e
+        @shared.error(e)
+      end
+
+      def reader
+        @reader ||= Gitlab::ImportExport::Reader.new(
+          shared: @shared,
+          config: Gitlab::ImportExport::Config.new(
+            config: Gitlab::ImportExport.group_config_file
+          ).to_h
+        )
+      end
+
+      def tree_saver
+        @tree_saver ||= RelationTreeSaver.new
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb
index 63c71105efe..386a4cfdfc6 100644
--- a/lib/gitlab/import_export/project_tree_saver.rb
+++ b/lib/gitlab/import_export/project_tree_saver.rb
@@ -3,25 +3,20 @@
 module Gitlab
   module ImportExport
     class ProjectTreeSaver
-      include Gitlab::ImportExport::CommandLineUtil
-
       attr_reader :full_path
 
       def initialize(project:, current_user:, shared:, params: {})
-        @params = params
-        @project = project
+        @params       = params
+        @project      = project
         @current_user = current_user
-        @shared = shared
-        @full_path = File.join(@shared.export_path, ImportExport.project_filename)
+        @shared       = shared
+        @full_path    = File.join(@shared.export_path, ImportExport.project_filename)
       end
 
       def save
-        mkdir_p(@shared.export_path)
-
-        project_tree = serialize_project_tree
+        project_tree = tree_saver.serialize(@project, reader.project_tree)
         fix_project_tree(project_tree)
-        project_tree_json = JSON.generate(project_tree)
-        File.write(full_path, project_tree_json)
+        tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
 
         true
       rescue => e
@@ -43,16 +38,6 @@ module Gitlab
         RelationRenameService.add_new_associations(project_tree)
       end
 
-      def serialize_project_tree
-        if Feature.enabled?(:export_fast_serialize, default_enabled: true)
-          Gitlab::ImportExport::FastHashSerializer
-            .new(@project, reader.project_tree)
-            .execute
-        else
-          @project.as_json(reader.project_tree)
-        end
-      end
-
       def reader
         @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
       end
@@ -74,6 +59,10 @@ module Gitlab
 
         GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
       end
+
+      def tree_saver
+        @tree_saver ||= RelationTreeSaver.new
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
index 9e81c6a3d07..1390770acef 100644
--- a/lib/gitlab/import_export/reader.rb
+++ b/lib/gitlab/import_export/reader.rb
@@ -5,24 +5,31 @@ module Gitlab
     class Reader
       attr_reader :tree, :attributes_finder
 
-      def initialize(shared:)
-        @shared = shared
-
-        @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(
-          config: ImportExport::Config.new.to_h)
+      def initialize(shared:, config: ImportExport::Config.new.to_h)
+        @shared            = shared
+        @config            = config
+        @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config)
       end
 
       # Outputs a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html
       # for outputting a project in JSON format, including its relations and sub relations.
       def project_tree
-        attributes_finder.find_root(:project)
-      rescue => e
-        @shared.error(e)
-        false
+        tree_by_key(:project)
+      end
+
+      def group_tree
+        tree_by_key(:group)
       end
 
       def group_members_tree
-        attributes_finder.find_root(:group_members)
+        tree_by_key(:group_members)
+      end
+
+      def tree_by_key(key)
+        attributes_finder.find_root(key)
+      rescue => e
+        @shared.error(e)
+        false
       end
     end
   end
diff --git a/lib/gitlab/import_export/relation_rename_service.rb b/lib/gitlab/import_export/relation_rename_service.rb
index 179bde5e21e..03aaa6aefc3 100644
--- a/lib/gitlab/import_export/relation_rename_service.rb
+++ b/lib/gitlab/import_export/relation_rename_service.rb
@@ -8,7 +8,7 @@
 # The behavior of these renamed relationships should be transient and it should
 # only last one release until you completely remove the renaming from the list.
 #
-# When importing, this class will check the project hash and:
+# When importing, this class will check the hash and:
 # - if only the old relationship name is found, it will rename it with the new one
 # - if only the new relationship name is found, it will do nothing
 # - if it finds both, it will use the new relationship data
diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/relation_tree_saver.rb
new file mode 100644
index 00000000000..a0452071ccf
--- /dev/null
+++ b/lib/gitlab/import_export/relation_tree_saver.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    class RelationTreeSaver
+      include Gitlab::ImportExport::CommandLineUtil
+
+      def serialize(exportable, relations_tree)
+        if Feature.enabled?(:export_fast_serialize, default_enabled: true)
+          Gitlab::ImportExport::FastHashSerializer
+            .new(exportable, relations_tree)
+            .execute
+        else
+          exportable.as_json(relations_tree)
+        end
+      end
+
+      def save(tree, dir_path, filename)
+        mkdir_p(dir_path)
+
+        tree_json = JSON.generate(tree)
+
+        File.write(File.join(dir_path, filename), tree_json)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index bea7a7cce65..ae82c380755 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -9,16 +9,16 @@ module Gitlab
         new(*args).save
       end
 
-      def initialize(project:, shared:)
-        @project = project
-        @shared = shared
+      def initialize(exportable:, shared:)
+        @exportable = exportable
+        @shared     = shared
       end
 
       def save
         if compress_and_save
           remove_export_path
 
-          Rails.logger.info("Saved project export #{archive_file}") # rubocop:disable Gitlab/RailsLogger
+          Rails.logger.info("Saved #{@exportable.class} export #{archive_file}") # rubocop:disable Gitlab/RailsLogger
 
           save_upload
         else
@@ -48,11 +48,11 @@ module Gitlab
       end
 
       def archive_file
-        @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
+        @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @exportable))
       end
 
       def save_upload
-        upload = ImportExportUpload.find_or_initialize_by(project: @project)
+        upload = initialize_upload
 
         File.open(archive_file) { |file| upload.export_file = file }
 
@@ -62,6 +62,12 @@ module Gitlab
       def error_message
         "Unable to save #{archive_file} into #{@shared.export_path}."
       end
+
+      def initialize_upload
+        exportable_kind = @exportable.class.name.downcase
+
+        ImportExportUpload.find_or_initialize_by(Hash[exportable_kind, @exportable])
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
index 02d46a1f498..2539a6828c3 100644
--- a/lib/gitlab/import_export/shared.rb
+++ b/lib/gitlab/import_export/shared.rb
@@ -23,21 +23,21 @@
 module Gitlab
   module ImportExport
     class Shared
-      attr_reader :errors, :project
+      attr_reader :errors, :exportable, :logger
 
       LOCKS_DIRECTORY = 'locks'
 
-      def initialize(project)
-        @project = project
-        @errors = []
-        @logger = Gitlab::Import::Logger.build
+      def initialize(exportable)
+        @exportable = exportable
+        @errors     = []
+        @logger     = Gitlab::Import::Logger.build
       end
 
       def active_export_count
         Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) }
       end
 
-      # The path where the project metadata and repository bundle is saved
+      # The path where the exportable metadata and repository bundle (in case of project) is saved
       def export_path
         @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path)
       end
@@ -84,11 +84,18 @@ module Gitlab
       end
 
       def relative_archive_path
-        @relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex)
+        @relative_archive_path ||= File.join(relative_base_path, SecureRandom.hex)
       end
 
       def relative_base_path
-        @project.disk_path
+        case exportable_type
+        when 'Project'
+          @exportable.disk_path
+        when 'Group'
+          @exportable.full_path
+        else
+          raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}")
+        end
       end
 
       def log_error(details)
@@ -100,17 +107,24 @@ module Gitlab
       end
 
       def log_base_data
-        {
-          importer: 'Import/Export',
-          import_jid: @project&.import_state&.jid,
-          project_id: @project&.id,
-          project_path: @project&.full_path
+        log = {
+          importer:        'Import/Export',
+          exportable_id:   @exportable&.id,
+          exportable_path: @exportable&.full_path
         }
+
+        log[:import_jid] = @exportable&.import_state&.jid if exportable_type == 'Project'
+
+        log
       end
 
       def filtered_error_message(message)
         Projects::ImportErrorFilter.filter_message(message)
       end
+
+      def exportable_type
+        @exportable.class.name
+      end
     end
   end
 end