Add latest changes from gitlab-org/gitlab@master

10 files changed, 150 insertions, 75 deletions

diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index d08848a65a8..b2ac60fe825 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -38,6 +38,10 @@ module Gitlab
       "lfs-objects"
     end
 
+    def wiki_repo_bundle_filename
+      "project.wiki.bundle"
+    end
+
     def config_file
       Rails.root.join('lib/gitlab/import_export/import_export.yml')
     end
@@ -61,3 +65,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::ImportExport.prepend_if_ee('EE::Gitlab::ImportExport')
diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb
index 1c62591ed5a..de1629d0e28 100644
--- a/lib/gitlab/import_export/group_project_object_builder.rb
+++ b/lib/gitlab/import_export/group_project_object_builder.rb
@@ -26,30 +26,60 @@ module Gitlab
       end
 
       def find
-        find_object || @klass.create(project_attributes)
+        find_object || klass.create(project_attributes)
       end
 
       private
 
+      attr_reader :klass, :attributes, :group, :project
+
       def find_object
-        @klass.where(where_clause).first
+        klass.where(where_clause).first
       end
 
       def where_clause
-        @attributes.slice('title').map do |key, value|
-          scope_clause = table[:project_id].eq(@project.id)
-          scope_clause = scope_clause.or(table[:group_id].eq(@group.id)) if @group
+        where_clauses.reduce(:and)
+      end
+
+      def where_clauses
+        [
+          where_clause_base,
+          where_clause_for_title,
+          where_clause_for_klass
+        ].compact
+      end
+
+      # Returns Arel clause `"{table_name}"."project_id" = {project.id}`
+      # or, if group is present:
+      # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}`
+      def where_clause_base
+        clause = table[:project_id].eq(project.id)
+        clause = clause.or(table[:group_id].eq(group.id)) if group
+
+        clause
+      end
 
-          table[key].eq(value).and(scope_clause)
-        end.reduce(:or)
+      # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'`
+      # if attributes has 'title key, otherwise `nil`.
+      def where_clause_for_title
+        attrs_to_arel(attributes.slice('title'))
+      end
+
+      # Returns Arel clause:
+      # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"`
+      # from the given Hash of attributes.
+      def attrs_to_arel(attrs)
+        attrs.map do |key, value|
+          table[key].eq(value)
+        end.reduce(:and)
       end
 
       def table
-        @table ||= @klass.arel_table
+        @table ||= klass.arel_table
       end
 
       def project_attributes
-        @attributes.except('group').tap do |atts|
+        attributes.except('group').tap do |atts|
           if label?
             atts['type'] = 'ProjectLabel' # Always create project labels
           elsif milestone?
@@ -60,15 +90,17 @@ module Gitlab
               claim_iid
             end
           end
+
+          atts['importing'] = true if klass.ancestors.include?(Importable)
         end
       end
 
       def label?
-        @klass == Label
+        klass == Label
       end
 
       def milestone?
-        @klass == Milestone
+        klass == Milestone
       end
 
       # If an existing group milestone used the IID
@@ -79,7 +111,7 @@ module Gitlab
       def claim_iid
         # The milestone has to be a group milestone, as it's the only case where
         # we set the IID as the maximum. The rest of them are fixed.
-        milestone = @project.milestones.find_by(iid: @attributes['iid'])
+        milestone = project.milestones.find_by(iid: attributes['iid'])
 
         return unless milestone
 
@@ -87,6 +119,15 @@ module Gitlab
         milestone.ensure_project_iid!
         milestone.save!
       end
+
+      protected
+
+      # Returns Arel clause for a particular model or `nil`.
+      def where_clause_for_klass
+        # no-op
+      end
     end
   end
 end
+
+Gitlab::ImportExport::GroupProjectObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::GroupProjectObjectBuilder')
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 994aa95dc5e..58df596bdde 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -248,7 +248,16 @@ preloads:
 ee:
   tree:
     project:
-      protected_branches:
+      - issues:
+        - designs:
+          - notes:
+            - :author
+            - events:
+              - :push_event_payload
+        - design_versions:
+          - actions:
+            - :design # Duplicate export of issues.designs in order to link the record to both Issue and DesignVersion
+      - protected_branches:
         - :unprotect_access_levels
-      protected_environments:
+      - protected_environments:
         - :deploy_access_levels
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 767f1b5de0e..f061a1916da 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -21,7 +21,7 @@ module Gitlab
         if import_file && check_version! && restorers.all?(&:restore) && overwrite_project
           project_tree.restored_project
         else
-          raise Projects::ImportService::Error.new(@shared.errors.join(', '))
+          raise Projects::ImportService::Error.new(shared.errors.to_sentence)
         end
       rescue => e
         raise Projects::ImportService::Error.new(e.message)
@@ -31,70 +31,72 @@ module Gitlab
 
       private
 
+      attr_accessor :archive_file, :current_user, :project, :shared
+
       def restorers
         [repo_restorer, wiki_restorer, project_tree, avatar_restorer,
          uploads_restorer, lfs_restorer, statistics_restorer]
       end
 
       def import_file
-        Gitlab::ImportExport::FileImporter.import(project: @project,
-                                                  archive_file: @archive_file,
-                                                  shared: @shared)
+        Gitlab::ImportExport::FileImporter.import(project: project,
+                                                  archive_file: archive_file,
+                                                  shared: shared)
       end
 
       def check_version!
-        Gitlab::ImportExport::VersionChecker.check!(shared: @shared)
+        Gitlab::ImportExport::VersionChecker.check!(shared: shared)
       end
 
       def project_tree
-        @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: @current_user,
-                                                                        shared: @shared,
-                                                                        project: @project)
+        @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: current_user,
+                                                                        shared: shared,
+                                                                        project: project)
       end
 
       def avatar_restorer
-        Gitlab::ImportExport::AvatarRestorer.new(project: project_tree.restored_project, shared: @shared)
+        Gitlab::ImportExport::AvatarRestorer.new(project: project_tree.restored_project, shared: shared)
       end
 
       def repo_restorer
         Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path,
-                                               shared: @shared,
+                                               shared: shared,
                                                project: project_tree.restored_project)
       end
 
       def wiki_restorer
         Gitlab::ImportExport::WikiRestorer.new(path_to_bundle: wiki_repo_path,
-                                               shared: @shared,
+                                               shared: shared,
                                                project: ProjectWiki.new(project_tree.restored_project),
-                                               wiki_enabled: @project.wiki_enabled?)
+                                               wiki_enabled: project.wiki_enabled?)
       end
 
       def uploads_restorer
-        Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared)
+        Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: shared)
       end
 
       def lfs_restorer
-        Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared)
+        Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: shared)
       end
 
       def statistics_restorer
-        Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: @shared)
+        Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: shared)
       end
 
       def path_with_namespace
-        File.join(@project.namespace.full_path, @project.path)
+        File.join(project.namespace.full_path, project.path)
       end
 
       def repo_path
-        File.join(@shared.export_path, 'project.bundle')
+        File.join(shared.export_path, Gitlab::ImportExport.project_bundle_filename)
       end
 
       def wiki_repo_path
-        File.join(@shared.export_path, 'project.wiki.bundle')
+        File.join(shared.export_path, Gitlab::ImportExport.wiki_repo_bundle_filename)
       end
 
       def remove_import_file
-        upload = @project.import_export_upload
+        upload = project.import_export_upload
 
         return unless upload&.import_file&.file
 
@@ -105,10 +107,10 @@ module Gitlab
       def overwrite_project
         project = project_tree.restored_project
 
-        return unless can?(@current_user, :admin_namespace, project.namespace)
+        return unless can?(current_user, :admin_namespace, project.namespace)
 
         if overwrite_project?
-          ::Projects::OverwriteProjectService.new(project, @current_user)
+          ::Projects::OverwriteProjectService.new(project, current_user)
                                              .execute(project_to_overwrite)
         end
 
@@ -116,7 +118,7 @@ module Gitlab
       end
 
       def original_path
-        @project.import_data&.data&.fetch('original_path', nil)
+        project.import_data&.data&.fetch('original_path', nil)
       end
 
       def overwrite_project?
@@ -125,9 +127,11 @@ module Gitlab
 
       def project_to_overwrite
         strong_memoize(:project_to_overwrite) do
-          Project.find_by_full_path("#{@project.namespace.full_path}/#{original_path}")
+          Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}")
         end
       end
     end
   end
 end
+
+Gitlab::ImportExport::Importer.prepend_if_ee('EE::Gitlab::ImportExport::Importer')
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 2dd18616cd6..39a243bd433 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -93,6 +93,10 @@ module Gitlab
         end
       end
 
+      def remove_feature_dependent_sub_relations(_relation_item)
+        # no-op
+      end
+
       def project_relations_without_project_members
         # We remove `project_members` as they are deserialized separately
         project_relations.except(:project_members)
@@ -171,6 +175,8 @@ module Gitlab
             next
           end
 
+          remove_feature_dependent_sub_relations(relation_item)
+
           # The transaction at this level is less speedy than one single transaction
           # But we can't have it in the upper level or GC won't get rid of the AR objects
           # after we save the batch.
@@ -238,3 +244,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::ImportExport::ProjectTreeRestorer.prepend_if_ee('::EE::Gitlab::ImportExport::ProjectTreeRestorer')
diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index 1e9dff405c5..33e7595cf43 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -34,13 +34,13 @@ module Gitlab
 
       PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
 
-      BUILD_MODELS = %w[Ci::Build commit_status].freeze
+      BUILD_MODELS = %i[Ci::Build commit_status].freeze
 
       IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
 
       EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature].freeze
 
-      TOKEN_RESET_MODELS = %w[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
+      TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
 
       def self.create(*args)
         new(*args).create
@@ -56,7 +56,7 @@ module Gitlab
       end
 
       def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: [])
-        @relation_name = self.class.overrides[relation_sym] || relation_sym
+        @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym
         @relation_hash = relation_hash.except('noteable_id')
         @members_mapper = members_mapper
         @user = user
@@ -92,6 +92,10 @@ module Gitlab
         OVERRIDES
       end
 
+      def self.existing_object_check
+        EXISTING_OBJECT_CHECK
+      end
+
       private
 
       def setup_models
@@ -105,7 +109,7 @@ module Gitlab
         update_group_references
         remove_duplicate_assignees
 
-        setup_pipeline if @relation_name == 'Ci::Pipeline'
+        setup_pipeline if @relation_name == :'Ci::Pipeline'
 
         reset_tokens!
         remove_encrypted_attributes!
@@ -184,14 +188,14 @@ module Gitlab
       end
 
       def update_group_references
-        return unless EXISTING_OBJECT_CHECK.include?(@relation_name)
+        return unless self.class.existing_object_check.include?(@relation_name)
         return unless @relation_hash['group_id']
 
         @relation_hash['group_id'] = @project.namespace_id
       end
 
       def reset_tokens!
-        return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s)
+        return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name)
 
         # If we import/export a project to the same instance, tokens will have to be reset.
         # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across.
@@ -255,7 +259,7 @@ module Gitlab
         # Only find existing records to avoid mapping tables such as milestones
         # Otherwise always create the record, skipping the extra SELECT clause.
         @existing_or_new_object ||= begin
-          if EXISTING_OBJECT_CHECK.include?(@relation_name)
+          if self.class.existing_object_check.include?(@relation_name)
             attribute_hash = attribute_hash_for(['events'])
 
             existing_object.assign_attributes(attribute_hash) if attribute_hash.any?
@@ -284,7 +288,7 @@ module Gitlab
       end
 
       def legacy_trigger?
-        @relation_name == 'Ci::Trigger' && @relation_hash['owner_id'].nil?
+        @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil?
       end
 
       def find_or_create_object!
@@ -293,7 +297,7 @@ module Gitlab
         # Can't use IDs as validation exists calling `group` or `project` attributes
         finder_hash = parsed_relation_hash.tap do |hash|
           hash['group'] = @project.group if relation_class.attribute_method?('group_id')
-          hash['project'] = @project
+          hash['project'] = @project if relation_class.reflect_on_association(:project)
           hash.delete('project_id')
         end
 
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index 91167a9c4fb..3123687453f 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -6,19 +6,23 @@ module Gitlab
       include Gitlab::ImportExport::CommandLineUtil
 
       def initialize(project:, shared:, path_to_bundle:)
-        @project = project
+        @repository = project.repository
         @path_to_bundle = path_to_bundle
         @shared = shared
       end
 
       def restore
-        return true unless File.exist?(@path_to_bundle)
+        return true unless File.exist?(path_to_bundle)
 
-        @project.repository.create_from_bundle(@path_to_bundle)
+        repository.create_from_bundle(path_to_bundle)
       rescue => e
-        @shared.error(e)
+        shared.error(e)
         false
       end
+
+      private
+
+      attr_accessor :repository, :path_to_bundle, :shared
     end
   end
 end
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
index a60618dfcec..898cd7898ba 100644
--- a/lib/gitlab/import_export/repo_saver.rb
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -5,27 +5,35 @@ module Gitlab
     class RepoSaver
       include Gitlab::ImportExport::CommandLineUtil
 
-      attr_reader :full_path
+      attr_reader :project, :repository, :shared
 
       def initialize(project:, shared:)
         @project = project
         @shared = shared
+        @repository = @project.repository
       end
 
       def save
-        return true if @project.empty_repo? # it's ok to have no repo
+        return true unless repository_exists? # it's ok to have no repo
 
-        @full_path = File.join(@shared.export_path, ImportExport.project_bundle_filename)
         bundle_to_disk
       end
 
       private
 
+      def repository_exists?
+        repository.exists? && !repository.empty?
+      end
+
+      def bundle_full_path
+        File.join(shared.export_path, ImportExport.project_bundle_filename)
+      end
+
       def bundle_to_disk
-        mkdir_p(@shared.export_path)
-        @project.repository.bundle_to_disk(@full_path)
+        mkdir_p(shared.export_path)
+        repository.bundle_to_disk(bundle_full_path)
       rescue => e
-        @shared.error(e)
+        shared.error(e)
         false
       end
     end
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
index 7303bcf61a4..93ae6f6b02a 100644
--- a/lib/gitlab/import_export/wiki_repo_saver.rb
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -4,28 +4,16 @@ module Gitlab
   module ImportExport
     class WikiRepoSaver < RepoSaver
       def save
-        @wiki = ProjectWiki.new(@project)
-        return true unless wiki_repository_exists? # it's okay to have no Wiki
+        wiki = ProjectWiki.new(project)
+        @repository = wiki.repository
 
-        bundle_to_disk(File.join(@shared.export_path, project_filename))
-      end
-
-      def bundle_to_disk(full_path)
-        mkdir_p(@shared.export_path)
-        @wiki.repository.bundle_to_disk(full_path)
-      rescue => e
-        @shared.error(e)
-        false
+        super
       end
 
       private
 
-      def project_filename
-        "project.wiki.bundle"
-      end
-
-      def wiki_repository_exists?
-        @wiki.repository.exists? && !@wiki.repository.empty?
+      def bundle_full_path
+        File.join(shared.export_path, ImportExport.wiki_repo_bundle_filename)
       end
     end
   end
diff --git a/lib/gitlab/import_export/wiki_restorer.rb b/lib/gitlab/import_export/wiki_restorer.rb
index 28b5e7449cd..359ba8ba769 100644
--- a/lib/gitlab/import_export/wiki_restorer.rb
+++ b/lib/gitlab/import_export/wiki_restorer.rb
@@ -6,19 +6,22 @@ module Gitlab
       def initialize(project:, shared:, path_to_bundle:, wiki_enabled:)
         super(project: project, shared: shared, path_to_bundle: path_to_bundle)
 
+        @project = project
         @wiki_enabled = wiki_enabled
       end
 
       def restore
-        @project.wiki if create_empty_wiki?
+        project.wiki if create_empty_wiki?
 
         super
       end
 
       private
 
+      attr_accessor :project, :wiki_enabled
+
       def create_empty_wiki?
-        !File.exist?(@path_to_bundle) && @wiki_enabled
+        !File.exist?(path_to_bundle) && wiki_enabled
       end
     end
   end