Add latest changes from gitlab-org/gitlab@master

7 files changed, 142 insertions, 127 deletions

diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb
index cbaa6929efa..247e39a68b9 100644
--- a/lib/gitlab/import_export/group/tree_restorer.rb
+++ b/lib/gitlab/import_export/group/tree_restorer.rb
@@ -17,9 +17,17 @@ module Gitlab
         end
 
         def restore
-          @tree_hash = @group_hash || read_tree_hash
-          @group_members = @tree_hash.delete('members')
-          @children = @tree_hash.delete('children')
+          @relation_reader ||=
+            if @group_hash.present?
+              ImportExport::JSON::LegacyReader::User.new(@group_hash, reader.group_relation_names)
+            else
+              ImportExport::JSON::LegacyReader::File.new(@path, reader.group_relation_names)
+            end
+
+          @group_members = @relation_reader.consume_relation('members')
+          @children = @relation_reader.consume_attribute('children')
+          @relation_reader.consume_attribute('name')
+          @relation_reader.consume_attribute('path')
 
           if members_mapper.map && restorer.restore
             @children&.each do |group_hash|
@@ -45,21 +53,12 @@ module Gitlab
 
         private
 
-        def read_tree_hash
-          json = IO.read(@path)
-          ActiveSupport::JSON.decode(json)
-        rescue => e
-          @shared.error(e)
-
-          raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
-        end
-
         def restorer
           @relation_tree_restorer ||= RelationTreeRestorer.new(
             user:             @user,
             shared:           @shared,
             importable:       @group,
-            tree_hash:        @tree_hash.except('name', 'path'),
+            relation_reader:  @relation_reader,
             members_mapper:   members_mapper,
             object_builder:   object_builder,
             relation_factory: relation_factory,
diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb
new file mode 100644
index 00000000000..477e41ae3eb
--- /dev/null
+++ b/lib/gitlab/import_export/json/legacy_reader.rb
@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module JSON
+      class LegacyReader
+        class File < LegacyReader
+          def initialize(path, relation_names)
+            @path = path
+            super(relation_names)
+          end
+
+          def valid?
+            ::File.exist?(@path)
+          end
+
+          private
+
+          def tree_hash
+            @tree_hash ||= read_hash
+          end
+
+          def read_hash
+            ActiveSupport::JSON.decode(IO.read(@path))
+          rescue => e
+            Gitlab::ErrorTracking.log_exception(e)
+            raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
+          end
+        end
+
+        class User < LegacyReader
+          def initialize(tree_hash, relation_names)
+            @tree_hash = tree_hash
+            super(relation_names)
+          end
+
+          def valid?
+            @tree_hash.present?
+          end
+
+          protected
+
+          attr_reader :tree_hash
+        end
+
+        def initialize(relation_names)
+          @relation_names = relation_names.map(&:to_s)
+        end
+
+        def valid?
+          raise NotImplementedError
+        end
+
+        def legacy?
+          true
+        end
+
+        def root_attributes(excluded_attributes = [])
+          attributes.except(*excluded_attributes.map(&:to_s))
+        end
+
+        def consume_relation(key)
+          value = relations.delete(key)
+
+          return value unless block_given?
+
+          return if value.nil?
+
+          if value.is_a?(Array)
+            value.each.with_index do |item, idx|
+              yield(item, idx)
+            end
+          else
+            yield(value, 0)
+          end
+        end
+
+        def consume_attribute(key)
+          attributes.delete(key)
+        end
+
+        def sort_ci_pipelines_by_id
+          relations['ci_pipelines']&.sort_by! { |hash| hash['id'] }
+        end
+
+        private
+
+        attr_reader :relation_names
+
+        def tree_hash
+          raise NotImplementedError
+        end
+
+        def attributes
+          @attributes ||= tree_hash.slice!(*relation_names)
+        end
+
+        def relations
+          @relations ||= tree_hash.extract!(*relation_names)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project/tree_loader.rb b/lib/gitlab/import_export/project/tree_loader.rb
deleted file mode 100644
index 6d4737a2d00..00000000000
--- a/lib/gitlab/import_export/project/tree_loader.rb
+++ /dev/null
@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    module Project
-      class TreeLoader
-        def load(path, dedup_entries: false)
-          tree_hash = ActiveSupport::JSON.decode(IO.read(path))
-
-          if dedup_entries
-            dedup_tree(tree_hash)
-          else
-            tree_hash
-          end
-        end
-
-        private
-
-        # This function removes duplicate entries from the given tree recursively
-        # by caching nodes it encounters repeatedly. We only consider nodes for
-        # which there can actually be multiple equivalent instances (e.g. strings,
-        # hashes and arrays, but not `nil`s, numbers or booleans.)
-        #
-        # The algorithm uses a recursive depth-first descent with 3 cases, starting
-        # with a root node (the tree/hash itself):
-        # - a node has already been cached; in this case we return it from the cache
-        # - a node has not been cached yet but should be; descend into its children
-        # - a node is neither cached nor qualifies for caching; this is a no-op
-        def dedup_tree(node, nodes_seen = {})
-          if nodes_seen.key?(node) && distinguishable?(node)
-            yield nodes_seen[node]
-          elsif should_dedup?(node)
-            nodes_seen[node] = node
-
-            case node
-            when Array
-              node.each_index do |idx|
-                dedup_tree(node[idx], nodes_seen) do |cached_node|
-                  node[idx] = cached_node
-                end
-              end
-            when Hash
-              node.each do |k, v|
-                dedup_tree(v, nodes_seen) do |cached_node|
-                  node[k] = cached_node
-                end
-              end
-            end
-          else
-            node
-          end
-        end
-
-        # We do not need to consider nodes for which there cannot be multiple instances
-        def should_dedup?(node)
-          node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass))
-        end
-
-        # We can only safely de-dup values that are distinguishable. True value objects
-        # are always distinguishable by nature. Hashes however can represent entities,
-        # which are identified by ID, not value. We therefore disallow de-duping hashes
-        # that do not have an `id` field, since we might risk dropping entities that
-        # have equal attributes yet different identities.
-        def distinguishable?(node)
-          if node.is_a?(Hash)
-            node.key?('id')
-          else
-            true
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb
index 295e0d5f348..f8d25e14c02 100644
--- a/lib/gitlab/import_export/project/tree_restorer.rb
+++ b/lib/gitlab/import_export/project/tree_restorer.rb
@@ -4,8 +4,6 @@ module Gitlab
   module ImportExport
     module Project
       class TreeRestorer
-        LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte
-
         attr_reader :user
         attr_reader :shared
         attr_reader :project
@@ -14,12 +12,12 @@ module Gitlab
           @user = user
           @shared = shared
           @project = project
-          @tree_loader = TreeLoader.new
         end
 
         def restore
-          @tree_hash = read_tree_hash
-          @project_members = @tree_hash.delete('project_members')
+          @relation_reader = ImportExport::JSON::LegacyReader::File.new(File.join(shared.export_path, 'project.json'), reader.project_relation_names)
+
+          @project_members = @relation_reader.consume_relation('project_members')
 
           if relation_tree_restorer.restore
             import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
@@ -37,24 +35,12 @@ module Gitlab
 
         private
 
-        def large_project?(path)
-          File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES
-        end
-
-        def read_tree_hash
-          path = File.join(@shared.export_path, 'project.json')
-          @tree_loader.load(path, dedup_entries: large_project?(path))
-        rescue => e
-          Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
-          raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
-        end
-
         def relation_tree_restorer
           @relation_tree_restorer ||= RelationTreeRestorer.new(
             user: @user,
             shared: @shared,
             importable: @project,
-            tree_hash: @tree_hash,
+            relation_reader: @relation_reader,
             object_builder: object_builder,
             members_mapper: members_mapper,
             relation_factory: relation_factory,
diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
index 1390770acef..8d36d05ca6f 100644
--- a/lib/gitlab/import_export/reader.rb
+++ b/lib/gitlab/import_export/reader.rb
@@ -17,10 +17,18 @@ module Gitlab
         tree_by_key(:project)
       end
 
+      def project_relation_names
+        attributes_finder.find_relations_tree(:project).keys
+      end
+
       def group_tree
         tree_by_key(:group)
       end
 
+      def group_relation_names
+        attributes_finder.find_relations_tree(:group).keys
+      end
+
       def group_members_tree
         tree_by_key(:group_members)
       end
diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb
index 8359eefc846..466cb03862e 100644
--- a/lib/gitlab/import_export/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/relation_tree_restorer.rb
@@ -9,13 +9,13 @@ module Gitlab
       attr_reader :user
       attr_reader :shared
       attr_reader :importable
-      attr_reader :tree_hash
+      attr_reader :relation_reader
 
-      def initialize(user:, shared:, importable:, tree_hash:, members_mapper:, object_builder:, relation_factory:, reader:)
+      def initialize(user:, shared:, importable:, relation_reader:, members_mapper:, object_builder:, relation_factory:, reader:)
         @user = user
         @shared = shared
         @importable = importable
-        @tree_hash = tree_hash
+        @relation_reader = relation_reader
         @members_mapper = members_mapper
         @object_builder = object_builder
         @relation_factory = relation_factory
@@ -30,7 +30,7 @@ module Gitlab
             bulk_inserts_enabled = @importable.class == ::Project &&
               Feature.enabled?(:import_bulk_inserts, @importable.group)
             BulkInsertableAssociations.with_bulk_insert(enabled: bulk_inserts_enabled) do
-              update_relation_hashes!
+              fix_ci_pipelines_not_sorted_on_legacy_project_json!
               create_relations!
             end
           end
@@ -57,18 +57,8 @@ module Gitlab
       end
 
       def process_relation!(relation_key, relation_definition)
-        data_hashes = @tree_hash.delete(relation_key)
-        return unless data_hashes
-
-        # we do not care if we process array or hash
-        data_hashes = [data_hashes] unless data_hashes.is_a?(Array)
-
-        relation_index = 0
-
-        # consume and remove objects from memory
-        while data_hash = data_hashes.shift
+        @relation_reader.consume_relation(relation_key) do |data_hash, relation_index|
           process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
-          relation_index += 1
         end
       end
 
@@ -103,10 +93,7 @@ module Gitlab
       end
 
       def update_params!
-        params = @tree_hash.reject do |key, _|
-          relations.include?(key)
-        end
-
+        params = @relation_reader.root_attributes(relations.keys)
         params = params.merge(present_override_params)
 
         # Cleaning all imported and overridden params
@@ -223,8 +210,13 @@ module Gitlab
         }
       end
 
-      def update_relation_hashes!
-        @tree_hash['ci_pipelines']&.sort_by! { |hash| hash['id'] }
+      # Temporary fix for https://gitlab.com/gitlab-org/gitlab/-/issues/27883 when import from legacy project.json
+      # This should be removed once legacy JSON format is deprecated.
+      # Ndjson export file will fix the order during project export.
+      def fix_ci_pipelines_not_sorted_on_legacy_project_json!
+        return unless relation_reader.legacy?
+
+        relation_reader.sort_ci_pipelines_by_id
       end
     end
   end
diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/relation_tree_saver.rb
index a0452071ccf..ed5392c13d0 100644
--- a/lib/gitlab/import_export/relation_tree_saver.rb
+++ b/lib/gitlab/import_export/relation_tree_saver.rb
@@ -18,7 +18,7 @@ module Gitlab
       def save(tree, dir_path, filename)
         mkdir_p(dir_path)
 
-        tree_json = JSON.generate(tree)
+        tree_json = ::JSON.generate(tree)
 
         File.write(File.join(dir_path, filename), tree_json)
       end