Add latest changes from gitlab-org/gitlab@master

8 files changed, 187 insertions, 84 deletions

diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb
index 3bfc059dcd3..018cb36fc58 100644
--- a/lib/gitlab/import_export/attribute_cleaner.rb
+++ b/lib/gitlab/import_export/attribute_cleaner.rb
@@ -11,7 +11,14 @@ module Gitlab
         'discussion_id',
         'custom_attributes'
       ].freeze
-      PROHIBITED_REFERENCES = Regexp.union(/\Acached_markdown_version\Z/, /_id\Z/, /_ids\Z/, /_html\Z/, /attributes/).freeze
+      PROHIBITED_REFERENCES = Regexp.union(
+        /\Acached_markdown_version\Z/,
+        /_id\Z/,
+        /_ids\Z/,
+        /_html\Z/,
+        /attributes/,
+        /\Aremote_\w+_(url|urls|request_header)\Z/ # carrierwave automatically creates these attribute methods for uploads
+      ).freeze
 
       def self.clean(*args)
         new(*args).clean
diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb
index 247e39a68b9..f6ebd83bfaa 100644
--- a/lib/gitlab/import_export/group/tree_restorer.rb
+++ b/lib/gitlab/import_export/group/tree_restorer.rb
@@ -4,12 +4,13 @@ module Gitlab
   module ImportExport
     module Group
       class TreeRestorer
+        include Gitlab::Utils::StrongMemoize
+
         attr_reader :user
         attr_reader :shared
         attr_reader :group
 
         def initialize(user:, shared:, group:, group_hash:)
-          @path = File.join(shared.export_path, 'group.json')
           @user = user
           @shared = shared
           @group = group
@@ -17,17 +18,14 @@ module Gitlab
         end
 
         def restore
-          @relation_reader ||=
-            if @group_hash.present?
-              ImportExport::JSON::LegacyReader::User.new(@group_hash, reader.group_relation_names)
-            else
-              ImportExport::JSON::LegacyReader::File.new(@path, reader.group_relation_names)
-            end
+          @group_attributes = relation_reader.consume_attributes(nil)
+          @group_members = relation_reader.consume_relation(nil, 'members')
+
+          # We need to remove `name` and `path` as we did consume it in previous pass
+          @group_attributes.delete('name')
+          @group_attributes.delete('path')
 
-          @group_members = @relation_reader.consume_relation('members')
-          @children = @relation_reader.consume_attribute('children')
-          @relation_reader.consume_attribute('name')
-          @relation_reader.consume_attribute('path')
+          @children = @group_attributes.delete('children')
 
           if members_mapper.map && restorer.restore
             @children&.each do |group_hash|
@@ -53,16 +51,32 @@ module Gitlab
 
         private
 
+        def relation_reader
+          strong_memoize(:relation_reader) do
+            if @group_hash.present?
+              ImportExport::JSON::LegacyReader::Hash.new(
+                @group_hash,
+                relation_names: reader.group_relation_names)
+            else
+              ImportExport::JSON::LegacyReader::File.new(
+                File.join(shared.export_path, 'group.json'),
+                relation_names: reader.group_relation_names)
+            end
+          end
+        end
+
         def restorer
           @relation_tree_restorer ||= RelationTreeRestorer.new(
-            user:             @user,
-            shared:           @shared,
-            importable:       @group,
-            relation_reader:  @relation_reader,
-            members_mapper:   members_mapper,
-            object_builder:   object_builder,
-            relation_factory: relation_factory,
-            reader:           reader
+            user:                  @user,
+            shared:                @shared,
+            relation_reader:       relation_reader,
+            members_mapper:        members_mapper,
+            object_builder:        object_builder,
+            relation_factory:      relation_factory,
+            reader:                reader,
+            importable:            @group,
+            importable_attributes: @group_attributes,
+            importable_path:       nil
           )
         end
 
@@ -88,7 +102,11 @@ module Gitlab
         end
 
         def members_mapper
-          @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group)
+          @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(
+            exported_members: @group_members,
+            user: @user,
+            importable: @group
+          )
         end
 
         def relation_factory
diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb
index 477e41ae3eb..57579fe9def 100644
--- a/lib/gitlab/import_export/json/legacy_reader.rb
+++ b/lib/gitlab/import_export/json/legacy_reader.rb
@@ -5,19 +5,25 @@ module Gitlab
     module JSON
       class LegacyReader
         class File < LegacyReader
-          def initialize(path, relation_names)
+          include Gitlab::Utils::StrongMemoize
+
+          def initialize(path, relation_names:, allowed_path: nil)
             @path = path
-            super(relation_names)
+            super(
+              relation_names: relation_names,
+              allowed_path: allowed_path)
           end
 
-          def valid?
+          def exist?
             ::File.exist?(@path)
           end
 
-          private
+          protected
 
           def tree_hash
-            @tree_hash ||= read_hash
+            strong_memoize(:tree_hash) do
+              read_hash
+            end
           end
 
           def read_hash
@@ -28,13 +34,15 @@ module Gitlab
           end
         end
 
-        class User < LegacyReader
-          def initialize(tree_hash, relation_names)
+        class Hash < LegacyReader
+          def initialize(tree_hash, relation_names:, allowed_path: nil)
             @tree_hash = tree_hash
-            super(relation_names)
+            super(
+              relation_names: relation_names,
+              allowed_path: allowed_path)
           end
 
-          def valid?
+          def exist?
             @tree_hash.present?
           end
 
@@ -43,11 +51,16 @@ module Gitlab
           attr_reader :tree_hash
         end
 
-        def initialize(relation_names)
+        def initialize(relation_names:, allowed_path:)
           @relation_names = relation_names.map(&:to_s)
+
+          # This is legacy reader, to be used in transition
+          # period before `.ndjson`,
+          # we strong validate what is being readed
+          @allowed_path = allowed_path
         end
 
-        def valid?
+        def exist?
           raise NotImplementedError
         end
 
@@ -55,15 +68,22 @@ module Gitlab
           true
         end
 
-        def root_attributes(excluded_attributes = [])
-          attributes.except(*excluded_attributes.map(&:to_s))
+        def consume_attributes(importable_path)
+          unless importable_path == @allowed_path
+            raise ArgumentError, "Invalid #{importable_path} passed to `consume_attributes`. Use #{@allowed_path} instead."
+          end
+
+          attributes
         end
 
-        def consume_relation(key)
+        def consume_relation(importable_path, key)
+          unless importable_path == @allowed_path
+            raise ArgumentError, "Invalid #{importable_name} passed to `consume_relation`. Use #{@allowed_path} instead."
+          end
+
           value = relations.delete(key)
 
           return value unless block_given?
-
           return if value.nil?
 
           if value.is_a?(Array)
@@ -75,17 +95,13 @@ module Gitlab
           end
         end
 
-        def consume_attribute(key)
-          attributes.delete(key)
-        end
-
         def sort_ci_pipelines_by_id
           relations['ci_pipelines']&.sort_by! { |hash| hash['id'] }
         end
 
         private
 
-        attr_reader :relation_names
+        attr_reader :relation_names, :allowed_path
 
         def tree_hash
           raise NotImplementedError
diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb
index c935e360a65..7be21410d26 100644
--- a/lib/gitlab/import_export/json/legacy_writer.rb
+++ b/lib/gitlab/import_export/json/legacy_writer.rb
@@ -8,11 +8,15 @@ module Gitlab
 
         attr_reader :path
 
-        def initialize(path)
+        def initialize(path, allowed_path:)
           @path = path
-          @last_array = nil
           @keys = Set.new
 
+          # This is legacy writer, to be used in transition
+          # period before `.ndjson`,
+          # we strong validate what is being written
+          @allowed_path = allowed_path
+
           mkdir_p(File.dirname(@path))
           file.write('{}')
         end
@@ -22,12 +26,44 @@ module Gitlab
           @file = nil
         end
 
-        def set(hash)
+        def write_attributes(exportable_path, hash)
+          unless exportable_path == @allowed_path
+            raise ArgumentError, "Invalid #{exportable_path}"
+          end
+
           hash.each do |key, value|
             write(key, value)
           end
         end
 
+        def write_relation(exportable_path, key, value)
+          unless exportable_path == @allowed_path
+            raise ArgumentError, "Invalid #{exportable_path}"
+          end
+
+          write(key, value)
+        end
+
+        def write_relation_array(exportable_path, key, items)
+          unless exportable_path == @allowed_path
+            raise ArgumentError, "Invalid #{exportable_path}"
+          end
+
+          write(key, [])
+
+          # rewind by two bytes, to overwrite ']}'
+          file.pos = file.size - 2
+
+          items.each_with_index do |item, idx|
+            file.write(',') if idx > 0
+            file.write(item.to_json)
+          end
+
+          file.write(']}')
+        end
+
+        private
+
         def write(key, value)
           raise ArgumentError, "key '#{key}' already written" if @keys.include?(key)
 
@@ -41,29 +77,8 @@ module Gitlab
           file.write('}')
 
           @keys.add(key)
-          @last_array = nil
-          @last_array_count = nil
-        end
-
-        def append(key, value)
-          unless @last_array == key
-            write(key, [])
-
-            @last_array = key
-            @last_array_count = 0
-          end
-
-          # rewind by two bytes, to overwrite ']}'
-          file.pos = file.size - 2
-
-          file.write(',') if @last_array_count > 0
-          file.write(value.to_json)
-          file.write(']}')
-          @last_array_count += 1
         end
 
-        private
-
         def file
           @file ||= File.open(@path, "wb")
         end
diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb
index d053bf16166..04ca598ff5d 100644
--- a/lib/gitlab/import_export/json/streaming_serializer.rb
+++ b/lib/gitlab/import_export/json/streaming_serializer.rb
@@ -14,8 +14,9 @@ module Gitlab
           end
         end
 
-        def initialize(exportable, relations_schema, json_writer)
+        def initialize(exportable, relations_schema, json_writer, exportable_path:)
           @exportable = exportable
+          @exportable_path = exportable_path
           @relations_schema = relations_schema
           @json_writer = json_writer
         end
@@ -35,7 +36,7 @@ module Gitlab
         def serialize_root
           attributes = exportable.as_json(
             relations_schema.merge(include: nil, preloads: nil))
-          json_writer.set(attributes)
+          json_writer.write_attributes(@exportable_path, attributes)
         end
 
         def serialize_relation(definition)
@@ -47,16 +48,28 @@ module Gitlab
           record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend
           if record.is_a?(ActiveRecord::Relation)
             serialize_many_relations(key, record, options)
+          elsif record.respond_to?(:each) # this is to support `project_members` that return an Array
+            serialize_many_each(key, record, options)
           else
             serialize_single_relation(key, record, options)
           end
         end
 
         def serialize_many_relations(key, records, options)
-          key_preloads = preloads&.dig(key)
-          records = records.preload(key_preloads) if key_preloads
+          enumerator = Enumerator.new do |items|
+            key_preloads = preloads&.dig(key)
+            records = records.preload(key_preloads) if key_preloads
 
-          records.find_each(batch_size: BATCH_SIZE) do |record|
+            records.find_each(batch_size: BATCH_SIZE) do |record|
+              items << Raw.new(record.to_json(options))
+            end
+          end
+
+          json_writer.write_relation_array(@exportable_path, key, enumerator)
+        end
+
+        def serialize_many_each(key, records, options)
+          records.each do |record|
             json = Raw.new(record.to_json(options))
 
             json_writer.append(key, json)
@@ -66,7 +79,7 @@ module Gitlab
         def serialize_single_relation(key, record, options)
           json = Raw.new(record.to_json(options))
 
-          json_writer.write(key, json)
+          json_writer.write_relation(@exportable_path, key, json)
         end
 
         def includes
diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb
index f8d25e14c02..99e57d9decd 100644
--- a/lib/gitlab/import_export/project/tree_restorer.rb
+++ b/lib/gitlab/import_export/project/tree_restorer.rb
@@ -4,6 +4,8 @@ module Gitlab
   module ImportExport
     module Project
       class TreeRestorer
+        include Gitlab::Utils::StrongMemoize
+
         attr_reader :user
         attr_reader :shared
         attr_reader :project
@@ -15,9 +17,8 @@ module Gitlab
         end
 
         def restore
-          @relation_reader = ImportExport::JSON::LegacyReader::File.new(File.join(shared.export_path, 'project.json'), reader.project_relation_names)
-
-          @project_members = @relation_reader.consume_relation('project_members')
+          @project_attributes = relation_reader.consume_attributes(importable_path)
+          @project_members = relation_reader.consume_relation(importable_path, 'project_members')
 
           if relation_tree_restorer.restore
             import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
@@ -35,16 +36,28 @@ module Gitlab
 
         private
 
+        def relation_reader
+          strong_memoize(:relation_reader) do
+            ImportExport::JSON::LegacyReader::File.new(
+              File.join(shared.export_path, 'project.json'),
+              relation_names: reader.project_relation_names,
+              allowed_path: importable_path
+            )
+          end
+        end
+
         def relation_tree_restorer
           @relation_tree_restorer ||= RelationTreeRestorer.new(
             user: @user,
             shared: @shared,
-            importable: @project,
-            relation_reader: @relation_reader,
+            relation_reader: relation_reader,
             object_builder: object_builder,
             members_mapper: members_mapper,
             relation_factory: relation_factory,
-            reader: reader
+            reader: reader,
+            importable: @project,
+            importable_attributes: @project_attributes,
+            importable_path: importable_path
           )
         end
 
@@ -69,6 +82,10 @@ module Gitlab
         def import_failure_service
           @import_failure_service ||= ImportFailureService.new(@project)
         end
+
+        def importable_path
+          "project"
+        end
       end
     end
   end
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
index 01000c9d6d9..988776fe600 100644
--- a/lib/gitlab/import_export/project/tree_saver.rb
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -15,10 +15,17 @@ module Gitlab
         end
 
         def save
-          json_writer = ImportExport::JSON::LegacyWriter.new(@full_path)
-
-          serializer = ImportExport::JSON::StreamingSerializer.new(exportable, reader.project_tree, json_writer)
-          serializer.execute
+          json_writer = ImportExport::JSON::LegacyWriter.new(
+            @full_path,
+            allowed_path: "project"
+          )
+
+          ImportExport::JSON::StreamingSerializer.new(
+            exportable,
+            reader.project_tree,
+            json_writer,
+            exportable_path: "project"
+          ).execute
 
           true
         rescue => e
diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb
index 1157e18c7f9..7f25222ba41 100644
--- a/lib/gitlab/import_export/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/relation_tree_restorer.rb
@@ -11,7 +11,15 @@ module Gitlab
       attr_reader :importable
       attr_reader :relation_reader
 
-      def initialize(user:, shared:, importable:, relation_reader:, members_mapper:, object_builder:, relation_factory:, reader:)
+      def initialize( # rubocop:disable Metrics/ParameterLists
+        user:, shared:, relation_reader:,
+        members_mapper:, object_builder:,
+        relation_factory:,
+        reader:,
+        importable:,
+        importable_attributes:,
+        importable_path:
+      )
         @user = user
         @shared = shared
         @importable = importable
@@ -20,6 +28,8 @@ module Gitlab
         @object_builder = object_builder
         @relation_factory = relation_factory
         @reader = reader
+        @importable_attributes = importable_attributes
+        @importable_path = importable_path
       end
 
       def restore
@@ -57,7 +67,7 @@ module Gitlab
       end
 
       def process_relation!(relation_key, relation_definition)
-        @relation_reader.consume_relation(relation_key) do |data_hash, relation_index|
+        @relation_reader.consume_relation(@importable_path, relation_key) do |data_hash, relation_index|
           process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
         end
       end
@@ -94,7 +104,7 @@ module Gitlab
       end
 
       def update_params!
-        params = @relation_reader.root_attributes(relations.keys)
+        params = @importable_attributes.except(*relations.keys.map(&:to_s))
         params = params.merge(present_override_params)
 
         # Cleaning all imported and overridden params