Add latest changes from gitlab-org/gitlab@master

6 files changed, 242 insertions, 31 deletions

diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb
index 48f6925884b..fd1eb329ad2 100644
--- a/lib/gitlab/import_export/group/tree_saver.rb
+++ b/lib/gitlab/import_export/group/tree_saver.rb
@@ -49,7 +49,7 @@ module Gitlab
         end
 
         def tree_saver
-          @tree_saver ||= RelationTreeSaver.new
+          @tree_saver ||= LegacyRelationTreeSaver.new
         end
       end
     end
diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb
new file mode 100644
index 00000000000..c935e360a65
--- /dev/null
+++ b/lib/gitlab/import_export/json/legacy_writer.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module JSON
+      class LegacyWriter
+        include Gitlab::ImportExport::CommandLineUtil
+
+        attr_reader :path
+
+        def initialize(path)
+          @path = path
+          @last_array = nil
+          @keys = Set.new
+
+          mkdir_p(File.dirname(@path))
+          file.write('{}')
+        end
+
+        def close
+          @file&.close
+          @file = nil
+        end
+
+        def set(hash)
+          hash.each do |key, value|
+            write(key, value)
+          end
+        end
+
+        def write(key, value)
+          raise ArgumentError, "key '#{key}' already written" if @keys.include?(key)
+
+          # rewind by one byte, to overwrite '}'
+          file.pos = file.size - 1
+
+          file.write(',') if @keys.any?
+          file.write(key.to_json)
+          file.write(':')
+          file.write(value.to_json)
+          file.write('}')
+
+          @keys.add(key)
+          @last_array = nil
+          @last_array_count = nil
+        end
+
+        def append(key, value)
+          unless @last_array == key
+            write(key, [])
+
+            @last_array = key
+            @last_array_count = 0
+          end
+
+          # rewind by two bytes, to overwrite ']}'
+          file.pos = file.size - 2
+
+          file.write(',') if @last_array_count > 0
+          file.write(value.to_json)
+          file.write(']}')
+          @last_array_count += 1
+        end
+
+        private
+
+        def file
+          @file ||= File.open(@path, "wb")
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb
new file mode 100644
index 00000000000..d053bf16166
--- /dev/null
+++ b/lib/gitlab/import_export/json/streaming_serializer.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module JSON
+      class StreamingSerializer
+        include Gitlab::ImportExport::CommandLineUtil
+
+        BATCH_SIZE = 100
+
+        class Raw < String
+          def to_json(*_args)
+            to_s
+          end
+        end
+
+        def initialize(exportable, relations_schema, json_writer)
+          @exportable = exportable
+          @relations_schema = relations_schema
+          @json_writer = json_writer
+        end
+
+        def execute
+          serialize_root
+
+          includes.each do |relation_definition|
+            serialize_relation(relation_definition)
+          end
+        end
+
+        private
+
+        attr_reader :json_writer, :relations_schema, :exportable
+
+        def serialize_root
+          attributes = exportable.as_json(
+            relations_schema.merge(include: nil, preloads: nil))
+          json_writer.set(attributes)
+        end
+
+        def serialize_relation(definition)
+          raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash)
+          raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one?
+
+          key, options = definition.first
+
+          record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend
+          if record.is_a?(ActiveRecord::Relation)
+            serialize_many_relations(key, record, options)
+          else
+            serialize_single_relation(key, record, options)
+          end
+        end
+
+        def serialize_many_relations(key, records, options)
+          key_preloads = preloads&.dig(key)
+          records = records.preload(key_preloads) if key_preloads
+
+          records.find_each(batch_size: BATCH_SIZE) do |record|
+            json = Raw.new(record.to_json(options))
+
+            json_writer.append(key, json)
+          end
+        end
+
+        def serialize_single_relation(key, record, options)
+          json = Raw.new(record.to_json(options))
+
+          json_writer.write(key, json)
+        end
+
+        def includes
+          relations_schema[:include]
+        end
+
+        def preloads
+          relations_schema[:preload]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb
index ed5392c13d0..fe3e64358e5 100644
--- a/lib/gitlab/import_export/relation_tree_saver.rb
+++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   module ImportExport
-    class RelationTreeSaver
+    class LegacyRelationTreeSaver
       include Gitlab::ImportExport::CommandLineUtil
 
       def serialize(exportable, relations_tree)
diff --git a/lib/gitlab/import_export/project/legacy_tree_saver.rb b/lib/gitlab/import_export/project/legacy_tree_saver.rb
new file mode 100644
index 00000000000..2ed98f52c58
--- /dev/null
+++ b/lib/gitlab/import_export/project/legacy_tree_saver.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Project
+      class LegacyTreeSaver
+        attr_reader :full_path
+
+        def initialize(project:, current_user:, shared:, params: {})
+          @params       = params
+          @project      = project
+          @current_user = current_user
+          @shared       = shared
+          @full_path    = File.join(@shared.export_path, ImportExport.project_filename)
+        end
+
+        def save
+          project_tree = tree_saver.serialize(@project, reader.project_tree)
+          fix_project_tree(project_tree)
+          tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
+
+          true
+        rescue => e
+          @shared.error(e)
+          false
+        end
+
+        private
+
+        # Aware that the resulting hash needs to be pure-hash and
+        # does not include any AR objects anymore, only objects that run `.to_json`
+        def fix_project_tree(project_tree)
+          if @params[:description].present?
+            project_tree['description'] = @params[:description]
+          end
+
+          project_tree['project_members'] += group_members_array
+        end
+
+        def reader
+          @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
+        end
+
+        def group_members_array
+          group_members.as_json(reader.group_members_tree).each do |group_member|
+            group_member['source_type'] = 'Project' # Make group members project members of the future import
+          end
+        end
+
+        def group_members
+          return [] unless @current_user.can?(:admin_group, @project.group)
+
+          # We need `.where.not(user_id: nil)` here otherwise when a group has an
+          # invitee, it would make the following query return 0 rows since a NULL
+          # user_id would be present in the subquery
+          # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
+          non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
+
+          GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
+        end
+
+        def tree_saver
+          @tree_saver ||= Gitlab::ImportExport::LegacyRelationTreeSaver.new
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
index 32b3b518ece..01000c9d6d9 100644
--- a/lib/gitlab/import_export/project/tree_saver.rb
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -15,52 +15,40 @@ module Gitlab
         end
 
         def save
-          project_tree = tree_saver.serialize(@project, reader.project_tree)
-          fix_project_tree(project_tree)
-          tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
+          json_writer = ImportExport::JSON::LegacyWriter.new(@full_path)
+
+          serializer = ImportExport::JSON::StreamingSerializer.new(exportable, reader.project_tree, json_writer)
+          serializer.execute
 
           true
         rescue => e
           @shared.error(e)
           false
+        ensure
+          json_writer&.close
         end
 
         private
 
-        # Aware that the resulting hash needs to be pure-hash and
-        # does not include any AR objects anymore, only objects that run `.to_json`
-        def fix_project_tree(project_tree)
-          if @params[:description].present?
-            project_tree['description'] = @params[:description]
-          end
-
-          project_tree['project_members'] += group_members_array
-        end
-
         def reader
           @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
         end
 
-        def group_members_array
-          group_members.as_json(reader.group_members_tree).each do |group_member|
-            group_member['source_type'] = 'Project' # Make group members project members of the future import
-          end
+        def exportable
+          @project.present(exportable_params)
         end
 
-        def group_members
-          return [] unless @current_user.can?(:admin_group, @project.group)
-
-          # We need `.where.not(user_id: nil)` here otherwise when a group has an
-          # invitee, it would make the following query return 0 rows since a NULL
-          # user_id would be present in the subquery
-          # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
-          non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
-
-          GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
+        def exportable_params
+          params = {
+            presenter_class: presenter_class,
+            current_user: @current_user
+          }
+          params[:override_description] = @params[:description] if @params[:description].present?
+          params
         end
 
-        def tree_saver
-          @tree_saver ||= RelationTreeSaver.new
+        def presenter_class
+          Projects::ImportExport::ProjectExportPresenter
         end
       end
     end