Add latest changes from gitlab-org/gitlab@master

15 files changed, 149 insertions, 71 deletions

diff --git a/lib/gitlab/graphql/connections.rb b/lib/gitlab/graphql/connections.rb
deleted file mode 100644
index 08d5cd0b72e..00000000000
--- a/lib/gitlab/graphql/connections.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Graphql
-    module Connections
-      def self.use(_schema)
-        GraphQL::Relay::BaseConnection.register_connection_implementation(
-          ActiveRecord::Relation,
-          Gitlab::Graphql::Connections::Keyset::Connection
-        )
-        GraphQL::Relay::BaseConnection.register_connection_implementation(
-          Gitlab::Graphql::FilterableArray,
-          Gitlab::Graphql::Connections::FilterableArrayConnection
-        )
-        GraphQL::Relay::BaseConnection.register_connection_implementation(
-          Gitlab::Graphql::ExternallyPaginatedArray,
-          Gitlab::Graphql::Connections::ExternallyPaginatedArrayConnection
-        )
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/graphql/pagination/connections.rb b/lib/gitlab/graphql/pagination/connections.rb
new file mode 100644
index 00000000000..febdc938317
--- /dev/null
+++ b/lib/gitlab/graphql/pagination/connections.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Graphql
+    module Pagination
+      module Connections
+        def self.use(schema)
+          schema.connections.add(
+            ActiveRecord::Relation,
+            Gitlab::Graphql::Pagination::Keyset::Connection)
+
+          schema.connections.add(
+            Gitlab::Graphql::FilterableArray,
+            Gitlab::Graphql::Pagination::FilterableArrayConnection)
+
+          schema.connections.add(
+            Gitlab::Graphql::ExternallyPaginatedArray,
+            Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/graphql/connections/externally_paginated_array_connection.rb b/lib/gitlab/graphql/pagination/externally_paginated_array_connection.rb
index f0861260691..1f01dd07571 100644
--- a/lib/gitlab/graphql/connections/externally_paginated_array_connection.rb
+++ b/lib/gitlab/graphql/pagination/externally_paginated_array_connection.rb
@@ -3,20 +3,14 @@
 # Make a customized connection type
 module Gitlab
   module Graphql
-    module Connections
-      class ExternallyPaginatedArrayConnection < GraphQL::Relay::ArrayConnection
-        # As the pagination happens externally
-        # we just return all the nodes here.
-        def sliced_nodes
-          @nodes
-        end
-
+    module Pagination
+      class ExternallyPaginatedArrayConnection < GraphQL::Pagination::ArrayConnection
         def start_cursor
-          nodes.previous_cursor
+          items.previous_cursor
         end
 
         def end_cursor
-          nodes.next_cursor
+          items.next_cursor
         end
 
         def next_page?
diff --git a/lib/gitlab/graphql/connections/filterable_array_connection.rb b/lib/gitlab/graphql/pagination/filterable_array_connection.rb
index 800f2c949c6..4a76cd5fb00 100644
--- a/lib/gitlab/graphql/connections/filterable_array_connection.rb
+++ b/lib/gitlab/graphql/pagination/filterable_array_connection.rb
@@ -2,14 +2,14 @@
 
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       # FilterableArrayConnection is useful especially for lazy-loaded values.
       # It allows us to call a callback only on the slice of array being
       # rendered in the "after loaded" phase.  For example we can check
       # permissions only on a small subset of items.
-      class FilterableArrayConnection < GraphQL::Relay::ArrayConnection
-        def paged_nodes
-          @filtered_nodes ||= nodes.filter_callback.call(super)
+      class FilterableArrayConnection < GraphQL::Pagination::ArrayConnection
+        def nodes
+          @nodes ||= items.filter_callback.call(super)
         end
       end
     end
diff --git a/lib/gitlab/graphql/connections/keyset/conditions/base_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb
index 26c9d77a8df..afea7c602be 100644
--- a/lib/gitlab/graphql/connections/keyset/conditions/base_condition.rb
+++ b/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       module Keyset
         module Conditions
           class BaseCondition
diff --git a/lib/gitlab/graphql/connections/keyset/conditions/not_null_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb
index 3239d27c0cd..3164598b7b9 100644
--- a/lib/gitlab/graphql/connections/keyset/conditions/not_null_condition.rb
+++ b/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       module Keyset
         module Conditions
           class NotNullCondition < BaseCondition
diff --git a/lib/gitlab/graphql/connections/keyset/conditions/null_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb
index 18ea0692e2c..fa25181d663 100644
--- a/lib/gitlab/graphql/connections/keyset/conditions/null_condition.rb
+++ b/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       module Keyset
         module Conditions
           class NullCondition < BaseCondition
diff --git a/lib/gitlab/graphql/connections/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb
index 5de075f2f7a..5466924a794 100644
--- a/lib/gitlab/graphql/connections/keyset/connection.rb
+++ b/lib/gitlab/graphql/pagination/keyset/connection.rb
@@ -27,21 +27,21 @@
 #
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       module Keyset
-        class Connection < GraphQL::Relay::BaseConnection
+        class Connection < GraphQL::Pagination::ActiveRecordRelationConnection
           include Gitlab::Utils::StrongMemoize
 
-          def cursor_from_node(node)
+          def cursor_for(node)
             encoded_json_from_ordering(node)
           end
 
           def sliced_nodes
             @sliced_nodes ||=
               begin
-                OrderInfo.validate_ordering(ordered_nodes, order_list)
+                OrderInfo.validate_ordering(ordered_items, order_list)
 
-                sliced = ordered_nodes
+                sliced = ordered_items
                 sliced = slice_nodes(sliced, before, :before) if before.present?
                 sliced = slice_nodes(sliced, after, :after) if after.present?
 
@@ -49,12 +49,12 @@ module Gitlab
               end
           end
 
-          def paged_nodes
+          def nodes
             # These are the nodes that will be loaded into memory for rendering
             # So we're ok loading them into memory here as that's bound to happen
             # anyway. Having them ready means we can modify the result while
             # rendering the fields.
-            @paged_nodes ||= load_paged_nodes.to_a
+            @nodes ||= load_paged_nodes.to_a
           end
 
           private
@@ -85,31 +85,31 @@ module Gitlab
             @limit_value ||= [first, last, max_page_size].compact.min
           end
 
-          def ordered_nodes
-            strong_memoize(:order_nodes) do
-              unless nodes.primary_key.present?
+          def ordered_items
+            strong_memoize(:ordered_items) do
+              unless items.primary_key.present?
                 raise ArgumentError.new('Relation must have a primary key')
               end
 
-              list = OrderInfo.build_order_list(nodes)
+              list = OrderInfo.build_order_list(items)
 
               # ensure there is a primary key ordering
-              if list&.last&.attribute_name != nodes.primary_key
-                nodes.order(arel_table[nodes.primary_key].desc) # rubocop: disable CodeReuse/ActiveRecord
+              if list&.last&.attribute_name != items.primary_key
+                items.order(arel_table[items.primary_key].desc) # rubocop: disable CodeReuse/ActiveRecord
               else
-                nodes
+                items
               end
             end
           end
 
           def order_list
             strong_memoize(:order_list) do
-              OrderInfo.build_order_list(ordered_nodes)
+              OrderInfo.build_order_list(ordered_items)
             end
           end
 
           def arel_table
-            nodes.arel_table
+            items.arel_table
           end
 
           # Storing the current order values in the cursor allows us to
diff --git a/lib/gitlab/graphql/connections/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb
index 7f61bf937b4..876d6114f3c 100644
--- a/lib/gitlab/graphql/connections/keyset/order_info.rb
+++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       module Keyset
         class OrderInfo
           attr_reader :attribute_name, :sort_direction, :named_function
diff --git a/lib/gitlab/graphql/connections/keyset/query_builder.rb b/lib/gitlab/graphql/pagination/keyset/query_builder.rb
index fe85898f638..331981ce723 100644
--- a/lib/gitlab/graphql/connections/keyset/query_builder.rb
+++ b/lib/gitlab/graphql/pagination/keyset/query_builder.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   module Graphql
-    module Connections
+    module Pagination
       module Keyset
         class QueryBuilder
           def initialize(arel_table, order_list, decoded_cursor, before_or_after)
diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb
index f6ebd83bfaa..323e6727a9f 100644
--- a/lib/gitlab/import_export/group/tree_restorer.rb
+++ b/lib/gitlab/import_export/group/tree_restorer.rb
@@ -20,6 +20,7 @@ module Gitlab
         def restore
           @group_attributes = relation_reader.consume_attributes(nil)
           @group_members = relation_reader.consume_relation(nil, 'members')
+            .map(&:first)
 
           # We need to remove `name` and `path` as we did consume it in previous pass
           @group_attributes.delete('name')
diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb
index 57579fe9def..12d6458aedc 100644
--- a/lib/gitlab/import_export/json/legacy_reader.rb
+++ b/lib/gitlab/import_export/json/legacy_reader.rb
@@ -53,6 +53,7 @@ module Gitlab
 
         def initialize(relation_names:, allowed_path:)
           @relation_names = relation_names.map(&:to_s)
+          @consumed_relations = Set.new
 
           # This is legacy reader, to be used in transition
           # period before `.ndjson`,
@@ -81,17 +82,19 @@ module Gitlab
             raise ArgumentError, "Invalid #{importable_name} passed to `consume_relation`. Use #{@allowed_path} instead."
           end
 
-          value = relations.delete(key)
+          Enumerator.new do |documents|
+            next unless @consumed_relations.add?("#{importable_path}/#{key}")
 
-          return value unless block_given?
-          return if value.nil?
+            value = relations.delete(key)
+            next if value.nil?
 
-          if value.is_a?(Array)
-            value.each.with_index do |item, idx|
-              yield(item, idx)
+            if value.is_a?(Array)
+              value.each.with_index do |item, idx|
+                documents << [item, idx]
+              end
+            else
+              documents << [value, 0]
             end
-          else
-            yield(value, 0)
           end
         end
 
diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb
new file mode 100644
index 00000000000..e9b05afc7d4
--- /dev/null
+++ b/lib/gitlab/import_export/json/ndjson_reader.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module JSON
+      class NdjsonReader
+        MAX_JSON_DOCUMENT_SIZE = 50.megabytes
+
+        attr_reader :dir_path
+
+        def initialize(dir_path)
+          @dir_path = dir_path
+          @consumed_relations = Set.new
+        end
+
+        def exist?
+          Dir.exist?(@dir_path)
+        end
+
+        # This can be removed once legacy_reader is deprecated.
+        def legacy?
+          false
+        end
+
+        def consume_attributes(importable_path)
+          # This reads from `tree/project.json`
+          path = file_path("#{importable_path}.json")
+          data = File.read(path, MAX_JSON_DOCUMENT_SIZE)
+          json_decode(data)
+        end
+
+        def consume_relation(importable_path, key)
+          Enumerator.new do |documents|
+            next unless @consumed_relations.add?("#{importable_path}/#{key}")
+
+            # This reads from `tree/project/merge_requests.ndjson`
+            path = file_path(importable_path, "#{key}.ndjson")
+            next unless File.exist?(path)
+
+            File.foreach(path, MAX_JSON_DOCUMENT_SIZE).with_index do |line, line_num|
+              documents << [json_decode(line), line_num]
+            end
+          end
+        end
+
+        private
+
+        def json_decode(string)
+          ActiveSupport::JSON.decode(string)
+        rescue ActiveSupport::JSON.parse_error => e
+          Gitlab::ErrorTracking.log_exception(e)
+          raise Gitlab::ImportExport::Error, 'Incorrect JSON format'
+        end
+
+        def file_path(*path)
+          File.join(dir_path, *path)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb
index 99e57d9decd..ad3720b56be 100644
--- a/lib/gitlab/import_export/project/tree_restorer.rb
+++ b/lib/gitlab/import_export/project/tree_restorer.rb
@@ -17,8 +17,13 @@ module Gitlab
         end
 
         def restore
+          unless relation_reader
+            raise Gitlab::ImportExport::Error, 'invalid import format'
+          end
+
           @project_attributes = relation_reader.consume_attributes(importable_path)
           @project_members = relation_reader.consume_relation(importable_path, 'project_members')
+            .map(&:first)
 
           if relation_tree_restorer.restore
             import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
@@ -38,14 +43,27 @@ module Gitlab
 
         def relation_reader
           strong_memoize(:relation_reader) do
-            ImportExport::JSON::LegacyReader::File.new(
-              File.join(shared.export_path, 'project.json'),
-              relation_names: reader.project_relation_names,
-              allowed_path: importable_path
-            )
+            [ndjson_relation_reader, legacy_relation_reader]
+              .compact.find(&:exist?)
           end
         end
 
+        def ndjson_relation_reader
+          return unless Feature.enabled?(:project_import_ndjson, project.namespace)
+
+          ImportExport::JSON::NdjsonReader.new(
+            File.join(shared.export_path, 'tree')
+          )
+        end
+
+        def legacy_relation_reader
+          ImportExport::JSON::LegacyReader::File.new(
+            File.join(shared.export_path, 'project.json'),
+            relation_names: reader.project_relation_names,
+            allowed_path: importable_path
+          )
+        end
+
         def relation_tree_restorer
           @relation_tree_restorer ||= RelationTreeRestorer.new(
             user: @user,
diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb
index 78ed365cea0..056945d0294 100644
--- a/lib/gitlab/import_export/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/relation_tree_restorer.rb
@@ -67,7 +67,7 @@ module Gitlab
       end
 
       def process_relation!(relation_key, relation_definition)
-        @relation_reader.consume_relation(@importable_path, relation_key) do |data_hash, relation_index|
+        @relation_reader.consume_relation(@importable_path, relation_key).each do |data_hash, relation_index|
           process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
         end
       end