Add latest changes from gitlab-org/gitlab@master

7 files changed, 10 insertions, 255 deletions

diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb
index 624acd3bb2a..5825db89201 100644
--- a/lib/gitlab/import_export/group/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb
@@ -34,7 +34,6 @@ module Gitlab
               update_params!
 
               BulkInsertableAssociations.with_bulk_insert(enabled: bulk_insert_enabled) do
-                fix_ci_pipelines_not_sorted_on_legacy_project_json!
                 create_relations!
               end
             end
@@ -275,15 +274,6 @@ module Gitlab
           }
         end
 
-        # Temporary fix for https://gitlab.com/gitlab-org/gitlab/-/issues/27883 when import from legacy project.json
-        # This should be removed once legacy JSON format is deprecated.
-        # Ndjson export file will fix the order during project export.
-        def fix_ci_pipelines_not_sorted_on_legacy_project_json!
-          return unless @relation_reader.legacy?
-
-          @relation_reader.sort_ci_pipelines_by_id
-        end
-
         # Enable logging of each top-level relation creation when Importing into a Group
         def log_relation_creation(importable, relation_key, relation_object)
           root_ancestor_group = importable.try(:root_ancestor)
diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb
deleted file mode 100644
index ee360020556..00000000000
--- a/lib/gitlab/import_export/json/legacy_reader.rb
+++ /dev/null
@@ -1,123 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    module Json
-      class LegacyReader
-        class File < LegacyReader
-          include Gitlab::Utils::StrongMemoize
-
-          def initialize(path, relation_names:, allowed_path: nil)
-            @path = path
-            super(
-              relation_names: relation_names,
-              allowed_path: allowed_path)
-          end
-
-          def exist?
-            ::File.exist?(@path)
-          end
-
-          protected
-
-          def tree_hash
-            strong_memoize(:tree_hash) do
-              read_hash
-            end
-          end
-
-          def read_hash
-            Gitlab::Json.parse(::File.read(@path))
-          rescue StandardError => e
-            Gitlab::ErrorTracking.log_exception(e)
-            raise Gitlab::ImportExport::Error, 'Incorrect JSON format'
-          end
-        end
-
-        class Hash < LegacyReader
-          def initialize(tree_hash, relation_names:, allowed_path: nil)
-            @tree_hash = tree_hash
-            super(
-              relation_names: relation_names,
-              allowed_path: allowed_path)
-          end
-
-          def exist?
-            @tree_hash.present?
-          end
-
-          protected
-
-          attr_reader :tree_hash
-        end
-
-        def initialize(relation_names:, allowed_path:)
-          @relation_names = relation_names.map(&:to_s)
-          @consumed_relations = Set.new
-
-          # This is legacy reader, to be used in transition
-          # period before `.ndjson`,
-          # we strong validate what is being readed
-          @allowed_path = allowed_path
-        end
-
-        def exist?
-          raise NotImplementedError
-        end
-
-        def legacy?
-          true
-        end
-
-        def consume_attributes(importable_path)
-          unless importable_path == @allowed_path
-            raise ArgumentError, "Invalid #{importable_path} passed to `consume_attributes`. Use #{@allowed_path} instead."
-          end
-
-          attributes
-        end
-
-        def consume_relation(importable_path, key)
-          unless importable_path == @allowed_path
-            raise ArgumentError, "Invalid #{importable_name} passed to `consume_relation`. Use #{@allowed_path} instead."
-          end
-
-          Enumerator.new do |documents|
-            next unless @consumed_relations.add?("#{importable_path}/#{key}")
-
-            value = relations.delete(key)
-            next if value.nil?
-
-            if value.is_a?(Array)
-              value.each.with_index do |item, idx|
-                documents << [item, idx]
-              end
-            else
-              documents << [value, 0]
-            end
-          end
-        end
-
-        def sort_ci_pipelines_by_id
-          relations['ci_pipelines']&.sort_by! { |hash| hash['id'] }
-        end
-
-        private
-
-        attr_reader :relation_names, :allowed_path
-
-        def tree_hash
-          raise NotImplementedError
-        end
-
-        def attributes
-          @attributes ||= tree_hash.slice!(*relation_names)
-        end
-
-        def relations
-          @relations ||= tree_hash.extract!(*relation_names)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb
deleted file mode 100644
index e03ab9f7650..00000000000
--- a/lib/gitlab/import_export/json/legacy_writer.rb
+++ /dev/null
@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module ImportExport
-    module Json
-      class LegacyWriter
-        include Gitlab::ImportExport::CommandLineUtil
-
-        attr_reader :path
-
-        def initialize(path, allowed_path:)
-          @path = path
-          @keys = Set.new
-
-          # This is legacy writer, to be used in transition
-          # period before `.ndjson`,
-          # we strong validate what is being written
-          @allowed_path = allowed_path
-
-          mkdir_p(File.dirname(@path))
-          file.write('{}')
-        end
-
-        def close
-          @file&.close
-          @file = nil
-        end
-
-        def write_attributes(exportable_path, hash)
-          unless exportable_path == @allowed_path
-            raise ArgumentError, "Invalid #{exportable_path}"
-          end
-
-          hash.each do |key, value|
-            write(key, value)
-          end
-        end
-
-        def write_relation(exportable_path, key, value)
-          unless exportable_path == @allowed_path
-            raise ArgumentError, "Invalid #{exportable_path}"
-          end
-
-          write(key, value)
-        end
-
-        def write_relation_array(exportable_path, key, items)
-          unless exportable_path == @allowed_path
-            raise ArgumentError, "Invalid #{exportable_path}"
-          end
-
-          write(key, [])
-
-          # rewind by two bytes, to overwrite ']}'
-          file.pos = file.size - 2
-
-          items.each_with_index do |item, idx|
-            file.write(',') if idx > 0
-            file.write(item.to_json)
-          end
-
-          file.write(']}')
-        end
-
-        private
-
-        def write(key, value)
-          raise ArgumentError, "key '#{key}' already written" if @keys.include?(key)
-
-          # rewind by one byte, to overwrite '}'
-          file.pos = file.size - 1
-
-          file.write(',') if @keys.any?
-          file.write(key.to_json)
-          file.write(':')
-          file.write(value.to_json)
-          file.write('}')
-
-          @keys.add(key)
-        end
-
-        def file
-          @file ||= File.open(@path, "wb")
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb
index 510da61d3ab..3de56aacf18 100644
--- a/lib/gitlab/import_export/json/ndjson_reader.rb
+++ b/lib/gitlab/import_export/json/ndjson_reader.rb
@@ -17,14 +17,12 @@ module Gitlab
           Dir.exist?(@dir_path)
         end
 
-        # This can be removed once legacy_reader is deprecated.
-        def legacy?
-          false
-        end
-
         def consume_attributes(importable_path)
           # This reads from `tree/project.json`
           path = file_path("#{importable_path}.json")
+
+          raise Gitlab::ImportExport::Error, 'Invalid file' if !File.exist?(path) || File.symlink?(path)
+
           data = File.read(path, MAX_JSON_DOCUMENT_SIZE)
           json_decode(data)
         end
@@ -36,7 +34,7 @@ module Gitlab
             # This reads from `tree/project/merge_requests.ndjson`
             path = file_path(importable_path, "#{key}.ndjson")
 
-            next unless File.exist?(path)
+            next if !File.exist?(path) || File.symlink?(path)
 
             File.foreach(path, MAX_JSON_DOCUMENT_SIZE).with_index do |line, line_num|
               documents << [json_decode(line), line_num]
diff --git a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb
index 034122a9f14..639f34980ff 100644
--- a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb
@@ -18,8 +18,6 @@ module Gitlab
           end
 
           def dates
-            return [] if @relation_reader.legacy?
-
             RelationFactory::DATE_MODELS.flat_map do |tag|
               @relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model|
                 model.first['due_date']
diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb
index 47f82a901b7..e791424875a 100644
--- a/lib/gitlab/import_export/project/tree_restorer.rb
+++ b/lib/gitlab/import_export/project/tree_restorer.rb
@@ -17,7 +17,7 @@ module Gitlab
         end
 
         def restore
-          unless relation_reader
+          unless relation_reader.exist?
             raise Gitlab::ImportExport::Error, 'invalid import format'
           end
 
@@ -47,28 +47,11 @@ module Gitlab
         private
 
         def relation_reader
-          strong_memoize(:relation_reader) do
-            [ndjson_relation_reader, legacy_relation_reader]
-              .compact.find(&:exist?)
-          end
-        end
-
-        def ndjson_relation_reader
-          return unless Feature.enabled?(:project_import_ndjson, project.namespace)
-
-          ImportExport::Json::NdjsonReader.new(
+          @relation_reader ||= ImportExport::Json::NdjsonReader.new(
             File.join(shared.export_path, 'tree')
           )
         end
 
-        def legacy_relation_reader
-          ImportExport::Json::LegacyReader::File.new(
-            File.join(shared.export_path, 'project.json'),
-            relation_names: reader.project_relation_names,
-            allowed_path: importable_path
-          )
-        end
-
         def relation_tree_restorer
           @relation_tree_restorer ||= relation_tree_restorer_class.new(
             user: @user,
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
index 05b96f7e8ce..fd5fa73764e 100644
--- a/lib/gitlab/import_export/project/tree_saver.rb
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -81,13 +81,10 @@ module Gitlab
         end
 
         def json_writer
-          @json_writer ||= if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace)
-                             full_path = File.join(@shared.export_path, 'tree')
-                             Gitlab::ImportExport::Json::NdjsonWriter.new(full_path)
-                           else
-                             full_path = File.join(@shared.export_path, ImportExport.project_filename)
-                             Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project')
-                           end
+          @json_writer ||= begin
+            full_path = File.join(@shared.export_path, 'tree')
+            Gitlab::ImportExport::Json::NdjsonWriter.new(full_path)
+          end
         end
       end
     end