Add latest changes from gitlab-org/gitlab@14-8-stable-eev14.8.0-rc42

14 files changed, 269 insertions, 87 deletions

diff --git a/lib/bulk_imports/common/extractors/graphql_extractor.rb b/lib/bulk_imports/common/extractors/graphql_extractor.rb
index cde3d1cad5b..bfdc0b13603 100644
--- a/lib/bulk_imports/common/extractors/graphql_extractor.rb
+++ b/lib/bulk_imports/common/extractors/graphql_extractor.rb
@@ -5,15 +5,16 @@ module BulkImports
     module Extractors
       class GraphqlExtractor
         def initialize(options = {})
-          @query = options[:query]
+          @query_klass = options[:query]
         end
 
         def extract(context)
           client = graphql_client(context)
+          query = query_klass.new(context: context)
 
           response = client.execute(
             client.parse(query.to_s),
-            query.variables(context)
+            query.variables
           ).original_hash.deep_dup
 
           BulkImports::Pipeline::ExtractedData.new(
@@ -24,7 +25,7 @@ module BulkImports
 
         private
 
-        attr_reader :query
+        attr_reader :query_klass
 
         def graphql_client(context)
           @graphql_client ||= BulkImports::Clients::Graphql.new(
diff --git a/lib/bulk_imports/groups/graphql/get_members_query.rb b/lib/bulk_imports/common/graphql/get_members_query.rb
index e76c87cc521..00977f694d7 100644
--- a/lib/bulk_imports/groups/graphql/get_members_query.rb
+++ b/lib/bulk_imports/common/graphql/get_members_query.rb
@@ -1,15 +1,20 @@
 # frozen_string_literal: true
 
 module BulkImports
-  module Groups
+  module Common
     module Graphql
-      module GetMembersQuery
-        extend self
+      class GetMembersQuery
+        attr_reader :context
+
+        def initialize(context:)
+          @context = context
+        end
+
         def to_s
-          <<-'GRAPHQL'
+          <<-GRAPHQL
           query($full_path: ID!, $cursor: String, $per_page: Int) {
-            group(fullPath: $full_path) {
-              group_members: groupMembers(relations: DIRECT, first: $per_page, after: $cursor) {
+            portable: #{context.entity.entity_type}(fullPath: $full_path) {
+              members: #{members_type}(relations: [DIRECT, INHERITED], first: $per_page, after: $cursor) {
                 page_info: pageInfo {
                   next_page: endCursor
                   has_next_page: hasNextPage
@@ -32,7 +37,7 @@ module BulkImports
           GRAPHQL
         end
 
-        def variables(context)
+        def variables
           {
             full_path: context.entity.source_full_path,
             cursor: context.tracker.next_page,
@@ -40,10 +45,6 @@ module BulkImports
           }
         end
 
-        def base_path
-          %w[data group group_members]
-        end
-
         def data_path
           base_path << 'nodes'
         end
@@ -51,6 +52,20 @@ module BulkImports
         def page_info_path
           base_path << 'page_info'
         end
+
+        private
+
+        def base_path
+          %w[data portable members]
+        end
+
+        def members_type
+          if context.entity.group?
+            'groupMembers'
+          else
+            'projectMembers'
+          end
+        end
       end
     end
   end
diff --git a/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb b/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb
new file mode 100644
index 00000000000..2e6a29f4738
--- /dev/null
+++ b/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb
@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Common
+    module Pipelines
+      class LfsObjectsPipeline
+        include Pipeline
+
+        def extract(_context)
+          download_service.execute
+          decompression_service.execute
+          extraction_service.execute
+
+          file_paths = Dir.glob(File.join(tmpdir, '*'))
+
+          BulkImports::Pipeline::ExtractedData.new(data: file_paths)
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def load(_context, file_path)
+          Gitlab::Utils.check_path_traversal!(file_path)
+          Gitlab::Utils.check_allowed_absolute_path!(file_path, [Dir.tmpdir])
+
+          return if tar_filepath?(file_path)
+          return if lfs_json_filepath?(file_path)
+          return if File.directory?(file_path)
+          return if File.lstat(file_path).symlink?
+
+          size = File.size(file_path)
+          oid = LfsObject.calculate_oid(file_path)
+
+          lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size)
+          lfs_object.file = File.open(file_path) unless lfs_object.file&.exists?
+          lfs_object.save! if lfs_object.changed?
+
+          repository_types(oid)&.each do |type|
+            create_lfs_objects_project(lfs_object, type)
+          end
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        def after_run(_)
+          FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir)
+        end
+
+        private
+
+        def download_service
+          BulkImports::FileDownloadService.new(
+            configuration: context.configuration,
+            relative_url: context.entity.relation_download_url_path(relation),
+            tmpdir: tmpdir,
+            filename: targz_filename
+          )
+        end
+
+        def decompression_service
+          BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: targz_filename)
+        end
+
+        def extraction_service
+          BulkImports::ArchiveExtractionService.new(tmpdir: tmpdir, filename: tar_filename)
+        end
+
+        def lfs_json
+          @lfs_json ||= Gitlab::Json.parse(File.read(lfs_json_filepath))
+        rescue StandardError
+          raise BulkImports::Error, 'LFS Objects JSON read failed'
+        end
+
+        def tmpdir
+          @tmpdir ||= Dir.mktmpdir('bulk_imports')
+        end
+
+        def relation
+          BulkImports::FileTransfer::ProjectConfig::LFS_OBJECTS_RELATION
+        end
+
+        def tar_filename
+          "#{relation}.tar"
+        end
+
+        def targz_filename
+          "#{tar_filename}.gz"
+        end
+
+        def lfs_json_filepath?(file_path)
+          file_path == lfs_json_filepath
+        end
+
+        def tar_filepath?(file_path)
+          File.join(tmpdir, tar_filename) == file_path
+        end
+
+        def lfs_json_filepath
+          File.join(tmpdir, "#{relation}.json")
+        end
+
+        def create_lfs_objects_project(lfs_object, repository_type)
+          return unless allowed_repository_types.include?(repository_type)
+
+          lfs_objects_project = LfsObjectsProject.create(
+            project: portable,
+            lfs_object: lfs_object,
+            repository_type: repository_type
+          )
+
+          return if lfs_objects_project.persisted?
+
+          logger.warn(
+            project_id: portable.id,
+            message: 'Failed to save lfs objects project',
+            errors: lfs_objects_project.errors.full_messages.to_sentence,
+            **Gitlab::ApplicationContext.current
+          )
+        end
+
+        def repository_types(oid)
+          types = lfs_json[oid]
+
+          return [] unless types
+          return [] unless types.is_a?(Array)
+
+          # only return allowed repository types
+          types.uniq & allowed_repository_types
+        end
+
+        def allowed_repository_types
+          @allowed_repository_types ||= LfsObjectsProject.repository_types.values.push(nil)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/common/pipelines/members_pipeline.rb b/lib/bulk_imports/common/pipelines/members_pipeline.rb
new file mode 100644
index 00000000000..f35eb5ccf5e
--- /dev/null
+++ b/lib/bulk_imports/common/pipelines/members_pipeline.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Common
+    module Pipelines
+      class MembersPipeline
+        include Pipeline
+
+        transformer Common::Transformers::ProhibitedAttributesTransformer
+        transformer BulkImports::Groups::Transformers::MemberAttributesTransformer
+
+        def extract(context)
+          graphql_extractor.extract(context)
+        end
+
+        def load(_context, data)
+          return unless data
+
+          user_id = data[:user_id]
+
+          # Current user is already a member
+          return if user_id == current_user.id
+
+          user_membership = existing_user_membership(user_id)
+
+          # User is already a member with higher existing (inherited) membership
+          return if user_membership && user_membership[:access_level] >= data[:access_level]
+
+          # Create new membership for any other access level
+          portable.members.create!(data)
+        end
+
+        private
+
+        def graphql_extractor
+          @graphql_extractor ||= BulkImports::Common::Extractors::GraphqlExtractor
+            .new(query: BulkImports::Common::Graphql::GetMembersQuery)
+        end
+
+        def existing_user_membership(user_id)
+          members_finder.execute.find_by_user_id(user_id)
+        end
+
+        def members_finder
+          @members_finder ||= if context.entity.group?
+                                ::GroupMembersFinder.new(portable, current_user)
+                              else
+                                ::MembersFinder.new(portable, current_user)
+                              end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/graphql/get_group_query.rb b/lib/bulk_imports/groups/graphql/get_group_query.rb
index 6852e25c87f..911b2b67d8c 100644
--- a/lib/bulk_imports/groups/graphql/get_group_query.rb
+++ b/lib/bulk_imports/groups/graphql/get_group_query.rb
@@ -3,8 +3,12 @@
 module BulkImports
   module Groups
     module Graphql
-      module GetGroupQuery
-        extend self
+      class GetGroupQuery
+        attr_reader :context
+
+        def initialize(context:)
+          @context = context
+        end
 
         def to_s
           <<-'GRAPHQL'
@@ -29,7 +33,7 @@ module BulkImports
           GRAPHQL
         end
 
-        def variables(context)
+        def variables
           { full_path: context.entity.source_full_path }
         end
 
diff --git a/lib/bulk_imports/groups/graphql/get_projects_query.rb b/lib/bulk_imports/groups/graphql/get_projects_query.rb
index 4cec1ad1462..3f74bbb8cce 100644
--- a/lib/bulk_imports/groups/graphql/get_projects_query.rb
+++ b/lib/bulk_imports/groups/graphql/get_projects_query.rb
@@ -3,8 +3,12 @@
 module BulkImports
   module Groups
     module Graphql
-      module GetProjectsQuery
-        extend self
+      class GetProjectsQuery
+        attr_reader :context
+
+        def initialize(context:)
+          @context = context
+        end
 
         def to_s
           <<-'GRAPHQL'
@@ -25,7 +29,7 @@ module BulkImports
           GRAPHQL
         end
 
-        def variables(context)
+        def variables
           {
             full_path: context.entity.source_full_path,
             cursor: context.tracker.next_page,
diff --git a/lib/bulk_imports/groups/pipelines/members_pipeline.rb b/lib/bulk_imports/groups/pipelines/members_pipeline.rb
deleted file mode 100644
index 265abd5e3a7..00000000000
--- a/lib/bulk_imports/groups/pipelines/members_pipeline.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module BulkImports
-  module Groups
-    module Pipelines
-      class MembersPipeline
-        include Pipeline
-
-        extractor BulkImports::Common::Extractors::GraphqlExtractor,
-          query: BulkImports::Groups::Graphql::GetMembersQuery
-
-        transformer Common::Transformers::ProhibitedAttributesTransformer
-        transformer BulkImports::Groups::Transformers::MemberAttributesTransformer
-
-        def load(context, data)
-          return unless data
-
-          # Current user is already a member
-          return if data['user_id'].to_i == context.current_user.id
-
-          context.group.members.create!(data)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb
index 1a3babe1679..bc27220391d 100644
--- a/lib/bulk_imports/groups/stage.rb
+++ b/lib/bulk_imports/groups/stage.rb
@@ -16,7 +16,7 @@ module BulkImports
             stage: 1
           },
           members: {
-            pipeline: BulkImports::Groups::Pipelines::MembersPipeline,
+            pipeline: BulkImports::Common::Pipelines::MembersPipeline,
             stage: 1
           },
           labels: {
diff --git a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb
index b9de375d0e9..da50a19ee62 100644
--- a/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb
+++ b/lib/bulk_imports/groups/transformers/member_attributes_transformer.rb
@@ -5,50 +5,35 @@ module BulkImports
     module Transformers
       class MemberAttributesTransformer
         def transform(context, data)
-          data
-            .then { |data| add_user(data, context) }
-            .then { |data| add_access_level(data) }
-            .then { |data| add_author(data, context) }
-        end
-
-        private
-
-        def add_user(data, context)
           user = find_user(data&.dig('user', 'public_email'))
+          access_level = data&.dig('access_level', 'integer_value')
 
+          return unless data
           return unless user
+          return unless valid_access_level?(access_level)
 
           cache_source_user_id(data, user, context)
 
-          data
-            .except('user')
-            .merge('user_id' => user.id)
+          {
+            user_id: user.id,
+            access_level: access_level,
+            created_at: data['created_at'],
+            updated_at: data['updated_at'],
+            expires_at: data['expires_at'],
+            created_by_id: context.current_user.id
+          }
         end
 
+        private
+
         def find_user(email)
           return unless email
 
           User.find_by_any_email(email, confirmed: true)
         end
 
-        def add_access_level(data)
-          access_level = data&.dig('access_level', 'integer_value')
-
-          return unless valid_access_level?(access_level)
-
-          data.merge('access_level' => access_level)
-        end
-
         def valid_access_level?(access_level)
-          Gitlab::Access
-            .options_with_owner
-            .value?(access_level)
-        end
-
-        def add_author(data, context)
-          return unless data
-
-          data.merge('created_by_id' => context.current_user.id)
+          Gitlab::Access.options_with_owner.value?(access_level)
         end
 
         def cache_source_user_id(data, user, context)
diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb
index 04ac0916bbc..b3d7f3f4683 100644
--- a/lib/bulk_imports/projects/graphql/get_project_query.rb
+++ b/lib/bulk_imports/projects/graphql/get_project_query.rb
@@ -3,9 +3,8 @@
 module BulkImports
   module Projects
     module Graphql
-      module GetProjectQuery
-        extend Queryable
-        extend self
+      class GetProjectQuery
+        include Queryable
 
         def to_s
           <<-'GRAPHQL'
diff --git a/lib/bulk_imports/projects/graphql/get_repository_query.rb b/lib/bulk_imports/projects/graphql/get_repository_query.rb
index 24efce9e276..ca777c1a7e0 100644
--- a/lib/bulk_imports/projects/graphql/get_repository_query.rb
+++ b/lib/bulk_imports/projects/graphql/get_repository_query.rb
@@ -3,9 +3,8 @@
 module BulkImports
   module Projects
     module Graphql
-      module GetRepositoryQuery
-        extend Queryable
-        extend self
+      class GetRepositoryQuery
+        include Queryable
 
         def to_s
           <<-'GRAPHQL'
diff --git a/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb b/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb
index 1ba57789612..c105b04c731 100644
--- a/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb
+++ b/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb
@@ -3,9 +3,8 @@
 module BulkImports
   module Projects
     module Graphql
-      module GetSnippetRepositoryQuery
-        extend Queryable
-        extend self
+      class GetSnippetRepositoryQuery
+        include Queryable
 
         def to_s
           <<-'GRAPHQL'
@@ -27,7 +26,7 @@ module BulkImports
           GRAPHQL
         end
 
-        def variables(context)
+        def variables
           {
             full_path: context.entity.source_full_path,
             cursor: context.tracker.next_page,
diff --git a/lib/bulk_imports/projects/graphql/queryable.rb b/lib/bulk_imports/projects/graphql/queryable.rb
index a897632dff3..bd3116cb838 100644
--- a/lib/bulk_imports/projects/graphql/queryable.rb
+++ b/lib/bulk_imports/projects/graphql/queryable.rb
@@ -4,7 +4,13 @@ module BulkImports
   module Projects
     module Graphql
       module Queryable
-        def variables(context)
+        attr_reader :context
+
+        def initialize(context:)
+          @context = context
+        end
+
+        def variables
           { full_path: context.entity.source_full_path }
         end
 
diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb
index 0556395ca66..b920c1bf355 100644
--- a/lib/bulk_imports/projects/stage.rb
+++ b/lib/bulk_imports/projects/stage.rb
@@ -19,6 +19,10 @@ module BulkImports
             pipeline: BulkImports::Projects::Pipelines::ProjectAttributesPipeline,
             stage: 1
           },
+          members: {
+            pipeline: BulkImports::Common::Pipelines::MembersPipeline,
+            stage: 1
+          },
           labels: {
             pipeline: BulkImports::Common::Pipelines::LabelsPipeline,
             stage: 2
@@ -83,6 +87,10 @@ module BulkImports
             pipeline: BulkImports::Common::Pipelines::UploadsPipeline,
             stage: 5
           },
+          lfs_objects: {
+            pipeline: BulkImports::Common::Pipelines::LfsObjectsPipeline,
+            stage: 5
+          },
           auto_devops: {
             pipeline: BulkImports::Projects::Pipelines::AutoDevopsPipeline,
             stage: 5