Add latest changes from gitlab-org/gitlab@15-7-stable-eev15.7.0-rc42

17 files changed, 213 insertions, 80 deletions

diff --git a/app/services/projects/batch_forks_count_service.rb b/app/services/projects/batch_forks_count_service.rb
index 6467744a435..78663d8dad5 100644
--- a/app/services/projects/batch_forks_count_service.rb
+++ b/app/services/projects/batch_forks_count_service.rb
@@ -7,11 +7,9 @@ module Projects
   class BatchForksCountService < Projects::BatchCountService
     # rubocop: disable CodeReuse/ActiveRecord
     def global_count
-      @global_count ||= begin
-        count_service.query(project_ids)
+      @global_count ||= count_service.query(project_ids)
                      .group(:forked_from_project_id)
                      .count
-      end
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
diff --git a/app/services/projects/batch_open_issues_count_service.rb b/app/services/projects/batch_open_issues_count_service.rb
index d6ff2291af8..c396d7c0cfc 100644
--- a/app/services/projects/batch_open_issues_count_service.rb
+++ b/app/services/projects/batch_open_issues_count_service.rb
@@ -7,9 +7,7 @@ module Projects
   class BatchOpenIssuesCountService < Projects::BatchCountService
     # rubocop: disable CodeReuse/ActiveRecord
     def global_count
-      @global_count ||= begin
-        count_service.query(project_ids).group(:project_id).count
-      end
+      @global_count ||= count_service.query(project_ids).group(:project_id).count
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
diff --git a/app/services/projects/container_repository/cleanup_tags_base_service.rb b/app/services/projects/container_repository/cleanup_tags_base_service.rb
index 5393c2c080d..45557d03502 100644
--- a/app/services/projects/container_repository/cleanup_tags_base_service.rb
+++ b/app/services/projects/container_repository/cleanup_tags_base_service.rb
@@ -6,6 +6,8 @@ module Projects
       private
 
       def filter_out_latest!(tags)
+        return unless keep_latest
+
         tags.reject!(&:latest?)
       end
 
@@ -84,6 +86,10 @@ module Projects
         params['keep_n']
       end
 
+      def keep_latest
+        params.fetch('keep_latest', true)
+      end
+
       def project
         container_repository.project
       end
diff --git a/app/services/projects/container_repository/destroy_service.rb b/app/services/projects/container_repository/destroy_service.rb
index 83bb8624bba..6db6b449671 100644
--- a/app/services/projects/container_repository/destroy_service.rb
+++ b/app/services/projects/container_repository/destroy_service.rb
@@ -3,12 +3,46 @@
 module Projects
   module ContainerRepository
     class DestroyService < BaseService
-      def execute(container_repository)
+      CLEANUP_TAGS_SERVICE_PARAMS = {
+        'name_regex_delete' => '.*',
+        'container_expiration_policy' => true, # to avoid permissions checks
+        'keep_latest' => false
+      }.freeze
+
+      def execute(container_repository, disable_timeout: true)
         return false unless can?(current_user, :update_container_image, project)
 
         # Delete tags outside of the transaction to avoid hitting an idle-in-transaction timeout
-        container_repository.delete_tags!
-        container_repository.delete_failed! unless container_repository.destroy
+        unless delete_tags(container_repository, disable_timeout) &&
+            destroy_container_repository(container_repository)
+          container_repository.delete_failed!
+        end
+      end
+
+      private
+
+      def delete_tags(container_repository, disable_timeout)
+        service = Projects::ContainerRepository::CleanupTagsService.new(
+          container_repository: container_repository,
+          params: CLEANUP_TAGS_SERVICE_PARAMS.merge('disable_timeout' => disable_timeout)
+        )
+        result = service.execute
+        return true if result[:status] == :success
+
+        log_error(error_message(container_repository, 'error in deleting tags'))
+        false
+      end
+
+      def destroy_container_repository(container_repository)
+        return true if container_repository.destroy
+
+        log_error(error_message(container_repository, container_repository.errors.full_messages.join('. ')))
+        false
+      end
+
+      def error_message(container_repository, message)
+        "Container repository with ID: #{container_repository.id} and path: #{container_repository.path}" \
+        " failed with message: #{message}"
       end
     end
   end
diff --git a/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb b/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb
index e947e9575e2..b69a3cc1a2c 100644
--- a/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb
+++ b/app/services/projects/container_repository/gitlab/cleanup_tags_service.rb
@@ -18,7 +18,7 @@ module Projects
             container_repository.each_tags_page(page_size: TAGS_PAGE_SIZE) do |tags|
               execute_for_tags(tags, result)
 
-              raise TimeoutError if timeout?(start_time)
+              raise TimeoutError if !timeout_disabled? && timeout?(start_time)
             end
           end
         end
@@ -72,6 +72,10 @@ module Projects
         def pushed_at(tag)
           tag.updated_at || tag.created_at
         end
+
+        def timeout_disabled?
+          params['disable_timeout'] || false
+        end
       end
     end
   end
diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb
index c72f9b4b602..a4b473f35c6 100644
--- a/app/services/projects/create_service.rb
+++ b/app/services/projects/create_service.rb
@@ -317,6 +317,3 @@ module Projects
 end
 
 Projects::CreateService.prepend_mod_with('Projects::CreateService')
-
-# Measurable should be at the bottom of the ancestor chain, so it will measure execution of EE::Projects::CreateService as well
-Projects::CreateService.prepend(Measurable)
diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb
index ddbcfbb675c..a1f55f547a1 100644
--- a/app/services/projects/import_export/export_service.rb
+++ b/app/services/projects/import_export/export_service.rb
@@ -3,8 +3,6 @@
 module Projects
   module ImportExport
     class ExportService < BaseService
-      prepend Measurable
-
       def initialize(*args)
         super
 
diff --git a/app/services/projects/import_export/parallel_export_service.rb b/app/services/projects/import_export/parallel_export_service.rb
new file mode 100644
index 00000000000..7e4c0279b06
--- /dev/null
+++ b/app/services/projects/import_export/parallel_export_service.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Projects
+  module ImportExport
+    class ParallelExportService
+      def initialize(export_job, current_user, after_export_strategy)
+        @export_job = export_job
+        @current_user = current_user
+        @after_export_strategy = after_export_strategy
+        @shared = project.import_export_shared
+        @logger = Gitlab::Export::Logger.build
+      end
+
+      def execute
+        log_info('Parallel project export started')
+
+        if save_exporters && save_export_archive
+          log_info('Parallel project export finished successfully')
+          execute_after_export_action(after_export_strategy)
+        else
+          notify_error
+        end
+
+      ensure
+        cleanup
+      end
+
+      private
+
+      attr_reader :export_job, :current_user, :after_export_strategy, :shared, :logger
+
+      delegate :project, to: :export_job
+
+      def execute_after_export_action(after_export_strategy)
+        return if after_export_strategy.execute(current_user, project)
+
+        notify_error
+      end
+
+      def exporters
+        [version_saver, exported_relations_merger]
+      end
+
+      def save_exporters
+        exporters.all? do |exporter|
+          log_info("Parallel project export - #{exporter.class.name} saver started")
+
+          exporter.save
+        end
+      end
+
+      def save_export_archive
+        Gitlab::ImportExport::Saver.save(exportable: project, shared: shared)
+      end
+
+      def version_saver
+        @version_saver ||= Gitlab::ImportExport::VersionSaver.new(shared: shared)
+      end
+
+      def exported_relations_merger
+        @relation_saver ||= Gitlab::ImportExport::Project::ExportedRelationsMerger.new(
+          export_job: export_job,
+          shared: shared)
+      end
+
+      def cleanup
+        FileUtils.rm_rf(shared.export_path) if File.exist?(shared.export_path)
+        FileUtils.rm_rf(shared.archive_path) if File.exist?(shared.archive_path)
+      end
+
+      def log_info(message)
+        logger.info(
+          message: message,
+          **log_base_data
+        )
+      end
+
+      def notify_error
+        logger.error(
+          message: 'Parallel project export error',
+          export_errors: shared.errors.join(', '),
+          export_job_id: export_job.id,
+          **log_base_data
+        )
+
+        NotificationService.new.project_not_exported(project, current_user, shared.errors)
+      end
+
+      def log_base_data
+        {
+          project_id: project.id,
+          project_name: project.name,
+          project_path: project.full_path
+        }
+      end
+    end
+  end
+end
diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb
index 6a13b8e38c1..967a1e990b2 100644
--- a/app/services/projects/import_service.rb
+++ b/app/services/projects/import_service.rb
@@ -179,6 +179,3 @@ module Projects
 end
 
 Projects::ImportService.prepend_mod_with('Projects::ImportService')
-
-# Measurable should be at the bottom of the ancestor chain, so it will measure execution of EE::Projects::ImportService as well
-Projects::ImportService.prepend(Measurable)
diff --git a/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb b/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb
index c91103f897f..f7de7f98768 100644
--- a/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb
+++ b/app/services/projects/lfs_pointers/lfs_download_link_list_service.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# This service lists the download link from a remote source based on the
+# This service yields operation on each download link from a remote source based on the
 # oids provided
 module Projects
   module LfsPointers
@@ -23,29 +23,22 @@ module Projects
         @remote_uri = remote_uri
       end
 
-      # This method accepts two parameters:
       # - oids: hash of oids to query. The structure is { lfs_file_oid => lfs_file_size }
-      #
-      # Returns an array of LfsDownloadObject
-      def execute(oids)
-        return [] unless project&.lfs_enabled? && remote_uri && oids.present?
+      # Yields operation for each link batch-by-batch
+      def each_link(oids, &block)
+        return unless project&.lfs_enabled? && remote_uri && oids.present?
 
-        get_download_links_in_batches(oids)
+        download_links_in_batches(oids, &block)
       end
 
       private
 
-      def get_download_links_in_batches(oids, batch_size = REQUEST_BATCH_SIZE)
-        download_links = []
-
+      def download_links_in_batches(oids, batch_size = REQUEST_BATCH_SIZE, &block)
         oids.each_slice(batch_size) do |batch|
-          download_links += get_download_links(batch)
+          download_links_for(batch).each(&block)
         end
-
-        download_links
-
       rescue DownloadLinksRequestEntityTooLargeError => e
-        # Log this exceptions to see how open it happens
+        # Log this exceptions to see how often it happens
         Gitlab::ErrorTracking
           .track_exception(e, project_id: project&.id, batch_size: batch_size, oids_count: oids.count)
 
@@ -57,7 +50,7 @@ module Projects
         raise DownloadLinksError, 'Unable to download due to RequestEntityTooLarge errors'
       end
 
-      def get_download_links(oids)
+      def download_links_for(oids)
         response = Gitlab::HTTP.post(remote_uri,
                                      body: request_body(oids),
                                      headers: headers)
diff --git a/app/services/projects/lfs_pointers/lfs_download_service.rb b/app/services/projects/lfs_pointers/lfs_download_service.rb
index eaf73b78c1c..26352198e5c 100644
--- a/app/services/projects/lfs_pointers/lfs_download_service.rb
+++ b/app/services/projects/lfs_pointers/lfs_download_service.rb
@@ -92,9 +92,15 @@ module Projects
       end
 
       def fetch_file(&block)
+        attempts ||= 1
         response = Gitlab::HTTP.get(lfs_sanitized_url, download_options, &block)
 
         raise ResponseError, "Received error code #{response.code}" unless response.success?
+      rescue Net::OpenTimeout
+        raise if attempts >= 3
+
+        attempts += 1
+        retry
       end
 
       def with_tmp_file
diff --git a/app/services/projects/lfs_pointers/lfs_import_service.rb b/app/services/projects/lfs_pointers/lfs_import_service.rb
index 3fc82f2c410..c9791041088 100644
--- a/app/services/projects/lfs_pointers/lfs_import_service.rb
+++ b/app/services/projects/lfs_pointers/lfs_import_service.rb
@@ -9,9 +9,7 @@ module Projects
       def execute
         return success unless project&.lfs_enabled?
 
-        lfs_objects_to_download = LfsObjectDownloadListService.new(project).execute
-
-        lfs_objects_to_download.each do |lfs_download_object|
+        LfsObjectDownloadListService.new(project).each_list_item do |lfs_download_object|
           LfsDownloadService.new(project, lfs_download_object).execute
         end
 
diff --git a/app/services/projects/lfs_pointers/lfs_object_download_list_service.rb b/app/services/projects/lfs_pointers/lfs_object_download_list_service.rb
index b4872cd9442..09fec9939b9 100644
--- a/app/services/projects/lfs_pointers/lfs_object_download_list_service.rb
+++ b/app/services/projects/lfs_pointers/lfs_object_download_list_service.rb
@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-# This service manages the whole worflow of discovering the Lfs files in a
-# repository, linking them to the project and downloading (and linking) the non
-# existent ones.
+# This service discovers the Lfs files that are linked in repository,
+# but not downloaded yet and yields the operation
+# on each Lfs file link (url) to remote repository.
 module Projects
   module LfsPointers
     class LfsObjectDownloadListService < BaseService
@@ -14,30 +14,31 @@ module Projects
 
       LfsObjectDownloadListError = Class.new(StandardError)
 
-      def execute
-        return [] unless project&.lfs_enabled?
-
-        if external_lfs_endpoint?
-          # If the endpoint host is different from the import_url it means
-          # that the repo is using a third party service for storing the LFS files.
-          # In this case, we have to disable lfs in the project
-          disable_lfs!
-
-          return []
-        end
+      def each_list_item(&block)
+        return unless context_valid?
 
         # Downloading the required information and gathering it inside an
         #   LfsDownloadObject for each oid
-        #
         LfsDownloadLinkListService
           .new(project, remote_uri: current_endpoint_uri)
-          .execute(missing_lfs_files)
+          .each_link(missing_lfs_files, &block)
       rescue LfsDownloadLinkListService::DownloadLinksError => e
         raise LfsObjectDownloadListError, "The LFS objects download list couldn't be imported. Error: #{e.message}"
       end
 
       private
 
+      def context_valid?
+        return false unless project&.lfs_enabled?
+        return true unless external_lfs_endpoint?
+
+        # If the endpoint host is different from the import_url it means
+        # that the repo is using a third party service for storing the LFS files.
+        # In this case, we have to disable lfs in the project
+        disable_lfs!
+        false
+      end
+
       def external_lfs_endpoint?
         lfsconfig_endpoint_uri && lfsconfig_endpoint_uri.host != import_uri.host
       end
@@ -99,12 +100,10 @@ module Projects
 
       # The import url must end with '.git' here we ensure it is
       def default_endpoint_uri
-        @default_endpoint_uri ||= begin
-          import_uri.dup.tap do |uri|
-            path = uri.path.gsub(%r(/$), '')
-            path += '.git' unless path.ends_with?('.git')
-            uri.path = path + LFS_BATCH_API_ENDPOINT
-          end
+        @default_endpoint_uri ||= import_uri.dup.tap do |uri|
+          path = uri.path.gsub(%r(/$), '')
+          path += '.git' unless path.ends_with?('.git')
+          uri.path = path + LFS_BATCH_API_ENDPOINT
         end
       end
     end
diff --git a/app/services/projects/refresh_build_artifacts_size_statistics_service.rb b/app/services/projects/refresh_build_artifacts_size_statistics_service.rb
index 1f86e5f4ba9..8e006dc8c34 100644
--- a/app/services/projects/refresh_build_artifacts_size_statistics_service.rb
+++ b/app/services/projects/refresh_build_artifacts_size_statistics_service.rb
@@ -18,7 +18,7 @@ module Projects
           # Mark the refresh ready for another worker to pick up and process the next batch
           refresh.requeue!(batch.last.id)
 
-          refresh.project.statistics.delayed_increment_counter(:build_artifacts_size, total_artifacts_size)
+          refresh.project.statistics.increment_counter(:build_artifacts_size, total_artifacts_size)
         end
       else
         # Remove the refresh job from the table if there are no more
diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb
index 6a963e7fcd1..0fadd75669e 100644
--- a/app/services/projects/update_pages_service.rb
+++ b/app/services/projects/update_pages_service.rb
@@ -63,16 +63,19 @@ module Projects
     end
 
     def build_commit_status
+      stage = create_stage
+
       GenericCommitStatus.new(
         user: build.user,
         ci_stage: stage,
         name: 'pages:deploy',
-        stage: 'deploy'
+        stage: 'deploy',
+        stage_idx: stage.position
       )
     end
 
     # rubocop: disable Performance/ActiveRecordSubtransactionMethods
-    def stage
+    def create_stage
       build.pipeline.stages.safe_find_or_create_by(name: 'deploy', pipeline_id: build.pipeline.id) do |stage|
         stage.position = GenericCommitStatus::EXTERNAL_STAGE_IDX
         stage.project = build.project
diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb
index f686f14b5b3..aca6fa91eb1 100644
--- a/app/services/projects/update_remote_mirror_service.rb
+++ b/app/services/projects/update_remote_mirror_service.rb
@@ -10,7 +10,7 @@ module Projects
       return success unless remote_mirror.enabled?
 
       # Blocked URLs are a hard failure, no need to attempt to retry
-      if Gitlab::UrlBlocker.blocked_url?(normalized_url(remote_mirror.url))
+      if Gitlab::UrlBlocker.blocked_url?(normalized_url(remote_mirror.url), schemes: Project::VALID_MIRROR_PROTOCOLS)
         hard_retry_or_fail(remote_mirror, _('The remote mirror URL is invalid.'), tries)
         return error(remote_mirror.last_error)
       end
diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb
index f9a2c825608..301d11d841c 100644
--- a/app/services/projects/update_service.rb
+++ b/app/services/projects/update_service.rb
@@ -10,7 +10,6 @@ module Projects
     def execute
       build_topics
       remove_unallowed_params
-      mirror_operations_access_level_changes
       validate!
 
       ensure_wiki_exists if enabling_wiki?
@@ -65,16 +64,36 @@ module Projects
       return unless changing_default_branch?
 
       previous_default_branch = project.default_branch
+      new_default_branch = params[:default_branch]
 
-      if project.change_head(params[:default_branch])
+      if project.change_head(new_default_branch)
         params[:previous_default_branch] = previous_default_branch
 
+        if !project.root_ref?(new_default_branch) && has_custom_head_branch?
+          raise ValidationError,
+            format(
+              s_("UpdateProject|Could not set the default branch. Do you have a branch named 'HEAD' in your repository? (%{linkStart}How do I fix this?%{linkEnd})"),
+              linkStart: ambiguous_head_documentation_link, linkEnd: '</a>'
+            ).html_safe
+        end
+
         after_default_branch_change(previous_default_branch)
       else
         raise ValidationError, s_("UpdateProject|Could not set the default branch")
       end
     end
 
+    def ambiguous_head_documentation_link
+      url = Rails.application.routes.url_helpers.help_page_path('user/project/repository/branches/index.md', anchor: 'error-ambiguous-head-branch-exists')
+
+      format('<a href="%{url}" target="_blank" rel="noopener noreferrer">', url: url)
+    end
+
+    # See issue: https://gitlab.com/gitlab-org/gitlab/-/issues/381731
+    def has_custom_head_branch?
+      project.repository.branch_names.any? { |name| name.casecmp('head') == 0 }
+    end
+
     def after_default_branch_change(previous_default_branch)
       # overridden by EE module
     end
@@ -83,21 +102,6 @@ module Projects
       params.delete(:emails_disabled) unless can?(current_user, :set_emails_disabled, project)
     end
 
-    # Temporary code to sync permissions changes as operations access setting
-    # is being split into monitor_access_level, deployments_access_level, infrastructure_access_level.
-    # To be removed as part of https://gitlab.com/gitlab-org/gitlab/-/issues/364240
-    def mirror_operations_access_level_changes
-      return if Feature.enabled?(:split_operations_visibility_permissions, project)
-
-      operations_access_level = params.dig(:project_feature_attributes, :operations_access_level)
-
-      return if operations_access_level.nil?
-
-      [:monitor_access_level, :infrastructure_access_level, :feature_flags_access_level, :environments_access_level].each do |key|
-        params[:project_feature_attributes][key] = operations_access_level
-      end
-    end
-
     def after_update
       todos_features_changes = %w(
         issues_access_level