Add latest changes from gitlab-org/gitlab@13-6-stable-eev13.6.0-rc42

9 files changed, 87 insertions, 65 deletions

diff --git a/app/services/projects/alerting/notify_service.rb b/app/services/projects/alerting/notify_service.rb
index affac45fc3d..ab8f53a3757 100644
--- a/app/services/projects/alerting/notify_service.rb
+++ b/app/services/projects/alerting/notify_service.rb
@@ -6,9 +6,11 @@ module Projects
       include Gitlab::Utils::StrongMemoize
       include ::IncidentManagement::Settings
 
-      def execute(token)
+      def execute(token, integration = nil)
+        @integration = integration
+
         return bad_request unless valid_payload_size?
-        return forbidden unless alerts_service_activated?
+        return forbidden unless active_integration?
         return unauthorized unless valid_token?(token)
 
         process_alert
@@ -22,7 +24,7 @@ module Projects
 
       private
 
-      delegate :alerts_service, :alerts_service_activated?, to: :project
+      attr_reader :integration
 
       def process_alert
         if alert.persisted?
@@ -66,14 +68,11 @@ module Projects
         return unless alert.save
 
         alert.execute_services
-        SystemNoteService.create_new_alert(
-          alert,
-          alert.monitoring_tool || 'Generic Alert Endpoint'
-        )
+        SystemNoteService.create_new_alert(alert, notification_source)
       end
 
       def process_incident_issues
-        return if alert.issue
+        return if alert.issue || alert.resolved?
 
         ::IncidentManagement::ProcessAlertWorker.perform_async(nil, nil, alert.id)
       end
@@ -81,7 +80,7 @@ module Projects
       def send_alert_email
         notification_service
           .async
-          .prometheus_alerts_fired(project, [alert.attributes])
+          .prometheus_alerts_fired(project, [alert])
       end
 
       def alert
@@ -106,12 +105,20 @@ module Projects
         end
       end
 
+      def notification_source
+        alert.monitoring_tool || integration&.name || 'Generic Alert Endpoint'
+      end
+
       def valid_payload_size?
         Gitlab::Utils::DeepSize.new(params).valid?
       end
 
+      def active_integration?
+        integration&.active?
+      end
+
       def valid_token?(token)
-        token == alerts_service.token
+        token == integration.token
       end
 
       def bad_request
diff --git a/app/services/projects/cleanup_service.rb b/app/services/projects/cleanup_service.rb
index 4ced9feff00..6e3b320afbe 100644
--- a/app/services/projects/cleanup_service.rb
+++ b/app/services/projects/cleanup_service.rb
@@ -11,6 +11,24 @@ module Projects
 
     include Gitlab::Utils::StrongMemoize
 
+    class << self
+      def enqueue(project, current_user, bfg_object_map)
+        Projects::UpdateService.new(project, current_user, bfg_object_map: bfg_object_map).execute.tap do |result|
+          next unless result[:status] == :success
+
+          project.set_repository_read_only!
+          RepositoryCleanupWorker.perform_async(project.id, current_user.id)
+        end
+      rescue Project::RepositoryReadOnlyError => err
+        { status: :error, message: (_('Failed to make repository read-only. %{reason}') % { reason: err.message }) }
+      end
+
+      def cleanup_after(project)
+        project.bfg_object_map.remove!
+        project.set_repository_writable!
+      end
+    end
+
     # Attempt to clean up the project following the push. Warning: this is
     # destructive!
     #
@@ -22,14 +40,14 @@ module Projects
       apply_bfg_object_map!
 
       # Remove older objects that are no longer referenced
-      GitGarbageCollectWorker.new.perform(project.id, :gc, "project_cleanup:gc:#{project.id}")
+      GitGarbageCollectWorker.new.perform(project.id, :prune, "project_cleanup:gc:#{project.id}")
 
       # The cache may now be inaccurate, and holding onto it could prevent
       # bugs assuming the presence of some object from manifesting for some
       # time. Better to feel the pain immediately.
       project.repository.expire_all_method_caches
 
-      project.bfg_object_map.remove!
+      self.class.cleanup_after(project)
     end
 
     private
diff --git a/app/services/projects/container_repository/delete_tags_service.rb b/app/services/projects/container_repository/delete_tags_service.rb
index 9fc3ec0aafb..505ddaf50e3 100644
--- a/app/services/projects/container_repository/delete_tags_service.rb
+++ b/app/services/projects/container_repository/delete_tags_service.rb
@@ -36,11 +36,11 @@ module Projects
       def log_response(response)
         log_data = LOG_DATA_BASE.merge(
           container_repository_id: @container_repository.id,
-          message: 'deleted tags'
-        )
+          message: 'deleted tags',
+          deleted_tags_count: response[:deleted]&.size
+        ).compact
 
         if response[:status] == :success
-          log_data[:deleted_tags_count] = response[:deleted].size
           log_info(log_data)
         else
           log_data[:message] = response[:message]
diff --git a/app/services/projects/container_repository/gitlab/delete_tags_service.rb b/app/services/projects/container_repository/gitlab/delete_tags_service.rb
index cee94b994a3..e4e22dd9543 100644
--- a/app/services/projects/container_repository/gitlab/delete_tags_service.rb
+++ b/app/services/projects/container_repository/gitlab/delete_tags_service.rb
@@ -14,6 +14,7 @@ module Projects
         def initialize(container_repository, tag_names)
           @container_repository = container_repository
           @tag_names = tag_names
+          @deleted_tags = []
         end
 
         # Delete tags by name with a single DELETE request. This is only supported
@@ -25,7 +26,7 @@ module Projects
           delete_tags
         rescue TimeoutError => e
           ::Gitlab::ErrorTracking.track_exception(e, tags_count: @tag_names&.size, container_repository_id: @container_repository&.id)
-          error('timeout while deleting tags')
+          error('timeout while deleting tags', nil, pass_back: { deleted: @deleted_tags })
         end
 
         private
@@ -33,13 +34,15 @@ module Projects
         def delete_tags
           start_time = Time.zone.now
 
-          deleted_tags = @tag_names.select do |name|
+          @tag_names.each do |name|
             raise TimeoutError if timeout?(start_time)
 
-            @container_repository.delete_tag_by_name(name)
+            if @container_repository.delete_tag_by_name(name)
+              @deleted_tags.append(name)
+            end
           end
 
-          deleted_tags.any? ? success(deleted: deleted_tags) : error('could not delete tags')
+          @deleted_tags.any? ? success(deleted: @deleted_tags) : error('could not delete tags')
         end
 
         def timeout?(start_time)
diff --git a/app/services/projects/hashed_storage/base_repository_service.rb b/app/services/projects/hashed_storage/base_repository_service.rb
index 065bf8725be..349d4d367be 100644
--- a/app/services/projects/hashed_storage/base_repository_service.rb
+++ b/app/services/projects/hashed_storage/base_repository_service.rb
@@ -79,13 +79,12 @@ module Projects
       end
 
       def try_to_set_repository_read_only!
-        # Mitigate any push operation to start during migration
-        unless project.set_repository_read_only!
-          migration_error = "Target repository '#{old_disk_path}' cannot be made read-only as there is a git transfer in progress"
-          logger.error migration_error
+        project.set_repository_read_only!
+      rescue Project::RepositoryReadOnlyError => err
+        migration_error = "Target repository '#{old_disk_path}' cannot be made read-only: #{err.message}"
+        logger.error migration_error
 
-          raise RepositoryInUseError, migration_error
-        end
+        raise RepositoryInUseError, migration_error
       end
 
       def wiki_path_suffix
diff --git a/app/services/projects/lfs_pointers/lfs_download_service.rb b/app/services/projects/lfs_pointers/lfs_download_service.rb
index d6e5b825e13..525f8a25d04 100644
--- a/app/services/projects/lfs_pointers/lfs_download_service.rb
+++ b/app/services/projects/lfs_pointers/lfs_download_service.rb
@@ -22,7 +22,7 @@ module Projects
       def execute
         return unless project&.lfs_enabled? && lfs_download_object
         return error("LFS file with oid #{lfs_oid} has invalid attributes") unless lfs_download_object.valid?
-        return link_existing_lfs_object! if Feature.enabled?(:lfs_link_existing_object, project, default_enabled: true) && lfs_size > LARGE_FILE_SIZE && lfs_object
+        return link_existing_lfs_object! if lfs_size > LARGE_FILE_SIZE && lfs_object
 
         wrap_download_errors do
           download_lfs_file!
diff --git a/app/services/projects/prometheus/alerts/notify_service.rb b/app/services/projects/prometheus/alerts/notify_service.rb
index c002aca32db..8ad4f59373d 100644
--- a/app/services/projects/prometheus/alerts/notify_service.rb
+++ b/app/services/projects/prometheus/alerts/notify_service.rb
@@ -17,13 +17,12 @@ module Projects
 
         SUPPORTED_VERSION = '4'
 
-        def execute(token)
+        def execute(token, _integration = nil)
           return bad_request unless valid_payload_size?
           return unprocessable_entity unless self.class.processable?(params)
           return unauthorized unless valid_alert_manager_token?(token)
 
           process_prometheus_alerts
-          send_alert_email if send_email?
 
           ServiceResponse.success
         end
@@ -120,14 +119,6 @@ module Projects
           ActiveSupport::SecurityUtils.secure_compare(expected, actual)
         end
 
-        def send_alert_email
-          return unless firings.any?
-
-          notification_service
-            .async
-            .prometheus_alerts_fired(project, alerts_attributes)
-        end
-
         def process_prometheus_alerts
           alerts.each do |alert|
             AlertManagement::ProcessPrometheusAlertService
@@ -136,18 +127,6 @@ module Projects
           end
         end
 
-        def alerts_attributes
-          firings.map do |payload|
-            alert_params = Gitlab::AlertManagement::Payload.parse(
-              project,
-              payload,
-              monitoring_tool: Gitlab::AlertManagement::Payload::MONITORING_TOOLS[:prometheus]
-            ).alert_params
-
-            AlertManagement::Alert.new(alert_params).attributes
-          end
-        end
-
         def bad_request
           ServiceResponse.error(message: 'Bad Request', http_status: :bad_request)
         end
diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb
index 64b9eca9014..b9c579a130f 100644
--- a/app/services/projects/update_pages_service.rb
+++ b/app/services/projects/update_pages_service.rb
@@ -12,6 +12,11 @@ module Projects
     # as it shares the namespace with groups
     TMP_EXTRACT_PATH = '@pages.tmp'
 
+    # old deployment can be cached by pages daemon
+    # so we need to give pages daemon some time update cache
+    # 10 minutes is enough, but 30 feels safer
+    OLD_DEPLOYMENTS_DESTRUCTION_DELAY = 30.minutes.freeze
+
     attr_reader :build
 
     def initialize(project, build)
@@ -97,7 +102,7 @@ module Projects
       build.artifacts_file.use_file do |artifacts_path|
         SafeZip::Extract.new(artifacts_path)
           .extract(directories: [PUBLIC_DIR], to: temp_path)
-        create_pages_deployment(artifacts_path)
+        create_pages_deployment(artifacts_path, build)
       end
     rescue SafeZip::Extract::Error => e
       raise FailedToExtractError, e.message
@@ -119,19 +124,28 @@ module Projects
       FileUtils.rm_r(previous_public_path, force: true)
     end
 
-    def create_pages_deployment(artifacts_path)
-      return unless Feature.enabled?(:zip_pages_deployments, project)
+    def create_pages_deployment(artifacts_path, build)
+      return unless Feature.enabled?(:zip_pages_deployments, project, default_enabled: true)
+
+      # we're using the full archive and pages daemon needs to read it
+      # so we want the total count from entries, not only "public/" directory
+      # because it better approximates work we need to do before we can serve the site
+      entries_count = build.artifacts_metadata_entry("", recursive: true).entries.count
+      sha256 = build.job_artifacts_archive.file_sha256
 
+      deployment = nil
       File.open(artifacts_path) do |file|
-        deployment = project.pages_deployments.create!(file: file)
-        project.pages_metadatum.update!(pages_deployment: deployment)
+        deployment = project.pages_deployments.create!(file: file,
+                                                       file_count: entries_count,
+                                                       file_sha256: sha256)
+        project.update_pages_deployment!(deployment)
       end
 
-      # TODO: schedule old deployment removal https://gitlab.com/gitlab-org/gitlab/-/issues/235730
-    rescue => e
-      # we don't want to break current pages deployment process if something goes wrong
-      # TODO: remove this rescue as part of https://gitlab.com/gitlab-org/gitlab/-/issues/245308
-      Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
+      DestroyPagesDeploymentsWorker.perform_in(
+        OLD_DEPLOYMENTS_DESTRUCTION_DELAY,
+        project.id,
+        deployment.id
+      )
     end
 
     def latest?
diff --git a/app/services/projects/update_repository_storage_service.rb b/app/services/projects/update_repository_storage_service.rb
index a479d53a43a..e0d2398bc66 100644
--- a/app/services/projects/update_repository_storage_service.rb
+++ b/app/services/projects/update_repository_storage_service.rb
@@ -54,7 +54,7 @@ module Projects
     end
 
     def mirror_repositories
-      mirror_repository
+      mirror_repository if project.repository_exists?
 
       if project.wiki.repository_exists?
         mirror_repository(type: Gitlab::GlRepository::WIKI)
@@ -92,12 +92,14 @@ module Projects
     end
 
     def remove_old_paths
-      Gitlab::Git::Repository.new(
-        source_storage_name,
-        "#{project.disk_path}.git",
-        nil,
-        nil
-      ).remove
+      if project.repository_exists?
+        Gitlab::Git::Repository.new(
+          source_storage_name,
+          "#{project.disk_path}.git",
+          nil,
+          nil
+        ).remove
+      end
 
       if project.wiki.repository_exists?
         Gitlab::Git::Repository.new(