Add latest changes from gitlab-org/gitlab@13-2-stable-ee

13 files changed, 111 insertions, 130 deletions

diff --git a/app/services/projects/after_import_service.rb b/app/services/projects/after_import_service.rb
index fad2290a47b..b37ae56ba0f 100644
--- a/app/services/projects/after_import_service.rb
+++ b/app/services/projects/after_import_service.rb
@@ -26,7 +26,7 @@ module Projects
         message: 'Project housekeeping failed',
         project_full_path: @project.full_path,
         project_id: @project.id,
-        error: e.message
+        'error.message' => e.message
       )
     end
 
diff --git a/app/services/projects/alerting/notify_service.rb b/app/services/projects/alerting/notify_service.rb
index 86c408aeec8..e08bc8efb15 100644
--- a/app/services/projects/alerting/notify_service.rb
+++ b/app/services/projects/alerting/notify_service.rb
@@ -4,7 +4,7 @@ module Projects
   module Alerting
     class NotifyService < BaseService
       include Gitlab::Utils::StrongMemoize
-      include IncidentManagement::Settings
+      include ::IncidentManagement::Settings
 
       def execute(token)
         return forbidden unless alerts_service_activated?
@@ -55,7 +55,7 @@ module Projects
       def find_alert_by_fingerprint(fingerprint)
         return unless fingerprint
 
-        AlertManagement::Alert.for_fingerprint(project, fingerprint).first
+        AlertManagement::Alert.not_resolved.for_fingerprint(project, fingerprint).first
       end
 
       def send_email?
@@ -65,8 +65,7 @@ module Projects
       def process_incident_issues(alert)
         return if alert.issue
 
-        IncidentManagement::ProcessAlertWorker
-          .perform_async(project.id, parsed_payload, alert.id)
+        ::IncidentManagement::ProcessAlertWorker.perform_async(nil, nil, alert.id)
       end
 
       def send_alert_email
@@ -76,7 +75,7 @@ module Projects
       end
 
       def parsed_payload
-        Gitlab::Alerting::NotificationPayloadParser.call(params.to_h)
+        Gitlab::Alerting::NotificationPayloadParser.call(params.to_h, project)
       end
 
       def valid_token?(token)
diff --git a/app/services/projects/batch_forks_count_service.rb b/app/services/projects/batch_forks_count_service.rb
index 6467744a435..d12772b40ff 100644
--- a/app/services/projects/batch_forks_count_service.rb
+++ b/app/services/projects/batch_forks_count_service.rb
@@ -5,6 +5,21 @@
 # because the service use maps to retrieve the project ids
 module Projects
   class BatchForksCountService < Projects::BatchCountService
+    def refresh_cache_and_retrieve_data
+      count_services = @projects.map { |project| count_service.new(project) }
+
+      values = Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
+        Rails.cache.fetch_multi(*(count_services.map { |ser| ser.cache_key } )) { |key| nil }
+      end
+
+      results_per_service = Hash[count_services.zip(values.values)]
+      projects_to_refresh = results_per_service.select { |_k, value| value.nil? }
+      projects_to_refresh = recreate_cache(projects_to_refresh)
+
+      results_per_service.update(projects_to_refresh)
+      results_per_service.transform_keys { |k| k.project }
+    end
+
     # rubocop: disable CodeReuse/ActiveRecord
     def global_count
       @global_count ||= begin
@@ -18,5 +33,13 @@ module Projects
     def count_service
       ::Projects::ForksCountService
     end
+
+    def recreate_cache(projects_to_refresh)
+      projects_to_refresh.each_with_object({}) do |(service, _v), hash|
+        count = global_count[service.project.id].to_i
+        service.refresh_cache { count }
+        hash[service] = count
+      end
+    end
   end
 end
diff --git a/app/services/projects/container_repository/delete_tags_service.rb b/app/services/projects/container_repository/delete_tags_service.rb
index 21081bd077f..5d4059710bb 100644
--- a/app/services/projects/container_repository/delete_tags_service.rb
+++ b/app/services/projects/container_repository/delete_tags_service.rb
@@ -3,6 +3,8 @@
 module Projects
   module ContainerRepository
     class DeleteTagsService < BaseService
+      LOG_DATA_BASE = { service_class: self.to_s }.freeze
+
       def execute(container_repository)
         return error('access denied') unless can?(current_user, :destroy_container_image, project)
 
@@ -51,10 +53,27 @@ module Projects
 
       def smart_delete(container_repository, tag_names)
         fast_delete_enabled = Feature.enabled?(:container_registry_fast_tag_delete, default_enabled: true)
-        if fast_delete_enabled && container_repository.client.supports_tag_delete?
-          fast_delete(container_repository, tag_names)
+        response = if fast_delete_enabled && container_repository.client.supports_tag_delete?
+                     fast_delete(container_repository, tag_names)
+                   else
+                     slow_delete(container_repository, tag_names)
+                   end
+
+        response.tap { |r| log_response(r, container_repository) }
+      end
+
+      def log_response(response, container_repository)
+        log_data = LOG_DATA_BASE.merge(
+          container_repository_id: container_repository.id,
+          message: 'deleted tags'
+        )
+
+        if response[:status] == :success
+          log_data[:deleted_tags_count] = response[:deleted].size
+          log_info(log_data)
         else
-          slow_delete(container_repository, tag_names)
+          log_data[:message] = response[:message]
+          log_error(log_data)
         end
       end
 
diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb
index bffd443c49f..6569277ad9d 100644
--- a/app/services/projects/create_service.rb
+++ b/app/services/projects/create_service.rb
@@ -84,8 +84,12 @@ module Projects
     def after_create_actions
       log_info("#{@project.owner.name} created a new project \"#{@project.full_name}\"")
 
+      # Skip writing the config for project imports/forks because it
+      # will always fail since the Git directory doesn't exist until
+      # a background job creates it (see Project#add_import_job).
+      @project.write_repository_config unless @project.import?
+
       unless @project.gitlab_project_import?
-        @project.write_repository_config
         @project.create_wiki unless skip_wiki?
       end
 
@@ -103,12 +107,13 @@ module Projects
       create_readme if @initialize_with_readme
     end
 
-    # Refresh the current user's authorizations inline (so they can access the
-    # project immediately after this request completes), and any other affected
-    # users in the background
+    # Add an authorization for the current user authorizations inline
+    # (so they can access the project immediately after this request
+    # completes), and any other affected users in the background
     def setup_authorizations
       if @project.group
-        current_user.refresh_authorized_projects
+        current_user.project_authorizations.create!(project: @project,
+                                                    access_level: @project.group.max_member_access_for_user(current_user))
 
         if Feature.enabled?(:specialized_project_authorization_workers)
           AuthorizedProjectUpdate::ProjectCreateWorker.perform_async(@project.id)
@@ -131,7 +136,7 @@ module Projects
 
     def create_readme
       commit_attrs = {
-        branch_name: 'master',
+        branch_name:  Gitlab::CurrentSettings.default_branch_name.presence || 'master',
         commit_message: 'Initial commit',
         file_path: 'README.md',
         file_content: "# #{@project.name}\n\n#{@project.description}"
diff --git a/app/services/projects/forks_count_service.rb b/app/services/projects/forks_count_service.rb
index ca85e2dc281..848d8d54104 100644
--- a/app/services/projects/forks_count_service.rb
+++ b/app/services/projects/forks_count_service.rb
@@ -3,6 +3,8 @@
 module Projects
   # Service class for getting and caching the number of forks of a project.
   class ForksCountService < Projects::CountService
+    attr_reader :project
+
     def cache_key_name
       'forks_count'
     end
diff --git a/app/services/projects/group_links/create_service.rb b/app/services/projects/group_links/create_service.rb
index 2ba3cd6694f..3c3cab26fb5 100644
--- a/app/services/projects/group_links/create_service.rb
+++ b/app/services/projects/group_links/create_service.rb
@@ -13,12 +13,32 @@ module Projects
         )
 
         if link.save
-          group.refresh_members_authorized_projects
+          setup_authorizations(group)
           success(link: link)
         else
           error(link.errors.full_messages.to_sentence, 409)
         end
       end
+
+      private
+
+      def setup_authorizations(group)
+        if Feature.enabled?(:specialized_project_authorization_project_share_worker)
+          AuthorizedProjectUpdate::ProjectGroupLinkCreateWorker.perform_async(project.id, group.id)
+
+          # AuthorizedProjectsWorker uses an exclusive lease per user but
+          # specialized workers might have synchronization issues. Until we
+          # compare the inconsistency rates of both approaches, we still run
+          # AuthorizedProjectsWorker but with some delay and lower urgency as a
+          # safety net.
+          group.refresh_members_authorized_projects(
+            blocking: false,
+            priority: UserProjectAccessChangedService::LOW_PRIORITY
+          )
+        else
+          group.refresh_members_authorized_projects(blocking: false)
+        end
+      end
     end
   end
 end
diff --git a/app/services/projects/operations/update_service.rb b/app/services/projects/operations/update_service.rb
index 7aa7ea73639..7af489c3751 100644
--- a/app/services/projects/operations/update_service.rb
+++ b/app/services/projects/operations/update_service.rb
@@ -108,7 +108,18 @@ module Projects
       end
 
       def incident_management_setting_params
-        params.slice(:incident_management_setting_attributes)
+        attrs = params[:incident_management_setting_attributes]
+        return {} unless attrs
+
+        regenerate_token = attrs.delete(:regenerate_token)
+
+        if regenerate_token
+          attrs[:pagerduty_token] = nil
+        else
+          attrs = attrs.except(:pagerduty_token)
+        end
+
+        { incident_management_setting_attributes: attrs }
       end
     end
   end
diff --git a/app/services/projects/prometheus/alerts/create_events_service.rb b/app/services/projects/prometheus/alerts/create_events_service.rb
deleted file mode 100644
index 4fcf841314b..00000000000
--- a/app/services/projects/prometheus/alerts/create_events_service.rb
+++ /dev/null
@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-module Projects
-  module Prometheus
-    module Alerts
-      # Persists a series of Prometheus alert events as list of PrometheusAlertEvent.
-      class CreateEventsService < BaseService
-        def execute
-          create_events_from(alerts)
-        end
-
-        private
-
-        def create_events_from(alerts)
-          Array.wrap(alerts).map { |alert| create_event(alert) }.compact
-        end
-
-        def create_event(payload)
-          parsed_alert = Gitlab::Alerting::Alert.new(project: project, payload: payload)
-
-          return unless parsed_alert.valid?
-
-          if parsed_alert.gitlab_managed?
-            create_managed_prometheus_alert_event(parsed_alert)
-          else
-            create_self_managed_prometheus_alert_event(parsed_alert)
-          end
-        end
-
-        def alerts
-          params['alerts']
-        end
-
-        def find_alert(metric)
-          Projects::Prometheus::AlertsFinder
-            .new(project: project, metric: metric)
-            .execute
-            .first
-        end
-
-        def create_managed_prometheus_alert_event(parsed_alert)
-          alert = find_alert(parsed_alert.metric_id)
-          event = PrometheusAlertEvent.find_or_initialize_by_payload_key(parsed_alert.project, alert, parsed_alert.gitlab_fingerprint)
-
-          set_status(parsed_alert, event)
-        end
-
-        def create_self_managed_prometheus_alert_event(parsed_alert)
-          event = SelfManagedPrometheusAlertEvent.find_or_initialize_by_payload_key(parsed_alert.project, parsed_alert.gitlab_fingerprint) do |event|
-            event.environment      = parsed_alert.environment
-            event.title            = parsed_alert.title
-            event.query_expression = parsed_alert.full_query
-          end
-
-          set_status(parsed_alert, event)
-        end
-
-        def set_status(parsed_alert, event)
-          persisted = case parsed_alert.status
-                      when 'firing'
-                        event.fire(parsed_alert.starts_at)
-                      when 'resolved'
-                        event.resolve(parsed_alert.ends_at)
-                      end
-
-          event if persisted
-        end
-      end
-    end
-  end
-end
diff --git a/app/services/projects/prometheus/alerts/notify_service.rb b/app/services/projects/prometheus/alerts/notify_service.rb
index 877a4f99a94..ea557ebe20f 100644
--- a/app/services/projects/prometheus/alerts/notify_service.rb
+++ b/app/services/projects/prometheus/alerts/notify_service.rb
@@ -5,7 +5,7 @@ module Projects
     module Alerts
       class NotifyService < BaseService
         include Gitlab::Utils::StrongMemoize
-        include IncidentManagement::Settings
+        include ::IncidentManagement::Settings
 
         # This set of keys identifies a payload as a valid Prometheus
         # payload and thus processable by this service. See also
@@ -23,9 +23,7 @@ module Projects
           return unauthorized unless valid_alert_manager_token?(token)
 
           process_prometheus_alerts
-          persist_events
           send_alert_email if send_email?
-          process_incident_issues if process_issues?
 
           ServiceResponse.success
         end
@@ -132,13 +130,6 @@ module Projects
             .prometheus_alerts_fired(project, firings)
         end
 
-        def process_incident_issues
-          alerts.each do |alert|
-            IncidentManagement::ProcessPrometheusAlertWorker
-              .perform_async(project.id, alert.to_h)
-          end
-        end
-
         def process_prometheus_alerts
           alerts.each do |alert|
             AlertManagement::ProcessPrometheusAlertService
@@ -147,10 +138,6 @@ module Projects
           end
         end
 
-        def persist_events
-          CreateEventsService.new(project, nil, params).execute
-        end
-
         def bad_request
           ServiceResponse.error(message: 'Bad Request', http_status: :bad_request)
         end
diff --git a/app/services/projects/propagate_service_template.rb b/app/services/projects/propagate_service_template.rb
index 4adcda042d1..b6465810fde 100644
--- a/app/services/projects/propagate_service_template.rb
+++ b/app/services/projects/propagate_service_template.rb
@@ -26,7 +26,7 @@ module Projects
 
     def propagate_projects_with_template
       loop do
-        batch = Project.uncached { project_ids_without_integration }
+        batch = Project.uncached { Project.ids_without_integration(template, BATCH_SIZE) }
 
         bulk_create_from_template(batch) unless batch.empty?
 
@@ -50,22 +50,6 @@ module Projects
       end
     end
 
-    # rubocop: disable CodeReuse/ActiveRecord
-    def project_ids_without_integration
-      services = Service
-        .select('1')
-        .where('services.project_id = projects.id')
-        .where(type: template.type)
-
-      Project
-        .where('NOT EXISTS (?)', services)
-        .where(pending_delete: false)
-        .where(archived: false)
-        .limit(BATCH_SIZE)
-        .pluck(:id)
-    end
-    # rubocop: enable CodeReuse/ActiveRecord
-
     def bulk_insert(klass, columns, values_array)
       items_to_insert = values_array.map { |array| Hash[columns.zip(array)] }
 
diff --git a/app/services/projects/update_remote_mirror_service.rb b/app/services/projects/update_remote_mirror_service.rb
index 5f8ef75a8d7..d6c0d647468 100644
--- a/app/services/projects/update_remote_mirror_service.rb
+++ b/app/services/projects/update_remote_mirror_service.rb
@@ -29,7 +29,7 @@ module Projects
       remote_mirror.ensure_remote!
 
       # https://gitlab.com/gitlab-org/gitaly/-/issues/2670
-      if Feature.disabled?(:gitaly_ruby_remote_branches_ls_remote)
+      if Feature.disabled?(:gitaly_ruby_remote_branches_ls_remote, default_enabled: true)
         repository.fetch_remote(remote_mirror.remote_name, ssh_auth: remote_mirror, no_tags: true)
       end
 
diff --git a/app/services/projects/update_repository_storage_service.rb b/app/services/projects/update_repository_storage_service.rb
index fa8d4c5aa5f..7b346c09635 100644
--- a/app/services/projects/update_repository_storage_service.rb
+++ b/app/services/projects/update_repository_storage_service.rb
@@ -14,7 +14,11 @@ module Projects
     end
 
     def execute
-      repository_storage_move.start!
+      repository_storage_move.with_lock do
+        return ServiceResponse.success unless repository_storage_move.scheduled? # rubocop:disable Cop/AvoidReturnFromBlocks
+
+        repository_storage_move.start!
+      end
 
       raise SameFilesystemError if same_filesystem?(repository.storage, destination_storage_name)
 
@@ -79,8 +83,6 @@ module Projects
         full_path
       )
 
-      new_repository.create_repository
-
       new_repository.replicate(raw_repository)
       new_checksum = new_repository.checksum
 
@@ -93,25 +95,25 @@ module Projects
       old_repository_storage = project.repository_storage
       new_project_path = moved_path(project.disk_path)
 
-      # Notice that the block passed to `run_after_commit` will run with `project`
+      # Notice that the block passed to `run_after_commit` will run with `repository_storage_move`
       # as its context
-      project.run_after_commit do
+      repository_storage_move.run_after_commit do
         GitlabShellWorker.perform_async(:mv_repository,
                                         old_repository_storage,
-                                        disk_path,
+                                        project.disk_path,
                                         new_project_path)
 
-        if wiki.repository_exists?
+        if project.wiki.repository_exists?
           GitlabShellWorker.perform_async(:mv_repository,
                                           old_repository_storage,
-                                          wiki.disk_path,
+                                          project.wiki.disk_path,
                                           "#{new_project_path}.wiki")
         end
 
-        if design_repository.exists?
+        if project.design_repository.exists?
           GitlabShellWorker.perform_async(:mv_repository,
                                           old_repository_storage,
-                                          design_repository.disk_path,
+                                          project.design_repository.disk_path,
                                           "#{new_project_path}.design")
         end
       end