Add latest changes from gitlab-org/gitlab@master

14 files changed, 86 insertions, 46 deletions

diff --git a/app/assets/javascripts/notifications_form.js b/app/assets/javascripts/notifications_form.js
index dcd226795a6..fa27994f598 100644
--- a/app/assets/javascripts/notifications_form.js
+++ b/app/assets/javascripts/notifications_form.js
@@ -26,7 +26,7 @@ export default class NotificationsForm {
       .addClass('is-loading')
       .find('.custom-notification-event-loading')
       .removeClass('fa-check')
-      .addClass('fa-spin fa-spinner')
+      .addClass('spinner align-middle')
       .removeClass('is-done');
   }
 
@@ -41,12 +41,12 @@ export default class NotificationsForm {
         if (data.saved) {
           $parent
             .find('.custom-notification-event-loading')
-            .toggleClass('fa-spin fa-spinner fa-check is-done');
+            .toggleClass('spinner fa-check is-done align-middle');
           setTimeout(() => {
             $parent
               .removeClass('is-loading')
               .find('.custom-notification-event-loading')
-              .toggleClass('fa-spin fa-spinner fa-check is-done');
+              .toggleClass('spinner fa-check is-done align-middle');
           }, 2000);
         }
       })
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index b0502f7a26e..27a394e4ab8 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -818,7 +818,7 @@ module Ci
       depended_jobs = depends_on_builds
 
       # find all jobs that are needed
-      if Feature.enabled?(:ci_dag_support, project, default_enabled: true) && needs.exists?
+      if Feature.enabled?(:ci_dag_support, project, default_enabled: true) && scheduling_type_dag?
         depended_jobs = depended_jobs.where(name: needs.artifacts.select(:name))
       end
 
diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb
index 95fb75688a9..6c080582cae 100644
--- a/app/models/ci/processable.rb
+++ b/app/models/ci/processable.rb
@@ -12,6 +12,18 @@ module Ci
 
     scope :preload_needs, -> { preload(:needs) }
 
+    scope :with_needs, -> (names = nil) do
+      needs = Ci::BuildNeed.scoped_build.select(1)
+      needs = needs.where(name: names) if names
+      where('EXISTS (?)', needs).preload(:needs)
+    end
+
+    scope :without_needs, -> (names = nil) do
+      needs = Ci::BuildNeed.scoped_build.select(1)
+      needs = needs.where(name: names) if names
+      where('NOT EXISTS (?)', needs)
+    end
+
     def self.select_with_aggregated_needs(project)
       return all unless Feature.enabled?(:ci_dag_support, project, default_enabled: true)
 
@@ -26,6 +38,18 @@ module Ci
       )
     end
 
+    # Old processables may have scheduling_type as nil,
+    # so we need to ensure the data exists before using it.
+    def self.populate_scheduling_type!
+      needs = Ci::BuildNeed.scoped_build.select(1)
+      where(scheduling_type: nil).update_all(
+        "scheduling_type = CASE WHEN (EXISTS (#{needs.to_sql}))
+         THEN #{scheduling_types[:dag]}
+         ELSE #{scheduling_types[:stage]}
+         END"
+      )
+    end
+
     validates :type, presence: true
     validates :scheduling_type, presence: true, on: :create, if: :validate_scheduling_type?
 
@@ -53,6 +77,11 @@ module Ci
       raise NotImplementedError
     end
 
+    # Overriding scheduling_type enum's method for nil `scheduling_type`s
+    def scheduling_type_dag?
+      super || find_legacy_scheduling_type == :dag
+    end
+
     # scheduling_type column of previous builds/bridges have not been populated,
     # so we calculate this value on runtime when we need it.
     def find_legacy_scheduling_type
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 5a917588a33..35b727720ba 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -62,18 +62,6 @@ class CommitStatus < ApplicationRecord
     preload(project: :namespace)
   end
 
-  scope :with_needs, -> (names = nil) do
-    needs = Ci::BuildNeed.scoped_build.select(1)
-    needs = needs.where(name: names) if names
-    where('EXISTS (?)', needs).preload(:needs)
-  end
-
-  scope :without_needs, -> (names = nil) do
-    needs = Ci::BuildNeed.scoped_build.select(1)
-    needs = needs.where(name: names) if names
-    where('NOT EXISTS (?)', needs)
-  end
-
   scope :match_id_and_lock_version, -> (slice) do
     # it expects that items are an array of attributes to match
     # each hash needs to have `id` and `lock_version`
diff --git a/app/models/project_services/youtrack_service.rb b/app/models/project_services/youtrack_service.rb
index 0416eaa5be0..02d06eeb405 100644
--- a/app/models/project_services/youtrack_service.rb
+++ b/app/models/project_services/youtrack_service.rb
@@ -6,9 +6,9 @@ class YoutrackService < IssueTrackerService
   # {PROJECT-KEY}-{NUMBER} Examples: YT-1, PRJ-1, gl-030
   def self.reference_pattern(only_long: false)
     if only_long
-      /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+)/
+      /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+\b)/
     else
-      /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+)|(#{Issue.reference_prefix}(?<issue>\d+))/
+      /(?<issue>\b[A-Za-z][A-Za-z0-9_]*-\d+\b)|(#{Issue.reference_prefix}(?<issue>\d+))/
     end
   end
 
diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb
index 2daf3a51235..52977034b70 100644
--- a/app/services/ci/create_pipeline_service.rb
+++ b/app/services/ci/create_pipeline_service.rb
@@ -61,7 +61,7 @@ module Ci
 
           Ci::ProcessPipelineService
             .new(pipeline)
-            .execute
+            .execute(nil, initial_process: true)
         end
       end
 
diff --git a/app/services/ci/pipeline_processing/atomic_processing_service.rb b/app/services/ci/pipeline_processing/atomic_processing_service.rb
index 1ed295f5950..55846c3cb5c 100644
--- a/app/services/ci/pipeline_processing/atomic_processing_service.rb
+++ b/app/services/ci/pipeline_processing/atomic_processing_service.rb
@@ -93,9 +93,9 @@ module Ci
       end
 
       def processable_status(processable)
-        if needs_names = processable.aggregated_needs_names
+        if Feature.enabled?(:ci_dag_support, project, default_enabled: true) && processable.scheduling_type_dag?
           # Processable uses DAG, get status of all dependent needs
-          @collection.status_for_names(needs_names)
+          @collection.status_for_names(processable.aggregated_needs_names.to_a)
         else
           # Processable uses Stages, get status of prior stage
           @collection.status_for_prior_stage_position(processable.stage_idx.to_i)
diff --git a/app/services/ci/pipeline_processing/legacy_processing_service.rb b/app/services/ci/pipeline_processing/legacy_processing_service.rb
index 400dc9f0abb..278fba20283 100644
--- a/app/services/ci/pipeline_processing/legacy_processing_service.rb
+++ b/app/services/ci/pipeline_processing/legacy_processing_service.rb
@@ -11,12 +11,13 @@ module Ci
         @pipeline = pipeline
       end
 
-      def execute(trigger_build_ids = nil)
-        success = process_stages_without_needs
+      def execute(trigger_build_ids = nil, initial_process: false)
+        success = process_stages_for_stage_scheduling
 
         # we evaluate dependent needs,
         # only when the another job has finished
-        success = process_builds_with_needs(trigger_build_ids) || success
+        success = process_dag_builds_without_needs || success if initial_process
+        success = process_dag_builds_with_needs(trigger_build_ids) || success
 
         @pipeline.update_legacy_status
 
@@ -25,23 +26,31 @@ module Ci
 
       private
 
-      def process_stages_without_needs
-        stage_indexes_of_created_processables_without_needs.flat_map do |index|
-          process_stage_without_needs(index)
+      def process_stages_for_stage_scheduling
+        stage_indexes_of_created_stage_scheduled_processables.flat_map do |index|
+          process_stage_for_stage_scheduling(index)
         end.any?
       end
 
-      def process_stage_without_needs(index)
+      def process_stage_for_stage_scheduling(index)
         current_status = status_for_prior_stages(index)
 
         return unless HasStatus::COMPLETED_STATUSES.include?(current_status)
 
-        created_processables_in_stage_without_needs(index).find_each.select do |build|
+        created_stage_scheduled_processables_in_stage(index).find_each.select do |build|
           process_build(build, current_status)
         end.any?
       end
 
-      def process_builds_with_needs(trigger_build_ids)
+      def process_dag_builds_without_needs
+        return false unless Feature.enabled?(:ci_dag_support, project, default_enabled: true)
+
+        created_processables.scheduling_type_dag.without_needs.each do |build|
+          process_build(build, 'success')
+        end
+      end
+
+      def process_dag_builds_with_needs(trigger_build_ids)
         return false unless trigger_build_ids.present?
         return false unless Feature.enabled?(:ci_dag_support, project, default_enabled: true)
 
@@ -56,14 +65,15 @@ module Ci
 
         # Each found processable is guaranteed here to have completed status
         created_processables
+          .scheduling_type_dag
           .with_needs(trigger_build_names)
           .without_needs(incomplete_build_names)
           .find_each
-          .map(&method(:process_build_with_needs))
+          .map(&method(:process_dag_build_with_needs))
           .any?
       end
 
-      def process_build_with_needs(build)
+      def process_dag_build_with_needs(build)
         current_status = status_for_build_needs(build.needs.map(&:name))
 
         return unless HasStatus::COMPLETED_STATUSES.include?(current_status)
@@ -87,23 +97,23 @@ module Ci
       end
 
       # rubocop: disable CodeReuse/ActiveRecord
-      def stage_indexes_of_created_processables_without_needs
-        created_processables_without_needs.order(:stage_idx)
+      def stage_indexes_of_created_stage_scheduled_processables
+        created_stage_scheduled_processables.order(:stage_idx)
           .pluck(Arel.sql('DISTINCT stage_idx'))
       end
       # rubocop: enable CodeReuse/ActiveRecord
 
-      def created_processables_in_stage_without_needs(index)
-        created_processables_without_needs
+      def created_stage_scheduled_processables_in_stage(index)
+        created_stage_scheduled_processables
           .with_preloads
           .for_stage(index)
       end
 
-      def created_processables_without_needs
+      def created_stage_scheduled_processables
         if Feature.enabled?(:ci_dag_support, project, default_enabled: true)
-          pipeline.processables.created.without_needs
+          created_processables.scheduling_type_stage
         else
-          pipeline.processables.created
+          created_processables
         end
       end
 
diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb
index 1ecef256233..d1efa19eb0d 100644
--- a/app/services/ci/process_pipeline_service.rb
+++ b/app/services/ci/process_pipeline_service.rb
@@ -8,8 +8,9 @@ module Ci
       @pipeline = pipeline
     end
 
-    def execute(trigger_build_ids = nil)
+    def execute(trigger_build_ids = nil, initial_process: false)
       update_retried
+      ensure_scheduling_type_for_processables
 
       if Feature.enabled?(:ci_atomic_processing, pipeline.project)
         Ci::PipelineProcessing::AtomicProcessingService
@@ -18,7 +19,7 @@ module Ci
       else
         Ci::PipelineProcessing::LegacyProcessingService
           .new(pipeline)
-          .execute(trigger_build_ids)
+          .execute(trigger_build_ids, initial_process: initial_process)
       end
     end
 
@@ -43,5 +44,17 @@ module Ci
         .update_all(retried: true) if latest_statuses.any?
     end
     # rubocop: enable CodeReuse/ActiveRecord
+
+    # Set scheduling type of processables if they were created before scheduling_type
+    # data was deployed (https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22246).
+    # Given that this service runs multiple times during the pipeline
+    # life cycle we need to ensure we populate the data once.
+    # See more: https://gitlab.com/gitlab-org/gitlab/issues/205426
+    def ensure_scheduling_type_for_processables
+      lease = Gitlab::ExclusiveLease.new("set-scheduling-types:#{pipeline.id}", timeout: 1.hour.to_i)
+      return unless lease.try_obtain
+
+      pipeline.processables.populate_scheduling_type!
+    end
   end
 end
diff --git a/app/services/ci/retry_pipeline_service.rb b/app/services/ci/retry_pipeline_service.rb
index 7d01de9ee68..9bb236ac44c 100644
--- a/app/services/ci/retry_pipeline_service.rb
+++ b/app/services/ci/retry_pipeline_service.rb
@@ -36,7 +36,7 @@ module Ci
 
       Ci::ProcessPipelineService
         .new(pipeline)
-        .execute(completed_build_ids)
+        .execute(completed_build_ids, initial_process: true)
     end
   end
 end
diff --git a/app/views/notify/_failed_builds.html.haml b/app/views/notify/_failed_builds.html.haml
index 7c563bb016c..1711c34a842 100644
--- a/app/views/notify/_failed_builds.html.haml
+++ b/app/views/notify/_failed_builds.html.haml
@@ -27,6 +27,6 @@
             - if build.has_trace?
               %td{ colspan: "2", style: "font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; padding: 0 0 16px;" }
                 %pre{ style: "font-family: Monaco,'Lucida Console','Courier New',Courier,monospace; background-color: #fafafa; border-radius: 4px; overflow: hidden; white-space: pre-wrap; word-break: break-all; font-size:13px; line-height: 1.4; padding: 16px 8px; color: #333333; margin: 0;" }
-                  = build.trace.html(last_lines: 10).html_safe
+                  = build.trace.html(last_lines: 30).html_safe
             - else
               %td{ colspan: "2" }
diff --git a/app/views/notify/autodevops_disabled_email.text.erb b/app/views/notify/autodevops_disabled_email.text.erb
index bf863952478..91092060e74 100644
--- a/app/views/notify/autodevops_disabled_email.text.erb
+++ b/app/views/notify/autodevops_disabled_email.text.erb
@@ -15,6 +15,6 @@ had <%= failed.size %> failed <%= 'build'.pluralize(failed.size) %>.
   Stage: <%= build.stage %>
   Name: <%= build.name %>
   <% if build.has_trace? -%>
-    Trace: <%= build.trace.raw(last_lines: 10) %>
+    Trace: <%= build.trace.raw(last_lines: 30) %>
   <% end -%>
 <% end -%>
diff --git a/app/views/notify/pipeline_failed_email.text.erb b/app/views/notify/pipeline_failed_email.text.erb
index 9cd479ef1e6..41b26842dbc 100644
--- a/app/views/notify/pipeline_failed_email.text.erb
+++ b/app/views/notify/pipeline_failed_email.text.erb
@@ -35,7 +35,7 @@ had <%= failed.size %> failed <%= 'build'.pluralize(failed.size) %>.
 Stage: <%= build.stage %>
 Name: <%= build.name %>
 <% if build.has_trace? -%>
-Trace: <%= build.trace.raw(last_lines: 10) %>
+Trace: <%= build.trace.raw(last_lines: 30) %>
 <% end -%>
 
 <% end -%>
diff --git a/app/views/shared/notifications/_custom_notifications.html.haml b/app/views/shared/notifications/_custom_notifications.html.haml
index be574155436..0e1e3beeb1c 100644
--- a/app/views/shared/notifications/_custom_notifications.html.haml
+++ b/app/views/shared/notifications/_custom_notifications.html.haml
@@ -30,4 +30,4 @@
                       %label.form-check-label{ for: field_id }
                         %strong
                           = notification_event_name(event)
-                          = icon("spinner spin", class: "custom-notification-event-loading")
+                          .fa.custom-notification-event-loading.spinner