Add latest changes from gitlab-org/gitlab@master

18 files changed, 144 insertions, 44 deletions

diff --git a/app/assets/javascripts/boards/components/issue_card_inner.vue b/app/assets/javascripts/boards/components/issue_card_inner.vue
index 2acd92069ca..1d53a21c8ac 100644
--- a/app/assets/javascripts/boards/components/issue_card_inner.vue
+++ b/app/assets/javascripts/boards/components/issue_card_inner.vue
@@ -104,13 +104,6 @@ export default {
     helpLink() {
       return boardsStore.scopedLabels.helpLink;
     },
-    validIssueWeight() {
-      if (_.isNumber(this.issue.weight)) {
-        return this.issue.weight >= 0;
-      }
-
-      return false;
-    },
   },
   methods: {
     isIndexLessThanlimit(index) {
diff --git a/app/assets/javascripts/boards/mixins/issue_card_inner.js b/app/assets/javascripts/boards/mixins/issue_card_inner.js
index 8000237da6d..04e971b756d 100644
--- a/app/assets/javascripts/boards/mixins/issue_card_inner.js
+++ b/app/assets/javascripts/boards/mixins/issue_card_inner.js
@@ -1,4 +1,9 @@
 export default {
+  computed: {
+    validIssueWeight() {
+      return false;
+    },
+  },
   methods: {
     filterByWeight() {},
   },
diff --git a/app/assets/javascripts/jobs/components/job_container_item.vue b/app/assets/javascripts/jobs/components/job_container_item.vue
index a55dffbe488..7bd299bcfa0 100644
--- a/app/assets/javascripts/jobs/components/job_container_item.vue
+++ b/app/assets/javascripts/jobs/components/job_container_item.vue
@@ -54,7 +54,7 @@ export default {
       :href="job.status.details_path"
       :title="tooltipText"
       data-boundary="viewport"
-      class="js-job-link"
+      class="js-job-link d-flex"
     >
       <icon
         v-if="isActive"
@@ -64,7 +64,7 @@ export default {
 
       <ci-icon :status="job.status" />
 
-      <span>{{ job.name ? job.name : job.id }}</span>
+      <span class="text-truncate w-100">{{ job.name ? job.name : job.id }}</span>
 
       <icon v-if="job.retried" name="retry" class="js-retry-icon" />
     </gl-link>
diff --git a/app/assets/stylesheets/pages/builds.scss b/app/assets/stylesheets/pages/builds.scss
index 73166940146..89fd160b575 100644
--- a/app/assets/stylesheets/pages/builds.scss
+++ b/app/assets/stylesheets/pages/builds.scss
@@ -308,12 +308,8 @@
     }
 
     a {
-      display: block;
       padding: $gl-padding 10px $gl-padding 40px;
       width: 270px;
-      white-space: nowrap;
-      overflow: hidden;
-      text-overflow: ellipsis;
 
       &:hover {
         color: $gl-text-color;
diff --git a/app/assets/stylesheets/pages/projects.scss b/app/assets/stylesheets/pages/projects.scss
index 7577112cb0e..b2c1d0b6dc5 100644
--- a/app/assets/stylesheets/pages/projects.scss
+++ b/app/assets/stylesheets/pages/projects.scss
@@ -1001,6 +1001,14 @@ pre.light-well {
       }
     }
 
+    &:not(.with-pipeline-status) {
+      .icon-wrapper:first-of-type {
+        @include media-breakpoint-up(lg) {
+          margin-left: $gl-padding-32;
+        }
+      }
+    }
+
     .ci-status-link {
       display: inline-flex;
     }
diff --git a/app/models/ci/group.rb b/app/models/ci/group.rb
index 9b2c3c807ac..0e05318b253 100644
--- a/app/models/ci/group.rb
+++ b/app/models/ci/group.rb
@@ -9,6 +9,7 @@ module Ci
   #
   class Group
     include StaticModel
+    include Gitlab::Utils::StrongMemoize
 
     attr_reader :stage, :name, :jobs
 
@@ -21,7 +22,17 @@ module Ci
     end
 
     def status
-      @status ||= commit_statuses.status
+      strong_memoize(:status) do
+        if Feature.enabled?(:ci_composite_status, default_enabled: false)
+          Gitlab::Ci::Status::Composite
+            .new(@jobs)
+            .status
+        else
+          CommitStatus
+            .where(id: @jobs)
+            .legacy_status
+        end
+      end
     end
 
     def detailed_status(current_user)
@@ -40,11 +51,5 @@ module Ci
           self.new(stage, name: group_name, jobs: grouped_statuses)
         end
     end
-
-    private
-
-    def commit_statuses
-      @commit_statuses ||= CommitStatus.where(id: jobs.map(&:id))
-    end
   end
 end
diff --git a/app/models/ci/legacy_stage.rb b/app/models/ci/legacy_stage.rb
index 930c8a71453..2fd369c9aff 100644
--- a/app/models/ci/legacy_stage.rb
+++ b/app/models/ci/legacy_stage.rb
@@ -14,7 +14,8 @@ module Ci
       @pipeline = pipeline
       @name = name
       @status = status
-      @warnings = warnings
+      # support ints and booleans
+      @has_warnings = ActiveRecord::Type::Boolean.new.cast(warnings)
     end
 
     def groups
@@ -30,7 +31,7 @@ module Ci
     end
 
     def status
-      @status ||= statuses.latest.status
+      @status ||= statuses.latest.slow_composite_status
     end
 
     def detailed_status(current_user)
@@ -52,11 +53,12 @@ module Ci
     end
 
     def has_warnings?
-      if @warnings.is_a?(Integer)
-        @warnings > 0
-      else
-        statuses.latest.failed_but_allowed.any?
+      # lazilly calculate the warnings
+      if @has_warnings.nil?
+        @has_warnings = statuses.latest.failed_but_allowed.any?
       end
+
+      @has_warnings
     end
 
     def manual_playable?
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index 9a96429d3a9..7fa290610aa 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -386,13 +386,12 @@ module Ci
       end
     end
 
-    def legacy_stages
+    def legacy_stages_using_sql
       # TODO, this needs refactoring, see gitlab-foss#26481.
-
       stages_query = statuses
         .group('stage').select(:stage).order('max(stage_idx)')
 
-      status_sql = statuses.latest.where('stage=sg.stage').status_sql
+      status_sql = statuses.latest.where('stage=sg.stage').legacy_status_sql
 
       warnings_sql = statuses.latest.select('COUNT(*)')
         .where('stage=sg.stage').failed_but_allowed.to_sql
@@ -405,6 +404,30 @@ module Ci
       end
     end
 
+    def legacy_stages_using_composite_status
+      stages = statuses.latest
+        .order(:stage_idx, :stage)
+        .group_by(&:stage)
+
+      stages.map do |stage_name, jobs|
+        composite_status = Gitlab::Ci::Status::Composite
+          .new(jobs)
+
+        Ci::LegacyStage.new(self,
+          name: stage_name,
+          status: composite_status.status,
+          warnings: composite_status.warnings?)
+      end
+    end
+
+    def legacy_stages
+      if Feature.enabled?(:ci_composite_status, default_enabled: false)
+        legacy_stages_using_composite_status
+      else
+        legacy_stages_using_sql
+      end
+    end
+
     def valid_commit_sha
       if self.sha == Gitlab::Git::BLANK_SHA
         self.errors.add(:sha, " cant be 00000000 (branch removal)")
@@ -635,7 +658,8 @@ module Ci
 
     def update_status
       retry_optimistic_lock(self) do
-        case latest_builds_status.to_s
+        new_status = latest_builds_status.to_s
+        case new_status
         when 'created' then nil
         when 'preparing' then prepare
         when 'pending' then enqueue
@@ -648,7 +672,7 @@ module Ci
         when 'scheduled' then delay
         else
           raise HasStatus::UnknownStatusError,
-                "Unknown status `#{latest_builds_status}`"
+                "Unknown status `#{new_status}`"
         end
       end
     end
@@ -907,7 +931,7 @@ module Ci
     def latest_builds_status
       return 'failed' unless yaml_errors.blank?
 
-      statuses.latest.status || 'skipped'
+      statuses.latest.slow_composite_status || 'skipped'
     end
 
     def keep_around_commits
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
index d90339d90dc..77ac8bfe875 100644
--- a/app/models/ci/stage.rb
+++ b/app/models/ci/stage.rb
@@ -78,7 +78,8 @@ module Ci
 
     def update_status
       retry_optimistic_lock(self) do
-        case statuses.latest.status
+        new_status = latest_stage_status.to_s
+        case new_status
         when 'created' then nil
         when 'preparing' then prepare
         when 'pending' then enqueue
@@ -91,7 +92,7 @@ module Ci
         when 'skipped', nil then skip
         else
           raise HasStatus::UnknownStatusError,
-                "Unknown status `#{statuses.latest.status}`"
+                "Unknown status `#{new_status}`"
         end
       end
     end
@@ -124,5 +125,9 @@ module Ci
     def manual_playable?
       blocked? || skipped?
     end
+
+    def latest_stage_status
+      statuses.latest.slow_composite_status || 'skipped'
+    end
   end
 end
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 5d9d3179f9d..39a6247b3b2 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -48,6 +48,10 @@ class CommitStatus < ApplicationRecord
   scope :processables, -> { where(type: %w[Ci::Build Ci::Bridge]) }
   scope :for_ids, -> (ids) { where(id: ids) }
 
+  scope :with_preloads, -> do
+    preload(:project, :user)
+  end
+
   scope :with_needs, -> (names = nil) do
     needs = Ci::BuildNeed.scoped_build.select(1)
     needs = needs.where(name: names) if names
@@ -161,11 +165,11 @@ class CommitStatus < ApplicationRecord
   end
 
   def self.status_for_prior_stages(index)
-    before_stage(index).latest.status || 'success'
+    before_stage(index).latest.slow_composite_status || 'success'
   end
 
   def self.status_for_names(names)
-    where(name: names).latest.status || 'success'
+    where(name: names).latest.slow_composite_status || 'success'
   end
 
   def locking_enabled?
diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb
index bcbbb27a9a8..c01fb4740e5 100644
--- a/app/models/concerns/has_status.rb
+++ b/app/models/concerns/has_status.rb
@@ -10,6 +10,8 @@ module HasStatus
   ACTIVE_STATUSES = %w[preparing pending running].freeze
   COMPLETED_STATUSES = %w[success failed canceled skipped].freeze
   ORDERED_STATUSES = %w[failed preparing pending running manual scheduled canceled success skipped created].freeze
+  PASSED_WITH_WARNINGS_STATUSES = %w[failed canceled].to_set.freeze
+  EXCLUDE_IGNORED_STATUSES = %w[manual failed canceled].to_set.freeze
   STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3,
                     failed: 4, canceled: 5, skipped: 6, manual: 7,
                     scheduled: 8, preparing: 9 }.freeze
@@ -17,7 +19,7 @@ module HasStatus
   UnknownStatusError = Class.new(StandardError)
 
   class_methods do
-    def status_sql
+    def legacy_status_sql
       scope_relevant = respond_to?(:exclude_ignored) ? exclude_ignored : all
       scope_warnings = respond_to?(:failed_but_allowed) ? failed_but_allowed : none
 
@@ -53,8 +55,22 @@ module HasStatus
       )
     end
 
-    def status
-      all.pluck(status_sql).first
+    def legacy_status
+      all.pluck(legacy_status_sql).first
+    end
+
+    # This method should not be used.
+    # This method performs expensive calculation of status:
+    # 1. By plucking all related objects,
+    # 2. Or executes expensive SQL query
+    def slow_composite_status
+      if Feature.enabled?(:ci_composite_status, default_enabled: false)
+        Gitlab::Ci::Status::Composite
+          .new(all, with_allow_failure: columns_hash.key?('allow_failure'))
+          .status
+      else
+        legacy_status
+      end
     end
 
     def started_at
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index fb90ddc1048..2fe691bd959 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -319,6 +319,12 @@ class Namespace < ApplicationRecord
   private
 
   def all_projects_with_pages
+    if all_projects.pages_metadata_not_migrated.exists?
+      Gitlab::BackgroundMigration::MigratePagesMetadata.new.perform_on_relation(
+        all_projects.pages_metadata_not_migrated
+      )
+    end
+
     all_projects.with_pages_deployed
   end
 
diff --git a/app/models/pages_domain.rb b/app/models/pages_domain.rb
index 6be3053f637..7903a2182dd 100644
--- a/app/models/pages_domain.rb
+++ b/app/models/pages_domain.rb
@@ -194,6 +194,16 @@ class PagesDomain < ApplicationRecord
   private
 
   def pages_deployed?
+    # TODO: remove once `pages_metadatum` is migrated
+    # https://gitlab.com/gitlab-org/gitlab/issues/33106
+    unless project.pages_metadatum
+      Gitlab::BackgroundMigration::MigratePagesMetadata
+        .new
+        .perform_on_relation(Project.where(id: project_id))
+
+      project.reset
+    end
+
     project.pages_metadatum&.deployed?
   end
 
diff --git a/app/models/project.rb b/app/models/project.rb
index 2a7d652678d..318d1473a70 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -433,6 +433,11 @@ class Project < ApplicationRecord
     joins(:pages_metadatum).merge(ProjectPagesMetadatum.deployed)
   end
 
+  scope :pages_metadata_not_migrated, -> do
+    left_outer_joins(:pages_metadatum)
+      .where(project_pages_metadata: { project_id: nil })
+  end
+
   enum auto_cancel_pending_pipelines: { disabled: 0, enabled: 1 }
 
   chronic_duration_attr :build_timeout_human_readable, :build_timeout,
diff --git a/app/models/project_services/issue_tracker_service.rb b/app/models/project_services/issue_tracker_service.rb
index 3ecd5390d79..278677edcdf 100644
--- a/app/models/project_services/issue_tracker_service.rb
+++ b/app/models/project_services/issue_tracker_service.rb
@@ -62,6 +62,7 @@ class IssueTrackerService < Service
     end
 
     data_values.reject! { |key| data_fields.changed.include?(key) }
+    data_values.slice!(*data_fields.attributes.keys)
     data_fields.assign_attributes(data_values) if data_values.present?
 
     self.properties = {}
@@ -71,6 +72,10 @@ class IssueTrackerService < Service
     @legacy_properties_data ||= {}
   end
 
+  def supports_data_fields?
+    true
+  end
+
   def data_fields
     issue_tracker_data || self.build_issue_tracker_data
   end
diff --git a/app/models/service.rb b/app/models/service.rb
index 43ed0c7dfaa..305cf7b78a2 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -291,6 +291,12 @@ class Service < ApplicationRecord
 
   def self.build_from_template(project_id, template)
     service = template.dup
+
+    if template.supports_data_fields?
+      data_fields = template.data_fields.dup
+      data_fields.service = service
+    end
+
     service.template = false
     service.project_id = project_id
     service.active = false if service.active? && !service.valid?
@@ -309,6 +315,11 @@ class Service < ApplicationRecord
     find_by(template: true)
   end
 
+  # override if needed
+  def supports_data_fields?
+    false
+  end
+
   private
 
   def cache_project_has_external_issue_tracker
diff --git a/app/services/ci/process_pipeline_service.rb b/app/services/ci/process_pipeline_service.rb
index 3b145a65d79..039670f58c8 100644
--- a/app/services/ci/process_pipeline_service.rb
+++ b/app/services/ci/process_pipeline_service.rb
@@ -2,6 +2,8 @@
 
 module Ci
   class ProcessPipelineService < BaseService
+    include Gitlab::Utils::StrongMemoize
+
     attr_reader :pipeline
 
     def execute(pipeline, trigger_build_ids = nil)
@@ -33,9 +35,9 @@ module Ci
 
       return unless HasStatus::COMPLETED_STATUSES.include?(current_status)
 
-      created_processables_in_stage_without_needs(index).select do |build|
+      created_processables_in_stage_without_needs(index).find_each.select do |build|
         process_build(build, current_status)
-      end
+      end.any?
     end
 
     def process_builds_with_needs(trigger_build_ids)
@@ -92,6 +94,7 @@ module Ci
 
     def created_processables_in_stage_without_needs(index)
       created_processables_without_needs
+        .with_preloads
         .for_stage(index)
     end
 
diff --git a/app/views/shared/projects/_project.html.haml b/app/views/shared/projects/_project.html.haml
index 5432607f82f..67dad9b7a75 100644
--- a/app/views/shared/projects/_project.html.haml
+++ b/app/views/shared/projects/_project.html.haml
@@ -12,7 +12,9 @@
 - css_class += " no-description" if project.description.blank? && !show_last_commit_as_description
 - cache_key = project_list_cache_key(project, pipeline_status: pipeline_status)
 - updated_tooltip = time_ago_with_tooltip(project.last_activity_date)
-- css_controls_class = compact_mode ? "" : "flex-lg-row justify-content-lg-between"
+- show_pipeline_status_icon = pipeline_status && can?(current_user, :read_cross_project) && project.pipeline_status.has_status? && can?(current_user, :read_build, project)
+- css_controls_class = compact_mode ? [] : ["flex-lg-row", "justify-content-lg-between"]
+- css_controls_class << "with-pipeline-status" if show_pipeline_status_icon
 - avatar_container_class = project.creator && use_creator_avatar ? '' : 'rect-avatar'
 
 %li.project-row.d-flex{ class: css_class }
@@ -58,9 +60,9 @@
           .description.d-none.d-sm-block.append-right-default
             = markdown_field(project, :description)
 
-      .controls.d-flex.flex-sm-column.align-items-center.align-items-sm-end.flex-wrap.flex-shrink-0.text-secondary{ class: css_controls_class }
+      .controls.d-flex.flex-sm-column.align-items-center.align-items-sm-end.flex-wrap.flex-shrink-0.text-secondary{ class: css_controls_class.join(" ") }
         .icon-container.d-flex.align-items-center
-          - if pipeline_status && can?(current_user, :read_cross_project) && project.pipeline_status.has_status? && can?(current_user, :read_build, project)
+          - if show_pipeline_status_icon
             - pipeline_path = pipelines_project_commit_path(project.pipeline_status.project, project.pipeline_status.sha, ref: project.pipeline_status.ref)
             %span.icon-wrapper.pipeline-status
               = render 'ci/status/icon', status: project.last_pipeline.detailed_status(current_user), tooltip_placement: 'top', path: pipeline_path