Merge branch '57115-just-in-time-k8s-resource-creation' into 'master'

Create project-specific Kubernetes resources just-in-time

See merge request gitlab-org/gitlab-ce!25586

17 files changed, 123 insertions, 21 deletions

diff --git a/app/assets/stylesheets/framework/icons.scss b/app/assets/stylesheets/framework/icons.scss
index 49b9b7014ae..3ab61cc5c47 100644
--- a/app/assets/stylesheets/framework/icons.scss
+++ b/app/assets/stylesheets/framework/icons.scss
@@ -31,6 +31,7 @@
   }
 }
 
+.ci-status-icon-preparing,
 .ci-status-icon-running {
   svg {
     fill: $blue-400;
diff --git a/app/assets/stylesheets/pages/merge_requests.scss b/app/assets/stylesheets/pages/merge_requests.scss
index df3a4be6559..7f8b8ea8100 100644
--- a/app/assets/stylesheets/pages/merge_requests.scss
+++ b/app/assets/stylesheets/pages/merge_requests.scss
@@ -166,6 +166,7 @@
       float: left;
 
       .accept-merge-request {
+        &.ci-preparing,
         &.ci-pending,
         &.ci-running {
           @include btn-blue;
diff --git a/app/assets/stylesheets/pages/pipelines.scss b/app/assets/stylesheets/pages/pipelines.scss
index ef9c87d0452..bb08440fda8 100644
--- a/app/assets/stylesheets/pages/pipelines.scss
+++ b/app/assets/stylesheets/pages/pipelines.scss
@@ -795,6 +795,7 @@
     @include mini-pipeline-graph-color($white, $orange-100, $orange-200, $orange-500, $orange-600, $orange-700);
   }
 
+  &.ci-status-icon-preparing,
   &.ci-status-icon-running {
     @include mini-pipeline-graph-color($white, $blue-100, $blue-200, $blue-500, $blue-600, $blue-700);
   }
diff --git a/app/assets/stylesheets/pages/status.scss b/app/assets/stylesheets/pages/status.scss
index f4d568d02ac..a59bb31bdcb 100644
--- a/app/assets/stylesheets/pages/status.scss
+++ b/app/assets/stylesheets/pages/status.scss
@@ -44,6 +44,7 @@
   }
 
   &.ci-info,
+  &.ci-preparing,
   &.ci-running {
     @include status-color($blue-100, $blue-500, $blue-600);
   }
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index a629db82c19..59f47effff7 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -172,6 +172,10 @@ module Ci
     end
 
     state_machine :status do
+      event :enqueue do
+        transition [:created, :skipped, :manual, :scheduled] => :preparing, if: :any_unmet_prerequisites?
+      end
+
       event :actionize do
         transition created: :manual
       end
@@ -185,8 +189,12 @@ module Ci
       end
 
       event :enqueue_scheduled do
+        transition scheduled: :preparing, if: ->(build) do
+          build.scheduled_at&.past? && build.any_unmet_prerequisites?
+        end
+
         transition scheduled: :pending, if: ->(build) do
-          build.scheduled_at && build.scheduled_at < Time.now
+          build.scheduled_at&.past? && !build.any_unmet_prerequisites?
         end
       end
 
@@ -204,6 +212,12 @@ module Ci
         end
       end
 
+      after_transition any => [:preparing] do |build|
+        build.run_after_commit do
+          Ci::BuildPrepareWorker.perform_async(id)
+        end
+      end
+
       after_transition any => [:pending] do |build|
         build.run_after_commit do
           BuildQueueWorker.perform_async(id)
@@ -355,6 +369,16 @@ module Ci
       !retried?
     end
 
+    def any_unmet_prerequisites?
+      return false unless Feature.enabled?(:ci_preparing_state, default_enabled: true)
+
+      prerequisites.present?
+    end
+
+    def prerequisites
+      Gitlab::Ci::Build::Prerequisite::Factory.new(self).unmet
+    end
+
     def expanded_environment_name
       return unless has_environment?
 
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index adffdc0355e..ae74f569415 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -82,10 +82,14 @@ module Ci
 
     state_machine :status, initial: :created do
       event :enqueue do
-        transition [:created, :skipped, :scheduled] => :pending
+        transition [:created, :preparing, :skipped, :scheduled] => :pending
         transition [:success, :failed, :canceled] => :running
       end
 
+      event :prepare do
+        transition any - [:preparing] => :preparing
+      end
+
       event :run do
         transition any - [:running] => :running
       end
@@ -118,7 +122,7 @@ module Ci
       # Do not add any operations to this state_machine
       # Create a separate worker for each new operation
 
-      before_transition [:created, :pending] => :running do |pipeline|
+      before_transition [:created, :preparing, :pending] => :running do |pipeline|
         pipeline.started_at = Time.now
       end
 
@@ -141,7 +145,7 @@ module Ci
         end
       end
 
-      after_transition [:created, :pending] => :running do |pipeline|
+      after_transition [:created, :preparing, :pending] => :running do |pipeline|
         pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
       end
 
@@ -149,7 +153,7 @@ module Ci
         pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
       end
 
-      after_transition [:created, :pending, :running] => :success do |pipeline|
+      after_transition [:created, :preparing, :pending, :running] => :success do |pipeline|
         pipeline.run_after_commit { PipelineSuccessWorker.perform_async(pipeline.id) }
       end
 
@@ -597,6 +601,7 @@ module Ci
       retry_optimistic_lock(self) do
         case latest_builds_status.to_s
         when 'created' then nil
+        when 'preparing' then prepare
         when 'pending' then enqueue
         when 'running' then run
         when 'success' then succeed
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
index 0389945191e..098f5189517 100644
--- a/app/models/ci/stage.rb
+++ b/app/models/ci/stage.rb
@@ -39,10 +39,14 @@ module Ci
 
     state_machine :status, initial: :created do
       event :enqueue do
-        transition created: :pending
+        transition [:created, :preparing] => :pending
         transition [:success, :failed, :canceled, :skipped] => :running
       end
 
+      event :prepare do
+        transition any - [:preparing] => :preparing
+      end
+
       event :run do
         transition any - [:running] => :running
       end
@@ -76,6 +80,7 @@ module Ci
       retry_optimistic_lock(self) do
         case statuses.latest.status
         when 'created' then nil
+        when 'preparing' then prepare
         when 'pending' then enqueue
         when 'running' then run
         when 'success' then succeed
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 7f6562b63e5..5f66a661324 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -66,7 +66,10 @@ class CommitStatus < ActiveRecord::Base
     end
 
     event :enqueue do
-      transition [:created, :skipped, :manual, :scheduled] => :pending
+      # A CommitStatus will never have prerequisites, but this event
+      # is shared by Ci::Build, which cannot progress unless prerequisites
+      # are satisfied.
+      transition [:created, :preparing, :skipped, :manual, :scheduled] => :pending, unless: :any_unmet_prerequisites?
     end
 
     event :run do
@@ -74,26 +77,26 @@ class CommitStatus < ActiveRecord::Base
     end
 
     event :skip do
-      transition [:created, :pending] => :skipped
+      transition [:created, :preparing, :pending] => :skipped
     end
 
     event :drop do
-      transition [:created, :pending, :running, :scheduled] => :failed
+      transition [:created, :preparing, :pending, :running, :scheduled] => :failed
     end
 
     event :success do
-      transition [:created, :pending, :running] => :success
+      transition [:created, :preparing, :pending, :running] => :success
     end
 
     event :cancel do
-      transition [:created, :pending, :running, :manual, :scheduled] => :canceled
+      transition [:created, :preparing, :pending, :running, :manual, :scheduled] => :canceled
     end
 
-    before_transition [:created, :skipped, :manual, :scheduled] => :pending do |commit_status|
+    before_transition [:created, :preparing, :skipped, :manual, :scheduled] => :pending do |commit_status|
       commit_status.queued_at = Time.now
     end
 
-    before_transition [:created, :pending] => :running do |commit_status|
+    before_transition [:created, :preparing, :pending] => :running do |commit_status|
       commit_status.started_at = Time.now
     end
 
@@ -180,6 +183,10 @@ class CommitStatus < ActiveRecord::Base
     false
   end
 
+  def any_unmet_prerequisites?
+    false
+  end
+
   def auto_canceled?
     canceled? && auto_canceled_by_id?
   end
diff --git a/app/models/commit_status_enums.rb b/app/models/commit_status_enums.rb
index 152105d9429..45e08fa18fe 100644
--- a/app/models/commit_status_enums.rb
+++ b/app/models/commit_status_enums.rb
@@ -14,7 +14,8 @@ module CommitStatusEnums
       runner_unsupported: 6,
       stale_schedule: 7,
       job_execution_timeout: 8,
-      archived_failure: 9
+      archived_failure: 9,
+      unmet_prerequisites: 10
     }
   end
 end
diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb
index 0d2be4c61ab..8882f48c281 100644
--- a/app/models/concerns/has_status.rb
+++ b/app/models/concerns/has_status.rb
@@ -5,14 +5,14 @@ module HasStatus
 
   DEFAULT_STATUS = 'created'.freeze
   BLOCKED_STATUS = %w[manual scheduled].freeze
-  AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped manual scheduled].freeze
+  AVAILABLE_STATUSES = %w[created preparing pending running success failed canceled skipped manual scheduled].freeze
   STARTED_STATUSES = %w[running success failed skipped manual scheduled].freeze
-  ACTIVE_STATUSES = %w[pending running].freeze
+  ACTIVE_STATUSES = %w[preparing pending running].freeze
   COMPLETED_STATUSES = %w[success failed canceled skipped].freeze
-  ORDERED_STATUSES = %w[failed pending running manual scheduled canceled success skipped created].freeze
+  ORDERED_STATUSES = %w[failed preparing pending running manual scheduled canceled success skipped created].freeze
   STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3,
                     failed: 4, canceled: 5, skipped: 6, manual: 7,
-                    scheduled: 8 }.freeze
+                    scheduled: 8, preparing: 9 }.freeze
 
   UnknownStatusError = Class.new(StandardError)
 
@@ -26,6 +26,7 @@ module HasStatus
       success = scope_relevant.success.select('count(*)').to_sql
       manual = scope_relevant.manual.select('count(*)').to_sql
       scheduled = scope_relevant.scheduled.select('count(*)').to_sql
+      preparing = scope_relevant.preparing.select('count(*)').to_sql
       pending = scope_relevant.pending.select('count(*)').to_sql
       running = scope_relevant.running.select('count(*)').to_sql
       skipped = scope_relevant.skipped.select('count(*)').to_sql
@@ -37,12 +38,14 @@ module HasStatus
         WHEN (#{builds})=(#{skipped}) THEN 'skipped'
         WHEN (#{builds})=(#{success}) THEN 'success'
         WHEN (#{builds})=(#{created}) THEN 'created'
+        WHEN (#{builds})=(#{preparing}) THEN 'preparing'
         WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success'
         WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled'
         WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending'
         WHEN (#{running})+(#{pending})>0 THEN 'running'
         WHEN (#{manual})>0 THEN 'manual'
         WHEN (#{scheduled})>0 THEN 'scheduled'
+        WHEN (#{preparing})>0 THEN 'preparing'
         WHEN (#{created})>0 THEN 'running'
         ELSE 'failed'
       END)"
@@ -70,6 +73,7 @@ module HasStatus
 
     state_machine :status, initial: :created do
       state :created, value: 'created'
+      state :preparing, value: 'preparing'
       state :pending, value: 'pending'
       state :running, value: 'running'
       state :failed, value: 'failed'
@@ -81,6 +85,7 @@ module HasStatus
     end
 
     scope :created, -> { where(status: 'created') }
+    scope :preparing, -> { where(status: 'preparing') }
     scope :relevant, -> { where(status: AVAILABLE_STATUSES - ['created']) }
     scope :running, -> { where(status: 'running') }
     scope :pending, -> { where(status: 'pending') }
@@ -90,14 +95,14 @@ module HasStatus
     scope :skipped, -> { where(status: 'skipped') }
     scope :manual, -> { where(status: 'manual') }
     scope :scheduled, -> { where(status: 'scheduled') }
-    scope :alive, -> { where(status: [:created, :pending, :running]) }
+    scope :alive, -> { where(status: [:created, :preparing, :pending, :running]) }
     scope :created_or_pending, -> { where(status: [:created, :pending]) }
     scope :running_or_pending, -> { where(status: [:running, :pending]) }
     scope :finished, -> { where(status: [:success, :failed, :canceled]) }
     scope :failed_or_canceled, -> { where(status: [:failed, :canceled]) }
 
     scope :cancelable, -> do
-      where(status: [:running, :pending, :created, :scheduled])
+      where(status: [:running, :preparing, :pending, :created, :scheduled])
     end
   end
 
diff --git a/app/models/deployment.rb b/app/models/deployment.rb
index 811e623b7f7..428edfd88de 100644
--- a/app/models/deployment.rb
+++ b/app/models/deployment.rb
@@ -78,6 +78,10 @@ class Deployment < ActiveRecord::Base
     Commit.truncate_sha(sha)
   end
 
+  def cluster
+    project.deployment_platform(environment: environment.name)&.cluster
+  end
+
   def last?
     self == environment.last_deployment
   end
diff --git a/app/presenters/commit_status_presenter.rb b/app/presenters/commit_status_presenter.rb
index 0cd77da6303..28a25c8b7a3 100644
--- a/app/presenters/commit_status_presenter.rb
+++ b/app/presenters/commit_status_presenter.rb
@@ -11,7 +11,8 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated
     runner_unsupported: 'Your runner is outdated, please upgrade your runner',
     stale_schedule: 'Delayed job could not be executed by some reason, please try again',
     job_execution_timeout: 'The script exceeded the maximum execution time set for the job',
-    archived_failure: 'The job is archived and cannot be run'
+    archived_failure: 'The job is archived and cannot be run',
+    unmet_prerequisites: 'The job failed to complete prerequisite tasks'
   }.freeze
 
   private_constant :CALLOUT_FAILURE_MESSAGES
diff --git a/app/services/ci/prepare_build_service.rb b/app/services/ci/prepare_build_service.rb
new file mode 100644
index 00000000000..32f11438b79
--- /dev/null
+++ b/app/services/ci/prepare_build_service.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Ci
+  class PrepareBuildService
+    attr_reader :build
+
+    def initialize(build)
+      @build = build
+    end
+
+    def execute
+      prerequisites.each(&:complete!)
+
+      unless build.enqueue
+        build.drop!(:unmet_prerequisites)
+      end
+    end
+
+    private
+
+    def prerequisites
+      build.prerequisites
+    end
+  end
+end
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index b2d88567e0e..6ebd756d3da 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -71,6 +71,7 @@
 - pipeline_hooks:build_hooks
 - pipeline_hooks:pipeline_hooks
 - pipeline_processing:build_finished
+- pipeline_processing:ci_build_prepare
 - pipeline_processing:build_queue
 - pipeline_processing:build_success
 - pipeline_processing:pipeline_process
diff --git a/app/workers/ci/build_prepare_worker.rb b/app/workers/ci/build_prepare_worker.rb
new file mode 100644
index 00000000000..1a35a74ae53
--- /dev/null
+++ b/app/workers/ci/build_prepare_worker.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Ci
+  class BuildPrepareWorker
+    include ApplicationWorker
+    include PipelineQueue
+
+    queue_namespace :pipeline_processing
+
+    def perform(build_id)
+      Ci::Build.find_by_id(build_id).try do |build|
+        Ci::PrepareBuildService.new(build).execute
+      end
+    end
+  end
+end
diff --git a/app/workers/cluster_configure_worker.rb b/app/workers/cluster_configure_worker.rb
index 63e6cc147be..b984dee5b21 100644
--- a/app/workers/cluster_configure_worker.rb
+++ b/app/workers/cluster_configure_worker.rb
@@ -5,6 +5,8 @@ class ClusterConfigureWorker
   include ClusterQueue
 
   def perform(cluster_id)
+    return if Feature.enabled?(:ci_preparing_state, default_enabled: true)
+
     Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
       Clusters::RefreshService.create_or_update_namespaces_for_cluster(cluster)
     end
diff --git a/app/workers/cluster_project_configure_worker.rb b/app/workers/cluster_project_configure_worker.rb
index 497e57c0d0b..d7bea69a01c 100644
--- a/app/workers/cluster_project_configure_worker.rb
+++ b/app/workers/cluster_project_configure_worker.rb
@@ -5,6 +5,8 @@ class ClusterProjectConfigureWorker
   include ClusterQueue
 
   def perform(project_id)
+    return if Feature.enabled?(:ci_preparing_state, default_enabled: true)
+
     project = Project.find(project_id)
 
     ::Clusters::RefreshService.create_or_update_namespaces_for_project(project)