Add latest changes from gitlab-org/gitlab@master

10 files changed, 175 insertions, 59 deletions

diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb
index bab1c73e2f1..aabdf7ce47d 100644
--- a/lib/gitlab/ci/pipeline/chain/base.rb
+++ b/lib/gitlab/ci/pipeline/chain/base.rb
@@ -5,7 +5,7 @@ module Gitlab
     module Pipeline
       module Chain
         class Base
-          attr_reader :pipeline, :command
+          attr_reader :pipeline, :command, :config
 
           delegate :project, :current_user, to: :command
 
diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb
index 899df81ea5c..9662209f88e 100644
--- a/lib/gitlab/ci/pipeline/chain/build.rb
+++ b/lib/gitlab/ci/pipeline/chain/build.rb
@@ -22,8 +22,6 @@ module Gitlab
               external_pull_request: @command.external_pull_request,
               variables_attributes: Array(@command.variables_attributes)
             )
-
-            @pipeline.set_config_source
           end
 
           def break?
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index 58f89a6be5e..c2df419cca0 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -10,7 +10,9 @@ module Gitlab
           :trigger_request, :schedule, :merge_request, :external_pull_request,
           :ignore_skip_ci, :save_incompleted,
           :seeds_block, :variables_attributes, :push_options,
-          :chat_data, :allow_mirror_update
+          :chat_data, :allow_mirror_update,
+          # These attributes are set by Chains during processing:
+          :config_content, :config_processor, :stage_seeds
         ) do
           include Gitlab::Utils::StrongMemoize
 
diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb
new file mode 100644
index 00000000000..a8cd99b8e92
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/config/content.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Content < Chain::Base
+            include Chain::Helpers
+
+            def perform!
+              return if @command.config_content
+
+              if content = content_from_repo
+                @command.config_content = content
+                @pipeline.config_source = :repository_source
+                # TODO: we should persist ci_config_path
+                # @pipeline.config_path = ci_config_path
+              elsif content = content_from_auto_devops
+                @command.config_content = content
+                @pipeline.config_source = :auto_devops_source
+              end
+
+              unless @command.config_content
+                return error("Missing #{ci_config_path} file")
+              end
+            end
+
+            def break?
+              @pipeline.errors.any? || @pipeline.persisted?
+            end
+
+            private
+
+            def content_from_repo
+              return unless project
+              return unless @pipeline.sha
+              return unless ci_config_path
+
+              project.repository.gitlab_ci_yml_for(@pipeline.sha, ci_config_path)
+            rescue GRPC::NotFound, GRPC::Internal
+              nil
+            end
+
+            def content_from_auto_devops
+              return unless project&.auto_devops_enabled?
+
+              Gitlab::Template::GitlabCiYmlTemplate.find('Auto-DevOps').content
+            end
+
+            def ci_config_path
+              project.ci_config_path.presence || '.gitlab-ci.yml'
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb
new file mode 100644
index 00000000000..731b0fdb286
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/config/process.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Config
+          class Process < Chain::Base
+            include Chain::Helpers
+
+            def perform!
+              raise ArgumentError, 'missing config content' unless @command.config_content
+
+              @command.config_processor = ::Gitlab::Ci::YamlProcessor.new(
+                @command.config_content, {
+                  project: project,
+                  sha: @pipeline.sha,
+                  user: current_user
+                }
+              )
+            rescue Gitlab::Ci::YamlProcessor::ValidationError => ex
+              error(ex.message, config_error: true)
+            rescue => ex
+              Gitlab::Sentry.track_acceptable_exception(ex, extra: {
+                project_id: project.id,
+                sha: @pipeline.sha
+              })
+
+              error("Undefined error (#{Labkit::Correlation::CorrelationId.current_id})",
+                config_error: true)
+            end
+
+            def break?
+              @pipeline.errors.any? || @pipeline.persisted?
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb
index 5b46a43b725..0ee9485eebc 100644
--- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb
+++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb
@@ -41,7 +41,7 @@ module Gitlab
           end
 
           def workflow_config
-            @pipeline.config_processor.workflow_attributes || {}
+            @command.config_processor.workflow_attributes || {}
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb
index 13eca5a9d28..3a40c7b167c 100644
--- a/lib/gitlab/ci/pipeline/chain/populate.rb
+++ b/lib/gitlab/ci/pipeline/chain/populate.rb
@@ -10,29 +10,12 @@ module Gitlab
           PopulateError = Class.new(StandardError)
 
           def perform!
-            # Allocate next IID. This operation must be outside of transactions of pipeline creations.
-            pipeline.ensure_project_iid!
-
-            # Protect the pipeline. This is assigned in Populate instead of
-            # Build to prevent erroring out on ambiguous refs.
-            pipeline.protected = @command.protected_ref?
-
-            ##
-            # Populate pipeline with block argument of CreatePipelineService#execute.
-            #
-            @command.seeds_block&.call(pipeline)
-
-            ##
-            # Gather all runtime build/stage errors
-            #
-            if seeds_errors = pipeline.stage_seeds.flat_map(&:errors).compact.presence
-              return error(seeds_errors.join("\n"), config_error: true)
-            end
+            raise ArgumentError, 'missing stage seeds' unless @command.stage_seeds
 
             ##
             # Populate pipeline with all stages, and stages with builds.
             #
-            pipeline.stages = pipeline.stage_seeds.map(&:to_resource)
+            pipeline.stages = @command.stage_seeds.map(&:to_resource)
 
             if pipeline.stages.none?
               return error('No stages / jobs for this pipeline.')
diff --git a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb
index 1e09b417311..9267c72efa4 100644
--- a/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb
+++ b/lib/gitlab/ci/pipeline/chain/remove_unwanted_chat_jobs.rb
@@ -6,11 +6,13 @@ module Gitlab
       module Chain
         class RemoveUnwantedChatJobs < Chain::Base
           def perform!
-            return unless pipeline.config_processor && pipeline.chat?
+            raise ArgumentError, 'missing config processor' unless @command.config_processor
+
+            return unless pipeline.chat?
 
             # When scheduling a chat pipeline we only want to run the build
             # that matches the chat command.
-            pipeline.config_processor.jobs.select! do |name, _|
+            @command.config_processor.jobs.select! do |name, _|
               name.to_s == command.chat_data[:command].to_s
             end
           end
diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb
new file mode 100644
index 00000000000..2e177cfec7e
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/seed.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class Seed < Chain::Base
+          include Chain::Helpers
+          include Gitlab::Utils::StrongMemoize
+
+          def perform!
+            raise ArgumentError, 'missing config processor' unless @command.config_processor
+
+            # Allocate next IID. This operation must be outside of transactions of pipeline creations.
+            pipeline.ensure_project_iid!
+
+            # Protect the pipeline. This is assigned in Populate instead of
+            # Build to prevent erroring out on ambiguous refs.
+            pipeline.protected = @command.protected_ref?
+
+            ##
+            # Populate pipeline with block argument of CreatePipelineService#execute.
+            #
+            @command.seeds_block&.call(pipeline)
+
+            ##
+            # Gather all runtime build/stage errors
+            #
+            if stage_seeds_errors
+              return error(stage_seeds_errors.join("\n"), config_error: true)
+            end
+
+            @command.stage_seeds = stage_seeds
+          end
+
+          def break?
+            pipeline.errors.any?
+          end
+
+          private
+
+          def stage_seeds_errors
+            stage_seeds.flat_map(&:errors).compact.presence
+          end
+
+          def stage_seeds
+            strong_memoize(:stage_seeds) do
+              seeds = stages_attributes.inject([]) do |previous_stages, attributes|
+                seed = Gitlab::Ci::Pipeline::Seed::Stage.new(pipeline, attributes, previous_stages)
+                previous_stages + [seed]
+              end
+
+              seeds.select(&:included?)
+            end
+          end
+
+          def stages_attributes
+            @command.config_processor.stages_attributes
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb
deleted file mode 100644
index 28c38cc3d18..00000000000
--- a/lib/gitlab/ci/pipeline/chain/validate/config.rb
+++ /dev/null
@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Ci
-    module Pipeline
-      module Chain
-        module Validate
-          class Config < Chain::Base
-            include Chain::Helpers
-
-            def perform!
-              unless @pipeline.config_processor
-                unless @pipeline.ci_yaml_file
-                  return error("Missing #{@pipeline.ci_yaml_file_path} file")
-                end
-
-                if @command.save_incompleted && @pipeline.has_yaml_errors?
-                  @pipeline.drop!(:config_error)
-                end
-
-                error(@pipeline.yaml_errors)
-              end
-            end
-
-            def break?
-              @pipeline.errors.any? || @pipeline.persisted?
-            end
-          end
-        end
-      end
-    end
-  end
-end