Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (671 commits)
  Make rubocop happy
  Use guard clause
  Improve language
  Prettify
  Use temp branch
  Pass info about who started the job and which job triggered it
  Docs: add indexes for monitoring and performance monitoring
  clearer-documentation-on-inline-diffs
  Add docs for commit diff discussion in merge requests
  sorting for tags api
  Clear BatchLoader after each spec to prevent holding onto records longer than necessary
  Include project in BatchLoader key to prevent returning blobs for the wrong project
  moved lfs_blob_ids method into ExtractsPath module
  Converted JS modules into exported modules
  spec fixes
  Bump gitlab-shell version to 5.10.3
  Clear caches before updating MR diffs
  Use new Ruby version 2.4 in GitLab QA images
  moved lfs blob fetch from extractspath file
  Update GitLab QA dependencies
  ...

8 files changed, 124 insertions, 36 deletions

diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb
index 8d82e1b288d..efed19da21c 100644
--- a/lib/gitlab/ci/pipeline/chain/base.rb
+++ b/lib/gitlab/ci/pipeline/chain/base.rb
@@ -3,14 +3,13 @@ module Gitlab
     module Pipeline
       module Chain
         class Base
-          attr_reader :pipeline, :project, :current_user
+          attr_reader :pipeline, :command
+
+          delegate :project, :current_user, to: :command
 
           def initialize(pipeline, command)
             @pipeline = pipeline
             @command = command
-
-            @project = command.project
-            @current_user = command.current_user
           end
 
           def perform!
diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb
new file mode 100644
index 00000000000..70732d26bbd
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/build.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class Build < Chain::Base
+          def perform!
+            @pipeline.assign_attributes(
+              source: @command.source,
+              project: @command.project,
+              ref: @command.ref,
+              sha: @command.sha,
+              before_sha: @command.before_sha,
+              tag: @command.tag_exists?,
+              trigger_requests: Array(@command.trigger_request),
+              user: @command.current_user,
+              pipeline_schedule: @command.schedule,
+              protected: @command.protected_ref?
+            )
+
+            @pipeline.set_config_source
+          end
+
+          def break?
+            false
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
new file mode 100644
index 00000000000..7b19b10e05b
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -0,0 +1,61 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        Command = Struct.new(
+          :source, :project, :current_user,
+          :origin_ref, :checkout_sha, :after_sha, :before_sha,
+          :trigger_request, :schedule,
+          :ignore_skip_ci, :save_incompleted,
+          :seeds_block
+        ) do
+          include Gitlab::Utils::StrongMemoize
+
+          def initialize(**params)
+            params.each do |key, value|
+              self[key] = value
+            end
+          end
+
+          def branch_exists?
+            strong_memoize(:is_branch) do
+              project.repository.branch_exists?(ref)
+            end
+          end
+
+          def tag_exists?
+            strong_memoize(:is_tag) do
+              project.repository.tag_exists?(ref)
+            end
+          end
+
+          def ref
+            strong_memoize(:ref) do
+              Gitlab::Git.ref_name(origin_ref)
+            end
+          end
+
+          def sha
+            strong_memoize(:sha) do
+              project.commit(origin_sha || origin_ref).try(:id)
+            end
+          end
+
+          def origin_sha
+            checkout_sha || after_sha
+          end
+
+          def before_sha
+            self[:before_sha] || checkout_sha || Gitlab::Git::BLANK_SHA
+          end
+
+          def protected_ref?
+            strong_memoize(:protected_ref) do
+              project.protected_for?(ref)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb
index d5e17a123df..d19a2519803 100644
--- a/lib/gitlab/ci/pipeline/chain/create.rb
+++ b/lib/gitlab/ci/pipeline/chain/create.rb
@@ -17,11 +17,27 @@ module Gitlab
             end
           rescue ActiveRecord::RecordInvalid => e
             error("Failed to persist the pipeline: #{e}")
+          ensure
+            if pipeline.builds.where(stage_id: nil).any?
+              invalid_builds_counter.increment(node: hostname)
+            end
           end
 
           def break?
             !pipeline.persisted?
           end
+
+          private
+
+          def invalid_builds_counter
+            @counter ||= Gitlab::Metrics
+              .counter(:gitlab_ci_invalid_builds_total,
+                       'Invalid builds without stage assigned counter')
+          end
+
+          def hostname
+            @hostname ||= Socket.gethostname
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb
index 7ab7a64c7e3..bf1380a1da9 100644
--- a/lib/gitlab/ci/pipeline/chain/helpers.rb
+++ b/lib/gitlab/ci/pipeline/chain/helpers.rb
@@ -3,20 +3,6 @@ module Gitlab
     module Pipeline
       module Chain
         module Helpers
-          include Gitlab::Utils::StrongMemoize
-
-          def branch_exists?
-            strong_memoize(:is_branch) do
-              project.repository.branch_exists?(pipeline.ref)
-            end
-          end
-
-          def tag_exists?
-            strong_memoize(:is_tag) do
-              project.repository.tag_exists?(pipeline.ref)
-            end
-          end
-
           def error(message)
             pipeline.errors.add(:base, message)
           end
diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb
index 015f2988327..e24630656d3 100644
--- a/lib/gitlab/ci/pipeline/chain/sequence.rb
+++ b/lib/gitlab/ci/pipeline/chain/sequence.rb
@@ -5,20 +5,19 @@ module Gitlab
         class Sequence
           def initialize(pipeline, command, sequence)
             @pipeline = pipeline
+            @command = command
+            @sequence = sequence
             @completed = []
-
-            @sequence = sequence.map do |chain|
-              chain.new(pipeline, command)
-            end
           end
 
           def build!
-            @sequence.each do |step|
-              step.perform!
+            @sequence.each do |chain|
+              step = chain.new(@pipeline, @command)
 
+              step.perform!
               break if step.break?
 
-              @completed << step
+              @completed.push(step)
             end
 
             @pipeline.tap do
diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
index 4913a604079..13c6fedd831 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
@@ -14,7 +14,7 @@ module Gitlab
 
               unless allowed_to_trigger_pipeline?
                 if can?(current_user, :create_pipeline, project)
-                  return error("Insufficient permissions for protected ref '#{pipeline.ref}'")
+                  return error("Insufficient permissions for protected ref '#{command.ref}'")
                 else
                   return error('Insufficient permissions to create a new pipeline')
                 end
@@ -29,7 +29,7 @@ module Gitlab
               if current_user
                 allowed_to_create?
               else # legacy triggers don't have a corresponding user
-                !project.protected_for?(@pipeline.ref)
+                !@command.protected_ref?
               end
             end
 
@@ -38,10 +38,10 @@ module Gitlab
 
               access = Gitlab::UserAccess.new(current_user, project: project)
 
-              if branch_exists?
-                access.can_update_branch?(@pipeline.ref)
-              elsif tag_exists?
-                access.can_create_tag?(@pipeline.ref)
+              if @command.branch_exists?
+                access.can_update_branch?(@command.ref)
+              elsif @command.tag_exists?
+                access.can_create_tag?(@command.ref)
               else
                 true # Allow it for now and we'll reject when we check ref existence
               end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb
index 70a4cfdbdea..9699c24e5b6 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/repository.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb
@@ -7,14 +7,11 @@ module Gitlab
             include Chain::Helpers
 
             def perform!
-              unless branch_exists? || tag_exists?
+              unless @command.branch_exists? || @command.tag_exists?
                 return error('Reference not found')
               end
 
-              ## TODO, we check commit in the service, that is why
-              # there is no repository access here.
-              #
-              unless pipeline.sha
+              unless @command.sha
                 return error('Commit not found')
               end
             end