Merge branch 'ce-to-ee-2017-09-27' into 'master'

CE upstream - Wednesday

Closes gitlab-ce#34415

See merge request gitlab-org/gitlab-ee!3026

13 files changed, 448 insertions, 151 deletions

diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index 28bbf3b384e..d1979bb7ed3 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -149,16 +149,21 @@ module Gitlab
             description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author)
             description += pull_request.description
 
+            source_branch_sha = pull_request.source_branch_sha
+            target_branch_sha = pull_request.target_branch_sha
+            source_branch_sha = project.repository.commit(source_branch_sha)&.sha || source_branch_sha
+            target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha
+
             merge_request = project.merge_requests.create!(
               iid: pull_request.iid,
               title: pull_request.title,
               description: description,
               source_project: project,
               source_branch: pull_request.source_branch_name,
-              source_branch_sha: pull_request.source_branch_sha,
+              source_branch_sha: source_branch_sha,
               target_project: project,
               target_branch: pull_request.target_branch_name,
-              target_branch_sha: pull_request.target_branch_sha,
+              target_branch_sha: target_branch_sha,
               state: pull_request.state,
               author_id: gitlab_user_id(project, pull_request.author),
               assignee_id: nil,
diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb
new file mode 100644
index 00000000000..8d82e1b288d
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/base.rb
@@ -0,0 +1,27 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class Base
+          attr_reader :pipeline, :project, :current_user
+
+          def initialize(pipeline, command)
+            @pipeline = pipeline
+            @command = command
+
+            @project = command.project
+            @current_user = command.current_user
+          end
+
+          def perform!
+            raise NotImplementedError
+          end
+
+          def break?
+            raise NotImplementedError
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb
new file mode 100644
index 00000000000..d5e17a123df
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/create.rb
@@ -0,0 +1,29 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class Create < Chain::Base
+          include Chain::Helpers
+
+          def perform!
+            ::Ci::Pipeline.transaction do
+              pipeline.save!
+
+              @command.seeds_block&.call(pipeline)
+
+              ::Ci::CreatePipelineStagesService
+                .new(project, current_user)
+                .execute(pipeline)
+            end
+          rescue ActiveRecord::RecordInvalid => e
+            error("Failed to persist the pipeline: #{e}")
+          end
+
+          def break?
+            !pipeline.persisted?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb
new file mode 100644
index 00000000000..02d81286f21
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/helpers.rb
@@ -0,0 +1,25 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Helpers
+          def branch_exists?
+            return @is_branch if defined?(@is_branch)
+
+            @is_branch = project.repository.branch_exists?(pipeline.ref)
+          end
+
+          def tag_exists?
+            return @is_tag if defined?(@is_tag)
+
+            @is_tag = project.repository.tag_exists?(pipeline.ref)
+          end
+
+          def error(message)
+            pipeline.errors.add(:base, message)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb
new file mode 100644
index 00000000000..015f2988327
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/sequence.rb
@@ -0,0 +1,36 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class Sequence
+          def initialize(pipeline, command, sequence)
+            @pipeline = pipeline
+            @completed = []
+
+            @sequence = sequence.map do |chain|
+              chain.new(pipeline, command)
+            end
+          end
+
+          def build!
+            @sequence.each do |step|
+              step.perform!
+
+              break if step.break?
+
+              @completed << step
+            end
+
+            @pipeline.tap do
+              yield @pipeline, self if block_given?
+            end
+          end
+
+          def complete?
+            @completed.size == @sequence.size
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb
new file mode 100644
index 00000000000..9a72de87bab
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/skip.rb
@@ -0,0 +1,33 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class Skip < Chain::Base
+          SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i
+
+          def perform!
+            if skipped?
+              @pipeline.skip if @command.save_incompleted
+            end
+          end
+
+          def skipped?
+            !@command.ignore_skip_ci && commit_message_skips_ci?
+          end
+
+          def break?
+            skipped?
+          end
+
+          private
+
+          def commit_message_skips_ci?
+            return false unless @pipeline.git_commit_message
+
+            @skipped ||= !!(@pipeline.git_commit_message =~ SKIP_PATTERN)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
new file mode 100644
index 00000000000..19b78b792a6
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
@@ -0,0 +1,58 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Validate
+          class Abilities < Chain::Base
+            include Gitlab::Allowable
+            include Chain::Helpers
+
+            def perform!
+              unless project.builds_enabled?
+                return error('Pipelines are disabled!')
+              end
+
+              if @command.allow_mirror_update && !project.mirror_trigger_builds?
+                return error('Pipeline is disabled for mirror updates')
+              end
+
+              unless allowed_to_trigger_pipeline?
+                if can?(current_user, :create_pipeline, project)
+                  return error("Insufficient permissions for protected ref '#{pipeline.ref}'")
+                else
+                  return error('Insufficient permissions to create a new pipeline')
+                end
+              end
+            end
+
+            def break?
+              @pipeline.errors.any?
+            end
+
+            def allowed_to_trigger_pipeline?
+              if current_user
+                allowed_to_create?
+              else # legacy triggers don't have a corresponding user
+                !project.protected_for?(@pipeline.ref)
+              end
+            end
+
+            def allowed_to_create?
+              return unless can?(current_user, :create_pipeline, project)
+
+              access = Gitlab::UserAccess.new(current_user, project: project)
+
+              if branch_exists?
+                access.can_update_branch?(@pipeline.ref)
+              elsif tag_exists?
+                access.can_create_tag?(@pipeline.ref)
+              else
+                true # Allow it for now and we'll reject when we check ref existence
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb
new file mode 100644
index 00000000000..489bcd79655
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Validate
+          class Config < Chain::Base
+            include Chain::Helpers
+
+            def perform!
+              unless @pipeline.config_processor
+                unless @pipeline.ci_yaml_file
+                  return error("Missing #{@pipeline.ci_yaml_file_path} file")
+                end
+
+                if @command.save_incompleted && @pipeline.has_yaml_errors?
+                  @pipeline.drop
+                end
+
+                return error(@pipeline.yaml_errors)
+              end
+
+              unless @pipeline.has_stage_seeds?
+                return error('No stages / jobs for this pipeline.')
+              end
+            end
+
+            def break?
+              @pipeline.errors.any? || @pipeline.persisted?
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb
new file mode 100644
index 00000000000..70a4cfdbdea
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb
@@ -0,0 +1,30 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        module Validate
+          class Repository < Chain::Base
+            include Chain::Helpers
+
+            def perform!
+              unless branch_exists? || tag_exists?
+                return error('Reference not found')
+              end
+
+              ## TODO, we check commit in the service, that is why
+              # there is no repository access here.
+              #
+              unless pipeline.sha
+                return error('Commit not found')
+              end
+            end
+
+            def break?
+              @pipeline.errors.any?
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb
new file mode 100644
index 00000000000..469fc094cc8
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/duration.rb
@@ -0,0 +1,143 @@
+module Gitlab
+  module Ci
+    module Pipeline
+      # # Introduction - total running time
+      #
+      # The problem this module is trying to solve is finding the total running
+      # time amongst all the jobs, excluding retries and pending (queue) time.
+      # We could reduce this problem down to finding the union of periods.
+      #
+      # So each job would be represented as a `Period`, which consists of
+      # `Period#first` as when the job started and `Period#last` as when the
+      # job was finished. A simple example here would be:
+      #
+      # * A (1, 3)
+      # * B (2, 4)
+      # * C (6, 7)
+      #
+      # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4.
+      # C begins from 6, and ends to 7. Visually it could be viewed as:
+      #
+      #     0  1  2  3  4  5  6  7
+      #        AAAAAAA
+      #           BBBBBBB
+      #                       CCCC
+      #
+      # The union of A, B, and C would be (1, 4) and (6, 7), therefore the
+      # total running time should be:
+      #
+      #     (4 - 1) + (7 - 6) => 4
+      #
+      # # The Algorithm
+      #
+      # The algorithm used here for union would be described as follow.
+      # First we make sure that all periods are sorted by `Period#first`.
+      # Then we try to merge periods by iterating through the first period
+      # to the last period. The goal would be merging all overlapped periods
+      # so that in the end all the periods are discrete. When all periods
+      # are discrete, we're free to just sum all the periods to get real
+      # running time.
+      #
+      # Here we begin from A, and compare it to B. We could find that
+      # before A ends, B already started. That is `B.first <= A.last`
+      # that is `2 <= 3` which means A and B are overlapping!
+      #
+      # When we found that two periods are overlapping, we would need to merge
+      # them into a new period and disregard the old periods. To make a new
+      # period, we take `A.first` as the new first because remember? we sorted
+      # them, so `A.first` must be smaller or equal to `B.first`. And we take
+      # `[A.last, B.last].max` as the new last because we want whoever ended
+      # later. This could be broken into two cases:
+      #
+      #     0  1  2  3  4
+      #        AAAAAAA
+      #           BBBBBBB
+      #
+      # Or:
+      #
+      #     0  1  2  3  4
+      #        AAAAAAAAAA
+      #           BBBB
+      #
+      # So that we need to take whoever ends later. Back to our example,
+      # after merging and discard A and B it could be visually viewed as:
+      #
+      #     0  1  2  3  4  5  6  7
+      #        DDDDDDDDDD
+      #                       CCCC
+      #
+      # Now we could go on and compare the newly created D and the old C.
+      # We could figure out that D and C are not overlapping by checking
+      # `C.first <= D.last` is `false`. Therefore we need to keep both C
+      # and D. The example would end here because there are no more jobs.
+      #
+      # After having the union of all periods, we just need to sum the length
+      # of all periods to get total time.
+      #
+      #     (4 - 1) + (7 - 6) => 4
+      #
+      # That is 4 is the answer in the example.
+      module Duration
+        extend self
+
+        Period = Struct.new(:first, :last) do
+          def duration
+            last - first
+          end
+        end
+
+        def from_pipeline(pipeline)
+          status = %w[success failed running canceled]
+          builds = pipeline.builds.latest
+            .where(status: status).where.not(started_at: nil).order(:started_at)
+
+          from_builds(builds)
+        end
+
+        def from_builds(builds)
+          now = Time.now
+
+          periods = builds.map do |b|
+            Period.new(b.started_at, b.finished_at || now)
+          end
+
+          from_periods(periods)
+        end
+
+        # periods should be sorted by `first`
+        def from_periods(periods)
+          process_duration(process_periods(periods))
+        end
+
+        private
+
+        def process_periods(periods)
+          return periods if periods.empty?
+
+          periods.drop(1).inject([periods.first]) do |result, current|
+            previous = result.last
+
+            if overlap?(previous, current)
+              result[-1] = merge(previous, current)
+              result
+            else
+              result << current
+            end
+          end
+        end
+
+        def overlap?(previous, current)
+          current.first <= previous.last
+        end
+
+        def merge(previous, current)
+          Period.new(previous.first, [previous.last, current.last].max)
+        end
+
+        def process_duration(periods)
+          periods.sum(&:duration)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb
deleted file mode 100644
index 3208cc2bef6..00000000000
--- a/lib/gitlab/ci/pipeline_duration.rb
+++ /dev/null
@@ -1,141 +0,0 @@
-module Gitlab
-  module Ci
-    # # Introduction - total running time
-    #
-    # The problem this module is trying to solve is finding the total running
-    # time amongst all the jobs, excluding retries and pending (queue) time.
-    # We could reduce this problem down to finding the union of periods.
-    #
-    # So each job would be represented as a `Period`, which consists of
-    # `Period#first` as when the job started and `Period#last` as when the
-    # job was finished. A simple example here would be:
-    #
-    # * A (1, 3)
-    # * B (2, 4)
-    # * C (6, 7)
-    #
-    # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4.
-    # C begins from 6, and ends to 7. Visually it could be viewed as:
-    #
-    #     0  1  2  3  4  5  6  7
-    #        AAAAAAA
-    #           BBBBBBB
-    #                       CCCC
-    #
-    # The union of A, B, and C would be (1, 4) and (6, 7), therefore the
-    # total running time should be:
-    #
-    #     (4 - 1) + (7 - 6) => 4
-    #
-    # # The Algorithm
-    #
-    # The algorithm used here for union would be described as follow.
-    # First we make sure that all periods are sorted by `Period#first`.
-    # Then we try to merge periods by iterating through the first period
-    # to the last period. The goal would be merging all overlapped periods
-    # so that in the end all the periods are discrete. When all periods
-    # are discrete, we're free to just sum all the periods to get real
-    # running time.
-    #
-    # Here we begin from A, and compare it to B. We could find that
-    # before A ends, B already started. That is `B.first <= A.last`
-    # that is `2 <= 3` which means A and B are overlapping!
-    #
-    # When we found that two periods are overlapping, we would need to merge
-    # them into a new period and disregard the old periods. To make a new
-    # period, we take `A.first` as the new first because remember? we sorted
-    # them, so `A.first` must be smaller or equal to `B.first`. And we take
-    # `[A.last, B.last].max` as the new last because we want whoever ended
-    # later. This could be broken into two cases:
-    #
-    #     0  1  2  3  4
-    #        AAAAAAA
-    #           BBBBBBB
-    #
-    # Or:
-    #
-    #     0  1  2  3  4
-    #        AAAAAAAAAA
-    #           BBBB
-    #
-    # So that we need to take whoever ends later. Back to our example,
-    # after merging and discard A and B it could be visually viewed as:
-    #
-    #     0  1  2  3  4  5  6  7
-    #        DDDDDDDDDD
-    #                       CCCC
-    #
-    # Now we could go on and compare the newly created D and the old C.
-    # We could figure out that D and C are not overlapping by checking
-    # `C.first <= D.last` is `false`. Therefore we need to keep both C
-    # and D. The example would end here because there are no more jobs.
-    #
-    # After having the union of all periods, we just need to sum the length
-    # of all periods to get total time.
-    #
-    #     (4 - 1) + (7 - 6) => 4
-    #
-    # That is 4 is the answer in the example.
-    module PipelineDuration
-      extend self
-
-      Period = Struct.new(:first, :last) do
-        def duration
-          last - first
-        end
-      end
-
-      def from_pipeline(pipeline)
-        status = %w[success failed running canceled]
-        builds = pipeline.builds.latest
-          .where(status: status).where.not(started_at: nil).order(:started_at)
-
-        from_builds(builds)
-      end
-
-      def from_builds(builds)
-        now = Time.now
-
-        periods = builds.map do |b|
-          Period.new(b.started_at, b.finished_at || now)
-        end
-
-        from_periods(periods)
-      end
-
-      # periods should be sorted by `first`
-      def from_periods(periods)
-        process_duration(process_periods(periods))
-      end
-
-      private
-
-      def process_periods(periods)
-        return periods if periods.empty?
-
-        periods.drop(1).inject([periods.first]) do |result, current|
-          previous = result.last
-
-          if overlap?(previous, current)
-            result[-1] = merge(previous, current)
-            result
-          else
-            result << current
-          end
-        end
-      end
-
-      def overlap?(previous, current)
-        current.first <= previous.last
-      end
-
-      def merge(previous, current)
-        Period.new(previous.first, [previous.last, current.last].max)
-      end
-
-      def process_duration(periods)
-        periods.sum(&:duration)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb
index 371cbe04b9b..c98eefbce25 100644
--- a/lib/gitlab/diff/diff_refs.rb
+++ b/lib/gitlab/diff/diff_refs.rb
@@ -13,9 +13,9 @@ module Gitlab
 
       def ==(other)
         other.is_a?(self.class) &&
-          base_sha == other.base_sha &&
-          start_sha == other.start_sha &&
-          head_sha == other.head_sha
+          shas_equal?(base_sha, other.base_sha) &&
+          shas_equal?(start_sha, other.start_sha) &&
+          shas_equal?(head_sha, other.head_sha)
       end
 
       alias_method :eql?, :==
@@ -47,6 +47,22 @@ module Gitlab
           CompareService.new(project, head_sha).execute(project, start_sha, straight: straight)
         end
       end
+
+      private
+
+      def shas_equal?(sha1, sha2)
+        return true if sha1 == sha2
+        return false if sha1.nil? || sha2.nil?
+        return false unless sha1.class == sha2.class
+
+        length = [sha1.length, sha2.length].min
+
+        # If either of the shas is below the minimum length, we cannot be sure
+        # that they actually refer to the same commit because of hash collision.
+        return false if length < Commit::MIN_SHA_LENGTH
+
+        sha1[0, length] == sha2[0, length]
+      end
     end
   end
 end
diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb
index f80afb20f0c..b8db3adef0a 100644
--- a/lib/gitlab/diff/position.rb
+++ b/lib/gitlab/diff/position.rb
@@ -49,12 +49,13 @@ module Gitlab
         coder['attributes'] = self.to_h
       end
 
-      def key
-        @key ||= [base_sha, start_sha, head_sha, Digest::SHA1.hexdigest(old_path || ""), Digest::SHA1.hexdigest(new_path || ""), old_line, new_line]
-      end
-
       def ==(other)
-        other.is_a?(self.class) && key == other.key
+        other.is_a?(self.class) &&
+          other.diff_refs == diff_refs &&
+          other.old_path == old_path &&
+          other.new_path == new_path &&
+          other.old_line == old_line &&
+          other.new_line == new_line
       end
 
       def to_h