Merge remote-tracking branch 'upstream/master' into fix-git-hooks-when-creating-file

* upstream/master: (60 commits)
  Refactor SSH keys docs
  Improvements to setting up ssh
  Do not reload diff for merge request made from fork when target branch in fork is updated
  Add 8.12.10, 8.12.11, and 8.12.12 CHANGELOG.md items
  Changes after review
  Fix broken test
  Adds CHANGELOG entry
  Adds tests
  Uniformize props name format
  Replace commit icon svg logic
  Replace play icon svg logic
  Update docs to reflect new defaults on omnibus
  Merge branch 'jej-23867-use-mr-finder-instead-of-access-check' into 'security'
  Merge branch 'html-safe-diff-line-content' into 'security'
  Merge branch 'rs-filter-authentication_token' into 'security'
  Merge branch 'destroy-session' into 'security'
  remove unnecessary issues event filter on comments tab
  Updating reference to database password
  Add CHANGELOG entry
  Displays milestone remaining days only when it's present
  ...

12 files changed, 226 insertions, 141 deletions

diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index caf6908505e..fda8228a1e9 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -21,8 +21,6 @@ module Ci
 
     after_create :keep_around_commits, unless: :importing?
 
-    delegate :stages, to: :statuses
-
     state_machine :status, initial: :created do
       event :enqueue do
         transition created: :pending
@@ -98,17 +96,35 @@ module Ci
       sha[0...8]
     end
 
-    def self.stages
-      # We use pluck here due to problems with MySQL which doesn't allow LIMIT/OFFSET in queries
-      CommitStatus.where(pipeline: pluck(:id)).stages
-    end
-
     def self.total_duration
       where.not(duration: nil).sum(:duration)
     end
 
-    def stages_with_latest_statuses
-      statuses.latest.includes(project: :namespace).order(:stage_idx).group_by(&:stage)
+    def stages_count
+      statuses.select(:stage).distinct.count
+    end
+
+    def stages_name
+      statuses.order(:stage_idx).distinct.
+        pluck(:stage, :stage_idx).map(&:first)
+    end
+
+    def stages
+      status_sql = statuses.latest.where('stage=sg.stage').status_sql
+
+      stages_query = statuses.group('stage').select(:stage)
+                       .order('max(stage_idx)')
+
+      stages_with_statuses = CommitStatus.from(stages_query, :sg).
+        pluck('sg.stage', status_sql)
+
+      stages_with_statuses.map do |stage|
+        Ci::Stage.new(self, name: stage.first, status: stage.last)
+      end
+    end
+
+    def artifacts
+      builds.latest.with_artifacts_not_expired
     end
 
     def project_id
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
new file mode 100644
index 00000000000..d2a37c0a827
--- /dev/null
+++ b/app/models/ci/stage.rb
@@ -0,0 +1,37 @@
+module Ci
+  # Currently this is artificial object, constructed dynamically
+  # We should migrate this object to actual database record in the future
+  class Stage
+    include StaticModel
+
+    attr_reader :pipeline, :name
+
+    delegate :project, to: :pipeline
+
+    def initialize(pipeline, name:, status: nil)
+      @pipeline = pipeline
+      @name = name
+      @status = status
+    end
+
+    def to_param
+      name
+    end
+
+    def status
+      @status ||= statuses.latest.status
+    end
+
+    def detailed_status
+      Gitlab::Ci::Status::Stage::Factory.new(self).fabricate!
+    end
+
+    def statuses
+      @statuses ||= pipeline.statuses.where(stage: name)
+    end
+
+    def builds
+      @builds ||= pipeline.builds.where(stage: name)
+    end
+  end
+end
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 248140f421b..1831cc7e175 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -245,44 +245,47 @@ class Commit
     project.repository.next_branch("cherry-pick-#{short_id}", mild: true)
   end
 
-  def revert_description
-    if merged_merge_request
-      "This reverts merge request #{merged_merge_request.to_reference}"
+  def revert_description(user)
+    if merged_merge_request?(user)
+      "This reverts merge request #{merged_merge_request(user).to_reference}"
     else
       "This reverts commit #{sha}"
     end
   end
 
-  def revert_message
-    %Q{Revert "#{title.strip}"\n\n#{revert_description}}
+  def revert_message(user)
+    %Q{Revert "#{title.strip}"\n\n#{revert_description(user)}}
   end
 
-  def reverts_commit?(commit)
-    description? && description.include?(commit.revert_description)
+  def reverts_commit?(commit, user)
+    description? && description.include?(commit.revert_description(user))
   end
 
   def merge_commit?
     parents.size > 1
   end
 
-  def merged_merge_request
-    return @merged_merge_request if defined?(@merged_merge_request)
-
-    @merged_merge_request = project.merge_requests.find_by(merge_commit_sha: id) if merge_commit?
+  def merged_merge_request(current_user)
+    # Memoize with per-user access check
+    @merged_merge_request_hash ||= Hash.new do |hash, user|
+      hash[user] = merged_merge_request_no_cache(user)
+    end
+    
+    @merged_merge_request_hash[current_user]
   end
 
-  def has_been_reverted?(current_user = nil, noteable = self)
+  def has_been_reverted?(current_user, noteable = self)
     ext = all_references(current_user)
 
     noteable.notes_with_associations.system.each do |note|
       note.all_references(current_user, extractor: ext)
     end
 
-    ext.commits.any? { |commit_ref| commit_ref.reverts_commit?(self) }
+    ext.commits.any? { |commit_ref| commit_ref.reverts_commit?(self, current_user) }
   end
 
-  def change_type_title
-    merged_merge_request ? 'merge request' : 'commit'
+  def change_type_title(user)
+    merged_merge_request?(user) ? 'merge request' : 'commit'
   end
 
   # Get the URI type of the given path
@@ -350,4 +353,12 @@ class Commit
 
     changes
   end
+
+  def merged_merge_request?(user)
+    !!merged_merge_request(user)
+  end
+
+  def merged_merge_request_no_cache(user)
+    MergeRequestsFinder.new(user, project_id: project.id).find_by(merge_commit_sha: id) if merge_commit?
+  end
 end
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index c345bf293c9..cf90475f4d4 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -31,18 +31,13 @@ class CommitStatus < ActiveRecord::Base
   end
 
   scope :exclude_ignored, -> do
-    quoted_when = connection.quote_column_name('when')
     # We want to ignore failed_but_allowed jobs
     where("allow_failure = ? OR status IN (?)",
-      false, all_state_names - [:failed, :canceled]).
-      # We want to ignore skipped manual jobs
-      where("#{quoted_when} <> ? OR status <> ?", 'manual', 'skipped').
-      # We want to ignore skipped on_failure
-      where("#{quoted_when} <> ? OR status <> ?", 'on_failure', 'skipped')
+      false, all_state_names - [:failed, :canceled])
   end
 
-  scope :latest_ci_stages, -> { latest.ordered.includes(project: :namespace) }
-  scope :retried_ci_stages, -> { retried.ordered.includes(project: :namespace) }
+  scope :latest_ordered, -> { latest.ordered.includes(project: :namespace) }
+  scope :retried_ordered, -> { retried.ordered.includes(project: :namespace) }
 
   state_machine :status do
     event :enqueue do
@@ -117,20 +112,6 @@ class CommitStatus < ActiveRecord::Base
     name.gsub(/\d+[\s:\/\\]+\d+\s*/, '').strip
   end
 
-  def self.stages
-    # We group by stage name, but order stages by theirs' index
-    unscoped.from(all, :sg).group('stage').order('max(stage_idx)', 'stage').pluck('sg.stage')
-  end
-
-  def self.stages_status
-    # We execute subquery for each stage to calculate a stage status
-    statuses = unscoped.from(all, :sg).group('stage').pluck('sg.stage', all.where('stage=sg.stage').status_sql)
-    statuses.inject({}) do |h, k|
-      h[k.first] = k.last
-      h
-    end
-  end
-
   def failed_but_allowed?
     allow_failure? && (failed? || canceled?)
   end
diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb
index 2f5aa91a964..90432fc4050 100644
--- a/app/models/concerns/has_status.rb
+++ b/app/models/concerns/has_status.rb
@@ -4,7 +4,7 @@ module HasStatus
   AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped]
   STARTED_STATUSES = %w[running success failed skipped]
   ACTIVE_STATUSES = %w[pending running]
-  COMPLETED_STATUSES = %w[success failed canceled]
+  COMPLETED_STATUSES = %w[success failed canceled skipped]
   ORDERED_STATUSES = %w[failed pending running canceled success skipped]
 
   class_methods do
@@ -23,9 +23,10 @@ module HasStatus
       canceled = scope.canceled.select('count(*)').to_sql
 
       "(CASE
+        WHEN (#{builds})=(#{skipped}) THEN 'skipped'
         WHEN (#{builds})=(#{success}) THEN 'success'
         WHEN (#{builds})=(#{created}) THEN 'created'
-        WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'skipped'
+        WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success'
         WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled'
         WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending'
         WHEN (#{running})+(#{pending})+(#{created})>0 THEN 'running'
diff --git a/app/models/concerns/milestoneish.rb b/app/models/concerns/milestoneish.rb
index e65fc9eaa09..875e9834487 100644
--- a/app/models/concerns/milestoneish.rb
+++ b/app/models/concerns/milestoneish.rb
@@ -1,17 +1,17 @@
 module Milestoneish
-  def closed_items_count(user = nil)
+  def closed_items_count(user)
     issues_visible_to_user(user).closed.size + merge_requests.closed_and_merged.size
   end
 
-  def total_items_count(user = nil)
+  def total_items_count(user)
     issues_visible_to_user(user).size + merge_requests.size
   end
 
-  def complete?(user = nil)
+  def complete?(user)
     total_items_count(user) > 0 && total_items_count(user) == closed_items_count(user)
   end
 
-  def percent_complete(user = nil)
+  def percent_complete(user)
     ((closed_items_count(user) * 100) / total_items_count(user)).abs
   rescue ZeroDivisionError
     0
@@ -29,7 +29,7 @@ module Milestoneish
     (Date.today - start_date).to_i
   end
 
-  def issues_visible_to_user(user = nil)
+  def issues_visible_to_user(user)
     issues.visible_to_user(user)
   end
 
diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb
new file mode 100644
index 00000000000..d36bb9da296
--- /dev/null
+++ b/app/models/concerns/routable.rb
@@ -0,0 +1,70 @@
+# Store object full path in separate table for easy lookup and uniq validation
+# Object must have path db field and respond to full_path and full_path_changed? methods.
+module Routable
+  extend ActiveSupport::Concern
+
+  included do
+    has_one :route, as: :source, autosave: true, dependent: :destroy
+
+    validates_associated :route
+
+    before_validation :update_route_path, if: :full_path_changed?
+  end
+
+  class_methods do
+    # Finds a single object by full path match in routes table.
+    #
+    # Usage:
+    #
+    #     Klass.find_by_full_path('gitlab-org/gitlab-ce')
+    #
+    # Returns a single object, or nil.
+    def find_by_full_path(path)
+      # On MySQL we want to ensure the ORDER BY uses a case-sensitive match so
+      # any literal matches come first, for this we have to use "BINARY".
+      # Without this there's still no guarantee in what order MySQL will return
+      # rows.
+      binary = Gitlab::Database.mysql? ? 'BINARY' : ''
+
+      order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)"
+
+      where_paths_in([path]).reorder(order_sql).take
+    end
+
+    # Builds a relation to find multiple objects by their full paths.
+    #
+    # Usage:
+    #
+    #     Klass.where_paths_in(%w{gitlab-org/gitlab-ce gitlab-org/gitlab-ee})
+    #
+    # Returns an ActiveRecord::Relation.
+    def where_paths_in(paths)
+      wheres = []
+      cast_lower = Gitlab::Database.postgresql?
+
+      paths.each do |path|
+        path = connection.quote(path)
+        where = "(routes.path = #{path})"
+
+        if cast_lower
+          where = "(#{where} OR (LOWER(routes.path) = LOWER(#{path})))"
+        end
+
+        wheres << where
+      end
+
+      if wheres.empty?
+        none
+      else
+        joins(:route).where(wheres.join(' OR '))
+      end
+    end
+  end
+
+  private
+
+  def update_route_path
+    route || build_route(source: self)
+    route.path = full_path
+  end
+end
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 33b578e12c1..ea3cf1cdaac 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -101,7 +101,9 @@ class MergeRequest < ActiveRecord::Base
   validate :validate_branches, unless: [:allow_broken, :importing?, :closed_without_fork?]
   validate :validate_fork, unless: :closed_without_fork?
 
-  scope :by_branch, ->(branch_name) { where("(source_branch LIKE :branch) OR (target_branch LIKE :branch)", branch: branch_name) }
+  scope :by_source_or_target_branch, ->(branch_name) do
+    where("source_branch = :branch OR target_branch = :branch", branch: branch_name)
+  end
   scope :cared, ->(user) { where('assignee_id = :user OR author_id = :user', user: user.id) }
   scope :by_milestone, ->(milestone) { where(milestone_id: milestone) }
   scope :of_projects, ->(ids) { where(target_project_id: ids) }
@@ -805,7 +807,7 @@ class MergeRequest < ActiveRecord::Base
     @merge_commit ||= project.commit(merge_commit_sha) if merge_commit_sha
   end
 
-  def can_be_reverted?(current_user = nil)
+  def can_be_reverted?(current_user)
     merge_commit && !merge_commit.has_been_reverted?(current_user, self)
   end
 
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index 7a545f752b6..37374044551 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -4,12 +4,16 @@ class Namespace < ActiveRecord::Base
   include CacheMarkdownField
   include Sortable
   include Gitlab::ShellAdapter
+  include Routable
 
   cache_markdown_field :description, pipeline: :description
 
   has_many :projects, dependent: :destroy
   belongs_to :owner, class_name: "User"
 
+  belongs_to :parent, class_name: "Namespace"
+  has_many :children, class_name: "Namespace", foreign_key: :parent_id
+
   validates :owner, presence: true, unless: ->(n) { n.type == "Group" }
   validates :name,
     presence: true,
@@ -86,7 +90,7 @@ class Namespace < ActiveRecord::Base
   end
 
   def to_param
-    path
+    full_path
   end
 
   def human_name
@@ -150,6 +154,14 @@ class Namespace < ActiveRecord::Base
     Gitlab.config.lfs.enabled
   end
 
+  def full_path
+    if parent
+      parent.full_path + '/' + path
+    else
+      path
+    end
+  end
+
   private
 
   def repository_storage_paths
@@ -185,4 +197,8 @@ class Namespace < ActiveRecord::Base
       where(projects: { namespace_id: id }).
       find_each(&:refresh_members_authorized_projects)
   end
+
+  def full_path_changed?
+    path_changed? || parent_id_changed?
+  end
 end
diff --git a/app/models/project.rb b/app/models/project.rb
index 590885c0177..77d740081c6 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -14,6 +14,7 @@ class Project < ActiveRecord::Base
   include TokenAuthenticatable
   include ProjectFeaturesCompatibility
   include SelectForProjectAuthorization
+  include Routable
 
   extend Gitlab::ConfigHelper
 
@@ -324,87 +325,6 @@ class Project < ActiveRecord::Base
       non_archived.where(table[:name].matches(pattern))
     end
 
-    # Finds a single project for the given path.
-    #
-    # path - The full project path (including namespace path).
-    #
-    # Returns a Project, or nil if no project could be found.
-    def find_with_namespace(path)
-      namespace_path, project_path = path.split('/', 2)
-
-      return unless namespace_path && project_path
-
-      namespace_path = connection.quote(namespace_path)
-      project_path = connection.quote(project_path)
-
-      # On MySQL we want to ensure the ORDER BY uses a case-sensitive match so
-      # any literal matches come first, for this we have to use "BINARY".
-      # Without this there's still no guarantee in what order MySQL will return
-      # rows.
-      binary = Gitlab::Database.mysql? ? 'BINARY' : ''
-
-      order_sql = "(CASE WHEN #{binary} namespaces.path = #{namespace_path} " \
-        "AND #{binary} projects.path = #{project_path} THEN 0 ELSE 1 END)"
-
-      where_paths_in([path]).reorder(order_sql).take
-    end
-
-    # Builds a relation to find multiple projects by their full paths.
-    #
-    # Each path must be in the following format:
-    #
-    #     namespace_path/project_path
-    #
-    # For example:
-    #
-    #     gitlab-org/gitlab-ce
-    #
-    # Usage:
-    #
-    #     Project.where_paths_in(%w{gitlab-org/gitlab-ce gitlab-org/gitlab-ee})
-    #
-    # This would return the projects with the full paths matching the values
-    # given.
-    #
-    # paths - An Array of full paths (namespace path + project path) for which
-    #         to find the projects.
-    #
-    # Returns an ActiveRecord::Relation.
-    def where_paths_in(paths)
-      wheres = []
-      cast_lower = Gitlab::Database.postgresql?
-
-      paths.each do |path|
-        namespace_path, project_path = path.split('/', 2)
-
-        next unless namespace_path && project_path
-
-        namespace_path = connection.quote(namespace_path)
-        project_path = connection.quote(project_path)
-
-        where = "(namespaces.path = #{namespace_path}
-          AND projects.path = #{project_path})"
-
-        if cast_lower
-          where = "(
-            #{where}
-            OR (
-              LOWER(namespaces.path) = LOWER(#{namespace_path})
-              AND LOWER(projects.path) = LOWER(#{project_path})
-            )
-          )"
-        end
-
-        wheres << where
-      end
-
-      if wheres.empty?
-        none
-      else
-        joins(:namespace).where(wheres.join(' OR '))
-      end
-    end
-
     def visibility_levels
       Gitlab::VisibilityLevel.options
     end
@@ -440,6 +360,10 @@ class Project < ActiveRecord::Base
     def group_ids
       joins(:namespace).where(namespaces: { type: 'Group' }).select(:namespace_id)
     end
+
+    # Add alias for Routable method for compatibility with old code.
+    # In future all calls `find_with_namespace` should be replaced with `find_by_full_path`
+    alias_method :find_with_namespace, :find_by_full_path
   end
 
   def lfs_enabled?
@@ -879,13 +803,14 @@ class Project < ActiveRecord::Base
   end
   alias_method :human_name, :name_with_namespace
 
-  def path_with_namespace
-    if namespace
-      namespace.path + '/' + path
+  def full_path
+    if namespace && path
+      namespace.full_path + '/' + path
     else
       path
     end
   end
+  alias_method :path_with_namespace, :full_path
 
   def execute_hooks(data, hooks_scope = :push_hooks)
     hooks.send(hooks_scope).each do |hook|
@@ -1373,4 +1298,8 @@ class Project < ActiveRecord::Base
   def validate_board_limit(board)
     raise BoardLimitExceeded, 'Number of permitted boards exceeded' if boards.size >= NUMBER_OF_PERMITTED_BOARDS
   end
+
+  def full_path_changed?
+    path_changed? || namespace_id_changed?
+  end
 end
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 4034a49ae63..44f66b89600 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -940,7 +940,7 @@ class Repository
       committer = user_to_committer(user)
 
       Rugged::Commit.create(rugged,
-        message: commit.revert_message,
+        message: commit.revert_message(user),
         author: committer,
         committer: committer,
         tree: revert_tree_id,
diff --git a/app/models/route.rb b/app/models/route.rb
new file mode 100644
index 00000000000..d40214b9da6
--- /dev/null
+++ b/app/models/route.rb
@@ -0,0 +1,22 @@
+class Route < ActiveRecord::Base
+  belongs_to :source, polymorphic: true
+
+  validates :source, presence: true
+
+  validates :path,
+    length: { within: 1..255 },
+    presence: true,
+    uniqueness: { case_sensitive: false }
+
+  after_update :rename_children, if: :path_changed?
+
+  def rename_children
+    # We update each row separately because MySQL does not have regexp_replace.
+    # rubocop:disable Rails/FindEach
+    Route.where('path LIKE ?', "#{path_was}%").each do |route|
+      # Note that update column skips validation and callbacks.
+      # We need this to avoid recursive call of rename_children method
+      route.update_column(:path, route.path.sub(path_was, path))
+    end
+  end
+end