Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (671 commits)
  Make rubocop happy
  Use guard clause
  Improve language
  Prettify
  Use temp branch
  Pass info about who started the job and which job triggered it
  Docs: add indexes for monitoring and performance monitoring
  clearer-documentation-on-inline-diffs
  Add docs for commit diff discussion in merge requests
  sorting for tags api
  Clear BatchLoader after each spec to prevent holding onto records longer than necessary
  Include project in BatchLoader key to prevent returning blobs for the wrong project
  moved lfs_blob_ids method into ExtractsPath module
  Converted JS modules into exported modules
  spec fixes
  Bump gitlab-shell version to 5.10.3
  Clear caches before updating MR diffs
  Use new Ruby version 2.4 in GitLab QA images
  moved lfs blob fetch from extractspath file
  Update GitLab QA dependencies
  ...

43 files changed, 750 insertions, 216 deletions

diff --git a/app/models/appearance.rb b/app/models/appearance.rb
index ff15689ecac..76cfe28742a 100644
--- a/app/models/appearance.rb
+++ b/app/models/appearance.rb
@@ -2,9 +2,8 @@ class Appearance < ActiveRecord::Base
   include CacheMarkdownField
 
   cache_markdown_field :description
+  cache_markdown_field :new_project_guidelines
 
-  validates :title,       presence: true
-  validates :description, presence: true
   validates :logo,        file_size: { maximum: 1.megabyte }
   validates :header_logo, file_size: { maximum: 1.megabyte }
 
diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb
index 3117c98c846..253e213af81 100644
--- a/app/models/application_setting.rb
+++ b/app/models/application_setting.rb
@@ -153,11 +153,10 @@ class ApplicationSetting < ActiveRecord::Base
             presence: true,
             numericality: { greater_than_or_equal_to: 0 }
 
-  validates :circuitbreaker_backoff_threshold,
-            :circuitbreaker_failure_count_threshold,
-            :circuitbreaker_failure_wait_time,
+  validates :circuitbreaker_failure_count_threshold,
             :circuitbreaker_failure_reset_time,
             :circuitbreaker_storage_timeout,
+            :circuitbreaker_check_interval,
             presence: true,
             numericality: { only_integer: true, greater_than_or_equal_to: 0 }
 
@@ -165,13 +164,6 @@ class ApplicationSetting < ActiveRecord::Base
             presence: true,
             numericality: { only_integer: true, greater_than_or_equal_to: 1 }
 
-  validates_each :circuitbreaker_backoff_threshold do |record, attr, value|
-    if value.to_i >= record.circuitbreaker_failure_count_threshold
-      record.errors.add(attr, _("The circuitbreaker backoff threshold should be "\
-                                "lower than the failure count threshold"))
-    end
-  end
-
   validates :gitaly_timeout_default,
             presence: true,
             numericality: { only_integer: true, greater_than_or_equal_to: 0 }
diff --git a/app/models/blob.rb b/app/models/blob.rb
index 29e762724e3..19ad110db58 100644
--- a/app/models/blob.rb
+++ b/app/models/blob.rb
@@ -77,9 +77,15 @@ class Blob < SimpleDelegator
   end
 
   def self.lazy(project, commit_id, path)
-    BatchLoader.for(commit_id: commit_id, path: path).batch do |items, loader|
-      project.repository.blobs_at(items.map(&:values)).each do |blob|
-        loader.call({ commit_id: blob.commit_id, path: blob.path }, blob) if blob
+    BatchLoader.for({ project: project, commit_id: commit_id, path: path }).batch do |items, loader|
+      items_by_project = items.group_by { |i| i[:project] }
+
+      items_by_project.each do |project, items|
+        items = items.map { |i| i.values_at(:commit_id, :path) }
+
+        project.repository.blobs_at(items).each do |blob|
+          loader.call({ project: blob.project, commit_id: blob.commit_id, path: blob.path }, blob) if blob
+        end
       end
     end
   end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index 4ea040dfad5..83fe23606d1 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -1,18 +1,26 @@
 module Ci
   class Build < CommitStatus
+    prepend ArtifactMigratable
     include TokenAuthenticatable
     include AfterCommitQueue
     include Presentable
     include Importable
 
+    MissingDependenciesError = Class.new(StandardError)
+
     belongs_to :runner
     belongs_to :trigger_request
     belongs_to :erased_by, class_name: 'User'
 
     has_many :deployments, as: :deployable
+
     has_one :last_deployment, -> { order('deployments.id DESC') }, as: :deployable, class_name: 'Deployment'
     has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
 
+    has_many :job_artifacts, class_name: 'Ci::JobArtifact', foreign_key: :job_id, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+    has_one :job_artifacts_archive, -> { where(file_type: Ci::JobArtifact.file_types[:archive]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+    has_one :job_artifacts_metadata, -> { where(file_type: Ci::JobArtifact.file_types[:metadata]) }, class_name: 'Ci::JobArtifact', inverse_of: :job, foreign_key: :job_id
+
     # The "environment" field for builds is a String, and is the unexpanded name
     def persisted_environment
       @persisted_environment ||= Environment.find_by(
@@ -31,15 +39,37 @@ module Ci
 
     scope :unstarted, ->() { where(runner_id: nil) }
     scope :ignore_failures, ->() { where(allow_failure: false) }
-    scope :with_artifacts, ->() { where.not(artifacts_file: [nil, '']) }
+    scope :with_artifacts, ->() do
+      where('(artifacts_file IS NOT NULL AND artifacts_file <> ?) OR EXISTS (?)',
+        '', Ci::JobArtifact.select(1).where('ci_builds.id = ci_job_artifacts.job_id'))
+    end
     scope :with_artifacts_not_expired, ->() { with_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.now) }
     scope :with_expired_artifacts, ->() { with_artifacts.where('artifacts_expire_at < ?', Time.now) }
     scope :last_month, ->() { where('created_at > ?', Date.today - 1.month) }
     scope :manual_actions, ->() { where(when: :manual, status: COMPLETED_STATUSES + [:manual]) }
     scope :ref_protected, -> { where(protected: true) }
 
-    mount_uploader :artifacts_file, ArtifactUploader
-    mount_uploader :artifacts_metadata, ArtifactUploader
+    scope :matches_tag_ids, -> (tag_ids) do
+      matcher = ::ActsAsTaggableOn::Tagging
+        .where(taggable_type: CommitStatus)
+        .where(context: 'tags')
+        .where('taggable_id = ci_builds.id')
+        .where.not(tag_id: tag_ids).select('1')
+
+      where("NOT EXISTS (?)", matcher)
+    end
+
+    scope :with_any_tags, -> do
+      matcher = ::ActsAsTaggableOn::Tagging
+        .where(taggable_type: CommitStatus)
+        .where(context: 'tags')
+        .where('taggable_id = ci_builds.id').select('1')
+
+      where("EXISTS (?)", matcher)
+    end
+
+    mount_uploader :legacy_artifacts_file, LegacyArtifactUploader, mount_on: :artifacts_file
+    mount_uploader :legacy_artifacts_metadata, LegacyArtifactUploader, mount_on: :artifacts_metadata
 
     acts_as_taggable
 
@@ -111,6 +141,10 @@ module Ci
           Ci::Build.retry(build, build.user)
         end
       end
+
+      before_transition any => [:running] do |build|
+        build.validates_dependencies! unless Feature.enabled?('ci_disable_validates_dependencies')
+      end
     end
 
     def detailed_status(current_user)
@@ -326,14 +360,6 @@ module Ci
       project.running_or_pending_build_count(force: true)
     end
 
-    def artifacts?
-      !artifacts_expired? && artifacts_file.exists?
-    end
-
-    def artifacts_metadata?
-      artifacts? && artifacts_metadata.exists?
-    end
-
     def artifacts_metadata_entry(path, **options)
       metadata = Gitlab::Ci::Build::Artifacts::Metadata.new(
         artifacts_metadata.path,
@@ -386,6 +412,7 @@ module Ci
 
     def keep_artifacts!
       self.update(artifacts_expire_at: nil)
+      self.job_artifacts.update_all(expire_at: nil)
     end
 
     def coverage_regex
@@ -457,6 +484,19 @@ module Ci
       options[:dependencies]&.empty?
     end
 
+    def validates_dependencies!
+      dependencies.each do |dependency|
+        raise MissingDependenciesError unless dependency.valid_dependency?
+      end
+    end
+
+    def valid_dependency?
+      return false if artifacts_expired?
+      return false if erased?
+
+      true
+    end
+
     def hide_secrets(trace)
       return unless trace
 
@@ -473,11 +513,7 @@ module Ci
     private
 
     def update_artifacts_size
-      self.artifacts_size = if artifacts_file.exists?
-                              artifacts_file.size
-                            else
-                              nil
-                            end
+      self.artifacts_size = legacy_artifacts_file&.size
     end
 
     def erase_trace!
diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb
new file mode 100644
index 00000000000..84fc6863567
--- /dev/null
+++ b/app/models/ci/job_artifact.rb
@@ -0,0 +1,36 @@
+module Ci
+  class JobArtifact < ActiveRecord::Base
+    extend Gitlab::Ci::Model
+
+    belongs_to :project
+    belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id
+
+    before_save :set_size, if: :file_changed?
+
+    mount_uploader :file, JobArtifactUploader
+
+    enum file_type: {
+      archive: 1,
+      metadata: 2
+    }
+
+    def self.artifacts_size_for(project)
+      self.where(project: project).sum(:size)
+    end
+
+    def set_size
+      self.size = file.size
+    end
+
+    def expire_in
+      expire_at - Time.now if expire_at
+    end
+
+    def expire_in=(value)
+      self.expire_at =
+        if value
+          ChronicDuration.parse(value)&.seconds&.from_now
+        end
+    end
+  end
+end
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index ebbefc51a4f..eebbf7c4218 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -40,7 +40,6 @@ module Ci
     validates :status, presence: { unless: :importing? }
     validate :valid_commit_sha, unless: :importing?
 
-    after_initialize :set_config_source, if: :new_record?
     after_create :keep_around_commits, unless: :importing?
 
     enum source: {
@@ -365,7 +364,7 @@ module Ci
     end
 
     def has_kubernetes_active?
-      project.kubernetes_service&.active?
+      project.deployment_platform&.active?
     end
 
     def has_stage_seeds?
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index d39610a8995..dcbb397fb78 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -112,7 +112,7 @@ module Ci
     def can_pick?(build)
       return false if self.ref_protected? && !build.protected?
 
-      assignable_for?(build.project) && accepting_tags?(build)
+      assignable_for?(build.project_id) && accepting_tags?(build)
     end
 
     def only_for?(project)
@@ -171,8 +171,8 @@ module Ci
       end
     end
 
-    def assignable_for?(project)
-      is_shared? || projects.exists?(id: project.id)
+    def assignable_for?(project_id)
+      is_shared? || projects.exists?(id: project_id)
     end
 
     def accepting_tags?(build)
diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb
index 185d9473aab..55419189282 100644
--- a/app/models/clusters/cluster.rb
+++ b/app/models/clusters/cluster.rb
@@ -17,8 +17,7 @@ module Clusters
     # we force autosave to happen when we save `Cluster` model
     has_one :provider_gcp, class_name: 'Clusters::Providers::Gcp', autosave: true
 
-    # We have to ":destroy" it today to ensure that we clean also the Kubernetes Integration
-    has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
+    has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', autosave: true
 
     has_one :application_helm, class_name: 'Clusters::Applications::Helm'
     has_one :application_ingress, class_name: 'Clusters::Applications::Ingress'
@@ -29,15 +28,9 @@ module Clusters
     validates :name, cluster_name: true
     validate :restrict_modification, on: :update
 
-    # TODO: Move back this into Clusters::Platforms::Kubernetes in 10.3
-    # We need callback here because `enabled` belongs to Clusters::Cluster
-    # Callbacks in Clusters::Platforms::Kubernetes will not be called after update
-    after_save :update_kubernetes_integration!
-
     delegate :status, to: :provider, allow_nil: true
     delegate :status_reason, to: :provider, allow_nil: true
     delegate :on_creation?, to: :provider, allow_nil: true
-    delegate :update_kubernetes_integration!, to: :platform, allow_nil: true
 
     delegate :active?, to: :platform_kubernetes, prefix: true, allow_nil: true
     delegate :installed?, to: :application_helm, prefix: true, allow_nil: true
@@ -62,6 +55,10 @@ module Clusters
       end
     end
 
+    def created?
+      status_name == :created
+    end
+
     def applications
       [
         application_helm || build_application_helm,
@@ -77,6 +74,10 @@ module Clusters
       return platform_kubernetes if kubernetes?
     end
 
+    def managed?
+      !user?
+    end
+
     def first_project
       return @first_project if defined?(@first_project)
 
diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb
index 6dc1ee810d3..9160a169452 100644
--- a/app/models/clusters/platforms/kubernetes.rb
+++ b/app/models/clusters/platforms/kubernetes.rb
@@ -1,7 +1,12 @@
 module Clusters
   module Platforms
     class Kubernetes < ActiveRecord::Base
+      include Gitlab::CurrentSettings
+      include Gitlab::Kubernetes
+      include ReactiveCaching
+
       self.table_name = 'cluster_platforms_kubernetes'
+      self.reactive_cache_key = ->(kubernetes) { [kubernetes.class.model_name.singular, kubernetes.id] }
 
       belongs_to :cluster, inverse_of: :platform_kubernetes, class_name: 'Clusters::Cluster'
 
@@ -29,19 +34,17 @@ module Clusters
       validates :api_url, url: true, presence: true
       validates :token, presence: true
 
-      # TODO: Glue code till we migrate Kubernetes Integration into Platforms::Kubernetes
-      after_destroy :destroy_kubernetes_integration!
+      validate :prevent_modification, on: :update
+
+      after_save :clear_reactive_cache!
 
       alias_attribute :ca_pem, :ca_cert
 
       delegate :project, to: :cluster, allow_nil: true
       delegate :enabled?, to: :cluster, allow_nil: true
+      delegate :managed?, to: :cluster, allow_nil: true
 
-      class << self
-        def namespace_for_project(project)
-          "#{project.path}-#{project.id}"
-        end
-      end
+      alias_method :active?, :enabled?
 
       def actual_namespace
         if namespace.present?
@@ -51,58 +54,138 @@ module Clusters
         end
       end
 
-      def default_namespace
-        self.class.namespace_for_project(project) if project
+      def predefined_variables
+        config = YAML.dump(kubeconfig)
+
+        variables = [
+          { key: 'KUBE_URL', value: api_url, public: true },
+          { key: 'KUBE_TOKEN', value: token, public: false },
+          { key: 'KUBE_NAMESPACE', value: actual_namespace, public: true },
+          { key: 'KUBECONFIG', value: config, public: false, file: true }
+        ]
+
+        if ca_pem.present?
+          variables << { key: 'KUBE_CA_PEM', value: ca_pem, public: true }
+          variables << { key: 'KUBE_CA_PEM_FILE', value: ca_pem, public: true, file: true }
+        end
+
+        variables
       end
 
-      def kubeclient
-        @kubeclient ||= kubernetes_service.kubeclient if manages_kubernetes_service?
+      # Constructs a list of terminals from the reactive cache
+      #
+      # Returns nil if the cache is empty, in which case you should try again a
+      # short time later
+      def terminals(environment)
+        with_reactive_cache do |data|
+          pods = filter_by_label(data[:pods], app: environment.slug)
+          terminals = pods.flat_map { |pod| terminals_for_pod(api_url, actual_namespace, pod) }
+          terminals.each { |terminal| add_terminal_auth(terminal, terminal_auth) }
+        end
       end
 
-      def update_kubernetes_integration!
-        raise 'Kubernetes service already configured' unless manages_kubernetes_service?
+      # Caches resources in the namespace so other calls don't need to block on
+      # network access
+      def calculate_reactive_cache
+        return unless enabled? && project && !project.pending_delete?
 
-        # This is neccesary, otheriwse enabled? returns true even though cluster updated with enabled: false
-        cluster.reload
+        # We may want to cache extra things in the future
+        { pods: read_pods }
+      end
+
+      def kubeclient
+        @kubeclient ||= build_kubeclient!
+      end
+
+      private
 
-        ensure_kubernetes_service&.update!(
-          active: enabled?,
-          api_url: api_url,
-          namespace: namespace,
+      def kubeconfig
+        to_kubeconfig(
+          url: api_url,
+          namespace: actual_namespace,
           token: token,
-          ca_pem: ca_cert
-        )
+          ca_pem: ca_pem)
       end
 
-      def active?
-        manages_kubernetes_service?
+      def default_namespace
+        return unless project
+
+        slug = "#{project.path}-#{project.id}".downcase
+        slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '')
       end
 
-      private
+      def build_kubeclient!(api_path: 'api', api_version: 'v1')
+        raise "Incomplete settings" unless api_url && actual_namespace
 
-      def enforce_namespace_to_lower_case
-        self.namespace = self.namespace&.downcase
+        unless (username && password) || token
+          raise "Either username/password or token is required to access API"
+        end
+
+        ::Kubeclient::Client.new(
+          join_api_url(api_path),
+          api_version,
+          auth_options: kubeclient_auth_options,
+          ssl_options: kubeclient_ssl_options,
+          http_proxy_uri: ENV['http_proxy']
+        )
       end
 
-      # TODO: glue code till we migrate Kubernetes Service into Platforms::Kubernetes class
-      def manages_kubernetes_service?
-        return true unless kubernetes_service&.active?
+      # Returns a hash of all pods in the namespace
+      def read_pods
+        kubeclient = build_kubeclient!
+
+        kubeclient.get_pods(namespace: actual_namespace).as_json
+      rescue KubeException => err
+        raise err unless err.error_code == 404
+
+        []
+      end
+
+      def kubeclient_ssl_options
+        opts = { verify_ssl: OpenSSL::SSL::VERIFY_PEER }
+
+        if ca_pem.present?
+          opts[:cert_store] = OpenSSL::X509::Store.new
+          opts[:cert_store].add_cert(OpenSSL::X509::Certificate.new(ca_pem))
+        end
+
+        opts
+      end
 
-        kubernetes_service.api_url == api_url
+      def kubeclient_auth_options
+        { bearer_token: token }
       end
 
-      def destroy_kubernetes_integration!
-        return unless manages_kubernetes_service?
+      def join_api_url(api_path)
+        url = URI.parse(api_url)
+        prefix = url.path.sub(%r{/+\z}, '')
+
+        url.path = [prefix, api_path].join("/")
 
-        kubernetes_service&.destroy!
+        url.to_s
       end
 
-      def kubernetes_service
-        @kubernetes_service ||= project&.kubernetes_service
+      def terminal_auth
+        {
+          token: token,
+          ca_pem: ca_pem,
+          max_session_time: current_application_settings.terminal_max_session_time
+        }
       end
 
-      def ensure_kubernetes_service
-        @kubernetes_service ||= kubernetes_service || project&.build_kubernetes_service
+      def enforce_namespace_to_lower_case
+        self.namespace = self.namespace&.downcase
+      end
+
+      def prevent_modification
+        return unless managed?
+
+        if api_url_changed? || token_changed? || ca_pem_changed?
+          errors.add(:base, "cannot modify managed cluster")
+          return false
+        end
+
+        true
       end
     end
   end
diff --git a/app/models/commit.rb b/app/models/commit.rb
index 6b28d290f99..13c31111134 100644
--- a/app/models/commit.rb
+++ b/app/models/commit.rb
@@ -1,3 +1,4 @@
+# coding: utf-8
 class Commit
   extend ActiveModel::Naming
   extend Gitlab::Cache::RequestCache
@@ -25,7 +26,7 @@ class Commit
   DIFF_HARD_LIMIT_FILES = 1000
   DIFF_HARD_LIMIT_LINES = 50000
 
-  MIN_SHA_LENGTH = 7
+  MIN_SHA_LENGTH = Gitlab::Git::Commit::MIN_SHA_LENGTH
   COMMIT_SHA_PATTERN = /\h{#{MIN_SHA_LENGTH},40}/.freeze
 
   def banzai_render_context(field)
@@ -51,6 +52,20 @@ class Commit
       diffs.reduce(0) { |sum, d| sum + Gitlab::Git::Util.count_lines(d.diff) }
     end
 
+    def order_by(collection:, order_by:, sort:)
+      return collection unless %w[email name commits].include?(order_by)
+      return collection unless %w[asc desc].include?(sort)
+
+      collection.sort do |a, b|
+        operands = [a, b].tap { |o| o.reverse! if sort == 'desc' }
+
+        attr1, attr2 = operands.first.public_send(order_by), operands.second.public_send(order_by) # rubocop:disable PublicSend
+
+        # use case insensitive comparison for string values
+        order_by.in?(%w[email name]) ? attr1.casecmp(attr2) : attr1 <=> attr2
+      end
+    end
+
     # Truncate sha to 8 characters
     def truncate_sha(sha)
       sha[0..MIN_SHA_LENGTH]
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index ee21ed8e420..c0263c0b4e2 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -43,7 +43,8 @@ class CommitStatus < ActiveRecord::Base
     script_failure: 1,
     api_failure: 2,
     stuck_or_timeout_failure: 3,
-    runner_system_failure: 4
+    runner_system_failure: 4,
+    missing_dependency_failure: 5
   }
 
   ##
diff --git a/app/models/concerns/artifact_migratable.rb b/app/models/concerns/artifact_migratable.rb
new file mode 100644
index 00000000000..0460439e9e6
--- /dev/null
+++ b/app/models/concerns/artifact_migratable.rb
@@ -0,0 +1,45 @@
+# Adapter class to unify the interface between mounted uploaders and the
+# Ci::Artifact model
+# Meant to be prepended so the interface can stay the same
+module ArtifactMigratable
+  def artifacts_file
+    job_artifacts_archive&.file || legacy_artifacts_file
+  end
+
+  def artifacts_metadata
+    job_artifacts_metadata&.file || legacy_artifacts_metadata
+  end
+
+  def artifacts?
+    !artifacts_expired? && artifacts_file.exists?
+  end
+
+  def artifacts_metadata?
+    artifacts? && artifacts_metadata.exists?
+  end
+
+  def artifacts_file_changed?
+    job_artifacts_archive&.file_changed? || attribute_changed?(:artifacts_file)
+  end
+
+  def remove_artifacts_file!
+    if job_artifacts_archive
+      job_artifacts_archive.destroy
+    else
+      remove_legacy_artifacts_file!
+    end
+  end
+
+  def remove_artifacts_metadata!
+    if job_artifacts_metadata
+      job_artifacts_metadata.destroy
+    else
+      remove_legacy_artifacts_metadata!
+    end
+  end
+
+  def artifacts_size
+    read_attribute(:artifacts_size).to_i +
+      job_artifacts_archive&.size.to_i + job_artifacts_metadata&.size.to_i
+  end
+end
diff --git a/app/models/concerns/bulk_member_access_load.rb b/app/models/concerns/bulk_member_access_load.rb
new file mode 100644
index 00000000000..984c4f53bf7
--- /dev/null
+++ b/app/models/concerns/bulk_member_access_load.rb
@@ -0,0 +1,46 @@
+# Returns and caches in thread max member access for a resource
+#
+module BulkMemberAccessLoad
+  extend ActiveSupport::Concern
+
+  included do
+    # Determine the maximum access level for a group of resources in bulk.
+    #
+    # Returns a Hash mapping resource ID -> maximum access level.
+    def max_member_access_for_resource_ids(resource_klass, resource_ids, memoization_index = self.id, &block)
+      raise 'Block is mandatory' unless block_given?
+
+      resource_ids = resource_ids.uniq
+      key = max_member_access_for_resource_key(resource_klass, memoization_index)
+      access = {}
+
+      if RequestStore.active?
+        RequestStore.store[key] ||= {}
+        access = RequestStore.store[key]
+      end
+
+      # Look up only the IDs we need
+      resource_ids = resource_ids - access.keys
+
+      return access if resource_ids.empty?
+
+      resource_access = yield(resource_ids)
+
+      access.merge!(resource_access)
+
+      missing_resource_ids = resource_ids - resource_access.keys
+
+      missing_resource_ids.each do |resource_id|
+        access[resource_id] = Gitlab::Access::NO_ACCESS
+      end
+
+      access
+    end
+
+    private
+
+    def max_member_access_for_resource_key(klass, memoization_index)
+      "max_member_access_for_#{klass.name.underscore.pluralize}:#{memoization_index}"
+    end
+  end
+end
diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb
index 98776eab424..90ad644ce34 100644
--- a/app/models/concerns/cache_markdown_field.rb
+++ b/app/models/concerns/cache_markdown_field.rb
@@ -85,8 +85,7 @@ module CacheMarkdownField
   def cached_html_up_to_date?(markdown_field)
     html_field = cached_markdown_fields.html_field(markdown_field)
 
-    cached = cached_html_for(markdown_field).present? && __send__(markdown_field).present? # rubocop:disable GitlabSecurity/PublicSend
-    return false unless cached
+    return false if cached_html_for(markdown_field).nil? && !__send__(markdown_field).nil? # rubocop:disable GitlabSecurity/PublicSend
 
     markdown_changed = attribute_changed?(markdown_field) || false
     html_changed = attribute_changed?(html_field) || false
diff --git a/app/models/concerns/discussion_on_diff.rb b/app/models/concerns/discussion_on_diff.rb
index f5cbb3becad..4b4d519f3df 100644
--- a/app/models/concerns/discussion_on_diff.rb
+++ b/app/models/concerns/discussion_on_diff.rb
@@ -32,6 +32,10 @@ module DiscussionOnDiff
     first_note.position.new_path
   end
 
+  def on_merge_request_commit?
+    for_merge_request? && commit_id.present?
+  end
+
   # Returns an array of at most 16 highlighted lines above a diff note
   def truncated_diff_lines(highlight: true)
     lines = highlight ? highlighted_diff_lines : diff_lines
diff --git a/app/models/concerns/throttled_touch.rb b/app/models/concerns/throttled_touch.rb
new file mode 100644
index 00000000000..ad0ff0f20d4
--- /dev/null
+++ b/app/models/concerns/throttled_touch.rb
@@ -0,0 +1,10 @@
+# ThrottledTouch can be used to throttle the number of updates triggered by
+# calling "touch" on an ActiveRecord model.
+module ThrottledTouch
+  # The amount of time to wait before "touch" can update a record again.
+  TOUCH_INTERVAL = 1.minute
+
+  def touch(*args)
+    super if (Time.zone.now - updated_at) > TOUCH_INTERVAL
+  end
+end
diff --git a/app/models/diff_discussion.rb b/app/models/diff_discussion.rb
index 6eba87da1a1..4a65738214b 100644
--- a/app/models/diff_discussion.rb
+++ b/app/models/diff_discussion.rb
@@ -24,7 +24,11 @@ class DiffDiscussion < Discussion
     return unless for_merge_request?
     return {} if active?
 
-    noteable.version_params_for(position.diff_refs)
+    if on_merge_request_commit?
+      { commit_id: commit_id }
+    else
+      noteable.version_params_for(position.diff_refs)
+    end
   end
 
   def reply_attributes
diff --git a/app/models/diff_note.rb b/app/models/diff_note.rb
index ae5f138a920..b53d44cda95 100644
--- a/app/models/diff_note.rb
+++ b/app/models/diff_note.rb
@@ -17,6 +17,7 @@ class DiffNote < Note
   validates :noteable_type, inclusion: { in: NOTEABLE_TYPES }
   validate :positions_complete
   validate :verify_supported
+  validate :diff_refs_match_commit, if: :for_commit?
 
   before_validation :set_original_position, on: :create
   before_validation :update_position, on: :create, if: :on_text?
@@ -135,6 +136,12 @@ class DiffNote < Note
     errors.add(:position, "is invalid")
   end
 
+  def diff_refs_match_commit
+    return if self.original_position.diff_refs == self.commit.diff_refs
+
+    errors.add(:commit_id, 'does not match the diff refs')
+  end
+
   def keep_around_commits
     project.repository.keep_around(self.original_position.base_sha)
     project.repository.keep_around(self.original_position.start_sha)
diff --git a/app/models/discussion.rb b/app/models/discussion.rb
index 437df923d2d..92482a1a875 100644
--- a/app/models/discussion.rb
+++ b/app/models/discussion.rb
@@ -11,6 +11,7 @@ class Discussion
             :author,
 
             :noteable,
+            :commit_id,
             :for_commit?,
             :for_merge_request?,
 
diff --git a/app/models/environment.rb b/app/models/environment.rb
index 21a028e351c..bf69b4c50f0 100644
--- a/app/models/environment.rb
+++ b/app/models/environment.rb
@@ -138,11 +138,11 @@ class Environment < ActiveRecord::Base
   end
 
   def has_terminals?
-    project.deployment_service.present? && available? && last_deployment.present?
+    project.deployment_platform.present? && available? && last_deployment.present?
   end
 
   def terminals
-    project.deployment_service.terminals(self) if has_terminals?
+    project.deployment_platform.terminals(self) if has_terminals?
   end
 
   def has_metrics?
diff --git a/app/models/epic.rb b/app/models/epic.rb
index 62898a02e2d..286b855de3f 100644
--- a/app/models/epic.rb
+++ b/app/models/epic.rb
@@ -1,7 +1,11 @@
 # Placeholder class for model that is implemented in EE
-# It will reserve (ee#3853) '&' as a reference prefix, but the table does not exists in CE
+# It reserves '&' as a reference prefix, but the table does not exists in CE
 class Epic < ActiveRecord::Base
-  # TODO: this will be implemented as part of #3853
-  def to_reference
+  def self.reference_prefix
+    '&'
+  end
+
+  def self.reference_prefix_escaped
+    '&amp;'
   end
 end
diff --git a/app/models/event.rb b/app/models/event.rb
index 0997b056c6a..6053594fab5 100644
--- a/app/models/event.rb
+++ b/app/models/event.rb
@@ -72,7 +72,7 @@ class Event < ActiveRecord::Base
     # We're using preload for "push_event_payload" as otherwise the association
     # is not always available (depending on the query being built).
     includes(:author, :project, project: :namespace)
-      .preload(:target, :push_event_payload)
+      .preload(:push_event_payload, target: :author)
   end
 
   scope :for_milestone_id, ->(milestone_id) { where(target_type: "Milestone", target_id: milestone_id) }
diff --git a/app/models/group.rb b/app/models/group.rb
index 76262acf50c..fddace03387 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -2,6 +2,7 @@ require 'carrierwave/orm/activerecord'
 
 class Group < Namespace
   include Gitlab::ConfigHelper
+  include AfterCommitQueue
   include AccessRequestable
   include Avatarable
   include Referable
@@ -289,6 +290,18 @@ class Group < Namespace
     "#{parent.full_path}/#{path_was}"
   end
 
+  def group_member(user)
+    if group_members.loaded?
+      group_members.find { |gm| gm.user_id == user.id }
+    else
+      group_members.find_by(user_id: user)
+    end
+  end
+
+  def hashed_storage?(_feature)
+    false
+  end
+
   private
 
   def update_two_factor_requirement
diff --git a/app/models/issue.rb b/app/models/issue.rb
index d6ef58d150b..dc64888b6fc 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -9,6 +9,10 @@ class Issue < ActiveRecord::Base
   include FasterCacheKeys
   include RelativePositioning
   include TimeTrackable
+  include ThrottledTouch
+  include IgnorableColumn
+
+  ignore_column :assignee_id, :branch_name
 
   DueDateStruct = Struct.new(:title, :name).freeze
   NoDueDate     = DueDateStruct.new('No Due Date', '0').freeze
diff --git a/app/models/key.rb b/app/models/key.rb
index 815fd1de909..a3f8a5d6dc7 100644
--- a/app/models/key.rb
+++ b/app/models/key.rb
@@ -2,6 +2,7 @@ require 'digest/md5'
 
 class Key < ActiveRecord::Base
   include Gitlab::CurrentSettings
+  include AfterCommitQueue
   include Sortable
 
   belongs_to :user
diff --git a/app/models/member.rb b/app/models/member.rb
index cbbd58f2eaf..c47145667b5 100644
--- a/app/models/member.rb
+++ b/app/models/member.rb
@@ -1,8 +1,10 @@
 class Member < ActiveRecord::Base
+  include AfterCommitQueue
   include Sortable
   include Importable
   include Expirable
   include Gitlab::Access
+  include Presentable
 
   attr_accessor :raw_invite_token
 
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index e232feaeada..c39789b047d 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -7,6 +7,8 @@ class MergeRequest < ActiveRecord::Base
   include TimeTrackable
   include ManualInverseAssociation
   include EachBatch
+  include ThrottledTouch
+  include Gitlab::Utils::StrongMemoize
 
   ignore_column :locked_at,
                 :ref_fetched
@@ -51,6 +53,7 @@ class MergeRequest < ActiveRecord::Base
   serialize :merge_params, Hash # rubocop:disable Cop/ActiveRecordSerialize
 
   after_create :ensure_merge_request_diff, unless: :importing?
+  after_update :clear_memoized_shas
   after_update :reload_diff_if_branch_changed
 
   # When this attribute is true some MR validation is ignored
@@ -82,6 +85,14 @@ class MergeRequest < ActiveRecord::Base
       transition locked: :opened
     end
 
+    before_transition any => :opened do |merge_request|
+      merge_request.merge_jid = nil
+
+      merge_request.run_after_commit do
+        UpdateHeadPipelineForMergeRequestWorker.perform_async(merge_request.id)
+      end
+    end
+
     state :opened
     state :closed
     state :merged
@@ -145,6 +156,13 @@ class MergeRequest < ActiveRecord::Base
     '!'
   end
 
+  # Use this method whenever you need to make sure the head_pipeline is synced with the
+  # branch head commit, for example checking if a merge request can be merged.
+  # For more information check: https://gitlab.com/gitlab-org/gitlab-ce/issues/40004
+  def actual_head_pipeline
+    head_pipeline&.sha == diff_head_sha ? head_pipeline : nil
+  end
+
   # Pattern used to extract `!123` merge request references from text
   #
   # This pattern supports cross-project references.
@@ -379,13 +397,17 @@ class MergeRequest < ActiveRecord::Base
   end
 
   def source_branch_head
-    return unless source_project
-
-    source_project.repository.commit(source_branch_ref) if source_branch_ref
+    strong_memoize(:source_branch_head) do
+      if source_project && source_branch_ref
+        source_project.repository.commit(source_branch_ref)
+      end
+    end
   end
 
   def target_branch_head
-    target_project.repository.commit(target_branch_ref)
+    strong_memoize(:target_branch_head) do
+      target_project.repository.commit(target_branch_ref)
+    end
   end
 
   def branch_merge_base_commit
@@ -517,6 +539,13 @@ class MergeRequest < ActiveRecord::Base
     end
   end
 
+  def clear_memoized_shas
+    @target_branch_sha = @source_branch_sha = nil
+
+    clear_memoization(:source_branch_head)
+    clear_memoization(:target_branch_head)
+  end
+
   def reload_diff_if_branch_changed
     if (source_branch_changed? || target_branch_changed?) &&
         (source_branch_head && target_branch_head)
@@ -641,6 +670,7 @@ class MergeRequest < ActiveRecord::Base
       .to_sql
 
     Note.from("(#{union}) #{Note.table_name}")
+      .includes(:noteable)
   end
 
   alias_method :discussion_notes, :related_notes
@@ -822,8 +852,9 @@ class MergeRequest < ActiveRecord::Base
 
   def mergeable_ci_state?
     return true unless project.only_allow_merge_if_pipeline_succeeds?
+    return true unless head_pipeline
 
-    !head_pipeline || head_pipeline.success? || head_pipeline.skipped?
+    actual_head_pipeline&.success? || actual_head_pipeline&.skipped?
   end
 
   def environments_for(current_user)
@@ -856,11 +887,11 @@ class MergeRequest < ActiveRecord::Base
 
   def state_icon_name
     if merged?
-      "check"
+      "git-merge"
     elsif closed?
-      "times"
+      "close"
     else
-      "circle-o"
+      "issue-open-m"
     end
   end
 
@@ -899,7 +930,8 @@ class MergeRequest < ActiveRecord::Base
   def compute_diverged_commits_count
     return 0 unless source_branch_sha && target_branch_sha
 
-    Gitlab::Git::Commit.between(target_project.repository.raw_repository, source_branch_sha, target_branch_sha).size
+    target_project.repository
+      .count_commits_between(source_branch_sha, target_branch_sha)
   end
   private :compute_diverged_commits_count
 
@@ -915,21 +947,27 @@ class MergeRequest < ActiveRecord::Base
       .order(id: :desc)
   end
 
-  # Note that this could also return SHA from now dangling commits
-  #
-  def all_commit_shas
-    return commit_shas unless persisted?
-
-    diffs_relation = merge_request_diffs
-
+  def all_commits
     # MySQL doesn't support LIMIT in a subquery.
-    diffs_relation = diffs_relation.recent if Gitlab::Database.postgresql?
+    diffs_relation = if Gitlab::Database.postgresql?
+                       merge_request_diffs.recent
+                     else
+                       merge_request_diffs
+                     end
 
     MergeRequestDiffCommit
       .where(merge_request_diff: diffs_relation)
       .limit(10_000)
-      .pluck('sha')
-      .uniq
+  end
+
+  # Note that this could also return SHA from now dangling commits
+  #
+  def all_commit_shas
+    @all_commit_shas ||= begin
+      return commit_shas unless persisted?
+
+      all_commits.pluck(:sha).uniq
+    end
   end
 
   def merge_commit
@@ -996,7 +1034,7 @@ class MergeRequest < ActiveRecord::Base
     return true if autocomplete_precheck
 
     return false unless mergeable?(skip_ci_check: true)
-    return false if head_pipeline && !(head_pipeline.success? || head_pipeline.active?)
+    return false if actual_head_pipeline && !(actual_head_pipeline.success? || actual_head_pipeline.active?)
     return false if last_diff_sha != diff_head_sha
 
     true
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index c37aa0a594b..e35de9b97ee 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -104,19 +104,19 @@ class MergeRequestDiff < ActiveRecord::Base
   def base_commit
     return unless base_commit_sha
 
-    project.commit(base_commit_sha)
+    project.commit_by(oid: base_commit_sha)
   end
 
   def start_commit
     return unless start_commit_sha
 
-    project.commit(start_commit_sha)
+    project.commit_by(oid: start_commit_sha)
   end
 
   def head_commit
     return unless head_commit_sha
 
-    project.commit(head_commit_sha)
+    project.commit_by(oid: head_commit_sha)
   end
 
   def commit_shas
diff --git a/app/models/milestone.rb b/app/models/milestone.rb
index c06ee8083f0..77c19380e66 100644
--- a/app/models/milestone.rb
+++ b/app/models/milestone.rb
@@ -84,6 +84,13 @@ class Milestone < ActiveRecord::Base
       else milestones.active
       end
     end
+
+    def predefined?(milestone)
+      milestone == Any ||
+        milestone == None ||
+        milestone == Upcoming ||
+        milestone == Started
+    end
   end
 
   def self.reference_prefix
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index fa76729a702..0ff169d4531 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -40,6 +40,7 @@ class Namespace < ActiveRecord::Base
     namespace_path: true
 
   validate :nesting_level_allowed
+  validate :allowed_path_by_redirects
 
   delegate :name, to: :owner, allow_nil: true, prefix: true
 
@@ -139,7 +140,17 @@ class Namespace < ActiveRecord::Base
   def find_fork_of(project)
     return nil unless project.fork_network
 
-    project.fork_network.find_forks_in(projects).first
+    if RequestStore.active?
+      forks_in_namespace = RequestStore.fetch("namespaces:#{id}:forked_projects") do
+        Hash.new do |found_forks, project|
+          found_forks[project] = project.fork_network.find_forks_in(projects).first
+        end
+      end
+
+      forks_in_namespace[project]
+    else
+      project.fork_network.find_forks_in(projects).first
+    end
   end
 
   def lfs_enabled?
@@ -247,4 +258,14 @@ class Namespace < ActiveRecord::Base
     Namespace.where(id: descendants.select(:id))
       .update_all(share_with_group_lock: true)
   end
+
+  def allowed_path_by_redirects
+    return if path.nil?
+
+    errors.add(:path, "#{path} has been taken before. Please use another one") if namespace_previously_created_with_same_path?
+  end
+
+  def namespace_previously_created_with_same_path?
+    RedirectRoute.permanent.exists?(path: path)
+  end
 end
diff --git a/app/models/note.rb b/app/models/note.rb
index 340fe087f82..184fbd5f5ae 100644
--- a/app/models/note.rb
+++ b/app/models/note.rb
@@ -15,6 +15,7 @@ class Note < ActiveRecord::Base
   include IgnorableColumn
   include Editable
   include Gitlab::SQL::Pattern
+  include ThrottledTouch
 
   module SpecialRole
     FIRST_TIME_CONTRIBUTOR = :first_time_contributor
@@ -55,7 +56,7 @@ class Note < ActiveRecord::Base
   participant :author
 
   belongs_to :project
-  belongs_to :noteable, polymorphic: true, touch: true # rubocop:disable Cop/PolymorphicAssociations
+  belongs_to :noteable, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations
   belongs_to :author, class_name: "User"
   belongs_to :updated_by, class_name: "User"
   belongs_to :last_edited_by, class_name: 'User'
@@ -118,6 +119,7 @@ class Note < ActiveRecord::Base
   before_validation :set_discussion_id, on: :create
   after_save :keep_around_commit, if: :for_project_noteable?
   after_save :expire_etag_cache
+  after_save :touch_noteable
   after_destroy :expire_etag_cache
 
   class << self
@@ -228,16 +230,18 @@ class Note < ActiveRecord::Base
     for_personal_snippet?
   end
 
+  def commit
+    @commit ||= project.commit(commit_id) if commit_id.present?
+  end
+
   # override to return commits, which are not active record
   def noteable
-    if for_commit?
-      @commit ||= project.commit(commit_id)
-    else
-      super
-    end
-  # Temp fix to prevent app crash
-  # if note commit id doesn't exist
+    return commit if for_commit?
+
+    super
   rescue
+    # Temp fix to prevent app crash
+    # if note commit id doesn't exist
     nil
   end
 
@@ -356,6 +360,16 @@ class Note < ActiveRecord::Base
     end
   end
 
+  def references
+    refs = [noteable]
+
+    if part_of_discussion?
+      refs += discussion.notes.take_while { |n| n.id < id }
+    end
+
+    refs
+  end
+
   def expire_etag_cache
     return unless noteable&.discussions_rendered_on_frontend?
 
@@ -367,6 +381,45 @@ class Note < ActiveRecord::Base
     Gitlab::EtagCaching::Store.new.touch(key)
   end
 
+  def touch(*args)
+    # We're not using an explicit transaction here because this would in all
+    # cases result in all future queries going to the primary, even if no writes
+    # are performed.
+    #
+    # We touch the noteable first so its SELECT query can run before our writes,
+    # ensuring it runs on a secondary (if no prior write took place).
+    touch_noteable
+    super
+  end
+
+  # By default Rails will issue an "SELECT *" for the relation, which is
+  # overkill for just updating the timestamps. To work around this we manually
+  # touch the data so we can SELECT only the columns we need.
+  def touch_noteable
+    # Commits are not stored in the DB so we can't touch them.
+    return if for_commit?
+
+    assoc = association(:noteable)
+
+    noteable_object =
+      if assoc.loaded?
+        noteable
+      else
+        # If the object is not loaded (e.g. when notes are loaded async) we
+        # _only_ want the data we actually need.
+        assoc.scope.select(:id, :updated_at).take
+      end
+
+    noteable_object&.touch
+
+    # We return the noteable object so we can re-use it in EE for ElasticSearch.
+    noteable_object
+  end
+
+  def banzai_render_context(field)
+    super.merge(noteable: noteable)
+  end
+
   private
 
   def keep_around_commit
diff --git a/app/models/personal_access_token.rb b/app/models/personal_access_token.rb
index cfcb03138b7..063dc521324 100644
--- a/app/models/personal_access_token.rb
+++ b/app/models/personal_access_token.rb
@@ -3,6 +3,8 @@ class PersonalAccessToken < ActiveRecord::Base
   include TokenAuthenticatable
   add_authentication_token_field :token
 
+  REDIS_EXPIRY_TIME = 3.minutes
+
   serialize :scopes, Array # rubocop:disable Cop/ActiveRecordSerialize
 
   belongs_to :user
@@ -27,6 +29,21 @@ class PersonalAccessToken < ActiveRecord::Base
     !revoked? && !expired?
   end
 
+  def self.redis_getdel(user_id)
+    Gitlab::Redis::SharedState.with do |redis|
+      token = redis.get(redis_shared_state_key(user_id))
+      redis.del(redis_shared_state_key(user_id))
+      token
+    end
+  end
+
+  def self.redis_store!(user_id, token)
+    Gitlab::Redis::SharedState.with do |redis|
+      redis.set(redis_shared_state_key(user_id), token, ex: REDIS_EXPIRY_TIME)
+      token
+    end
+  end
+
   protected
 
   def validate_scopes
@@ -38,4 +55,8 @@ class PersonalAccessToken < ActiveRecord::Base
   def set_default_scopes
     self.scopes = Gitlab::Auth::DEFAULT_SCOPES if self.scopes.empty?
   end
+
+  def self.redis_shared_state_key(user_id)
+    "gitlab:personal_access_token:#{user_id}"
+  end
 end
diff --git a/app/models/project.rb b/app/models/project.rb
index 5a3f591c2e7..5183a216c53 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -189,7 +189,6 @@ class Project < ActiveRecord::Base
   has_one :statistics, class_name: 'ProjectStatistics'
 
   has_one :cluster_project, class_name: 'Clusters::Project'
-  has_one :cluster, through: :cluster_project, class_name: 'Clusters::Cluster'
   has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster'
 
   # Container repositories need to remove data from the container registry,
@@ -228,14 +227,13 @@ class Project < ActiveRecord::Base
   delegate :members, to: :team, prefix: true
   delegate :add_user, :add_users, to: :team
   delegate :add_guest, :add_reporter, :add_developer, :add_master, to: :team
-  delegate :empty_repo?, to: :repository
 
   # Validations
   validates :creator, presence: true, on: :create
   validates :description, length: { maximum: 2000 }, allow_blank: true
   validates :ci_config_path,
-    format: { without: /\.{2}/,
-              message: 'cannot include directory traversal.' },
+    format: { without: /(\.{2}|\A\/)/,
+              message: 'cannot include leading slash or directory traversal.' },
     length: { maximum: 255 },
     allow_blank: true
   validates :name,
@@ -500,6 +498,10 @@ class Project < ActiveRecord::Base
     auto_devops&.enabled.nil? && !current_application_settings.auto_devops_enabled?
   end
 
+  def empty_repo?
+    repository.empty?
+  end
+
   def repository_storage_path
     Gitlab.config.repositories.storages[repository_storage].try(:[], 'path')
   end
@@ -562,8 +564,7 @@ class Project < ActiveRecord::Base
       if forked?
         RepositoryForkWorker.perform_async(id,
                                            forked_from_project.repository_storage_path,
-                                           forked_from_project.full_path,
-                                           self.namespace.full_path)
+                                           forked_from_project.disk_path)
       else
         RepositoryImportWorker.perform_async(self.id)
       end
@@ -599,7 +600,7 @@ class Project < ActiveRecord::Base
 
   def ci_config_path=(value)
     # Strip all leading slashes so that //foo -> foo
-    super(value&.sub(%r{\A/+}, '')&.delete("\0"))
+    super(value&.delete("\0"))
   end
 
   def import_url=(value)
@@ -658,7 +659,8 @@ class Project < ActiveRecord::Base
   end
 
   def import_started?
-    import? && import_status == 'started'
+    # import? does SQL work so only run it if it looks like there's an import running
+    import_status == 'started' && import?
   end
 
   def import_scheduled?
@@ -753,13 +755,14 @@ class Project < ActiveRecord::Base
     Gitlab::Routing.url_helpers.project_url(self)
   end
 
-  def new_issue_address(author)
+  def new_issuable_address(author, address_type)
     return unless Gitlab::IncomingEmail.supports_issue_creation? && author
 
     author.ensure_incoming_email_token!
 
+    suffix = address_type == 'merge_request' ? '+merge-request' : ''
     Gitlab::IncomingEmail.reply_address(
-      "#{full_path}+#{author.incoming_email_token}")
+      "#{full_path}#{suffix}+#{author.incoming_email_token}")
   end
 
   def build_commit_note(commit)
@@ -897,12 +900,10 @@ class Project < ActiveRecord::Base
     @ci_service ||= ci_services.reorder(nil).find_by(active: true)
   end
 
-  def deployment_services
-    services.where(category: :deployment)
-  end
-
-  def deployment_service
-    @deployment_service ||= deployment_services.reorder(nil).find_by(active: true)
+  # TODO: This will be extended for multiple enviroment clusters
+  def deployment_platform
+    @deployment_platform ||= clusters.find_by(enabled: true)&.platform_kubernetes
+    @deployment_platform ||= services.where(category: :deployment).reorder(nil).find_by(active: true)
   end
 
   def monitoring_services
@@ -1116,7 +1117,11 @@ class Project < ActiveRecord::Base
   end
 
   def project_member(user)
-    project_members.find_by(user_id: user)
+    if project_members.loaded?
+      project_members.find { |member| member.user_id == user.id }
+    else
+      project_members.find_by(user_id: user)
+    end
   end
 
   def default_branch
@@ -1143,7 +1148,7 @@ class Project < ActiveRecord::Base
   def change_head(branch)
     if repository.branch_exists?(branch)
       repository.before_change_head
-      repository.write_ref('HEAD', "refs/heads/#{branch}")
+      repository.write_ref('HEAD', "refs/heads/#{branch}", force: true)
       repository.copy_gitattributes(branch)
       repository.after_change_head
       reload_default_branch
@@ -1547,9 +1552,9 @@ class Project < ActiveRecord::Base
   end
 
   def deployment_variables
-    return [] unless deployment_service
+    return [] unless deployment_platform
 
-    deployment_service.predefined_variables
+    deployment_platform.predefined_variables
   end
 
   def auto_devops_variables
diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb
index bc62972dbb0..b82567ce2b3 100644
--- a/app/models/project_services/kubernetes_service.rb
+++ b/app/models/project_services/kubernetes_service.rb
@@ -1,3 +1,8 @@
+##
+# NOTE:
+# We'll move this class to Clusters::Platforms::Kubernetes, which contains exactly the same logic.
+# After we've migrated data, we'll remove KubernetesService. This would happen in a few months.
+# If you're modyfiyng this class, please note that you should update the same change in Clusters::Platforms::Kubernetes.
 class KubernetesService < DeploymentService
   include Gitlab::CurrentSettings
   include Gitlab::Kubernetes
diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb
index 715b215d1db..17b9d2cf7b4 100644
--- a/app/models/project_statistics.rb
+++ b/app/models/project_statistics.rb
@@ -35,7 +35,9 @@ class ProjectStatistics < ActiveRecord::Base
   end
 
   def update_build_artifacts_size
-    self.build_artifacts_size = project.builds.sum(:artifacts_size)
+    self.build_artifacts_size =
+      project.builds.sum(:artifacts_size) +
+      Ci::JobArtifact.artifacts_size_for(self)
   end
 
   def update_storage_size
diff --git a/app/models/project_team.rb b/app/models/project_team.rb
index 1d35426050e..c679758973a 100644
--- a/app/models/project_team.rb
+++ b/app/models/project_team.rb
@@ -1,4 +1,6 @@
 class ProjectTeam
+  include BulkMemberAccessLoad
+
   attr_accessor :project
 
   def initialize(project)
@@ -157,39 +159,16 @@ class ProjectTeam
   #
   # Returns a Hash mapping user ID -> maximum access level.
   def max_member_access_for_user_ids(user_ids)
-    user_ids = user_ids.uniq
-    key = "max_member_access:#{project.id}"
-
-    access = {}
-
-    if RequestStore.active?
-      RequestStore.store[key] ||= {}
-      access = RequestStore.store[key]
+    max_member_access_for_resource_ids(User, user_ids, project.id) do |user_ids|
+      project.project_authorizations
+             .where(user: user_ids)
+             .group(:user_id)
+             .maximum(:access_level)
     end
-
-    # Look up only the IDs we need
-    user_ids = user_ids - access.keys
-
-    return access if user_ids.empty?
-
-    users_access = project.project_authorizations
-      .where(user: user_ids)
-      .group(:user_id)
-      .maximum(:access_level)
-
-    access.merge!(users_access)
-
-    missing_user_ids = user_ids - users_access.keys
-
-    missing_user_ids.each do |user_id|
-      access[user_id] = Gitlab::Access::NO_ACCESS
-    end
-
-    access
   end
 
   def max_member_access(user_id)
-    max_member_access_for_user_ids([user_id])[user_id] || Gitlab::Access::NO_ACCESS
+    max_member_access_for_user_ids([user_id])[user_id]
   end
 
   private
diff --git a/app/models/protected_branch.rb b/app/models/protected_branch.rb
index 89bfc5f9a9c..d28fed11ca8 100644
--- a/app/models/protected_branch.rb
+++ b/app/models/protected_branch.rb
@@ -10,7 +10,9 @@ class ProtectedBranch < ActiveRecord::Base
   def self.protected?(project, ref_name)
     return true if project.empty_repo? && default_branch_protected?
 
-    self.matching(ref_name, protected_refs: project.protected_branches).present?
+    refs = project.protected_branches.select(:name)
+
+    self.matching(ref_name, protected_refs: refs).present?
   end
 
   def self.default_branch_protected?
diff --git a/app/models/protected_tag.rb b/app/models/protected_tag.rb
index f38109c0e52..42a9bcf7723 100644
--- a/app/models/protected_tag.rb
+++ b/app/models/protected_tag.rb
@@ -5,6 +5,8 @@ class ProtectedTag < ActiveRecord::Base
   protected_ref_access_levels :create
 
   def self.protected?(project, ref_name)
-    self.matching(ref_name, protected_refs: project.protected_tags).present?
+    refs = project.protected_tags.select(:name)
+
+    self.matching(ref_name, protected_refs: refs).present?
   end
 end
diff --git a/app/models/redirect_route.rb b/app/models/redirect_route.rb
index 31de204d824..20532527346 100644
--- a/app/models/redirect_route.rb
+++ b/app/models/redirect_route.rb
@@ -17,4 +17,32 @@ class RedirectRoute < ActiveRecord::Base
 
     where(wheres, path, "#{sanitize_sql_like(path)}/%")
   end
+
+  scope :permanent, -> do
+    if column_permanent_exists?
+      where(permanent: true)
+    else
+      none
+    end
+  end
+
+  scope :temporary, -> do
+    if column_permanent_exists?
+      where(permanent: [false, nil])
+    else
+      all
+    end
+  end
+
+  default_value_for :permanent, false
+
+  def permanent=(value)
+    if self.class.column_permanent_exists?
+      super
+    end
+  end
+
+  def self.column_permanent_exists?
+    ActiveRecord::Base.connection.column_exists?(:redirect_routes, :permanent)
+  end
 end
diff --git a/app/models/repository.rb b/app/models/repository.rb
index 165dafd83fd..552a354d1ce 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -19,6 +19,7 @@ class Repository
   attr_accessor :full_path, :disk_path, :project, :is_wiki
 
   delegate :ref_name_for_sha, to: :raw_repository
+  delegate :write_ref, to: :raw_repository
 
   CreateTreeError = Class.new(StandardError)
 
@@ -37,7 +38,7 @@ class Repository
                       issue_template_names merge_request_template_names).freeze
 
   # Methods that use cache_method but only memoize the value
-  MEMOIZED_CACHED_METHODS = %i(license empty_repo?).freeze
+  MEMOIZED_CACHED_METHODS = %i(license).freeze
 
   # Certain method caches should be refreshed when certain types of files are
   # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to
@@ -237,11 +238,10 @@ class Repository
 
     # This will still fail if the file is corrupted (e.g. 0 bytes)
     begin
-      write_ref(keep_around_ref_name(sha), sha)
-    rescue Rugged::ReferenceError => ex
-      Rails.logger.error "Unable to create #{REF_KEEP_AROUND} reference for repository #{path}: #{ex}"
-    rescue Rugged::OSError => ex
-      raise unless ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/
+      write_ref(keep_around_ref_name(sha), sha, force: true)
+    rescue Gitlab::Git::Repository::GitError => ex
+      # Necessary because https://gitlab.com/gitlab-org/gitlab-ce/issues/20156
+      return true if ex.message =~ /Failed to create locked file/ && ex.message =~ /File exists/
 
       Rails.logger.error "Unable to create #{REF_KEEP_AROUND} reference for repository #{path}: #{ex}"
     end
@@ -251,12 +251,8 @@ class Repository
     ref_exists?(keep_around_ref_name(sha))
   end
 
-  def write_ref(ref_path, sha)
-    rugged.references.create(ref_path, sha, force: true)
-  end
-
   def diverging_commit_counts(branch)
-    root_ref_hash = raw_repository.rev_parse_target(root_ref).oid
+    root_ref_hash = raw_repository.commit(root_ref).id
     cache.fetch(:"diverging_commit_counts_#{branch.name}") do
       # Rugged seems to throw a `ReferenceError` when given branch_names rather
       # than SHA-1 hashes
@@ -497,7 +493,11 @@ class Repository
   end
   cache_method :exists?
 
-  delegate :empty?, to: :raw_repository
+  def empty?
+    return true unless exists?
+
+    !has_visible_content?
+  end
   cache_method :empty?
 
   # The size of this repository in megabytes.
@@ -686,7 +686,9 @@ class Repository
 
   def tags_sorted_by(value)
     case value
-    when 'name'
+    when 'name_asc'
+      VersionSorter.sort(tags) { |tag| tag.name }
+    when 'name_desc'
       VersionSorter.rsort(tags) { |tag| tag.name }
     when 'updated_desc'
       tags_sorted_by_committed_date.reverse
@@ -697,10 +699,14 @@ class Repository
     end
   end
 
-  def contributors
+  # Params:
+  #
+  # order_by: name|email|commits
+  # sort: asc|desc default: 'asc'
+  def contributors(order_by: nil, sort: 'asc')
     commits = self.commits(nil, limit: 2000, offset: 0, skip_merges: true)
 
-    commits.group_by(&:author_email).map do |email, commits|
+    commits = commits.group_by(&:author_email).map do |email, commits|
       contributor = Gitlab::Contributor.new
       contributor.email = email
 
@@ -714,6 +720,7 @@ class Repository
 
       contributor
     end
+    Commit.order_by(collection: commits, order_by: order_by, sort: sort)
   end
 
   def refs_contains_sha(ref_type, sha)
@@ -927,7 +934,7 @@ class Repository
   def merge_base(first_commit_id, second_commit_id)
     first_commit_id = commit(first_commit_id).try(:id) || first_commit_id
     second_commit_id = commit(second_commit_id).try(:id) || second_commit_id
-    rugged.merge_base(first_commit_id, second_commit_id)
+    raw_repository.merge_base(first_commit_id, second_commit_id)
   rescue Rugged::ReferenceError
     nil
   end
@@ -944,13 +951,8 @@ class Repository
     end
   end
 
-  def empty_repo?
-    !exists? || !has_visible_content?
-  end
-  cache_method :empty_repo?, memoize_only: true
-
   def search_files_by_content(query, ref)
-    return [] if empty_repo? || query.blank?
+    return [] if empty? || query.blank?
 
     offset = 2
     args = %W(grep -i -I -n --before-context #{offset} --after-context #{offset} -E -e #{Regexp.escape(query)} #{ref || root_ref})
@@ -959,7 +961,7 @@ class Repository
   end
 
   def search_files_by_name(query, ref)
-    return [] if empty_repo? || query.blank?
+    return [] if empty? || query.blank?
 
     args = %W(ls-tree --full-tree -r #{ref || root_ref} --name-status | #{Regexp.escape(query)})
 
@@ -972,8 +974,7 @@ class Repository
       tmp_remote_name = true
     end
 
-    add_remote(remote_name, url)
-    set_remote_as_mirror(remote_name, refmap: refmap)
+    add_remote(remote_name, url, mirror_refmap: refmap)
     fetch_remote(remote_name, forced: forced)
   ensure
     remove_remote(remote_name) if tmp_remote_name
@@ -996,7 +997,7 @@ class Repository
   end
 
   def create_ref(ref, ref_path)
-    raw_repository.write_ref(ref_path, ref)
+    write_ref(ref_path, ref)
   end
 
   def ls_files(ref)
diff --git a/app/models/route.rb b/app/models/route.rb
index 97e8a6ad9e9..7ba3ec06041 100644
--- a/app/models/route.rb
+++ b/app/models/route.rb
@@ -8,6 +8,8 @@ class Route < ActiveRecord::Base
     presence: true,
     uniqueness: { case_sensitive: false }
 
+  validate :ensure_permanent_paths
+
   after_create :delete_conflicting_redirects
   after_update :delete_conflicting_redirects, if: :path_changed?
   after_update :create_redirect_for_old_path
@@ -40,7 +42,7 @@ class Route < ActiveRecord::Base
         # We are not calling route.delete_conflicting_redirects here, in hopes
         # of avoiding deadlocks. The parent (self, in this method) already
         # called it, which deletes conflicts for all descendants.
-        route.create_redirect(old_path) if attributes[:path]
+        route.create_redirect(old_path, permanent: permanent_redirect?) if attributes[:path]
       end
     end
   end
@@ -50,16 +52,30 @@ class Route < ActiveRecord::Base
   end
 
   def conflicting_redirects
-    RedirectRoute.matching_path_and_descendants(path)
+    RedirectRoute.temporary.matching_path_and_descendants(path)
   end
 
-  def create_redirect(path)
-    RedirectRoute.create(source: source, path: path)
+  def create_redirect(path, permanent: false)
+    RedirectRoute.create(source: source, path: path, permanent: permanent)
   end
 
   private
 
   def create_redirect_for_old_path
-    create_redirect(path_was) if path_changed?
+    create_redirect(path_was, permanent: permanent_redirect?) if path_changed?
+  end
+
+  def permanent_redirect?
+    source_type != "Project"
+  end
+
+  def ensure_permanent_paths
+    return if path.nil?
+
+    errors.add(:path, "#{path} has been taken before. Please use another one") if conflicting_redirect_exists?
+  end
+
+  def conflicting_redirect_exists?
+    RedirectRoute.permanent.matching_path_and_descendants(path).exists?
   end
 end
diff --git a/app/models/service.rb b/app/models/service.rb
index fdd2605e3e3..3c4f1885dd0 100644
--- a/app/models/service.rb
+++ b/app/models/service.rb
@@ -211,7 +211,7 @@ class Service < ActiveRecord::Base
   def async_execute(data)
     return unless supported_events.include?(data[:object_kind])
 
-    Sidekiq::Client.enqueue(ProjectServiceWorker, id, data)
+    ProjectServiceWorker.perform_async(id, data)
   end
 
   def issue_tracker?
diff --git a/app/models/user.rb b/app/models/user.rb
index 14941fd7f98..92b461ce3ed 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -7,6 +7,7 @@ class User < ActiveRecord::Base
   include Gitlab::ConfigHelper
   include Gitlab::CurrentSettings
   include Gitlab::SQL::Pattern
+  include AfterCommitQueue
   include Avatarable
   include Referable
   include Sortable
@@ -16,6 +17,7 @@ class User < ActiveRecord::Base
   include FeatureGate
   include CreatedAtFilterable
   include IgnorableColumn
+  include BulkMemberAccessLoad
 
   DEFAULT_NOTIFICATION_LEVEL = :participating
 
@@ -313,6 +315,8 @@ class User < ActiveRecord::Base
     #
     # Returns an ActiveRecord::Relation.
     def search(query)
+      query = query.downcase
+
       order = <<~SQL
         CASE
           WHEN users.name = %{query} THEN 0
@@ -322,8 +326,11 @@ class User < ActiveRecord::Base
         END
       SQL
 
-      fuzzy_search(query, [:name, :email, :username])
-        .reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name)
+      where(
+        fuzzy_arel_match(:name, query)
+          .or(fuzzy_arel_match(:username, query))
+          .or(arel_table[:email].eq(query))
+      ).reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name)
     end
 
     # searches user by given pattern
@@ -331,15 +338,17 @@ class User < ActiveRecord::Base
     # This method uses ILIKE on PostgreSQL and LIKE on MySQL.
 
     def search_with_secondary_emails(query)
+      query = query.downcase
+
       email_table = Email.arel_table
       matched_by_emails_user_ids = email_table
         .project(email_table[:user_id])
-        .where(Email.fuzzy_arel_match(:email, query))
+        .where(email_table[:email].eq(query))
 
       where(
         fuzzy_arel_match(:name, query)
-          .or(fuzzy_arel_match(:email, query))
           .or(fuzzy_arel_match(:username, query))
+          .or(arel_table[:email].eq(query))
           .or(arel_table[:id].in(matched_by_emails_user_ids))
       )
     end
@@ -487,7 +496,11 @@ class User < ActiveRecord::Base
   end
 
   def two_factor_u2f_enabled?
-    u2f_registrations.exists?
+    if u2f_registrations.loaded?
+      u2f_registrations.any?
+    else
+      u2f_registrations.exists?
+    end
   end
 
   def namespace_uniq
@@ -899,6 +912,7 @@ class User < ActiveRecord::Base
 
   def post_destroy_hook
     log_info("User \"#{name}\" (#{email})  was removed")
+
     system_hook_service.execute_hooks_for(self, :destroy)
   end
 
@@ -998,7 +1012,11 @@ class User < ActiveRecord::Base
   end
 
   def notification_settings_for(source)
-    notification_settings.find_or_initialize_by(source: source)
+    if notification_settings.loaded?
+      notification_settings.find { |notification| notification.source == source }
+    else
+      notification_settings.find_or_initialize_by(source: source)
+    end
   end
 
   # Lazy load global notification setting
@@ -1043,13 +1061,13 @@ class User < ActiveRecord::Base
   end
 
   def todos_done_count(force: false)
-    Rails.cache.fetch(['users', id, 'todos_done_count'], force: force) do
+    Rails.cache.fetch(['users', id, 'todos_done_count'], force: force, expires_in: 20.minutes) do
       TodosFinder.new(self, state: :done).execute.count
     end
   end
 
   def todos_pending_count(force: false)
-    Rails.cache.fetch(['users', id, 'todos_pending_count'], force: force) do
+    Rails.cache.fetch(['users', id, 'todos_pending_count'], force: force, expires_in: 20.minutes) do
       TodosFinder.new(self, state: :pending).execute.count
     end
   end
@@ -1134,6 +1152,34 @@ class User < ActiveRecord::Base
     super
   end
 
+  # Determine the maximum access level for a group of projects in bulk.
+  #
+  # Returns a Hash mapping project ID -> maximum access level.
+  def max_member_access_for_project_ids(project_ids)
+    max_member_access_for_resource_ids(Project, project_ids) do |project_ids|
+      project_authorizations.where(project: project_ids)
+                            .group(:project_id)
+                            .maximum(:access_level)
+    end
+  end
+
+  def max_member_access_for_project(project_id)
+    max_member_access_for_project_ids([project_id])[project_id]
+  end
+
+  # Determine the maximum access level for a group of groups in bulk.
+  #
+  # Returns a Hash mapping project ID -> maximum access level.
+  def max_member_access_for_group_ids(group_ids)
+    max_member_access_for_resource_ids(Group, group_ids) do |group_ids|
+      group_members.where(source: group_ids).group(:source_id).maximum(:access_level)
+    end
+  end
+
+  def max_member_access_for_group(group_id)
+    max_member_access_for_group_ids([group_id])[group_id]
+  end
+
   protected
 
   # override, from Devise::Validatable