Merge branch 'master' into sh-break-out-invited-group-members

22 files changed, 196 insertions, 163 deletions

diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb
index 4bb09bf3b53..b7a4d7aa803 100644
--- a/app/models/application_setting_implementation.rb
+++ b/app/models/application_setting_implementation.rb
@@ -21,7 +21,8 @@ module ApplicationSettingImplementation
       {
         after_sign_up_text: nil,
         akismet_enabled: false,
-        allow_local_requests_from_hooks_and_services: false,
+        allow_local_requests_from_web_hooks_and_services: false,
+        allow_local_requests_from_system_hooks: true,
         dns_rebinding_protection_enabled: true,
         authorized_keys_enabled: true, # TODO default to false if the instance is configured to use AuthorizedKeysCommand
         container_registry_token_expire_delay: 5,
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index ffab4e82f90..3b28eb246db 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -612,8 +612,8 @@ module Ci
     end
 
     # rubocop: disable CodeReuse/ServiceClass
-    def process!(trigger_build_name = nil)
-      Ci::ProcessPipelineService.new(project, user).execute(self, trigger_build_name)
+    def process!(trigger_build_ids = nil)
+      Ci::ProcessPipelineService.new(project, user).execute(self, trigger_build_ids)
     end
     # rubocop: enable CodeReuse/ServiceClass
 
diff --git a/app/models/clusters/applications/cert_manager.rb b/app/models/clusters/applications/cert_manager.rb
index 7d5a6dec519..2fc1b67dfd2 100644
--- a/app/models/clusters/applications/cert_manager.rb
+++ b/app/models/clusters/applications/cert_manager.rb
@@ -24,12 +24,6 @@ module Clusters
         'stable/cert-manager'
       end
 
-      # We will implement this in future MRs.
-      # Need to reverse postinstall step
-      def allowed_to_uninstall?
-        false
-      end
-
       def install_command
         Gitlab::Kubernetes::Helm::InstallCommand.new(
           name: 'certmanager',
@@ -41,12 +35,42 @@ module Clusters
         )
       end
 
+      def uninstall_command
+        Gitlab::Kubernetes::Helm::DeleteCommand.new(
+          name: 'certmanager',
+          rbac: cluster.platform_kubernetes_rbac?,
+          files: files,
+          postdelete: post_delete_script
+        )
+      end
+
       private
 
       def post_install_script
         ["kubectl create -f /data/helm/certmanager/config/cluster_issuer.yaml"]
       end
 
+      def post_delete_script
+        [
+          delete_private_key,
+          delete_crd('certificates.certmanager.k8s.io'),
+          delete_crd('clusterissuers.certmanager.k8s.io'),
+          delete_crd('issuers.certmanager.k8s.io')
+        ].compact
+      end
+
+      def private_key_name
+        @private_key_name ||= cluster_issuer_content.dig('spec', 'acme', 'privateKeySecretRef', 'name')
+      end
+
+      def delete_private_key
+        "kubectl delete secret -n #{Gitlab::Kubernetes::Helm::NAMESPACE} #{private_key_name} --ignore-not-found" if private_key_name.present?
+      end
+
+      def delete_crd(definition)
+        "kubectl delete crd #{definition} --ignore-not-found"
+      end
+
       def cluster_issuer_file
         {
           'cluster_issuer.yaml': cluster_issuer_yaml_content
diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb
index 5eb535cab58..08e52f32bb3 100644
--- a/app/models/clusters/applications/prometheus.rb
+++ b/app/models/clusters/applications/prometheus.rb
@@ -83,7 +83,7 @@ module Clusters
 
         # ensures headers containing auth data are appended to original k8s client options
         options = kube_client.rest_client.options.merge(headers: kube_client.headers)
-        RestClient::Resource.new(proxy_url, options)
+        Gitlab::PrometheusClient.new(proxy_url, options)
       rescue Kubeclient::HttpError
         # If users have mistakenly set parameters or removed the depended clusters,
         # `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb
index 8bb44b0ce40..97d39491b73 100644
--- a/app/models/clusters/cluster.rb
+++ b/app/models/clusters/cluster.rb
@@ -53,6 +53,7 @@ module Clusters
     validates :name, cluster_name: true
     validates :cluster_type, presence: true
     validates :domain, allow_blank: true, hostname: { allow_numeric_hostname: true }
+    validates :namespace_per_environment, inclusion: { in: [true, false] }
 
     validate :restrict_modification, on: :update
     validate :no_groups, unless: :group_type?
@@ -100,16 +101,6 @@ module Clusters
 
     scope :default_environment, -> { where(environment_scope: DEFAULT_ENVIRONMENT) }
 
-    scope :with_knative_installed, -> { joins(:application_knative).merge(Clusters::Applications::Knative.available) }
-
-    scope :preload_knative, -> {
-      preload(
-        :kubernetes_namespaces,
-        :platform_kubernetes,
-        :application_knative
-      )
-    }
-
     def self.ancestor_clusters_for_clusterable(clusterable, hierarchy_order: :asc)
       return [] if clusterable.is_a?(Instance)
 
@@ -177,36 +168,15 @@ module Clusters
       platform_kubernetes.kubeclient if kubernetes?
     end
 
-    ##
-    # This is subtly different to #find_or_initialize_kubernetes_namespace_for_project
-    # below because it will ignore any namespaces that have not got a service account
-    # token. This provides a guarantee that any namespace selected here can be used
-    # for cluster operations - a namespace needs to have a service account configured
-    # before it it can be used.
-    #
-    # This is used for selecting a namespace to use when querying a cluster, or
-    # generating variables to pass to CI.
-    def kubernetes_namespace_for(project)
-      find_or_initialize_kubernetes_namespace_for_project(
-        project, scope: kubernetes_namespaces.has_service_account_token
-      ).namespace
-    end
-
-    ##
-    # This is subtly different to #kubernetes_namespace_for because it will include
-    # namespaces that have yet to receive a service account token. This allows
-    # the namespace configuration process to be repeatable - if a namespace has
-    # already been created without a token we don't need to create another
-    # record entirely, just set the token on the pre-existing namespace.
-    #
-    # This is used for configuring cluster namespaces.
-    def find_or_initialize_kubernetes_namespace_for_project(project, scope: kubernetes_namespaces)
-      attributes = { project: project }
-      attributes[:cluster_project] = cluster_project if project_type?
+    def kubernetes_namespace_for(environment)
+      project = environment.project
+      persisted_namespace = Clusters::KubernetesNamespaceFinder.new(
+        self,
+        project: project,
+        environment_slug: environment.slug
+      ).execute
 
-      scope.find_or_initialize_by(attributes).tap do |namespace|
-        namespace.set_defaults
-      end
+      persisted_namespace&.namespace || Gitlab::Kubernetes::DefaultNamespace.new(self, project: project).from_environment_slug(environment.slug)
     end
 
     def allow_user_defined_namespace?
@@ -225,10 +195,6 @@ module Clusters
       end
     end
 
-    def knative_services_finder(project)
-      @knative_services_finder ||= KnativeServicesFinder.new(self, project)
-    end
-
     private
 
     def instance_domain
diff --git a/app/models/clusters/kubernetes_namespace.rb b/app/models/clusters/kubernetes_namespace.rb
index b0c4900546e..69a2b99fcb6 100644
--- a/app/models/clusters/kubernetes_namespace.rb
+++ b/app/models/clusters/kubernetes_namespace.rb
@@ -9,12 +9,12 @@ module Clusters
     belongs_to :cluster_project, class_name: 'Clusters::Project'
     belongs_to :cluster, class_name: 'Clusters::Cluster'
     belongs_to :project, class_name: '::Project'
+    belongs_to :environment, optional: true
     has_one :platform_kubernetes, through: :cluster
 
-    before_validation :set_defaults
-
     validates :namespace, presence: true
     validates :namespace, uniqueness: { scope: :cluster_id }
+    validates :environment_id, uniqueness: { scope: [:cluster_id, :project_id] }, allow_nil: true
 
     validates :service_account_name, presence: true
 
@@ -27,6 +27,7 @@ module Clusters
         algorithm: 'aes-256-cbc'
 
     scope :has_service_account_token, -> { where.not(encrypted_service_account_token: nil) }
+    scope :with_environment_slug, -> (slug) { joins(:environment).where(environments: { slug: slug }) }
 
     def token_name
       "#{namespace}-token"
@@ -42,34 +43,8 @@ module Clusters
       end
     end
 
-    def set_defaults
-      self.namespace ||= default_platform_kubernetes_namespace
-      self.namespace ||= default_project_namespace
-      self.service_account_name ||= default_service_account_name
-    end
-
     private
 
-    def default_service_account_name
-      return unless namespace
-
-      "#{namespace}-service-account"
-    end
-
-    def default_platform_kubernetes_namespace
-      platform_kubernetes&.namespace.presence
-    end
-
-    def default_project_namespace
-      Gitlab::NamespaceSanitizer.sanitize(project_slug) if project_slug
-    end
-
-    def project_slug
-      return unless project
-
-      "#{project.path}-#{project.id}".downcase
-    end
-
     def kubeconfig
       to_kubeconfig(
         url: api_url,
diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb
index 9296c28776b..37614fbe3ca 100644
--- a/app/models/clusters/platforms/kubernetes.rb
+++ b/app/models/clusters/platforms/kubernetes.rb
@@ -51,11 +51,6 @@ module Clusters
       delegate :provided_by_user?, to: :cluster, allow_nil: true
       delegate :allow_user_defined_namespace?, to: :cluster, allow_nil: true
 
-      # This is just to maintain compatibility with KubernetesService, which
-      # will be removed in https://gitlab.com/gitlab-org/gitlab-ce/issues/39217.
-      # It can be removed once KubernetesService is gone.
-      delegate :kubernetes_namespace_for, to: :cluster, allow_nil: true
-
       alias_method :active?, :enabled?
 
       enum_with_nil authorization_type: {
@@ -66,7 +61,7 @@ module Clusters
 
       default_value_for :authorization_type, :rbac
 
-      def predefined_variables(project:)
+      def predefined_variables(project:, environment_name:)
         Gitlab::Ci::Variables::Collection.new.tap do |variables|
           variables.append(key: 'KUBE_URL', value: api_url)
 
@@ -77,15 +72,14 @@ module Clusters
           end
 
           if !cluster.managed?
-            project_namespace = namespace.presence || "#{project.path}-#{project.id}".downcase
+            namespace = Gitlab::Kubernetes::DefaultNamespace.new(cluster, project: project).from_environment_name(environment_name)
 
             variables
-              .append(key: 'KUBE_URL', value: api_url)
               .append(key: 'KUBE_TOKEN', value: token, public: false, masked: true)
-              .append(key: 'KUBE_NAMESPACE', value: project_namespace)
-              .append(key: 'KUBECONFIG', value: kubeconfig(project_namespace), public: false, file: true)
+              .append(key: 'KUBE_NAMESPACE', value: namespace)
+              .append(key: 'KUBECONFIG', value: kubeconfig(namespace), public: false, file: true)
 
-          elsif kubernetes_namespace = cluster.kubernetes_namespaces.has_service_account_token.find_by(project: project)
+          elsif kubernetes_namespace = find_persisted_namespace(project, environment_name: environment_name)
             variables.concat(kubernetes_namespace.predefined_variables)
           end
 
@@ -111,6 +105,22 @@ module Clusters
 
       private
 
+      ##
+      # Environment slug can be predicted given an environment
+      # name, so even if the environment isn't persisted yet we
+      # still know what to look for.
+      def environment_slug(name)
+        Gitlab::Slug::Environment.new(name).generate
+      end
+
+      def find_persisted_namespace(project, environment_name:)
+        Clusters::KubernetesNamespaceFinder.new(
+          cluster,
+          project: project,
+          environment_slug: environment_slug(environment_name)
+        ).execute
+      end
+
       def kubeconfig(namespace)
         to_kubeconfig(
           url: api_url,
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index a9c29fb390b..a88cac6b8e6 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -126,7 +126,7 @@ class CommitStatus < ApplicationRecord
       commit_status.run_after_commit do
         if pipeline_id
           if complete? || manual?
-            BuildProcessWorker.perform_async(id)
+            PipelineProcessWorker.perform_async(pipeline_id, [id])
           else
             PipelineUpdateWorker.perform_async(pipeline_id)
           end
diff --git a/app/models/concerns/prometheus_adapter.rb b/app/models/concerns/prometheus_adapter.rb
index c2542dbe743..9ac4722c6b1 100644
--- a/app/models/concerns/prometheus_adapter.rb
+++ b/app/models/concerns/prometheus_adapter.rb
@@ -14,10 +14,6 @@ module PrometheusAdapter
       raise NotImplementedError
     end
 
-    def prometheus_client_wrapper
-      Gitlab::PrometheusClient.new(prometheus_client)
-    end
-
     def can_query?
       prometheus_client.present?
     end
@@ -35,7 +31,7 @@ module PrometheusAdapter
     def calculate_reactive_cache(query_class_name, *args)
       return unless prometheus_client
 
-      data = Object.const_get(query_class_name, false).new(prometheus_client_wrapper).query(*args)
+      data = Object.const_get(query_class_name, false).new(prometheus_client).query(*args)
       {
         success: true,
         data: data,
diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb
index 4a1441805fc..6d3c7a7ed68 100644
--- a/app/models/concerns/relative_positioning.rb
+++ b/app/models/concerns/relative_positioning.rb
@@ -29,10 +29,6 @@ module RelativePositioning
   MAX_POSITION = Gitlab::Database::MAX_INT_VALUE
   IDEAL_DISTANCE = 500
 
-  included do
-    after_save :save_positionable_neighbours
-  end
-
   class_methods do
     def move_nulls_to_end(objects)
       objects = objects.reject(&:relative_position)
@@ -114,11 +110,12 @@ module RelativePositioning
     return move_after(before) unless after
     return move_before(after) unless before
 
-    # If there is no place to insert an item we need to create one by moving the before item closer
-    # to its predecessor. This process will recursively move all the predecessors until we have a place
+    # If there is no place to insert an item we need to create one by moving the item
+    # before this and all preceding items until there is a gap
+    before, after = after, before if after.relative_position < before.relative_position
     if (after.relative_position - before.relative_position) < 2
-      before.move_before
-      @positionable_neighbours = [before] # rubocop:disable Gitlab/ModuleWithInstanceVariables
+      after.move_sequence_before
+      before.reset
     end
 
     self.relative_position = self.class.position_between(before.relative_position, after.relative_position)
@@ -128,12 +125,8 @@ module RelativePositioning
     pos_before = before.relative_position
     pos_after = before.next_relative_position
 
-    if before.shift_after?
-      item_to_move = self.class.relative_positioning_query_base(self).find_by!(relative_position: pos_after)
-      item_to_move.move_after
-      @positionable_neighbours = [item_to_move] # rubocop:disable Gitlab/ModuleWithInstanceVariables
-
-      pos_after = item_to_move.relative_position
+    if pos_after && (pos_after - pos_before) < 2
+      before.move_sequence_after
     end
 
     self.relative_position = self.class.position_between(pos_before, pos_after)
@@ -143,12 +136,8 @@ module RelativePositioning
     pos_after = after.relative_position
     pos_before = after.prev_relative_position
 
-    if after.shift_before?
-      item_to_move = self.class.relative_positioning_query_base(self).find_by!(relative_position: pos_before)
-      item_to_move.move_before
-      @positionable_neighbours = [item_to_move] # rubocop:disable Gitlab/ModuleWithInstanceVariables
-
-      pos_before = item_to_move.relative_position
+    if pos_before && (pos_after - pos_before) < 2
+      after.move_sequence_before
     end
 
     self.relative_position = self.class.position_between(pos_before, pos_after)
@@ -162,36 +151,82 @@ module RelativePositioning
     self.relative_position = self.class.position_between(min_relative_position || START_POSITION, MIN_POSITION)
   end
 
-  # Indicates if there is an item that should be shifted to free the place
-  def shift_after?
-    next_pos = next_relative_position
-    next_pos && (next_pos - relative_position) == 1
+  # Moves the sequence before the current item to the middle of the next gap
+  # For example, we have 5 11 12 13 14 15 and the current item is 15
+  # This moves the sequence 11 12 13 14 to 8 9 10 11
+  def move_sequence_before
+    next_gap = find_next_gap_before
+    delta = optimum_delta_for_gap(next_gap)
+
+    move_sequence(next_gap[:start], relative_position, -delta)
   end
 
-  # Indicates if there is an item that should be shifted to free the place
-  def shift_before?
-    prev_pos = prev_relative_position
-    prev_pos && (relative_position - prev_pos) == 1
+  # Moves the sequence after the current item to the middle of the next gap
+  # For example, we have 11 12 13 14 15 21 and the current item is 11
+  # This moves the sequence 12 13 14 15 to 15 16 17 18
+  def move_sequence_after
+    next_gap = find_next_gap_after
+    delta = optimum_delta_for_gap(next_gap)
+
+    move_sequence(relative_position, next_gap[:start], delta)
   end
 
   private
 
-  # rubocop:disable Gitlab/ModuleWithInstanceVariables
-  def save_positionable_neighbours
-    return unless @positionable_neighbours
+  # Supposing that we have a sequence of items: 1 5 11 12 13 and the current item is 13
+  # This would return: `{ start: 11, end: 5 }`
+  def find_next_gap_before
+    items_with_next_pos = scoped_items
+                            .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position DESC) AS next_pos')
+                            .where('relative_position <= ?', relative_position)
+                            .order(relative_position: :desc)
+
+    find_next_gap(items_with_next_pos).tap do |gap|
+      gap[:end] ||= MIN_POSITION
+    end
+  end
+
+  # Supposing that we have a sequence of items: 13 14 15 20 24 and the current item is 13
+  # This would return: `{ start: 15, end: 20 }`
+  def find_next_gap_after
+    items_with_next_pos = scoped_items
+                            .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position ASC) AS next_pos')
+                            .where('relative_position >= ?', relative_position)
+                            .order(:relative_position)
 
-    status = @positionable_neighbours.all? { |item| item.save(touch: false) }
-    @positionable_neighbours = nil
+    find_next_gap(items_with_next_pos).tap do |gap|
+      gap[:end] ||= MAX_POSITION
+    end
+  end
+
+  def find_next_gap(items_with_next_pos)
+    gap = self.class.from(items_with_next_pos, :items_with_next_pos)
+                    .where('ABS(pos - next_pos) > 1 OR next_pos IS NULL')
+                    .limit(1)
+                    .pluck(:pos, :next_pos)
+                    .first
+
+    { start: gap[0], end: gap[1] }
+  end
 
-    status
+  def optimum_delta_for_gap(gap)
+    delta = ((gap[:start] - gap[:end]) / 2.0).abs.ceil
+
+    [delta, IDEAL_DISTANCE].min
+  end
+
+  def move_sequence(start_pos, end_pos, delta)
+    scoped_items
+      .where.not(id: self.id)
+      .where('relative_position BETWEEN ? AND ?', start_pos, end_pos)
+      .update_all("relative_position = relative_position + #{delta}")
   end
-  # rubocop:enable Gitlab/ModuleWithInstanceVariables
 
   def calculate_relative_position(calculation)
     # When calculating across projects, this is much more efficient than
     # MAX(relative_position) without the GROUP BY, due to index usage:
     # https://gitlab.com/gitlab-org/gitlab-ce/issues/54276#note_119340977
-    relation = self.class.relative_positioning_query_base(self)
+    relation = scoped_items
                  .order(Gitlab::Database.nulls_last_order('position', 'DESC'))
                  .group(self.class.relative_positioning_parent_column)
                  .limit(1)
@@ -203,4 +238,8 @@ module RelativePositioning
       .first&.
       last
   end
+
+  def scoped_items
+    self.class.relative_positioning_query_base(self)
+  end
 end
diff --git a/app/models/deploy_key.rb b/app/models/deploy_key.rb
index db501b4b506..0bd90bd28e3 100644
--- a/app/models/deploy_key.rb
+++ b/app/models/deploy_key.rb
@@ -2,12 +2,14 @@
 
 class DeployKey < Key
   include IgnorableColumn
+  include FromUnion
 
   has_many :deploy_keys_projects, inverse_of: :deploy_key, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
   has_many :projects, through: :deploy_keys_projects
 
   scope :in_projects, ->(projects) { joins(:deploy_keys_projects).where('deploy_keys_projects.project_id in (?)', projects) }
   scope :are_public,  -> { where(public: true) }
+  scope :with_projects, -> { includes(deploy_keys_projects: { project: [:route, :namespace] }) }
 
   ignore_column :can_push
 
@@ -22,7 +24,7 @@ class DeployKey < Key
   end
 
   def almost_orphaned?
-    self.deploy_keys_projects.length == 1
+    self.deploy_keys_projects.count == 1
   end
 
   def destroyed_when_orphaned?
@@ -46,6 +48,6 @@ class DeployKey < Key
   end
 
   def projects_with_write_access
-    Project.preload(:route).where(id: deploy_keys_projects.with_write_access.select(:project_id))
+    Project.with_route.where(id: deploy_keys_projects.with_write_access.select(:project_id))
   end
 end
diff --git a/app/models/deploy_keys_project.rb b/app/models/deploy_keys_project.rb
index 15906ed8e06..40c66d5bc4c 100644
--- a/app/models/deploy_keys_project.rb
+++ b/app/models/deploy_keys_project.rb
@@ -1,9 +1,8 @@
 # frozen_string_literal: true
 
 class DeployKeysProject < ApplicationRecord
-  belongs_to :project
+  belongs_to :project, inverse_of: :deploy_keys_projects
   belongs_to :deploy_key, inverse_of: :deploy_keys_projects
-
   scope :without_project_deleted,  -> { joins(:project).where(projects: { pending_delete: false }) }
   scope :in_project, ->(project) { where(project: project) }
   scope :with_write_access, -> { where(can_push: true) }
diff --git a/app/models/environment.rb b/app/models/environment.rb
index 513427ac2c5..1b53c4b45f9 100644
--- a/app/models/environment.rb
+++ b/app/models/environment.rb
@@ -48,6 +48,7 @@ class Environment < ApplicationRecord
   end
   scope :in_review_folder, -> { where(environment_type: "review") }
   scope :for_name, -> (name) { where(name: name) }
+  scope :preload_cluster, -> { preload(last_deployment: :cluster) }
 
   ##
   # Search environments which have names like the given query.
@@ -170,7 +171,7 @@ class Environment < ApplicationRecord
 
   def deployment_namespace
     strong_memoize(:kubernetes_namespace) do
-      deployment_platform&.kubernetes_namespace_for(project)
+      deployment_platform.cluster.kubernetes_namespace_for(self) if deployment_platform
     end
   end
 
@@ -233,6 +234,12 @@ class Environment < ApplicationRecord
     end
   end
 
+  def knative_services_finder
+    if last_deployment&.cluster
+      Clusters::KnativeServicesFinder.new(last_deployment.cluster, self)
+    end
+  end
+
   private
 
   def generate_slug
diff --git a/app/models/group.rb b/app/models/group.rb
index 74eb556b1b5..6c868b1d1f0 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -44,6 +44,8 @@ class Group < Namespace
   has_many :cluster_groups, class_name: 'Clusters::Group'
   has_many :clusters, through: :cluster_groups, class_name: 'Clusters::Cluster'
 
+  has_many :container_repositories, through: :projects
+
   has_many :todos
 
   accepts_nested_attributes_for :variables, allow_destroy: true
diff --git a/app/models/hooks/system_hook.rb b/app/models/hooks/system_hook.rb
index 90b4588a325..3d54d17e787 100644
--- a/app/models/hooks/system_hook.rb
+++ b/app/models/hooks/system_hook.rb
@@ -14,8 +14,10 @@ class SystemHook < WebHook
   default_value_for :repository_update_events, true
   default_value_for :merge_requests_events, false
 
+  validates :url, system_hook_url: true
+
   # Allow urls pointing localhost and the local network
   def allow_local_requests?
-    true
+    Gitlab::CurrentSettings.allow_local_requests_from_system_hooks?
   end
 end
diff --git a/app/models/hooks/web_hook.rb b/app/models/hooks/web_hook.rb
index daf7ff4b771..16fc7fdbd48 100644
--- a/app/models/hooks/web_hook.rb
+++ b/app/models/hooks/web_hook.rb
@@ -15,8 +15,8 @@ class WebHook < ApplicationRecord
 
   has_many :web_hook_logs, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
-  validates :url, presence: true, public_url: { allow_localhost: lambda(&:allow_local_requests?),
-                                                allow_local_network: lambda(&:allow_local_requests?) }
+  validates :url, presence: true
+  validates :url, public_url: true, unless: ->(hook) { hook.is_a?(SystemHook) }
 
   validates :token, format: { without: /\n/ }
   validates :push_events_branch_filter, branch_filter: true
@@ -35,6 +35,6 @@ class WebHook < ApplicationRecord
 
   # Allow urls pointing localhost and the local network
   def allow_local_requests?
-    false
+    Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?
   end
 end
diff --git a/app/models/list.rb b/app/models/list.rb
index d28a9bda82d..ccadd39bda2 100644
--- a/app/models/list.rb
+++ b/app/models/list.rb
@@ -3,10 +3,11 @@
 class List < ApplicationRecord
   belongs_to :board
   belongs_to :label
+  include Importable
 
   enum list_type: { backlog: 0, label: 1, closed: 2, assignee: 3, milestone: 4 }
 
-  validates :board, :list_type, presence: true
+  validates :board, :list_type, presence: true, unless: :importing?
   validates :label, :position, presence: true, if: :label?
   validates :label_id, uniqueness: { scope: :board_id }, if: :label?
   validates :position, numericality: { only_integer: true, greater_than_or_equal_to: 0 }, if: :movable?
diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb
index f45bd0e03de..2c9dbf2585c 100644
--- a/app/models/merge_request_diff.rb
+++ b/app/models/merge_request_diff.rb
@@ -196,6 +196,12 @@ class MergeRequestDiff < ApplicationRecord
     real_size.presence || raw_diffs.size
   end
 
+  def lines_count
+    strong_memoize(:lines_count) do
+      diffs.diff_files.sum(&:line_count)
+    end
+  end
+
   def raw_diffs(options = {})
     if options[:ignore_whitespace_change]
       @diffs_no_whitespace ||= compare.diffs(options)
diff --git a/app/models/project.rb b/app/models/project.rb
index 8f234fba04f..960795b73cb 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -214,7 +214,7 @@ class Project < ApplicationRecord
     as: :source, class_name: 'ProjectMember', dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
   has_many :members_and_requesters, as: :source, class_name: 'ProjectMember'
 
-  has_many :deploy_keys_projects
+  has_many :deploy_keys_projects, inverse_of: :project
   has_many :deploy_keys, through: :deploy_keys_projects
   has_many :users_star_projects
   has_many :starrers, through: :users_star_projects, source: :user
@@ -1487,6 +1487,9 @@ class Project < ApplicationRecord
   end
 
   def pipeline_for(ref, sha = nil, id = nil)
+    sha ||= commit(ref).try(:sha)
+    return unless sha
+
     if id.present?
       pipelines_for(ref, sha).find_by(id: id)
     else
@@ -1494,11 +1497,7 @@ class Project < ApplicationRecord
     end
   end
 
-  def pipelines_for(ref, sha = nil)
-    sha ||= commit(ref).try(:sha)
-
-    return unless sha
-
+  def pipelines_for(ref, sha)
     ci_pipelines.order(id: :desc).where(sha: sha, ref: ref)
   end
 
@@ -1856,8 +1855,12 @@ class Project < ApplicationRecord
     end
   end
 
-  def deployment_variables(environment: nil)
-    deployment_platform(environment: environment)&.predefined_variables(project: self) || []
+  def deployment_variables(environment:)
+    platform = deployment_platform(environment: environment)
+
+    return [] unless platform.present?
+
+    platform.predefined_variables(project: self, environment_name: environment)
   end
 
   def auto_devops_variables
diff --git a/app/models/project_services/mock_deployment_service.rb b/app/models/project_services/mock_deployment_service.rb
index 1103cb11e73..6f2b0f7747f 100644
--- a/app/models/project_services/mock_deployment_service.rb
+++ b/app/models/project_services/mock_deployment_service.rb
@@ -24,7 +24,7 @@ class MockDeploymentService < Service
     %w()
   end
 
-  def predefined_variables(project:)
+  def predefined_variables(project:, environment_name:)
     []
   end
 
diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb
index c68a9d923c8..6eff2ea2e3a 100644
--- a/app/models/project_services/prometheus_service.rb
+++ b/app/models/project_services/prometheus_service.rb
@@ -63,15 +63,16 @@ class PrometheusService < MonitoringService
 
   # Check we can connect to the Prometheus API
   def test(*args)
-    Gitlab::PrometheusClient.new(prometheus_client).ping
-
+    prometheus_client.ping
     { success: true, result: 'Checked API endpoint' }
   rescue Gitlab::PrometheusClient::Error => err
     { success: false, result: err }
   end
 
   def prometheus_client
-    RestClient::Resource.new(api_url, max_redirects: 0) if should_return_client?
+    return unless should_return_client?
+
+    Gitlab::PrometheusClient.new(api_url)
   end
 
   def prometheus_available?
@@ -84,7 +85,7 @@ class PrometheusService < MonitoringService
   private
 
   def should_return_client?
-    api_url && manual_configuration? && active? && valid?
+    api_url.present? && manual_configuration? && active? && valid?
   end
 
   def synchronize_service_state
diff --git a/app/models/user.rb b/app/models/user.rb
index b439d1c0c16..4630552e02e 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -933,7 +933,7 @@ class User < ApplicationRecord
   end
 
   def project_deploy_keys
-    DeployKey.unscoped.in_projects(authorized_projects.pluck(:id)).distinct(:id)
+    DeployKey.in_projects(authorized_projects.select(:id)).distinct(:id)
   end
 
   def highest_role
@@ -941,11 +941,10 @@ class User < ApplicationRecord
   end
 
   def accessible_deploy_keys
-    @accessible_deploy_keys ||= begin
-      key_ids = project_deploy_keys.pluck(:id)
-      key_ids.push(*DeployKey.are_public.pluck(:id))
-      DeployKey.where(id: key_ids)
-    end
+    DeployKey.from_union([
+      DeployKey.where(id: project_deploy_keys.select(:deploy_key_id)),
+      DeployKey.are_public
+    ])
   end
 
   def created_by