Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43

9 files changed, 129 insertions, 45 deletions

diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb
index 9d79887b574..d42279502c5 100644
--- a/app/models/clusters/agent_token.rb
+++ b/app/models/clusters/agent_token.rb
@@ -2,17 +2,45 @@
 
 module Clusters
   class AgentToken < ApplicationRecord
+    include RedisCacheable
     include TokenAuthenticatable
+
     add_authentication_token_field :token, encrypted: :required, token_generator: -> { Devise.friendly_token(50) }
+    cached_attr_reader :last_contacted_at
 
     self.table_name = 'cluster_agent_tokens'
 
+    # The `UPDATE_USED_COLUMN_EVERY` defines how often the token DB entry can be updated
+    UPDATE_USED_COLUMN_EVERY = (40.minutes..55.minutes).freeze
+
     belongs_to :agent, class_name: 'Clusters::Agent', optional: false
     belongs_to :created_by_user, class_name: 'User', optional: true
 
     before_save :ensure_token
 
     validates :description, length: { maximum: 1024 }
-    validates :name, presence: true, length: { maximum: 255 }, on: :create
+    validates :name, presence: true, length: { maximum: 255 }
+
+    def track_usage
+      track_values = { last_used_at: Time.current.utc }
+
+      cache_attributes(track_values)
+
+      # Use update_column so updated_at is skipped
+      update_columns(track_values) if can_update_track_values?
+    end
+
+    private
+
+    def can_update_track_values?
+      # Use a random threshold to prevent beating DB updates.
+      last_used_at_max_age = Random.rand(UPDATE_USED_COLUMN_EVERY)
+
+      real_last_used_at = read_attribute(:last_used_at)
+
+      # Handle too many updates from high token traffic
+      real_last_used_at.nil? ||
+        (Time.current - real_last_used_at) >= last_used_at_max_age
+    end
   end
 end
diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb
index 55a9a0ccb81..b9c136abab4 100644
--- a/app/models/clusters/applications/prometheus.rb
+++ b/app/models/clusters/applications/prometheus.rb
@@ -3,7 +3,7 @@
 module Clusters
   module Applications
     class Prometheus < ApplicationRecord
-      include PrometheusAdapter
+      include ::Clusters::Concerns::PrometheusClient
 
       VERSION = '10.4.1'
 
@@ -32,7 +32,7 @@ module Clusters
       end
 
       state_machine :status do
-        after_transition any => [:installed] do |application|
+        after_transition any => [:installed, :externally_installed] do |application|
           application.run_after_commit do
             Clusters::Applications::ActivateServiceWorker
               .perform_async(application.cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass
@@ -58,14 +58,6 @@ module Clusters
         'https://gitlab-org.gitlab.io/cluster-integration/helm-stable-archive'
       end
 
-      def service_name
-        'prometheus-prometheus-server'
-      end
-
-      def service_port
-        80
-      end
-
       def install_command
         helm_command_module::InstallCommand.new(
           name: name,
@@ -106,29 +98,6 @@ module Clusters
         files.merge('values.yaml': replaced_values)
       end
 
-      def prometheus_client
-        return unless kube_client
-
-        proxy_url = kube_client.proxy_url('service', service_name, service_port, Gitlab::Kubernetes::Helm::NAMESPACE)
-
-        # ensures headers containing auth data are appended to original k8s client options
-        options = kube_client.rest_client.options
-          .merge(prometheus_client_default_options)
-          .merge(headers: kube_client.headers)
-        Gitlab::PrometheusClient.new(proxy_url, options)
-      rescue Kubeclient::HttpError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::ENETUNREACH
-        # If users have mistakenly set parameters or removed the depended clusters,
-        # `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
-        # Since `PrometheusAdapter#can_query?` is eargely loaded on environement pages in gitlab,
-        # we need to silence the exceptions
-      end
-
-      def configured?
-        kube_client.present? && available?
-      rescue Gitlab::UrlBlocker::BlockedUrlError
-        false
-      end
-
       def generate_alert_manager_token!
         unless alert_manager_token.present?
           update!(alert_manager_token: generate_token)
@@ -146,10 +115,6 @@ module Clusters
           .perform_async(cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass
       end
 
-      def kube_client
-        cluster&.kubeclient&.core_client
-      end
-
       def install_knative_metrics
         return [] unless cluster.application_knative_available?
 
diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb
index 8a49d476ba7..bc80bcd0b06 100644
--- a/app/models/clusters/applications/runner.rb
+++ b/app/models/clusters/applications/runner.rb
@@ -3,7 +3,7 @@
 module Clusters
   module Applications
     class Runner < ApplicationRecord
-      VERSION = '0.26.0'
+      VERSION = '0.27.0'
 
       self.table_name = 'clusters_applications_runners'
 
diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb
index a34d8a6b98d..a1e2aa194a0 100644
--- a/app/models/clusters/cluster.rb
+++ b/app/models/clusters/cluster.rb
@@ -51,6 +51,8 @@ module Clusters
 
     has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', inverse_of: :cluster, autosave: true
 
+    has_one :integration_prometheus, class_name: 'Clusters::Integrations::Prometheus', inverse_of: :cluster
+
     def self.has_one_cluster_application(name) # rubocop:disable Naming/PredicateName
       application = APPLICATIONS[name.to_s]
       has_one application.association_name, class_name: application.to_s, inverse_of: :cluster # rubocop:disable Rails/ReflectionClassName
@@ -100,7 +102,6 @@ module Clusters
     delegate :rbac?, to: :platform_kubernetes, prefix: true, allow_nil: true
     delegate :available?, to: :application_helm, prefix: true, allow_nil: true
     delegate :available?, to: :application_ingress, prefix: true, allow_nil: true
-    delegate :available?, to: :application_prometheus, prefix: true, allow_nil: true
     delegate :available?, to: :application_knative, prefix: true, allow_nil: true
     delegate :available?, to: :application_elastic_stack, prefix: true, allow_nil: true
     delegate :external_ip, to: :application_ingress, prefix: true, allow_nil: true
@@ -148,6 +149,9 @@ module Clusters
     scope :with_management_project, -> { where.not(management_project: nil) }
 
     scope :for_project_namespace, -> (namespace_id) { joins(:projects).where(projects: { namespace_id: namespace_id }) }
+
+    # with_application_prometheus scope is deprecated, and scheduled for removal
+    # in %14.0. See https://gitlab.com/groups/gitlab-org/-/epics/4280
     scope :with_application_prometheus, -> { includes(:application_prometheus).joins(:application_prometheus) }
     scope :with_project_http_integrations, -> (project_ids) do
       conditions = { projects: :alert_management_http_integrations }
@@ -276,6 +280,10 @@ module Clusters
       public_send(association_name) || public_send("build_#{association_name}") # rubocop:disable GitlabSecurity/PublicSend
     end
 
+    def find_or_build_integration_prometheus
+      integration_prometheus || build_integration_prometheus
+    end
+
     def provider
       if gcp?
         provider_gcp
@@ -361,8 +369,12 @@ module Clusters
       end
     end
 
+    def application_prometheus_available?
+      integration_prometheus&.available? || application_prometheus&.available?
+    end
+
     def prometheus_adapter
-      application_prometheus
+      integration_prometheus || application_prometheus
     end
 
     private
diff --git a/app/models/clusters/clusters_hierarchy.rb b/app/models/clusters/clusters_hierarchy.rb
index c9c18d8c96a..125783e6ee1 100644
--- a/app/models/clusters/clusters_hierarchy.rb
+++ b/app/models/clusters/clusters_hierarchy.rb
@@ -16,7 +16,7 @@ module Clusters
 
       model
         .unscoped
-        .where('clusters.id IS NOT NULL')
+        .where.not('clusters.id' => nil)
         .with
         .recursive(cte.to_arel)
         .from(cte_alias)
diff --git a/app/models/clusters/concerns/application_status.rb b/app/models/clusters/concerns/application_status.rb
index 95ac95448dd..7485ee079ce 100644
--- a/app/models/clusters/concerns/application_status.rb
+++ b/app/models/clusters/concerns/application_status.rb
@@ -9,6 +9,7 @@ module Clusters
         scope :available, -> do
           where(
             status: [
+              self.state_machines[:status].states[:externally_installed].value,
               self.state_machines[:status].states[:installed].value,
               self.state_machines[:status].states[:updated].value
             ]
@@ -28,6 +29,7 @@ module Clusters
           state :uninstalling, value: 7
           state :uninstall_errored, value: 8
           state :uninstalled, value: 10
+          state :externally_installed, value: 11
 
           # Used for applications that are pre-installed by the cluster,
           # e.g. Knative in GCP Cloud Run enabled clusters
@@ -37,7 +39,7 @@ module Clusters
           state :pre_installed, value: 9
 
           event :make_externally_installed do
-            transition any => :installed
+            transition any => :externally_installed
           end
 
           event :make_externally_uninstalled do
@@ -79,7 +81,7 @@ module Clusters
             transition [:scheduled] => :uninstalling
           end
 
-          before_transition any => [:scheduled, :installed, :uninstalled] do |application, _|
+          before_transition any => [:scheduled, :installed, :uninstalled, :externally_installed] do |application, _|
             application.status_reason = nil
           end
 
@@ -114,7 +116,7 @@ module Clusters
       end
 
       def available?
-        pre_installed? || installed? || updated?
+        pre_installed? || installed? || externally_installed? || updated?
       end
 
       def update_in_progress?
diff --git a/app/models/clusters/concerns/application_version.rb b/app/models/clusters/concerns/application_version.rb
index 6c0b014662c..dab0bd23e2e 100644
--- a/app/models/clusters/concerns/application_version.rb
+++ b/app/models/clusters/concerns/application_version.rb
@@ -5,11 +5,17 @@ module Clusters
     module ApplicationVersion
       extend ActiveSupport::Concern
 
+      EXTERNAL_VERSION = 'EXTERNALLY_INSTALLED'
+
       included do
         state_machine :status do
           before_transition any => [:installed, :updated] do |application|
             application.version = application.class.const_get(:VERSION, false)
           end
+
+          before_transition any => [:externally_installed] do |application|
+            application.version = EXTERNAL_VERSION
+          end
         end
       end
 
diff --git a/app/models/clusters/concerns/prometheus_client.rb b/app/models/clusters/concerns/prometheus_client.rb
new file mode 100644
index 00000000000..10cb307addd
--- /dev/null
+++ b/app/models/clusters/concerns/prometheus_client.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Clusters
+  module Concerns
+    module PrometheusClient
+      extend ActiveSupport::Concern
+
+      included do
+        include PrometheusAdapter
+
+        def service_name
+          'prometheus-prometheus-server'
+        end
+
+        def service_port
+          80
+        end
+
+        def prometheus_client
+          return unless kube_client
+
+          proxy_url = kube_client.proxy_url('service', service_name, service_port, Gitlab::Kubernetes::Helm::NAMESPACE)
+
+          # ensures headers containing auth data are appended to original k8s client options
+          options = kube_client.rest_client.options
+            .merge(prometheus_client_default_options)
+            .merge(headers: kube_client.headers)
+          Gitlab::PrometheusClient.new(proxy_url, options)
+        rescue Kubeclient::HttpError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::ENETUNREACH
+          # If users have mistakenly set parameters or removed the depended clusters,
+          # `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
+          # Since `PrometheusAdapter#can_query?` is eargely loaded on environement pages in gitlab,
+          # we need to silence the exceptions
+        end
+
+        def configured?
+          kube_client.present? && available?
+        rescue Gitlab::UrlBlocker::BlockedUrlError
+          false
+        end
+
+        private
+
+        def kube_client
+          cluster&.kubeclient&.core_client
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/clusters/integrations/prometheus.rb b/app/models/clusters/integrations/prometheus.rb
new file mode 100644
index 00000000000..1496d8ff1dd
--- /dev/null
+++ b/app/models/clusters/integrations/prometheus.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Clusters
+  module Integrations
+    class Prometheus < ApplicationRecord
+      include ::Clusters::Concerns::PrometheusClient
+
+      self.table_name = 'clusters_integration_prometheus'
+      self.primary_key = :cluster_id
+
+      belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
+
+      validates :cluster, presence: true
+      validates :enabled, inclusion: { in: [true, false] }
+
+      def available?
+        enabled?
+      end
+    end
+  end
+end