Add latest changes from gitlab-org/gitlab@13-12-stable-eev13.12.0-rc42

12 files changed, 199 insertions, 78 deletions

diff --git a/app/models/clusters/agent.rb b/app/models/clusters/agent.rb
index c5b9dddb1da..9fb8cd024c5 100644
--- a/app/models/clusters/agent.rb
+++ b/app/models/clusters/agent.rb
@@ -8,6 +8,7 @@ module Clusters
     belongs_to :project, class_name: '::Project' # Otherwise, it will load ::Clusters::Project
 
     has_many :agent_tokens, class_name: 'Clusters::AgentToken'
+    has_many :last_used_agent_tokens, -> { order_last_used_at_desc }, class_name: 'Clusters::AgentToken', inverse_of: :agent
 
     scope :ordered_by_name, -> { order(:name) }
     scope :with_name, -> (name) { where(name: name) }
diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb
index d42279502c5..27a3cd8d13d 100644
--- a/app/models/clusters/agent_token.rb
+++ b/app/models/clusters/agent_token.rb
@@ -6,7 +6,7 @@ module Clusters
     include TokenAuthenticatable
 
     add_authentication_token_field :token, encrypted: :required, token_generator: -> { Devise.friendly_token(50) }
-    cached_attr_reader :last_contacted_at
+    cached_attr_reader :last_used_at
 
     self.table_name = 'cluster_agent_tokens'
 
@@ -21,6 +21,8 @@ module Clusters
     validates :description, length: { maximum: 1024 }
     validates :name, presence: true, length: { maximum: 255 }
 
+    scope :order_last_used_at_desc, -> { order(::Gitlab::Database.nulls_last_order('last_used_at', 'DESC')) }
+
     def track_usage
       track_values = { last_used_at: Time.current.utc }
 
diff --git a/app/models/clusters/applications/elastic_stack.rb b/app/models/clusters/applications/elastic_stack.rb
index db18a29ec84..73c731aab1a 100644
--- a/app/models/clusters/applications/elastic_stack.rb
+++ b/app/models/clusters/applications/elastic_stack.rb
@@ -3,9 +3,9 @@
 module Clusters
   module Applications
     class ElasticStack < ApplicationRecord
-      VERSION = '3.0.0'
+      include ::Clusters::Concerns::ElasticsearchClient
 
-      ELASTICSEARCH_PORT = 9200
+      VERSION = '3.0.0'
 
       self.table_name = 'clusters_applications_elastic_stacks'
 
@@ -13,10 +13,23 @@ module Clusters
       include ::Clusters::Concerns::ApplicationStatus
       include ::Clusters::Concerns::ApplicationVersion
       include ::Clusters::Concerns::ApplicationData
-      include ::Gitlab::Utils::StrongMemoize
 
       default_value_for :version, VERSION
 
+      after_destroy do
+        cluster&.find_or_build_integration_elastic_stack&.update(enabled: false, chart_version: nil)
+      end
+
+      state_machine :status do
+        after_transition any => [:installed] do |application|
+          application.cluster&.find_or_build_integration_elastic_stack&.update(enabled: true, chart_version: application.version)
+        end
+
+        after_transition any => [:uninstalled] do |application|
+          application.cluster&.find_or_build_integration_elastic_stack&.update(enabled: false, chart_version: nil)
+        end
+      end
+
       def chart
         'elastic-stack/elastic-stack'
       end
@@ -51,31 +64,6 @@ module Clusters
         super.merge('wait-for-elasticsearch.sh': File.read("#{Rails.root}/vendor/elastic_stack/wait-for-elasticsearch.sh"))
       end
 
-      def elasticsearch_client(timeout: nil)
-        strong_memoize(:elasticsearch_client) do
-          next unless kube_client
-
-          proxy_url = kube_client.proxy_url('service', service_name, ::Clusters::Applications::ElasticStack::ELASTICSEARCH_PORT, Gitlab::Kubernetes::Helm::NAMESPACE)
-
-          Elasticsearch::Client.new(url: proxy_url) do |faraday|
-            # ensures headers containing auth data are appended to original client options
-            faraday.headers.merge!(kube_client.headers)
-            # ensure TLS certs are properly verified
-            faraday.ssl[:verify] = kube_client.ssl_options[:verify_ssl]
-            faraday.ssl[:cert_store] = kube_client.ssl_options[:cert_store]
-            faraday.options.timeout = timeout unless timeout.nil?
-          end
-
-        rescue Kubeclient::HttpError => error
-          # If users have mistakenly set parameters or removed the depended clusters,
-          # `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
-          # We check for a nil client in downstream use and behaviour is equivalent to an empty state
-          log_exception(error, :failed_to_create_elasticsearch_client)
-
-          nil
-        end
-      end
-
       def chart_above_v2?
         Gem::Version.new(version) >= Gem::Version.new('2.0.0')
       end
@@ -106,10 +94,6 @@ module Clusters
         ]
       end
 
-      def kube_client
-        cluster&.kubeclient&.core_client
-      end
-
       def migrate_to_3_script
         return [] if !updating? || chart_above_v3?
 
diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb
index b9c136abab4..21f7e410843 100644
--- a/app/models/clusters/applications/prometheus.rb
+++ b/app/models/clusters/applications/prometheus.rb
@@ -22,21 +22,18 @@ module Clusters
 
       attr_encrypted :alert_manager_token,
         mode: :per_attribute_iv,
-        key: Settings.attr_encrypted_db_key_base_truncated,
+        key: Settings.attr_encrypted_db_key_base_32,
         algorithm: 'aes-256-gcm'
 
+      default_value_for(:alert_manager_token) { SecureRandom.hex }
+
       after_destroy do
-        run_after_commit do
-          disable_prometheus_integration
-        end
+        cluster.find_or_build_integration_prometheus.destroy
       end
 
       state_machine :status do
         after_transition any => [:installed, :externally_installed] do |application|
-          application.run_after_commit do
-            Clusters::Applications::ActivateServiceWorker
-              .perform_async(application.cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass
-          end
+          application.cluster.find_or_build_integration_prometheus.update(enabled: true, alert_manager_token: application.alert_manager_token)
         end
 
         after_transition any => :updating do |application|
@@ -44,6 +41,10 @@ module Clusters
         end
       end
 
+      def managed_prometheus?
+        !externally_installed? && !uninstalled?
+      end
+
       def updated_since?(timestamp)
         last_update_started_at &&
           last_update_started_at > timestamp &&
@@ -70,6 +71,7 @@ module Clusters
         )
       end
 
+      # Deprecated, to be removed in %14.0 as part of https://gitlab.com/groups/gitlab-org/-/epics/4280
       def patch_command(values)
         helm_command_module::PatchCommand.new(
           name: name,
@@ -98,23 +100,8 @@ module Clusters
         files.merge('values.yaml': replaced_values)
       end
 
-      def generate_alert_manager_token!
-        unless alert_manager_token.present?
-          update!(alert_manager_token: generate_token)
-        end
-      end
-
       private
 
-      def generate_token
-        SecureRandom.hex
-      end
-
-      def disable_prometheus_integration
-        ::Clusters::Applications::DeactivateServiceWorker
-          .perform_async(cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass
-      end
-
       def install_knative_metrics
         return [] unless cluster.application_knative_available?
 
diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb
index bc80bcd0b06..e8d56072b89 100644
--- a/app/models/clusters/applications/runner.rb
+++ b/app/models/clusters/applications/runner.rb
@@ -3,7 +3,7 @@
 module Clusters
   module Applications
     class Runner < ApplicationRecord
-      VERSION = '0.27.0'
+      VERSION = '0.28.0'
 
       self.table_name = 'clusters_applications_runners'
 
diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb
index a1e2aa194a0..4877ced795c 100644
--- a/app/models/clusters/cluster.rb
+++ b/app/models/clusters/cluster.rb
@@ -52,6 +52,7 @@ module Clusters
     has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', inverse_of: :cluster, autosave: true
 
     has_one :integration_prometheus, class_name: 'Clusters::Integrations::Prometheus', inverse_of: :cluster
+    has_one :integration_elastic_stack, class_name: 'Clusters::Integrations::ElasticStack', inverse_of: :cluster
 
     def self.has_one_cluster_application(name) # rubocop:disable Naming/PredicateName
       application = APPLICATIONS[name.to_s]
@@ -104,6 +105,7 @@ module Clusters
     delegate :available?, to: :application_ingress, prefix: true, allow_nil: true
     delegate :available?, to: :application_knative, prefix: true, allow_nil: true
     delegate :available?, to: :application_elastic_stack, prefix: true, allow_nil: true
+    delegate :available?, to: :integration_elastic_stack, prefix: true, allow_nil: true
     delegate :external_ip, to: :application_ingress, prefix: true, allow_nil: true
     delegate :external_hostname, to: :application_ingress, prefix: true, allow_nil: true
 
@@ -284,6 +286,10 @@ module Clusters
       integration_prometheus || build_integration_prometheus
     end
 
+    def find_or_build_integration_elastic_stack
+      integration_elastic_stack || build_integration_elastic_stack
+    end
+
     def provider
       if gcp?
         provider_gcp
@@ -318,6 +324,22 @@ module Clusters
       platform_kubernetes.kubeclient if kubernetes?
     end
 
+    def elastic_stack_adapter
+      application_elastic_stack || integration_elastic_stack
+    end
+
+    def elasticsearch_client
+      elastic_stack_adapter&.elasticsearch_client
+    end
+
+    def elastic_stack_available?
+      if application_elastic_stack_available? || integration_elastic_stack_available?
+        true
+      else
+        false
+      end
+    end
+
     def kubernetes_namespace_for(environment, deployable: environment.last_deployable)
       if deployable && environment.project_id != deployable.project_id
         raise ArgumentError, 'environment.project_id must match deployable.project_id'
@@ -470,4 +492,4 @@ module Clusters
   end
 end
 
-Clusters::Cluster.prepend_if_ee('EE::Clusters::Cluster')
+Clusters::Cluster.prepend_mod_with('Clusters::Cluster')
diff --git a/app/models/clusters/concerns/application_core.rb b/app/models/clusters/concerns/application_core.rb
index ad6699daa78..2e40689a650 100644
--- a/app/models/clusters/concerns/application_core.rb
+++ b/app/models/clusters/concerns/application_core.rb
@@ -6,6 +6,8 @@ module Clusters
       extend ActiveSupport::Concern
 
       included do
+        include ::Clusters::Concerns::KubernetesLogger
+
         belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
 
         validates :cluster, presence: true
@@ -79,27 +81,9 @@ module Clusters
           # Override if your application needs any action after
           # being uninstalled by Helm
         end
-
-        def logger
-          @logger ||= Gitlab::Kubernetes::Logger.build
-        end
-
-        def log_exception(error, event)
-          logger.error({
-            exception: error.class.name,
-            status_code: error.error_code,
-            cluster_id: cluster&.id,
-            application_id: id,
-            class_name: self.class.name,
-            event: event,
-            message: error.message
-          })
-
-          Gitlab::ErrorTracking.track_exception(error, cluster_id: cluster&.id, application_id: id)
-        end
       end
     end
   end
 end
 
-Clusters::Concerns::ApplicationCore.prepend_if_ee('EE::Clusters::Concerns::ApplicationCore')
+Clusters::Concerns::ApplicationCore.prepend_mod_with('Clusters::Concerns::ApplicationCore')
diff --git a/app/models/clusters/concerns/elasticsearch_client.rb b/app/models/clusters/concerns/elasticsearch_client.rb
new file mode 100644
index 00000000000..7b0b6bdae02
--- /dev/null
+++ b/app/models/clusters/concerns/elasticsearch_client.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Clusters
+  module Concerns
+    module ElasticsearchClient
+      include ::Gitlab::Utils::StrongMemoize
+
+      ELASTICSEARCH_PORT = 9200
+      ELASTICSEARCH_NAMESPACE = 'gitlab-managed-apps'
+
+      def elasticsearch_client(timeout: nil)
+        strong_memoize(:elasticsearch_client) do
+          kube_client = cluster&.kubeclient&.core_client
+          next unless kube_client
+
+          proxy_url = kube_client.proxy_url('service', service_name, ELASTICSEARCH_PORT, ELASTICSEARCH_NAMESPACE)
+
+          Elasticsearch::Client.new(url: proxy_url) do |faraday|
+            # ensures headers containing auth data are appended to original client options
+            faraday.headers.merge!(kube_client.headers)
+            # ensure TLS certs are properly verified
+            faraday.ssl[:verify] = kube_client.ssl_options[:verify_ssl]
+            faraday.ssl[:cert_store] = kube_client.ssl_options[:cert_store]
+            faraday.options.timeout = timeout unless timeout.nil?
+          end
+
+        rescue Kubeclient::HttpError => error
+          # If users have mistakenly set parameters or removed the depended clusters,
+          # `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
+          # We check for a nil client in downstream use and behaviour is equivalent to an empty state
+          log_exception(error, :failed_to_create_elasticsearch_client)
+
+          nil
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/clusters/concerns/kubernetes_logger.rb b/app/models/clusters/concerns/kubernetes_logger.rb
new file mode 100644
index 00000000000..2eca33a7610
--- /dev/null
+++ b/app/models/clusters/concerns/kubernetes_logger.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Clusters
+  module Concerns
+    module KubernetesLogger
+      def logger
+        @logger ||= Gitlab::Kubernetes::Logger.build
+      end
+
+      def log_exception(error, event)
+        logger.error(
+          {
+            exception: error.class.name,
+            status_code: error.error_code,
+            cluster_id: cluster&.id,
+            application_id: id,
+            class_name: self.class.name,
+            event: event,
+            message: error.message
+          }
+        )
+
+        Gitlab::ErrorTracking.track_exception(error, cluster_id: cluster&.id, application_id: id)
+      end
+    end
+  end
+end
diff --git a/app/models/clusters/integrations/elastic_stack.rb b/app/models/clusters/integrations/elastic_stack.rb
new file mode 100644
index 00000000000..565d268259a
--- /dev/null
+++ b/app/models/clusters/integrations/elastic_stack.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Clusters
+  module Integrations
+    class ElasticStack < ApplicationRecord
+      include ::Clusters::Concerns::ElasticsearchClient
+      include ::Clusters::Concerns::KubernetesLogger
+
+      self.table_name = 'clusters_integration_elasticstack'
+      self.primary_key = :cluster_id
+
+      belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
+
+      validates :cluster, presence: true
+      validates :enabled, inclusion: { in: [true, false] }
+
+      def available?
+        enabled
+      end
+
+      def service_name
+        chart_above_v3? ? 'elastic-stack-elasticsearch-master' : 'elastic-stack-elasticsearch-client'
+      end
+
+      def chart_above_v2?
+        return true if chart_version.nil?
+
+        Gem::Version.new(chart_version) >= Gem::Version.new('2.0.0')
+      end
+
+      def chart_above_v3?
+        return true if chart_version.nil?
+
+        Gem::Version.new(chart_version) >= Gem::Version.new('3.0.0')
+      end
+    end
+  end
+end
diff --git a/app/models/clusters/integrations/prometheus.rb b/app/models/clusters/integrations/prometheus.rb
index 1496d8ff1dd..0a01ac5d1ce 100644
--- a/app/models/clusters/integrations/prometheus.rb
+++ b/app/models/clusters/integrations/prometheus.rb
@@ -4,6 +4,7 @@ module Clusters
   module Integrations
     class Prometheus < ApplicationRecord
       include ::Clusters::Concerns::PrometheusClient
+      include AfterCommitQueue
 
       self.table_name = 'clusters_integration_prometheus'
       self.primary_key = :cluster_id
@@ -13,9 +14,46 @@ module Clusters
       validates :cluster, presence: true
       validates :enabled, inclusion: { in: [true, false] }
 
+      attr_encrypted :alert_manager_token,
+        mode: :per_attribute_iv,
+        key: Settings.attr_encrypted_db_key_base_32,
+        algorithm: 'aes-256-gcm'
+
+      default_value_for(:alert_manager_token) { SecureRandom.hex }
+
+      after_destroy do
+        run_after_commit do
+          deactivate_project_services
+        end
+      end
+
+      after_save do
+        next unless enabled_before_last_save != enabled
+
+        run_after_commit do
+          if enabled
+            activate_project_services
+          else
+            deactivate_project_services
+          end
+        end
+      end
+
       def available?
         enabled?
       end
+
+      private
+
+      def activate_project_services
+        ::Clusters::Applications::ActivateServiceWorker
+          .perform_async(cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass
+      end
+
+      def deactivate_project_services
+        ::Clusters::Applications::DeactivateServiceWorker
+          .perform_async(cluster_id, ::PrometheusService.to_param) # rubocop:disable CodeReuse/ServiceClass
+      end
     end
   end
 end
diff --git a/app/models/clusters/providers/aws.rb b/app/models/clusters/providers/aws.rb
index bfd01775620..af2eba42721 100644
--- a/app/models/clusters/providers/aws.rb
+++ b/app/models/clusters/providers/aws.rb
@@ -18,7 +18,7 @@ module Clusters
 
       attr_encrypted :secret_access_key,
         mode: :per_attribute_iv,
-        key: Settings.attr_encrypted_db_key_base_truncated,
+        key: Settings.attr_encrypted_db_key_base_32,
         algorithm: 'aes-256-gcm'
 
       validates :role_arn,