Add latest changes from gitlab-org/gitlab@13-0-stable-ee

10 files changed, 115 insertions, 94 deletions

diff --git a/app/services/clusters/applications/base_service.rb b/app/services/clusters/applications/base_service.rb
index 86b48b5228d..39a2d6bf758 100644
--- a/app/services/clusters/applications/base_service.rb
+++ b/app/services/clusters/applications/base_service.rb
@@ -5,6 +5,8 @@ module Clusters
     class BaseService
       InvalidApplicationError = Class.new(StandardError)
 
+      FLUENTD_KNOWN_ATTRS = %i[host protocol port waf_log_enabled cilium_log_enabled].freeze
+
       attr_reader :cluster, :current_user, :params
 
       def initialize(cluster, user, params = {})
@@ -35,17 +37,7 @@ module Clusters
             application.modsecurity_mode = params[:modsecurity_mode] || 0
           end
 
-          if application.has_attribute?(:host)
-            application.host = params[:host]
-          end
-
-          if application.has_attribute?(:protocol)
-            application.protocol = params[:protocol]
-          end
-
-          if application.has_attribute?(:port)
-            application.port = params[:port]
-          end
+          apply_fluentd_related_attributes(application)
 
           if application.respond_to?(:oauth_application)
             application.oauth_application = create_oauth_application(application, request)
@@ -111,6 +103,12 @@ module Clusters
 
         ::Applications::CreateService.new(current_user, oauth_application_params).execute(request)
       end
+
+      def apply_fluentd_related_attributes(application)
+        FLUENTD_KNOWN_ATTRS.each do |attr|
+          application[attr] = params[attr] if application.has_attribute?(attr)
+        end
+      end
     end
   end
 end
diff --git a/app/services/clusters/applications/check_installation_progress_service.rb b/app/services/clusters/applications/check_installation_progress_service.rb
index 7d064abfaa3..249abd3ff9d 100644
--- a/app/services/clusters/applications/check_installation_progress_service.rb
+++ b/app/services/clusters/applications/check_installation_progress_service.rb
@@ -33,7 +33,7 @@ module Clusters
       end
 
       def timed_out?
-        Time.now.utc - app.updated_at.utc > ClusterWaitForAppInstallationWorker::TIMEOUT
+        Time.current.utc - app.updated_at.utc > ClusterWaitForAppInstallationWorker::TIMEOUT
       end
 
       def remove_installation_pod
diff --git a/app/services/clusters/applications/check_uninstall_progress_service.rb b/app/services/clusters/applications/check_uninstall_progress_service.rb
index fe9c488bdfd..cd213c3ebbf 100644
--- a/app/services/clusters/applications/check_uninstall_progress_service.rb
+++ b/app/services/clusters/applications/check_uninstall_progress_service.rb
@@ -31,7 +31,7 @@ module Clusters
       end
 
       def timed_out?
-        Time.now.utc - app.updated_at.utc > WaitForUninstallAppWorker::TIMEOUT
+        Time.current.utc - app.updated_at.utc > WaitForUninstallAppWorker::TIMEOUT
       end
 
       def remove_uninstallation_pod
diff --git a/app/services/clusters/applications/check_upgrade_progress_service.rb b/app/services/clusters/applications/check_upgrade_progress_service.rb
index 8502ea69f27..bc161218618 100644
--- a/app/services/clusters/applications/check_upgrade_progress_service.rb
+++ b/app/services/clusters/applications/check_upgrade_progress_service.rb
@@ -46,7 +46,7 @@ module Clusters
       end
 
       def timed_out?
-        Time.now.utc - app.updated_at.to_time.utc > ::ClusterWaitForAppUpdateWorker::TIMEOUT
+        Time.current.utc - app.updated_at.to_time.utc > ::ClusterWaitForAppUpdateWorker::TIMEOUT
       end
 
       def remove_pod
diff --git a/app/services/clusters/applications/ingress_modsecurity_usage_service.rb b/app/services/clusters/applications/ingress_modsecurity_usage_service.rb
deleted file mode 100644
index 4aac8bb3cbd..00000000000
--- a/app/services/clusters/applications/ingress_modsecurity_usage_service.rb
+++ /dev/null
@@ -1,69 +0,0 @@
-# frozen_string_literal: true
-
-# rubocop: disable CodeReuse/ActiveRecord
-module Clusters
-  module Applications
-    ##
-    # This service measures usage of the Modsecurity Web Application Firewall across the entire
-    # instance's deployed environments.
-    #
-    # The default configuration is`AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE=DetectionOnly` so we
-    # measure non-default values via definition of either ci_variables or ci_pipeline_variables.
-    # Since both these values are encrypted, we must decrypt and count them in memory.
-    #
-    # NOTE: this service is an approximation as it does not yet take into account `environment_scope` or `ci_group_variables`.
-    ##
-    class IngressModsecurityUsageService
-      ADO_MODSEC_KEY = "AUTO_DEVOPS_MODSECURITY_SEC_RULE_ENGINE"
-
-      def initialize(blocking_count: 0, disabled_count: 0)
-        @blocking_count = blocking_count
-        @disabled_count = disabled_count
-      end
-
-      def execute
-        conditions = -> { merge(::Environment.available).merge(::Deployment.success).where(key: ADO_MODSEC_KEY) }
-
-        ci_pipeline_var_enabled =
-          ::Ci::PipelineVariable
-            .joins(pipeline: { environments: :last_visible_deployment })
-            .merge(conditions)
-            .order('deployments.environment_id, deployments.id DESC')
-
-        ci_var_enabled =
-          ::Ci::Variable
-            .joins(project: { environments: :last_visible_deployment })
-            .merge(conditions)
-            .merge(
-              # Give priority to pipeline variables by excluding from dataset
-              ::Ci::Variable.joins(project: :environments).where.not(
-                environments: { id: ci_pipeline_var_enabled.select('DISTINCT ON (deployments.environment_id) deployments.environment_id') }
-              )
-            ).select('DISTINCT ON (deployments.environment_id) ci_variables.*')
-
-        sum_modsec_config_counts(
-          ci_pipeline_var_enabled.select('DISTINCT ON (deployments.environment_id) ci_pipeline_variables.*')
-        )
-        sum_modsec_config_counts(ci_var_enabled)
-
-        {
-          ingress_modsecurity_blocking: @blocking_count,
-          ingress_modsecurity_disabled: @disabled_count
-        }
-      end
-
-      private
-
-      # These are encrypted so we must decrypt and count in memory
-      def sum_modsec_config_counts(dataset)
-        dataset.each do |var|
-          case var.value
-          when "On" then @blocking_count += 1
-          when "Off" then @disabled_count += 1
-            # `else` could be default or any unsupported user input
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/app/services/clusters/applications/schedule_update_service.rb b/app/services/clusters/applications/schedule_update_service.rb
index b7639c771a8..41718df9a98 100644
--- a/app/services/clusters/applications/schedule_update_service.rb
+++ b/app/services/clusters/applications/schedule_update_service.rb
@@ -16,9 +16,9 @@ module Clusters
         return unless application
 
         if recently_scheduled?
-          worker_class.perform_in(BACKOFF_DELAY, application.name, application.id, project.id, Time.now)
+          worker_class.perform_in(BACKOFF_DELAY, application.name, application.id, project.id, Time.current)
         else
-          worker_class.perform_async(application.name, application.id, project.id, Time.now)
+          worker_class.perform_async(application.name, application.id, project.id, Time.current)
         end
       end
 
@@ -31,7 +31,7 @@ module Clusters
       def recently_scheduled?
         return false unless application.last_update_started_at
 
-        application.last_update_started_at.utc >= Time.now.utc - BACKOFF_DELAY
+        application.last_update_started_at.utc >= Time.current.utc - BACKOFF_DELAY
       end
     end
   end
diff --git a/app/services/clusters/gcp/verify_provision_status_service.rb b/app/services/clusters/gcp/verify_provision_status_service.rb
index b24246f5c4b..ddb2832aae6 100644
--- a/app/services/clusters/gcp/verify_provision_status_service.rb
+++ b/app/services/clusters/gcp/verify_provision_status_service.rb
@@ -35,7 +35,7 @@ module Clusters
       end
 
       def elapsed_time_from_creation(operation)
-        Time.now.utc - operation.start_time.to_time.utc
+        Time.current.utc - operation.start_time.to_time.utc
       end
 
       def finalize_creation
diff --git a/app/services/clusters/kubernetes/configure_istio_ingress_service.rb b/app/services/clusters/kubernetes/configure_istio_ingress_service.rb
index a81014d99ff..53c3c686f07 100644
--- a/app/services/clusters/kubernetes/configure_istio_ingress_service.rb
+++ b/app/services/clusters/kubernetes/configure_istio_ingress_service.rb
@@ -54,8 +54,8 @@ module Clusters
         cert = OpenSSL::X509::Certificate.new
         cert.version = 2
         cert.serial = 0
-        cert.not_before = Time.now
-        cert.not_after = Time.now + 1000.years
+        cert.not_before = Time.current
+        cert.not_after = Time.current + 1000.years
 
         cert.public_key = key.public_key
         cert.subject = name
diff --git a/app/services/clusters/management/create_project_service.rb b/app/services/clusters/management/create_project_service.rb
index 0a33582be98..5a0176edd12 100644
--- a/app/services/clusters/management/create_project_service.rb
+++ b/app/services/clusters/management/create_project_service.rb
@@ -15,11 +15,8 @@ module Clusters
       def execute
         return unless management_project_required?
 
-        ActiveRecord::Base.transaction do
-          project = create_management_project!
-
-          update_cluster!(project)
-        end
+        project = create_management_project!
+        update_cluster!(project)
       end
 
       private
diff --git a/app/services/clusters/parse_cluster_applications_artifact_service.rb b/app/services/clusters/parse_cluster_applications_artifact_service.rb
new file mode 100644
index 00000000000..b8e1c80cfe7
--- /dev/null
+++ b/app/services/clusters/parse_cluster_applications_artifact_service.rb
@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Clusters
+  class ParseClusterApplicationsArtifactService < ::BaseService
+    include Gitlab::Utils::StrongMemoize
+
+    MAX_ACCEPTABLE_ARTIFACT_SIZE = 5.kilobytes
+    RELEASE_NAMES = %w[prometheus].freeze
+
+    def initialize(job, current_user)
+      @job = job
+
+      super(job.project, current_user)
+    end
+
+    def execute(artifact)
+      return success unless Feature.enabled?(:cluster_applications_artifact, project)
+
+      raise ArgumentError, 'Artifact is not cluster_applications file type' unless artifact&.cluster_applications?
+
+      unless artifact.file.size < MAX_ACCEPTABLE_ARTIFACT_SIZE
+        return error(too_big_error_message, :bad_request)
+      end
+
+      unless cluster
+        return error(s_('ClusterIntegration|No deployment cluster found for this job'))
+      end
+
+      parse!(artifact)
+
+      success
+    rescue Gitlab::Kubernetes::Helm::Parsers::ListV2::ParserError, ActiveRecord::RecordInvalid => error
+      Gitlab::ErrorTracking.track_exception(error, job_id: artifact.job_id)
+      error(error.message, :bad_request)
+    end
+
+    private
+
+    attr_reader :job
+
+    def cluster
+      strong_memoize(:cluster) do
+        deployment_cluster = job.deployment&.cluster
+
+        deployment_cluster if Ability.allowed?(current_user, :admin_cluster, deployment_cluster)
+      end
+    end
+
+    def parse!(artifact)
+      releases = []
+
+      artifact.each_blob do |blob|
+        releases.concat(Gitlab::Kubernetes::Helm::Parsers::ListV2.new(blob).releases)
+      end
+
+      update_cluster_application_statuses!(releases)
+    end
+
+    def update_cluster_application_statuses!(releases)
+      release_by_name = releases.index_by { |release| release['Name'] }
+
+      Clusters::Cluster.transaction do
+        RELEASE_NAMES.each do |release_name|
+          application = find_or_build_application(release_name)
+
+          release = release_by_name[release_name]
+
+          if release
+            case release['Status']
+            when 'DEPLOYED'
+              application.make_externally_installed!
+            when 'FAILED'
+              application.make_errored!(s_('ClusterIntegration|Helm release failed to install'))
+            end
+          else
+            # missing, so by definition, we consider this uninstalled
+            application.make_externally_uninstalled! if application.persisted?
+          end
+        end
+      end
+    end
+
+    def find_or_build_application(application_name)
+      application_class = Clusters::Cluster::APPLICATIONS[application_name]
+
+      cluster.find_or_build_application(application_class)
+    end
+
+    def too_big_error_message
+      human_size = ActiveSupport::NumberHelper.number_to_human_size(MAX_ACCEPTABLE_ARTIFACT_SIZE)
+
+      s_('ClusterIntegration|Cluster_applications artifact too big. Maximum allowable size: %{human_size}') % { human_size: human_size }
+    end
+  end
+end