Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42

7 files changed, 94 insertions, 34 deletions

diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb
index d1490d5d9b6..8a176be30a2 100644
--- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb
+++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb
@@ -13,11 +13,12 @@ module Gitlab
             @dashboard_hash = dashboard_hash
             @project = project
             @dashboard_path = dashboard_path
+            @affected_environment_ids = []
           end
 
           def execute
             import
-          rescue ActiveRecord::RecordInvalid, ::Gitlab::Metrics::Dashboard::Transformers::TransformerError
+          rescue ActiveRecord::RecordInvalid, Dashboard::Transformers::Errors::BaseError
             false
           end
 
@@ -32,28 +33,51 @@ module Gitlab
           def import
             delete_stale_metrics
             create_or_update_metrics
+            update_prometheus_environments
           end
 
           # rubocop: disable CodeReuse/ActiveRecord
           def create_or_update_metrics
             # TODO: use upsert and worker for callbacks?
+
+            affected_metric_ids = []
             prometheus_metrics_attributes.each do |attributes|
-              prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:identifier, :project))
+              prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:dashboard_path, :identifier, :project))
               prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES))
+
+              affected_metric_ids << prometheus_metric.id
             end
+
+            @affected_environment_ids += find_alerts(affected_metric_ids).get_environment_id
           end
           # rubocop: enable CodeReuse/ActiveRecord
 
           def delete_stale_metrics
-            identifiers = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] }
+            identifiers_from_yml = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] }
 
             stale_metrics = PrometheusMetric.for_project(project)
               .for_dashboard_path(dashboard_path)
               .for_group(Enums::PrometheusMetric.groups[:custom])
-              .not_identifier(identifiers)
+              .not_identifier(identifiers_from_yml)
+
+            return unless stale_metrics.exists?
+
+            delete_stale_alerts(stale_metrics)
+            stale_metrics.each_batch { |batch| batch.delete_all }
+          end
+
+          def delete_stale_alerts(stale_metrics)
+            stale_alerts = find_alerts(stale_metrics)
+
+            affected_environment_ids = stale_alerts.get_environment_id
+            return unless affected_environment_ids.present?
 
-            # TODO: use destroy_all and worker for callbacks?
-            stale_metrics.each(&:destroy)
+            @affected_environment_ids += affected_environment_ids
+            stale_alerts.each_batch { |batch| batch.delete_all }
+          end
+
+          def find_alerts(metrics)
+            Projects::Prometheus::AlertsFinder.new(project: project, metric: metrics).execute
           end
 
           def prometheus_metrics_attributes
@@ -65,6 +89,19 @@ module Gitlab
               ).execute
             end
           end
+
+          def update_prometheus_environments
+            affected_environments = ::Environment.for_id(@affected_environment_ids.flatten.uniq).for_project(project)
+
+            return unless affected_environments.exists?
+
+            affected_environments.each do |affected_environment|
+              ::Clusters::Applications::ScheduleUpdateService.new(
+                affected_environment.cluster_prometheus_adapter,
+                project
+              ).execute
+            end
+          end
         end
       end
     end
diff --git a/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb
new file mode 100644
index 00000000000..5ed4466f440
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        # Acts on metrics which have been ingested from source controlled dashboards
+        class CustomDashboardMetricsInserter < BaseStage
+          # For each metric in the dashboard config, attempts to
+          # find a corresponding database record. If found, includes
+          # the record's id in the dashboard config.
+          def transform!
+            database_metrics = ::PrometheusMetricsFinder.new(common: false, group: :custom, project: project).execute
+
+            for_metrics do |metric|
+              metric_record = database_metrics.find { |m| m.identifier == metric[:id] }
+              metric[:metric_id] = metric_record.id if metric_record
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb
index 9e2bb0d1a70..ad9d78133af 100644
--- a/lib/gitlab/metrics/dashboard/stages/url_validator.rb
+++ b/lib/gitlab/metrics/dashboard/stages/url_validator.rb
@@ -46,7 +46,7 @@ module Gitlab
             links&.each do |link|
               next unless link.is_a? Hash
 
-              Gitlab::UrlBlocker.validate!(link[:url], blocker_args)
+              Gitlab::UrlBlocker.validate!(link[:url], **blocker_args)
             rescue Gitlab::UrlBlocker::BlockedUrlError
               link[:url] = ''
             end
diff --git a/lib/gitlab/metrics/dashboard/transformers/errors.rb b/lib/gitlab/metrics/dashboard/transformers/errors.rb
index 4d94ab098ae..bc85dc4e131 100644
--- a/lib/gitlab/metrics/dashboard/transformers/errors.rb
+++ b/lib/gitlab/metrics/dashboard/transformers/errors.rb
@@ -4,10 +4,10 @@ module Gitlab
   module Metrics
     module Dashboard
       module Transformers
-        TransformerError = Class.new(StandardError)
-
         module Errors
-          class MissingAttribute < TransformerError
+          BaseError = Class.new(StandardError)
+
+          class MissingAttribute < BaseError
             def initialize(attribute_name)
               super("Missing attribute: '#{attribute_name}'")
             end
diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb
index 15db3999fa4..f6bda0dbea4 100644
--- a/lib/gitlab/metrics/requests_rack_middleware.rb
+++ b/lib/gitlab/metrics/requests_rack_middleware.rb
@@ -3,20 +3,13 @@
 module Gitlab
   module Metrics
     class RequestsRackMiddleware
-      HTTP_METHODS = {
-        "delete" => %w(200 202 204 303 400 401 403 404 410 422 500 503),
-        "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 412 422 429 500 503),
-        "head" => %w(200 204 301 302 303 304 400 401 403 404 410 429 500 503),
-        "options" => %w(200 404),
-        "patch" => %w(200 202 204 400 403 404 409 416 422 500),
-        "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 413 415 422 429 500 503),
-        "propfind" => %w(404),
-        "put" => %w(200 202 204 400 401 403 404 405 406 409 410 415 422 500),
-        "report" =>  %w(404)
-      }.freeze
+      HTTP_METHODS = %w(delete get head options patch post put).to_set.freeze
 
       HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze
 
+      FEATURE_CATEGORY_HEADER = 'X-Gitlab-Feature-Category'
+      FEATURE_CATEGORY_DEFAULT = 'unknown'
+
       def initialize(app)
         @app = app
       end
@@ -39,33 +32,39 @@ module Gitlab
       end
 
       def self.initialize_http_request_duration_seconds
-        HTTP_METHODS.each do |method, statuses|
-          statuses.each do |status|
-            http_request_duration_seconds.get({ method: method, status: status.to_s })
-          end
+        HTTP_METHODS.each do |method|
+          http_request_duration_seconds.get({ method: method })
         end
       end
 
       def call(env)
         method = env['REQUEST_METHOD'].downcase
+        method = 'INVALID' unless HTTP_METHODS.include?(method)
         started = Time.now.to_f
+        health_endpoint = health_endpoint?(env['PATH_INFO'])
+        status = 'undefined'
+        feature_category = nil
 
         begin
-          if health_endpoint?(env['PATH_INFO'])
-            RequestsRackMiddleware.http_health_requests_total.increment(method: method)
-          else
-            RequestsRackMiddleware.http_request_total.increment(method: method)
-          end
-
           status, headers, body = @app.call(env)
 
           elapsed = Time.now.to_f - started
-          RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status.to_s }, elapsed)
+          feature_category = headers&.fetch(FEATURE_CATEGORY_HEADER, nil)
+
+          unless health_endpoint
+            RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method }, elapsed)
+          end
 
           [status, headers, body]
         rescue
           RequestsRackMiddleware.rack_uncaught_errors_count.increment
           raise
+        ensure
+          if health_endpoint
+            RequestsRackMiddleware.http_health_requests_total.increment(status: status, method: method)
+          else
+            RequestsRackMiddleware.http_request_total.increment(status: status, method: method, feature_category: feature_category || FEATURE_CATEGORY_DEFAULT)
+          end
         end
       end
 
diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb
index 8c4d150adad..d7935d65e12 100644
--- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb
+++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb
@@ -54,7 +54,7 @@ module Gitlab
         end
 
         def unicorn_workers_count
-          http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord
+          http_servers.sum(&:worker_processes)
         end
 
         # Traversal of ObjectSpace is expensive, on fully loaded application
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index e53ac00e77f..f9ba0a69b0e 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -61,7 +61,7 @@ module Gitlab
         end
 
         def current_transaction
-          Transaction.current
+          ::Gitlab::Metrics::Transaction.current
         end
       end
     end