diff options
Diffstat (limited to 'lib/gitlab/metrics')
7 files changed, 94 insertions, 34 deletions
diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb index d1490d5d9b6..8a176be30a2 100644 --- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb +++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb @@ -13,11 +13,12 @@ module Gitlab @dashboard_hash = dashboard_hash @project = project @dashboard_path = dashboard_path + @affected_environment_ids = [] end def execute import - rescue ActiveRecord::RecordInvalid, ::Gitlab::Metrics::Dashboard::Transformers::TransformerError + rescue ActiveRecord::RecordInvalid, Dashboard::Transformers::Errors::BaseError false end @@ -32,28 +33,51 @@ module Gitlab def import delete_stale_metrics create_or_update_metrics + update_prometheus_environments end # rubocop: disable CodeReuse/ActiveRecord def create_or_update_metrics # TODO: use upsert and worker for callbacks? + + affected_metric_ids = [] prometheus_metrics_attributes.each do |attributes| - prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:identifier, :project)) + prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:dashboard_path, :identifier, :project)) prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES)) + + affected_metric_ids << prometheus_metric.id end + + @affected_environment_ids += find_alerts(affected_metric_ids).get_environment_id end # rubocop: enable CodeReuse/ActiveRecord def delete_stale_metrics - identifiers = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } + identifiers_from_yml = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } stale_metrics = PrometheusMetric.for_project(project) .for_dashboard_path(dashboard_path) .for_group(Enums::PrometheusMetric.groups[:custom]) - .not_identifier(identifiers) + .not_identifier(identifiers_from_yml) + + return unless stale_metrics.exists? + + delete_stale_alerts(stale_metrics) + stale_metrics.each_batch { |batch| batch.delete_all } + end + + def delete_stale_alerts(stale_metrics) + stale_alerts = find_alerts(stale_metrics) + + affected_environment_ids = stale_alerts.get_environment_id + return unless affected_environment_ids.present? - # TODO: use destroy_all and worker for callbacks? - stale_metrics.each(&:destroy) + @affected_environment_ids += affected_environment_ids + stale_alerts.each_batch { |batch| batch.delete_all } + end + + def find_alerts(metrics) + Projects::Prometheus::AlertsFinder.new(project: project, metric: metrics).execute end def prometheus_metrics_attributes @@ -65,6 +89,19 @@ module Gitlab ).execute end end + + def update_prometheus_environments + affected_environments = ::Environment.for_id(@affected_environment_ids.flatten.uniq).for_project(project) + + return unless affected_environments.exists? + + affected_environments.each do |affected_environment| + ::Clusters::Applications::ScheduleUpdateService.new( + affected_environment.cluster_prometheus_adapter, + project + ).execute + end + end end end end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb new file mode 100644 index 00000000000..5ed4466f440 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + # Acts on metrics which have been ingested from source controlled dashboards + class CustomDashboardMetricsInserter < BaseStage + # For each metric in the dashboard config, attempts to + # find a corresponding database record. If found, includes + # the record's id in the dashboard config. + def transform! + database_metrics = ::PrometheusMetricsFinder.new(common: false, group: :custom, project: project).execute + + for_metrics do |metric| + metric_record = database_metrics.find { |m| m.identifier == metric[:id] } + metric[:metric_id] = metric_record.id if metric_record + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb index 9e2bb0d1a70..ad9d78133af 100644 --- a/lib/gitlab/metrics/dashboard/stages/url_validator.rb +++ b/lib/gitlab/metrics/dashboard/stages/url_validator.rb @@ -46,7 +46,7 @@ module Gitlab links&.each do |link| next unless link.is_a? Hash - Gitlab::UrlBlocker.validate!(link[:url], blocker_args) + Gitlab::UrlBlocker.validate!(link[:url], **blocker_args) rescue Gitlab::UrlBlocker::BlockedUrlError link[:url] = '' end diff --git a/lib/gitlab/metrics/dashboard/transformers/errors.rb b/lib/gitlab/metrics/dashboard/transformers/errors.rb index 4d94ab098ae..bc85dc4e131 100644 --- a/lib/gitlab/metrics/dashboard/transformers/errors.rb +++ b/lib/gitlab/metrics/dashboard/transformers/errors.rb @@ -4,10 +4,10 @@ module Gitlab module Metrics module Dashboard module Transformers - TransformerError = Class.new(StandardError) - module Errors - class MissingAttribute < TransformerError + BaseError = Class.new(StandardError) + + class MissingAttribute < BaseError def initialize(attribute_name) super("Missing attribute: '#{attribute_name}'") end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 15db3999fa4..f6bda0dbea4 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -3,20 +3,13 @@ module Gitlab module Metrics class RequestsRackMiddleware - HTTP_METHODS = { - "delete" => %w(200 202 204 303 400 401 403 404 410 422 500 503), - "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 412 422 429 500 503), - "head" => %w(200 204 301 302 303 304 400 401 403 404 410 429 500 503), - "options" => %w(200 404), - "patch" => %w(200 202 204 400 403 404 409 416 422 500), - "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 413 415 422 429 500 503), - "propfind" => %w(404), - "put" => %w(200 202 204 400 401 403 404 405 406 409 410 415 422 500), - "report" => %w(404) - }.freeze + HTTP_METHODS = %w(delete get head options patch post put).to_set.freeze HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze + FEATURE_CATEGORY_HEADER = 'X-Gitlab-Feature-Category' + FEATURE_CATEGORY_DEFAULT = 'unknown' + def initialize(app) @app = app end @@ -39,33 +32,39 @@ module Gitlab end def self.initialize_http_request_duration_seconds - HTTP_METHODS.each do |method, statuses| - statuses.each do |status| - http_request_duration_seconds.get({ method: method, status: status.to_s }) - end + HTTP_METHODS.each do |method| + http_request_duration_seconds.get({ method: method }) end end def call(env) method = env['REQUEST_METHOD'].downcase + method = 'INVALID' unless HTTP_METHODS.include?(method) started = Time.now.to_f + health_endpoint = health_endpoint?(env['PATH_INFO']) + status = 'undefined' + feature_category = nil begin - if health_endpoint?(env['PATH_INFO']) - RequestsRackMiddleware.http_health_requests_total.increment(method: method) - else - RequestsRackMiddleware.http_request_total.increment(method: method) - end - status, headers, body = @app.call(env) elapsed = Time.now.to_f - started - RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status.to_s }, elapsed) + feature_category = headers&.fetch(FEATURE_CATEGORY_HEADER, nil) + + unless health_endpoint + RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method }, elapsed) + end [status, headers, body] rescue RequestsRackMiddleware.rack_uncaught_errors_count.increment raise + ensure + if health_endpoint + RequestsRackMiddleware.http_health_requests_total.increment(status: status, method: method) + else + RequestsRackMiddleware.http_request_total.increment(status: status, method: method, feature_category: feature_category || FEATURE_CATEGORY_DEFAULT) + end end end diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index 8c4d150adad..d7935d65e12 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -54,7 +54,7 @@ module Gitlab end def unicorn_workers_count - http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord + http_servers.sum(&:worker_processes) end # Traversal of ObjectSpace is expensive, on fully loaded application diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index e53ac00e77f..f9ba0a69b0e 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -61,7 +61,7 @@ module Gitlab end def current_transaction - Transaction.current + ::Gitlab::Metrics::Transaction.current end end end |