diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-09-20 02:18:09 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-09-20 02:18:09 +0300 |
commit | 6ed4ec3e0b1340f96b7c043ef51d1b33bbe85fde (patch) | |
tree | dc4d20fe6064752c0bd323187252c77e0a89144b /lib/gitlab/metrics | |
parent | 9868dae7fc0655bd7ce4a6887d4e6d487690eeed (diff) |
Add latest changes from gitlab-org/gitlab@15-4-stable-eev15.4.0-rc42
Diffstat (limited to 'lib/gitlab/metrics')
-rw-r--r-- | lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb | 10 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/validator/client.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/metrics/exporter/metrics_middleware.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/metrics/global_search_slis.rb | 114 | ||||
-rw-r--r-- | lib/gitlab/metrics/samplers/puma_sampler.rb | 14 | ||||
-rw-r--r-- | lib/gitlab/metrics/samplers/ruby_sampler.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/metrics/system.rb | 16 |
8 files changed, 156 insertions, 42 deletions
diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb index 55d14d6f94a..622b6adec7e 100644 --- a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb +++ b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb @@ -40,8 +40,8 @@ module Gitlab def formatted_panel { - title: panel[:title], - type: CHART_TYPE, + title: panel[:title], + type: CHART_TYPE, y_label: '', # Grafana panels do not include a Y-Axis label metrics: panel[:targets].map.with_index do |target, idx| formatted_metric(target, idx) @@ -51,9 +51,9 @@ module Gitlab def formatted_metric(metric, idx) { - id: "#{metric[:legendFormat]}_#{idx}", - query_range: format_query(metric), - label: replace_variables(metric[:legendFormat]), + id: "#{metric[:legendFormat]}_#{idx}", + query_range: format_query(metric), + label: replace_variables(metric[:legendFormat]), prometheus_endpoint_path: prometheus_endpoint_for_metric(metric) }.compact end diff --git a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb index 4e46eec17d6..3650ddf698a 100644 --- a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb +++ b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb @@ -24,15 +24,15 @@ module Gitlab panel_group[:panels].each do |panel| panel[:metrics].each do |metric| prometheus_metrics << { - project: project, - title: panel[:title], - y_label: panel[:y_label], - query: metric[:query_range] || metric[:query], - unit: metric[:unit], - legend: metric[:label], - identifier: metric[:id], - group: Enums::PrometheusMetric.groups[:custom], - common: false, + project: project, + title: panel[:title], + y_label: panel[:y_label], + query: metric[:query_range] || metric[:query], + unit: metric[:unit], + legend: metric[:label], + identifier: metric[:id], + group: Enums::PrometheusMetric.groups[:custom], + common: false, dashboard_path: dashboard_path }.compact end diff --git a/lib/gitlab/metrics/dashboard/validator/client.rb b/lib/gitlab/metrics/dashboard/validator/client.rb index 588c677ca28..29f1274a097 100644 --- a/lib/gitlab/metrics/dashboard/validator/client.rb +++ b/lib/gitlab/metrics/dashboard/validator/client.rb @@ -34,8 +34,8 @@ module Gitlab def post_schema_validator PostSchemaValidator.new( - project: project, - metric_ids: custom_formats.metric_ids_cache, + project: project, + metric_ids: custom_formats.metric_ids_cache, dashboard_path: dashboard_path ) end diff --git a/lib/gitlab/metrics/exporter/metrics_middleware.rb b/lib/gitlab/metrics/exporter/metrics_middleware.rb index e17f1c13cf0..258b655229e 100644 --- a/lib/gitlab/metrics/exporter/metrics_middleware.rb +++ b/lib/gitlab/metrics/exporter/metrics_middleware.rb @@ -27,8 +27,8 @@ module Gitlab labels = { method: env['REQUEST_METHOD'].downcase, - path: env['PATH_INFO'].to_s, - code: response.first.to_s + path: env['PATH_INFO'].to_s, + code: response.first.to_s } @requests_total.increment(labels) diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb new file mode 100644 index 00000000000..e37129fed38 --- /dev/null +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module GlobalSearchSlis + class << self + # The following targets are the 99.95th percentile of code searches + # gathered on 24-08-2022 + # from https://log.gprd.gitlab.net/goto/0c89cd80-23af-11ed-8656-f5f2137823ba (internal only) + BASIC_CONTENT_TARGET_S = 7.031 + BASIC_CODE_TARGET_S = 21.903 + ADVANCED_CONTENT_TARGET_S = 4.865 + ADVANCED_CODE_TARGET_S = 13.546 + + def initialize_slis! + if Feature.enabled?(:global_search_custom_slis) + Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) + end + + return unless Feature.enabled?(:global_search_error_rate_sli) + + Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:global_search, possible_labels) + end + + def record_apdex(elapsed:, search_type:, search_level:, search_scope:) + return unless Feature.enabled?(:global_search_custom_slis) + + Gitlab::Metrics::Sli::Apdex[:global_search].increment( + labels: labels(search_type: search_type, search_level: search_level, search_scope: search_scope), + success: elapsed < duration_target(search_type, search_scope) + ) + end + + def record_error_rate(error:, search_type:, search_level:, search_scope:) + return unless Feature.enabled?(:global_search_error_rate_sli) + + Gitlab::Metrics::Sli::ErrorRate[:global_search].increment( + labels: labels(search_type: search_type, search_level: search_level, search_scope: search_scope), + error: error + ) + end + + private + + def duration_target(search_type, search_scope) + if search_type == 'basic' && content_search?(search_scope) + BASIC_CONTENT_TARGET_S + elsif search_type == 'basic' && code_search?(search_scope) + BASIC_CODE_TARGET_S + elsif search_type == 'advanced' && content_search?(search_scope) + ADVANCED_CONTENT_TARGET_S + elsif search_type == 'advanced' && code_search?(search_scope) + ADVANCED_CODE_TARGET_S + end + end + + def search_types + %w[basic advanced] + end + + def search_levels + %w[project group global] + end + + def search_scopes + Gitlab::Search::AbuseDetection::ALLOWED_SCOPES + end + + def endpoint_ids + ['SearchController#show', 'GET /api/:version/search', 'GET /api/:version/projects/:id/(-/)search', + 'GET /api/:version/groups/:id/(-/)search'] + end + + def possible_labels + search_types.flat_map do |search_type| + search_levels.flat_map do |search_level| + search_scopes.flat_map do |search_scope| + endpoint_ids.flat_map do |endpoint_id| + { + search_type: search_type, + search_level: search_level, + search_scope: search_scope, + endpoint_id: endpoint_id + } + end + end + end + end + end + + def labels(search_type:, search_level:, search_scope:) + { + search_type: search_type, + search_level: search_level, + search_scope: search_scope, + endpoint_id: endpoint_id + } + end + + def endpoint_id + ::Gitlab::ApplicationContext.current_context_attribute(:caller_id) + end + + def code_search?(search_scope) + search_scope == 'blobs' + end + + def content_search?(search_scope) + !code_search?(search_scope) + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index 848a55e59ff..d818aa43853 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -12,15 +12,15 @@ module Gitlab def init_metrics { - puma_workers: ::Gitlab::Metrics.gauge(:puma_workers, 'Total number of workers'), - puma_running_workers: ::Gitlab::Metrics.gauge(:puma_running_workers, 'Number of active workers'), - puma_stale_workers: ::Gitlab::Metrics.gauge(:puma_stale_workers, 'Number of stale workers'), - puma_running: ::Gitlab::Metrics.gauge(:puma_running, 'Number of running threads'), + puma_workers: ::Gitlab::Metrics.gauge(:puma_workers, 'Total number of workers'), + puma_running_workers: ::Gitlab::Metrics.gauge(:puma_running_workers, 'Number of active workers'), + puma_stale_workers: ::Gitlab::Metrics.gauge(:puma_stale_workers, 'Number of stale workers'), + puma_running: ::Gitlab::Metrics.gauge(:puma_running, 'Number of running threads'), puma_queued_connections: ::Gitlab::Metrics.gauge(:puma_queued_connections, 'Number of connections in that worker\'s "todo" set waiting for a worker thread'), puma_active_connections: ::Gitlab::Metrics.gauge(:puma_active_connections, 'Number of threads processing a request'), - puma_pool_capacity: ::Gitlab::Metrics.gauge(:puma_pool_capacity, 'Number of requests the worker is capable of taking right now'), - puma_max_threads: ::Gitlab::Metrics.gauge(:puma_max_threads, 'Maximum number of worker threads'), - puma_idle_threads: ::Gitlab::Metrics.gauge(:puma_idle_threads, 'Number of spawned threads which are not processing a request') + puma_pool_capacity: ::Gitlab::Metrics.gauge(:puma_pool_capacity, 'Number of requests the worker is capable of taking right now'), + puma_max_threads: ::Gitlab::Metrics.gauge(:puma_max_threads, 'Maximum number of worker threads'), + puma_idle_threads: ::Gitlab::Metrics.gauge(:puma_idle_threads, 'Number of spawned threads which are not processing a request') } end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 8e002293347..4fe338ffc7f 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -31,16 +31,16 @@ module Gitlab def init_metrics metrics = { - file_descriptors: ::Gitlab::Metrics.gauge(metric_name(:file, :descriptors), 'File descriptors used', labels), - process_cpu_seconds_total: ::Gitlab::Metrics.gauge(metric_name(:process, :cpu_seconds_total), 'Process CPU seconds total'), - process_max_fds: ::Gitlab::Metrics.gauge(metric_name(:process, :max_fds), 'Process max fds'), - process_resident_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :resident_memory_bytes), 'Memory used (RSS)', labels), - process_unique_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :unique_memory_bytes), 'Memory used (USS)', labels), + file_descriptors: ::Gitlab::Metrics.gauge(metric_name(:file, :descriptors), 'File descriptors used', labels), + process_cpu_seconds_total: ::Gitlab::Metrics.gauge(metric_name(:process, :cpu_seconds_total), 'Process CPU seconds total'), + process_max_fds: ::Gitlab::Metrics.gauge(metric_name(:process, :max_fds), 'Process max fds'), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :resident_memory_bytes), 'Memory used (RSS)', labels), + process_unique_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :unique_memory_bytes), 'Memory used (USS)', labels), process_proportional_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :proportional_memory_bytes), 'Memory used (PSS)', labels), - process_start_time_seconds: ::Gitlab::Metrics.gauge(metric_name(:process, :start_time_seconds), 'Process start time seconds'), - sampler_duration: ::Gitlab::Metrics.counter(metric_name(:sampler, :duration_seconds_total), 'Sampler time', labels), - gc_duration_seconds: ::Gitlab::Metrics.histogram(metric_name(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS), - heap_fragmentation: ::Gitlab::Metrics.gauge(metric_name(:gc_stat_ext, :heap_fragmentation), 'Ruby heap fragmentation', labels) + process_start_time_seconds: ::Gitlab::Metrics.gauge(metric_name(:process, :start_time_seconds), 'Process start time seconds'), + sampler_duration: ::Gitlab::Metrics.counter(metric_name(:sampler, :duration_seconds_total), 'Sampler time', labels), + gc_duration_seconds: ::Gitlab::Metrics.histogram(metric_name(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS), + heap_fragmentation: ::Gitlab::Metrics.gauge(metric_name(:gc_stat_ext, :heap_fragmentation), 'Ruby heap fragmentation', labels) } GC.stat.keys.each do |key| diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index e646846face..d7eef722d6e 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -10,8 +10,8 @@ module Gitlab extend self PROC_STAT_PATH = '/proc/self/stat' - PROC_STATUS_PATH = '/proc/self/status' - PROC_SMAPS_ROLLUP_PATH = '/proc/self/smaps_rollup' + PROC_STATUS_PATH = '/proc/%s/status' + PROC_SMAPS_ROLLUP_PATH = '/proc/%s/smaps_rollup' PROC_LIMITS_PATH = '/proc/self/limits' PROC_FD_GLOB = '/proc/self/fd/*' @@ -34,14 +34,14 @@ module Gitlab } end - # Returns the current process' RSS (resident set size) in bytes. - def memory_usage_rss - sum_matches(PROC_STATUS_PATH, rss: RSS_PATTERN)[:rss].kilobytes + # Returns the given process' RSS (resident set size) in bytes. + def memory_usage_rss(pid: 'self') + sum_matches(PROC_STATUS_PATH % pid, rss: RSS_PATTERN)[:rss].kilobytes end - # Returns the current process' USS/PSS (unique/proportional set size) in bytes. - def memory_usage_uss_pss - sum_matches(PROC_SMAPS_ROLLUP_PATH, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN) + # Returns the given process' USS/PSS (unique/proportional set size) in bytes. + def memory_usage_uss_pss(pid: 'self') + sum_matches(PROC_SMAPS_ROLLUP_PATH % pid, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN) .transform_values(&:kilobytes) end |