diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-09-20 14:18:08 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-09-20 14:18:08 +0300 |
commit | 5afcbe03ead9ada87621888a31a62652b10a7e4f (patch) | |
tree | 9918b67a0d0f0bafa6542e839a8be37adf73102d /lib/gitlab/metrics | |
parent | c97c0201564848c1f53226fe19d71fdcc472f7d0 (diff) |
Add latest changes from gitlab-org/gitlab@16-4-stable-eev16.4.0-rc42
Diffstat (limited to 'lib/gitlab/metrics')
-rw-r--r-- | lib/gitlab/metrics/background_transaction.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/cache.rb | 63 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/errors.rb | 41 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/processor.rb | 33 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb | 37 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/stages/base_stage.rb | 74 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/stages/url_validator.rb | 58 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/transformers/errors.rb | 19 | ||||
-rw-r--r-- | lib/gitlab/metrics/dashboard/url.rb | 133 | ||||
-rw-r--r-- | lib/gitlab/metrics/requests_rack_middleware.rb | 14 | ||||
-rw-r--r-- | lib/gitlab/metrics/samplers/database_sampler.rb | 4 | ||||
-rw-r--r-- | lib/gitlab/metrics/subscribers/action_view.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/metrics/subscribers/active_record.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/metrics/transaction.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/metrics/web_transaction.rb | 2 |
15 files changed, 17 insertions, 475 deletions
diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb index 54095461dd4..7ffec7b0141 100644 --- a/lib/gitlab/metrics/background_transaction.rb +++ b/lib/gitlab/metrics/background_transaction.rb @@ -8,7 +8,7 @@ module Gitlab # metadata such as endpoint_id, queue, and feature category. class BackgroundTransaction < Transaction THREAD_KEY = :_gitlab_metrics_background_transaction - BASE_LABEL_KEYS = %i(queue endpoint_id feature_category).freeze + BASE_LABEL_KEYS = %i[queue endpoint_id feature_category].freeze class << self def current diff --git a/lib/gitlab/metrics/dashboard/cache.rb b/lib/gitlab/metrics/dashboard/cache.rb deleted file mode 100644 index 54b5250d209..00000000000 --- a/lib/gitlab/metrics/dashboard/cache.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -require 'set' - -module Gitlab - module Metrics - module Dashboard - class Cache - CACHE_KEYS = 'all_cached_metric_dashboards' - - class << self - # This class method (Gitlab::Metrics::Dashboard::Cache.fetch) can be used - # when the key does not need to be deleted by `delete_all!`. - # For example, out of the box dashboard caches do not need to be deleted. - delegate :fetch, to: :"Rails.cache" - - alias_method :for, :new - end - - def initialize(project) - @project = project - end - - # Stores a dashboard in the cache, documenting the key - # so the cache can be cleared in bulk at another time. - def fetch(key) - register_key(key) - - Rails.cache.fetch(key) { yield } - end - - # Resets all dashboard caches, such that all - # dashboard content will be loaded from source on - # subsequent dashboard calls. - def delete_all! - all_keys.each { |key| Rails.cache.delete(key) } - - Rails.cache.delete(catalog_key) - end - - private - - def register_key(key) - new_keys = all_keys.add(key).to_a.join('|') - - Rails.cache.write(catalog_key, new_keys) - end - - def all_keys - keys = Rails.cache.read(catalog_key)&.split('|') - Set.new(keys) - end - - # One key to store them all... - # This key is used to store the names of all the keys that contain this - # project's dashboards. - def catalog_key - "#{CACHE_KEYS}_#{@project.id}" - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/errors.rb b/lib/gitlab/metrics/dashboard/errors.rb deleted file mode 100644 index 1a951172f74..00000000000 --- a/lib/gitlab/metrics/dashboard/errors.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -# Central point for managing errors from within the metrics -# dashboard module. Handles errors from dashboard retrieval -# and processing steps, as well as defines shared error classes. -module Gitlab - module Metrics - module Dashboard - module Errors - DashboardProcessingError = Class.new(StandardError) - PanelNotFoundError = Class.new(StandardError) - MissingIntegrationError = Class.new(StandardError) - LayoutError = Class.new(DashboardProcessingError) - MissingQueryError = Class.new(DashboardProcessingError) - - NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError - - def handle_errors(error) - case error - when DashboardProcessingError - error(error.message, :unprocessable_entity) - when NOT_FOUND_ERROR - error(_("%{dashboard_path} could not be found.") % { dashboard_path: dashboard_path }, :not_found) - when PanelNotFoundError - error(error.message, :not_found) - when ::Grafana::Client::Error - error(error.message, :service_unavailable) - when MissingIntegrationError - error(_('Proxy support for this API is not available currently'), :bad_request) - else - raise error - end - end - - def panels_not_found!(opts) - raise PanelNotFoundError, _("No panels matching properties %{opts}") % { opts: opts } - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/processor.rb b/lib/gitlab/metrics/dashboard/processor.rb deleted file mode 100644 index 9566e5afb9a..00000000000 --- a/lib/gitlab/metrics/dashboard/processor.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - # Responsible for processesing a dashboard hash, inserting - # relevant DB records & sorting for proper rendering in - # the UI. These includes shared metric info, custom metrics - # info, and alerts (only in EE). - class Processor - def initialize(project, dashboard, sequence, params) - @project = project - @dashboard = dashboard - @sequence = sequence - @params = params - end - - # Returns a new dashboard hash with the results of - # running transforms on the dashboard. - # @return [Hash, nil] - def process - return unless @dashboard - - @dashboard.deep_symbolize_keys.tap do |dashboard| - @sequence.each do |stage| - stage.new(@project, dashboard, @params).transform! - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb b/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb deleted file mode 100644 index 8b791e110ba..00000000000 --- a/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -# Provides methods to list and read dashboard yaml files from a project's repository. -module Gitlab - module Metrics - module Dashboard - class RepoDashboardFinder - DASHBOARD_ROOT = ".gitlab/dashboards" - DASHBOARD_EXTENSION = '.yml' - - class << self - # Returns list of all user-defined dashboard paths. Used to populate - # Repository model cache (Repository#user_defined_metrics_dashboard_paths). - # Also deletes all dashboard cache entries. - # @return [Array] ex) ['.gitlab/dashboards/dashboard1.yml'] - def list_dashboards(project) - Gitlab::Metrics::Dashboard::Cache.for(project).delete_all! - - file_finder(project).list_files_for(DASHBOARD_ROOT) - end - - # Reads the given dashboard from repository, and returns the content as a string. - # @return [String] - def read_dashboard(project, dashboard_path) - file_finder(project).read(dashboard_path) - end - - private - - def file_finder(project) - Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, DASHBOARD_EXTENSION) - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb deleted file mode 100644 index b869a633030..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class BaseStage - attr_reader :project, :dashboard, :params - - def initialize(project, dashboard, params) - @project = project - @dashboard = dashboard - @params = params - end - - # Entry-point to the stage - def transform! - raise NotImplementedError - end - - protected - - def missing_panel_groups! - raise Errors::LayoutError, 'Top-level key :panel_groups must be an array' - end - - def missing_panels! - raise Errors::LayoutError, 'Each "panel_group" must define an array :panels' - end - - def missing_metrics! - raise Errors::LayoutError, 'Each "panel" must define an array :metrics' - end - - def for_metrics - missing_panel_groups! unless dashboard[:panel_groups].is_a?(Array) - - for_panel_groups do |panel_group| - for_panels_in(panel_group) do |panel| - missing_metrics! unless panel[:metrics].is_a?(Array) - - panel[:metrics].each do |metric| - yield metric - end - end - end - end - - def for_variables - return unless dashboard.dig(:templating, :variables).is_a?(Hash) - - dashboard.dig(:templating, :variables).each do |variable_name, variable| - yield variable_name, variable - end - end - - def for_panel_groups - dashboard[:panel_groups].each do |panel_group| - yield panel_group - end - end - - def for_panels_in(panel_group) - missing_panels! unless panel_group[:panels].is_a?(Array) - - panel_group[:panels].each do |panel| - yield panel - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb deleted file mode 100644 index ad9d78133af..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/url_validator.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class UrlValidator < BaseStage - def transform! - validate_dashboard_links(dashboard) - - validate_chart_links(dashboard) - end - - private - - def blocker_args - { - schemes: %w(http https), - ports: [], - allow_localhost: allow_setting_local_requests?, - allow_local_network: allow_setting_local_requests?, - ascii_only: false, - enforce_user: false, - enforce_sanitization: false, - dns_rebind_protection: true - } - end - - def allow_setting_local_requests? - Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? - end - - def validate_dashboard_links(dashboard) - validate_links(dashboard[:links]) - end - - def validate_chart_links(dashboard) - dashboard[:panel_groups].each do |panel_group| - panel_group[:panels].each do |panel| - validate_links(panel[:links]) - end - end - end - - def validate_links(links) - links&.each do |link| - next unless link.is_a? Hash - - Gitlab::UrlBlocker.validate!(link[:url], **blocker_args) - rescue Gitlab::UrlBlocker::BlockedUrlError - link[:url] = '' - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/transformers/errors.rb b/lib/gitlab/metrics/dashboard/transformers/errors.rb deleted file mode 100644 index bc85dc4e131..00000000000 --- a/lib/gitlab/metrics/dashboard/transformers/errors.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Transformers - module Errors - BaseError = Class.new(StandardError) - - class MissingAttribute < BaseError - def initialize(attribute_name) - super("Missing attribute: '#{attribute_name}'") - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb deleted file mode 100644 index e7b901861ef..00000000000 --- a/lib/gitlab/metrics/dashboard/url.rb +++ /dev/null @@ -1,133 +0,0 @@ -# frozen_string_literal: true - -# Manages url matching for metrics dashboards. -module Gitlab - module Metrics - module Dashboard - class Url - class << self - include Gitlab::Utils::StrongMemoize - - QUERY_PATTERN = '(?<query>\?[a-zA-Z0-9%.()+_=-]+(&[a-zA-Z0-9%.()+_=-]+)*)?' - ANCHOR_PATTERN = '(?<anchor>\#[a-z0-9_-]+)?' - DASH_PATTERN = '(?:/-)' - - # Matches dashboard urls for a Grafana embed. - # - # EX - https://<host>/<namespace>/<project>/grafana/metrics_dashboard - def grafana_regex - strong_memoize(:grafana_regex) do - regex_for_project_metrics( - %r{ - #{DASH_PATTERN}? - /grafana - /metrics_dashboard - }xo - ) - end - end - - # Matches dashboard urls for a metric chart embed - # for cluster metrics. - # This regex needs to match the dashboard URL as well, not just the trigger URL. - # The inline_metrics_redactor_filter.rb uses this regex to match against - # the dashboard URL. - # - # EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB) - # dashboard URL - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/metrics_dashboard?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB) - def clusters_regex - strong_memoize(:clusters_regex) do - regex_for_project_metrics( - %r{ - #{DASH_PATTERN}? - /clusters - /(?<cluster_id>\d+) - /? - ( (/metrics) | ( /metrics_dashboard\.json ) )? - }xo - ) - end - end - - # Matches dashboard urls for a metric chart embed - # for a specifc firing GitLab alert - # - # EX - https://<host>/<namespace>/<project>/prometheus/alerts/<alert_id>/metrics_dashboard - def alert_regex - strong_memoize(:alert_regex) do - regex_for_project_metrics( - %r{ - #{DASH_PATTERN}? - /prometheus - /alerts - /(?<alert>\d+) - /metrics_dashboard(\.json)? - }xo - ) - end - end - - # Parses query params out from full url string into hash. - # - # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group' - # --> { title: 'Title', group: 'Group' } - def parse_query(url) - query_string = URI.parse(url).query.to_s - - CGI.parse(query_string) - .transform_values { |value| value.first } - .symbolize_keys - end - - private - - def environment_metrics_regex - %r{ - #{DASH_PATTERN}? - /environments - /(?<environment>\d+) - /(metrics_dashboard|metrics) - }xo - end - - def non_environment_metrics_regex - %r{ - #{DASH_PATTERN} - /metrics - (?= # Lookahead to ensure there is an environment query param - \? - .* - environment=(?<environment>\d+) - .* - ) - }xo - end - - def regex_for_project_metrics(path_suffix_pattern) - %r{ - ^(?<url> - #{gitlab_host_pattern} - #{project_path_pattern} - #{path_suffix_pattern} - #{QUERY_PATTERN} - #{ANCHOR_PATTERN} - )$ - }x - end - - def gitlab_host_pattern - Regexp.escape(gitlab_domain) - end - - def project_path_pattern - "\/#{Project.reference_pattern}" - end - - def gitlab_domain - Gitlab.config.gitlab.url - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index b4baeba72e8..c299fa37e7a 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -4,13 +4,13 @@ module Gitlab module Metrics class RequestsRackMiddleware HTTP_METHODS = { - "delete" => %w(200 202 204 303 400 401 403 404 500 503), - "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 422 429 500 503), - "head" => %w(200 204 301 302 303 401 403 404 410 500), - "options" => %w(200 404), - "patch" => %w(200 202 204 400 403 404 409 416 500), - "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 422 429 500 503), - "put" => %w(200 202 204 400 401 403 404 405 406 409 410 422 500) + "delete" => %w[200 202 204 303 400 401 403 404 500 503], + "get" => %w[200 204 301 302 303 304 307 400 401 403 404 410 422 429 500 503], + "head" => %w[200 204 301 302 303 401 403 404 410 500], + "options" => %w[200 404], + "patch" => %w[200 202 204 400 403 404 409 416 500], + "post" => %w[200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 422 429 500 503], + "put" => %w[200 202 204 400 401 403 404 405 406 409 410 422 500] }.freeze HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$}.freeze diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index 86372973c82..8d4f4570c21 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -54,7 +54,7 @@ module Gitlab next if load_balancer.primary_only? load_balancer.host_list.hosts.each do |host| - stats << { labels: labels_for_replica_host(load_balancer, host), stats: host.connection.pool.stat } + stats << { labels: labels_for_replica_host(load_balancer, host), stats: host.pool.stat } end end end @@ -73,7 +73,7 @@ module Gitlab host: host.host, port: host.port, class: load_balancer.configuration.connection_specification_name, - db_config_name: Gitlab::Database.db_config_name(host.connection) + db_config_name: host.pool.db_config.name } end end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index bc9032a6942..ce2cc88e035 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -21,7 +21,7 @@ module Gitlab tags = tags_for(event) current_transaction.observe(:gitlab_view_rendering_duration_seconds, event.duration, tags) do docstring 'View rendering time' - label_keys %i(view) + label_keys %i[view] buckets [0.001, 0.01, 0.1, 1, 10.0] with_feature :prometheus_metrics_view_instrumentation end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 10bb358a292..f9749b65888 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -9,15 +9,15 @@ module Gitlab attach_to :active_record - DB_COUNTERS = %i{count write_count cached_count}.freeze + DB_COUNTERS = %i[count write_count cached_count].freeze SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i.freeze SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze - DB_LOAD_BALANCING_ROLES = %i{replica primary}.freeze - DB_LOAD_BALANCING_COUNTERS = %i{count cached_count wal_count wal_cached_count}.freeze - DB_LOAD_BALANCING_DURATIONS = %i{duration_s}.freeze + DB_LOAD_BALANCING_ROLES = %i[replica primary].freeze + DB_LOAD_BALANCING_COUNTERS = %i[count cached_count wal_count wal_cached_count].freeze + DB_LOAD_BALANCING_DURATIONS = %i[duration_s].freeze SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 56a310548a7..a57a859acfe 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -7,7 +7,7 @@ module Gitlab include Gitlab::Metrics::Methods # labels that potentially contain sensitive information and will be filtered - FILTERED_LABEL_KEYS = %i(branch path).freeze + FILTERED_LABEL_KEYS = %i[branch path].freeze # The series to store events (e.g. Git pushes) in. EVENT_SERIES = 'events' diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index fcfa86734e8..f3c1e6897af 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -9,7 +9,7 @@ module Gitlab # etc. class WebTransaction < Transaction THREAD_KEY = :_gitlab_metrics_transaction - BASE_LABEL_KEYS = %i(controller action feature_category).freeze + BASE_LABEL_KEYS = %i[controller action feature_category].freeze CONTROLLER_KEY = 'action_controller.instance' ENDPOINT_KEY = 'api.endpoint' |