diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-08-18 13:50:51 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-08-18 13:50:51 +0300 |
commit | db384e6b19af03b4c3c82a5760d83a3fd79f7982 (patch) | |
tree | 34beaef37df5f47ccbcf5729d7583aae093cffa0 /lib/gitlab/metrics | |
parent | 54fd7b1bad233e3944434da91d257fa7f63c3996 (diff) |
Add latest changes from gitlab-org/gitlab@16-3-stable-eev16.3.0-rc42
Diffstat (limited to 'lib/gitlab/metrics')
31 files changed, 4 insertions, 1223 deletions
diff --git a/lib/gitlab/metrics/dashboard/defaults.rb b/lib/gitlab/metrics/dashboard/defaults.rb deleted file mode 100644 index 6a5f98a18c8..00000000000 --- a/lib/gitlab/metrics/dashboard/defaults.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -# Central point for managing default attributes from within -# the metrics dashboard module. -module Gitlab - module Metrics - module Dashboard - module Defaults - DEFAULT_PANEL_TYPE = 'area-chart' - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb deleted file mode 100644 index 12f7c347b2d..00000000000 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ /dev/null @@ -1,95 +0,0 @@ -# frozen_string_literal: true - -# Returns DB-supplmented dashboard info for determining -# the layout of UI. Intended entry-point for the Metrics::Dashboard -# module. -module Gitlab - module Metrics - module Dashboard - class Finder - PREDEFINED_DASHBOARD_LIST = [ - ::Metrics::Dashboard::PodDashboardService, - ::Metrics::Dashboard::SystemDashboardService - ].freeze - - class << self - # Returns a formatted dashboard packed with DB info. - # @param project [Project] - # @param user [User] - # @param environment [Environment] - # @param options [Hash<Symbol,Any>] - # @param options - embedded [Boolean] Determines whether the - # dashboard is to be rendered as part of an - # issue or location other than the primary - # metrics dashboard UI. Returns only the - # Memory/CPU charts of the system dash. - # @param options - dashboard_path [String] Path at which the - # dashboard can be found. Nil values will - # default to the system dashboard. - # @param options - group [String, Group] Title of the group - # to which a panel might belong. Used by - # embedded dashboards. If cluster dashboard, - # refers to the Group corresponding to the cluster. - # @param options - title [String] Title of the panel. - # Used by embedded dashboards. - # @param options - y_label [String] Y-Axis label of - # a panel. Used by embedded dashboards. - # @param options - cluster [Cluster]. Used by - # embedded and un-embedded dashboards. - # @param options - cluster_type [Symbol] The level of - # cluster, one of [:admin, :project, :group]. Used by - # embedded and un-embedded dashboards. - # @param options - grafana_url [String] URL pointing - # to a grafana dashboard panel - # @param options - prometheus_alert_id [Integer] ID of - # a PrometheusAlert. For dashboard embeds. - # @return [Hash] - def find(project, user, options = {}) - service_for(options) - .new(project, user, options) - .get_dashboard - end - - # Returns a dashboard without any supplemental info. - # Returns only full, yml-defined dashboards. - # @return [Hash] - def find_raw(project, dashboard_path: nil) - service_for(dashboard_path: dashboard_path) - .new(project, nil, dashboard_path: dashboard_path) - .raw_dashboard - end - - # Summary of all known dashboards. - # @return [Array<Hash>] ex) [{ path: String, - # display_name: String, - # default: Boolean }] - def find_all_paths(project) - dashboards = user_facing_dashboard_services.flat_map do |service| - service.all_dashboard_paths(project) - end - - Gitlab::Utils.stable_sort_by(dashboards) { |dashboard| dashboard[:display_name].downcase } - end - - private - - def user_facing_dashboard_services - PREDEFINED_DASHBOARD_LIST + [project_service] - end - - def system_service - ::Metrics::Dashboard::SystemDashboardService - end - - def project_service - ::Metrics::Dashboard::CustomDashboardService - end - - def service_for(options) - Gitlab::Metrics::Dashboard::ServiceSelector.call(options) - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/importer.rb b/lib/gitlab/metrics/dashboard/importer.rb deleted file mode 100644 index ca835650648..00000000000 --- a/lib/gitlab/metrics/dashboard/importer.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - class Importer - def initialize(dashboard_path, project) - @dashboard_path = dashboard_path.to_s - @project = project - end - - def execute - return false unless Dashboard::Validator.validate(dashboard_hash, project: project, dashboard_path: dashboard_path) - - Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute - rescue Gitlab::Config::Loader::FormatError - false - end - - def execute! - Dashboard::Validator.validate!(dashboard_hash, project: project, dashboard_path: dashboard_path) - - Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute! - end - - private - - attr_accessor :dashboard_path, :project - - def dashboard_hash - @dashboard_hash ||= begin - raw_dashboard = Dashboard::RepoDashboardFinder.read_dashboard(project, dashboard_path) - return unless raw_dashboard.present? - - ::Gitlab::Config::Loader::Yaml.new(raw_dashboard).load_raw! - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb deleted file mode 100644 index 531e4079632..00000000000 --- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Importers - class PrometheusMetrics - ALLOWED_ATTRIBUTES = %i(title query y_label unit legend group dashboard_path).freeze - - # Takes a JSON schema validated dashboard hash and - # imports metrics to database - def initialize(dashboard_hash, project:, dashboard_path:) - @dashboard_hash = dashboard_hash - @project = project - @dashboard_path = dashboard_path - @affected_environment_ids = [] - end - - def execute - import - rescue ActiveRecord::RecordInvalid, Dashboard::Transformers::Errors::BaseError - false - end - - def execute! - import - end - - private - - attr_reader :dashboard_hash, :project, :dashboard_path - - def import - delete_stale_metrics - create_or_update_metrics - end - - # rubocop: disable CodeReuse/ActiveRecord - def create_or_update_metrics - # TODO: use upsert and worker for callbacks? - - affected_metric_ids = [] - prometheus_metrics_attributes.each do |attributes| - prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:dashboard_path, :identifier, :project)) - prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES)) - - affected_metric_ids << prometheus_metric.id - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def delete_stale_metrics - identifiers_from_yml = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } - - stale_metrics = PrometheusMetric.for_project(project) - .for_dashboard_path(dashboard_path) - .for_group(Enums::PrometheusMetric.groups[:custom]) - .not_identifier(identifiers_from_yml) - - return unless stale_metrics.exists? - - stale_metrics.each_batch { |batch| batch.delete_all } - end - - def prometheus_metrics_attributes - @prometheus_metrics_attributes ||= Dashboard::Transformers::Yml::V1::PrometheusMetrics.new( - dashboard_hash, - project: project, - dashboard_path: dashboard_path - ).execute - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb deleted file mode 100644 index 67bf4ce7e9a..00000000000 --- a/lib/gitlab/metrics/dashboard/service_selector.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -# Responsible for determining which dashboard service should -# be used to fetch or generate a dashboard hash. -# The services can be considered in two categories - embeds -# and dashboards. Embed hashes are identical to dashboard hashes except -# that they contain a subset of panels. -module Gitlab - module Metrics - module Dashboard - class ServiceSelector - class << self - include Gitlab::Utils::StrongMemoize - - SERVICES = [ - ::Metrics::Dashboard::ClusterMetricsEmbedService, - ::Metrics::Dashboard::ClusterDashboardService, - ::Metrics::Dashboard::GitlabAlertEmbedService, - ::Metrics::Dashboard::CustomMetricEmbedService, - ::Metrics::Dashboard::GrafanaMetricEmbedService, - ::Metrics::Dashboard::TransientEmbedService, - ::Metrics::Dashboard::DynamicEmbedService, - ::Metrics::Dashboard::DefaultEmbedService, - ::Metrics::Dashboard::SystemDashboardService, - ::Metrics::Dashboard::PodDashboardService, - ::Metrics::Dashboard::CustomDashboardService - ].freeze - - # Returns a class which inherits from the BaseService - # class that can be used to obtain a dashboard for - # the provided params. - # @return [Metrics::Dashboard::BaseService] - def call(params) - service = services.find do |service_class| - service_class.valid_params?(params) - end - - service || default_service - end - - private - - def services - SERVICES - end - - def default_service - ::Metrics::Dashboard::SystemDashboardService - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb index c2a8a88108f..b869a633030 100644 --- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -5,8 +5,6 @@ module Gitlab module Dashboard module Stages class BaseStage - include Gitlab::Metrics::Dashboard::Defaults - attr_reader :project, :dashboard, :params def initialize(project, dashboard, params) diff --git a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb deleted file mode 100644 index 56a82d1df46..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class ClusterEndpointInserter < BaseStage - def transform! - verify_params - end - - private - - def error!(message) - raise Errors::DashboardProcessingError, message - end - - def query_type(metric) - metric[:query] ? :query : :query_range - end - - def query_for_metric(metric) - query = metric[query_type(metric)] - - raise Errors::MissingQueryError, 'Each "metric" must define one of :query or :query_range' unless query - - query - end - - def verify_params - raise Errors::DashboardProcessingError, _('Cluster is required for Stages::ClusterEndpointInserter') unless params[:cluster] - raise Errors::DashboardProcessingError, _('Cluster type must be specified for Stages::ClusterEndpointInserter') unless params[:cluster_type] - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb deleted file mode 100644 index 62479ed6de4..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class CommonMetricsInserter < BaseStage - # For each metric in the dashboard config, attempts to - # find a corresponding database record. If found, - # includes the record's id in the dashboard config. - def transform! - common_metrics = ::PrometheusMetricsFinder.new(common: true).execute - - for_metrics do |metric| - metric_record = common_metrics.find { |m| m.identifier == metric[:id] } - metric[:metric_id] = metric_record.id if metric_record - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb deleted file mode 100644 index 5ed4466f440..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - # Acts on metrics which have been ingested from source controlled dashboards - class CustomDashboardMetricsInserter < BaseStage - # For each metric in the dashboard config, attempts to - # find a corresponding database record. If found, includes - # the record's id in the dashboard config. - def transform! - database_metrics = ::PrometheusMetricsFinder.new(common: false, group: :custom, project: project).execute - - for_metrics do |metric| - metric_record = database_metrics.find { |m| m.identifier == metric[:id] } - metric[:metric_id] = metric_record.id if metric_record - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb deleted file mode 100644 index 06cfa5cc58e..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/custom_metrics_details_inserter.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class CustomMetricsDetailsInserter < BaseStage - def transform! - dashboard[:panel_groups].each do |panel_group| - next unless panel_group - - has_custom_metrics = custom_group_titles.include?(panel_group[:group]) - panel_group[:has_custom_metrics] = has_custom_metrics - - panel_group[:panels].each do |panel| - next unless panel - - panel[:metrics].each do |metric| - next unless metric - - metric[:edit_path] = has_custom_metrics ? edit_path(metric) : nil - end - end - end - end - - private - - def custom_group_titles - @custom_group_titles ||= Enums::PrometheusMetric.custom_group_details.values.map { |group_details| group_details[:group_title] } - end - - def edit_path(metric) - Gitlab::Routing.url_helpers.edit_project_prometheus_metric_path(project, metric[:metric_id]) - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb deleted file mode 100644 index 3b49eb1c837..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb +++ /dev/null @@ -1,109 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class CustomMetricsInserter < BaseStage - # Inserts project-specific metrics into the dashboard - # config. If there are no project-specific metrics, - # this will have no effect. - def transform! - custom_metrics = PrometheusMetricsFinder.new(project: project, ordered: true).execute - custom_metrics = Gitlab::Utils.stable_sort_by(custom_metrics) { |metric| -metric.priority } - - custom_metrics.each do |project_metric| - group = find_or_create_panel_group(dashboard[:panel_groups], project_metric) - panel = find_or_create_panel(group[:panels], project_metric) - find_or_create_metric(panel[:metrics], project_metric) - end - end - - private - - # Looks for a panel_group corresponding to the - # provided metric object. If unavailable, inserts one. - # @param panel_groups [Array<Hash>] - # @param metric [PrometheusMetric] - def find_or_create_panel_group(panel_groups, metric) - panel_group = find_panel_group(panel_groups, metric) - return panel_group if panel_group - - panel_group = new_panel_group(metric) - panel_groups << panel_group - - panel_group - end - - # Looks for a panel corresponding to the provided - # metric object. If unavailable, inserts one. - # @param panels [Array<Hash>] - # @param metric [PrometheusMetric] - def find_or_create_panel(panels, metric) - panel = find_panel(panels, metric) - return panel if panel - - panel = new_panel(metric) - panels << panel - - panel - end - - # Looks for a metric corresponding to the provided - # metric object. If unavailable, inserts one. - # @param metrics [Array<Hash>] - # @param metric [PrometheusMetric] - def find_or_create_metric(metrics, metric) - target_metric = find_metric(metrics, metric) - return target_metric if target_metric - - target_metric = new_metric(metric) - metrics << target_metric - - target_metric - end - - def find_panel_group(panel_groups, metric) - return unless panel_groups - - panel_groups.find { |group| group[:group] == metric.group_title } - end - - def find_panel(panels, metric) - return unless panels - - panel_identifiers = [DEFAULT_PANEL_TYPE, metric.title, metric.y_label] - panels.find { |panel| panel.values_at(:type, :title, :y_label) == panel_identifiers } - end - - def find_metric(metrics, metric) - return unless metrics - return unless metric.identifier - - metrics.find { |m| m[:id] == metric.identifier } - end - - def new_panel_group(metric) - { - group: metric.group_title, - panels: [] - } - end - - def new_panel(metric) - { - type: DEFAULT_PANEL_TYPE, - title: metric.title, - y_label: metric.y_label, - metrics: [] - } - end - - def new_metric(metric) - metric.to_metric_hash - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb deleted file mode 100644 index 03370ae7370..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb +++ /dev/null @@ -1,151 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class GrafanaFormatter < BaseStage - include Gitlab::Utils::StrongMemoize - - CHART_TYPE = 'area-chart' - PROXY_PATH = 'api/v1/query_range' - - # Reformats the specified panel in the Gitlab - # dashboard-yml format - def transform! - validate_input! - - new_dashboard = formatted_dashboard - - dashboard.clear - dashboard.merge!(new_dashboard) - end - - private - - def validate_input! - ::Grafana::Validator.new( - grafana_dashboard, - datasource, - panel, - query_params - ).validate! - rescue ::Grafana::Validator::Error => e - raise ::Gitlab::Metrics::Dashboard::Errors::DashboardProcessingError, e.message - end - - def formatted_dashboard - { panel_groups: [{ panels: [formatted_panel] }] } - end - - def formatted_panel - { - title: panel[:title], - type: CHART_TYPE, - y_label: '', # Grafana panels do not include a Y-Axis label - metrics: panel[:targets].map.with_index do |target, idx| - formatted_metric(target, idx) - end - } - end - - def formatted_metric(metric, idx) - { - id: "#{metric[:legendFormat]}_#{idx}", - query_range: format_query(metric), - label: replace_variables(metric[:legendFormat]) - }.compact - end - - # Panel specified by the url from the Grafana dashboard - def panel - strong_memoize(:panel) do - grafana_dashboard[:dashboard][:panels].find do |panel| - query_params[:panelId] ? matching_panel?(panel) : valid_panel?(panel) - end - end - end - - # Determines whether a given panel is the one - # specified by the linked grafana url - def matching_panel?(panel) - panel[:id].to_s == query_params[:panelId] - end - - # Determines whether any given panel has the potenial - # to return valid results from grafana/prometheus - def valid_panel?(panel) - ::Grafana::Validator - .new(grafana_dashboard, datasource, panel, query_params) - .valid? - end - - # Grafana url query parameters. Includes information - # on which panel to select and time range. - def query_params - strong_memoize(:query_params) do - Gitlab::Metrics::Dashboard::Url.parse_query(grafana_url) - end - end - - # Reformats query for compatibility with prometheus api. - def format_query(metric) - expression = remove_new_lines(metric[:expr]) - expression = replace_variables(expression) - replace_global_variables(expression, metric) - end - - # Accomodates instance-defined Grafana variables. - # These are variables defined by users, and values - # must be provided in the query parameters. - def replace_variables(expression) - return expression unless grafana_dashboard[:dashboard][:templating] - - grafana_dashboard[:dashboard][:templating][:list] - .sort_by { |variable| variable[:name].length } - .each do |variable| - variable_value = query_params[:"var-#{variable[:name]}"] - - expression = expression.gsub("$#{variable[:name]}", variable_value) - expression = expression.gsub("[[#{variable[:name]}]]", variable_value) - expression = expression.gsub("{{#{variable[:name]}}}", variable_value) - end - - expression - end - - # Replaces Grafana global built-in variables with values. - # Only $__interval and $__from and $__to are supported. - # - # See https://grafana.com/docs/reference/templating/#global-built-in-variables - def replace_global_variables(expression, metric) - expression = expression.gsub('$__interval', metric[:interval]) if metric[:interval] - expression = expression.gsub('$__from', query_params[:from]) - expression.gsub('$__to', query_params[:to]) - end - - # Removes new lines from expression. - def remove_new_lines(expression) - expression.gsub(/\R+/, '') - end - - # Grafana datasource object corresponding to the - # specified dashboard - def datasource - params[:datasource] - end - - # The specified Grafana dashboard - def grafana_dashboard - params[:grafana_dashboard] - end - - # The URL specifying which Grafana panel to embed - def grafana_url - params[:grafana_url] - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb deleted file mode 100644 index d885d978524..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class MetricEndpointInserter < BaseStage - def transform! - raise Errors::DashboardProcessingError, _('Environment is required for Stages::MetricEndpointInserter') unless params[:environment] - - for_metrics do |metric| - metric[:prometheus_endpoint_path] = endpoint_for_metric(metric) - end - end - - private - - def endpoint_for_metric(metric) - if params[:sample_metrics] - Gitlab::Routing.url_helpers.sample_metrics_project_environment_path( - project, - params[:environment], - identifier: metric[:id] - ) - else - Gitlab::Routing.url_helpers.prometheus_api_project_environment_path( - project, - params[:environment], - proxy_path: query_type(metric), - query: query_for_metric(metric) - ) - end - end - - def query_type(metric) - if metric[:query] - ::Prometheus::ProxyService::PROMETHEUS_QUERY_API.to_sym - else - ::Prometheus::ProxyService::PROMETHEUS_QUERY_RANGE_API.to_sym - end - end - - def query_for_metric(metric) - query = metric[query_type(metric)] - - raise Errors::MissingQueryError, 'Each "metric" must define one of :query or :query_range' unless query - - # We need to remove any newlines since our UrlBlocker does not allow - # multiline URLs. - query.to_s.squish - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/panel_ids_inserter.rb b/lib/gitlab/metrics/dashboard/stages/panel_ids_inserter.rb deleted file mode 100644 index 239b5161256..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/panel_ids_inserter.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class PanelIdsInserter < BaseStage - # For each panel within given dashboard inserts panel_id unique in scope of the dashboard - def transform! - missing_panel_groups! unless dashboard[:panel_groups] - - for_panels_group_with_panels do |panel_group, panel| - id = generate_panel_id(panel_group, panel) - remove_panel_ids! && break if duplicated_panel_id?(id) - - insert_panel_id(id, panel) - end - rescue ActiveModel::UnknownAttributeError => error - remove_panel_ids! - Gitlab::ErrorTracking.log_exception(error) - end - - private - - def generate_panel_id(group, panel) - ::PerformanceMonitoring::PrometheusPanel.new(panel.with_indifferent_access).id(group[:group]) - end - - def insert_panel_id(id, panel) - track_inserted_panel_ids(id, panel) - panel[:id] = id - end - - def track_inserted_panel_ids(id, panel) - panel_ids[id] = panel - end - - def duplicated_panel_id?(id) - panel_ids.key?(id) - end - - def remove_panel_ids! - panel_ids.each_value { |panel| panel.delete(:id) } - end - - def panel_ids - @_panel_ids ||= {} - end - - def for_panels_group_with_panels - for_panel_groups do |panel_group| - for_panels_in(panel_group) do |panel| - yield panel_group, panel - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb b/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb deleted file mode 100644 index 71da779d16c..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class TrackPanelType < BaseStage - def transform! - for_panel_groups do |panel_group| - for_panels_in(panel_group) do |panel| - track_panel_type(panel) - end - end - end - - private - - def track_panel_type(panel) - panel_type = panel[:type] - - Gitlab::Tracking.event('MetricsDashboard::Chart', 'chart_rendered', label: panel_type) - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb deleted file mode 100644 index b3ce0b79675..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class VariableEndpointInserter < BaseStage - VARIABLE_TYPE_METRIC_LABEL_VALUES = 'metric_label_values' - - def transform! - raise Errors::DashboardProcessingError, _('Environment is required for Stages::VariableEndpointInserter') unless params[:environment] - - for_variables do |variable_name, variable| - if variable.is_a?(Hash) && variable[:type] == VARIABLE_TYPE_METRIC_LABEL_VALUES - variable[:options][:prometheus_endpoint_path] = endpoint_for_variable(variable.dig(:options, :series_selector)) - end - end - end - - private - - def endpoint_for_variable(series_selector) - Gitlab::Routing.url_helpers.prometheus_api_project_environment_path( - project, - params[:environment], - proxy_path: ::Prometheus::ProxyService::PROMETHEUS_SERIES_API, - match: Array(series_selector) - ) - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb deleted file mode 100644 index 3650ddf698a..00000000000 --- a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Transformers - module Yml - module V1 - # Takes a JSON schema validated dashboard hash and - # maps it to PrometheusMetric model attributes - class PrometheusMetrics - def initialize(dashboard_hash, project: nil, dashboard_path: nil) - @dashboard_hash = dashboard_hash.with_indifferent_access - @project = project - @dashboard_path = dashboard_path - - @dashboard_hash.default_proc = -> (h, k) { raise Transformers::Errors::MissingAttribute, k.to_s } - end - - def execute - prometheus_metrics = [] - - dashboard_hash[:panel_groups].each do |panel_group| - panel_group[:panels].each do |panel| - panel[:metrics].each do |metric| - prometheus_metrics << { - project: project, - title: panel[:title], - y_label: panel[:y_label], - query: metric[:query_range] || metric[:query], - unit: metric[:unit], - legend: metric[:label], - identifier: metric[:id], - group: Enums::PrometheusMetric.groups[:custom], - common: false, - dashboard_path: dashboard_path - }.compact - end - end - end - - prometheus_metrics - end - - private - - attr_reader :dashboard_hash, :project, :dashboard_path - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/validator.rb b/lib/gitlab/metrics/dashboard/validator.rb deleted file mode 100644 index 57b4b5c068d..00000000000 --- a/lib/gitlab/metrics/dashboard/validator.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Validator - DASHBOARD_SCHEMA_PATH = Rails.root.join(*%w[lib gitlab metrics dashboard validator schemas dashboard.json]).freeze - - class << self - def validate(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) - errors(content, schema_path, dashboard_path: dashboard_path, project: project).empty? - end - - def validate!(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) - errors = errors(content, schema_path, dashboard_path: dashboard_path, project: project) - errors.empty? || raise(errors.first) - end - - private - - def errors(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) - Validator::Client - .new(content, schema_path, dashboard_path: dashboard_path, project: project) - .execute - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/validator/client.rb b/lib/gitlab/metrics/dashboard/validator/client.rb deleted file mode 100644 index 29f1274a097..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/client.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Validator - class Client - # @param content [Hash] Representing a raw, unprocessed - # dashboard object - # @param schema_path [String] Representing path to dashboard schema file - # @param dashboard_path[String] Representing path to dashboard content file - # @param project [Project] Project to validate dashboard against - def initialize(content, schema_path, dashboard_path: nil, project: nil) - @content = content - @schema_path = schema_path - @dashboard_path = dashboard_path - @project = project - end - - def execute - errors = validate_against_schema - errors += post_schema_validator.validate - - errors.compact - end - - private - - attr_reader :content, :schema_path, :project, :dashboard_path - - def custom_formats - @custom_formats ||= CustomFormats.new - end - - def post_schema_validator - PostSchemaValidator.new( - project: project, - metric_ids: custom_formats.metric_ids_cache, - dashboard_path: dashboard_path - ) - end - - def schemer - @schemer ||= ::JSONSchemer.schema(Pathname.new(schema_path), formats: custom_formats.format_handlers) - end - - def validate_against_schema - schemer.validate(content).map do |error| - ::Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError.new(error) - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/validator/custom_formats.rb b/lib/gitlab/metrics/dashboard/validator/custom_formats.rb deleted file mode 100644 index 485e80ad1b7..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/custom_formats.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Validator - class CustomFormats - def format_handlers - # Key is custom JSON Schema format name. Value is a proc that takes data and schema and handles - # validations. - @format_handlers ||= { - "add_to_metric_id_cache" => ->(data, schema) { metric_ids_cache << data } - } - end - - def metric_ids_cache - @metric_ids_cache ||= [] - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/validator/errors.rb b/lib/gitlab/metrics/dashboard/validator/errors.rb deleted file mode 100644 index 0f6e687d291..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/errors.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Validator - module Errors - InvalidDashboardError = Class.new(StandardError) - - class SchemaValidationError < InvalidDashboardError - def initialize(error = {}) - super(error_message(error)) - end - - private - - def error_message(error) - if error.is_a?(Hash) && error.present? - pretty(error) - else - "Dashboard failed schema validation" - end - end - - # based on https://github.com/davishmcclurg/json_schemer/blob/master/lib/json_schemer/errors.rb - # with addition ability to translate error messages - def pretty(error) - data, data_pointer, type, schema = error.values_at('data', 'data_pointer', 'type', 'schema') - location = data_pointer.empty? ? 'root' : data_pointer - - case type - when 'required' - keys = error.fetch('details').fetch('missing_keys').join(', ') - _("%{location} is missing required keys: %{keys}") % { location: location, keys: keys } - when 'null', 'string', 'boolean', 'integer', 'number', 'array', 'object' - _("'%{data}' at %{location} is not of type: %{type}") % { data: data, location: location, type: type } - when 'pattern' - _("'%{data}' at %{location} does not match pattern: %{pattern}") % { data: data, location: location, pattern: schema.fetch('pattern') } - when 'format' - _("'%{data}' at %{location} does not match format: %{format}") % { data: data, location: location, format: schema.fetch('format') } - when 'const' - _("'%{data}' at %{location} is not: %{const}") % { data: data, location: location, const: schema.fetch('const').inspect } - when 'enum' - _("'%{data}' at %{location} is not one of: %{enum}") % { data: data, location: location, enum: schema.fetch('enum') } - else - _("'%{data}' at %{location} is invalid: error_type=%{type}") % { data: data, location: location, type: type } - end - end - end - - class DuplicateMetricIds < InvalidDashboardError - def initialize - super(_("metric_id must be unique across a project")) - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb b/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb deleted file mode 100644 index 73bfc5a6294..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Validator - class PostSchemaValidator - def initialize(metric_ids:, project: nil, dashboard_path: nil) - @metric_ids = metric_ids - @project = project - @dashboard_path = dashboard_path - end - - def validate - errors = [] - errors << uniq_metric_ids - errors.compact - end - - private - - attr_reader :project, :metric_ids, :dashboard_path - - def uniq_metric_ids - return Validator::Errors::DuplicateMetricIds.new if metric_ids.uniq! - - uniq_metric_ids_across_project if project.present? || dashboard_path.present? - end - - # rubocop: disable CodeReuse/ActiveRecord - def uniq_metric_ids_across_project - return ArgumentError.new(_('Both project and dashboard_path are required')) unless - dashboard_path.present? && project.present? - - # If PrometheusMetric identifier is not unique across project and dashboard_path, - # we need to error because we don't know if the user is trying to create a new metric - # or update an existing one. - identifier_on_other_dashboard = PrometheusMetric.where( - project: project, - identifier: metric_ids - ).where.not( - dashboard_path: dashboard_path - ).exists? - - Validator::Errors::DuplicateMetricIds.new if identifier_on_other_dashboard - end - # rubocop: enable CodeReuse/ActiveRecord - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/axis.json b/lib/gitlab/metrics/dashboard/validator/schemas/axis.json deleted file mode 100644 index 54334022426..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/axis.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "type": "object", - "properties": { - "name": { "type": "string" }, - "format": { - "type": "string", - "default": "engineering" - }, - "precision": { - "type": "number", - "default": 2 - } - } -} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json b/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json deleted file mode 100644 index 313f03be7dc..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": "object", - "required": ["dashboard", "panel_groups"], - "properties": { - "dashboard": { "type": "string" }, - "panel_groups": { - "type": "array", - "items": { "$ref": "./panel_group.json" } - }, - "templating": { - "$ref": "./templating.json" - }, - "links": { - "type": "array", - "items": { "$ref": "./link.json" } - } - } -} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/link.json b/lib/gitlab/metrics/dashboard/validator/schemas/link.json deleted file mode 100644 index 4ea7b5dd324..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/link.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "required": ["url"], - "properties": { - "url": { "type": "string" }, - "title": { "type": "string" }, - "type": { - "type": "string", - "enum": ["grafana"] - } - } -} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/metric.json b/lib/gitlab/metrics/dashboard/validator/schemas/metric.json deleted file mode 100644 index 13831b77e3e..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/metric.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "type": "object", - "required": ["unit"], - "oneOf": [{ "required": ["query"] }, { "required": ["query_range"] }], - "properties": { - "id": { - "type": "string", - "format": "add_to_metric_id_cache" - }, - "unit": { "type": "string" }, - "label": { "type": "string" }, - "query": { "type": ["string", "number"] }, - "query_range": { "type": ["string", "number"] }, - "step": { "type": "number" } - } -} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/panel.json b/lib/gitlab/metrics/dashboard/validator/schemas/panel.json deleted file mode 100644 index 2ae9608036e..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/panel.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "type": "object", - "required": ["title", "metrics"], - "properties": { - "type": { - "type": "string", - "enum": ["area-chart", "line-chart", "anomaly-chart", "bar", "column", "stacked-column", "single-stat", "heatmap", "gauge"], - "default": "area-chart" - }, - "title": { "type": "string" }, - "y_label": { "type": "string" }, - "y_axis": { "$ref": "./axis.json" }, - "max_value": { "type": "number" }, - "weight": { "type": "number" }, - "metrics": { - "type": "array", - "items": { "$ref": "./metric.json" } - }, - "links": { - "type": "array", - "items": { "$ref": "./link.json" } - } - } -} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json b/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json deleted file mode 100644 index 1306fc475db..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "required": ["group", "panels"], - "properties": { - "group": { "type": "string" }, - "priority": { "type": "number" }, - "panels": { - "type": "array", - "items": { "$ref": "./panel.json" } - } - } -} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/templating.json b/lib/gitlab/metrics/dashboard/validator/schemas/templating.json deleted file mode 100644 index 6f8664c89af..00000000000 --- a/lib/gitlab/metrics/dashboard/validator/schemas/templating.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "type": "object", - "required": ["variables"], - "properties": { - "variables": { "type": "object" } - } -} diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb index c361d755a12..530bebd72ab 100644 --- a/lib/gitlab/metrics/global_search_slis.rb +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -11,6 +11,7 @@ module Gitlab BASIC_CODE_TARGET_S = 27.538 ADVANCED_CONTENT_TARGET_S = 2.452 ADVANCED_CODE_TARGET_S = 15.52 + ZOEKT_TARGET_S = 15.52 def initialize_slis! Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) @@ -42,6 +43,8 @@ module Gitlab ADVANCED_CONTENT_TARGET_S elsif search_type == 'advanced' && code_search?(search_scope) ADVANCED_CODE_TARGET_S + elsif search_type == 'zoekt' && code_search?(search_scope) + ZOEKT_TARGET_S end end diff --git a/lib/gitlab/metrics/samplers/threads_sampler.rb b/lib/gitlab/metrics/samplers/threads_sampler.rb index a460594fb59..1357e0a5d9b 100644 --- a/lib/gitlab/metrics/samplers/threads_sampler.rb +++ b/lib/gitlab/metrics/samplers/threads_sampler.rb @@ -54,7 +54,7 @@ module Gitlab if thread_name.presence.nil? 'unnamed' - elsif thread_name =~ /puma threadpool \d+/ + elsif /puma threadpool \d+/.match?(thread_name) # These are the puma workers processing requests 'puma threadpool' elsif use_thread_name?(thread_name) |