diff options
Diffstat (limited to 'lib/gitlab/metrics/dashboard')
21 files changed, 477 insertions, 91 deletions
diff --git a/lib/gitlab/metrics/dashboard/cache.rb b/lib/gitlab/metrics/dashboard/cache.rb index a9ccf0fea9b..54b5250d209 100644 --- a/lib/gitlab/metrics/dashboard/cache.rb +++ b/lib/gitlab/metrics/dashboard/cache.rb @@ -9,34 +9,53 @@ module Gitlab CACHE_KEYS = 'all_cached_metric_dashboards' class << self - # Stores a dashboard in the cache, documenting the key - # so the cached can be cleared in bulk at another time. - def fetch(key) - register_key(key) + # This class method (Gitlab::Metrics::Dashboard::Cache.fetch) can be used + # when the key does not need to be deleted by `delete_all!`. + # For example, out of the box dashboard caches do not need to be deleted. + delegate :fetch, to: :"Rails.cache" - Rails.cache.fetch(key) { yield } - end + alias_method :for, :new + end + + def initialize(project) + @project = project + end + + # Stores a dashboard in the cache, documenting the key + # so the cache can be cleared in bulk at another time. + def fetch(key) + register_key(key) + + Rails.cache.fetch(key) { yield } + end - # Resets all dashboard caches, such that all - # dashboard content will be loaded from source on - # subsequent dashboard calls. - def delete_all! - all_keys.each { |key| Rails.cache.delete(key) } + # Resets all dashboard caches, such that all + # dashboard content will be loaded from source on + # subsequent dashboard calls. + def delete_all! + all_keys.each { |key| Rails.cache.delete(key) } - Rails.cache.delete(CACHE_KEYS) - end + Rails.cache.delete(catalog_key) + end - private + private - def register_key(key) - new_keys = all_keys.add(key).to_a.join('|') + def register_key(key) + new_keys = all_keys.add(key).to_a.join('|') - Rails.cache.write(CACHE_KEYS, new_keys) - end + Rails.cache.write(catalog_key, new_keys) + end + + def all_keys + keys = Rails.cache.read(catalog_key)&.split('|') + Set.new(keys) + end - def all_keys - Set.new(Rails.cache.read(CACHE_KEYS)&.split('|')) - end + # One key to store them all... + # This key is used to store the names of all the keys that contain this + # project's dashboards. + def catalog_key + "#{CACHE_KEYS}_#{@project.id}" end end end diff --git a/lib/gitlab/metrics/dashboard/defaults.rb b/lib/gitlab/metrics/dashboard/defaults.rb index 3c39a7c6911..6a5f98a18c8 100644 --- a/lib/gitlab/metrics/dashboard/defaults.rb +++ b/lib/gitlab/metrics/dashboard/defaults.rb @@ -7,7 +7,6 @@ module Gitlab module Dashboard module Defaults DEFAULT_PANEL_TYPE = 'area-chart' - DEFAULT_PANEL_WEIGHT = 0 end end end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index 5e2d78e10a4..2c4793eb75f 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -14,10 +14,7 @@ module Gitlab ::Metrics::Dashboard::SelfMonitoringDashboardService, # This dashboard is displayed on the K8s cluster settings health page. - ::Metrics::Dashboard::ClusterDashboardService, - - # This dashboard is not yet ready for the world. - ::Metrics::Dashboard::PodDashboardService + ::Metrics::Dashboard::ClusterDashboardService ].freeze class << self @@ -72,17 +69,11 @@ module Gitlab # display_name: String, # default: Boolean }] def find_all_paths(project) - project.repository.metrics_dashboard_paths - end - - # Summary of all known dashboards. Used to populate repo cache. - # Prefer #find_all_paths. - def find_all_paths_from_source(project) - Gitlab::Metrics::Dashboard::Cache.delete_all! - - user_facing_dashboard_services(project).flat_map do |service| + dashboards = user_facing_dashboard_services(project).flat_map do |service| service.all_dashboard_paths(project) end + + Gitlab::Utils.stable_sort_by(dashboards) { |dashboard| dashboard[:display_name].downcase } end private diff --git a/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb b/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb new file mode 100644 index 00000000000..8b791e110ba --- /dev/null +++ b/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +# Provides methods to list and read dashboard yaml files from a project's repository. +module Gitlab + module Metrics + module Dashboard + class RepoDashboardFinder + DASHBOARD_ROOT = ".gitlab/dashboards" + DASHBOARD_EXTENSION = '.yml' + + class << self + # Returns list of all user-defined dashboard paths. Used to populate + # Repository model cache (Repository#user_defined_metrics_dashboard_paths). + # Also deletes all dashboard cache entries. + # @return [Array] ex) ['.gitlab/dashboards/dashboard1.yml'] + def list_dashboards(project) + Gitlab::Metrics::Dashboard::Cache.for(project).delete_all! + + file_finder(project).list_files_for(DASHBOARD_ROOT) + end + + # Reads the given dashboard from repository, and returns the content as a string. + # @return [String] + def read_dashboard(project, dashboard_path) + file_finder(project).read(dashboard_path) + end + + private + + def file_finder(project) + Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, DASHBOARD_EXTENSION) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb index 3444a01bccd..3b49eb1c837 100644 --- a/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb @@ -9,7 +9,10 @@ module Gitlab # config. If there are no project-specific metrics, # this will have no effect. def transform! - PrometheusMetricsFinder.new(project: project).execute.each do |project_metric| + custom_metrics = PrometheusMetricsFinder.new(project: project, ordered: true).execute + custom_metrics = Gitlab::Utils.stable_sort_by(custom_metrics) { |metric| -metric.priority } + + custom_metrics.each do |project_metric| group = find_or_create_panel_group(dashboard[:panel_groups], project_metric) panel = find_or_create_panel(group[:panels], project_metric) find_or_create_metric(panel[:metrics], project_metric) @@ -83,7 +86,6 @@ module Gitlab def new_panel_group(metric) { group: metric.group_title, - priority: metric.priority, panels: [] } end diff --git a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb index c48a7ff25a5..dd85bd0beb1 100644 --- a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb @@ -45,7 +45,9 @@ module Gitlab raise Errors::MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query - query + # We need to remove any newlines since our UrlBlocker does not allow + # multiline URLs. + query.to_s.squish end end end diff --git a/lib/gitlab/metrics/dashboard/stages/sorter.rb b/lib/gitlab/metrics/dashboard/stages/sorter.rb deleted file mode 100644 index 882211e1441..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/sorter.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class Sorter < BaseStage - def transform! - missing_panel_groups! unless dashboard[:panel_groups].is_a? Array - - sort_groups! - sort_panels! - end - - private - - # Sorts the groups in the dashboard by the :priority key - def sort_groups! - dashboard[:panel_groups] = Gitlab::Utils.stable_sort_by(dashboard[:panel_groups]) { |group| -group[:priority].to_i } - end - - # Sorts the panels in the dashboard by the :weight key - def sort_panels! - dashboard[:panel_groups].each do |group| - missing_panels! unless group[:panels].is_a? Array - - group[:panels] = Gitlab::Utils.stable_sort_by(group[:panels]) { |panel| -panel[:weight].to_i } - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb b/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb new file mode 100644 index 00000000000..71da779d16c --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class TrackPanelType < BaseStage + def transform! + for_panel_groups do |panel_group| + for_panels_in(panel_group) do |panel| + track_panel_type(panel) + end + end + end + + private + + def track_panel_type(panel) + panel_type = panel[:type] + + Gitlab::Tracking.event('MetricsDashboard::Chart', 'chart_rendered', label: panel_type) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb index 10a2f3c2397..160ecfb85c9 100644 --- a/lib/gitlab/metrics/dashboard/url.rb +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -43,6 +43,39 @@ module Gitlab end end + # Matches dashboard urls for a metric chart embed + # for cluster metrics + # + # EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB) + def clusters_regex + strong_memoize(:clusters_regex) do + regex_for_project_metrics( + %r{ + /clusters + /(?<cluster_id>\d+) + /? + }x + ) + end + end + + # Matches dashboard urls for a metric chart embed + # for a specifc firing GitLab alert + # + # EX - https://<host>/<namespace>/<project>/prometheus/alerts/<alert_id>/metrics_dashboard + def alert_regex + strong_memoize(:alert_regex) do + regex_for_project_metrics( + %r{ + /prometheus + /alerts + /(?<alert>\d+) + /metrics_dashboard + }x + ) + end + end + # Parses query params out from full url string into hash. # # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group' @@ -60,22 +93,6 @@ module Gitlab Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args) end - # Matches dashboard urls for a metric chart embed - # for cluster metrics - # - # EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB) - def clusters_regex - strong_memoize(:clusters_regex) do - regex_for_project_metrics( - %r{ - /clusters - /(?<cluster_id>\d+) - /? - }x - ) - end - end - private def regex_for_project_metrics(path_suffix_pattern) @@ -92,16 +109,18 @@ module Gitlab end def gitlab_host_pattern - Regexp.escape(Gitlab.config.gitlab.url) + Regexp.escape(gitlab_domain) end def project_path_pattern "\/#{Project.reference_pattern}" end + + def gitlab_domain + Gitlab.config.gitlab.url + end end end end end end - -Gitlab::Metrics::Dashboard::Url.extend_if_ee('::EE::Gitlab::Metrics::Dashboard::Url') diff --git a/lib/gitlab/metrics/dashboard/validator.rb b/lib/gitlab/metrics/dashboard/validator.rb new file mode 100644 index 00000000000..8edd9c397f9 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Validator + DASHBOARD_SCHEMA_PATH = 'lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json'.freeze + + class << self + def validate(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) + errors = _validate(content, schema_path, dashboard_path: dashboard_path, project: project) + errors.empty? + end + + def validate!(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil) + errors = _validate(content, schema_path, dashboard_path: dashboard_path, project: project) + errors.empty? || raise(errors.first) + end + + private + + def _validate(content, schema_path, dashboard_path: nil, project: nil) + client = Validator::Client.new(content, schema_path, dashboard_path: dashboard_path, project: project) + client.execute + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/validator/client.rb b/lib/gitlab/metrics/dashboard/validator/client.rb new file mode 100644 index 00000000000..c63415abcfc --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/client.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Validator + class Client + # @param content [Hash] Representing a raw, unprocessed + # dashboard object + # @param schema_path [String] Representing path to dashboard schema file + # @param dashboard_path[String] Representing path to dashboard content file + # @param project [Project] Project to validate dashboard against + def initialize(content, schema_path, dashboard_path: nil, project: nil) + @content = content + @schema_path = schema_path + @dashboard_path = dashboard_path + @project = project + end + + def execute + errors = validate_against_schema + errors += post_schema_validator.validate + + errors.compact + end + + private + + attr_reader :content, :schema_path, :project, :dashboard_path + + def custom_formats + @custom_formats ||= CustomFormats.new + end + + def post_schema_validator + PostSchemaValidator.new( + project: project, + metric_ids: custom_formats.metric_ids_cache, + dashboard_path: dashboard_path + ) + end + + def schemer + @schemer ||= ::JSONSchemer.schema(Pathname.new(schema_path), formats: custom_formats.format_handlers) + end + + def validate_against_schema + schemer.validate(content).map do |error| + Errors::SchemaValidationError.new(error) + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/validator/custom_formats.rb b/lib/gitlab/metrics/dashboard/validator/custom_formats.rb new file mode 100644 index 00000000000..485e80ad1b7 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/custom_formats.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Validator + class CustomFormats + def format_handlers + # Key is custom JSON Schema format name. Value is a proc that takes data and schema and handles + # validations. + @format_handlers ||= { + "add_to_metric_id_cache" => ->(data, schema) { metric_ids_cache << data } + } + end + + def metric_ids_cache + @metric_ids_cache ||= [] + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/validator/errors.rb b/lib/gitlab/metrics/dashboard/validator/errors.rb new file mode 100644 index 00000000000..0f6e687d291 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/errors.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Validator + module Errors + InvalidDashboardError = Class.new(StandardError) + + class SchemaValidationError < InvalidDashboardError + def initialize(error = {}) + super(error_message(error)) + end + + private + + def error_message(error) + if error.is_a?(Hash) && error.present? + pretty(error) + else + "Dashboard failed schema validation" + end + end + + # based on https://github.com/davishmcclurg/json_schemer/blob/master/lib/json_schemer/errors.rb + # with addition ability to translate error messages + def pretty(error) + data, data_pointer, type, schema = error.values_at('data', 'data_pointer', 'type', 'schema') + location = data_pointer.empty? ? 'root' : data_pointer + + case type + when 'required' + keys = error.fetch('details').fetch('missing_keys').join(', ') + _("%{location} is missing required keys: %{keys}") % { location: location, keys: keys } + when 'null', 'string', 'boolean', 'integer', 'number', 'array', 'object' + _("'%{data}' at %{location} is not of type: %{type}") % { data: data, location: location, type: type } + when 'pattern' + _("'%{data}' at %{location} does not match pattern: %{pattern}") % { data: data, location: location, pattern: schema.fetch('pattern') } + when 'format' + _("'%{data}' at %{location} does not match format: %{format}") % { data: data, location: location, format: schema.fetch('format') } + when 'const' + _("'%{data}' at %{location} is not: %{const}") % { data: data, location: location, const: schema.fetch('const').inspect } + when 'enum' + _("'%{data}' at %{location} is not one of: %{enum}") % { data: data, location: location, enum: schema.fetch('enum') } + else + _("'%{data}' at %{location} is invalid: error_type=%{type}") % { data: data, location: location, type: type } + end + end + end + + class DuplicateMetricIds < InvalidDashboardError + def initialize + super(_("metric_id must be unique across a project")) + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb b/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb new file mode 100644 index 00000000000..73bfc5a6294 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Validator + class PostSchemaValidator + def initialize(metric_ids:, project: nil, dashboard_path: nil) + @metric_ids = metric_ids + @project = project + @dashboard_path = dashboard_path + end + + def validate + errors = [] + errors << uniq_metric_ids + errors.compact + end + + private + + attr_reader :project, :metric_ids, :dashboard_path + + def uniq_metric_ids + return Validator::Errors::DuplicateMetricIds.new if metric_ids.uniq! + + uniq_metric_ids_across_project if project.present? || dashboard_path.present? + end + + # rubocop: disable CodeReuse/ActiveRecord + def uniq_metric_ids_across_project + return ArgumentError.new(_('Both project and dashboard_path are required')) unless + dashboard_path.present? && project.present? + + # If PrometheusMetric identifier is not unique across project and dashboard_path, + # we need to error because we don't know if the user is trying to create a new metric + # or update an existing one. + identifier_on_other_dashboard = PrometheusMetric.where( + project: project, + identifier: metric_ids + ).where.not( + dashboard_path: dashboard_path + ).exists? + + Validator::Errors::DuplicateMetricIds.new if identifier_on_other_dashboard + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/axis.json b/lib/gitlab/metrics/dashboard/validator/schemas/axis.json new file mode 100644 index 00000000000..54334022426 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/axis.json @@ -0,0 +1,14 @@ +{ + "type": "object", + "properties": { + "name": { "type": "string" }, + "format": { + "type": "string", + "default": "engineering" + }, + "precision": { + "type": "number", + "default": 2 + } + } +} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json b/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json new file mode 100644 index 00000000000..313f03be7dc --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json @@ -0,0 +1,18 @@ +{ + "type": "object", + "required": ["dashboard", "panel_groups"], + "properties": { + "dashboard": { "type": "string" }, + "panel_groups": { + "type": "array", + "items": { "$ref": "./panel_group.json" } + }, + "templating": { + "$ref": "./templating.json" + }, + "links": { + "type": "array", + "items": { "$ref": "./link.json" } + } + } +} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/link.json b/lib/gitlab/metrics/dashboard/validator/schemas/link.json new file mode 100644 index 00000000000..4ea7b5dd324 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/link.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "required": ["url"], + "properties": { + "url": { "type": "string" }, + "title": { "type": "string" }, + "type": { + "type": "string", + "enum": ["grafana"] + } + } +} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/metric.json b/lib/gitlab/metrics/dashboard/validator/schemas/metric.json new file mode 100644 index 00000000000..13831b77e3e --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/metric.json @@ -0,0 +1,16 @@ +{ + "type": "object", + "required": ["unit"], + "oneOf": [{ "required": ["query"] }, { "required": ["query_range"] }], + "properties": { + "id": { + "type": "string", + "format": "add_to_metric_id_cache" + }, + "unit": { "type": "string" }, + "label": { "type": "string" }, + "query": { "type": ["string", "number"] }, + "query_range": { "type": ["string", "number"] }, + "step": { "type": "number" } + } +} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/panel.json b/lib/gitlab/metrics/dashboard/validator/schemas/panel.json new file mode 100644 index 00000000000..011eef53e40 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/panel.json @@ -0,0 +1,24 @@ +{ + "type": "object", + "required": ["title", "metrics"], + "properties": { + "type": { + "type": "string", + "enum": ["area-chart", "anomaly-chart", "bar", "column", "stacked-column", "single-stat", "heatmap"], + "default": "area-chart" + }, + "title": { "type": "string" }, + "y_label": { "type": "string" }, + "y_axis": { "$ref": "./axis.json" }, + "max_value": { "type": "number" }, + "weight": { "type": "number" }, + "metrics": { + "type": "array", + "items": { "$ref": "./metric.json" } + }, + "links": { + "type": "array", + "items": { "$ref": "./link.json" } + } + } +} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json b/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json new file mode 100644 index 00000000000..1306fc475db --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "required": ["group", "panels"], + "properties": { + "group": { "type": "string" }, + "priority": { "type": "number" }, + "panels": { + "type": "array", + "items": { "$ref": "./panel.json" } + } + } +} diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/templating.json b/lib/gitlab/metrics/dashboard/validator/schemas/templating.json new file mode 100644 index 00000000000..6f8664c89af --- /dev/null +++ b/lib/gitlab/metrics/dashboard/validator/schemas/templating.json @@ -0,0 +1,7 @@ +{ + "type": "object", + "required": ["variables"], + "properties": { + "variables": { "type": "object" } + } +} |