diff options
Diffstat (limited to 'lib/gitlab')
7 files changed, 193 insertions, 2 deletions
diff --git a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb index dcc336b8929..4990a5a6eb5 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb @@ -9,7 +9,7 @@ module Gitlab class Remote < Source def content strong_memoize(:content) do - next unless ci_config_path =~ URI.regexp(%w[http https]) + next unless ci_config_path =~ URI::DEFAULT_PARSER.make_regexp(%w[http https]) YAML.dump('include' => [{ 'remote' => ci_config_path }]) end diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb index fb0fcc5a93b..8a5f53be20f 100644 --- a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb +++ b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb @@ -18,6 +18,7 @@ module Gitlab business: 0, response: 1, system: 2, + custom: 3, cluster_health: -100 } @@ -34,7 +35,8 @@ module Gitlab aws_elb: _('Response metrics (AWS ELB)'), nginx: _('Response metrics (NGINX)'), kubernetes: _('System metrics (Kubernetes)'), - cluster_health: _('Cluster Health') + cluster_health: _('Cluster Health'), + custom: _('Custom metrics') } end end diff --git a/lib/gitlab/metrics/dashboard/importer.rb b/lib/gitlab/metrics/dashboard/importer.rb new file mode 100644 index 00000000000..ca835650648 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/importer.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + class Importer + def initialize(dashboard_path, project) + @dashboard_path = dashboard_path.to_s + @project = project + end + + def execute + return false unless Dashboard::Validator.validate(dashboard_hash, project: project, dashboard_path: dashboard_path) + + Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute + rescue Gitlab::Config::Loader::FormatError + false + end + + def execute! + Dashboard::Validator.validate!(dashboard_hash, project: project, dashboard_path: dashboard_path) + + Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute! + end + + private + + attr_accessor :dashboard_path, :project + + def dashboard_hash + @dashboard_hash ||= begin + raw_dashboard = Dashboard::RepoDashboardFinder.read_dashboard(project, dashboard_path) + return unless raw_dashboard.present? + + ::Gitlab::Config::Loader::Yaml.new(raw_dashboard).load_raw! + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb new file mode 100644 index 00000000000..d1490d5d9b6 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Importers + class PrometheusMetrics + ALLOWED_ATTRIBUTES = %i(title query y_label unit legend group dashboard_path).freeze + + # Takes a JSON schema validated dashboard hash and + # imports metrics to database + def initialize(dashboard_hash, project:, dashboard_path:) + @dashboard_hash = dashboard_hash + @project = project + @dashboard_path = dashboard_path + end + + def execute + import + rescue ActiveRecord::RecordInvalid, ::Gitlab::Metrics::Dashboard::Transformers::TransformerError + false + end + + def execute! + import + end + + private + + attr_reader :dashboard_hash, :project, :dashboard_path + + def import + delete_stale_metrics + create_or_update_metrics + end + + # rubocop: disable CodeReuse/ActiveRecord + def create_or_update_metrics + # TODO: use upsert and worker for callbacks? + prometheus_metrics_attributes.each do |attributes| + prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:identifier, :project)) + prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES)) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + def delete_stale_metrics + identifiers = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] } + + stale_metrics = PrometheusMetric.for_project(project) + .for_dashboard_path(dashboard_path) + .for_group(Enums::PrometheusMetric.groups[:custom]) + .not_identifier(identifiers) + + # TODO: use destroy_all and worker for callbacks? + stale_metrics.each(&:destroy) + end + + def prometheus_metrics_attributes + @prometheus_metrics_attributes ||= begin + Dashboard::Transformers::Yml::V1::PrometheusMetrics.new( + dashboard_hash, + project: project, + dashboard_path: dashboard_path + ).execute + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/transformers/errors.rb b/lib/gitlab/metrics/dashboard/transformers/errors.rb new file mode 100644 index 00000000000..4d94ab098ae --- /dev/null +++ b/lib/gitlab/metrics/dashboard/transformers/errors.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Transformers + TransformerError = Class.new(StandardError) + + module Errors + class MissingAttribute < TransformerError + def initialize(attribute_name) + super("Missing attribute: '#{attribute_name}'") + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb new file mode 100644 index 00000000000..4e46eec17d6 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Transformers + module Yml + module V1 + # Takes a JSON schema validated dashboard hash and + # maps it to PrometheusMetric model attributes + class PrometheusMetrics + def initialize(dashboard_hash, project: nil, dashboard_path: nil) + @dashboard_hash = dashboard_hash.with_indifferent_access + @project = project + @dashboard_path = dashboard_path + + @dashboard_hash.default_proc = -> (h, k) { raise Transformers::Errors::MissingAttribute, k.to_s } + end + + def execute + prometheus_metrics = [] + + dashboard_hash[:panel_groups].each do |panel_group| + panel_group[:panels].each do |panel| + panel[:metrics].each do |metric| + prometheus_metrics << { + project: project, + title: panel[:title], + y_label: panel[:y_label], + query: metric[:query_range] || metric[:query], + unit: metric[:unit], + legend: metric[:label], + identifier: metric[:id], + group: Enums::PrometheusMetric.groups[:custom], + common: false, + dashboard_path: dashboard_path + }.compact + end + end + end + + prometheus_metrics + end + + private + + attr_reader :dashboard_hash, :project, :dashboard_path + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 1dd6eab1743..a9e2ba4d85c 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -269,6 +269,9 @@ module Gitlab database: { adapter: alt_usage_data { Gitlab::Database.adapter_name }, version: alt_usage_data { Gitlab::Database.version } + }, + mail: { + smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] } } } end |