diff options
16 files changed, 282 insertions, 239 deletions
diff --git a/app/models/environment.rb b/app/models/environment.rb index 7ab4a23ab16..8d98b02c05a 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -159,7 +159,7 @@ class Environment < ActiveRecord::Base def additional_metrics if has_additional_metrics? - project.monitoring_service.reactive_query(Gitlab::Prometheus::Queries::AdditionalMetricsQuery.name, self.id, &:itself) + project.monitoring_service.reactive_query(Gitlab::Prometheus::Queries::AdditionalMetricsEnvironmentQuery.name, self.id, &:itself) end end diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb new file mode 100644 index 00000000000..eb168ce8f9c --- /dev/null +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -0,0 +1,36 @@ +module Gitlab + module Prometheus + module AdditionalMetricsParser + extend self + + def load_groups_from_yaml + additional_metrics_raw.map(&method(:new)) + end + + private + + def metrics_from_list(list) + list.map { |entry| metric_from_entry(entry) } + end + + def metric_from_entry(entry) + missing_fields = [:title, :required_metrics, :weight, :queries].select { |key| !entry.has_key?(key) } + raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? + + Metric.new(entry[:title], entry[:required_metrics], entry[:weight], entry[:y_label], entry[:queries]) + end + + def group_from_entry(entry) + missing_fields = [:group, :priority, :metrics].select { |key| !entry.has_key?(key) } + raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? + + group = MetricGroup.new(entry[:group], entry[:priority]) + group.tap { |g| g.metrics = Metric.metrics_from_list(entry[:metrics]) } + end + + def additional_metrics_raw + @additional_metrics_raw ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml'))&.map(&:deep_symbolize_keys).freeze + end + end + end +end diff --git a/lib/gitlab/prometheus/metric.rb b/lib/gitlab/prometheus/metric.rb index d7cd4237a7b..5155064317c 100644 --- a/lib/gitlab/prometheus/metric.rb +++ b/lib/gitlab/prometheus/metric.rb @@ -1,24 +1,15 @@ -module Gitlab::Prometheus - class Metric - attr_reader :group, :title, :required_metrics, :weight, :y_label, :queries - - def initialize(title, required_metrics, weight, y_label, queries = []) - @title = title - @required_metrics = required_metrics - @weight = weight - @y_label = y_label || 'Values' - @queries = queries - end - - def self.metric_from_entry(entry) - missing_fields = [:title, :required_metrics, :weight, :queries].select { |key| !entry.has_key?(key) } - raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? - - Metric.new(entry[:title], entry[:required_metrics], entry[:weight], entry[:y_label], entry[:queries]) - end - - def self.metrics_from_list(list) - list.map { |entry| metric_from_entry(entry) } +module Gitlab + module Prometheus + class Metric + attr_reader :group, :title, :required_metrics, :weight, :y_label, :queries + + def initialize(title, required_metrics, weight, y_label, queries = []) + @title = title + @required_metrics = required_metrics + @weight = weight + @y_label = y_label || 'Values' + @queries = queries + end end end end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index 0dcd9dc0f36..c3b24dc1fa5 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -1,33 +1,18 @@ -module Gitlab::Prometheus - class MetricGroup - attr_reader :priority, :name - attr_accessor :metrics +module Gitlab + module Prometheus + class MetricGroup + attr_reader :priority, :name + attr_accessor :metrics - def initialize(name, priority, metrics = []) - @name = name - @priority = priority - @metrics = metrics - end - - def self.all - load_groups_from_yaml - end - - def self.load_groups_from_yaml - additional_metrics_raw.map(&method(:group_from_entry)) - end - - def self.group_from_entry(entry) - missing_fields = [:group, :priority, :metrics].select { |key| !entry.has_key?(key) } - raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty? - - group = MetricGroup.new(entry[:group], entry[:priority]) - group.metrics = Metric.metrics_from_list(entry[:metrics]) - group - end + def initialize(name:, priority:, metrics: []) + @name = name + @priority = priority + @metrics = metrics + end - def self.additional_metrics_raw - @additional_metrics_raw ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml'))&.map(&:deep_symbolize_keys).freeze + def self.all + AdditionalMetricsParser.load_groups_from_yaml + end end end end diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb index 067ea7f878a..49cc0e16080 100644 --- a/lib/gitlab/prometheus/parsing_error.rb +++ b/lib/gitlab/prometheus/parsing_error.rb @@ -1,3 +1,5 @@ -module Gitlab::Prometheus - ParsingError = Class.new(StandardError) +module Gitlab + module Prometheus + ParsingError = Class.new(StandardError) + end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 7693772bf81..cfe6bed188b 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -1,15 +1,21 @@ -module Gitlab::Prometheus::Queries - class AdditionalMetricsDeploymentQuery < AdditionalMetricsQuery - def query(deployment_id) - deployment = Deployment.find_by(id: deployment_id) - query_context = { - environment_slug: deployment.environment.slug, - environment_filter: %{container_name!="POD",environment="#{deployment.environment.slug}"}, - timeframe_start: (deployment.created_at - 30.minutes).to_f, - timeframe_end: (deployment.created_at + 30.minutes).to_f - } +module Gitlab + module Prometheus + module Queries + class AdditionalMetricsDeploymentQuery < BaseQuery + include QueryAdditionalMetrics - query_metrics(query_context) + def query(deployment_id) + deployment = Deployment.find_by(id: deployment_id) + query_context = { + environment_slug: deployment.environment.slug, + environment_filter: %{container_name!="POD",environment="#{deployment.environment.slug}"}, + timeframe_start: (deployment.created_at - 30.minutes).to_f, + timeframe_end: (deployment.created_at + 30.minutes).to_f + } + + query_metrics(query_context) + end + end end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb new file mode 100644 index 00000000000..e261988b234 --- /dev/null +++ b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb @@ -0,0 +1,21 @@ +module Gitlab + module Prometheus + module Queries + class AdditionalMetricsEnvironmentQuery < BaseQuery + include QueryAdditionalMetrics + + def query(environment_id) + environment = Environment.find_by(id: environment_id) + query_context = { + environment_slug: environment.slug, + environment_filter: %{container_name!="POD",environment="#{environment.slug}"}, + timeframe_start: 8.hours.ago.to_f, + timeframe_end: Time.now.to_f + } + + query_metrics(query_context) + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_query.rb deleted file mode 100644 index 7ef4ee3a91a..00000000000 --- a/lib/gitlab/prometheus/queries/additional_metrics_query.rb +++ /dev/null @@ -1,82 +0,0 @@ -module Gitlab::Prometheus::Queries - class AdditionalMetricsQuery < BaseQuery - def query(environment_id) - environment = Environment.find_by(id: environment_id) - query_context = { - environment_slug: environment.slug, - environment_filter: %{container_name!="POD",environment="#{environment.slug}"}, - timeframe_start: 8.hours.ago.to_f, - timeframe_end: Time.now.to_f - } - - query_metrics(query_context) - end - - protected - - def query_metrics(query_context) - query_processor = method(:process_query).curry[query_context] - - groups = matched_metrics.map do |group| - metrics = group.metrics.map do |metric| - { - title: metric.title, - weight: metric.weight, - y_label: metric.y_label, - queries: metric.queries.map(&query_processor).select(&method(:query_with_result)) - } - end - - { - group: group.name, - priority: group.priority, - metrics: metrics.select(&method(:metric_with_any_queries)) - } - end - - groups.select(&method(:group_with_any_metrics)) - end - - private - - def metric_with_any_queries(metric) - metric[:queries]&.count&.> 0 - end - - def group_with_any_metrics(group) - group[:metrics]&.count&.> 0 - end - - def query_with_result(query) - query[:result]&.any? do |item| - item&.[](:values)&.any? || item&.[](:value)&.any? - end - end - - def process_query(context, query) - query_with_result = query.dup - query_with_result[:result] = - if query.has_key?(:query_range) - client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end]) - else - client_query(query[:query] % context, time: context[:timeframe_end]) - end - query_with_result - end - - def available_metrics - @available_metrics ||= client_label_values || [] - end - - def matched_metrics - result = Gitlab::Prometheus::MetricGroup.all.map do |group| - group.metrics.select! do |metric| - metric.required_metrics.all?(&available_metrics.method(:include?)) - end - group - end - - result.select { |group| group.metrics.any? } - end - end -end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 2cc08731f8d..be3527f0f72 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -1,26 +1,30 @@ -module Gitlab::Prometheus::Queries - class DeploymentQuery < BaseQuery - def query(deployment_id) - deployment = Deployment.find_by(id: deployment_id) - environment_slug = deployment.environment.slug +module Gitlab + module Prometheus + module Queries + class DeploymentQuery < BaseQuery + def query(deployment_id) + deployment = Deployment.find_by(id: deployment_id) + environment_slug = deployment.environment.slug - memory_query = raw_memory_usage_query(environment_slug) - memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} - cpu_query = raw_cpu_usage_query(environment_slug) - cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} + memory_query = raw_memory_usage_query(environment_slug) + memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} + cpu_query = raw_cpu_usage_query(environment_slug) + cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} - timeframe_start = (deployment.created_at - 30.minutes).to_f - timeframe_end = (deployment.created_at + 30.minutes).to_f + timeframe_start = (deployment.created_at - 30.minutes).to_f + timeframe_end = (deployment.created_at + 30.minutes).to_f - { - memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), - memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), - memory_after: client_query(memory_avg_query, time: timeframe_end), + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), + memory_after: client_query(memory_avg_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), - cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), - cpu_after: client_query(cpu_avg_query, time: timeframe_end) - } + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), + cpu_after: client_query(cpu_avg_query, time: timeframe_end) + } + end + end end end end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 01d756d7284..9aa9da6d858 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -1,20 +1,24 @@ -module Gitlab::Prometheus::Queries - class EnvironmentQuery < BaseQuery - def query(environment_id) - environment = Environment.find_by(id: environment_id) - environment_slug = environment.slug - timeframe_start = 8.hours.ago.to_f - timeframe_end = Time.now.to_f +module Gitlab + module Prometheus + module Queries + class EnvironmentQuery < BaseQuery + def query(environment_id) + environment = Environment.find_by(id: environment_id) + environment_slug = environment.slug + timeframe_start = 8.hours.ago.to_f + timeframe_end = Time.now.to_f - memory_query = raw_memory_usage_query(environment_slug) - cpu_query = raw_cpu_usage_query(environment_slug) + memory_query = raw_memory_usage_query(environment_slug) + cpu_query = raw_cpu_usage_query(environment_slug) - { - memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), - memory_current: client_query(memory_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), - cpu_current: client_query(cpu_query, time: timeframe_end) - } + { + memory_values: client_query_range(memory_query, start: timeframe_start, stop: timeframe_end), + memory_current: client_query(memory_query, time: timeframe_end), + cpu_values: client_query_range(cpu_query, start: timeframe_start, stop: timeframe_end), + cpu_current: client_query(cpu_query, time: timeframe_end) + } + end + end end end end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb index fc97bca486c..d4894c87f8d 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metrics_query.rb @@ -1,74 +1,78 @@ -module Gitlab::Prometheus::Queries - class MatchedMetricsQuery < BaseQuery - MAX_QUERY_ITEMS = 40.freeze +module Gitlab + module Prometheus + module Queries + class MatchedMetricsQuery < BaseQuery + MAX_QUERY_ITEMS = 40.freeze - def query - groups_data.map do |group, data| - { - group: group.name, - priority: group.priority, - active_metrics: data[:active_metrics], - metrics_missing_requirements: data[:metrics_missing_requirements] - } - end - end + def query + groups_data.map do |group, data| + { + group: group.name, + priority: group.priority, + active_metrics: data[:active_metrics], + metrics_missing_requirements: data[:metrics_missing_requirements] + } + end + end - private + private - def groups_data - metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all) - lookup = active_series_lookup(metrics_groups) + def groups_data + metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.all) + lookup = active_series_lookup(metrics_groups) - groups = {} + groups = {} - metrics_groups.each do |group| - groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } - active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) } + metrics_groups.each do |group| + groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } + active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) } - groups[group][:active_metrics] += active_metrics - groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics - end + groups[group][:active_metrics] += active_metrics + groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics + end - groups - end + groups + end - def active_series_lookup(metric_groups) - timeframe_start = 8.hours.ago - timeframe_end = Time.now + def active_series_lookup(metric_groups) + timeframe_start = 8.hours.ago + timeframe_end = Time.now - series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq + series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq - lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| - client_series(*batched_series, start: timeframe_start, stop: timeframe_end) - .select(&method(:has_matching_label)) - .map { |series_info| [series_info['__name__'], true] } - end - lookup.to_h - end + lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| + client_series(*batched_series, start: timeframe_start, stop: timeframe_end) + .select(&method(:has_matching_label)) + .map { |series_info| [series_info['__name__'], true] } + end + lookup.to_h + end - def has_matching_label(series_info) - series_info.key?('environment') - end + def has_matching_label(series_info) + series_info.key?('environment') + end - def available_metrics - @available_metrics ||= client_label_values || [] - end + def available_metrics + @available_metrics ||= client_label_values || [] + end - def filter_active_metrics(metric_group) - metric_group.metrics.select! do |metric| - metric.required_metrics.all?(&available_metrics.method(:include?)) - end - metric_group - end + def filter_active_metrics(metric_group) + metric_group.metrics.select! do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + metric_group + end - def groups_with_active_metrics(metric_groups) - metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? } - end + def groups_with_active_metrics(metric_groups) + metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? } + end - def metrics_with_required_series(metric_groups) - metric_groups.flat_map do |group| - group.metrics.select do |metric| - metric.required_metrics.all?(&available_metrics.method(:include?)) + def metrics_with_required_series(metric_groups) + metric_groups.flat_map do |group| + group.metrics.select do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + end end end end diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb new file mode 100644 index 00000000000..46029a1889a --- /dev/null +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -0,0 +1,72 @@ +module Gitlab + module Prometheus + module Queries + module QueryAdditionalMetrics + def query_metrics(query_context) + query_processor = method(:process_query).curry[query_context] + + groups = matched_metrics.map do |group| + metrics = group.metrics.map do |metric| + { + title: metric.title, + weight: metric.weight, + y_label: metric.y_label, + queries: metric.queries.map(&query_processor).select(&method(:query_with_result)) + } + end + + { + group: group.name, + priority: group.priority, + metrics: metrics.select(&method(:metric_with_any_queries)) + } + end + + groups.select(&method(:group_with_any_metrics)) + end + + private + + def metric_with_any_queries(metric) + metric[:queries]&.count&.> 0 + end + + def group_with_any_metrics(group) + group[:metrics]&.count&.> 0 + end + + def query_with_result(query) + query[:result]&.any? do |item| + item&.[](:values)&.any? || item&.[](:value)&.any? + end + end + + def process_query(context, query) + query_with_result = query.dup + query_with_result[:result] = + if query.has_key?(:query_range) + client_query_range(query[:query_range] % context, start: context[:timeframe_start], stop: context[:timeframe_end]) + else + client_query(query[:query] % context, time: context[:timeframe_end]) + end + query_with_result + end + + def available_metrics + @available_metrics ||= client_label_values || [] + end + + def matched_metrics + result = Gitlab::Prometheus::MetricGroup.all.map do |group| + group.metrics.select! do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + group + end + + result.select { |group| group.metrics.any? } + end + end + end + end +end diff --git a/spec/controllers/projects/prometheus_controller_spec.rb b/spec/controllers/projects/prometheus_controller_spec.rb index 5e57b19e042..7c976cfad83 100644 --- a/spec/controllers/projects/prometheus_controller_spec.rb +++ b/spec/controllers/projects/prometheus_controller_spec.rb @@ -57,4 +57,4 @@ describe Projects::PrometheusController do def project_params(opts = {}) opts.reverse_merge(namespace_id: project.namespace, project_id: project) end -end
\ No newline at end of file +end diff --git a/spec/lib/gitlab/prometheus/queries/additional_metrics_query_spec.rb b/spec/lib/gitlab/prometheus/queries/additional_metrics_environment_query_spec.rb index 6fbd2fd17d6..8e6e3bb5946 100644 --- a/spec/lib/gitlab/prometheus/queries/additional_metrics_query_spec.rb +++ b/spec/lib/gitlab/prometheus/queries/additional_metrics_environment_query_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Prometheus::Queries::AdditionalMetricsQuery, lib: true do +describe Gitlab::Prometheus::Queries::AdditionalMetricsEnvironmentQuery, lib: true do include Prometheus::MetricBuilders let(:client) { double('prometheus_client') } diff --git a/spec/models/environment_spec.rb b/spec/models/environment_spec.rb index 12de9c9111b..e25d2bb6955 100644 --- a/spec/models/environment_spec.rb +++ b/spec/models/environment_spec.rb @@ -453,7 +453,7 @@ describe Environment, models: true do it 'returns the additional metrics from the deployment service' do expect(project.monitoring_service).to receive(:reactive_query) - .with(Gitlab::Prometheus::Queries::AdditionalMetricsQuery.name, environment.id) + .with(Gitlab::Prometheus::Queries::AdditionalMetricsEnvironmentQuery.name, environment.id) .and_return(:fake_metrics) is_expected.to eq(:fake_metrics) diff --git a/spec/support/prometheus/metric_builders.rb b/spec/support/prometheus/metric_builders.rb index 4c0e01bd5f6..cc733bfe1b4 100644 --- a/spec/support/prometheus/metric_builders.rb +++ b/spec/support/prometheus/metric_builders.rb @@ -21,7 +21,7 @@ module Prometheus end def simple_metric_group(name: 'name', metrics: simple_metrics) - Gitlab::Prometheus::MetricGroup.new(name, 1, metrics) + Gitlab::Prometheus::MetricGroup.new(name: name, priority: 1, metrics: metrics) end end end |