From 57de968a92f84a15736bbe37a5804fee7c47778c Mon Sep 17 00:00:00 2001 From: Pawel Chojnacki Date: Fri, 23 Feb 2018 21:33:33 +0100 Subject: Squashed commit of the following: commit 22e1cb8f4b98f71d21026f69aa3e68d79946d6ae Merge: 766a42a9639 e966c6aea25 Author: Pawel Chojnacki Date: Fri Feb 23 21:30:46 2018 +0100 Merge remote-tracking branch 'upstream/master' into 38783-add-cluster-metrics.yml # Conflicts: # app/controllers/projects/prometheus/metrics_controller.rb # app/controllers/projects/prometheus_controller.rb # app/models/project_services/prometheus_service.rb # lib/gitlab/prometheus/queries/query_additional_metrics.rb # spec/controllers/projects/prometheus/metrics_controller_spec.rb # spec/models/project_services/prometheus_service_spec.rb commit 766a42a96393f502d439c1f0beb0b6cfb2c228d4 Author: Pawel Chojnacki Date: Fri Feb 23 17:58:45 2018 +0100 Move prometheus adapter to app/models/concerns commit ca84eed49811cf3064a2e5ea611af1c947c590d7 Merge: 66702099586 cb504cedc2a Author: Pawel Chojnacki Date: Fri Feb 23 15:36:03 2018 +0100 Merge remote-tracking branch 'upstream/backport_custom_metrics_ce_components' into 38783-add-cluster-metrics.yml + fix failing tests # Conflicts: # app/controllers/projects/prometheus_controller.rb # app/models/project_services/prometheus_service.rb # lib/gitlab/prometheus/queries/query_additional_metrics.rb # spec/models/project_services/prometheus_service_spec.rb commit cb504cedc2a6e353ffb56833334681e3da09fc14 Author: Pawel Chojnacki Date: Fri Feb 23 14:46:15 2018 +0100 Fix prometheus_service found by find_or_initialize_service commit 928b84c72c2c4c46e1785b9a943c1822a137de16 Author: Pawel Chojnacki Date: Thu Feb 22 23:15:11 2018 +0100 additional metrics and backported tests commit 41291383b4b6976af94eaf9e1a7b2e2a172310e7 Author: Pawel Chojnacki Date: Thu Feb 22 21:57:45 2018 +0100 Introduce Metrics controller and retire prometheus controller commit 6e7492e4c7ffa9d8621f09198071bc14be875976 Author: Pawel Chojnacki Date: Thu Feb 22 21:46:37 2018 +0100 Backport PrometheusClient::Error and all->common_metrics rename commit 66702099586c864a7f78970a0ee0dd9f3c2beeef Author: Pawel Chojnacki Date: Thu Feb 22 20:23:03 2018 +0100 fix failing tests commit ce921ea723cde61b2441ba98c9aca97606c719c8 Author: Mike Greiling Date: Wed Feb 21 23:47:30 2018 -0600 add labels to cluster metrics commit 993830c6892e4fbf53d3f72da00002d642c7e9b2 Author: Pawel Chojnacki Date: Thu Feb 22 01:20:49 2018 +0100 Fix formatting probelms and few small tests commit 66ee65d8e3f747c90d986bc3056178422156bc8e Author: Pawel Chojnacki Date: Thu Feb 22 01:08:03 2018 +0100 stop using in deployment tests environment.id commit 62c91978d15f0369988521363dae24bd7510d68d Author: Pawel Chojnacki Date: Thu Feb 22 01:06:33 2018 +0100 fix prometheus_controller and adapter tests commit 977b1d34c1d03c7233582e8328f85caf634895ed Author: Pawel Chojnacki Date: Thu Feb 22 00:19:39 2018 +0100 finish up active? -> can_query? rename commit e614f7daee58a9758d83ba3efe1649c8b80bc1e2 Author: Pawel Chojnacki Date: Wed Feb 21 23:51:33 2018 +0100 deployment prometheus adapter tests fix commit ebd726c114a6026fef0adf3eba6ee1972530148a Author: Pawel Chojnacki Date: Wed Feb 21 21:40:40 2018 +0100 Move environment dependant tests to environment commit 6d31311cd3729c29233283dded70e03a4a9a3c97 Author: Pawel Chojnacki Date: Wed Feb 21 20:40:24 2018 +0100 update monitoring service and move adding dpeloyment_time to deployment model commit 60b6bf391ab36846dce122bc6b0c5196a186267c Author: Pawel Chojnacki Date: Wed Feb 21 20:15:46 2018 +0100 adjust deployment spec and prometheus specs commit 6681662cf1c028aff2ff94aa0501732cb7119ba1 Author: Pawel Chojnacki Date: Tue Feb 20 22:30:20 2018 +0100 Revert changes to reactive caching commit e282f86c45a056889f57d3f7fd23a81c88efff6a Merge: 5751c73df59 6844a2df873 Author: Pawel Chojnacki Date: Tue Feb 20 22:08:17 2018 +0100 Merge remote-tracking branch 'upstream/master' into 38783-add-cluster-metrics.yml commit 5751c73df59d0a03840a1b4b71b0637670f971a6 Author: Pawel Chojnacki Date: Tue Feb 20 22:07:43 2018 +0100 rename active? to can_query? and cleanup environment prometheus router commit 3f3c6e1d33dcd9315979daf26a95f2aab83a7de9 Author: Pawel Chojnacki Date: Tue Feb 20 21:51:43 2018 +0100 Fix tests, and only use prometheus service if its active commit 6345838bac584c213b665d334252ccab202cb271 Author: Pawel Chojnacki Date: Tue Feb 20 16:24:27 2018 +0100 Fix typo commit 7a585d32afe8da050b5615b1d036a550e06479f5 Author: Pawel Chojnacki Date: Tue Feb 20 15:29:40 2018 +0100 Cluster id is not required commit e6af62afb11fa380f6aff1c31a81bcc9bab3b1eb Author: Pawel Chojnacki Date: Tue Feb 20 15:15:46 2018 +0100 Result transformation support commit f3b1bd7c67894f44efe33591ddb70093bd620c03 Author: Pawel Chojnacki Date: Tue Feb 20 15:07:07 2018 +0100 Fix rubocop warning and exten cluster query timeframe commit be77947cea64261a4d3dead33c3c57f413a9880c Author: Pawel Chojnacki Date: Tue Feb 20 14:42:47 2018 +0100 Fix additional metrics test commit eb3922e16221abe16f59fae1c38122f227643343 Author: Pawel Chojnacki Date: Tue Feb 20 14:34:44 2018 +0100 rename prometheus adapter methods commit 045476cd08b21593818b274ae8a44d19b705523f Author: Pawel Chojnacki Date: Tue Feb 20 13:19:36 2018 +0100 Make prometheus adapter a module commit f2daf050d8c689f72c4c61207930bc53c331f12e Author: Pawel Chojnacki Date: Tue Feb 20 12:00:44 2018 +0100 refactoring wip commit 52e4ef5587794e811dc10a0f2dca522342a865da Author: Pawel Chojnacki Date: Sun Feb 18 20:25:55 2018 +0100 cleanup prometheus adapter concept commit 3887365faab9dfcd9c00bcfc501d09ac62431a03 Author: Pawel Chojnacki Date: Sun Feb 18 19:22:11 2018 +0100 Refactor out deployment id, Rename PrometheusQuerier to PrometheusAdapter commit aa2fc2df57bd72c9a5e94f66d1f1e23990be6c3f Author: Pawel Chojnacki Date: Fri Feb 16 23:59:54 2018 +0100 Refactor prometheus client commit e43c1ca9d9874d6cf1569f40fa1aca158d9d5d91 Author: Pawel Chojnacki Date: Thu Feb 15 20:24:45 2018 +0100 Use initial version of cluster_metrics.yml commit 867821ce0b2609ebf8994220aa8e3a94d66a01e0 Author: Pawel Chojnacki Date: Thu Feb 15 05:24:14 2018 +0100 Fix querying cluster metrics commit 1601e002a064cbb10ffe110a19433c5662858f1d Author: Pawel Chojnacki Date: Thu Feb 15 04:49:03 2018 +0100 Queues for unicersal querier commit 5db198fdc925c0223be24939b76da1d544dd569c Author: Pawel Chojnacki Date: Thu Feb 15 03:35:17 2018 +0100 refactor reactive caching and prometheus querying commit b0fc00e8c9d21e961ef44b0129103e2a62928b52 Author: Pawel Chojnacki Date: Thu Feb 15 00:12:03 2018 +0100 Add generic query additional metrics commit ffe76e6a9ba196bccff22c4880e0384959ad5e48 Author: Pawel Chojnacki Date: Tue Feb 13 15:40:15 2018 +0100 Cluster Metric yml initial + Remove cluster query + remove cluster_metrics.yml + Prometheus adapter tests --- lib/gitlab/prometheus/additional_metrics_parser.rb | 21 ++++-- .../queries/additional_metrics_deployment_query.rb | 2 +- lib/gitlab/prometheus/queries/base_query.rb | 4 ++ lib/gitlab/prometheus/queries/deployment_query.rb | 7 +- lib/gitlab/prometheus/queries/environment_query.rb | 5 ++ .../prometheus/queries/matched_metric_query.rb | 80 ++++++++++++++++++++++ .../prometheus/queries/matched_metrics_query.rb | 80 ---------------------- .../prometheus/queries/query_additional_metrics.rb | 22 ++++-- lib/gitlab/prometheus_client.rb | 6 +- 9 files changed, 132 insertions(+), 95 deletions(-) create mode 100644 lib/gitlab/prometheus/queries/matched_metric_query.rb delete mode 100644 lib/gitlab/prometheus/queries/matched_metrics_query.rb (limited to 'lib') diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb index cb95daf2260..bb1172f82a1 100644 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -1,10 +1,12 @@ module Gitlab module Prometheus module AdditionalMetricsParser + CONFIG_ROOT = 'config/prometheus'.freeze + MUTEX = Mutex.new extend self - def load_groups_from_yaml - additional_metrics_raw.map(&method(:group_from_entry)) + def load_groups_from_yaml(file_name = 'additional_metrics.yml') + yaml_metrics_raw(file_name).map(&method(:group_from_entry)) end private @@ -22,13 +24,20 @@ module Gitlab MetricGroup.new(entry).tap(&method(:validate!)) end - def additional_metrics_raw - load_yaml_file&.map(&:deep_symbolize_keys).freeze + def yaml_metrics_raw(file_name) + load_yaml_file(file_name)&.map(&:deep_symbolize_keys).freeze end - def load_yaml_file - @loaded_yaml_file ||= YAML.load_file(Rails.root.join('config/prometheus/additional_metrics.yml')) + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def load_yaml_file(file_name) + return YAML.load_file(Rails.root.join(CONFIG_ROOT, file_name)) if Rails.env.development? + + MUTEX.synchronize do + @loaded_yaml_cache ||= {} + @loaded_yaml_cache[file_name] ||= YAML.load_file(Rails.root.join(CONFIG_ROOT, file_name)) + end end + # rubocop:enable Gitlab/ModuleWithInstanceVariables end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 972ab75d1d5..e677ec84cd4 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -4,7 +4,7 @@ module Gitlab class AdditionalMetricsDeploymentQuery < BaseQuery include QueryAdditionalMetrics - def query(environment_id, deployment_id) + def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( deployment.project, diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb index c60828165bd..29cab6e9c15 100644 --- a/lib/gitlab/prometheus/queries/base_query.rb +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -20,6 +20,10 @@ module Gitlab def query(*args) raise NotImplementedError end + + def self.transform_reactive_result(result) + result + end end end end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 6e6da593178..c2626581897 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -2,7 +2,7 @@ module Gitlab module Prometheus module Queries class DeploymentQuery < BaseQuery - def query(environment_id, deployment_id) + def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| environment_slug = deployment.environment.slug @@ -25,6 +25,11 @@ module Gitlab } end end + + def self.transform_reactive_result(result) + result[:metrics] = result.delete :data + result + end end end end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 1d17d3cfd56..b62910c8de6 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -19,6 +19,11 @@ module Gitlab } end end + + def self.transform_reactive_result(result) + result[:metrics] = result.delete :data + result + end end end end diff --git a/lib/gitlab/prometheus/queries/matched_metric_query.rb b/lib/gitlab/prometheus/queries/matched_metric_query.rb new file mode 100644 index 00000000000..d920e9a749f --- /dev/null +++ b/lib/gitlab/prometheus/queries/matched_metric_query.rb @@ -0,0 +1,80 @@ +module Gitlab + module Prometheus + module Queries + class MatchedMetricQuery < BaseQuery + MAX_QUERY_ITEMS = 40.freeze + + def query + groups_data.map do |group, data| + { + group: group.name, + priority: group.priority, + active_metrics: data[:active_metrics], + metrics_missing_requirements: data[:metrics_missing_requirements] + } + end + end + + private + + def groups_data + metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.common_metrics) + lookup = active_series_lookup(metrics_groups) + + groups = {} + + metrics_groups.each do |group| + groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } + active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) } + + groups[group][:active_metrics] += active_metrics + groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics + end + + groups + end + + def active_series_lookup(metric_groups) + timeframe_start = 8.hours.ago + timeframe_end = Time.now + + series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq + + lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| + client_series(*batched_series, start: timeframe_start, stop: timeframe_end) + .select(&method(:has_matching_label?)) + .map { |series_info| [series_info['__name__'], true] } + end + lookup.to_h + end + + def has_matching_label?(series_info) + series_info.key?('environment') + end + + def available_metrics + @available_metrics ||= client_label_values || [] + end + + def filter_active_metrics(metric_group) + metric_group.metrics.select! do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + metric_group + end + + def groups_with_active_metrics(metric_groups) + metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? } + end + + def metrics_with_required_series(metric_groups) + metric_groups.flat_map do |group| + group.metrics.select do |metric| + metric.required_metrics.all?(&available_metrics.method(:include?)) + end + end + end + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metrics_query.rb deleted file mode 100644 index 5710ad47c1a..00000000000 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ /dev/null @@ -1,80 +0,0 @@ -module Gitlab - module Prometheus - module Queries - class MatchedMetricsQuery < BaseQuery - MAX_QUERY_ITEMS = 40.freeze - - def query - groups_data.map do |group, data| - { - group: group.name, - priority: group.priority, - active_metrics: data[:active_metrics], - metrics_missing_requirements: data[:metrics_missing_requirements] - } - end - end - - private - - def groups_data - metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.common_metrics) - lookup = active_series_lookup(metrics_groups) - - groups = {} - - metrics_groups.each do |group| - groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } - active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) } - - groups[group][:active_metrics] += active_metrics - groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics - end - - groups - end - - def active_series_lookup(metric_groups) - timeframe_start = 8.hours.ago - timeframe_end = Time.now - - series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq - - lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| - client_series(*batched_series, start: timeframe_start, stop: timeframe_end) - .select(&method(:has_matching_label?)) - .map { |series_info| [series_info['__name__'], true] } - end - lookup.to_h - end - - def has_matching_label?(series_info) - series_info.key?('environment') - end - - def available_metrics - @available_metrics ||= client_label_values || [] - end - - def filter_active_metrics(metric_group) - metric_group.metrics.select! do |metric| - metric.required_metrics.all?(&available_metrics.method(:include?)) - end - metric_group - end - - def groups_with_active_metrics(metric_groups) - metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? } - end - - def metrics_with_required_series(metric_groups) - metric_groups.flat_map do |group| - group.metrics.select do |metric| - metric.required_metrics.all?(&available_metrics.method(:include?)) - end - end - end - end - end - end -end diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index 0c280dc9a3c..aad76e335af 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -3,9 +3,16 @@ module Gitlab module Queries module QueryAdditionalMetrics def query_metrics(project, query_context) + matched_metrics(project).map(&query_group(query_context)) + .select(&method(:group_with_any_metrics)) + end + + protected + + def query_group(query_context) query_processor = method(:process_query).curry[query_context] - groups = matched_metrics(project).map do |group| + lambda do |group| metrics = group.metrics.map do |metric| { title: metric.title, @@ -21,8 +28,6 @@ module Gitlab metrics: metrics.select(&method(:metric_with_any_queries)) } end - - groups.select(&method(:group_with_any_metrics)) end private @@ -72,12 +77,17 @@ module Gitlab end def common_query_context(environment, timeframe_start:, timeframe_end:) - { - timeframe_start: timeframe_start, - timeframe_end: timeframe_end, + base_query_context(timeframe_start, timeframe_end).merge({ ci_environment_slug: environment.slug, kube_namespace: environment.project.deployment_platform&.actual_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} + }) + end + + def base_query_context(timeframe_start, timeframe_end) + { + timeframe_start: timeframe_start, + timeframe_end: timeframe_end } end end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 659021c9ac9..b66253a10e0 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -57,7 +57,11 @@ module Gitlab rescue OpenSSL::SSL::SSLError raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data" rescue RestClient::ExceptionWithResponse => ex - handle_exception_response(ex.response) + if ex.response + handle_exception_response(ex.response) + else + raise PrometheusClient::Error, "Network connection error" + end rescue RestClient::Exception raise PrometheusClient::Error, "Network connection error" end -- cgit v1.2.3