Add latest changes from gitlab-org/gitlab@13-2-stable-ee

16 files changed, 263 insertions, 39 deletions

diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb
index fe1722b1095..7b05ae29b02 100644
--- a/lib/gitlab/metrics/background_transaction.rb
+++ b/lib/gitlab/metrics/background_transaction.rb
@@ -9,7 +9,7 @@ module Gitlab
       end
 
       def labels
-        { controller: @worker_class.name, action: 'perform' }
+        { controller: @worker_class.name, action: 'perform', feature_category: @worker_class.try(:get_feature_category).to_s }
       end
     end
   end
diff --git a/lib/gitlab/metrics/dashboard/errors.rb b/lib/gitlab/metrics/dashboard/errors.rb
index 264ea0488e7..07ddd315bcc 100644
--- a/lib/gitlab/metrics/dashboard/errors.rb
+++ b/lib/gitlab/metrics/dashboard/errors.rb
@@ -20,20 +20,20 @@ module Gitlab
           when DashboardProcessingError
             error(error.message, :unprocessable_entity)
           when NOT_FOUND_ERROR
-            error("#{dashboard_path} could not be found.", :not_found)
+            error(_("%{dashboard_path} could not be found.") % { dashboard_path: dashboard_path }, :not_found)
           when PanelNotFoundError
             error(error.message, :not_found)
           when ::Grafana::Client::Error
             error(error.message, :service_unavailable)
           when MissingIntegrationError
-            error('Proxy support for this API is not available currently', :bad_request)
+            error(_('Proxy support for this API is not available currently'), :bad_request)
           else
             raise error
           end
         end
 
         def panels_not_found!(opts)
-          raise PanelNotFoundError.new("No panels matching properties #{opts}")
+          raise PanelNotFoundError.new(_("No panels matching properties %{opts}") % { opts: opts })
         end
       end
     end
diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb
index d80985e0a0e..5e2d78e10a4 100644
--- a/lib/gitlab/metrics/dashboard/finder.rb
+++ b/lib/gitlab/metrics/dashboard/finder.rb
@@ -7,6 +7,19 @@ module Gitlab
   module Metrics
     module Dashboard
       class Finder
+        # Dashboards that should not be part of the list of all dashboards
+        # displayed on the metrics dashboard page.
+        PREDEFINED_DASHBOARD_EXCLUSION_LIST = [
+          # This dashboard is only useful in the self monitoring project.
+          ::Metrics::Dashboard::SelfMonitoringDashboardService,
+
+          # This dashboard is displayed on the K8s cluster settings health page.
+          ::Metrics::Dashboard::ClusterDashboardService,
+
+          # This dashboard is not yet ready for the world.
+          ::Metrics::Dashboard::PodDashboardService
+        ].freeze
+
         class << self
           # Returns a formatted dashboard packed with DB info.
           # @param project [Project]
@@ -67,12 +80,32 @@ module Gitlab
           def find_all_paths_from_source(project)
             Gitlab::Metrics::Dashboard::Cache.delete_all!
 
-            default_dashboard_path(project)
-            .+ project_service.all_dashboard_paths(project)
+            user_facing_dashboard_services(project).flat_map do |service|
+              service.all_dashboard_paths(project)
+            end
           end
 
           private
 
+          def user_facing_dashboard_services(project)
+            predefined_dashboard_services_for(project) + [project_service]
+          end
+
+          def predefined_dashboard_services_for(project)
+            # Only list the self monitoring dashboard on the self monitoring project,
+            # since it is the only dashboard (at time of writing) that shows data
+            # about GitLab itself.
+            if project.self_monitoring?
+              return [self_monitoring_service]
+            end
+
+            predefined_dashboard_services
+          end
+
+          def predefined_dashboard_services
+            ::Metrics::Dashboard::PredefinedDashboardService.descendants - PREDEFINED_DASHBOARD_EXCLUSION_LIST
+          end
+
           def system_service
             ::Metrics::Dashboard::SystemDashboardService
           end
@@ -85,14 +118,6 @@ module Gitlab
             ::Metrics::Dashboard::SelfMonitoringDashboardService
           end
 
-          def default_dashboard_path(project)
-            if project.self_monitoring?
-              self_monitoring_service.all_dashboard_paths(project)
-            else
-              system_service.all_dashboard_paths(project)
-            end
-          end
-
           def service_for(options)
             Gitlab::Metrics::Dashboard::ServiceSelector.call(options)
           end
diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb
index 49682da320c..641c0c76f8f 100644
--- a/lib/gitlab/metrics/dashboard/service_selector.rb
+++ b/lib/gitlab/metrics/dashboard/service_selector.rb
@@ -13,6 +13,8 @@ module Gitlab
           include Gitlab::Utils::StrongMemoize
 
           SERVICES = [
+            ::Metrics::Dashboard::ClusterMetricsEmbedService,
+            ::Metrics::Dashboard::ClusterDashboardService,
             ::Metrics::Dashboard::GitlabAlertEmbedService,
             ::Metrics::Dashboard::CustomMetricEmbedService,
             ::Metrics::Dashboard::GrafanaMetricEmbedService,
@@ -51,5 +53,3 @@ module Gitlab
     end
   end
 end
-
-Gitlab::Metrics::Dashboard::ServiceSelector.prepend_if_ee('EE::Gitlab::Metrics::Dashboard::ServiceSelector')
diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb
index 622d5aa8cdb..ee2d36621b4 100644
--- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb
+++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb
@@ -48,6 +48,14 @@ module Gitlab
             end
           end
 
+          def for_variables
+            return unless dashboard.dig(:templating, :variables).is_a?(Hash)
+
+            dashboard.dig(:templating, :variables).each do |variable_name, variable|
+              yield variable_name, variable
+            end
+          end
+
           def for_panel_groups
             dashboard[:panel_groups].each do |panel_group|
               yield panel_group
diff --git a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb
new file mode 100644
index 00000000000..a12082b704c
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        class ClusterEndpointInserter < BaseStage
+          def transform!
+            verify_params
+
+            for_metrics do |metric|
+              metric[:prometheus_endpoint_path] = endpoint_for_metric(metric)
+            end
+          end
+
+          private
+
+          def admin_url(metric)
+            Gitlab::Routing.url_helpers.prometheus_api_admin_cluster_path(
+              params[:cluster],
+              proxy_path: query_type(metric),
+              query: query_for_metric(metric)
+            )
+          end
+
+          def endpoint_for_metric(metric)
+            case params[:cluster_type]
+            when :admin
+              admin_url(metric)
+            when :group
+              error!(_('Group is required when cluster_type is :group')) unless params[:group]
+              group_url(metric)
+            when :project
+              error!(_('Project is required when cluster_type is :project')) unless project
+              project_url(metric)
+            else
+              error!(_('Unrecognized cluster type'))
+            end
+          end
+
+          def error!(message)
+            raise Errors::DashboardProcessingError.new(message)
+          end
+
+          def group_url(metric)
+            Gitlab::Routing.url_helpers.prometheus_api_group_cluster_path(
+              params[:group],
+              params[:cluster],
+              proxy_path: query_type(metric),
+              query: query_for_metric(metric)
+            )
+          end
+
+          def project_url(metric)
+            Gitlab::Routing.url_helpers.prometheus_api_project_cluster_path(
+              project,
+              params[:cluster],
+              proxy_path: query_type(metric),
+              query: query_for_metric(metric)
+            )
+          end
+
+          def query_type(metric)
+            metric[:query] ? :query : :query_range
+          end
+
+          def query_for_metric(metric)
+            query = metric[query_type(metric)]
+
+            raise Errors::MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query
+
+            query
+          end
+
+          def verify_params
+            raise Errors::DashboardProcessingError.new(_('Cluster is required for Stages::ClusterEndpointInserter')) unless params[:cluster]
+            raise Errors::DashboardProcessingError.new(_('Cluster type must be specificed for Stages::ClusterEndpointInserter')) unless params[:cluster_type]
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb
index e085f551952..c48a7ff25a5 100644
--- a/lib/gitlab/metrics/dashboard/stages/endpoint_inserter.rb
+++ b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb
@@ -4,9 +4,9 @@ module Gitlab
   module Metrics
     module Dashboard
       module Stages
-        class EndpointInserter < BaseStage
+        class MetricEndpointInserter < BaseStage
           def transform!
-            raise Errors::DashboardProcessingError.new('Environment is required for Stages::EndpointInserter') unless params[:environment]
+            raise Errors::DashboardProcessingError.new(_('Environment is required for Stages::MetricEndpointInserter')) unless params[:environment]
 
             for_metrics do |metric|
               metric[:prometheus_endpoint_path] = endpoint_for_metric(metric)
@@ -33,7 +33,11 @@ module Gitlab
           end
 
           def query_type(metric)
-            metric[:query] ? :query : :query_range
+            if metric[:query]
+              ::Prometheus::ProxyService::PROMETHEUS_QUERY_API.to_sym
+            else
+              ::Prometheus::ProxyService::PROMETHEUS_QUERY_RANGE_API.to_sym
+            end
           end
 
           def query_for_metric(metric)
diff --git a/lib/gitlab/metrics/dashboard/stages/sorter.rb b/lib/gitlab/metrics/dashboard/stages/sorter.rb
index ba5aa78059c..882211e1441 100644
--- a/lib/gitlab/metrics/dashboard/stages/sorter.rb
+++ b/lib/gitlab/metrics/dashboard/stages/sorter.rb
@@ -16,7 +16,7 @@ module Gitlab
 
           # Sorts the groups in the dashboard by the :priority key
           def sort_groups!
-            dashboard[:panel_groups] = dashboard[:panel_groups].sort_by { |group| -group[:priority].to_i }
+            dashboard[:panel_groups] = Gitlab::Utils.stable_sort_by(dashboard[:panel_groups]) { |group| -group[:priority].to_i }
           end
 
           # Sorts the panels in the dashboard by the :weight key
@@ -24,7 +24,7 @@ module Gitlab
             dashboard[:panel_groups].each do |group|
               missing_panels! unless group[:panels].is_a? Array
 
-              group[:panels] = group[:panels].sort_by { |panel| -panel[:weight].to_i }
+              group[:panels] = Gitlab::Utils.stable_sort_by(group[:panels]) { |panel| -panel[:weight].to_i }
             end
           end
         end
diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb
index ff36f7b605e..9e2bb0d1a70 100644
--- a/lib/gitlab/metrics/dashboard/stages/url_validator.rb
+++ b/lib/gitlab/metrics/dashboard/stages/url_validator.rb
@@ -6,8 +6,47 @@ module Gitlab
       module Stages
         class UrlValidator < BaseStage
           def transform!
-            dashboard[:links]&.each do |link|
-              Gitlab::UrlBlocker.validate!(link[:url])
+            validate_dashboard_links(dashboard)
+
+            validate_chart_links(dashboard)
+          end
+
+          private
+
+          def blocker_args
+            {
+              schemes: %w(http https),
+              ports: [],
+              allow_localhost: allow_setting_local_requests?,
+              allow_local_network: allow_setting_local_requests?,
+              ascii_only: false,
+              enforce_user: false,
+              enforce_sanitization: false,
+              dns_rebind_protection: true
+            }
+          end
+
+          def allow_setting_local_requests?
+            Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?
+          end
+
+          def validate_dashboard_links(dashboard)
+            validate_links(dashboard[:links])
+          end
+
+          def validate_chart_links(dashboard)
+            dashboard[:panel_groups].each do |panel_group|
+              panel_group[:panels].each do |panel|
+                validate_links(panel[:links])
+              end
+            end
+          end
+
+          def validate_links(links)
+            links&.each do |link|
+              next unless link.is_a? Hash
+
+              Gitlab::UrlBlocker.validate!(link[:url], blocker_args)
             rescue Gitlab::UrlBlocker::BlockedUrlError
               link[:url] = ''
             end
diff --git a/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb
new file mode 100644
index 00000000000..20e7fe477e5
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/stages/variable_endpoint_inserter.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Stages
+        class VariableEndpointInserter < BaseStage
+          VARIABLE_TYPE_METRIC_LABEL_VALUES = 'metric_label_values'
+
+          def transform!
+            raise Errors::DashboardProcessingError.new(_('Environment is required for Stages::VariableEndpointInserter')) unless params[:environment]
+
+            for_variables do |variable_name, variable|
+              if variable.is_a?(Hash) && variable[:type] == VARIABLE_TYPE_METRIC_LABEL_VALUES
+                variable[:options][:prometheus_endpoint_path] = endpoint_for_variable(variable.dig(:options, :series_selector))
+              end
+            end
+          end
+
+          private
+
+          def endpoint_for_variable(series_selector)
+            Gitlab::Routing.url_helpers.prometheus_api_project_environment_path(
+              project,
+              params[:environment],
+              proxy_path: ::Prometheus::ProxyService::PROMETHEUS_SERIES_API,
+              match: Array(series_selector)
+            )
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb
index 31670a3f533..10a2f3c2397 100644
--- a/lib/gitlab/metrics/dashboard/url.rb
+++ b/lib/gitlab/metrics/dashboard/url.rb
@@ -60,6 +60,22 @@ module Gitlab
             Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args)
           end
 
+          # Matches dashboard urls for a metric chart embed
+          # for cluster metrics
+          #
+          # EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB)
+          def clusters_regex
+            strong_memoize(:clusters_regex) do
+              regex_for_project_metrics(
+                %r{
+                  /clusters
+                  /(?<cluster_id>\d+)
+                  /?
+                }x
+              )
+            end
+          end
+
           private
 
           def regex_for_project_metrics(path_suffix_pattern)
diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb
index 5955987541c..83a7b925392 100644
--- a/lib/gitlab/metrics/methods.rb
+++ b/lib/gitlab/metrics/methods.rb
@@ -35,7 +35,7 @@ module Gitlab
         end
 
         def init_metric(type, name, opts = {}, &block)
-          options = MetricOptions.new(opts)
+          options = ::Gitlab::Metrics::Methods::MetricOptions.new(opts)
           options.evaluate(&block)
 
           if disabled_by_feature(options)
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index de8e1ca3256..1c99e1e730c 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -26,9 +26,7 @@ module Gitlab
       private
 
       def add_info_to_payload(payload, trans)
-        payload[:db_count] = trans.get(:db_count, :counter).to_i
-        payload[:db_write_count] = trans.get(:db_write_count, :counter).to_i
-        payload[:db_cached_count] = trans.get(:db_cached_count, :counter).to_i
+        payload.merge!(::Gitlab::Metrics::Subscribers::ActiveRecord.db_counter_payload)
       end
     end
   end
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index 1628eeb5a95..d2736882432 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -23,6 +23,14 @@ module Gitlab
           increment_db_counters(payload)
         end
 
+        def self.db_counter_payload
+          return {} unless Gitlab::SafeRequestStore.active?
+
+          DB_COUNTERS.map do |counter|
+            [counter, Gitlab::SafeRequestStore[counter].to_i]
+          end.to_h
+        end
+
         private
 
         define_histogram :gitlab_sql_duration_seconds do
@@ -36,13 +44,21 @@ module Gitlab
         end
 
         def increment_db_counters(payload)
-          current_transaction.increment(:db_count, 1)
+          increment(:db_count)
 
           if payload.fetch(:cached, payload[:name] == 'CACHE')
-            current_transaction.increment(:db_cached_count, 1)
+            increment(:db_cached_count)
           end
 
-          current_transaction.increment(:db_write_count, 1) unless select_sql_command?(payload)
+          increment(:db_write_count) unless select_sql_command?(payload)
+        end
+
+        def increment(counter)
+          current_transaction.increment(counter, 1)
+
+          if Gitlab::SafeRequestStore.active?
+            Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1
+          end
         end
 
         def current_transaction
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 822f5243e9d..da06be9c79c 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -7,7 +7,7 @@ module Gitlab
       include Gitlab::Metrics::Methods
 
       # base labels shared among all transactions
-      BASE_LABELS = { controller: nil, action: nil }.freeze
+      BASE_LABELS = { controller: nil, action: nil, feature_category: nil }.freeze
       # labels that potentially contain sensitive information and will be filtered
       FILTERED_LABELS = [:branch, :path].freeze
 
@@ -92,12 +92,6 @@ module Gitlab
         self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus
       end
 
-      def get(name, type, tags = {})
-        metric = self.class.transaction_metric(name, type)
-
-        metric.get(filter_tags(tags).merge(labels))
-      end
-
       def labels
         BASE_LABELS
       end
diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb
index fa17548723e..2064f9290d3 100644
--- a/lib/gitlab/metrics/web_transaction.rb
+++ b/lib/gitlab/metrics/web_transaction.rb
@@ -32,6 +32,10 @@ module Gitlab
 
         action = "#{controller.action_name}"
 
+        # Try to get the feature category, but don't fail when the controller is
+        # not an ApplicationController.
+        feature_category = controller.class.try(:feature_category_for_action, action).to_s
+
         # Devise exposes a method called "request_format" that does the below.
         # However, this method is not available to all controllers (e.g. certain
         # Doorkeeper controllers). As such we use the underlying code directly.
@@ -45,7 +49,7 @@ module Gitlab
           action = "#{action}.#{suffix}"
         end
 
-        { controller: controller.class.name, action: action }
+        { controller: controller.class.name, action: action, feature_category: feature_category }
       end
 
       def labels_from_endpoint
@@ -61,7 +65,10 @@ module Gitlab
 
         if route
           path = endpoint_paths_cache[route.request_method][route.path]
-          { controller: 'Grape', action: "#{route.request_method} #{path}" }
+
+          # Feature categories will be added for grape endpoints in
+          # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/462
+          { controller: 'Grape', action: "#{route.request_method} #{path}", feature_category: '' }
         end
       end