Add latest changes from gitlab-org/gitlab@14-9-stable-eev14.9.0-rc42

6 files changed, 137 insertions, 3 deletions

diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb
index 6e5196ecdbd..1031f38792b 100644
--- a/lib/gitlab/usage/metric_definition.rb
+++ b/lib/gitlab/usage/metric_definition.rb
@@ -80,6 +80,10 @@ module Gitlab
           @all ||= definitions.map { |_key_path, definition| definition }
         end
 
+        def not_removed
+          all.select { |definition| definition.attributes[:status] != 'removed' }.index_by(&:key_path)
+        end
+
         def with_instrumentation_class
           all.select { |definition| definition.attributes[:instrumentation_class].present? && definition.available? }
         end
diff --git a/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb
new file mode 100644
index 00000000000..6df6fef5d3a
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Instrumentations
+        class CertBasedClustersFfMetric < GenericMetric
+          value do
+            Feature.enabled?(:certificate_based_clusters, default_enabled: :yaml, type: :ops)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
index d7fc798ebe2..34a8bfd08b5 100644
--- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
@@ -33,6 +33,12 @@ module Gitlab
               @metric_relation = block
             end
 
+            def metric_options(&block)
+              return @metric_options&.call.to_h unless block_given?
+
+              @metric_options = block
+            end
+
             def operation(symbol, column: nil, &block)
               @metric_operation = symbol
               @column = column
@@ -54,6 +60,7 @@ module Gitlab
                     self.class.column,
                     start: start,
                     finish: finish,
+                    **self.class.metric_options,
                     &self.class.metric_operation_block)
           end
 
diff --git a/lib/gitlab/usage/service_ping/instrumented_payload.rb b/lib/gitlab/usage/service_ping/instrumented_payload.rb
new file mode 100644
index 00000000000..e04e2e589b2
--- /dev/null
+++ b/lib/gitlab/usage/service_ping/instrumented_payload.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Service Ping payload build using the instrumentation classes
+# for given metrics key_paths and output method
+module Gitlab
+  module Usage
+    module ServicePing
+      class InstrumentedPayload
+        attr_reader :metrics_key_paths
+        attr_reader :output_method
+
+        def initialize(metrics_key_paths, output_method)
+          @metrics_key_paths = metrics_key_paths
+          @output_method = output_method
+        end
+
+        def build
+          metrics_key_paths.map do |key_path|
+            compute_instrumental_value(key_path, output_method)
+          end.reduce({}, :deep_merge)
+        end
+
+        private
+
+        # Not all metrics defintions have instrumentation classes
+        # The value can be computed only for those that have it
+        def instrumented_metrics_defintions
+          Gitlab::Usage::MetricDefinition.with_instrumentation_class
+        end
+
+        def compute_instrumental_value(key_path, output_method)
+          definition = instrumented_metrics_defintions.find { |df| df.key_path == key_path }
+
+          return {} unless definition.present?
+
+          Gitlab::Usage::Metric.new(definition).method(output_method).call
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage/service_ping/payload_keys_processor.rb b/lib/gitlab/usage/service_ping/payload_keys_processor.rb
new file mode 100644
index 00000000000..ea2043ffb83
--- /dev/null
+++ b/lib/gitlab/usage/service_ping/payload_keys_processor.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# Process the UsageData payload to get the keys that have a metric defintion
+# Get the missing keys from the payload
+module Gitlab
+  module Usage
+    module ServicePing
+      class PayloadKeysProcessor
+        attr_reader :old_payload
+
+        def initialize(old_payload)
+          @old_payload = old_payload
+        end
+
+        def key_paths
+          @key_paths ||= payload_keys.to_a.flatten.compact
+        end
+
+        def missing_instrumented_metrics_key_paths
+          @missing_key_paths ||= metrics_with_instrumentation.map(&:key) - key_paths
+        end
+
+        private
+
+        def payload_keys(payload = old_payload, parents = [])
+          return unless payload.is_a?(Hash)
+
+          payload.map do |key, value|
+            if has_metric_definition?(key, parents)
+              parents.dup.append(key).join('.')
+            else
+              payload_keys(value, parents.dup << key) if value.is_a?(Hash)
+            end
+          end
+        end
+
+        def has_metric_definition?(key, parent_keys)
+          key_path = parent_keys.dup.append(key).join('.')
+          metric_definitions.key?(key_path)
+        end
+
+        def metric_definitions
+          ::Gitlab::Usage::MetricDefinition.not_removed
+        end
+
+        def metrics_with_instrumentation
+          ::Gitlab::Usage::MetricDefinition.with_instrumentation_class
+        end
+      end
+    end
+  end
+end
+
+Gitlab::Usage::ServicePing::PayloadKeysProcessor.prepend_mod_with('Gitlab::Usage::ServicePing::PayloadKeysProcessor')
diff --git a/lib/gitlab/usage/service_ping_report.rb b/lib/gitlab/usage/service_ping_report.rb
index d9e30c46498..794f3373043 100644
--- a/lib/gitlab/usage/service_ping_report.rb
+++ b/lib/gitlab/usage/service_ping_report.rb
@@ -7,16 +7,29 @@ module Gitlab
         def for(output:, cached: false)
           case output.to_sym
           when :all_metrics_values
-            all_metrics_values(cached)
+            with_instrumentation_classes(all_metrics_values(cached), :with_value)
           when :metrics_queries
-            metrics_queries
+            with_instrumentation_classes(metrics_queries, :with_instrumentation)
           when :non_sql_metrics_values
-            non_sql_metrics_values
+            with_instrumentation_classes(non_sql_metrics_values, :with_instrumentation)
           end
         end
 
         private
 
+        def with_instrumentation_classes(old_payload, output_method)
+          if Feature.enabled?(:merge_service_ping_instrumented_metrics, default_enabled: :yaml)
+
+            instrumented_metrics_key_paths = Gitlab::Usage::ServicePing::PayloadKeysProcessor.new(old_payload).missing_instrumented_metrics_key_paths
+
+            instrumented_payload = Gitlab::Usage::ServicePing::InstrumentedPayload.new(instrumented_metrics_key_paths, output_method).build
+
+            old_payload.deep_merge(instrumented_payload)
+          else
+            old_payload
+          end
+        end
+
         def all_metrics_values(cached)
           Rails.cache.fetch('usage_data', force: !cached, expires_in: 2.weeks) do
             Gitlab::UsageData.data