Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42

17 files changed, 169 insertions, 238 deletions

diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb
deleted file mode 100644
index bfe674b945e..00000000000
--- a/lib/gitlab/usage/docs/helper.rb
+++ /dev/null
@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Usage
-    module Docs
-      # Helper with functions to be used by HAML templates
-      module Helper
-        def auto_generated_comment
-          <<-MARKDOWN.strip_heredoc
-            ---
-            stage: Growth
-            group: Product Intelligence
-            info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers
-            ---
-
-            <!---
-              This documentation is auto generated by a script.
-
-              Please do not edit this file directly, check generate_metrics_dictionary task on lib/tasks/gitlab/usage_data.rake.
-            --->
-          MARKDOWN
-        end
-
-        def render_name(name)
-          "### `#{name}`"
-        end
-
-        def render_description(object)
-          return 'Missing description' unless object[:description].present?
-
-          object[:description]
-        end
-
-        def render_object_schema(object)
-          "[Object JSON schema](#{object.json_schema_path})"
-        end
-
-        def render_yaml_link(yaml_path)
-          "[YAML definition](#{yaml_path})"
-        end
-
-        def render_status(object)
-          "Status: #{format(:status, object[:status])}"
-        end
-
-        def render_owner(object)
-          "Group: `#{object[:product_group]}`"
-        end
-
-        def render_tiers(object)
-          "Tiers:#{format(:tier, object[:tier])}"
-        end
-
-        def render_data_category(object)
-          "Data Category: `#{object[:data_category]}`"
-        end
-
-        def format(key, value)
-          Gitlab::Usage::Docs::ValueFormatter.format(key, value)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/usage/docs/renderer.rb b/lib/gitlab/usage/docs/renderer.rb
deleted file mode 100644
index 7a7c58005bb..00000000000
--- a/lib/gitlab/usage/docs/renderer.rb
+++ /dev/null
@@ -1,32 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Usage
-    module Docs
-      class Renderer
-        include Gitlab::Usage::Docs::Helper
-        DICTIONARY_PATH = Rails.root.join('doc', 'development', 'usage_ping')
-        TEMPLATE_PATH = Rails.root.join('lib', 'gitlab', 'usage', 'docs', 'templates', 'default.md.haml')
-
-        def initialize(metrics_definitions)
-          @layout = Haml::Engine.new(File.read(TEMPLATE_PATH))
-          @metrics_definitions = metrics_definitions.sort
-        end
-
-        def contents
-          # Render and remove an extra trailing new line
-          @contents ||= @layout.render(self, metrics_definitions: @metrics_definitions).sub!(/\n(?=\Z)/, '')
-        end
-
-        def write
-          filename = DICTIONARY_PATH.join('dictionary.md').to_s
-
-          FileUtils.mkdir_p(DICTIONARY_PATH)
-          File.write(filename, contents)
-
-          filename
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml
deleted file mode 100644
index 83a3a5b6698..00000000000
--- a/lib/gitlab/usage/docs/templates/default.md.haml
+++ /dev/null
@@ -1,48 +0,0 @@
-= auto_generated_comment
-
-:plain
-  # Metrics Dictionary
-
-  This file is autogenerated, please do not edit directly.
-
-  To generate these files from the GitLab repository, run:
-
-  ```shell
-  bundle exec rake gitlab:usage_data:generate_metrics_dictionary
-  ```
-
-  The Metrics Dictionary is based on the following metrics definition YAML files:
-
-  - [`config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics)
-  - [`ee/config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/config/metrics)
-
-  Each table includes a `milestone`, which corresponds to the GitLab version when the metric
-  was released.
-
-  <!-- vale off -->
-  <!-- Docs linting disabled after this line. -->
-  <!-- See https://docs.gitlab.com/ee/development/documentation/testing.html#disable-vale-tests -->
-
-  ## Metrics Definitions
-
-\
-- metrics_definitions.each do |name, object|
-
-  = render_name(name)
-  \
-  = render_description(object.attributes)
-  - if object.has_json_schema?
-    \
-    = render_object_schema(object)
-  \
-  = render_yaml_link(object.yaml_path)
-  \
-  = render_owner(object.attributes)
-  - if object.attributes[:data_category].present?
-    \
-    = render_data_category(object.attributes)
-  \
-  = render_status(object.attributes)
-  \
-  = render_tiers(object.attributes)
-  \
diff --git a/lib/gitlab/usage/docs/value_formatter.rb b/lib/gitlab/usage/docs/value_formatter.rb
deleted file mode 100644
index 379e5df4d52..00000000000
--- a/lib/gitlab/usage/docs/value_formatter.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Usage
-    module Docs
-      class ValueFormatter
-        def self.format(key, value)
-          return '' unless value.present?
-
-          case key
-          when :key_path
-            "**`#{value}`**"
-          when :data_source
-            value.to_s.capitalize
-          when :product_group, :product_category, :status
-            "`#{value}`"
-          when :introduced_by_url
-            "[Introduced by](#{value})"
-          when :distribution, :tier
-            Array(value).map { |tier| " `#{tier}`" }.join(',')
-          else
-            value
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb
index f3469209f48..5b1ac189c13 100644
--- a/lib/gitlab/usage/metric.rb
+++ b/lib/gitlab/usage/metric.rb
@@ -3,40 +3,43 @@
 module Gitlab
   module Usage
     class Metric
-      include ActiveModel::Model
+      attr_reader :definition
 
-      InvalidMetricError = Class.new(RuntimeError)
-
-      attr_accessor :key_path, :value
+      def initialize(definition)
+        @definition = definition
+      end
 
-      validates :key_path, presence: true
+      class << self
+        def all
+          @all ||= Gitlab::Usage::MetricDefinition.with_instrumentation_class.map do |definition|
+            self.new(definition)
+          end
+        end
+      end
 
-      def definition
-        self.class.definitions[key_path]
+      def with_value
+        unflatten_key_path(intrumentation_object.value)
       end
 
-      def unflatten_key_path
-        unflatten(key_path.split('.'), value)
+      def with_instrumentation
+        unflatten_key_path(intrumentation_object.instrumentation)
       end
 
-      class << self
-        def definitions
-          @definitions ||= Gitlab::Usage::MetricDefinition.definitions
-        end
+      private
 
-        def dictionary
-          definitions.map { |key, definition| definition.to_dictionary }
-        end
+      def unflatten_key_path(value)
+        ::Gitlab::Usage::Metrics::KeyPathProcessor.process(definition.key_path, value)
       end
 
-      private
+      def instrumentation_class
+        "Gitlab::Usage::Metrics::Instrumentations::#{definition.instrumentation_class}"
+      end
 
-      def unflatten(keys, value)
-        loop do
-          value = { keys.pop.to_sym => value }
-          break if keys.blank?
-        end
-        value
+      def intrumentation_object
+        instrumentation_class.constantize.new(
+          time_frame: definition.time_frame,
+          options: definition.attributes[:options]
+        )
       end
     end
   end
diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb
index ccd2c69e2e7..db0cb4c6326 100644
--- a/lib/gitlab/usage/metric_definition.rb
+++ b/lib/gitlab/usage/metric_definition.rb
@@ -7,6 +7,8 @@ module Gitlab
       BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master'
       SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze
 
+      InvalidError = Class.new(RuntimeError)
+
       attr_reader :path
       attr_reader :attributes
 
@@ -48,11 +50,15 @@ module Gitlab
               Metric file: #{path}
             ERROR_MSG
 
-            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(error_message))
+            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(error_message))
           end
         end
       end
 
+      def category_to_lowercase
+        attributes[:data_category]&.downcase!
+      end
+
       alias_method :to_dictionary, :to_h
 
       class << self
@@ -69,6 +75,10 @@ module Gitlab
           @all ||= definitions.map { |_key_path, definition| definition }
         end
 
+        def with_instrumentation_class
+          all.select { |definition| definition.attributes[:instrumentation_class].present? }
+        end
+
         def schemer
           @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH))
         end
@@ -90,9 +100,9 @@ module Gitlab
           definition = YAML.safe_load(definition)
           definition.deep_symbolize_keys!
 
-          self.new(path, definition).tap(&:validate!)
+          self.new(path, definition).tap(&:validate!).tap(&:category_to_lowercase)
         rescue StandardError => e
-          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(e.message))
+          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(e.message))
         end
 
         def load_all_from_path!(definitions, glob_path)
@@ -100,7 +110,7 @@ module Gitlab
             definition = load_from_file(path)
 
             if previous = definitions[definition.key]
-              Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new("Metric '#{definition.key}' is already defined in '#{previous.path}'"))
+              Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new("Metric '#{definition.key}' is already defined in '#{previous.path}'"))
             end
 
             definitions[definition.key] = definition
@@ -114,6 +124,10 @@ module Gitlab
         attributes[method] || super
       end
 
+      def respond_to_missing?(method, *args)
+        attributes[method].present? || super
+      end
+
       def skip_validation?
         !!attributes[:skip_validation] || @skip_validation || SKIP_VALIDATION_STATUSES.include?(attributes[:status])
       end
diff --git a/lib/gitlab/usage/metrics/aggregates.rb b/lib/gitlab/usage/metrics/aggregates.rb
new file mode 100644
index 00000000000..a32c413dba8
--- /dev/null
+++ b/lib/gitlab/usage/metrics/aggregates.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Aggregates
+        UNION_OF_AGGREGATED_METRICS = 'OR'
+        INTERSECTION_OF_AGGREGATED_METRICS = 'AND'
+        ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze
+        AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml')
+        AggregatedMetricError = Class.new(StandardError)
+        UnknownAggregationOperator = Class.new(AggregatedMetricError)
+        UnknownAggregationSource = Class.new(AggregatedMetricError)
+        DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError)
+
+        DATABASE_SOURCE = 'database'
+        REDIS_SOURCE = 'redis'
+
+        SOURCES = {
+          DATABASE_SOURCE => Sources::PostgresHll,
+          REDIS_SOURCE => Sources::RedisHll
+        }.freeze
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
index 3ec06fba5d1..2545a505984 100644
--- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb
+++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
@@ -4,23 +4,6 @@ module Gitlab
   module Usage
     module Metrics
       module Aggregates
-        UNION_OF_AGGREGATED_METRICS = 'OR'
-        INTERSECTION_OF_AGGREGATED_METRICS = 'AND'
-        ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze
-        AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml')
-        AggregatedMetricError = Class.new(StandardError)
-        UnknownAggregationOperator = Class.new(AggregatedMetricError)
-        UnknownAggregationSource = Class.new(AggregatedMetricError)
-        DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError)
-
-        DATABASE_SOURCE = 'database'
-        REDIS_SOURCE = 'redis'
-
-        SOURCES = {
-          DATABASE_SOURCE => Sources::PostgresHll,
-          REDIS_SOURCE => Sources::RedisHll
-        }.freeze
-
         class Aggregate
           include Gitlab::Usage::TimeFrame
 
diff --git a/lib/gitlab/usage/metrics/aggregates/sources.rb b/lib/gitlab/usage/metrics/aggregates/sources.rb
new file mode 100644
index 00000000000..f782a64f3b5
--- /dev/null
+++ b/lib/gitlab/usage/metrics/aggregates/sources.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Aggregates
+        module Sources
+          UnionNotAvailable = Class.new(AggregatedMetricError)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb
index 009b8e62543..1bdf3a7f9d8 100644
--- a/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb
+++ b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb
@@ -5,8 +5,6 @@ module Gitlab
     module Metrics
       module Aggregates
         module Sources
-          UnionNotAvailable = Class.new(AggregatedMetricError)
-
           class RedisHll
             extend Calculations::Intersection
             def self.calculate_metrics_union(metric_names:, start_date:, end_date:, recorded_at: nil)
diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb
index 7b5bee3f8bd..a264f9484f3 100644
--- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb
@@ -15,6 +15,10 @@ module Gitlab
             @time_frame = time_frame
             @options = options
           end
+
+          def instrumentation
+            value
+          end
         end
       end
     end
diff --git a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb
index dd1f9948815..ee51180973c 100644
--- a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb
@@ -5,8 +5,8 @@ module Gitlab
     module Metrics
       module Instrumentations
         class CollectedDataCategoriesMetric < GenericMetric
-          def value
-            ::ServicePing::PermitDataCategoriesService.new.execute
+          value do
+            ::ServicePing::PermitDataCategoriesService.new.execute.to_a
           end
         end
       end
diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
index 7b3a545185b..d7fc798ebe2 100644
--- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
@@ -33,16 +33,17 @@ module Gitlab
               @metric_relation = block
             end
 
-            def operation(symbol, column: nil)
+            def operation(symbol, column: nil, &block)
               @metric_operation = symbol
               @column = column
+              @metric_operation_block = block if block_given?
             end
 
             def cache_start_and_finish_as(cache_key)
               @cache_key = cache_key
             end
 
-            attr_reader :metric_operation, :metric_relation, :metric_start, :metric_finish, :column, :cache_key
+            attr_reader :metric_operation, :metric_relation, :metric_start, :metric_finish, :metric_operation_block, :column, :cache_key
           end
 
           def value
@@ -52,13 +53,18 @@ module Gitlab
               .call(relation,
                     self.class.column,
                     start: start,
-                    finish: finish)
+                    finish: finish,
+                    &self.class.metric_operation_block)
           end
 
           def to_sql
             Gitlab::Usage::Metrics::Query.for(self.class.metric_operation, relation, self.class.column)
           end
 
+          def instrumentation
+            to_sql
+          end
+
           def suggested_name
             Gitlab::Usage::Metrics::NameSuggestion.for(
               self.class.metric_operation,
diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb
index 1849773e33d..0f4b903b99c 100644
--- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb
@@ -12,27 +12,35 @@ module Gitlab
           #     Gitlab::CurrentSettings.uuid
           #   end
           # end
+          FALLBACK = -1
+
           class << self
-            attr_reader :metric_operation
-            @metric_operation = :alt
+            attr_reader :metric_value
+
+            def fallback(custom_fallback = FALLBACK)
+              return @metric_fallback if defined?(@metric_fallback)
+
+              @metric_fallback = custom_fallback
+            end
 
             def value(&block)
               @metric_value = block
             end
+          end
 
-            attr_reader :metric_value
+          def initialize(time_frame: 'none', options: {})
+            @time_frame = time_frame
+            @options = options
           end
 
           def value
-            alt_usage_data do
+            alt_usage_data(fallback: self.class.fallback) do
               self.class.metric_value.call
             end
           end
 
           def suggested_name
-            Gitlab::Usage::Metrics::NameSuggestion.for(
-              self.class.metric_operation
-            )
+            Gitlab::Usage::Metrics::NameSuggestion.for(:alt)
           end
         end
       end
diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb
index a36e612a1cb..bb27cca1bb9 100644
--- a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb
@@ -12,11 +12,6 @@ module Gitlab
           #   events:
           #     - g_analytics_valuestream
           # end
-          class << self
-            attr_reader :metric_operation
-            @metric_operation = :redis
-          end
-
           def initialize(time_frame:, options: {})
             super
 
@@ -36,9 +31,7 @@ module Gitlab
           end
 
           def suggested_name
-            Gitlab::Usage::Metrics::NameSuggestion.for(
-              self.class.metric_operation
-            )
+            Gitlab::Usage::Metrics::NameSuggestion.for(:redis)
           end
 
           private
diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb
new file mode 100644
index 00000000000..a25bad2436b
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module Instrumentations
+        # Usage example
+        #
+        # In metric YAML definition:
+        #
+        # instrumentation_class: RedisMetric
+        # options:
+        #   event: pushes
+        #   counter_class: SourceCodeCounter
+        #
+        class RedisMetric < BaseMetric
+          def initialize(time_frame:, options: {})
+            super
+
+            raise ArgumentError, "'event' option is required" unless metric_event.present?
+            raise ArgumentError, "'counter class' option is required" unless counter_class.present?
+          end
+
+          def metric_event
+            options[:event]
+          end
+
+          def counter_class_name
+            options[:counter_class]
+          end
+
+          def counter_class
+            "Gitlab::UsageDataCounters::#{counter_class_name}".constantize
+          end
+
+          def value
+            redis_usage_data do
+              counter_class.read(metric_event)
+            end
+          end
+
+          def suggested_name
+            Gitlab::Usage::Metrics::NameSuggestion.for(:redis)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
index a669b43f395..b47dc5689d4 100644
--- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb
+++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
@@ -10,6 +10,12 @@ module Gitlab
               uncached_data.deep_stringify_keys.dig(*key_path.split('.'))
             end
 
+            def add_metric(metric, time_frame: 'none')
+              metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize
+
+              metric_class.new(time_frame: time_frame).suggested_name
+            end
+
             private
 
             def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)