Add latest changes from gitlab-org/gitlab@13-10-stable-eev13.10.0-rc40

8 files changed, 204 insertions, 39 deletions

diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb
index 8483334800b..1dc660e574b 100644
--- a/lib/gitlab/usage/docs/helper.rb
+++ b/lib/gitlab/usage/docs/helper.rb
@@ -5,9 +5,6 @@ module Gitlab
     module Docs
       # Helper with functions to be used by HAML templates
       module Helper
-        HEADER = %w(field value).freeze
-        SKIP_KEYS = %i(description).freeze
-
         def auto_generated_comment
           <<-MARKDOWN.strip_heredoc
             ---
@@ -27,35 +24,33 @@ module Gitlab
         end
 
         def render_name(name)
-          "## `#{name}`\n"
+          "### `#{name}`"
         end
 
         def render_description(object)
-          object.description
+          return 'Missing description' unless object[:description].present?
+
+          object[:description]
         end
 
-        def render_attribute_row(key, value)
-          value = Gitlab::Usage::Docs::ValueFormatter.format(key, value)
-          table_row(["`#{key}`", value])
+        def render_yaml_link(yaml_path)
+          "[YAML definition](#{yaml_path})"
         end
 
-        def render_attributes_table(object)
-          <<~MARKDOWN
+        def render_status(object)
+          "Status: #{format(:status, object[:status])}"
+        end
 
-            #{table_row(HEADER)}
-            #{table_row(HEADER.map { '---' })}
-            #{table_value_rows(object.attributes)}
-          MARKDOWN
+        def render_owner(object)
+          "Group: `#{object[:product_group]}`"
         end
 
-        def table_value_rows(attributes)
-          attributes.reject { |k, _| k.in?(SKIP_KEYS) }.map do |key, value|
-            render_attribute_row(key, value)
-          end.join("\n")
+        def render_tiers(object)
+          "Tiers:#{format(:tier, object[:tier])}"
         end
 
-        def table_row(array)
-          "| #{array.join(' | ')} |"
+        def format(key, value)
+          Gitlab::Usage::Docs::ValueFormatter.format(key, value)
         end
       end
     end
diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml
index 86e93be66c7..19ad668019e 100644
--- a/lib/gitlab/usage/docs/templates/default.md.haml
+++ b/lib/gitlab/usage/docs/templates/default.md.haml
@@ -13,16 +13,26 @@
 
   The Metrics Dictionary is based on the following metrics definition YAML files:
 
-  - [`config/metrics`]('https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics')
+  - [`config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics)
   - [`ee/config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/config/metrics)
 
-Each table includes a `milestone`, which corresponds to the GitLab version when the metric
-was released.
+  Each table includes a `milestone`, which corresponds to the GitLab version when the metric
+  was released.
+
+  ## Metrics Definitions
+
 \
 - metrics_definitions.each do |name, object|
 
   = render_name(name)
-
-  = render_description(object)
-
-  = render_attributes_table(object)
+  \
+  = render_description(object.attributes)
+  \
+  = render_yaml_link(object.yaml_path)
+  \
+  = render_owner(object.attributes)
+  \
+  = render_status(object.attributes)
+  \
+  = render_tiers(object.attributes)
+  \
diff --git a/lib/gitlab/usage/docs/value_formatter.rb b/lib/gitlab/usage/docs/value_formatter.rb
index a2dc9b081f8..379e5df4d52 100644
--- a/lib/gitlab/usage/docs/value_formatter.rb
+++ b/lib/gitlab/usage/docs/value_formatter.rb
@@ -5,17 +5,19 @@ module Gitlab
     module Docs
       class ValueFormatter
         def self.format(key, value)
+          return '' unless value.present?
+
           case key
           when :key_path
             "**`#{value}`**"
           when :data_source
             value.to_s.capitalize
-          when :product_group
+          when :product_group, :product_category, :status
             "`#{value}`"
           when :introduced_by_url
             "[Introduced by](#{value})"
           when :distribution, :tier
-            Array(value).join(', ')
+            Array(value).map { |tier| " `#{tier}`" }.join(',')
           else
             value
           end
diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb
index 01d202e4d45..4cb83348478 100644
--- a/lib/gitlab/usage/metric_definition.rb
+++ b/lib/gitlab/usage/metric_definition.rb
@@ -4,6 +4,7 @@ module Gitlab
   module Usage
     class MetricDefinition
       METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json')
+      BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master'
 
       attr_reader :path
       attr_reader :attributes
@@ -21,10 +22,14 @@ module Gitlab
         attributes
       end
 
+      def yaml_path
+        "#{BASE_REPO_PATH}#{path.delete_prefix(Rails.root.to_s)}"
+      end
+
       def validate!
         unless skip_validation?
           self.class.schemer.validate(attributes.stringify_keys).each do |error|
-            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`"))
+            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`"))
           end
         end
       end
@@ -59,7 +64,7 @@ module Gitlab
 
           self.new(path, definition).tap(&:validate!)
         rescue => e
-          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new(e.message))
+          Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(e.message))
         end
 
         def load_all_from_path!(definitions, glob_path)
@@ -67,7 +72,7 @@ module Gitlab
             definition = load_from_file(path)
 
             if previous = definitions[definition.key]
-              Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("Metric '#{definition.key}' is already defined in '#{previous.path}'"))
+              Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new("Metric '#{definition.key}' is already defined in '#{previous.path}'"))
             end
 
             definitions[definition.key] = definition
diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
index 1fc40798320..1aeca87d849 100644
--- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb
+++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
@@ -11,6 +11,7 @@ module Gitlab
         AggregatedMetricError = Class.new(StandardError)
         UnknownAggregationOperator = Class.new(AggregatedMetricError)
         UnknownAggregationSource = Class.new(AggregatedMetricError)
+        DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError)
 
         DATABASE_SOURCE = 'database'
         REDIS_SOURCE = 'redis'
@@ -30,25 +31,38 @@ module Gitlab
             @recorded_at = recorded_at
           end
 
+          def all_time_data
+            aggregated_metrics_data(start_date: nil, end_date: nil, time_frame: Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME)
+          end
+
           def monthly_data
-            aggregated_metrics_data(**monthly_time_range)
+            aggregated_metrics_data(**monthly_time_range.merge(time_frame: Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME))
           end
 
           def weekly_data
-            aggregated_metrics_data(**weekly_time_range)
+            aggregated_metrics_data(**weekly_time_range.merge(time_frame: Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME))
           end
 
           private
 
           attr_accessor :aggregated_metrics, :recorded_at
 
-          def aggregated_metrics_data(start_date:, end_date:)
+          def aggregated_metrics_data(start_date:, end_date:, time_frame:)
             aggregated_metrics.each_with_object({}) do |aggregation, data|
               next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: :yaml, type: :development)
+              next unless aggregation[:time_frame].include?(time_frame)
 
               case aggregation[:source]
               when REDIS_SOURCE
-                data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date)
+                if time_frame == Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME
+                  data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK
+                  Gitlab::ErrorTracking
+                    .track_and_raise_for_dev_exception(
+                      DisallowedAggregationTimeFrame.new("Aggregation time frame: 'all' is not allowed for aggregation with source: '#{REDIS_SOURCE}'")
+                    )
+                else
+                  data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date)
+                end
               when DATABASE_SOURCE
                 next unless Feature.enabled?('database_sourced_aggregated_metrics', default_enabled: false, type: :development)
 
@@ -155,3 +169,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::Usage::Metrics::Aggregates::Aggregate.prepend_if_ee('EE::Gitlab::Usage::Metrics::Aggregates::Aggregate')
diff --git a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb
index 33678d2b813..a01efbdb1a6 100644
--- a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb
+++ b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb
@@ -55,15 +55,15 @@ module Gitlab
               end
 
               def time_period_to_human_name(time_period)
-                return Gitlab::Utils::UsageData::ALL_TIME_PERIOD_HUMAN_NAME if time_period.blank?
+                return Gitlab::Utils::UsageData::ALL_TIME_TIME_FRAME_NAME if time_period.blank?
 
                 start_date = time_period.first.to_date
                 end_date = time_period.last.to_date
 
                 if (end_date - start_date).to_i > 7
-                  Gitlab::Utils::UsageData::MONTHLY_PERIOD_HUMAN_NAME
+                  Gitlab::Utils::UsageData::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME
                 else
-                  Gitlab::Utils::UsageData::WEEKLY_PERIOD_HUMAN_NAME
+                  Gitlab::Utils::UsageData::SEVEN_DAYS_TIME_FRAME_NAME
                 end
               end
             end
diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
new file mode 100644
index 00000000000..33f025770e0
--- /dev/null
+++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module NamesSuggestions
+        class Generator < ::Gitlab::UsageData
+          FREE_TEXT_METRIC_NAME = "<please fill metric name>"
+
+          class << self
+            def generate(key_path)
+              uncached_data.deep_stringify_keys.dig(*key_path.split('.'))
+            end
+
+            private
+
+            def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
+              name_suggestion(column: column, relation: relation, prefix: 'count')
+            end
+
+            def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
+              name_suggestion(column: column, relation: relation, prefix: 'count_distinct', distinct: :distinct)
+            end
+
+            def redis_usage_counter
+              FREE_TEXT_METRIC_NAME
+            end
+
+            def alt_usage_data(*)
+              FREE_TEXT_METRIC_NAME
+            end
+
+            def redis_usage_data_totals(counter)
+              counter.fallback_totals.transform_values { |_| FREE_TEXT_METRIC_NAME}
+            end
+
+            def sum(relation, column, *rest)
+              name_suggestion(column: column, relation: relation, prefix: 'sum')
+            end
+
+            def estimate_batch_distinct_count(relation, column = nil, *rest)
+              name_suggestion(column: column, relation: relation, prefix: 'estimate_distinct_count')
+            end
+
+            def add(*args)
+              "add_#{args.join('_and_')}"
+            end
+
+            def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil)
+              parts = [prefix]
+
+              if column
+                parts << parse_target(column)
+                parts << 'from'
+              end
+
+              source = parse_source(relation)
+              constraints = parse_constraints(relation: relation, column: column, distinct: distinct)
+
+              if constraints.include?(source)
+                parts << "<adjective describing: '#{constraints}'>"
+              end
+
+              parts << source
+              parts.compact.join('_')
+            end
+
+            def parse_constraints(relation:, column: nil, distinct: nil)
+              connection = relation.connection
+              ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints
+                .new(connection)
+                .accept(arel(relation: relation, column: column, distinct: distinct), collector(connection))
+                .value
+            end
+
+            def parse_target(column)
+              if column.is_a?(Arel::Attribute)
+                "#{column.relation.name}.#{column.name}"
+              else
+                column
+              end
+            end
+
+            def parse_source(relation)
+              relation.table_name
+            end
+
+            def collector(connection)
+              Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new)
+            end
+
+            def arel(relation:, column: nil, distinct: nil)
+              column ||= relation.primary_key
+
+              if column.is_a?(Arel::Attribute)
+                relation.select(column.count(distinct)).arel
+              else
+                relation.select(relation.all.table[column].count(distinct)).arel
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/constraints.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/constraints.rb
new file mode 100644
index 00000000000..199395e4b20
--- /dev/null
+++ b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/constraints.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module NamesSuggestions
+        module RelationParsers
+          class Constraints < ::Arel::Visitors::PostgreSQL
+            # rubocop:disable Naming/MethodName
+            def visit_Arel_Nodes_SelectCore(object, collector)
+              collect_nodes_for(object.wheres, collector, "") || collector
+            end
+            # rubocop:enable Naming/MethodName
+
+            def quote(value)
+              "#{value}"
+            end
+
+            def quote_table_name(name)
+              "#{name}"
+            end
+
+            def quote_column_name(name)
+              "#{name}"
+            end
+          end
+        end
+      end
+    end
+  end
+end