Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43

6 files changed, 244 insertions, 26 deletions

diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb
index 1dc660e574b..6b185a5a1e9 100644
--- a/lib/gitlab/usage/docs/helper.rb
+++ b/lib/gitlab/usage/docs/helper.rb
@@ -33,6 +33,10 @@ module Gitlab
           object[:description]
         end
 
+        def render_object_schema(object)
+          "[Object JSON schema](#{object.json_schema_path})"
+        end
+
         def render_yaml_link(yaml_path)
           "[YAML definition](#{yaml_path})"
         end
diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml
index 19ad668019e..26f1aa4396d 100644
--- a/lib/gitlab/usage/docs/templates/default.md.haml
+++ b/lib/gitlab/usage/docs/templates/default.md.haml
@@ -27,6 +27,9 @@
   = render_name(name)
   \
   = render_description(object.attributes)
+  - if object.has_json_schema?
+    \
+    = render_object_schema(object)
   \
   = render_yaml_link(object.yaml_path)
   \
diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb
index 4cb83348478..9c4255a7c92 100644
--- a/lib/gitlab/usage/metric_definition.rb
+++ b/lib/gitlab/usage/metric_definition.rb
@@ -5,6 +5,7 @@ module Gitlab
     class MetricDefinition
       METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json')
       BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master'
+      SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze
 
       attr_reader :path
       attr_reader :attributes
@@ -22,6 +23,16 @@ module Gitlab
         attributes
       end
 
+      def json_schema_path
+        return '' unless has_json_schema?
+
+        "#{BASE_REPO_PATH}/#{attributes[:object_json_schema]}"
+      end
+
+      def has_json_schema?
+        attributes[:value_type] == 'object' && attributes[:object_json_schema].present?
+      end
+
       def yaml_path
         "#{BASE_REPO_PATH}#{path.delete_prefix(Rails.root.to_s)}"
       end
@@ -29,7 +40,15 @@ module Gitlab
       def validate!
         unless skip_validation?
           self.class.schemer.validate(attributes.stringify_keys).each do |error|
-            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`"))
+            error_message = <<~ERROR_MSG
+              Error type: #{error['type']}
+              Data: #{error['data']}
+              Path: #{error['data_pointer']}
+              Details: #{error['details']}
+              Metric file: #{path}
+            ERROR_MSG
+
+            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(error_message))
           end
         end
       end
@@ -38,10 +57,11 @@ module Gitlab
 
       class << self
         def paths
-          @paths ||= [Rails.root.join('config', 'metrics', '**', '*.yml')]
+          @paths ||= [Rails.root.join('config', 'metrics', '[^agg]*', '*.yml')]
         end
 
-        def definitions
+        def definitions(skip_validation: false)
+          @skip_validation = skip_validation
           @definitions ||= load_all!
         end
 
@@ -49,6 +69,10 @@ module Gitlab
           @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH))
         end
 
+        def dump_metrics_yaml
+          @metrics_yaml ||= definitions.values.map(&:to_h).map(&:deep_stringify_keys).to_yaml
+        end
+
         private
 
         def load_all!
@@ -87,7 +111,7 @@ module Gitlab
       end
 
       def skip_validation?
-        !!attributes[:skip_validation]
+        !!attributes[:skip_validation] || @skip_validation || SKIP_VALIDATION_STATUSES.include?(attributes[:status])
       end
     end
   end
diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
index 1aeca87d849..f77c8cab39c 100644
--- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb
+++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
@@ -7,7 +7,7 @@ module Gitlab
         UNION_OF_AGGREGATED_METRICS = 'OR'
         INTERSECTION_OF_AGGREGATED_METRICS = 'AND'
         ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze
-        AGGREGATED_METRICS_PATH = Rails.root.join('lib/gitlab/usage_data_counters/aggregated_metrics/*.yml')
+        AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml')
         AggregatedMetricError = Class.new(StandardError)
         UnknownAggregationOperator = Class.new(AggregatedMetricError)
         UnknownAggregationSource = Class.new(AggregatedMetricError)
diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
index 33f025770e0..49581169452 100644
--- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb
+++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
@@ -6,6 +6,8 @@ module Gitlab
       module NamesSuggestions
         class Generator < ::Gitlab::UsageData
           FREE_TEXT_METRIC_NAME = "<please fill metric name>"
+          REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>"
+          CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>"
 
           class << self
             def generate(key_path)
@@ -23,7 +25,7 @@ module Gitlab
             end
 
             def redis_usage_counter
-              FREE_TEXT_METRIC_NAME
+              REDIS_EVENT_METRIC_NAME
             end
 
             def alt_usage_data(*)
@@ -31,7 +33,7 @@ module Gitlab
             end
 
             def redis_usage_data_totals(counter)
-              counter.fallback_totals.transform_values { |_| FREE_TEXT_METRIC_NAME}
+              counter.fallback_totals.transform_values { |_| REDIS_EVENT_METRIC_NAME }
             end
 
             def sum(relation, column, *rest)
@@ -47,49 +49,160 @@ module Gitlab
             end
 
             def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil)
-              parts = [prefix]
+              # rubocop: disable CodeReuse/ActiveRecord
+              relation = relation.unscope(where: :created_at)
+              # rubocop: enable CodeReuse/ActiveRecord
 
-              if column
-                parts << parse_target(column)
+              parts = [prefix]
+              arel_column = arelize_column(relation, column)
+
+              # nil as column indicates that the counting would use fallback value of primary key.
+              # Because counting primary key from relation is the conceptual equal to counting all
+              # records from given relation, in order to keep name suggestion more condensed
+              # primary key column is skipped.
+              # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not
+              # as count_id_from_issues since it does not add more information to the name suggestion
+              if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key]
+                parts << arel_column.name
                 parts << 'from'
               end
 
-              source = parse_source(relation)
-              constraints = parse_constraints(relation: relation, column: column, distinct: distinct)
+              arel = arel_query(relation: relation, column: arel_column, distinct: distinct)
+              constraints = parse_constraints(relation: relation, arel: arel)
+
+              # In some cases due to performance reasons metrics are instrumented with joined relations
+              # where relation listed in FROM statement is not the one that includes counted attribute
+              # in such situations to make name suggestion more intuitive source should be inferred based
+              # on the relation that provide counted attribute
+              # EG: SELECT COUNT(deployments.environment_id) FROM clusters
+              #       JOIN deployments ON deployments.cluster_id = cluster.id
+              # should be translated into:
+              #   count_environment_id_from_deployments_with_clusters
+              # instead of
+              #   count_environment_id_from_clusters_with_deployments
+              actual_source = parse_source(relation, arel_column)
+
+              append_constraints_prompt(actual_source, [constraints], parts)
+
+              parts << actual_source
+              parts += process_joined_relations(actual_source, arel, relation, constraints)
+              parts.compact.join('_').delete('"')
+            end
 
-              if constraints.include?(source)
-                parts << "<adjective describing: '#{constraints}'>"
-              end
+            def append_constraints_prompt(target, constraints, parts)
+              applicable_constraints = constraints.select { |constraint| constraint.include?(target) }
+              return unless applicable_constraints.any?
 
-              parts << source
-              parts.compact.join('_')
+              parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') }
             end
 
-            def parse_constraints(relation:, column: nil, distinct: nil)
+            def parse_constraints(relation:, arel:)
               connection = relation.connection
               ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints
                 .new(connection)
-                .accept(arel(relation: relation, column: column, distinct: distinct), collector(connection))
+                .accept(arel, collector(connection))
                 .value
             end
 
-            def parse_target(column)
-              if column.is_a?(Arel::Attribute)
-                "#{column.relation.name}.#{column.name}"
-              else
+            # TODO: joins with `USING` keyword
+            def process_joined_relations(actual_source, arel, relation, where_constraints)
+              joins = parse_joins(connection: relation.connection, arel: arel)
+              return [] unless joins.any?
+
+              sources = [relation.table_name, *joins.map { |join| join[:source] }]
+              joins = extract_joins_targets(joins, sources)
+
+              relations = if actual_source != relation.table_name
+                            build_relations_tree(joins + [{ source: relation.table_name }], actual_source)
+                          else
+                            # in case where counter attribute comes from joined relations, the relations
+                            # diagram has to be built bottom up, thus source and target are reverted
+                            build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source)
+                          end
+
+              collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints)
+            end
+
+            def parse_joins(connection:, arel:)
+              ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins
+                .new(connection)
+                .accept(arel)
+            end
+
+            def extract_joins_targets(joins, sources)
+              joins.map do |join|
+                source_regex = /(#{join[:source]})\.(\w+_)*id/i
+
+                tables_except_src = (sources - [join[:source]]).join('|')
+                target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i
+
+                join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i
+                matched = join_cond_regex.match(join[:constraints])
+
+                if matched
+                  join[:target] = matched[:target]
+                  join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '')
+                end
+
+                join
+              end
+            end
+
+            def build_relations_tree(joins, parent, source_key: :source, target_key: :target)
+              return [] if joins.blank?
+
+              tree = {}
+              tree[parent] = []
+
+              joins.each do |join|
+                if join[source_key] == parent
+                  tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key)
+                end
+              end
+              tree
+            end
+
+            def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle)
+              conjunction = conjunctions.next
+              relations.each do |subtree|
+                subtree.each do |parent, children|
+                  parts << "<#{conjunction}>"
+                  join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints)
+                  append_constraints_prompt(parent, [wheres, join_constraints].compact, parts)
+                  parts << parent
+                  collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions)
+                end
+              end
+              parts
+            end
+
+            def arelize_column(relation, column)
+              case column
+              when Arel::Attribute
                 column
+              when NilClass
+                Arel::Table.new(relation.table_name)[relation.primary_key]
+              when String
+                if column.include?('.')
+                  table, col = column.split('.')
+                  Arel::Table.new(table)[col]
+                else
+                  Arel::Table.new(relation.table_name)[column]
+                end
+              when Symbol
+                arelize_column(relation, column.to_s)
               end
             end
 
-            def parse_source(relation)
-              relation.table_name
+            def parse_source(relation, column)
+              column.relation.name || relation.table_name
             end
 
             def collector(connection)
               Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new)
             end
 
-            def arel(relation:, column: nil, distinct: nil)
+            def arel_query(relation:, column: nil, distinct: nil)
               column ||= relation.primary_key
 
               if column.is_a?(Arel::Attribute)
diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb
new file mode 100644
index 00000000000..d52e4903f3c
--- /dev/null
+++ b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Usage
+    module Metrics
+      module NamesSuggestions
+        module RelationParsers
+          class Joins < ::Arel::Visitors::PostgreSQL
+            def accept(object)
+              object.source.right.map do |join|
+                visit(join, collector)
+              end
+            end
+
+            private
+
+            # rubocop:disable Naming/MethodName
+            def visit_Arel_Nodes_StringJoin(object, collector)
+              result = visit(object.left, collector)
+              source, constraints = result.value.split('ON')
+              {
+                source: source.split('JOIN').last&.strip,
+                constraints: constraints&.strip
+              }.compact
+            end
+
+            def visit_Arel_Nodes_FullOuterJoin(object, _)
+              parse_join(object)
+            end
+
+            def visit_Arel_Nodes_OuterJoin(object, _)
+              parse_join(object)
+            end
+
+            def visit_Arel_Nodes_RightOuterJoin(object, _)
+              parse_join(object)
+            end
+
+            def visit_Arel_Nodes_InnerJoin(object, _)
+              {
+                source: visit(object.left, collector).value,
+                constraints: object.right ? visit(object.right.expr, collector).value : nil
+              }.compact
+            end
+            # rubocop:enable Naming/MethodName
+
+            def parse_join(object)
+              {
+                source: visit(object.left, collector).value,
+                constraints: visit(object.right.expr, collector).value
+              }
+            end
+
+            def quote(value)
+              "#{value}"
+            end
+
+            def quote_table_name(name)
+              "#{name}"
+            end
+
+            def quote_column_name(name)
+              "#{name}"
+            end
+
+            def collector
+              Arel::Collectors::SubstituteBinds.new(@connection, Arel::Collectors::SQLString.new)
+            end
+          end
+        end
+      end
+    end
+  end
+end