Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42

10 files changed, 199 insertions, 166 deletions

diff --git a/spec/lib/gitlab/usage/docs/helper_spec.rb b/spec/lib/gitlab/usage/docs/helper_spec.rb
deleted file mode 100644
index e2bb1d8d818..00000000000
--- a/spec/lib/gitlab/usage/docs/helper_spec.rb
+++ /dev/null
@@ -1,79 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::Usage::Docs::Helper do
-  subject(:helper) { klass.new }
-
-  let_it_be(:klass) do
-    Class.new do
-      include Gitlab::Usage::Docs::Helper
-    end
-  end
-
-  let(:metric_definition) do
-    {
-      data_category: 'Standard',
-      name: 'test_metric',
-      description: description,
-      product_group: 'group::product intelligence',
-      status: 'data_available',
-      tier: %w(free premium)
-    }
-  end
-
-  let(:description) { 'Metric description' }
-
-  describe '#render_name' do
-    it { expect(helper.render_name(metric_definition[:name])).to eq('### `test_metric`') }
-  end
-
-  describe '#render_description' do
-    context 'without description' do
-      let(:description) { nil }
-
-      it { expect(helper.render_description(metric_definition)).to eq('Missing description') }
-    end
-
-    context 'without description' do
-      it { expect(helper.render_description(metric_definition)).to eq('Metric description') }
-    end
-  end
-
-  describe '#render_yaml_link' do
-    let(:yaml_link) { 'config/metrics/license/test_metric.yml' }
-    let(:expected) { "[YAML definition](#{yaml_link})" }
-
-    it { expect(helper.render_yaml_link(yaml_link)).to eq(expected) }
-  end
-
-  describe '#render_status' do
-    let(:expected) { "Status: `data_available`" }
-
-    it { expect(helper.render_status(metric_definition)).to eq(expected) }
-  end
-
-  describe '#render_owner' do
-    let(:expected) { "Group: `group::product intelligence`" }
-
-    it { expect(helper.render_owner(metric_definition)).to eq(expected) }
-  end
-
-  describe '#render_tiers' do
-    let(:expected) { "Tiers: `free`, `premium`" }
-
-    it { expect(helper.render_tiers(metric_definition)).to eq(expected) }
-  end
-
-  describe '#render_data_category' do
-    let(:expected) { 'Data Category: `Standard`' }
-
-    it { expect(helper.render_data_category(metric_definition)).to eq(expected) }
-  end
-
-  describe '#render_owner' do
-    let(:expected) { "Group: `group::product intelligence`" }
-
-    it { expect(helper.render_owner(metric_definition)).to eq(expected) }
-  end
-end
diff --git a/spec/lib/gitlab/usage/docs/renderer_spec.rb b/spec/lib/gitlab/usage/docs/renderer_spec.rb
deleted file mode 100644
index f3b83a4a4b3..00000000000
--- a/spec/lib/gitlab/usage/docs/renderer_spec.rb
+++ /dev/null
@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-CODE_REGEX = %r{<code>(.*)</code>}.freeze
-
-RSpec.describe Gitlab::Usage::Docs::Renderer do
-  describe 'contents' do
-    let(:dictionary_path) { Gitlab::Usage::Docs::Renderer::DICTIONARY_PATH }
-    let(:items) { Gitlab::Usage::MetricDefinition.definitions.first(10).to_h }
-
-    it 'generates dictionary for given items' do
-      generated_dictionary = described_class.new(items).contents
-
-      generated_dictionary_keys = RDoc::Markdown
-        .parse(generated_dictionary)
-        .table_of_contents
-        .select { |metric_doc| metric_doc.level == 3 }
-        .map { |item| item.text.match(CODE_REGEX)&.captures&.first }
-
-      expect(generated_dictionary_keys).to match_array(items.keys)
-    end
-  end
-end
diff --git a/spec/lib/gitlab/usage/docs/value_formatter_spec.rb b/spec/lib/gitlab/usage/docs/value_formatter_spec.rb
deleted file mode 100644
index f21656df894..00000000000
--- a/spec/lib/gitlab/usage/docs/value_formatter_spec.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::Usage::Docs::ValueFormatter do
-  describe '.format' do
-    using RSpec::Parameterized::TableSyntax
-    where(:key, :value, :expected_value) do
-      :product_group     | 'growth::product intelligence' | '`growth::product intelligence`'
-      :data_source       | 'redis'                        | 'Redis'
-      :data_source       | 'ruby'                         | 'Ruby'
-      :introduced_by_url | 'http://test.com'              | '[Introduced by](http://test.com)'
-      :tier              | %w(gold premium)               | ' `gold`, `premium`'
-      :distribution      | %w(ce ee)                      | ' `ce`, `ee`'
-      :key_path          | 'key.path'                     | '**`key.path`**'
-      :milestone         | '13.4'                         | '13.4'
-      :status            | 'data_available'               | '`data_available`'
-    end
-
-    with_them do
-      subject { described_class.format(key, value) }
-
-      it { is_expected.to eq(expected_value) }
-    end
-  end
-end
diff --git a/spec/lib/gitlab/usage/metric_definition_spec.rb b/spec/lib/gitlab/usage/metric_definition_spec.rb
index f3c3e5fc550..1ae8a0881ef 100644
--- a/spec/lib/gitlab/usage/metric_definition_spec.rb
+++ b/spec/lib/gitlab/usage/metric_definition_spec.rb
@@ -18,7 +18,7 @@ RSpec.describe Gitlab::Usage::MetricDefinition do
       distribution: %w(ee ce),
       tier: %w(free starter premium ultimate bronze silver gold),
       name: 'uuid',
-      data_category: 'Standard'
+      data_category: 'standard'
     }
   end
 
@@ -87,14 +87,14 @@ RSpec.describe Gitlab::Usage::MetricDefinition do
       end
 
       it 'raise exception' do
-        expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::Metric::InvalidMetricError))
+        expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError))
 
         described_class.new(path, attributes).validate!
       end
 
       context 'with skip_validation' do
         it 'raise exception if skip_validation: false' do
-          expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::Metric::InvalidMetricError))
+          expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError))
 
           described_class.new(path, attributes.merge( { skip_validation: false } )).validate!
         end
@@ -113,7 +113,7 @@ RSpec.describe Gitlab::Usage::MetricDefinition do
           attributes[:status] = 'broken'
           attributes.delete(:repair_issue_url)
 
-          expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::Metric::InvalidMetricError))
+          expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).at_least(:once).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError))
 
           described_class.new(path, attributes).validate!
         end
@@ -173,7 +173,7 @@ RSpec.describe Gitlab::Usage::MetricDefinition do
       write_metric(metric1, path, yaml_content)
       write_metric(metric2, path, yaml_content)
 
-      expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).with(instance_of(Gitlab::Usage::Metric::InvalidMetricError))
+      expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).with(instance_of(Gitlab::Usage::MetricDefinition::InvalidError))
 
       subject
     end
@@ -199,7 +199,7 @@ RSpec.describe Gitlab::Usage::MetricDefinition do
         data_source: 'database',
         distribution: %w(ee ce),
         tier: %w(free starter premium ultimate bronze silver gold),
-        data_category: 'Optional'
+        data_category: 'optional'
       }
     end
 
diff --git a/spec/lib/gitlab/usage/metric_spec.rb b/spec/lib/gitlab/usage/metric_spec.rb
index d4a789419a4..d83f59e4a7d 100644
--- a/spec/lib/gitlab/usage/metric_spec.rb
+++ b/spec/lib/gitlab/usage/metric_spec.rb
@@ -3,27 +3,46 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Usage::Metric do
-  describe '#definition' do
-    it 'returns key_path metric definiton' do
-      expect(described_class.new(key_path: 'uuid').definition).to be_an(Gitlab::Usage::MetricDefinition)
-    end
+  let!(:issue) { create(:issue) }
+
+  let(:attributes) do
+    {
+      data_category: "Operational",
+      key_path: "counts.issues",
+      description: "Count of Issues created",
+      product_section: "dev",
+      product_stage: "plan",
+      product_group: "group::plan",
+      product_category: "issue_tracking",
+      value_type: "number",
+      status: "data_available",
+      time_frame: "all",
+      data_source: "database",
+      instrumentation_class: "CountIssuesMetric",
+      distribution: %w(ce ee),
+      tier: %w(free premium ultimate)
+    }
   end
 
-  describe '#unflatten_default_path' do
-    using RSpec::Parameterized::TableSyntax
+  let(:issue_count_metric_definiton) do
+    double(:issue_count_metric_definiton,
+      attributes.merge({ attributes: attributes })
+    )
+  end
 
-    where(:key_path, :value, :expected_hash) do
-      'uuid'                                     | nil    | { uuid: nil }
-      'uuid'                                     | '1111' | { uuid: '1111' }
-      'counts.issues'                            | nil    | { counts: { issues: nil } }
-      'counts.issues'                            | 100    | { counts: { issues: 100 } }
-      'usage_activity_by_stage.verify.ci_builds' | 100    | { usage_activity_by_stage: { verify: { ci_builds: 100 } } }
-    end
+  before do
+    allow(ApplicationRecord.connection).to receive(:transaction_open?).and_return(false)
+  end
 
-    with_them do
-      subject { described_class.new(key_path: key_path, value: value).unflatten_key_path }
+  describe '#with_value' do
+    it 'returns key_path metric with the corresponding value' do
+      expect(described_class.new(issue_count_metric_definiton).with_value).to eq({ counts: { issues: 1 } })
+    end
+  end
 
-      it { is_expected.to eq(expected_hash) }
+  describe '#with_instrumentation' do
+    it 'returns key_path metric with the corresponding generated query' do
+      expect(described_class.new(issue_count_metric_definiton).with_instrumentation).to eq({ counts: { issues: "SELECT COUNT(\"issues\".\"id\") FROM \"issues\"" } })
     end
   end
 end
diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric_spec.rb
index 8f52d550e5c..1b2170baf17 100644
--- a/spec/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric_spec.rb
+++ b/spec/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric_spec.rb
@@ -4,7 +4,7 @@ require 'spec_helper'
 
 RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CollectedDataCategoriesMetric do
   it_behaves_like 'a correct instrumented metric value', {} do
-    let(:expected_value) { %w[Standard Subscription Operational Optional] }
+    let(:expected_value) { %w[standard subscription operational optional] }
 
     before do
       allow_next_instance_of(ServicePing::PermitDataCategoriesService) do |instance|
diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb
index 5e36820df5e..0a32bdb95d3 100644
--- a/spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb
+++ b/spec/lib/gitlab/usage/metrics/instrumentations/database_metric_spec.rb
@@ -4,11 +4,11 @@ require 'spec_helper'
 
 RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DatabaseMetric do
   subject do
-    described_class.tap do |m|
-      m.relation { Issue }
-      m.operation :count
-      m.start { m.relation.minimum(:id) }
-      m.finish { m.relation.maximum(:id) }
+    described_class.tap do |metric_class|
+      metric_class.relation { Issue }
+      metric_class.operation :count
+      metric_class.start { metric_class.relation.minimum(:id) }
+      metric_class.finish { metric_class.relation.maximum(:id) }
     end.new(time_frame: 'all')
   end
 
@@ -38,9 +38,9 @@ RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DatabaseMetric do
 
     context 'with start and finish not called' do
       subject do
-        described_class.tap do |m|
-          m.relation { Issue }
-          m.operation :count
+        described_class.tap do |metric_class|
+          metric_class.relation { Issue }
+          metric_class.operation :count
         end.new(time_frame: 'all')
       end
 
@@ -51,12 +51,12 @@ RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DatabaseMetric do
 
     context 'with cache_start_and_finish_as called' do
       subject do
-        described_class.tap do |m|
-          m.relation { Issue }
-          m.operation :count
-          m.start { m.relation.minimum(:id) }
-          m.finish { m.relation.maximum(:id) }
-          m.cache_start_and_finish_as :special_issue_count
+        described_class.tap do |metric_class|
+          metric_class.relation { Issue }
+          metric_class.operation :count
+          metric_class.start { metric_class.relation.minimum(:id) }
+          metric_class.finish { metric_class.relation.maximum(:id) }
+          metric_class.cache_start_and_finish_as :special_issue_count
         end.new(time_frame: 'all')
       end
 
@@ -71,5 +71,45 @@ RSpec.describe Gitlab::Usage::Metrics::Instrumentations::DatabaseMetric do
         expect(Rails.cache.read('metric_instrumentation/special_issue_count_maximum_id')).to eq(issues.max_by(&:id).id)
       end
     end
+
+    context 'with estimate_batch_distinct_count' do
+      subject do
+        described_class.tap do |metric_class|
+          metric_class.relation { Issue }
+          metric_class.operation(:estimate_batch_distinct_count)
+          metric_class.start { metric_class.relation.minimum(:id) }
+          metric_class.finish { metric_class.relation.maximum(:id) }
+        end.new(time_frame: 'all')
+      end
+
+      it 'calculates a correct result' do
+        expect(subject.value).to be_within(Gitlab::Database::PostgresHll::BatchDistinctCounter::ERROR_RATE).percent_of(3)
+      end
+
+      context 'with block passed to operation' do
+        let(:buckets) { double('Buckets').as_null_object }
+
+        subject do
+          described_class.tap do |metric_class|
+            metric_class.relation { Issue }
+            metric_class.operation(:estimate_batch_distinct_count) do |result|
+              result.foo
+            end
+            metric_class.start { metric_class.relation.minimum(:id) }
+            metric_class.finish { metric_class.relation.maximum(:id) }
+          end.new(time_frame: 'all')
+        end
+
+        before do
+          allow(Gitlab::Database::PostgresHll::Buckets).to receive(:new).and_return(buckets)
+        end
+
+        it 'calls the block passing HLL buckets as an argument' do
+          expect(buckets).to receive(:foo)
+
+          subject.value
+        end
+      end
+    end
   end
 end
diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/generic_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/generic_metric_spec.rb
new file mode 100644
index 00000000000..158be34d39c
--- /dev/null
+++ b/spec/lib/gitlab/usage/metrics/instrumentations/generic_metric_spec.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Usage::Metrics::Instrumentations::GenericMetric do
+  shared_examples 'custom fallback' do |custom_fallback|
+    subject do
+      Class.new(described_class) do
+        fallback(custom_fallback)
+        value { Gitlab::Database.main.version }
+      end.new(time_frame: 'none')
+    end
+
+    describe '#value' do
+      it 'gives the correct value' do
+        expect(subject.value).to eq(Gitlab::Database.main.version)
+      end
+
+      context 'when raising an exception' do
+        it 'return the custom fallback' do
+          expect(Gitlab::Database.main).to receive(:version).and_raise('Error')
+          expect(subject.value).to eq(custom_fallback)
+        end
+      end
+    end
+  end
+
+  context 'with default fallback' do
+    subject do
+      Class.new(described_class) do
+        value { Gitlab::Database.main.version }
+      end.new(time_frame: 'none')
+    end
+
+    describe '#value' do
+      it 'gives the correct value' do
+        expect(subject.value).to eq(Gitlab::Database.main.version )
+      end
+
+      context 'when raising an exception' do
+        it 'return the default fallback' do
+          expect(Gitlab::Database.main).to receive(:version).and_raise('Error')
+          expect(subject.value).to eq(described_class::FALLBACK)
+        end
+      end
+    end
+  end
+
+  context 'with custom fallback -2' do
+    it_behaves_like 'custom fallback', -2
+  end
+
+  context 'with custom fallback nil' do
+    it_behaves_like 'custom fallback', nil
+  end
+
+  context 'with custom fallback false' do
+    it_behaves_like 'custom fallback', false
+  end
+
+  context 'with custom fallback true' do
+    it_behaves_like 'custom fallback', true
+  end
+
+  context 'with custom fallback []' do
+    it_behaves_like 'custom fallback', []
+  end
+
+  context 'with custom fallback { major: -1 }' do
+    it_behaves_like 'custom fallback', { major: -1 }
+  end
+end
diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/redis_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/redis_metric_spec.rb
new file mode 100644
index 00000000000..fb3bd1ba834
--- /dev/null
+++ b/spec/lib/gitlab/usage/metrics/instrumentations/redis_metric_spec.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Usage::Metrics::Instrumentations::RedisMetric, :clean_gitlab_redis_shared_state do
+  before do
+    4.times do
+      Gitlab::UsageDataCounters::SourceCodeCounter.count(:pushes)
+    end
+  end
+
+  let(:expected_value) { 4 }
+
+  it_behaves_like 'a correct instrumented metric value', { options: { event: 'pushes', counter_class: 'SourceCodeCounter' } }
+
+  it 'raises an exception if event option is not present' do
+    expect { described_class.new(counter_class: 'SourceCodeCounter') }.to raise_error(ArgumentError)
+  end
+
+  it 'raises an exception if counter_class option is not present' do
+    expect { described_class.new(event: 'pushes') }.to raise_error(ArgumentError)
+  end
+end
diff --git a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb
index b4ab9d4861b..0f95da74ff9 100644
--- a/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb
+++ b/spec/lib/gitlab/usage/metrics/names_suggestions/generator_spec.rb
@@ -16,6 +16,14 @@ RSpec.describe Gitlab::Usage::Metrics::NamesSuggestions::Generator do
       end
     end
 
+    describe '#add_metric' do
+      let(:metric) {'CountIssuesMetric' }
+
+      it 'computes the suggested name for given metric' do
+        expect(described_class.add_metric(metric)).to eq('count_issues')
+      end
+    end
+
     context 'for count with default column metrics' do
       it_behaves_like 'name suggestion' do
         # corresponding metric is collected with count(Board)