Merge branch 'alerts-for-built-in-metrics' into 'master'

Import common metrics into database.

Closes gitlab-ee#6948

See merge request gitlab-org/gitlab-ce!21459

10 files changed, 341 insertions, 1 deletions

diff --git a/spec/db/development/import_common_metrics_spec.rb b/spec/db/development/import_common_metrics_spec.rb
new file mode 100644
index 00000000000..25061ef0887
--- /dev/null
+++ b/spec/db/development/import_common_metrics_spec.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'Import metrics on development seed' do
+  subject { load Rails.root.join('db', 'fixtures', 'development', '99_common_metrics.rb') }
+
+  it "imports all prometheus metrics" do
+    expect(PrometheusMetric.common).to be_empty
+
+    subject
+
+    expect(PrometheusMetric.common).not_to be_empty
+  end
+end
diff --git a/spec/db/importers/common_metrics_importer_spec.rb b/spec/db/importers/common_metrics_importer_spec.rb
new file mode 100644
index 00000000000..16b59e1dfe8
--- /dev/null
+++ b/spec/db/importers/common_metrics_importer_spec.rb
@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+require Rails.root.join("db", "importers", "common_metrics_importer.rb")
+
+describe Importers::PrometheusMetric do
+  it 'group enum equals ::PrometheusMetric' do
+    expect(described_class.groups).to eq(::PrometheusMetric.groups)
+  end
+
+  it 'GROUP_TITLES equals ::PrometheusMetric' do
+    expect(described_class::GROUP_TITLES).to eq(::PrometheusMetric::GROUP_TITLES)
+  end
+end
+
+describe Importers::CommonMetricsImporter do
+  subject { described_class.new }
+
+  context "does import common_metrics.yml" do
+    let(:groups) { subject.content }
+    let(:metrics) { groups.map { |group| group['metrics'] }.flatten }
+    let(:queries) { metrics.map { |group| group['queries'] }.flatten }
+    let(:query_ids) { queries.map { |query| query['id'] } }
+
+    before do
+      subject.execute
+    end
+
+    it "has the same amount of groups" do
+      expect(PrometheusMetric.common.group(:group).count.count).to eq(groups.count)
+    end
+
+    it "has the same amount of metrics" do
+      expect(PrometheusMetric.common.group(:group, :title).count.count).to eq(metrics.count)
+    end
+
+    it "has the same amount of queries" do
+      expect(PrometheusMetric.common.count).to eq(queries.count)
+    end
+
+    it "does not have duplicate IDs" do
+      expect(query_ids).to eq(query_ids.uniq)
+    end
+
+    it "imports all IDs" do
+      expect(PrometheusMetric.common.pluck(:identifier)).to contain_exactly(*query_ids)
+    end
+  end
+
+  context 'does import properly all fields' do
+    let(:query_identifier) { 'response-metric' }
+    let(:group) do
+      {
+        group: 'Response metrics (NGINX Ingress)',
+        metrics: [{
+          title: "Throughput",
+          y_label: "Requests / Sec",
+          queries: [{
+            id: query_identifier,
+            query_range: 'my-query',
+            unit: 'my-unit',
+            label: 'status code'
+          }]
+        }]
+      }
+    end
+
+    before do
+      expect(subject).to receive(:content) { [group.deep_stringify_keys] }
+    end
+
+    shared_examples 'stores metric' do
+      let(:metric) { PrometheusMetric.find_by(identifier: query_identifier) }
+
+      it 'with all data' do
+        expect(metric.group).to eq('nginx_ingress')
+        expect(metric.title).to eq('Throughput')
+        expect(metric.y_label).to eq('Requests / Sec')
+        expect(metric.unit).to eq('my-unit')
+        expect(metric.legend).to eq('status code')
+        expect(metric.query).to eq('my-query')
+      end
+    end
+
+    context 'if ID is missing' do
+      let(:query_identifier) { }
+
+      it 'raises exception' do
+        expect { subject.execute }.to raise_error(described_class::MissingQueryId)
+      end
+    end
+
+    context 'for existing common metric with different ID' do
+      let!(:existing_metric) { create(:prometheus_metric, :common, identifier: 'my-existing-metric') }
+
+      before do
+        subject.execute
+      end
+
+      it_behaves_like 'stores metric' do
+        it 'and existing metric is not changed' do
+          expect(metric).not_to eq(existing_metric)
+        end
+      end
+    end
+
+    context 'when metric with ID exists ' do
+      let!(:existing_metric) { create(:prometheus_metric, :common, identifier: 'response-metric') }
+
+      before do
+        subject.execute
+      end
+
+      it_behaves_like 'stores metric' do
+        it 'and existing metric is changed' do
+          expect(metric).to eq(existing_metric)
+        end
+      end
+    end
+  end
+end
diff --git a/spec/db/production/import_common_metrics_spec.rb b/spec/db/production/import_common_metrics_spec.rb
new file mode 100644
index 00000000000..1e4ff818a86
--- /dev/null
+++ b/spec/db/production/import_common_metrics_spec.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'Import metrics on production seed' do
+  subject { load Rails.root.join('db', 'fixtures', 'production', '999_common_metrics.rb') }
+
+  it "imports all prometheus metrics" do
+    expect(PrometheusMetric.common).to be_empty
+
+    subject
+
+    expect(PrometheusMetric.common).not_to be_empty
+  end
+end
diff --git a/spec/factories/prometheus_metrics.rb b/spec/factories/prometheus_metrics.rb
new file mode 100644
index 00000000000..c56644bfb96
--- /dev/null
+++ b/spec/factories/prometheus_metrics.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :prometheus_metric, class: PrometheusMetric do
+    title 'title'
+    query 'avg(metric)'
+    y_label 'y_label'
+    unit 'm/s'
+    group :business
+    project
+    legend 'legend'
+
+    trait :common do
+      common true
+      project nil
+    end
+  end
+end
diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml
index b4269bd5786..2412cc3010a 100644
--- a/spec/lib/gitlab/import_export/all_models.yml
+++ b/spec/lib/gitlab/import_export/all_models.yml
@@ -288,6 +288,7 @@ project:
 - fork_network_member
 - fork_network
 - custom_attributes
+- prometheus_metrics
 - lfs_file_locks
 - project_badges
 - source_of_merge_requests
@@ -303,6 +304,8 @@ award_emoji:
 - user
 priorities:
 - label
+prometheus_metrics:
+- project
 timelogs:
 - issue
 - merge_request
diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml
index 579f175c4a8..1be448b0486 100644
--- a/spec/lib/gitlab/import_export/safe_model_attributes.yml
+++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml
@@ -555,6 +555,19 @@ ProjectCustomAttribute:
 - project_id
 - key
 - value
+PrometheusMetric:
+- id
+- created_at
+- updated_at
+- project_id
+- y_label
+- unit
+- legend
+- title
+- query
+- group
+- common
+- identifier
 Badge:
 - id
 - link_url
diff --git a/spec/lib/gitlab/prometheus/additional_metrics_parser_spec.rb b/spec/lib/gitlab/prometheus/additional_metrics_parser_spec.rb
index 5589db92b1d..1a108003bc2 100644
--- a/spec/lib/gitlab/prometheus/additional_metrics_parser_spec.rb
+++ b/spec/lib/gitlab/prometheus/additional_metrics_parser_spec.rb
@@ -6,7 +6,7 @@ describe Gitlab::Prometheus::AdditionalMetricsParser do
   let(:parser_error_class) { Gitlab::Prometheus::ParsingError }
 
   describe '#load_groups_from_yaml' do
-    subject { described_class.load_groups_from_yaml }
+    subject { described_class.load_groups_from_yaml('dummy.yaml') }
 
     describe 'parsing sample yaml' do
       let(:sample_yaml) do
diff --git a/spec/lib/gitlab/prometheus/metric_group_spec.rb b/spec/lib/gitlab/prometheus/metric_group_spec.rb
new file mode 100644
index 00000000000..e7d16e73663
--- /dev/null
+++ b/spec/lib/gitlab/prometheus/metric_group_spec.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Gitlab::Prometheus::MetricGroup do
+  describe '.common_metrics' do
+    let!(:project_metric) { create(:prometheus_metric) }
+    let!(:common_metric_group_a) { create(:prometheus_metric, :common, group: :aws_elb) }
+    let!(:common_metric_group_b_q1) { create(:prometheus_metric, :common, group: :kubernetes) }
+    let!(:common_metric_group_b_q2) { create(:prometheus_metric, :common, group: :kubernetes) }
+
+    subject { described_class.common_metrics }
+
+    it 'returns exactly two groups' do
+      expect(subject.map(&:name)).to contain_exactly(
+        'Response metrics (AWS ELB)', 'System metrics (Kubernetes)')
+    end
+
+    it 'returns exactly three metric queries' do
+      expect(subject.map(&:metrics).flatten.map(&:id)).to contain_exactly(
+        common_metric_group_a.id, common_metric_group_b_q1.id,
+        common_metric_group_b_q2.id)
+    end
+  end
+
+  describe '.for_project' do
+    let!(:other_project) { create(:project) }
+    let!(:project_metric) { create(:prometheus_metric) }
+    let!(:common_metric) { create(:prometheus_metric, :common, group: :aws_elb) }
+
+    subject do
+      described_class.for_project(other_project)
+        .map(&:metrics).flatten
+        .map(&:id)
+    end
+
+    it 'returns exactly one common metric' do
+      is_expected.to contain_exactly(common_metric.id)
+    end
+  end
+end
diff --git a/spec/migrations/import_common_metrics_spec.rb b/spec/migrations/import_common_metrics_spec.rb
new file mode 100644
index 00000000000..1001629007c
--- /dev/null
+++ b/spec/migrations/import_common_metrics_spec.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'migrate', '20180831164910_import_common_metrics.rb')
+
+describe ImportCommonMetrics, :migration do
+  describe '#up' do
+    it "imports all prometheus metrics" do
+      expect(PrometheusMetric.common).to be_empty
+
+      migrate!
+
+      expect(PrometheusMetric.common).not_to be_empty
+    end
+  end
+end
diff --git a/spec/models/prometheus_metric_spec.rb b/spec/models/prometheus_metric_spec.rb
new file mode 100644
index 00000000000..a83a31ae88c
--- /dev/null
+++ b/spec/models/prometheus_metric_spec.rb
@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe PrometheusMetric do
+  subject { build(:prometheus_metric) }
+  let(:other_project) { build(:project) }
+
+  it { is_expected.to belong_to(:project) }
+  it { is_expected.to validate_presence_of(:title) }
+  it { is_expected.to validate_presence_of(:query) }
+  it { is_expected.to validate_presence_of(:group) }
+
+  describe 'common metrics' do
+    using RSpec::Parameterized::TableSyntax
+
+    where(:common, :project, :result) do
+      false | other_project | true
+      false | nil           | false
+      true  | other_project | false
+      true  | nil           | true
+    end
+
+    with_them do
+      before do
+        subject.common = common
+        subject.project = project
+      end
+
+      it { expect(subject.valid?).to eq(result) }
+    end
+  end
+
+  describe '#query_series' do
+    using RSpec::Parameterized::TableSyntax
+
+    where(:legend, :type) do
+      'Some other legend' | NilClass
+      'Status Code'       | Array
+    end
+
+    with_them do
+      before do
+        subject.legend = legend
+      end
+
+      it { expect(subject.query_series).to be_a(type) }
+    end
+  end
+
+  describe '#group_title' do
+    shared_examples 'group_title' do |group, title|
+      subject { build(:prometheus_metric, group: group).group_title }
+
+      it "returns text #{title} for group #{group}" do
+        expect(subject).to eq(title)
+      end
+    end
+
+    it_behaves_like 'group_title', :business, 'Business metrics (Custom)'
+    it_behaves_like 'group_title', :response, 'Response metrics (Custom)'
+    it_behaves_like 'group_title', :system, 'System metrics (Custom)'
+  end
+
+  describe '#to_query_metric' do
+    it 'converts to queryable metric object' do
+      expect(subject.to_query_metric).to be_instance_of(Gitlab::Prometheus::Metric)
+    end
+
+    it 'queryable metric object has title' do
+      expect(subject.to_query_metric.title).to eq(subject.title)
+    end
+
+    it 'queryable metric object has y_label' do
+      expect(subject.to_query_metric.y_label).to eq(subject.y_label)
+    end
+
+    it 'queryable metric has no required_metric' do
+      expect(subject.to_query_metric.required_metrics).to eq([])
+    end
+
+    it 'queryable metric has weight 0' do
+      expect(subject.to_query_metric.weight).to eq(0)
+    end
+
+    it 'queryable metrics has query description' do
+      queries = [
+        {
+          query_range: subject.query,
+          unit: subject.unit,
+          label: subject.legend
+        }
+      ]
+
+      expect(subject.to_query_metric.queries).to eq(queries)
+    end
+  end
+end