Import common metrics into database.

This MR backports PrometheusMetric model to CE
and adds: common, identifier to figure out what kind of metric is used.

10 files changed, 210 insertions, 1 deletions

diff --git a/db/fixtures/development/99_common_metrics.rb b/db/fixtures/development/99_common_metrics.rb
new file mode 100644
index 00000000000..89424905bd4
--- /dev/null
+++ b/db/fixtures/development/99_common_metrics.rb
@@ -0,0 +1,3 @@
+require_relative '../importers/common_metrics_importer.rb'
+
+::Importers::CommonMetricsImporter.new.execute
diff --git a/db/fixtures/production/999_common_metrics.rb b/db/fixtures/production/999_common_metrics.rb
new file mode 100644
index 00000000000..89424905bd4
--- /dev/null
+++ b/db/fixtures/production/999_common_metrics.rb
@@ -0,0 +1,3 @@
+require_relative '../importers/common_metrics_importer.rb'
+
+::Importers::CommonMetricsImporter.new.execute
diff --git a/db/importers/common_metrics_importer.rb b/db/importers/common_metrics_importer.rb
new file mode 100644
index 00000000000..9e6d2dd0518
--- /dev/null
+++ b/db/importers/common_metrics_importer.rb
@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Importers
+  class PrometheusMetric < ActiveRecord::Base
+    enum group: {
+      # built-in groups
+      nginx_ingress: -1,
+      ha_proxy: -2,
+      aws_elb: -3,
+      nginx: -4,
+      kubernetes: -5,
+  
+      # custom groups
+      business: 0,
+      response: 1,
+      system: 2,
+    }
+
+    scope :common, -> { where(common: true) }
+
+    GROUP_TITLES = {
+      business: _('Business metrics (Custom)'),
+      response: _('Response metrics (Custom)'),
+      system: _('System metrics (Custom)'),
+      nginx_ingress: _('Response metrics (NGINX Ingress)'),
+      ha_proxy: _('Response metrics (HA Proxy)'),
+      aws_elb: _('Response metrics (AWS ELB)'),
+      nginx: _('Response metrics (NGINX)'),
+      kubernetes: _('System metrics (Kubernetes)')
+    }.freeze
+  end
+
+  class CommonMetricsImporter
+    MissingQueryId = Class.new(StandardError)
+
+    attr_reader :content
+
+    def initialize(file = 'config/prometheus/common_metrics.yml')
+      @content = YAML.load_file(file)
+    end
+    
+    def execute
+      process_content do |id, attributes|
+        find_or_build_metric!(id)
+          .update!(**attributes)
+      end
+    end
+
+    private
+
+    def process_content(&blk)
+      content.map do |group|
+        process_group(group, &blk)
+      end
+    end
+
+    def process_group(group, &blk)
+      attributes = {
+        group: find_group_title_key(group['group'])
+      }
+
+      group['metrics'].map do |metric|
+        process_metric(metric, attributes, &blk)
+      end
+    end
+
+    def process_metric(metric, attributes, &blk)
+      attributes = attributes.merge(
+        title: metric['title'],
+        y_label: metric['y_label'])
+
+      metric['queries'].map do |query|
+        process_metric_query(query, attributes, &blk)
+      end
+    end
+
+    def process_metric_query(query, attributes, &blk)
+      attributes = attributes.merge(
+        legend: query['label'],
+        query: query['query_range'],
+        unit: query['unit'])
+
+      blk.call(query['id'], attributes)
+    end
+
+    def find_or_build_metric!(id)
+      raise MissingQueryId unless id
+    
+      PrometheusMetric.common.find_by(identifier: id) ||
+        PrometheusMetric.new(common: true, identifier: id)
+    end
+
+    def find_group_title_key(title)
+      PrometheusMetric.groups[find_group_title(title)]
+    end
+
+    def find_group_title(title)
+      PrometheusMetric::GROUP_TITLES.invert[title]
+    end
+  end
+end
diff --git a/db/migrate/20180101160629_create_prometheus_metrics.rb b/db/migrate/20180101160629_create_prometheus_metrics.rb
new file mode 100644
index 00000000000..da6371084cb
--- /dev/null
+++ b/db/migrate/20180101160629_create_prometheus_metrics.rb
@@ -0,0 +1,16 @@
+class CreatePrometheusMetrics < ActiveRecord::Migration
+  DOWNTIME = false
+
+  def change
+    create_table :prometheus_metrics do |t|
+      t.references :project, index: true, foreign_key: { on_delete: :cascade }, null: false
+      t.string :title, null: false
+      t.string :query, null: false
+      t.string :y_label
+      t.string :unit
+      t.string :legend
+      t.integer :group, null: false, index: true
+      t.timestamps_with_timezone null: false
+    end
+  end
+end
diff --git a/db/migrate/20180831164905_add_common_to_prometheus_metrics.rb b/db/migrate/20180831164905_add_common_to_prometheus_metrics.rb
new file mode 100644
index 00000000000..43e36d2047c
--- /dev/null
+++ b/db/migrate/20180831164905_add_common_to_prometheus_metrics.rb
@@ -0,0 +1,15 @@
+class AddCommonToPrometheusMetrics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_column_with_default(:prometheus_metrics, :common, :boolean, default: false)
+  end
+
+  def down
+    remove_column(:prometheus_metrics, :common)
+  end
+end
diff --git a/db/migrate/20180831164906_change_project_id_for_prometheus_metrics.rb b/db/migrate/20180831164906_change_project_id_for_prometheus_metrics.rb
new file mode 100644
index 00000000000..c9aae868e52
--- /dev/null
+++ b/db/migrate/20180831164906_change_project_id_for_prometheus_metrics.rb
@@ -0,0 +1,11 @@
+class ChangeProjectIdForPrometheusMetrics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def change
+    change_column_null :prometheus_metrics, :project_id, true
+  end
+end
diff --git a/db/migrate/20180831164907_add_index_on_default_prometheus_metrics.rb b/db/migrate/20180831164907_add_index_on_default_prometheus_metrics.rb
new file mode 100644
index 00000000000..2e2014ccdce
--- /dev/null
+++ b/db/migrate/20180831164907_add_index_on_default_prometheus_metrics.rb
@@ -0,0 +1,15 @@
+class AddIndexOnDefaultPrometheusMetrics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :prometheus_metrics, :common
+  end
+
+  def down
+    remove_concurrent_index :prometheus_metrics, :project_id
+  end
+end
diff --git a/db/migrate/20180831164908_add_identifier_to_prometheus_metric.rb b/db/migrate/20180831164908_add_identifier_to_prometheus_metric.rb
new file mode 100644
index 00000000000..d5f33a293cc
--- /dev/null
+++ b/db/migrate/20180831164908_add_identifier_to_prometheus_metric.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class AddIdentifierToPrometheusMetric < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    add_column :prometheus_metrics, :identifier, :string, unique: true
+  end
+end
diff --git a/db/migrate/20180831164909_import_common_metrics.rb b/db/migrate/20180831164909_import_common_metrics.rb
new file mode 100644
index 00000000000..adb35a9b211
--- /dev/null
+++ b/db/migrate/20180831164909_import_common_metrics.rb
@@ -0,0 +1,15 @@
+class ImportCommonMetrics < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  require_relative '../importers/common_metrics_importer.rb'
+
+  DOWNTIME = false
+
+  def up
+    Importers::CommonMetricsImporter.new.execute
+  end
+
+  def down
+    # no-op
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 02e545bec7d..c3d4eccd826 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180826111825) do
+ActiveRecord::Schema.define(version: 20180831164909) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -1696,6 +1696,24 @@ ActiveRecord::Schema.define(version: 20180826111825) do
   add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree
   add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree
 
+  create_table "prometheus_metrics", force: :cascade do |t|
+    t.integer "project_id"
+    t.string "title", null: false
+    t.string "query", null: false
+    t.string "y_label"
+    t.string "unit"
+    t.string "legend"
+    t.integer "group", null: false
+    t.datetime_with_timezone "created_at", null: false
+    t.datetime_with_timezone "updated_at", null: false
+    t.boolean "common", default: false, null: false
+    t.string "identifier"
+  end
+
+  add_index "prometheus_metrics", ["common"], name: "index_prometheus_metrics_on_common", using: :btree
+  add_index "prometheus_metrics", ["group"], name: "index_prometheus_metrics_on_group", using: :btree
+  add_index "prometheus_metrics", ["project_id"], name: "index_prometheus_metrics_on_project_id", using: :btree
+
   create_table "protected_branch_merge_access_levels", force: :cascade do |t|
     t.integer "protected_branch_id", null: false
     t.integer "access_level", default: 40, null: false
@@ -2375,6 +2393,7 @@ ActiveRecord::Schema.define(version: 20180826111825) do
   add_foreign_key "project_import_data", "projects", name: "fk_ffb9ee3a10", on_delete: :cascade
   add_foreign_key "project_mirror_data", "projects", on_delete: :cascade
   add_foreign_key "project_statistics", "projects", on_delete: :cascade
+  add_foreign_key "prometheus_metrics", "projects", on_delete: :cascade
   add_foreign_key "protected_branch_merge_access_levels", "protected_branches", name: "fk_8a3072ccb3", on_delete: :cascade
   add_foreign_key "protected_branch_push_access_levels", "protected_branches", name: "fk_9ffc86a3d9", on_delete: :cascade
   add_foreign_key "protected_branches", "projects", name: "fk_7a9c6d93e7", on_delete: :cascade