Add latest changes from gitlab-org/gitlab@master

7 files changed, 193 insertions, 2 deletions

diff --git a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb
index dcc336b8929..4990a5a6eb5 100644
--- a/lib/gitlab/ci/pipeline/chain/config/content/remote.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/content/remote.rb
@@ -9,7 +9,7 @@ module Gitlab
             class Remote < Source
               def content
                 strong_memoize(:content) do
-                  next unless ci_config_path =~ URI.regexp(%w[http https])
+                  next unless ci_config_path =~ URI::DEFAULT_PARSER.make_regexp(%w[http https])
 
                   YAML.dump('include' => [{ 'remote' => ci_config_path }])
                 end
diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb
index fb0fcc5a93b..8a5f53be20f 100644
--- a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb
+++ b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb
@@ -18,6 +18,7 @@ module Gitlab
             business: 0,
             response: 1,
             system: 2,
+            custom: 3,
 
             cluster_health: -100
           }
@@ -34,7 +35,8 @@ module Gitlab
             aws_elb: _('Response metrics (AWS ELB)'),
             nginx: _('Response metrics (NGINX)'),
             kubernetes: _('System metrics (Kubernetes)'),
-            cluster_health: _('Cluster Health')
+            cluster_health: _('Cluster Health'),
+            custom: _('Custom metrics')
           }
         end
       end
diff --git a/lib/gitlab/metrics/dashboard/importer.rb b/lib/gitlab/metrics/dashboard/importer.rb
new file mode 100644
index 00000000000..ca835650648
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/importer.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      class Importer
+        def initialize(dashboard_path, project)
+          @dashboard_path = dashboard_path.to_s
+          @project = project
+        end
+
+        def execute
+          return false unless Dashboard::Validator.validate(dashboard_hash, project: project, dashboard_path: dashboard_path)
+
+          Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute
+        rescue Gitlab::Config::Loader::FormatError
+          false
+        end
+
+        def execute!
+          Dashboard::Validator.validate!(dashboard_hash, project: project, dashboard_path: dashboard_path)
+
+          Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute!
+        end
+
+        private
+
+        attr_accessor :dashboard_path, :project
+
+        def dashboard_hash
+          @dashboard_hash ||= begin
+            raw_dashboard = Dashboard::RepoDashboardFinder.read_dashboard(project, dashboard_path)
+            return unless raw_dashboard.present?
+
+            ::Gitlab::Config::Loader::Yaml.new(raw_dashboard).load_raw!
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb
new file mode 100644
index 00000000000..d1490d5d9b6
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Importers
+        class PrometheusMetrics
+          ALLOWED_ATTRIBUTES = %i(title query y_label unit legend group dashboard_path).freeze
+
+          # Takes a JSON schema validated dashboard hash and
+          # imports metrics to database
+          def initialize(dashboard_hash, project:, dashboard_path:)
+            @dashboard_hash = dashboard_hash
+            @project = project
+            @dashboard_path = dashboard_path
+          end
+
+          def execute
+            import
+          rescue ActiveRecord::RecordInvalid, ::Gitlab::Metrics::Dashboard::Transformers::TransformerError
+            false
+          end
+
+          def execute!
+            import
+          end
+
+          private
+
+          attr_reader :dashboard_hash, :project, :dashboard_path
+
+          def import
+            delete_stale_metrics
+            create_or_update_metrics
+          end
+
+          # rubocop: disable CodeReuse/ActiveRecord
+          def create_or_update_metrics
+            # TODO: use upsert and worker for callbacks?
+            prometheus_metrics_attributes.each do |attributes|
+              prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:identifier, :project))
+              prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES))
+            end
+          end
+          # rubocop: enable CodeReuse/ActiveRecord
+
+          def delete_stale_metrics
+            identifiers = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] }
+
+            stale_metrics = PrometheusMetric.for_project(project)
+              .for_dashboard_path(dashboard_path)
+              .for_group(Enums::PrometheusMetric.groups[:custom])
+              .not_identifier(identifiers)
+
+            # TODO: use destroy_all and worker for callbacks?
+            stale_metrics.each(&:destroy)
+          end
+
+          def prometheus_metrics_attributes
+            @prometheus_metrics_attributes ||= begin
+              Dashboard::Transformers::Yml::V1::PrometheusMetrics.new(
+                dashboard_hash,
+                project: project,
+                dashboard_path: dashboard_path
+              ).execute
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/transformers/errors.rb b/lib/gitlab/metrics/dashboard/transformers/errors.rb
new file mode 100644
index 00000000000..4d94ab098ae
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/transformers/errors.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Transformers
+        TransformerError = Class.new(StandardError)
+
+        module Errors
+          class MissingAttribute < TransformerError
+            def initialize(attribute_name)
+              super("Missing attribute: '#{attribute_name}'")
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb
new file mode 100644
index 00000000000..4e46eec17d6
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    module Dashboard
+      module Transformers
+        module Yml
+          module V1
+            # Takes a JSON schema validated dashboard hash and
+            # maps it to PrometheusMetric model attributes
+            class PrometheusMetrics
+              def initialize(dashboard_hash, project: nil, dashboard_path: nil)
+                @dashboard_hash = dashboard_hash.with_indifferent_access
+                @project = project
+                @dashboard_path = dashboard_path
+
+                @dashboard_hash.default_proc = -> (h, k) { raise Transformers::Errors::MissingAttribute, k.to_s }
+              end
+
+              def execute
+                prometheus_metrics = []
+
+                dashboard_hash[:panel_groups].each do |panel_group|
+                  panel_group[:panels].each do |panel|
+                    panel[:metrics].each do |metric|
+                      prometheus_metrics << {
+                        project:        project,
+                        title:          panel[:title],
+                        y_label:        panel[:y_label],
+                        query:          metric[:query_range] || metric[:query],
+                        unit:           metric[:unit],
+                        legend:         metric[:label],
+                        identifier:     metric[:id],
+                        group:          Enums::PrometheusMetric.groups[:custom],
+                        common:         false,
+                        dashboard_path: dashboard_path
+                      }.compact
+                    end
+                  end
+                end
+
+                prometheus_metrics
+              end
+
+              private
+
+              attr_reader :dashboard_hash, :project, :dashboard_path
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 1dd6eab1743..a9e2ba4d85c 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -269,6 +269,9 @@ module Gitlab
           database: {
             adapter: alt_usage_data { Gitlab::Database.adapter_name },
             version: alt_usage_data { Gitlab::Database.version }
+          },
+          mail: {
+            smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] }
           }
         }
       end