Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2022-05-19 10:33:21 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2022-05-19 10:33:21 +0300
commit36a59d088eca61b834191dacea009677a96c052f (patch)
treee4f33972dab5d8ef79e3944a9f403035fceea43f /lib/gitlab/usage
parenta1761f15ec2cae7c7f7bbda39a75494add0dfd6f (diff)
Add latest changes from gitlab-org/gitlab@15-0-stable-eev15.0.0-rc42
Diffstat (limited to 'lib/gitlab/usage')
-rw-r--r--lib/gitlab/usage/metric.rb16
-rw-r--r--lib/gitlab/usage/metric_definition.rb20
-rw-r--r--lib/gitlab/usage/metrics/aggregates/aggregate.rb2
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/base_metric.rb16
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb2
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb2
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb39
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb57
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/database_metric.rb29
-rw-r--r--lib/gitlab/usage/metrics/query.rb24
-rw-r--r--lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb18
-rw-r--r--lib/gitlab/usage/service_ping_report.rb4
12 files changed, 206 insertions, 23 deletions
diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb
index 24e044c5740..cf48aa49938 100644
--- a/lib/gitlab/usage/metric.rb
+++ b/lib/gitlab/usage/metric.rb
@@ -18,19 +18,25 @@ module Gitlab
end
def with_value
- unflatten_key_path(intrumentation_object.value)
+ with_availability(proc { instrumentation_object.value })
end
def with_instrumentation
- unflatten_key_path(intrumentation_object.instrumentation)
+ with_availability(proc { instrumentation_object.instrumentation })
end
def with_suggested_name
- unflatten_key_path(intrumentation_object.suggested_name)
+ with_availability(proc { instrumentation_object.suggested_name })
end
private
+ def with_availability(value_proc)
+ return {} unless instrumentation_object.available?
+
+ unflatten_key_path(value_proc.call)
+ end
+
def unflatten_key_path(value)
::Gitlab::Usage::Metrics::KeyPathProcessor.process(definition.key_path, value)
end
@@ -39,8 +45,8 @@ module Gitlab
"Gitlab::Usage::Metrics::Instrumentations::#{definition.instrumentation_class}"
end
- def intrumentation_object
- instrumentation_class.constantize.new(
+ def instrumentation_object
+ @instrumentation_object ||= instrumentation_class.constantize.new(
time_frame: definition.time_frame,
options: definition.attributes[:options]
)
diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb
index 1031f38792b..2c50678c6bf 100644
--- a/lib/gitlab/usage/metric_definition.rb
+++ b/lib/gitlab/usage/metric_definition.rb
@@ -4,9 +4,9 @@ module Gitlab
module Usage
class MetricDefinition
METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json')
- BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master'
SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze
- AVAILABLE_STATUSES = %w[active data_available implemented deprecated].freeze
+ AVAILABLE_STATUSES = %w[active data_available implemented deprecated].to_set.freeze
+ VALID_SERVICE_PING_STATUSES = %w[active data_available implemented deprecated broken].to_set.freeze
InvalidError = Class.new(RuntimeError)
@@ -26,20 +26,22 @@ module Gitlab
attributes
end
+ def json_schema
+ return unless has_json_schema?
+
+ @json_schema ||= Gitlab::Json.parse(File.read(json_schema_path))
+ end
+
def json_schema_path
return '' unless has_json_schema?
- "#{BASE_REPO_PATH}/#{attributes[:value_json_schema]}"
+ Rails.root.join(attributes[:value_json_schema])
end
def has_json_schema?
attributes[:value_type] == 'object' && attributes[:value_json_schema].present?
end
- def yaml_path
- "#{BASE_REPO_PATH}#{path.delete_prefix(Rails.root.to_s)}"
- end
-
def validate!
unless skip_validation?
self.class.schemer.validate(attributes.stringify_keys).each do |error|
@@ -64,6 +66,10 @@ module Gitlab
AVAILABLE_STATUSES.include?(attributes[:status])
end
+ def valid_service_ping_status?
+ VALID_SERVICE_PING_STATUSES.include?(attributes[:status])
+ end
+
alias_method :to_dictionary, :to_h
class << self
diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
index 2545a505984..11e2fd22638 100644
--- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb
+++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
@@ -30,7 +30,7 @@ module Gitlab
def aggregated_metrics_data(start_date:, end_date:, time_frame:)
aggregated_metrics.each_with_object({}) do |aggregation, data|
- next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: :yaml, type: :development)
+ next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], type: :development)
next unless aggregation[:time_frame].include?(time_frame)
case aggregation[:source]
diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb
index a264f9484f3..f76ed1753b2 100644
--- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb
@@ -11,6 +11,18 @@ module Gitlab
attr_reader :time_frame
attr_reader :options
+ class << self
+ def available?(&block)
+ return @metric_available = block if block_given?
+
+ return @metric_available.call if instance_variable_defined?('@metric_available')
+
+ true
+ end
+
+ attr_reader :metric_available
+ end
+
def initialize(time_frame:, options: {})
@time_frame = time_frame
@options = options
@@ -19,6 +31,10 @@ module Gitlab
def instrumentation
value
end
+
+ def available?
+ self.class.available?
+ end
end
end
end
diff --git a/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb
index 6df6fef5d3a..d42250c9297 100644
--- a/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb
@@ -6,7 +6,7 @@ module Gitlab
module Instrumentations
class CertBasedClustersFfMetric < GenericMetric
value do
- Feature.enabled?(:certificate_based_clusters, default_enabled: :yaml, type: :ops)
+ Feature.enabled?(:certificate_based_clusters, type: :ops)
end
end
end
diff --git a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb
index ee51180973c..51be4bf3ccf 100644
--- a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb
@@ -6,7 +6,7 @@ module Gitlab
module Instrumentations
class CollectedDataCategoriesMetric < GenericMetric
value do
- ::ServicePing::PermitDataCategoriesService.new.execute.to_a
+ ::ServicePing::PermitDataCategories.new.execute.to_a
end
end
end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb
new file mode 100644
index 00000000000..c0d53b1b21a
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountBulkImportsEntitiesMetric < DatabaseMetric
+ operation :count
+
+ def initialize(time_frame:, options: {})
+ super
+
+ if source_type.present? && !source_type.in?(allowed_source_types)
+ raise ArgumentError, "source_type '#{source_type}' must be one of: #{allowed_source_types.join(', ')}"
+ end
+ end
+
+ relation { ::BulkImports::Entity }
+
+ private
+
+ def relation
+ return super.where(source_type: source_type) if source_type.present? # rubocop: disable CodeReuse/ActiveRecord
+
+ super
+ end
+
+ def source_type
+ options[:source_type].to_s
+ end
+
+ def allowed_source_types
+ BulkImports::Entity.source_types.keys.map(&:to_s)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb
new file mode 100644
index 00000000000..c5498ce530f
--- /dev/null
+++ b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module Instrumentations
+ class CountImportedProjectsMetric < DatabaseMetric
+ operation :count
+
+ def initialize(time_frame:, options: {})
+ super
+
+ raise ArgumentError, "import_type options attribute is required" unless import_type.present?
+ end
+
+ relation { ::Project }
+
+ start do |time_constraints|
+ unless time_constraints.nil?
+ start = time_constraints[:created_at]&.first
+
+ unless start.nil?
+ ::Project
+ .select(:id)
+ .where(Project.arel_table[:created_at].gteq(start)) # rubocop:disable UsageData/LargeTable
+ .order(created_at: :asc).limit(1).first&.id
+ end
+ end
+ end
+
+ finish do |time_constraints|
+ unless time_constraints.nil?
+ finish = time_constraints[:created_at]&.last
+
+ unless finish.nil?
+ ::Project
+ .select(:id)
+ .where(Project.arel_table[:created_at].lteq(finish)) # rubocop:disable UsageData/LargeTable
+ .order(created_at: :desc).limit(1).first&.id
+ end
+ end
+ end
+
+ private
+
+ def relation
+ super.imported_from(import_type) # rubocop: disable CodeReuse/ActiveRecord
+ end
+
+ def import_type
+ options[:import_type]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
index 34a8bfd08b5..a000b4509c6 100644
--- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
@@ -14,7 +14,14 @@ module Gitlab
# ::Issue.where(database_time_constraints)
# end
# end
+
+ UnimplementedOperationError = Class.new(StandardError) # rubocop:disable UsageData/InstrumentationSuperclass
+
class << self
+ IMPLEMENTED_OPERATIONS = %i(count distinct_count estimate_batch_distinct_count).freeze
+
+ private_constant :IMPLEMENTED_OPERATIONS
+
def start(&block)
return @metric_start&.call unless block_given?
@@ -40,6 +47,8 @@ module Gitlab
end
def operation(symbol, column: nil, &block)
+ raise UnimplementedOperationError unless symbol.in?(IMPLEMENTED_OPERATIONS)
+
@metric_operation = symbol
@column = column
@metric_operation_block = block if block_given?
@@ -82,6 +91,14 @@ module Gitlab
private
+ def start
+ self.class.metric_start&.call(time_constraints)
+ end
+
+ def finish
+ self.class.metric_finish&.call(time_constraints)
+ end
+
def relation
self.class.metric_relation.call.where(time_constraints)
end
@@ -100,19 +117,19 @@ module Gitlab
end
def get_or_cache_batch_ids
- return [self.class.start, self.class.finish] unless self.class.cache_key.present?
+ return [start, finish] unless self.class.cache_key.present?
key_name = "metric_instrumentation/#{self.class.cache_key}"
- start = Gitlab::Cache.fetch_once("#{key_name}_minimum_id", expires_in: 1.day) do
- self.class.start
+ cached_start = Gitlab::Cache.fetch_once("#{key_name}_minimum_id", expires_in: 1.day) do
+ start
end
- finish = Gitlab::Cache.fetch_once("#{key_name}_maximum_id", expires_in: 1.day) do
- self.class.finish
+ cached_finish = Gitlab::Cache.fetch_once("#{key_name}_maximum_id", expires_in: 1.day) do
+ finish
end
- [start, finish]
+ [cached_start, cached_finish]
end
end
end
diff --git a/lib/gitlab/usage/metrics/query.rb b/lib/gitlab/usage/metrics/query.rb
index f6947c4c8ff..851aa7a50e8 100644
--- a/lib/gitlab/usage/metrics/query.rb
+++ b/lib/gitlab/usage/metrics/query.rb
@@ -61,9 +61,31 @@ module Gitlab
end
# rubocop: enable CodeReuse/ActiveRecord
+ # rubocop: disable CodeReuse/ActiveRecord
def raw_sql(relation, column, distinct = false)
column ||= relation.primary_key
- relation.select(relation.all.table[column].count(distinct)).to_sql
+ node = node_to_count(relation, column)
+
+ relation.unscope(:order).select(node.count(distinct)).to_sql
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ def node_to_count(relation, column)
+ if join_relation?(relation) && joined_column?(column)
+ table_name, column_name = column.split(".")
+ Arel::Table.new(table_name)[column_name]
+ else
+ relation.all.table[column]
+ end
+ end
+
+ def join_relation?(relation)
+ relation.is_a?(ActiveRecord::Relation) && relation.joins_values.present?
+ end
+
+ # checks if the passed column is of format "table.column"
+ def joined_column?(column)
+ column.is_a?(String) && column.include?(".")
end
end
end
diff --git a/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb b/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb
new file mode 100644
index 00000000000..e32dcd3777b
--- /dev/null
+++ b/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module ServicePing
+ class LegacyMetricTimingDecorator < SimpleDelegator
+ attr_reader :duration
+
+ delegate :class, :is_a?, :kind_of?, to: :__getobj__
+
+ def initialize(value, duration)
+ @duration = duration
+ super(value)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/service_ping_report.rb b/lib/gitlab/usage/service_ping_report.rb
index 3e653b186a0..e73200cbd4a 100644
--- a/lib/gitlab/usage/service_ping_report.rb
+++ b/lib/gitlab/usage/service_ping_report.rb
@@ -3,6 +3,8 @@
module Gitlab
module Usage
class ServicePingReport
+ CACHE_KEY = 'usage_data'
+
class << self
def for(output:, cached: false)
case output.to_sym
@@ -26,7 +28,7 @@ module Gitlab
end
def all_metrics_values(cached)
- Rails.cache.fetch('usage_data', force: !cached, expires_in: 2.weeks) do
+ Rails.cache.fetch(CACHE_KEY, force: !cached, expires_in: 2.weeks) do
Gitlab::UsageData.data
end
end