diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-05-19 10:33:21 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-05-19 10:33:21 +0300 |
commit | 36a59d088eca61b834191dacea009677a96c052f (patch) | |
tree | e4f33972dab5d8ef79e3944a9f403035fceea43f /lib/gitlab/usage | |
parent | a1761f15ec2cae7c7f7bbda39a75494add0dfd6f (diff) |
Add latest changes from gitlab-org/gitlab@15-0-stable-eev15.0.0-rc42
Diffstat (limited to 'lib/gitlab/usage')
12 files changed, 206 insertions, 23 deletions
diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb index 24e044c5740..cf48aa49938 100644 --- a/lib/gitlab/usage/metric.rb +++ b/lib/gitlab/usage/metric.rb @@ -18,19 +18,25 @@ module Gitlab end def with_value - unflatten_key_path(intrumentation_object.value) + with_availability(proc { instrumentation_object.value }) end def with_instrumentation - unflatten_key_path(intrumentation_object.instrumentation) + with_availability(proc { instrumentation_object.instrumentation }) end def with_suggested_name - unflatten_key_path(intrumentation_object.suggested_name) + with_availability(proc { instrumentation_object.suggested_name }) end private + def with_availability(value_proc) + return {} unless instrumentation_object.available? + + unflatten_key_path(value_proc.call) + end + def unflatten_key_path(value) ::Gitlab::Usage::Metrics::KeyPathProcessor.process(definition.key_path, value) end @@ -39,8 +45,8 @@ module Gitlab "Gitlab::Usage::Metrics::Instrumentations::#{definition.instrumentation_class}" end - def intrumentation_object - instrumentation_class.constantize.new( + def instrumentation_object + @instrumentation_object ||= instrumentation_class.constantize.new( time_frame: definition.time_frame, options: definition.attributes[:options] ) diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 1031f38792b..2c50678c6bf 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -4,9 +4,9 @@ module Gitlab module Usage class MetricDefinition METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') - BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze - AVAILABLE_STATUSES = %w[active data_available implemented deprecated].freeze + AVAILABLE_STATUSES = %w[active data_available implemented deprecated].to_set.freeze + VALID_SERVICE_PING_STATUSES = %w[active data_available implemented deprecated broken].to_set.freeze InvalidError = Class.new(RuntimeError) @@ -26,20 +26,22 @@ module Gitlab attributes end + def json_schema + return unless has_json_schema? + + @json_schema ||= Gitlab::Json.parse(File.read(json_schema_path)) + end + def json_schema_path return '' unless has_json_schema? - "#{BASE_REPO_PATH}/#{attributes[:value_json_schema]}" + Rails.root.join(attributes[:value_json_schema]) end def has_json_schema? attributes[:value_type] == 'object' && attributes[:value_json_schema].present? end - def yaml_path - "#{BASE_REPO_PATH}#{path.delete_prefix(Rails.root.to_s)}" - end - def validate! unless skip_validation? self.class.schemer.validate(attributes.stringify_keys).each do |error| @@ -64,6 +66,10 @@ module Gitlab AVAILABLE_STATUSES.include?(attributes[:status]) end + def valid_service_ping_status? + VALID_SERVICE_PING_STATUSES.include?(attributes[:status]) + end + alias_method :to_dictionary, :to_h class << self diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 2545a505984..11e2fd22638 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -30,7 +30,7 @@ module Gitlab def aggregated_metrics_data(start_date:, end_date:, time_frame:) aggregated_metrics.each_with_object({}) do |aggregation, data| - next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: :yaml, type: :development) + next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], type: :development) next unless aggregation[:time_frame].include?(time_frame) case aggregation[:source] diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb index a264f9484f3..f76ed1753b2 100644 --- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -11,6 +11,18 @@ module Gitlab attr_reader :time_frame attr_reader :options + class << self + def available?(&block) + return @metric_available = block if block_given? + + return @metric_available.call if instance_variable_defined?('@metric_available') + + true + end + + attr_reader :metric_available + end + def initialize(time_frame:, options: {}) @time_frame = time_frame @options = options @@ -19,6 +31,10 @@ module Gitlab def instrumentation value end + + def available? + self.class.available? + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb index 6df6fef5d3a..d42250c9297 100644 --- a/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb @@ -6,7 +6,7 @@ module Gitlab module Instrumentations class CertBasedClustersFfMetric < GenericMetric value do - Feature.enabled?(:certificate_based_clusters, default_enabled: :yaml, type: :ops) + Feature.enabled?(:certificate_based_clusters, type: :ops) end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb index ee51180973c..51be4bf3ccf 100644 --- a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb @@ -6,7 +6,7 @@ module Gitlab module Instrumentations class CollectedDataCategoriesMetric < GenericMetric value do - ::ServicePing::PermitDataCategoriesService.new.execute.to_a + ::ServicePing::PermitDataCategories.new.execute.to_a end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb new file mode 100644 index 00000000000..c0d53b1b21a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountBulkImportsEntitiesMetric < DatabaseMetric + operation :count + + def initialize(time_frame:, options: {}) + super + + if source_type.present? && !source_type.in?(allowed_source_types) + raise ArgumentError, "source_type '#{source_type}' must be one of: #{allowed_source_types.join(', ')}" + end + end + + relation { ::BulkImports::Entity } + + private + + def relation + return super.where(source_type: source_type) if source_type.present? # rubocop: disable CodeReuse/ActiveRecord + + super + end + + def source_type + options[:source_type].to_s + end + + def allowed_source_types + BulkImports::Entity.source_types.keys.map(&:to_s) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb new file mode 100644 index 00000000000..c5498ce530f --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountImportedProjectsMetric < DatabaseMetric + operation :count + + def initialize(time_frame:, options: {}) + super + + raise ArgumentError, "import_type options attribute is required" unless import_type.present? + end + + relation { ::Project } + + start do |time_constraints| + unless time_constraints.nil? + start = time_constraints[:created_at]&.first + + unless start.nil? + ::Project + .select(:id) + .where(Project.arel_table[:created_at].gteq(start)) # rubocop:disable UsageData/LargeTable + .order(created_at: :asc).limit(1).first&.id + end + end + end + + finish do |time_constraints| + unless time_constraints.nil? + finish = time_constraints[:created_at]&.last + + unless finish.nil? + ::Project + .select(:id) + .where(Project.arel_table[:created_at].lteq(finish)) # rubocop:disable UsageData/LargeTable + .order(created_at: :desc).limit(1).first&.id + end + end + end + + private + + def relation + super.imported_from(import_type) # rubocop: disable CodeReuse/ActiveRecord + end + + def import_type + options[:import_type] + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index 34a8bfd08b5..a000b4509c6 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -14,7 +14,14 @@ module Gitlab # ::Issue.where(database_time_constraints) # end # end + + UnimplementedOperationError = Class.new(StandardError) # rubocop:disable UsageData/InstrumentationSuperclass + class << self + IMPLEMENTED_OPERATIONS = %i(count distinct_count estimate_batch_distinct_count).freeze + + private_constant :IMPLEMENTED_OPERATIONS + def start(&block) return @metric_start&.call unless block_given? @@ -40,6 +47,8 @@ module Gitlab end def operation(symbol, column: nil, &block) + raise UnimplementedOperationError unless symbol.in?(IMPLEMENTED_OPERATIONS) + @metric_operation = symbol @column = column @metric_operation_block = block if block_given? @@ -82,6 +91,14 @@ module Gitlab private + def start + self.class.metric_start&.call(time_constraints) + end + + def finish + self.class.metric_finish&.call(time_constraints) + end + def relation self.class.metric_relation.call.where(time_constraints) end @@ -100,19 +117,19 @@ module Gitlab end def get_or_cache_batch_ids - return [self.class.start, self.class.finish] unless self.class.cache_key.present? + return [start, finish] unless self.class.cache_key.present? key_name = "metric_instrumentation/#{self.class.cache_key}" - start = Gitlab::Cache.fetch_once("#{key_name}_minimum_id", expires_in: 1.day) do - self.class.start + cached_start = Gitlab::Cache.fetch_once("#{key_name}_minimum_id", expires_in: 1.day) do + start end - finish = Gitlab::Cache.fetch_once("#{key_name}_maximum_id", expires_in: 1.day) do - self.class.finish + cached_finish = Gitlab::Cache.fetch_once("#{key_name}_maximum_id", expires_in: 1.day) do + finish end - [start, finish] + [cached_start, cached_finish] end end end diff --git a/lib/gitlab/usage/metrics/query.rb b/lib/gitlab/usage/metrics/query.rb index f6947c4c8ff..851aa7a50e8 100644 --- a/lib/gitlab/usage/metrics/query.rb +++ b/lib/gitlab/usage/metrics/query.rb @@ -61,9 +61,31 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + # rubocop: disable CodeReuse/ActiveRecord def raw_sql(relation, column, distinct = false) column ||= relation.primary_key - relation.select(relation.all.table[column].count(distinct)).to_sql + node = node_to_count(relation, column) + + relation.unscope(:order).select(node.count(distinct)).to_sql + end + # rubocop: enable CodeReuse/ActiveRecord + + def node_to_count(relation, column) + if join_relation?(relation) && joined_column?(column) + table_name, column_name = column.split(".") + Arel::Table.new(table_name)[column_name] + else + relation.all.table[column] + end + end + + def join_relation?(relation) + relation.is_a?(ActiveRecord::Relation) && relation.joins_values.present? + end + + # checks if the passed column is of format "table.column" + def joined_column?(column) + column.is_a?(String) && column.include?(".") end end end diff --git a/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb b/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb new file mode 100644 index 00000000000..e32dcd3777b --- /dev/null +++ b/lib/gitlab/usage/service_ping/legacy_metric_timing_decorator.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module ServicePing + class LegacyMetricTimingDecorator < SimpleDelegator + attr_reader :duration + + delegate :class, :is_a?, :kind_of?, to: :__getobj__ + + def initialize(value, duration) + @duration = duration + super(value) + end + end + end + end +end diff --git a/lib/gitlab/usage/service_ping_report.rb b/lib/gitlab/usage/service_ping_report.rb index 3e653b186a0..e73200cbd4a 100644 --- a/lib/gitlab/usage/service_ping_report.rb +++ b/lib/gitlab/usage/service_ping_report.rb @@ -3,6 +3,8 @@ module Gitlab module Usage class ServicePingReport + CACHE_KEY = 'usage_data' + class << self def for(output:, cached: false) case output.to_sym @@ -26,7 +28,7 @@ module Gitlab end def all_metrics_values(cached) - Rails.cache.fetch('usage_data', force: !cached, expires_in: 2.weeks) do + Rails.cache.fetch(CACHE_KEY, force: !cached, expires_in: 2.weeks) do Gitlab::UsageData.data end end |