Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2021-06-07 15:10:00 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2021-06-07 15:10:00 +0300
commitde8e5077c3671b0b29642faf1b5e562bc4f99453 (patch)
tree315d59367b7ff609ed4293f369c14be9e7e91cba /lib
parentf4c6fbb86fbec3e5917e317b3490232d98531881 (diff)
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r--lib/api/composer_packages.rb4
-rw-r--r--lib/api/concerns/packages/nuget_endpoints.rb5
-rw-r--r--lib/api/debian_project_packages.rb2
-rw-r--r--lib/api/group_container_repositories.rb2
-rw-r--r--lib/api/helm_packages.rb2
-rw-r--r--lib/api/maven_packages.rb8
-rw-r--r--lib/api/npm_project_packages.rb4
-rw-r--r--lib/api/nuget_group_packages.rb4
-rw-r--r--lib/api/nuget_project_packages.rb8
-rw-r--r--lib/api/project_container_repositories.rb10
-rw-r--r--lib/api/pypi_packages.rb6
-rw-r--r--lib/api/rubygem_packages.rb4
-rw-r--r--lib/api/terraform/modules/v1/packages.rb4
-rw-r--r--lib/feature.rb7
-rw-r--r--lib/feature/active_support_cache_store_adapter.rb36
-rw-r--r--lib/gitlab/emoji.rb12
-rw-r--r--lib/gitlab/health_checks/redis/redis_check.rb3
-rw-r--r--lib/gitlab/health_checks/redis/trace_chunks_check.rb35
-rw-r--r--lib/gitlab/instrumentation/redis.rb3
-rw-r--r--lib/gitlab/redis/trace_chunks.rb12
-rw-r--r--lib/gitlab/redis/wrapper.rb15
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/database_metric.rb8
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/generic_metric.rb9
-rw-r--r--lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb11
-rw-r--r--lib/gitlab/usage/metrics/name_suggestion.rb200
-rw-r--r--lib/gitlab/usage/metrics/names_suggestions/generator.rb182
-rw-r--r--lib/gitlab/usage_data_counters/known_events/ecosystem.yml4
27 files changed, 290 insertions, 310 deletions
diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb
index 115a6b8ac4f..7b3750b37ee 100644
--- a/lib/api/composer_packages.rb
+++ b/lib/api/composer_packages.rb
@@ -137,7 +137,7 @@ module API
bad_request!
end
- track_package_event('push_package', :composer)
+ track_package_event('push_package', :composer, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace)
::Packages::Composer::CreatePackageService
.new(authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job))
@@ -161,7 +161,7 @@ module API
not_found! unless metadata
- track_package_event('pull_package', :composer)
+ track_package_event('pull_package', :composer, project: unauthorized_user_project, namespace: unauthorized_user_project.namespace)
send_git_archive unauthorized_user_project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true
end
diff --git a/lib/api/concerns/packages/nuget_endpoints.rb b/lib/api/concerns/packages/nuget_endpoints.rb
index 5364eeb1880..208daeb3037 100644
--- a/lib/api/concerns/packages/nuget_endpoints.rb
+++ b/lib/api/concerns/packages/nuget_endpoints.rb
@@ -58,7 +58,8 @@ module API
end
get 'index', format: :json do
authorize_read_package!(project_or_group)
- track_package_event('cli_metadata', :nuget, category: 'API::NugetPackages')
+
+ track_package_event('cli_metadata', :nuget, **snowplow_gitlab_standard_context.merge(category: 'API::NugetPackages'))
present ::Packages::Nuget::ServiceIndexPresenter.new(project_or_group),
with: ::API::Entities::Nuget::ServiceIndex
@@ -117,7 +118,7 @@ module API
results = search_packages(params[:q], search_options)
- track_package_event('search_package', :nuget, category: 'API::NugetPackages')
+ track_package_event('search_package', :nuget, **snowplow_gitlab_standard_context.merge(category: 'API::NugetPackages'))
present ::Packages::Nuget::SearchResultsPresenter.new(results),
with: ::API::Entities::Nuget::SearchResults
diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb
index feb83b52695..eadb0646a67 100644
--- a/lib/api/debian_project_packages.rb
+++ b/lib/api/debian_project_packages.rb
@@ -35,7 +35,7 @@ module API
authorize_upload!(authorized_user_project)
bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:debian_max_file_size, params[:file].size)
- track_package_event('push_package', :debian)
+ track_package_event('push_package', :debian, user: current_user, project: authorized_user_project, namespace: authorized_user_project.namespace)
file_params = {
file: params['file'],
diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb
index 4fede0ad583..96175f31696 100644
--- a/lib/api/group_container_repositories.rb
+++ b/lib/api/group_container_repositories.rb
@@ -31,7 +31,7 @@ module API
user: current_user, subject: user_group
).execute
- track_package_event('list_repositories', :container)
+ track_package_event('list_repositories', :container, user: current_user, namespace: user_group)
present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count]
end
diff --git a/lib/api/helm_packages.rb b/lib/api/helm_packages.rb
index 446dcaf1e3a..dc5630a1395 100644
--- a/lib/api/helm_packages.rb
+++ b/lib/api/helm_packages.rb
@@ -46,7 +46,7 @@ module API
package_file = Packages::Helm::PackageFilesFinder.new(authorized_user_project, params[:channel], file_name: "#{params[:file_name]}.tgz").execute.last!
- track_package_event('pull_package', :helm)
+ track_package_event('pull_package', :helm, project: authorized_user_project, namespace: authorized_user_project.namespace)
present_carrierwave_file!(package_file.file)
end
diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb
index cdce92d5c46..9e5705abe88 100644
--- a/lib/api/maven_packages.rb
+++ b/lib/api/maven_packages.rb
@@ -130,7 +130,7 @@ module API
when 'sha1'
package_file.file_sha1
else
- track_package_event('pull_package', :maven) if jar_file?(format)
+ track_package_event('pull_package', :maven, project: project, namespace: project.namespace) if jar_file?(format)
present_carrierwave_file_with_head_support!(package_file.file)
end
end
@@ -170,7 +170,7 @@ module API
when 'sha1'
package_file.file_sha1
else
- track_package_event('pull_package', :maven) if jar_file?(format)
+ track_package_event('pull_package', :maven, project: package.project, namespace: package.project.namespace) if jar_file?(format)
present_carrierwave_file_with_head_support!(package_file.file)
end
@@ -208,7 +208,7 @@ module API
when 'sha1'
package_file.file_sha1
else
- track_package_event('pull_package', :maven) if jar_file?(format)
+ track_package_event('pull_package', :maven, project: user_project, namespace: user_project.namespace) if jar_file?(format)
present_carrierwave_file_with_head_support!(package_file.file)
end
@@ -264,7 +264,7 @@ module API
when 'md5'
''
else
- track_package_event('push_package', :maven) if jar_file?(format)
+ track_package_event('push_package', :maven, user: current_user, project: user_project, namespace: user_project.namespace) if jar_file?(format)
file_params = {
file: params[:file],
diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb
index 887084dc9ae..7ff4439ce04 100644
--- a/lib/api/npm_project_packages.rb
+++ b/lib/api/npm_project_packages.rb
@@ -32,7 +32,7 @@ module API
package_file = ::Packages::PackageFileFinder
.new(package, params[:file_name]).execute!
- track_package_event('pull_package', package, category: 'API::NpmPackages')
+ track_package_event('pull_package', package, category: 'API::NpmPackages', project: project, namespace: project.namespace)
present_carrierwave_file!(package_file.file)
end
@@ -48,7 +48,7 @@ module API
put ':package_name', requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do
authorize_create_package!(project)
- track_package_event('push_package', :npm, category: 'API::NpmPackages')
+ track_package_event('push_package', :npm, category: 'API::NpmPackages', project: project, user: current_user, namespace: project.namespace)
created_package = ::Packages::Npm::CreatePackageService
.new(project, current_user, params.merge(build: current_authenticated_job)).execute
diff --git a/lib/api/nuget_group_packages.rb b/lib/api/nuget_group_packages.rb
index a80de06d6b0..eb55e4cbf70 100644
--- a/lib/api/nuget_group_packages.rb
+++ b/lib/api/nuget_group_packages.rb
@@ -38,6 +38,10 @@ module API
def require_authenticated!
unauthorized! unless current_user
end
+
+ def snowplow_gitlab_standard_context
+ { namespace: find_authorized_group! }
+ end
end
params do
diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb
index 73ecc140959..5bae08d4dae 100644
--- a/lib/api/nuget_project_packages.rb
+++ b/lib/api/nuget_project_packages.rb
@@ -36,6 +36,10 @@ module API
def project_or_group
authorized_user_project
end
+
+ def snowplow_gitlab_standard_context
+ { project: authorized_user_project, namespace: authorized_user_project.namespace }
+ end
end
params do
@@ -69,7 +73,7 @@ module API
package_file = ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job))
.execute
- track_package_event('push_package', :nuget, category: 'API::NugetPackages')
+ track_package_event('push_package', :nuget, category: 'API::NugetPackages', user: current_user, project: package.project, namespace: package.project.namespace)
::Packages::Nuget::ExtractionWorker.perform_async(package_file.id) # rubocop:disable CodeReuse/Worker
@@ -118,7 +122,7 @@ module API
not_found!('Package') unless package_file
- track_package_event('pull_package', :nuget, category: 'API::NugetPackages')
+ track_package_event('pull_package', :nuget, category: 'API::NugetPackages', project: package_file.project, namespace: package_file.project.namespace)
# nuget and dotnet don't support 302 Moved status codes, supports_direct_download has to be set to false
present_carrierwave_file!(package_file.file, supports_direct_download: false)
diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb
index 2580f7adbc9..28cfa9e3ae0 100644
--- a/lib/api/project_container_repositories.rb
+++ b/lib/api/project_container_repositories.rb
@@ -31,7 +31,7 @@ module API
user: current_user, subject: user_project
).execute
- track_package_event('list_repositories', :container)
+ track_package_event('list_repositories', :container, user: current_user, project: user_project, namespace: user_project.namespace)
present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count]
end
@@ -46,7 +46,7 @@ module API
authorize_admin_container_image!
DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) # rubocop:disable CodeReuse/Worker
- track_package_event('delete_repository', :container)
+ track_package_event('delete_repository', :container, user: current_user, project: user_project, namespace: user_project.namespace)
status :accepted
end
@@ -63,7 +63,7 @@ module API
authorize_read_container_image!
tags = Kaminari.paginate_array(repository.tags)
- track_package_event('list_tags', :container)
+ track_package_event('list_tags', :container, user: current_user, project: user_project, namespace: user_project.namespace)
present paginate(tags), with: Entities::ContainerRegistry::Tag
end
@@ -92,7 +92,7 @@ module API
declared_params.except(:repository_id).merge(container_expiration_policy: false))
# rubocop:enable CodeReuse/Worker
- track_package_event('delete_tag_bulk', :container)
+ track_package_event('delete_tag_bulk', :container, user: current_user, project: user_project, namespace: user_project.namespace)
status :accepted
end
@@ -128,7 +128,7 @@ module API
.execute(repository)
if result[:status] == :success
- track_package_event('delete_tag', :container)
+ track_package_event('delete_tag', :container, user: current_user, project: user_project, namespace: user_project.namespace)
status :ok
else
diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb
index 969b619c1cd..7c5f8bb4d99 100644
--- a/lib/api/pypi_packages.rb
+++ b/lib/api/pypi_packages.rb
@@ -121,7 +121,7 @@ module API
package = Packages::Pypi::PackageFinder.new(current_user, project, { filename: filename, sha256: params[:sha256] }).execute
package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute
- track_package_event('pull_package', :pypi)
+ track_package_event('pull_package', :pypi, project: project, namespace: project.namespace)
present_carrierwave_file!(package_file.file, supports_direct_download: true)
end
@@ -140,7 +140,7 @@ module API
get 'simple/*package_name', format: :txt do
authorize_read_package!(authorized_user_project)
- track_package_event('list_package', :pypi)
+ track_package_event('list_package', :pypi, project: authorized_user_project, namespace: authorized_user_project.namespace)
packages = Packages::Pypi::PackagesFinder.new(current_user, authorized_user_project, { package_name: params[:package_name] }).execute!
presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project)
@@ -171,7 +171,7 @@ module API
authorize_upload!(authorized_user_project)
bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:pypi_max_file_size, params[:content].size)
- track_package_event('push_package', :pypi)
+ track_package_event('push_package', :pypi, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace)
::Packages::Pypi::CreatePackageService
.new(authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job))
diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb
index 1d17148e0df..d7f9c584c67 100644
--- a/lib/api/rubygem_packages.rb
+++ b/lib/api/rubygem_packages.rb
@@ -70,7 +70,7 @@ module API
user_project, params[:file_name]
).last!
- track_package_event('pull_package', :rubygems)
+ track_package_event('pull_package', :rubygems, project: user_project, namespace: user_project.namespace)
present_carrierwave_file!(package_file.file)
end
@@ -97,7 +97,7 @@ module API
authorize_upload!(user_project)
bad_request!('File is too large') if user_project.actual_limits.exceeded?(:rubygems_max_file_size, params[:file].size)
- track_package_event('push_package', :rubygems)
+ track_package_event('push_package', :rubygems, user: current_user, project: user_project, namespace: user_project.namespace)
package_file = nil
diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/packages.rb
index 34e77e09800..aa59b6a4fee 100644
--- a/lib/api/terraform/modules/v1/packages.rb
+++ b/lib/api/terraform/modules/v1/packages.rb
@@ -124,7 +124,7 @@ module API
end
get do
- track_package_event('pull_package', :terraform_module)
+ track_package_event('pull_package', :terraform_module, project: package.project, namespace: module_namespace, user: current_user)
present_carrierwave_file!(package_file.file)
end
@@ -183,7 +183,7 @@ module API
render_api_error!(result[:message], result[:http_status]) if result[:status] == :error
- track_package_event('push_package', :terraform_module)
+ track_package_event('push_package', :terraform_module, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace)
created!
rescue ObjectStorage::RemoteStoreError => e
diff --git a/lib/feature.rb b/lib/feature.rb
index 87abd2689d0..453ecc8255a 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -18,6 +18,10 @@ class Feature
superclass.table_name = 'feature_gates'
end
+ # To enable EE overrides
+ class ActiveSupportCacheStoreAdapter < Flipper::Adapters::ActiveSupportCacheStore
+ end
+
InvalidFeatureFlagError = Class.new(Exception) # rubocop:disable Lint/InheritException
class << self
@@ -167,7 +171,8 @@ class Feature
ActiveSupportCacheStoreAdapter.new(
active_record_adapter,
l2_cache_backend,
- expires_in: 1.hour)
+ expires_in: 1.hour,
+ write_through: true)
# Thread-local L1 cache: use a short timeout since we don't have a
# way to expire this cache all at once
diff --git a/lib/feature/active_support_cache_store_adapter.rb b/lib/feature/active_support_cache_store_adapter.rb
deleted file mode 100644
index 431f1169a86..00000000000
--- a/lib/feature/active_support_cache_store_adapter.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-# rubocop:disable Gitlab/NamespacedClass
-# This class was already nested this way before moving to a separate file
-class Feature
- class ActiveSupportCacheStoreAdapter < Flipper::Adapters::ActiveSupportCacheStore
- # This patch represents https://github.com/jnunemaker/flipper/pull/512. In
- # Flipper 0.21.0 and later, we can remove this and just pass `write_through:
- # true` to the constructor in `Feature.build_flipper_instance`.
-
- extend ::Gitlab::Utils::Override
-
- override :enable
- def enable(feature, gate, thing)
- result = @adapter.enable(feature, gate, thing)
- @cache.write(key_for(feature.key), @adapter.get(feature), @write_options)
- result
- end
-
- override :disable
- def disable(feature, gate, thing)
- result = @adapter.disable(feature, gate, thing)
- @cache.write(key_for(feature.key), @adapter.get(feature), @write_options)
- result
- end
-
- override :remove
- def remove(feature)
- result = @adapter.remove(feature)
- @cache.delete(FeaturesKey)
- @cache.write(key_for(feature.key), {}, @write_options)
- result
- end
- end
-end
-# rubocop:disable Gitlab/NamespacedClass
diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb
index e6f71e3ad3c..2b5f465d3c5 100644
--- a/lib/gitlab/emoji.rb
+++ b/lib/gitlab/emoji.rb
@@ -41,7 +41,17 @@ module Gitlab
end
def emoji_image_tag(name, src)
- "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{src}' height='20' width='20' align='absmiddle' />"
+ image_options = {
+ class: 'emoji',
+ src: src,
+ title: ":#{name}:",
+ alt: ":#{name}:",
+ height: 20,
+ width: 20,
+ align: 'absmiddle'
+ }
+
+ ActionController::Base.helpers.tag(:img, image_options)
end
def emoji_exists?(name)
diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb
index 44b85bf886e..f7e46fce134 100644
--- a/lib/gitlab/health_checks/redis/redis_check.rb
+++ b/lib/gitlab/health_checks/redis/redis_check.rb
@@ -20,8 +20,7 @@ module Gitlab
def check
::Gitlab::HealthChecks::Redis::CacheCheck.check_up &&
::Gitlab::HealthChecks::Redis::QueuesCheck.check_up &&
- ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up &&
- ::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up
+ ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up
end
end
end
diff --git a/lib/gitlab/health_checks/redis/trace_chunks_check.rb b/lib/gitlab/health_checks/redis/trace_chunks_check.rb
deleted file mode 100644
index cf9fa700b0a..00000000000
--- a/lib/gitlab/health_checks/redis/trace_chunks_check.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module HealthChecks
- module Redis
- class TraceChunksCheck
- extend SimpleAbstractCheck
-
- class << self
- def check_up
- check
- end
-
- private
-
- def metric_prefix
- 'redis_trace_chunks_ping'
- end
-
- def successful?(result)
- result == 'PONG'
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def check
- catch_timeout 10.seconds do
- Gitlab::Redis::TraceChunks.with(&:ping)
- end
- end
- # rubocop: enable CodeReuse/ActiveRecord
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb
index ab0e56adc32..9a9d3a866b1 100644
--- a/lib/gitlab/instrumentation/redis.rb
+++ b/lib/gitlab/instrumentation/redis.rb
@@ -8,9 +8,8 @@ module Gitlab
Cache = Class.new(RedisBase).enable_redis_cluster_validation
Queues = Class.new(RedisBase)
SharedState = Class.new(RedisBase).enable_redis_cluster_validation
- TraceChunks = Class.new(RedisBase).enable_redis_cluster_validation
- STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks].freeze
+ STORAGES = [ActionCable, Cache, Queues, SharedState].freeze
# Milliseconds represented in seconds (from 1 millisecond to 2 seconds).
QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze
diff --git a/lib/gitlab/redis/trace_chunks.rb b/lib/gitlab/redis/trace_chunks.rb
deleted file mode 100644
index a2e77cb5df5..00000000000
--- a/lib/gitlab/redis/trace_chunks.rb
+++ /dev/null
@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Redis
- class TraceChunks < ::Gitlab::Redis::Wrapper
- # The data we store on TraceChunks used to be stored on SharedState.
- def self.config_fallback
- SharedState
- end
- end
- end
-end
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
index fa4e7f387ed..32447d39c02 100644
--- a/lib/gitlab/redis/wrapper.rb
+++ b/lib/gitlab/redis/wrapper.rb
@@ -64,19 +64,8 @@ module Gitlab
def config_file_name
[
- # Instance specific config sources:
ENV["GITLAB_REDIS_#{store_name.underscore.upcase}_CONFIG_FILE"],
config_file_path("redis.#{store_name.underscore}.yml"),
-
- # The current Redis instance may have been split off from another one
- # (e.g. TraceChunks was split off from SharedState). There are
- # installations out there where the lowest priority config source
- # (resque.yml) contains bogus values. In those cases, config_file_name
- # should resolve to the instance we originated from (the
- # "config_fallback") rather than resque.yml.
- config_fallback&.config_file_name,
-
- # Global config sources:
ENV['GITLAB_REDIS_CONFIG_FILE'],
config_file_path('resque.yml')
].compact.first
@@ -86,10 +75,6 @@ module Gitlab
name.demodulize
end
- def config_fallback
- nil
- end
-
def instrumentation_class
"::Gitlab::Instrumentation::Redis::#{store_name}".constantize
end
diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
index 012727dd475..69a288e5b6e 100644
--- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb
@@ -47,6 +47,14 @@ module Gitlab
Gitlab::Usage::Metrics::Query.for(self.class.metric_operation, relation, self.class.column)
end
+ def suggested_name
+ Gitlab::Usage::Metrics::NameSuggestion.for(
+ self.class.metric_operation,
+ relation: relation,
+ column: self.class.column
+ )
+ end
+
private
def relation
diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb
index 7c97cc37d17..1849773e33d 100644
--- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb
@@ -13,6 +13,9 @@ module Gitlab
# end
# end
class << self
+ attr_reader :metric_operation
+ @metric_operation = :alt
+
def value(&block)
@metric_value = block
end
@@ -25,6 +28,12 @@ module Gitlab
self.class.metric_value.call
end
end
+
+ def suggested_name
+ Gitlab::Usage::Metrics::NameSuggestion.for(
+ self.class.metric_operation
+ )
+ end
end
end
end
diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb
index 502a8147473..a36e612a1cb 100644
--- a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb
+++ b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb
@@ -12,6 +12,11 @@ module Gitlab
# events:
# - g_analytics_valuestream
# end
+ class << self
+ attr_reader :metric_operation
+ @metric_operation = :redis
+ end
+
def initialize(time_frame:, options: {})
super
@@ -30,6 +35,12 @@ module Gitlab
end
end
+ def suggested_name
+ Gitlab::Usage::Metrics::NameSuggestion.for(
+ self.class.metric_operation
+ )
+ end
+
private
def time_constraints
diff --git a/lib/gitlab/usage/metrics/name_suggestion.rb b/lib/gitlab/usage/metrics/name_suggestion.rb
new file mode 100644
index 00000000000..0728af9e2ca
--- /dev/null
+++ b/lib/gitlab/usage/metrics/name_suggestion.rb
@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ class NameSuggestion
+ FREE_TEXT_METRIC_NAME = "<please fill metric name>"
+ REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>"
+ CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>"
+
+ class << self
+ def for(operation, relation: nil, column: nil)
+ case operation
+ when :count
+ name_suggestion(column: column, relation: relation, prefix: 'count')
+ when :distinct_count
+ name_suggestion(column: column, relation: relation, prefix: 'count_distinct', distinct: :distinct)
+ when :estimate_batch_distinct_count
+ name_suggestion(column: column, relation: relation, prefix: 'estimate_distinct_count')
+ when :sum
+ name_suggestion(column: column, relation: relation, prefix: 'sum')
+ when :redis
+ REDIS_EVENT_METRIC_NAME
+ when :alt
+ FREE_TEXT_METRIC_NAME
+ else
+ raise ArgumentError, "#{operation} operation not supported"
+ end
+ end
+
+ private
+
+ def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil)
+ # rubocop: disable CodeReuse/ActiveRecord
+ relation = relation.unscope(where: :created_at)
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ parts = [prefix]
+ arel_column = arelize_column(relation, column)
+
+ # nil as column indicates that the counting would use fallback value of primary key.
+ # Because counting primary key from relation is the conceptual equal to counting all
+ # records from given relation, in order to keep name suggestion more condensed
+ # primary key column is skipped.
+ # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not
+ # as count_id_from_issues since it does not add more information to the name suggestion
+ if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key]
+ parts << arel_column.name
+ parts << 'from'
+ end
+
+ arel = arel_query(relation: relation, column: arel_column, distinct: distinct)
+ constraints = parse_constraints(relation: relation, arel: arel)
+
+ # In some cases due to performance reasons metrics are instrumented with joined relations
+ # where relation listed in FROM statement is not the one that includes counted attribute
+ # in such situations to make name suggestion more intuitive source should be inferred based
+ # on the relation that provide counted attribute
+ # EG: SELECT COUNT(deployments.environment_id) FROM clusters
+ # JOIN deployments ON deployments.cluster_id = cluster.id
+ # should be translated into:
+ # count_environment_id_from_deployments_with_clusters
+ # instead of
+ # count_environment_id_from_clusters_with_deployments
+ actual_source = parse_source(relation, arel_column)
+
+ append_constraints_prompt(actual_source, [constraints], parts)
+
+ parts << actual_source
+ parts += process_joined_relations(actual_source, arel, relation, constraints)
+ parts.compact.join('_').delete('"')
+ end
+
+ def append_constraints_prompt(target, constraints, parts)
+ applicable_constraints = constraints.select { |constraint| constraint.include?(target) }
+ return unless applicable_constraints.any?
+
+ parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') }
+ end
+
+ def parse_constraints(relation:, arel:)
+ connection = relation.connection
+ ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints
+ .new(connection)
+ .accept(arel, collector(connection))
+ .value
+ end
+
+ # TODO: joins with `USING` keyword
+ def process_joined_relations(actual_source, arel, relation, where_constraints)
+ joins = parse_joins(connection: relation.connection, arel: arel)
+ return [] unless joins.any?
+
+ sources = [relation.table_name, *joins.map { |join| join[:source] }]
+ joins = extract_joins_targets(joins, sources)
+
+ relations = if actual_source != relation.table_name
+ build_relations_tree(joins + [{ source: relation.table_name }], actual_source)
+ else
+ # in case where counter attribute comes from joined relations, the relations
+ # diagram has to be built bottom up, thus source and target are reverted
+ build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source)
+ end
+
+ collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints)
+ end
+
+ def parse_joins(connection:, arel:)
+ ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins
+ .new(connection)
+ .accept(arel)
+ end
+
+ def extract_joins_targets(joins, sources)
+ joins.map do |join|
+ source_regex = /(#{join[:source]})\.(\w+_)*id/i
+
+ tables_except_src = (sources - [join[:source]]).join('|')
+ target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i
+
+ join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i
+ matched = join_cond_regex.match(join[:constraints])
+
+ if matched
+ join[:target] = matched[:target]
+ join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '')
+ end
+
+ join
+ end
+ end
+
+ def build_relations_tree(joins, parent, source_key: :source, target_key: :target)
+ return [] if joins.blank?
+
+ tree = {}
+ tree[parent] = []
+
+ joins.each do |join|
+ if join[source_key] == parent
+ tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key)
+ end
+ end
+ tree
+ end
+
+ def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle)
+ conjunction = conjunctions.next
+ relations.each do |subtree|
+ subtree.each do |parent, children|
+ parts << "<#{conjunction}>"
+ join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints)
+ append_constraints_prompt(parent, [wheres, join_constraints].compact, parts)
+ parts << parent
+ collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions)
+ end
+ end
+ parts
+ end
+
+ def arelize_column(relation, column)
+ case column
+ when Arel::Attribute
+ column
+ when NilClass
+ Arel::Table.new(relation.table_name)[relation.primary_key]
+ when String
+ if column.include?('.')
+ table, col = column.split('.')
+ Arel::Table.new(table)[col]
+ else
+ Arel::Table.new(relation.table_name)[column]
+ end
+ when Symbol
+ arelize_column(relation, column.to_s)
+ end
+ end
+
+ def parse_source(relation, column)
+ column.relation.name || relation.table_name
+ end
+
+ def collector(connection)
+ Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new)
+ end
+
+ def arel_query(relation:, column: nil, distinct: nil)
+ column ||= relation.primary_key
+
+ if column.is_a?(Arel::Attribute)
+ relation.select(column.count(distinct)).arel
+ else
+ relation.select(relation.all.table[column].count(distinct)).arel
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
index 49581169452..a669b43f395 100644
--- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb
+++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
@@ -5,10 +5,6 @@ module Gitlab
module Metrics
module NamesSuggestions
class Generator < ::Gitlab::UsageData
- FREE_TEXT_METRIC_NAME = "<please fill metric name>"
- REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>"
- CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>"
-
class << self
def generate(key_path)
uncached_data.deep_stringify_keys.dig(*key_path.split('.'))
@@ -17,200 +13,36 @@ module Gitlab
private
def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
- name_suggestion(column: column, relation: relation, prefix: 'count')
+ Gitlab::Usage::Metrics::NameSuggestion.for(:count, column: column, relation: relation)
end
def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
- name_suggestion(column: column, relation: relation, prefix: 'count_distinct', distinct: :distinct)
+ Gitlab::Usage::Metrics::NameSuggestion.for(:distinct_count, column: column, relation: relation)
end
def redis_usage_counter
- REDIS_EVENT_METRIC_NAME
+ Gitlab::Usage::Metrics::NameSuggestion.for(:redis)
end
def alt_usage_data(*)
- FREE_TEXT_METRIC_NAME
+ Gitlab::Usage::Metrics::NameSuggestion.for(:alt)
end
def redis_usage_data_totals(counter)
- counter.fallback_totals.transform_values { |_| REDIS_EVENT_METRIC_NAME }
+ counter.fallback_totals.transform_values { |_| Gitlab::Usage::Metrics::NameSuggestion.for(:redis) }
end
def sum(relation, column, *rest)
- name_suggestion(column: column, relation: relation, prefix: 'sum')
+ Gitlab::Usage::Metrics::NameSuggestion.for(:sum, column: column, relation: relation)
end
def estimate_batch_distinct_count(relation, column = nil, *rest)
- name_suggestion(column: column, relation: relation, prefix: 'estimate_distinct_count')
+ Gitlab::Usage::Metrics::NameSuggestion.for(:estimate_batch_distinct_count, column: column, relation: relation)
end
def add(*args)
"add_#{args.join('_and_')}"
end
-
- def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil)
- # rubocop: disable CodeReuse/ActiveRecord
- relation = relation.unscope(where: :created_at)
- # rubocop: enable CodeReuse/ActiveRecord
-
- parts = [prefix]
- arel_column = arelize_column(relation, column)
-
- # nil as column indicates that the counting would use fallback value of primary key.
- # Because counting primary key from relation is the conceptual equal to counting all
- # records from given relation, in order to keep name suggestion more condensed
- # primary key column is skipped.
- # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not
- # as count_id_from_issues since it does not add more information to the name suggestion
- if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key]
- parts << arel_column.name
- parts << 'from'
- end
-
- arel = arel_query(relation: relation, column: arel_column, distinct: distinct)
- constraints = parse_constraints(relation: relation, arel: arel)
-
- # In some cases due to performance reasons metrics are instrumented with joined relations
- # where relation listed in FROM statement is not the one that includes counted attribute
- # in such situations to make name suggestion more intuitive source should be inferred based
- # on the relation that provide counted attribute
- # EG: SELECT COUNT(deployments.environment_id) FROM clusters
- # JOIN deployments ON deployments.cluster_id = cluster.id
- # should be translated into:
- # count_environment_id_from_deployments_with_clusters
- # instead of
- # count_environment_id_from_clusters_with_deployments
- actual_source = parse_source(relation, arel_column)
-
- append_constraints_prompt(actual_source, [constraints], parts)
-
- parts << actual_source
- parts += process_joined_relations(actual_source, arel, relation, constraints)
- parts.compact.join('_').delete('"')
- end
-
- def append_constraints_prompt(target, constraints, parts)
- applicable_constraints = constraints.select { |constraint| constraint.include?(target) }
- return unless applicable_constraints.any?
-
- parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') }
- end
-
- def parse_constraints(relation:, arel:)
- connection = relation.connection
- ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints
- .new(connection)
- .accept(arel, collector(connection))
- .value
- end
-
- # TODO: joins with `USING` keyword
- def process_joined_relations(actual_source, arel, relation, where_constraints)
- joins = parse_joins(connection: relation.connection, arel: arel)
- return [] unless joins.any?
-
- sources = [relation.table_name, *joins.map { |join| join[:source] }]
- joins = extract_joins_targets(joins, sources)
-
- relations = if actual_source != relation.table_name
- build_relations_tree(joins + [{ source: relation.table_name }], actual_source)
- else
- # in case where counter attribute comes from joined relations, the relations
- # diagram has to be built bottom up, thus source and target are reverted
- build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source)
- end
-
- collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints)
- end
-
- def parse_joins(connection:, arel:)
- ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins
- .new(connection)
- .accept(arel)
- end
-
- def extract_joins_targets(joins, sources)
- joins.map do |join|
- source_regex = /(#{join[:source]})\.(\w+_)*id/i
-
- tables_except_src = (sources - [join[:source]]).join('|')
- target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i
-
- join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i
- matched = join_cond_regex.match(join[:constraints])
-
- if matched
- join[:target] = matched[:target]
- join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '')
- end
-
- join
- end
- end
-
- def build_relations_tree(joins, parent, source_key: :source, target_key: :target)
- return [] if joins.blank?
-
- tree = {}
- tree[parent] = []
-
- joins.each do |join|
- if join[source_key] == parent
- tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key)
- end
- end
- tree
- end
-
- def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle)
- conjunction = conjunctions.next
- relations.each do |subtree|
- subtree.each do |parent, children|
- parts << "<#{conjunction}>"
- join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints)
- append_constraints_prompt(parent, [wheres, join_constraints].compact, parts)
- parts << parent
- collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions)
- end
- end
- parts
- end
-
- def arelize_column(relation, column)
- case column
- when Arel::Attribute
- column
- when NilClass
- Arel::Table.new(relation.table_name)[relation.primary_key]
- when String
- if column.include?('.')
- table, col = column.split('.')
- Arel::Table.new(table)[col]
- else
- Arel::Table.new(relation.table_name)[column]
- end
- when Symbol
- arelize_column(relation, column.to_s)
- end
- end
-
- def parse_source(relation, column)
- column.relation.name || relation.table_name
- end
-
- def collector(connection)
- Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new)
- end
-
- def arel_query(relation:, column: nil, distinct: nil)
- column ||= relation.primary_key
-
- if column.is_a?(Arel::Attribute)
- relation.select(column.count(distinct)).arel
- else
- relation.select(relation.all.table[column].count(distinct)).arel
- end
- end
end
end
end
diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml
index adc5ba36ad7..f594c6a1b7c 100644
--- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml
+++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml
@@ -4,22 +4,18 @@
category: ecosystem
redis_slot: ecosystem
aggregation: weekly
- feature_flag: usage_data_track_ecosystem_jira_service
- name: i_ecosystem_jira_service_cross_reference
category: ecosystem
redis_slot: ecosystem
aggregation: weekly
- feature_flag: usage_data_track_ecosystem_jira_service
- name: i_ecosystem_jira_service_list_issues
category: ecosystem
redis_slot: ecosystem
aggregation: weekly
- feature_flag: usage_data_track_ecosystem_jira_service
- name: i_ecosystem_jira_service_create_issue
category: ecosystem
redis_slot: ecosystem
aggregation: weekly
- feature_flag: usage_data_track_ecosystem_jira_service
- name: i_ecosystem_slack_service_issue_notification
category: ecosystem
redis_slot: ecosystem