diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-06-07 15:10:00 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-06-07 15:10:00 +0300 |
commit | de8e5077c3671b0b29642faf1b5e562bc4f99453 (patch) | |
tree | 315d59367b7ff609ed4293f369c14be9e7e91cba /lib | |
parent | f4c6fbb86fbec3e5917e317b3490232d98531881 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
27 files changed, 290 insertions, 310 deletions
diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index 115a6b8ac4f..7b3750b37ee 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -137,7 +137,7 @@ module API bad_request! end - track_package_event('push_package', :composer) + track_package_event('push_package', :composer, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace) ::Packages::Composer::CreatePackageService .new(authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job)) @@ -161,7 +161,7 @@ module API not_found! unless metadata - track_package_event('pull_package', :composer) + track_package_event('pull_package', :composer, project: unauthorized_user_project, namespace: unauthorized_user_project.namespace) send_git_archive unauthorized_user_project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true end diff --git a/lib/api/concerns/packages/nuget_endpoints.rb b/lib/api/concerns/packages/nuget_endpoints.rb index 5364eeb1880..208daeb3037 100644 --- a/lib/api/concerns/packages/nuget_endpoints.rb +++ b/lib/api/concerns/packages/nuget_endpoints.rb @@ -58,7 +58,8 @@ module API end get 'index', format: :json do authorize_read_package!(project_or_group) - track_package_event('cli_metadata', :nuget, category: 'API::NugetPackages') + + track_package_event('cli_metadata', :nuget, **snowplow_gitlab_standard_context.merge(category: 'API::NugetPackages')) present ::Packages::Nuget::ServiceIndexPresenter.new(project_or_group), with: ::API::Entities::Nuget::ServiceIndex @@ -117,7 +118,7 @@ module API results = search_packages(params[:q], search_options) - track_package_event('search_package', :nuget, category: 'API::NugetPackages') + track_package_event('search_package', :nuget, **snowplow_gitlab_standard_context.merge(category: 'API::NugetPackages')) present ::Packages::Nuget::SearchResultsPresenter.new(results), with: ::API::Entities::Nuget::SearchResults diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index feb83b52695..eadb0646a67 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -35,7 +35,7 @@ module API authorize_upload!(authorized_user_project) bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:debian_max_file_size, params[:file].size) - track_package_event('push_package', :debian) + track_package_event('push_package', :debian, user: current_user, project: authorized_user_project, namespace: authorized_user_project.namespace) file_params = { file: params['file'], diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb index 4fede0ad583..96175f31696 100644 --- a/lib/api/group_container_repositories.rb +++ b/lib/api/group_container_repositories.rb @@ -31,7 +31,7 @@ module API user: current_user, subject: user_group ).execute - track_package_event('list_repositories', :container) + track_package_event('list_repositories', :container, user: current_user, namespace: user_group) present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] end diff --git a/lib/api/helm_packages.rb b/lib/api/helm_packages.rb index 446dcaf1e3a..dc5630a1395 100644 --- a/lib/api/helm_packages.rb +++ b/lib/api/helm_packages.rb @@ -46,7 +46,7 @@ module API package_file = Packages::Helm::PackageFilesFinder.new(authorized_user_project, params[:channel], file_name: "#{params[:file_name]}.tgz").execute.last! - track_package_event('pull_package', :helm) + track_package_event('pull_package', :helm, project: authorized_user_project, namespace: authorized_user_project.namespace) present_carrierwave_file!(package_file.file) end diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb index cdce92d5c46..9e5705abe88 100644 --- a/lib/api/maven_packages.rb +++ b/lib/api/maven_packages.rb @@ -130,7 +130,7 @@ module API when 'sha1' package_file.file_sha1 else - track_package_event('pull_package', :maven) if jar_file?(format) + track_package_event('pull_package', :maven, project: project, namespace: project.namespace) if jar_file?(format) present_carrierwave_file_with_head_support!(package_file.file) end end @@ -170,7 +170,7 @@ module API when 'sha1' package_file.file_sha1 else - track_package_event('pull_package', :maven) if jar_file?(format) + track_package_event('pull_package', :maven, project: package.project, namespace: package.project.namespace) if jar_file?(format) present_carrierwave_file_with_head_support!(package_file.file) end @@ -208,7 +208,7 @@ module API when 'sha1' package_file.file_sha1 else - track_package_event('pull_package', :maven) if jar_file?(format) + track_package_event('pull_package', :maven, project: user_project, namespace: user_project.namespace) if jar_file?(format) present_carrierwave_file_with_head_support!(package_file.file) end @@ -264,7 +264,7 @@ module API when 'md5' '' else - track_package_event('push_package', :maven) if jar_file?(format) + track_package_event('push_package', :maven, user: current_user, project: user_project, namespace: user_project.namespace) if jar_file?(format) file_params = { file: params[:file], diff --git a/lib/api/npm_project_packages.rb b/lib/api/npm_project_packages.rb index 887084dc9ae..7ff4439ce04 100644 --- a/lib/api/npm_project_packages.rb +++ b/lib/api/npm_project_packages.rb @@ -32,7 +32,7 @@ module API package_file = ::Packages::PackageFileFinder .new(package, params[:file_name]).execute! - track_package_event('pull_package', package, category: 'API::NpmPackages') + track_package_event('pull_package', package, category: 'API::NpmPackages', project: project, namespace: project.namespace) present_carrierwave_file!(package_file.file) end @@ -48,7 +48,7 @@ module API put ':package_name', requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do authorize_create_package!(project) - track_package_event('push_package', :npm, category: 'API::NpmPackages') + track_package_event('push_package', :npm, category: 'API::NpmPackages', project: project, user: current_user, namespace: project.namespace) created_package = ::Packages::Npm::CreatePackageService .new(project, current_user, params.merge(build: current_authenticated_job)).execute diff --git a/lib/api/nuget_group_packages.rb b/lib/api/nuget_group_packages.rb index a80de06d6b0..eb55e4cbf70 100644 --- a/lib/api/nuget_group_packages.rb +++ b/lib/api/nuget_group_packages.rb @@ -38,6 +38,10 @@ module API def require_authenticated! unauthorized! unless current_user end + + def snowplow_gitlab_standard_context + { namespace: find_authorized_group! } + end end params do diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index 73ecc140959..5bae08d4dae 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -36,6 +36,10 @@ module API def project_or_group authorized_user_project end + + def snowplow_gitlab_standard_context + { project: authorized_user_project, namespace: authorized_user_project.namespace } + end end params do @@ -69,7 +73,7 @@ module API package_file = ::Packages::CreatePackageFileService.new(package, file_params.merge(build: current_authenticated_job)) .execute - track_package_event('push_package', :nuget, category: 'API::NugetPackages') + track_package_event('push_package', :nuget, category: 'API::NugetPackages', user: current_user, project: package.project, namespace: package.project.namespace) ::Packages::Nuget::ExtractionWorker.perform_async(package_file.id) # rubocop:disable CodeReuse/Worker @@ -118,7 +122,7 @@ module API not_found!('Package') unless package_file - track_package_event('pull_package', :nuget, category: 'API::NugetPackages') + track_package_event('pull_package', :nuget, category: 'API::NugetPackages', project: package_file.project, namespace: package_file.project.namespace) # nuget and dotnet don't support 302 Moved status codes, supports_direct_download has to be set to false present_carrierwave_file!(package_file.file, supports_direct_download: false) diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb index 2580f7adbc9..28cfa9e3ae0 100644 --- a/lib/api/project_container_repositories.rb +++ b/lib/api/project_container_repositories.rb @@ -31,7 +31,7 @@ module API user: current_user, subject: user_project ).execute - track_package_event('list_repositories', :container) + track_package_event('list_repositories', :container, user: current_user, project: user_project, namespace: user_project.namespace) present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] end @@ -46,7 +46,7 @@ module API authorize_admin_container_image! DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) # rubocop:disable CodeReuse/Worker - track_package_event('delete_repository', :container) + track_package_event('delete_repository', :container, user: current_user, project: user_project, namespace: user_project.namespace) status :accepted end @@ -63,7 +63,7 @@ module API authorize_read_container_image! tags = Kaminari.paginate_array(repository.tags) - track_package_event('list_tags', :container) + track_package_event('list_tags', :container, user: current_user, project: user_project, namespace: user_project.namespace) present paginate(tags), with: Entities::ContainerRegistry::Tag end @@ -92,7 +92,7 @@ module API declared_params.except(:repository_id).merge(container_expiration_policy: false)) # rubocop:enable CodeReuse/Worker - track_package_event('delete_tag_bulk', :container) + track_package_event('delete_tag_bulk', :container, user: current_user, project: user_project, namespace: user_project.namespace) status :accepted end @@ -128,7 +128,7 @@ module API .execute(repository) if result[:status] == :success - track_package_event('delete_tag', :container) + track_package_event('delete_tag', :container, user: current_user, project: user_project, namespace: user_project.namespace) status :ok else diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index 969b619c1cd..7c5f8bb4d99 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -121,7 +121,7 @@ module API package = Packages::Pypi::PackageFinder.new(current_user, project, { filename: filename, sha256: params[:sha256] }).execute package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute - track_package_event('pull_package', :pypi) + track_package_event('pull_package', :pypi, project: project, namespace: project.namespace) present_carrierwave_file!(package_file.file, supports_direct_download: true) end @@ -140,7 +140,7 @@ module API get 'simple/*package_name', format: :txt do authorize_read_package!(authorized_user_project) - track_package_event('list_package', :pypi) + track_package_event('list_package', :pypi, project: authorized_user_project, namespace: authorized_user_project.namespace) packages = Packages::Pypi::PackagesFinder.new(current_user, authorized_user_project, { package_name: params[:package_name] }).execute! presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project) @@ -171,7 +171,7 @@ module API authorize_upload!(authorized_user_project) bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:pypi_max_file_size, params[:content].size) - track_package_event('push_package', :pypi) + track_package_event('push_package', :pypi, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace) ::Packages::Pypi::CreatePackageService .new(authorized_user_project, current_user, declared_params.merge(build: current_authenticated_job)) diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb index 1d17148e0df..d7f9c584c67 100644 --- a/lib/api/rubygem_packages.rb +++ b/lib/api/rubygem_packages.rb @@ -70,7 +70,7 @@ module API user_project, params[:file_name] ).last! - track_package_event('pull_package', :rubygems) + track_package_event('pull_package', :rubygems, project: user_project, namespace: user_project.namespace) present_carrierwave_file!(package_file.file) end @@ -97,7 +97,7 @@ module API authorize_upload!(user_project) bad_request!('File is too large') if user_project.actual_limits.exceeded?(:rubygems_max_file_size, params[:file].size) - track_package_event('push_package', :rubygems) + track_package_event('push_package', :rubygems, user: current_user, project: user_project, namespace: user_project.namespace) package_file = nil diff --git a/lib/api/terraform/modules/v1/packages.rb b/lib/api/terraform/modules/v1/packages.rb index 34e77e09800..aa59b6a4fee 100644 --- a/lib/api/terraform/modules/v1/packages.rb +++ b/lib/api/terraform/modules/v1/packages.rb @@ -124,7 +124,7 @@ module API end get do - track_package_event('pull_package', :terraform_module) + track_package_event('pull_package', :terraform_module, project: package.project, namespace: module_namespace, user: current_user) present_carrierwave_file!(package_file.file) end @@ -183,7 +183,7 @@ module API render_api_error!(result[:message], result[:http_status]) if result[:status] == :error - track_package_event('push_package', :terraform_module) + track_package_event('push_package', :terraform_module, project: authorized_user_project, user: current_user, namespace: authorized_user_project.namespace) created! rescue ObjectStorage::RemoteStoreError => e diff --git a/lib/feature.rb b/lib/feature.rb index 87abd2689d0..453ecc8255a 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -18,6 +18,10 @@ class Feature superclass.table_name = 'feature_gates' end + # To enable EE overrides + class ActiveSupportCacheStoreAdapter < Flipper::Adapters::ActiveSupportCacheStore + end + InvalidFeatureFlagError = Class.new(Exception) # rubocop:disable Lint/InheritException class << self @@ -167,7 +171,8 @@ class Feature ActiveSupportCacheStoreAdapter.new( active_record_adapter, l2_cache_backend, - expires_in: 1.hour) + expires_in: 1.hour, + write_through: true) # Thread-local L1 cache: use a short timeout since we don't have a # way to expire this cache all at once diff --git a/lib/feature/active_support_cache_store_adapter.rb b/lib/feature/active_support_cache_store_adapter.rb deleted file mode 100644 index 431f1169a86..00000000000 --- a/lib/feature/active_support_cache_store_adapter.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -# rubocop:disable Gitlab/NamespacedClass -# This class was already nested this way before moving to a separate file -class Feature - class ActiveSupportCacheStoreAdapter < Flipper::Adapters::ActiveSupportCacheStore - # This patch represents https://github.com/jnunemaker/flipper/pull/512. In - # Flipper 0.21.0 and later, we can remove this and just pass `write_through: - # true` to the constructor in `Feature.build_flipper_instance`. - - extend ::Gitlab::Utils::Override - - override :enable - def enable(feature, gate, thing) - result = @adapter.enable(feature, gate, thing) - @cache.write(key_for(feature.key), @adapter.get(feature), @write_options) - result - end - - override :disable - def disable(feature, gate, thing) - result = @adapter.disable(feature, gate, thing) - @cache.write(key_for(feature.key), @adapter.get(feature), @write_options) - result - end - - override :remove - def remove(feature) - result = @adapter.remove(feature) - @cache.delete(FeaturesKey) - @cache.write(key_for(feature.key), {}, @write_options) - result - end - end -end -# rubocop:disable Gitlab/NamespacedClass diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index e6f71e3ad3c..2b5f465d3c5 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -41,7 +41,17 @@ module Gitlab end def emoji_image_tag(name, src) - "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{src}' height='20' width='20' align='absmiddle' />" + image_options = { + class: 'emoji', + src: src, + title: ":#{name}:", + alt: ":#{name}:", + height: 20, + width: 20, + align: 'absmiddle' + } + + ActionController::Base.helpers.tag(:img, image_options) end def emoji_exists?(name) diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb index 44b85bf886e..f7e46fce134 100644 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -20,8 +20,7 @@ module Gitlab def check ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && - ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up && - ::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up end end end diff --git a/lib/gitlab/health_checks/redis/trace_chunks_check.rb b/lib/gitlab/health_checks/redis/trace_chunks_check.rb deleted file mode 100644 index cf9fa700b0a..00000000000 --- a/lib/gitlab/health_checks/redis/trace_chunks_check.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class TraceChunksCheck - extend SimpleAbstractCheck - - class << self - def check_up - check - end - - private - - def metric_prefix - 'redis_trace_chunks_ping' - end - - def successful?(result) - result == 'PONG' - end - - # rubocop: disable CodeReuse/ActiveRecord - def check - catch_timeout 10.seconds do - Gitlab::Redis::TraceChunks.with(&:ping) - end - end - # rubocop: enable CodeReuse/ActiveRecord - end - end - end - end -end diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index ab0e56adc32..9a9d3a866b1 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -8,9 +8,8 @@ module Gitlab Cache = Class.new(RedisBase).enable_redis_cluster_validation Queues = Class.new(RedisBase) SharedState = Class.new(RedisBase).enable_redis_cluster_validation - TraceChunks = Class.new(RedisBase).enable_redis_cluster_validation - STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks].freeze + STORAGES = [ActionCable, Cache, Queues, SharedState].freeze # Milliseconds represented in seconds (from 1 millisecond to 2 seconds). QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze diff --git a/lib/gitlab/redis/trace_chunks.rb b/lib/gitlab/redis/trace_chunks.rb deleted file mode 100644 index a2e77cb5df5..00000000000 --- a/lib/gitlab/redis/trace_chunks.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - class TraceChunks < ::Gitlab::Redis::Wrapper - # The data we store on TraceChunks used to be stored on SharedState. - def self.config_fallback - SharedState - end - end - end -end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index fa4e7f387ed..32447d39c02 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -64,19 +64,8 @@ module Gitlab def config_file_name [ - # Instance specific config sources: ENV["GITLAB_REDIS_#{store_name.underscore.upcase}_CONFIG_FILE"], config_file_path("redis.#{store_name.underscore}.yml"), - - # The current Redis instance may have been split off from another one - # (e.g. TraceChunks was split off from SharedState). There are - # installations out there where the lowest priority config source - # (resque.yml) contains bogus values. In those cases, config_file_name - # should resolve to the instance we originated from (the - # "config_fallback") rather than resque.yml. - config_fallback&.config_file_name, - - # Global config sources: ENV['GITLAB_REDIS_CONFIG_FILE'], config_file_path('resque.yml') ].compact.first @@ -86,10 +75,6 @@ module Gitlab name.demodulize end - def config_fallback - nil - end - def instrumentation_class "::Gitlab::Instrumentation::Redis::#{store_name}".constantize end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index 012727dd475..69a288e5b6e 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -47,6 +47,14 @@ module Gitlab Gitlab::Usage::Metrics::Query.for(self.class.metric_operation, relation, self.class.column) end + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for( + self.class.metric_operation, + relation: relation, + column: self.class.column + ) + end + private def relation diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb index 7c97cc37d17..1849773e33d 100644 --- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -13,6 +13,9 @@ module Gitlab # end # end class << self + attr_reader :metric_operation + @metric_operation = :alt + def value(&block) @metric_value = block end @@ -25,6 +28,12 @@ module Gitlab self.class.metric_value.call end end + + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for( + self.class.metric_operation + ) + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb index 502a8147473..a36e612a1cb 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb @@ -12,6 +12,11 @@ module Gitlab # events: # - g_analytics_valuestream # end + class << self + attr_reader :metric_operation + @metric_operation = :redis + end + def initialize(time_frame:, options: {}) super @@ -30,6 +35,12 @@ module Gitlab end end + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for( + self.class.metric_operation + ) + end + private def time_constraints diff --git a/lib/gitlab/usage/metrics/name_suggestion.rb b/lib/gitlab/usage/metrics/name_suggestion.rb new file mode 100644 index 00000000000..0728af9e2ca --- /dev/null +++ b/lib/gitlab/usage/metrics/name_suggestion.rb @@ -0,0 +1,200 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + class NameSuggestion + FREE_TEXT_METRIC_NAME = "<please fill metric name>" + REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>" + CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>" + + class << self + def for(operation, relation: nil, column: nil) + case operation + when :count + name_suggestion(column: column, relation: relation, prefix: 'count') + when :distinct_count + name_suggestion(column: column, relation: relation, prefix: 'count_distinct', distinct: :distinct) + when :estimate_batch_distinct_count + name_suggestion(column: column, relation: relation, prefix: 'estimate_distinct_count') + when :sum + name_suggestion(column: column, relation: relation, prefix: 'sum') + when :redis + REDIS_EVENT_METRIC_NAME + when :alt + FREE_TEXT_METRIC_NAME + else + raise ArgumentError, "#{operation} operation not supported" + end + end + + private + + def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil) + # rubocop: disable CodeReuse/ActiveRecord + relation = relation.unscope(where: :created_at) + # rubocop: enable CodeReuse/ActiveRecord + + parts = [prefix] + arel_column = arelize_column(relation, column) + + # nil as column indicates that the counting would use fallback value of primary key. + # Because counting primary key from relation is the conceptual equal to counting all + # records from given relation, in order to keep name suggestion more condensed + # primary key column is skipped. + # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not + # as count_id_from_issues since it does not add more information to the name suggestion + if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key] + parts << arel_column.name + parts << 'from' + end + + arel = arel_query(relation: relation, column: arel_column, distinct: distinct) + constraints = parse_constraints(relation: relation, arel: arel) + + # In some cases due to performance reasons metrics are instrumented with joined relations + # where relation listed in FROM statement is not the one that includes counted attribute + # in such situations to make name suggestion more intuitive source should be inferred based + # on the relation that provide counted attribute + # EG: SELECT COUNT(deployments.environment_id) FROM clusters + # JOIN deployments ON deployments.cluster_id = cluster.id + # should be translated into: + # count_environment_id_from_deployments_with_clusters + # instead of + # count_environment_id_from_clusters_with_deployments + actual_source = parse_source(relation, arel_column) + + append_constraints_prompt(actual_source, [constraints], parts) + + parts << actual_source + parts += process_joined_relations(actual_source, arel, relation, constraints) + parts.compact.join('_').delete('"') + end + + def append_constraints_prompt(target, constraints, parts) + applicable_constraints = constraints.select { |constraint| constraint.include?(target) } + return unless applicable_constraints.any? + + parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') } + end + + def parse_constraints(relation:, arel:) + connection = relation.connection + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints + .new(connection) + .accept(arel, collector(connection)) + .value + end + + # TODO: joins with `USING` keyword + def process_joined_relations(actual_source, arel, relation, where_constraints) + joins = parse_joins(connection: relation.connection, arel: arel) + return [] unless joins.any? + + sources = [relation.table_name, *joins.map { |join| join[:source] }] + joins = extract_joins_targets(joins, sources) + + relations = if actual_source != relation.table_name + build_relations_tree(joins + [{ source: relation.table_name }], actual_source) + else + # in case where counter attribute comes from joined relations, the relations + # diagram has to be built bottom up, thus source and target are reverted + build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source) + end + + collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints) + end + + def parse_joins(connection:, arel:) + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins + .new(connection) + .accept(arel) + end + + def extract_joins_targets(joins, sources) + joins.map do |join| + source_regex = /(#{join[:source]})\.(\w+_)*id/i + + tables_except_src = (sources - [join[:source]]).join('|') + target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i + + join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i + matched = join_cond_regex.match(join[:constraints]) + + if matched + join[:target] = matched[:target] + join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '') + end + + join + end + end + + def build_relations_tree(joins, parent, source_key: :source, target_key: :target) + return [] if joins.blank? + + tree = {} + tree[parent] = [] + + joins.each do |join| + if join[source_key] == parent + tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key) + end + end + tree + end + + def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle) + conjunction = conjunctions.next + relations.each do |subtree| + subtree.each do |parent, children| + parts << "<#{conjunction}>" + join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints) + append_constraints_prompt(parent, [wheres, join_constraints].compact, parts) + parts << parent + collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions) + end + end + parts + end + + def arelize_column(relation, column) + case column + when Arel::Attribute + column + when NilClass + Arel::Table.new(relation.table_name)[relation.primary_key] + when String + if column.include?('.') + table, col = column.split('.') + Arel::Table.new(table)[col] + else + Arel::Table.new(relation.table_name)[column] + end + when Symbol + arelize_column(relation, column.to_s) + end + end + + def parse_source(relation, column) + column.relation.name || relation.table_name + end + + def collector(connection) + Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new) + end + + def arel_query(relation:, column: nil, distinct: nil) + column ||= relation.primary_key + + if column.is_a?(Arel::Attribute) + relation.select(column.count(distinct)).arel + else + relation.select(relation.all.table[column].count(distinct)).arel + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb index 49581169452..a669b43f395 100644 --- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb +++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb @@ -5,10 +5,6 @@ module Gitlab module Metrics module NamesSuggestions class Generator < ::Gitlab::UsageData - FREE_TEXT_METRIC_NAME = "<please fill metric name>" - REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>" - CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>" - class << self def generate(key_path) uncached_data.deep_stringify_keys.dig(*key_path.split('.')) @@ -17,200 +13,36 @@ module Gitlab private def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - name_suggestion(column: column, relation: relation, prefix: 'count') + Gitlab::Usage::Metrics::NameSuggestion.for(:count, column: column, relation: relation) end def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) - name_suggestion(column: column, relation: relation, prefix: 'count_distinct', distinct: :distinct) + Gitlab::Usage::Metrics::NameSuggestion.for(:distinct_count, column: column, relation: relation) end def redis_usage_counter - REDIS_EVENT_METRIC_NAME + Gitlab::Usage::Metrics::NameSuggestion.for(:redis) end def alt_usage_data(*) - FREE_TEXT_METRIC_NAME + Gitlab::Usage::Metrics::NameSuggestion.for(:alt) end def redis_usage_data_totals(counter) - counter.fallback_totals.transform_values { |_| REDIS_EVENT_METRIC_NAME } + counter.fallback_totals.transform_values { |_| Gitlab::Usage::Metrics::NameSuggestion.for(:redis) } end def sum(relation, column, *rest) - name_suggestion(column: column, relation: relation, prefix: 'sum') + Gitlab::Usage::Metrics::NameSuggestion.for(:sum, column: column, relation: relation) end def estimate_batch_distinct_count(relation, column = nil, *rest) - name_suggestion(column: column, relation: relation, prefix: 'estimate_distinct_count') + Gitlab::Usage::Metrics::NameSuggestion.for(:estimate_batch_distinct_count, column: column, relation: relation) end def add(*args) "add_#{args.join('_and_')}" end - - def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil) - # rubocop: disable CodeReuse/ActiveRecord - relation = relation.unscope(where: :created_at) - # rubocop: enable CodeReuse/ActiveRecord - - parts = [prefix] - arel_column = arelize_column(relation, column) - - # nil as column indicates that the counting would use fallback value of primary key. - # Because counting primary key from relation is the conceptual equal to counting all - # records from given relation, in order to keep name suggestion more condensed - # primary key column is skipped. - # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not - # as count_id_from_issues since it does not add more information to the name suggestion - if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key] - parts << arel_column.name - parts << 'from' - end - - arel = arel_query(relation: relation, column: arel_column, distinct: distinct) - constraints = parse_constraints(relation: relation, arel: arel) - - # In some cases due to performance reasons metrics are instrumented with joined relations - # where relation listed in FROM statement is not the one that includes counted attribute - # in such situations to make name suggestion more intuitive source should be inferred based - # on the relation that provide counted attribute - # EG: SELECT COUNT(deployments.environment_id) FROM clusters - # JOIN deployments ON deployments.cluster_id = cluster.id - # should be translated into: - # count_environment_id_from_deployments_with_clusters - # instead of - # count_environment_id_from_clusters_with_deployments - actual_source = parse_source(relation, arel_column) - - append_constraints_prompt(actual_source, [constraints], parts) - - parts << actual_source - parts += process_joined_relations(actual_source, arel, relation, constraints) - parts.compact.join('_').delete('"') - end - - def append_constraints_prompt(target, constraints, parts) - applicable_constraints = constraints.select { |constraint| constraint.include?(target) } - return unless applicable_constraints.any? - - parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') } - end - - def parse_constraints(relation:, arel:) - connection = relation.connection - ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints - .new(connection) - .accept(arel, collector(connection)) - .value - end - - # TODO: joins with `USING` keyword - def process_joined_relations(actual_source, arel, relation, where_constraints) - joins = parse_joins(connection: relation.connection, arel: arel) - return [] unless joins.any? - - sources = [relation.table_name, *joins.map { |join| join[:source] }] - joins = extract_joins_targets(joins, sources) - - relations = if actual_source != relation.table_name - build_relations_tree(joins + [{ source: relation.table_name }], actual_source) - else - # in case where counter attribute comes from joined relations, the relations - # diagram has to be built bottom up, thus source and target are reverted - build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source) - end - - collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints) - end - - def parse_joins(connection:, arel:) - ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins - .new(connection) - .accept(arel) - end - - def extract_joins_targets(joins, sources) - joins.map do |join| - source_regex = /(#{join[:source]})\.(\w+_)*id/i - - tables_except_src = (sources - [join[:source]]).join('|') - target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i - - join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i - matched = join_cond_regex.match(join[:constraints]) - - if matched - join[:target] = matched[:target] - join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '') - end - - join - end - end - - def build_relations_tree(joins, parent, source_key: :source, target_key: :target) - return [] if joins.blank? - - tree = {} - tree[parent] = [] - - joins.each do |join| - if join[source_key] == parent - tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key) - end - end - tree - end - - def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle) - conjunction = conjunctions.next - relations.each do |subtree| - subtree.each do |parent, children| - parts << "<#{conjunction}>" - join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints) - append_constraints_prompt(parent, [wheres, join_constraints].compact, parts) - parts << parent - collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions) - end - end - parts - end - - def arelize_column(relation, column) - case column - when Arel::Attribute - column - when NilClass - Arel::Table.new(relation.table_name)[relation.primary_key] - when String - if column.include?('.') - table, col = column.split('.') - Arel::Table.new(table)[col] - else - Arel::Table.new(relation.table_name)[column] - end - when Symbol - arelize_column(relation, column.to_s) - end - end - - def parse_source(relation, column) - column.relation.name || relation.table_name - end - - def collector(connection) - Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new) - end - - def arel_query(relation:, column: nil, distinct: nil) - column ||= relation.primary_key - - if column.is_a?(Arel::Attribute) - relation.select(column.count(distinct)).arel - else - relation.select(relation.all.table[column].count(distinct)).arel - end - end end end end diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml index adc5ba36ad7..f594c6a1b7c 100644 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -4,22 +4,18 @@ category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_jira_service - name: i_ecosystem_jira_service_cross_reference category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_jira_service - name: i_ecosystem_jira_service_list_issues category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_jira_service - name: i_ecosystem_jira_service_create_issue category: ecosystem redis_slot: ecosystem aggregation: weekly - feature_flag: usage_data_track_ecosystem_jira_service - name: i_ecosystem_slack_service_issue_notification category: ecosystem redis_slot: ecosystem |