diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-17 21:09:00 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-17 21:09:00 +0300 |
commit | eb1f5a3e087b7d6a3e85f2724b5a475cc9d9d37d (patch) | |
tree | d572a6d931152ae0dd10427237f5811893438552 /lib | |
parent | b304a72312465ed4c0a568ee6a6ea5e97f705c9b (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | lib/gitlab/current_settings.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/database/batch_count.rb | 89 | ||||
-rw-r--r-- | lib/gitlab/diff/diff_refs.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/diff/formatters/image_formatter.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/diff/formatters/text_formatter.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/usage_data.rb | 32 |
7 files changed, 117 insertions, 14 deletions
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index f5dc83ef472..47cc6caa192 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.9.1" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.9.3" review: extends: .auto-deploy diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 6ce47650562..891fd8c1bb5 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -50,7 +50,7 @@ module Gitlab # need to be added to the application settings. To prevent Rake tasks # and other callers from failing, use any loaded settings and return # defaults for missing columns. - if ActiveRecord::Base.connection.migration_context.needs_migration? + if Gitlab::Runtime.rake? && ActiveRecord::Base.connection.migration_context.needs_migration? db_attributes = current_settings&.attributes || {} fake_application_settings(db_attributes) elsif current_settings.present? diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb new file mode 100644 index 00000000000..a9d4665bc5f --- /dev/null +++ b/lib/gitlab/database/batch_count.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +# For large tables, PostgreSQL can take a long time to count rows due to MVCC. +# Implements a distinct and ordinary batch counter +# Needs indexes on the column below to calculate max, min and range queries +# For larger tables just set use higher batch_size with index optimization +# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 +# Examples: +# extend ::Gitlab::Database::BatchCount +# batch_count(User.active) +# batch_count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) +# batch_distinct_count(::Project, :creator_id) +module Gitlab + module Database + module BatchCount + def batch_count(relation, column = nil, batch_size: nil) + BatchCounter.new(relation, column: column).count(batch_size: batch_size) + end + + def batch_distinct_count(relation, column = nil, batch_size: nil) + BatchCounter.new(relation, column: column).count(mode: :distinct, batch_size: batch_size) + end + + class << self + include BatchCount + end + end + + class BatchCounter + FALLBACK = -1 + MIN_REQUIRED_BATCH_SIZE = 2_000 + MAX_ALLOWED_LOOPS = 10_000 + SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep + # Each query should take <<500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 + DEFAULT_DISTINCT_BATCH_SIZE = 100_000 + DEFAULT_BATCH_SIZE = 10_000 + + def initialize(relation, column: nil) + @relation = relation + @column = column || relation.primary_key + end + + def unwanted_configuration?(finish, batch_size, start) + batch_size <= MIN_REQUIRED_BATCH_SIZE || + (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || + start > finish + end + + def count(batch_size: nil, mode: :itself) + raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? + raise "The mode #{mode.inspect} is not supported" unless [:itself, :distinct].include?(mode) + + # non-distinct have better performance + batch_size ||= mode == :distinct ? DEFAULT_BATCH_SIZE : DEFAULT_DISTINCT_BATCH_SIZE + + start = @relation.minimum(@column) || 0 + finish = @relation.maximum(@column) || 0 + + raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 + return FALLBACK if unwanted_configuration?(finish, batch_size, start) + + counter = 0 + batch_start = start + + while batch_start <= finish + begin + counter += batch_fetch(batch_start, batch_start + batch_size, mode) + batch_start += batch_size + rescue ActiveRecord::QueryCanceled + # retry with a safe batch size & warmer cache + if batch_size >= 2 * MIN_REQUIRED_BATCH_SIZE + batch_size /= 2 + else + return FALLBACK + end + end + sleep(SLEEP_TIME_IN_SECONDS) + end + + counter + end + + def batch_fetch(start, finish, mode) + # rubocop:disable GitlabSecurity/PublicSend + @relation.select(@column).public_send(mode).where(@column => start..(finish - 1)).count + end + end + end +end diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index dc245377ccc..12b93af3f26 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -37,7 +37,7 @@ module Gitlab # We have `base_sha` directly available on `DiffRefs` because it's faster# # than having to look it up in the repo every time. def complete? - start_sha && head_sha + start_sha.present? && head_sha.present? end def compare_in(project) diff --git a/lib/gitlab/diff/formatters/image_formatter.rb b/lib/gitlab/diff/formatters/image_formatter.rb index 90cd74c84bb..d0c13dee1aa 100644 --- a/lib/gitlab/diff/formatters/image_formatter.rb +++ b/lib/gitlab/diff/formatters/image_formatter.rb @@ -23,7 +23,7 @@ module Gitlab end def complete? - x && y && width && height + [x, y, width, height].all?(&:present?) end def to_h diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb index f6e247ef665..5b670b1f83b 100644 --- a/lib/gitlab/diff/formatters/text_formatter.rb +++ b/lib/gitlab/diff/formatters/text_formatter.rb @@ -19,7 +19,7 @@ module Gitlab end def complete? - old_line || new_line + old_line.present? || new_line.present? end def to_h diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 23c3259b828..6e29a3e4cc4 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -67,8 +67,8 @@ module Gitlab clusters_disabled: count(::Clusters::Cluster.disabled), project_clusters_disabled: count(::Clusters::Cluster.disabled.project_type), group_clusters_disabled: count(::Clusters::Cluster.disabled.group_type), - clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled), - clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), + clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled, batch: false), + clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled, batch: false), clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), clusters_applications_helm: count(::Clusters::Applications::Helm.available), clusters_applications_ingress: count(::Clusters::Applications::Ingress.available), @@ -85,7 +85,7 @@ module Gitlab issues: count(Issue), issues_created_from_gitlab_error_tracking_ui: count(SentryIssue), issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue), - issues_using_zoom_quick_actions: count(ZoomMeeting.select(:issue_id).distinct), + issues_using_zoom_quick_actions: count(ZoomMeeting.select(:issue_id).distinct, batch: false), issues_with_embedded_grafana_charts_approx: ::Gitlab::GrafanaEmbedUsageData.issue_count, incident_issues: count(::Issue.authored(::User.alert_bot)), keys: count(Key), @@ -99,7 +99,7 @@ module Gitlab projects_imported_from_github: count(Project.where(import_type: 'github')), projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), - projects_with_alerts_service_enabled: count(AlertsService.active), + projects_with_alerts_service_enabled: count(AlertsService.active, batch: false), protected_branches: count(ProtectedBranch), releases: count(Release), remote_mirrors: count(RemoteMirror), @@ -181,7 +181,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def services_usage - service_counts = count(Service.active.where(template: false).where.not(type: 'JiraService').group(:type), fallback: Hash.new(-1)) + service_counts = count(Service.active.where(template: false).where.not(type: 'JiraService').group(:type), fallback: Hash.new(-1), batch: false) results = Service.available_services_names.each_with_object({}) do |service_name, response| response["projects_#{service_name}_active".to_sym] = service_counts["#{service_name}_service".camelize] || 0 @@ -217,9 +217,9 @@ module Gitlab results[:projects_jira_server_active] += counts[:server].count if counts[:server] results[:projects_jira_cloud_active] += counts[:cloud].count if counts[:cloud] if results[:projects_jira_active] == -1 - results[:projects_jira_active] = count(services) + results[:projects_jira_active] = count(services, batch: false) else - results[:projects_jira_active] += count(services) + results[:projects_jira_active] += count(services, batch: false) end end @@ -231,8 +231,22 @@ module Gitlab {} # augmented in EE end - def count(relation, fallback: -1) - relation.count + def count(relation, column = nil, fallback: -1, batch: true) + if batch && Feature.enabled?(:usage_ping_batch_counter) + Gitlab::Database::BatchCount.batch_count(relation, column) + else + relation.count + end + rescue ActiveRecord::StatementInvalid + fallback + end + + def distinct_count(relation, column = nil, fallback: -1, batch: true) + if batch && Feature.enabled?(:usage_ping_batch_counter) + Gitlab::Database::BatchCount.batch_distinct_count(relation, column) + else + relation.distinct_count_by(column) + end rescue ActiveRecord::StatementInvalid fallback end |