diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-07-20 18:40:28 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-07-20 18:40:28 +0300 |
commit | b595cb0c1dec83de5bdee18284abe86614bed33b (patch) | |
tree | 8c3d4540f193c5ff98019352f554e921b3a41a72 /lib/gitlab/database | |
parent | 2f9104a328fc8a4bddeaa4627b595166d24671d0 (diff) |
Add latest changes from gitlab-org/gitlab@15-2-stable-eev15.2.0-rc42
Diffstat (limited to 'lib/gitlab/database')
14 files changed, 242 insertions, 25 deletions
diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index ebc3ee240bd..72aa1cfe00b 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -26,6 +26,7 @@ module Gitlab scope :successful_in_execution_order, -> { where.not(finished_at: nil).with_status(:succeeded).order(:finished_at) } scope :with_preloads, -> { preload(:batched_migration) } scope :created_since, ->(date_time) { where('created_at >= ?', date_time) } + scope :blocked_by_max_attempts, -> { where('attempts >= ?', MAX_ATTEMPTS) } state_machine :status, initial: :pending do state :pending, value: 0 @@ -128,7 +129,8 @@ module Gitlab batched_migration.column_name, batch_min_value: min_value, batch_size: new_batch_size, - job_arguments: batched_migration.job_arguments + job_arguments: batched_migration.job_arguments, + job_class: batched_migration.job_class ) midpoint = next_batch_bounds.last diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index d052d5adc4c..9c8db2243f9 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -104,6 +104,12 @@ module Gitlab .sum(:batch_size) end + def reset_attempts_of_blocked_jobs! + batched_jobs.blocked_by_max_attempts.each_batch(of: 100) do |batch| + batch.update_all(attempts: 0) + end + end + def interval_elapsed?(variance: 0) return true unless last_job @@ -199,10 +205,32 @@ module Gitlab BatchOptimizer.new(self).optimize! end + def health_context + HealthStatus::Context.new([table_name]) + end + def hold!(until_time: 10.minutes.from_now) + duration_s = (until_time - Time.current).round + Gitlab::AppLogger.info( + message: "#{self} put on hold until #{until_time}", + migration_id: id, + job_class_name: job_class_name, + duration_s: duration_s + ) + update!(on_hold_until: until_time) end + def on_hold? + return false unless on_hold_until + + on_hold_until > Time.zone.now + end + + def to_s + "BatchedMigration[id: #{id}]" + end + private def validate_batched_jobs_status diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 388eb596ce2..1bc2e931391 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -29,7 +29,8 @@ module Gitlab if next_batched_job = find_or_create_next_batched_job(active_migration) migration_wrapper.perform(next_batched_job) - active_migration.optimize! + adjust_migration(active_migration) + active_migration.failure! if next_batched_job.failed? && active_migration.should_stop? else finish_active_migration(active_migration) @@ -71,12 +72,17 @@ module Gitlab elsif migration.finished? Gitlab::AppLogger.warn "Batched background migration for the given configuration is already finished: #{configuration}" else + migration.reset_attempts_of_blocked_jobs! + migration.finalize! migration.batched_jobs.with_status(:pending).each { |job| migration_wrapper.perform(job) } run_migration_while(migration, :finalizing) - raise FailedToFinalize unless migration.finished? + error_message = "Batched migration #{migration.job_class_name} could not be completed and a manual action is required."\ + "Check the admin panel at (`/admin/background_migrations`) for more details." + + raise FailedToFinalize, error_message unless migration.finished? end end @@ -101,7 +107,8 @@ module Gitlab active_migration.column_name, batch_min_value: batch_min_value, batch_size: active_migration.batch_size, - job_arguments: active_migration.job_arguments) + job_arguments: active_migration.job_arguments, + job_class: active_migration.job_class) return if next_batch_bounds.nil? @@ -135,6 +142,16 @@ module Gitlab migration.reload_last_job end end + + def adjust_migration(active_migration) + signal = HealthStatus.evaluate(active_migration) + + if signal.is_a?(HealthStatus::Signals::Stop) + active_migration.hold! + else + active_migration.optimize! + end + end end end end diff --git a/lib/gitlab/database/background_migration/health_status.rb b/lib/gitlab/database/background_migration/health_status.rb new file mode 100644 index 00000000000..01f9c5eb5fd --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + # Rather than passing along the migration, we use a more explicitly defined context + Context = Struct.new(:tables) + + def self.evaluate(migration, indicator = Indicators::AutovacuumActiveOnTable) + signal = begin + indicator.new(migration.health_context).evaluate + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e, migration_id: migration.id, + job_class_name: migration.job_class_name) + Signals::Unknown.new(indicator, reason: "unexpected error: #{e.message} (#{e.class})") + end + + log_signal(signal, migration) if signal.log_info? + + signal + end + + def self.log_signal(signal, migration) + Gitlab::AppLogger.info( + message: "#{migration} signaled: #{signal}", + migration_id: migration.id + ) + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/health_status/indicators.rb b/lib/gitlab/database/background_migration/health_status/indicators.rb new file mode 100644 index 00000000000..69503e5b61f --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status/indicators.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + module Indicators + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb b/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb new file mode 100644 index 00000000000..48e12609a13 --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + module Indicators + class AutovacuumActiveOnTable + def initialize(context) + @context = context + end + + def evaluate + return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? + + autovacuum_active_on = active_autovacuums_for(context.tables) + + if autovacuum_active_on.empty? + Signals::Normal.new(self.class, reason: 'no autovacuum running on any relevant tables') + else + Signals::Stop.new(self.class, reason: "autovacuum running on: #{autovacuum_active_on.join(', ')}") + end + end + + private + + attr_reader :context + + def enabled? + Feature.enabled?(:batched_migrations_health_status_autovacuum, type: :ops) + end + + def active_autovacuums_for(tables) + Gitlab::Database::PostgresAutovacuumActivity.for_tables(tables) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/health_status/signals.rb b/lib/gitlab/database/background_migration/health_status/signals.rb new file mode 100644 index 00000000000..6cd0ebd1bd0 --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status/signals.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + module Signals + # Base class for a signal + class Base + attr_reader :indicator_class, :reason + + def initialize(indicator_class, reason:) + @indicator_class = indicator_class + @reason = reason + end + + def to_s + "#{short_name} (indicator: #{indicator_class}; reason: #{reason})" + end + + # :nocov: + def log_info? + false + end + # :nocov: + + private + + def short_name + self.class.name.demodulize + end + end + + # A Signals::Stop is an indication to put a migration on hold or stop it entirely: + # In general, we want to slow down or pause the migration. + class Stop < Base + # :nocov: + def log_info? + true + end + # :nocov: + end + + # A Signals::Normal indicates normal system state: We carry on with the migration + # and may even attempt to optimize its throughput etc. + class Normal < Base; end + + # When given an Signals::Unknown, something unexpected happened while + # we evaluated system indicators. + class Unknown < Base + # :nocov: + def log_info? + true + end + # :nocov: + end + + # No signal could be determined, e.g. because the indicator + # was disabled. + class NotAvailable < Base; end + end + end + end + end +end diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index 0d876f5124f..02f008abf85 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -36,8 +36,7 @@ module Gitlab private def select_base_models(names) - base_models = Gitlab::Database.database_base_models - + base_models = Gitlab::Database.database_base_models_with_gitlab_shared return base_models if names.empty? names.each_with_object(HashWithIndifferentAccess.new) do |name, hash| @@ -48,7 +47,7 @@ module Gitlab end def with_shared_model_connections(shared_model, selected_databases, &blk) - Gitlab::Database.database_base_models.each_pair do |connection_name, connection_model| + Gitlab::Database.database_base_models_with_gitlab_shared.each_pair do |connection_name, connection_model| if shared_model.limit_connection_names next unless shared_model.limit_connection_names.include?(connection_name.to_sym) end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index baf4cc48424..365a4283d4c 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -13,6 +13,8 @@ module Gitlab module Database module GitlabSchema + GITLAB_SCHEMAS_FILE = 'lib/gitlab/database/gitlab_schemas.yml' + # These tables are deleted/renamed, but still referenced by migrations. # This is needed for now, but should be removed in the future DELETED_TABLES = { @@ -93,7 +95,7 @@ module Gitlab end def self.tables_to_schema - @tables_to_schema ||= YAML.load_file(Rails.root.join('lib/gitlab/database/gitlab_schemas.yml')) + @tables_to_schema ||= YAML.load_file(Rails.root.join(GITLAB_SCHEMAS_FILE)) end def self.schema_names @@ -102,3 +104,5 @@ module Gitlab end end end + +Gitlab::Database::GitlabSchema.prepend_mod diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 71c323cb393..4a467d18f0a 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -108,10 +108,12 @@ ci_resource_groups: :gitlab_ci ci_resources: :gitlab_ci ci_runner_namespaces: :gitlab_ci ci_runner_projects: :gitlab_ci +ci_runner_versions: :gitlab_ci ci_runners: :gitlab_ci ci_running_builds: :gitlab_ci ci_sources_pipelines: :gitlab_ci ci_secure_files: :gitlab_ci +ci_secure_file_states: :gitlab_ci ci_sources_projects: :gitlab_ci ci_stages: :gitlab_ci ci_subscriptions_projects: :gitlab_ci @@ -131,7 +133,6 @@ cluster_providers_gcp: :gitlab_main clusters_applications_cert_managers: :gitlab_main clusters_applications_cilium: :gitlab_main clusters_applications_crossplane: :gitlab_main -clusters_applications_elastic_stacks: :gitlab_main clusters_applications_helm: :gitlab_main clusters_applications_ingress: :gitlab_main clusters_applications_jupyter: :gitlab_main @@ -139,7 +140,6 @@ clusters_applications_knative: :gitlab_main clusters_applications_prometheus: :gitlab_main clusters_applications_runners: :gitlab_main clusters: :gitlab_main -clusters_integration_elasticstack: :gitlab_main clusters_integration_prometheus: :gitlab_main clusters_kubernetes_namespaces: :gitlab_main commit_user_mentions: :gitlab_main @@ -325,6 +325,7 @@ milestone_releases: :gitlab_main milestones: :gitlab_main namespace_admin_notes: :gitlab_main namespace_aggregation_schedules: :gitlab_main +namespace_bans: :gitlab_main namespace_limits: :gitlab_main namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main @@ -423,6 +424,8 @@ project_incident_management_settings: :gitlab_main project_metrics_settings: :gitlab_main project_mirror_data: :gitlab_main project_pages_metadata: :gitlab_main +project_relation_export_uploads: :gitlab_main +project_relation_exports: :gitlab_main project_repositories: :gitlab_main project_repository_states: :gitlab_main project_repository_storage_moves: :gitlab_main @@ -432,7 +435,6 @@ projects: :gitlab_main projects_sync_events: :gitlab_main project_statistics: :gitlab_main project_topics: :gitlab_main -project_tracing_settings: :gitlab_main prometheus_alert_events: :gitlab_main prometheus_alerts: :gitlab_main prometheus_metrics: :gitlab_main @@ -467,6 +469,10 @@ routes: :gitlab_main saml_group_links: :gitlab_main saml_providers: :gitlab_main saved_replies: :gitlab_main +sbom_components: :gitlab_main +sbom_occurrences: :gitlab_main +sbom_component_versions: :gitlab_main +sbom_sources: :gitlab_main schema_migrations: :gitlab_internal scim_identities: :gitlab_main scim_oauth_access_tokens: :gitlab_main @@ -547,6 +553,7 @@ vulnerability_flags: :gitlab_main vulnerability_historical_statistics: :gitlab_main vulnerability_identifiers: :gitlab_main vulnerability_issue_links: :gitlab_main +vulnerability_merge_request_links: :gitlab_main vulnerability_occurrence_identifiers: :gitlab_main vulnerability_occurrence_pipelines: :gitlab_main vulnerability_occurrences: :gitlab_main @@ -571,3 +578,4 @@ zentao_tracker_data: :gitlab_main dingtalk_tracker_data: :gitlab_main zoom_meetings: :gitlab_main batched_background_migration_job_transition_logs: :gitlab_shared +user_namespace_callouts: :gitlab_main diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index 6517923d23e..5f9416fb4db 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -19,7 +19,7 @@ module Gitlab ].freeze def self.base_models - @base_models ||= ::Gitlab::Database.database_base_models.values.freeze + @base_models ||= ::Gitlab::Database.database_base_models_using_load_balancing.values.freeze end def self.each_load_balancer diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 4aaeaa7b365..936b986ea07 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -72,12 +72,6 @@ module Gitlab ) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! - if transaction_open? - raise 'The `queue_batched_background_migration` cannot be run inside a transaction. ' \ - 'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \ - 'your migration class.' - end - gitlab_schema ||= gitlab_schema_from_context Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information @@ -182,12 +176,6 @@ module Gitlab def delete_batched_background_migration(job_class_name, table_name, column_name, job_arguments) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! - if transaction_open? - raise 'The `#delete_batched_background_migration` cannot be run inside a transaction. ' \ - 'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \ - 'your migration class.' - end - Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information Gitlab::Database::BackgroundMigration::BatchedMigration diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index 0c6a8d3d856..f38d847b0e8 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -14,7 +14,7 @@ module Gitlab def jobs_by_migration_name Gitlab::Database::BackgroundMigration::BatchedMigration .executable - .created_after(2.days.ago) # Simple way to exclude migrations already running before migration testing + .created_after(3.hours.ago) # Simple way to exclude migrations already running before migration testing .to_h do |migration| batching_strategy = migration.batch_class.new(connection: connection) diff --git a/lib/gitlab/database/postgres_autovacuum_activity.rb b/lib/gitlab/database/postgres_autovacuum_activity.rb new file mode 100644 index 00000000000..a4dc199c259 --- /dev/null +++ b/lib/gitlab/database/postgres_autovacuum_activity.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PostgresAutovacuumActivity < SharedModel + self.table_name = 'postgres_autovacuum_activity' + self.primary_key = 'table_identifier' + + def self.for_tables(tables) + Gitlab::Database::LoadBalancing::Session.current.use_primary do + where('schema = current_schema()').where(table: tables) + end + end + + def to_s + "table #{table_identifier} (started: #{vacuum_start})" + end + end + end +end |