diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-08-18 13:50:51 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-08-18 13:50:51 +0300 |
commit | db384e6b19af03b4c3c82a5760d83a3fd79f7982 (patch) | |
tree | 34beaef37df5f47ccbcf5729d7583aae093cffa0 /lib/gitlab/database | |
parent | 54fd7b1bad233e3944434da91d257fa7f63c3996 (diff) |
Add latest changes from gitlab-org/gitlab@16-3-stable-eev16.3.0-rc42
Diffstat (limited to 'lib/gitlab/database')
24 files changed, 341 insertions, 382 deletions
diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb index abb62140503..7abcef5c93b 100644 --- a/lib/gitlab/database/batch_counter.rb +++ b/lib/gitlab/database/batch_counter.rb @@ -16,17 +16,18 @@ module Gitlab DEFAULT_DISTINCT_BATCH_SIZE = 10_000 DEFAULT_BATCH_SIZE = 100_000 - def initialize(relation, column: nil, operation: :count, operation_args: nil) + def initialize(relation, column: nil, operation: :count, operation_args: nil, max_allowed_loops: nil) @relation = relation @column = column || relation.primary_key @operation = operation @operation_args = operation_args + @max_allowed_loops = max_allowed_loops || MAX_ALLOWED_LOOPS end def unwanted_configuration?(finish, batch_size, start) (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || - (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || + (finish - start) / batch_size >= @max_allowed_loops || start >= finish end @@ -139,6 +140,7 @@ module Gitlab relation: @relation.table_name, operation: @operation, operation_args: @operation_args, + max_allowed_loops: @max_allowed_loops, start: batch_start, mode: mode, query: query, diff --git a/lib/gitlab/database/bump_sequences.rb b/lib/gitlab/database/bump_sequences.rb new file mode 100644 index 00000000000..f1b725b2de9 --- /dev/null +++ b/lib/gitlab/database/bump_sequences.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class BumpSequences + SEQUENCE_NAME_MATCHER = /nextval\('([a-z_]+)'::regclass\)/ + + # gitlab_schema: can be 'gitlab_main', 'gitlab_ci', 'gitlab_main_cell', 'gitlab_shared' + # increase_by: positive number, to increase the sequence by + # base_model: is to choose which connection to use to query the tables + def initialize(gitlab_schema, increase_by, base_model = ApplicationRecord) + @base_model = base_model + @gitlab_schema = gitlab_schema + @increase_by = increase_by + end + + def execute + sequences_by_gitlab_schema(base_model, gitlab_schema).each do |sequence_name| + increment_sequence_by(base_model.connection, sequence_name, increase_by) + end + end + + private + + attr_reader :base_model, :gitlab_schema, :increase_by + + def sequences_by_gitlab_schema(base_model, gitlab_schema) + tables = Gitlab::Database::GitlabSchema.tables_to_schema.select do |_table_name, schema_name| + schema_name == gitlab_schema + end.keys + + sequences = [] + + tables.each do |table| + model = Class.new(base_model) do + self.table_name = table + end + + model.columns.each do |column| + match_result = column.default_function&.match(SEQUENCE_NAME_MATCHER) + next unless match_result + + sequences << match_result[1] + end + end + + sequences + end + + # This method is going to increase the sequence next_value by: + # - increment_by + 1 if the sequence has the attribute is_called = True (which is the common case) + # - increment_by if the sequence has the attribute is_called = False (for example, a newly created sequence) + # It uses ALTER SEQUENCE as a safety mechanism to avoid that no concurrent insertions + # will cause conflicts on the sequence. + # This is because ALTER SEQUENCE blocks concurrent nextval, currval, lastval, and setval calls. + def increment_sequence_by(connection, sequence_name, increment_by) + connection.transaction do + # The first call is to make sure that the sequence's is_called value is set to `true` + # This guarantees that the next call to `nextval` will increase the sequence by `increment_by` + connection.select_value("SELECT nextval($1)", nil, [sequence_name]) + connection.execute("ALTER SEQUENCE #{sequence_name} INCREMENT BY #{increment_by}") + connection.select_value("select nextval($1)", nil, [sequence_name]) + connection.execute("ALTER SEQUENCE #{sequence_name} INCREMENT BY 1") + end + end + end + end +end diff --git a/lib/gitlab/database/ci_builds_partitioning.rb b/lib/gitlab/database/ci_builds_partitioning.rb deleted file mode 100644 index 9f8b19f2d23..00000000000 --- a/lib/gitlab/database/ci_builds_partitioning.rb +++ /dev/null @@ -1,224 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - class CiBuildsPartitioning - include AsyncDdlExclusiveLeaseGuard - - ATTEMPTS = 5 - LOCK_TIMEOUT = 10.seconds - LEASE_TIMEOUT = 30.minutes - - FK_NAME = :fk_e20479742e_p - TEMP_FK_NAME = :temp_fk_e20479742e_p - NEXT_PARTITION_ID = 101 - BUILDS_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_builds_101' - ANNOTATION_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_job_annotations_101' - RUNNER_MACHINE_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_runner_machine_builds_101' - - def initialize(logger: Gitlab::AppLogger) - @connection = ::Ci::ApplicationRecord.connection - @timing_configuration = Array.new(ATTEMPTS) { [LOCK_TIMEOUT, 3.minutes] } - @logger = logger - end - - def execute - return unless can_execute? - - try_obtain_lease do - swap_foreign_keys - create_new_ci_builds_partition - create_new_job_annotations_partition - create_new_runner_machine_partition - end - - rescue StandardError => e - log_info("Failed to execute: #{e.message}") - end - - private - - attr_reader :connection, :timing_configuration, :logger - - delegate :quote_table_name, :quote_column_name, to: :connection - - def swap_foreign_keys - if new_foreign_key_exists? - log_info('Foreign key already renamed, nothing to do') - - return - end - - with_lock_retries do - connection.execute drop_old_foreign_key_sql - - rename_constraint :p_ci_builds_metadata, TEMP_FK_NAME, FK_NAME - - each_partition do |partition| - rename_constraint partition.identifier, TEMP_FK_NAME, FK_NAME - end - end - - log_info('Foreign key successfully renamed') - end - - def create_new_ci_builds_partition - if connection.table_exists?(BUILDS_PARTITION_NAME) - log_info('p_ci_builds partition exists, nothing to do') - return - end - - with_lock_retries do - connection.execute new_ci_builds_partition_sql - end - - log_info('Partition for p_ci_builds successfully created') - end - - def create_new_job_annotations_partition - if connection.table_exists?(ANNOTATION_PARTITION_NAME) - log_info('p_ci_job_annotations partition exists, nothing to do') - return - end - - with_lock_retries do - connection.execute new_job_annotations_partition_sql - end - - log_info('Partition for p_ci_job_annotations successfully created') - end - - def create_new_runner_machine_partition - if connection.table_exists?(RUNNER_MACHINE_PARTITION_NAME) - log_info('p_ci_runner_machine_builds partition exists, nothing to do') - return - end - - with_lock_retries do - connection.execute new_runner_machine_partition_sql - end - - log_info('Partition for p_ci_runner_machine_builds successfully created') - end - - def can_execute? - return false if process_disabled? - return false unless Gitlab.com? - - if vacuum_running? - log_info('Autovacuum detected') - - return false - end - - true - end - - def process_disabled? - ::Feature.disabled?(:complete_p_ci_builds_partitioning) - end - - def new_foreign_key_exists? - Gitlab::Database::SharedModel.using_connection(connection) do - Gitlab::Database::PostgresForeignKey - .by_constrained_table_name_or_identifier(:p_ci_builds_metadata) - .by_referenced_table_name(:p_ci_builds) - .by_name(FK_NAME) - .exists? - end - end - - def vacuum_running? - Gitlab::Database::SharedModel.using_connection(connection) do - Gitlab::Database::PostgresAutovacuumActivity - .wraparound_prevention - .for_tables(%i[ci_builds ci_builds_metadata]) - .any? - end - end - - def drop_old_foreign_key_sql - <<~SQL.squish - SET LOCAL statement_timeout TO '11s'; - - LOCK TABLE ci_builds, p_ci_builds_metadata IN ACCESS EXCLUSIVE MODE; - - ALTER TABLE p_ci_builds_metadata DROP CONSTRAINT #{FK_NAME}; - SQL - end - - def rename_constraint(table_name, old_name, new_name) - connection.execute <<~SQL - ALTER TABLE #{quote_table_name(table_name)} - RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)} - SQL - end - - def new_ci_builds_partition_sql - <<~SQL - SET LOCAL statement_timeout TO '11s'; - - LOCK ci_pipelines, ci_stages IN SHARE ROW EXCLUSIVE MODE; - LOCK TABLE ONLY p_ci_builds IN ACCESS EXCLUSIVE MODE; - - CREATE TABLE IF NOT EXISTS #{BUILDS_PARTITION_NAME} - PARTITION OF p_ci_builds - FOR VALUES IN (#{NEXT_PARTITION_ID}); - SQL - end - - def new_job_annotations_partition_sql - <<~SQL - SET LOCAL statement_timeout TO '11s'; - - LOCK TABLE p_ci_builds IN SHARE ROW EXCLUSIVE MODE; - LOCK TABLE ONLY p_ci_job_annotations IN ACCESS EXCLUSIVE MODE; - - CREATE TABLE IF NOT EXISTS #{ANNOTATION_PARTITION_NAME} - PARTITION OF p_ci_job_annotations - FOR VALUES IN (#{NEXT_PARTITION_ID}); - SQL - end - - def new_runner_machine_partition_sql - <<~SQL - SET LOCAL statement_timeout TO '11s'; - - LOCK TABLE p_ci_builds IN SHARE ROW EXCLUSIVE MODE; - LOCK TABLE ONLY p_ci_runner_machine_builds IN ACCESS EXCLUSIVE MODE; - - CREATE TABLE IF NOT EXISTS #{RUNNER_MACHINE_PARTITION_NAME} - PARTITION OF p_ci_runner_machine_builds - FOR VALUES IN (#{NEXT_PARTITION_ID}); - SQL - end - - def with_lock_retries(&block) - Gitlab::Database::WithLockRetries.new( - timing_configuration: timing_configuration, - connection: connection, - logger: logger, - klass: self.class - ).run(raise_on_exhaustion: true, &block) - end - - def each_partition(&block) - Gitlab::Database::SharedModel.using_connection(connection) do - Gitlab::Database::PostgresPartitionedTable.each_partition(:p_ci_builds_metadata, &block) - end - end - - def log_info(message) - logger.info(message: message, class: self.class.to_s) - end - - def connection_db_config - ::Ci::ApplicationRecord.connection_db_config - end - - def lease_timeout - LEASE_TIMEOUT - end - end - end -end diff --git a/lib/gitlab/database/health_status.rb b/lib/gitlab/database/health_status.rb index 69bb8a70afd..b8dba802b56 100644 --- a/lib/gitlab/database/health_status.rb +++ b/lib/gitlab/database/health_status.rb @@ -6,7 +6,8 @@ module Gitlab DEFAULT_INIDICATORS = [ Indicators::AutovacuumActiveOnTable, Indicators::WriteAheadLog, - Indicators::PatroniApdex + Indicators::PatroniApdex, + Indicators::WalRate ].freeze class << self diff --git a/lib/gitlab/database/health_status/indicators/patroni_apdex.rb b/lib/gitlab/database/health_status/indicators/patroni_apdex.rb index 680c86cf7b2..5631b6db6ff 100644 --- a/lib/gitlab/database/health_status/indicators/patroni_apdex.rb +++ b/lib/gitlab/database/health_status/indicators/patroni_apdex.rb @@ -4,82 +4,19 @@ module Gitlab module Database module HealthStatus module Indicators - class PatroniApdex - include Gitlab::Utils::StrongMemoize - - def initialize(context) - @gitlab_schema = context.gitlab_schema.to_sym - end - - def evaluate - return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? - - connection_error_message = fetch_connection_error_message - return unknown_signal(connection_error_message) if connection_error_message.present? - - apdex_sli = fetch_sli(apdex_sli_query) - return unknown_signal('Patroni service apdex can not be calculated') unless apdex_sli.present? - - if apdex_sli.to_f > apdex_slo.to_f - Signals::Normal.new(self.class, reason: 'Patroni service apdex is above SLO') - else - Signals::Stop.new(self.class, reason: 'Patroni service apdex is below SLO') - end - end - + class PatroniApdex < PrometheusAlertIndicator private - attr_reader :gitlab_schema - def enabled? Feature.enabled?(:batched_migrations_health_status_patroni_apdex, type: :ops) end - def unknown_signal(reason) - Signals::Unknown.new(self.class, reason: reason) - end - - def fetch_connection_error_message - return 'Patroni Apdex Settings not configured' unless database_apdex_settings.present? - return 'Prometheus client is not ready' unless client.ready? - return 'Apdex SLI query is not configured' unless apdex_sli_query - return 'Apdex SLO is not configured' unless apdex_slo - end - - def client - @client ||= Gitlab::PrometheusClient.new( - database_apdex_settings[:prometheus_api_url], - allow_local_requests: true, - verify: true - ) - end - - def database_apdex_settings - @database_apdex_settings ||= Gitlab::CurrentSettings.database_apdex_settings&.with_indifferent_access + def sli_query_key + :apdex_sli_query end - def apdex_sli_query - { - gitlab_main: database_apdex_settings[:apdex_sli_query][:main], - gitlab_ci: database_apdex_settings[:apdex_sli_query][:ci] - }.fetch(gitlab_schema) - end - strong_memoize_attr :apdex_sli_query - - def apdex_slo - { - gitlab_main: database_apdex_settings[:apdex_slo][:main], - gitlab_ci: database_apdex_settings[:apdex_slo][:ci] - }.fetch(gitlab_schema) - end - strong_memoize_attr :apdex_slo - - def fetch_sli(query) - response = client.query(query) - metric = response&.first || {} - value = metric.fetch('value', []) - - Array.wrap(value).second + def slo_key + :apdex_slo end end end diff --git a/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb b/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb new file mode 100644 index 00000000000..3d630d21d4c --- /dev/null +++ b/lib/gitlab/database/health_status/indicators/prometheus_alert_indicator.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Indicators + class PrometheusAlertIndicator + include Gitlab::Utils::StrongMemoize + + ALERT_CONDITIONS = { + above: :above, + below: :below + }.freeze + + def initialize(context) + @gitlab_schema = context.gitlab_schema.to_sym + end + + def evaluate + return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? + + connection_error_message = fetch_connection_error_message + return unknown_signal(connection_error_message) if connection_error_message.present? + + sli = fetch_sli(sli_query) + return unknown_signal("#{indicator_name} can not be calculated") unless sli.present? + + if alert_condition == ALERT_CONDITIONS[:above] ? sli.to_f > slo.to_f : sli.to_f < slo.to_f + Signals::Normal.new(self.class, reason: "#{indicator_name} SLI condition met") + else + Signals::Stop.new(self.class, reason: "#{indicator_name} SLI condition not met") + end + end + + private + + attr_reader :gitlab_schema + + def indicator_name + self.class.name.demodulize + end + + # By default SLIs are expected to be above SLOs, but there can be cases + # where we want it to be below SLO (eg: WAL rate). For such indicators + # the sub-class should override this default alert_condition. + def alert_condition + ALERT_CONDITIONS[:above] + end + + def enabled? + raise NotImplementedError, "prometheus alert based indicators must implement #{__method__}" + end + + def slo_key + raise NotImplementedError, "prometheus alert based indicators must implement #{__method__}" + end + + def sli_key + raise NotImplementedError, "prometheus alert based indicators must implement #{__method__}" + end + + def fetch_connection_error_message + return 'Prometheus Settings not configured' unless prometheus_alert_db_indicators_settings.present? + return 'Prometheus client is not ready' unless client.ready? + return "#{indicator_name} SLI query is not configured" unless sli_query + return "#{indicator_name} SLO is not configured" unless slo + end + + def prometheus_alert_db_indicators_settings + @prometheus_alert_db_indicators_settings ||= Gitlab::CurrentSettings + .prometheus_alert_db_indicators_settings&.with_indifferent_access + end + + def client + @client ||= Gitlab::PrometheusClient.new( + prometheus_alert_db_indicators_settings[:prometheus_api_url], + allow_local_requests: true, + verify: true + ) + end + + def sli_query + { + gitlab_main: prometheus_alert_db_indicators_settings[sli_query_key][:main], + gitlab_ci: prometheus_alert_db_indicators_settings[sli_query_key][:ci] + }.fetch(gitlab_schema) + end + strong_memoize_attr :sli_query + + def slo + { + gitlab_main: prometheus_alert_db_indicators_settings[slo_key][:main], + gitlab_ci: prometheus_alert_db_indicators_settings[slo_key][:ci] + }.fetch(gitlab_schema) + end + strong_memoize_attr :slo + + def fetch_sli(query) + response = client.query(query) + metric = response&.first || {} + value = metric.fetch('value', []) + + Array.wrap(value).second + end + + def unknown_signal(reason) + Signals::Unknown.new(self.class, reason: reason) + end + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/indicators/wal_rate.rb b/lib/gitlab/database/health_status/indicators/wal_rate.rb new file mode 100644 index 00000000000..de31b5899eb --- /dev/null +++ b/lib/gitlab/database/health_status/indicators/wal_rate.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Indicators + class WalRate < PrometheusAlertIndicator + private + + def enabled? + Feature.enabled?(:db_health_check_wal_rate, type: :ops) + end + + def sli_query_key + :wal_rate_sli_query + end + + def slo_key + :wal_rate_slo + end + + def alert_condition + ALERT_CONDITIONS[:below] + end + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 256c524e989..60cec12b4b5 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1191,6 +1191,19 @@ into similar problems in the future (e.g. when new tables are created). .present? end + # While it is safe to call `change_column_default` on a column without + # default it would still require access exclusive lock on the table + # and for tables with high autovacuum(wraparound prevention) it will + # fail if their executions overlap. + # + def remove_column_default(table_name, column_name) + column = connection.columns(table_name).find { |col| col.name == column_name.to_s } + + if column.default || column.default_function + change_column_default(table_name, column_name, to: nil) + end + end + private def multiple_columns(columns, separator: ', ') diff --git a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb index 63928d7dc09..11f1e62e8b9 100644 --- a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb +++ b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb @@ -15,6 +15,20 @@ module Gitlab Gitlab.com? && !Gitlab.jh? end + + def temp_column_removed?(table_name, column_name) + !column_exists?(table_name.to_s, convert_to_bigint_column(column_name)) + end + + def columns_swapped?(table_name, column_name) + table_columns = columns(table_name.to_s) + temp_column_name = convert_to_bigint_column(column_name) + + column = table_columns.find { |c| c.name == column_name.to_s } + temp_column = table_columns.find { |c| c.name == temp_column_name } + + column.sql_type == 'bigint' && temp_column.sql_type == 'integer' + end end end end diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index cb2a98b553f..efb1957d5e7 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -242,7 +242,7 @@ module Gitlab "\n\n" \ "For more information, check the documentation" \ "\n\n" \ - "\thttps://docs.gitlab.com/ee/user/admin_area/monitoring/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished" + "\thttps://docs.gitlab.com/ee/update/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished" end end end diff --git a/lib/gitlab/database/migrations/squasher.rb b/lib/gitlab/database/migrations/squasher.rb new file mode 100644 index 00000000000..98fdf873aa5 --- /dev/null +++ b/lib/gitlab/database/migrations/squasher.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'set' + +module Gitlab + module Database + module Migrations + class Squasher + RSPEC_FILENAME_REGEXP = /\A([0-9]+_)?([_a-z0-9]*)\.rb\z/ + + def initialize(git_output) + @migration_data = migration_files_from_git(git_output).filter_map do |mf| + basename = Pathname(mf).basename.to_s + file_name_match = ActiveRecord::Migration::MigrationFilenameRegexp.match(basename) + slug = file_name_match[2] + unless slug == 'init_schema' + { + path: mf, + basename: basename, + timestamp: file_name_match[1], + slug: slug + } + end + end + end + + def files_to_delete + @migration_data.pluck(:path) + schema_migrations + find_migration_specs + end + + private + + def schema_migrations + @migration_data.map { |m| "db/schema_migrations/#{m[:timestamp]}" } + end + + def find_migration_specs + @file_slugs = Set.new @migration_data.pluck(:slug) + (migration_specs + ee_migration_specs).select { |f| file_has_slug?(f) } + end + + def migration_files_from_git(body) + body.chomp + .split("\n") + .select { |fn| fn.end_with?('.rb') } + end + + def match_file_slug(filename) + m = RSPEC_FILENAME_REGEXP.match(filename) + return if m.nil? + + m[2].sub(/_spec$/, '') + end + + def file_has_slug?(filename) + spec_slug = match_file_slug(Pathname(filename).basename.to_s) + return false if spec_slug.nil? + + @file_slugs.include?(spec_slug) + end + + def migration_specs + Dir.glob(Rails.root.join('spec/migrations/*.rb')) + end + + def ee_migration_specs + Dir.glob(Rails.root.join('ee/spec/migrations/*.rb')) + end + end + end + end +end diff --git a/lib/gitlab/database/postgres_constraint.rb b/lib/gitlab/database/postgres_constraint.rb index fa3870cb9c7..9c49251664b 100644 --- a/lib/gitlab/database/postgres_constraint.rb +++ b/lib/gitlab/database/postgres_constraint.rb @@ -17,7 +17,7 @@ module Gitlab scope :valid, -> { where(constraint_valid: true) } scope :by_table_identifier, ->(identifier) do - unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + unless Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier) raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" end diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb index bb3e1d45f15..9fb3098efe0 100644 --- a/lib/gitlab/database/postgres_foreign_key.rb +++ b/lib/gitlab/database/postgres_foreign_key.rb @@ -21,7 +21,7 @@ module Gitlab enum on_update_action: ACTION_TYPES, _prefix: :on_update scope :by_referenced_table_identifier, ->(identifier) do - unless identifier =~ Database::FULLY_QUALIFIED_IDENTIFIER + unless Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier) raise ArgumentError, "Referenced table name is not fully qualified with a schema: #{identifier}" end @@ -31,7 +31,7 @@ module Gitlab scope :by_referenced_table_name, ->(name) { where(referenced_table_name: name) } scope :by_constrained_table_identifier, ->(identifier) do - unless identifier =~ Database::FULLY_QUALIFIED_IDENTIFIER + unless Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier) raise ArgumentError, "Constrained table name is not fully qualified with a schema: #{identifier}" end @@ -41,7 +41,7 @@ module Gitlab scope :by_constrained_table_name, ->(name) { where(constrained_table_name: name) } scope :by_constrained_table_name_or_identifier, ->(name) do - if name =~ Database::FULLY_QUALIFIED_IDENTIFIER + if Database::FULLY_QUALIFIED_IDENTIFIER.match?(name) by_constrained_table_identifier(name) else by_constrained_table_name(name) diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index 50009cadf5d..1c775482e7e 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -14,7 +14,7 @@ module Gitlab has_many :queued_reindexing_actions, class_name: 'Gitlab::Database::Reindexing::QueuedAction', foreign_key: :index_identifier scope :by_identifier, ->(identifier) do - unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + unless Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier) raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" end diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb index e63c6fc86ea..f79b8b5e32c 100644 --- a/lib/gitlab/database/postgres_partition.rb +++ b/lib/gitlab/database/postgres_partition.rb @@ -10,7 +10,7 @@ module Gitlab # identifier includes the partition schema. # For example 'gitlab_partitions_static.events_03', or 'gitlab_partitions_dynamic.logs_03' scope :for_identifier, ->(identifier) do - unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + unless Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier) raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" end @@ -22,7 +22,7 @@ module Gitlab end scope :for_parent_table, ->(parent_table) do - if parent_table =~ Database::FULLY_QUALIFIED_IDENTIFIER + if Database::FULLY_QUALIFIED_IDENTIFIER.match?(parent_table) where(parent_identifier: parent_table).order(:name) else where("parent_identifier = concat(current_schema(), '.', ?)", parent_table).order(:name) diff --git a/lib/gitlab/database/postgres_partitioned_table.rb b/lib/gitlab/database/postgres_partitioned_table.rb index fead7379e43..76e2cd48f80 100644 --- a/lib/gitlab/database/postgres_partitioned_table.rb +++ b/lib/gitlab/database/postgres_partitioned_table.rb @@ -10,7 +10,7 @@ module Gitlab has_many :postgres_partitions, foreign_key: 'parent_identifier', primary_key: 'identifier' scope :by_identifier, ->(identifier) do - unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER + unless Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER.match?(identifier) raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" end diff --git a/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb b/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb index 71d2554844e..21392283ccf 100644 --- a/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb +++ b/lib/gitlab/database/postgresql_adapter/force_disconnectable_mixin.rb @@ -11,6 +11,8 @@ module Gitlab end def force_disconnect_if_old! + return if Rails.env.test? && transaction_open? + if force_disconnect_timer.expired? disconnect! reset_force_disconnect_timer! diff --git a/lib/gitlab/database/query_analyzers/query_recorder.rb b/lib/gitlab/database/query_analyzers/query_recorder.rb deleted file mode 100644 index 63b4fbb8c1d..00000000000 --- a/lib/gitlab/database/query_analyzers/query_recorder.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module QueryAnalyzers - class QueryRecorder < Base - LOG_PATH = 'query_recorder/' - LIST_PARAMETER_REGEX = %r{\$\d+(?:\s*,\s*\$\d+)+}.freeze - SINGLE_PARAMETER_REGEX = %r{\$\d+}.freeze - - class << self - def enabled? - # Only enable QueryRecorder in CI on database MRs or default branch - ENV['CI_MERGE_REQUEST_LABELS']&.include?('database') || - (ENV['CI_COMMIT_REF_NAME'].present? && ENV['CI_COMMIT_REF_NAME'] == ENV['CI_DEFAULT_BRANCH']) - end - - def analyze(parsed) - payload = { - normalized: normalize_query(parsed.sql) - } - - log_query(payload) - end - - def log_file - Rails.root.join(LOG_PATH, "#{ENV.fetch('CI_JOB_NAME_SLUG', 'rspec')}.ndjson") - end - - private - - def log_query(payload) - log_dir = Rails.root.join(LOG_PATH) - - # Create log directory if it does not exist since it is only created - # ahead of time by certain CI jobs - FileUtils.mkdir_p(log_dir) unless Dir.exist?(log_dir) - - log_line = "#{Gitlab::Json.dump(payload)}\n" - - File.write(log_file, log_line, mode: 'a') - end - - def normalize_query(query) - query - .gsub(LIST_PARAMETER_REGEX, '?,?,?') # Replace list parameters with ?,?,? - .gsub(SINGLE_PARAMETER_REGEX, '?') # Replace single parameters with ? - end - end - end - end - end -end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 9c860ebc6aa..4a1b0be848e 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -59,7 +59,6 @@ module Gitlab # most bloated indexes for reindexing. def self.perform_with_heuristic(candidate_indexes = Gitlab::Database::PostgresIndex.reindexing_support, maximum_records: DEFAULT_INDEXES_PER_INVOCATION) IndexSelection.new(candidate_indexes).take(maximum_records).each do |index| - Gitlab::Database::CiBuildsPartitioning.new.execute Coordinator.new(index).perform end end diff --git a/lib/gitlab/database/reindexing/reindex_concurrently.rb b/lib/gitlab/database/reindexing/reindex_concurrently.rb index 60fa4deda39..aa445082aa9 100644 --- a/lib/gitlab/database/reindexing/reindex_concurrently.rb +++ b/lib/gitlab/database/reindexing/reindex_concurrently.rb @@ -20,7 +20,7 @@ module Gitlab def perform raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion? - raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name =~ /#{TEMPORARY_INDEX_PATTERN}/o + raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if /#{TEMPORARY_INDEX_PATTERN}/o.match?(index.name) # Expression indexes require additional statistics in `pg_statistic`: # select * from pg_statistic where starelid = (select oid from pg_class where relname = 'some_index'); diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 562e651cabc..72ae2849911 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -23,6 +23,7 @@ module Gitlab with_paths = MigrationClasses::Route.arel_table[:path] .matches_any(path_patterns) namespaces.joins(:route).where(with_paths) + .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") end def rename_namespace(namespace) @@ -45,6 +46,7 @@ module Gitlab reverts_for_type('namespace') do |path_before_rename, current_path| matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) namespace = MigrationClasses::Namespace.joins(:route) + .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") .find_by(matches_path)&.becomes(MigrationClasses::Namespace) # rubocop: disable Cop/AvoidBecomes if namespace diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 5dbf30bad4e..155e35b64f4 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -37,6 +37,8 @@ module Gitlab reverts_for_type('project') do |path_before_rename, current_path| matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) project = MigrationClasses::Project.joins(:route) + .allow_cross_joins_across_databases(url: + 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') .find_by(matches_path) if project @@ -67,6 +69,7 @@ module Gitlab .matches_any(path_patterns) @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') end end end diff --git a/lib/gitlab/database/schema_validation/schema_inconsistency.rb b/lib/gitlab/database/schema_validation/schema_inconsistency.rb deleted file mode 100644 index 9f39db5b4c0..00000000000 --- a/lib/gitlab/database/schema_validation/schema_inconsistency.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class SchemaInconsistency < ApplicationRecord - self.table_name = :schema_inconsistencies - - belongs_to :issue - - validates :object_name, :valitador_name, :table_name, :diff, presence: true - - scope :with_open_issues, -> { joins(:issue).where('issue.state_id': Issue.available_states[:opened]) } - end - end - end -end diff --git a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb index b2a7f5442e9..9593cb45947 100644 --- a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb +++ b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb @@ -34,7 +34,7 @@ module Gitlab def all_foreign_keys @all_foreign_keys ||= @tables.each_with_object(Hash.new { |h, k| h[k] = [] }) do |table, hash| foreign_keys_for(table).each do |fk| - hash[fk.to_table] << table + hash[fk.referenced_table_name] << table end end end @@ -45,12 +45,10 @@ module Gitlab # # See spec/lib/gitlab/database/tables_sorted_by_foreign_keys_spec.rb # for an example - name = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(table) + table = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(table) - if name.schema == ::Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA.to_s - @connection.foreign_keys(name.identifier) - else - @connection.foreign_keys(table) + Gitlab::Database::SharedModel.using_connection(@connection) do + Gitlab::Database::PostgresForeignKey.by_constrained_table_name_or_identifier(table.identifier).load end end end |