diff options
Diffstat (limited to 'lib/gitlab/database')
48 files changed, 1694 insertions, 478 deletions
diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb index 07809c5b592..a04ea97117d 100644 --- a/lib/gitlab/database/as_with_materialized.rb +++ b/lib/gitlab/database/as_with_materialized.rb @@ -19,7 +19,7 @@ module Gitlab # Note: to be deleted after the minimum PG version is set to 12.0 def self.materialized_supported? strong_memoize(:materialized_supported) do - Gitlab::Database.main.version.match?(/^1[2-9]\./) # version 12.x and above + ApplicationRecord.database.version.match?(/^1[2-9]\./) # version 12.x and above end end diff --git a/lib/gitlab/database/async_indexes/index_creator.rb b/lib/gitlab/database/async_indexes/index_creator.rb index 00de79ec970..994a1deba57 100644 --- a/lib/gitlab/database/async_indexes/index_creator.rb +++ b/lib/gitlab/database/async_indexes/index_creator.rb @@ -40,7 +40,7 @@ module Gitlab end def connection - @connection ||= ApplicationRecord.connection + @connection ||= async_index.connection end def lease_timeout diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb index 236459e6216..6cb40729061 100644 --- a/lib/gitlab/database/async_indexes/postgres_async_index.rb +++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb @@ -3,7 +3,7 @@ module Gitlab module Database module AsyncIndexes - class PostgresAsyncIndex < ApplicationRecord + class PostgresAsyncIndex < SharedModel self.table_name = 'postgres_async_indexes' MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH diff --git a/lib/gitlab/database/background_migration_job.rb b/lib/gitlab/database/background_migration_job.rb index 1121793917b..c046571a111 100644 --- a/lib/gitlab/database/background_migration_job.rb +++ b/lib/gitlab/database/background_migration_job.rb @@ -4,6 +4,7 @@ module Gitlab module Database class BackgroundMigrationJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord include EachBatch + include BulkInsertSafe self.table_name = :background_migration_jobs diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb index 7efa5b46ecb..6c0ce9e481a 100644 --- a/lib/gitlab/database/batch_counter.rb +++ b/lib/gitlab/database/batch_counter.rb @@ -31,7 +31,7 @@ module Gitlab end def count(batch_size: nil, mode: :itself, start: nil, finish: nil) - raise 'BatchCount can not be run inside a transaction' if @relation.connection.transaction_open? + raise 'BatchCount can not be run inside a transaction' if transaction_open? check_mode!(mode) @@ -87,6 +87,10 @@ module Gitlab results end + def transaction_open? + @relation.connection.transaction_open? + end + def merge_results(results, object) return object unless results diff --git a/lib/gitlab/database/connection.rb b/lib/gitlab/database/connection.rb deleted file mode 100644 index cda6220ee6c..00000000000 --- a/lib/gitlab/database/connection.rb +++ /dev/null @@ -1,260 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - # Configuration settings and methods for interacting with a PostgreSQL - # database, with support for multiple databases. - class Connection - attr_reader :scope - - # Initializes a new `Database`. - # - # The `scope` argument must be an object (such as `ActiveRecord::Base`) - # that supports retrieving connections and connection pools. - def initialize(scope = ActiveRecord::Base) - @config = nil - @scope = scope - @version = nil - @open_transactions_baseline = 0 - end - - def config - # The result of this method must not be cached, as other methods may use - # it after making configuration changes and expect those changes to be - # present. For example, `disable_prepared_statements` expects the - # configuration settings to always be up to date. - # - # See the following for more information: - # - # - https://gitlab.com/gitlab-org/release/retrospectives/-/issues/39 - # - https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5238 - scope.connection_db_config.configuration_hash.with_indifferent_access - end - - def pool_size - config[:pool] || Database.default_pool_size - end - - def username - config[:username] || ENV['USER'] - end - - def database_name - config[:database] - end - - def adapter_name - config[:adapter] - end - - def human_adapter_name - if postgresql? - 'PostgreSQL' - else - 'Unknown' - end - end - - def postgresql? - adapter_name.casecmp('postgresql') == 0 - end - - def db_config_with_default_pool_size - db_config_object = scope.connection_db_config - config = db_config_object - .configuration_hash - .merge(pool: Database.default_pool_size) - - ActiveRecord::DatabaseConfigurations::HashConfig.new( - db_config_object.env_name, - db_config_object.name, - config - ) - end - - # Disables prepared statements for the current database connection. - def disable_prepared_statements - db_config_object = scope.connection_db_config - config = db_config_object.configuration_hash.merge(prepared_statements: false) - - hash_config = ActiveRecord::DatabaseConfigurations::HashConfig.new( - db_config_object.env_name, - db_config_object.name, - config - ) - - scope.establish_connection(hash_config) - end - - # Check whether the underlying database is in read-only mode - def db_read_only? - pg_is_in_recovery = - scope - .connection - .execute('SELECT pg_is_in_recovery()') - .first - .fetch('pg_is_in_recovery') - - Gitlab::Utils.to_boolean(pg_is_in_recovery) - end - - def db_read_write? - !db_read_only? - end - - def version - @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] - end - - def database_version - connection.execute("SELECT VERSION()").first['version'] - end - - def postgresql_minimum_supported_version? - version.to_f >= MINIMUM_POSTGRES_VERSION - end - - # Bulk inserts a number of rows into a table, optionally returning their - # IDs. - # - # table - The name of the table to insert the rows into. - # rows - An Array of Hash instances, each mapping the columns to their - # values. - # return_ids - When set to true the return value will be an Array of IDs of - # the inserted rows - # disable_quote - A key or an Array of keys to exclude from quoting (You - # become responsible for protection from SQL injection for - # these keys!) - # on_conflict - Defines an upsert. Values can be: :disabled (default) or - # :do_nothing - def bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil) - return if rows.empty? - - keys = rows.first.keys - columns = keys.map { |key| connection.quote_column_name(key) } - - disable_quote = Array(disable_quote).to_set - tuples = rows.map do |row| - keys.map do |k| - disable_quote.include?(k) ? row[k] : connection.quote(row[k]) - end - end - - sql = <<-EOF - INSERT INTO #{table} (#{columns.join(', ')}) - VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} - EOF - - sql = "#{sql} ON CONFLICT DO NOTHING" if on_conflict == :do_nothing - - sql = "#{sql} RETURNING id" if return_ids - - result = connection.execute(sql) - - if return_ids - result.values.map { |tuple| tuple[0].to_i } - else - [] - end - end - - def cached_column_exists?(table_name, column_name) - connection - .schema_cache.columns_hash(table_name) - .has_key?(column_name.to_s) - end - - def cached_table_exists?(table_name) - exists? && connection.schema_cache.data_source_exists?(table_name) - end - - def exists? - # We can't _just_ check if `connection` raises an error, as it will - # point to a `ConnectionProxy`, and obtaining those doesn't involve any - # database queries. So instead we obtain the database version, which is - # cached after the first call. - connection.schema_cache.database_version - true - rescue StandardError - false - end - - def system_id - row = connection - .execute('SELECT system_identifier FROM pg_control_system()') - .first - - row['system_identifier'] - end - - def pg_wal_lsn_diff(location1, location2) - lsn1 = connection.quote(location1) - lsn2 = connection.quote(location2) - - query = <<-SQL.squish - SELECT pg_wal_lsn_diff(#{lsn1}, #{lsn2}) - AS result - SQL - - row = connection.select_all(query).first - row['result'] if row - end - - # @param [ActiveRecord::Connection] ar_connection - # @return [String] - def get_write_location(ar_connection) - use_new_load_balancer_query = Gitlab::Utils - .to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: true) - - sql = - if use_new_load_balancer_query - <<~NEWSQL - SELECT CASE - WHEN pg_is_in_recovery() = true AND EXISTS (SELECT 1 FROM pg_stat_get_wal_senders()) - THEN pg_last_wal_replay_lsn()::text - WHEN pg_is_in_recovery() = false - THEN pg_current_wal_insert_lsn()::text - ELSE NULL - END AS location; - NEWSQL - else - <<~SQL - SELECT pg_current_wal_insert_lsn()::text AS location - SQL - end - - row = ar_connection.select_all(sql).first - row['location'] if row - end - - # inside_transaction? will return true if the caller is running within a - # transaction. Handles special cases when running inside a test - # environment, where tests may be wrapped in transactions - def inside_transaction? - base = Rails.env.test? ? @open_transactions_baseline : 0 - - scope.connection.open_transactions > base - end - - # These methods that access @open_transactions_baseline are not - # thread-safe. These are fine though because we only call these in - # RSpec's main thread. If we decide to run specs multi-threaded, we would - # need to use something like ThreadGroup to keep track of this value - def set_open_transactions_baseline - @open_transactions_baseline = scope.connection.open_transactions - end - - def reset_open_transactions_baseline - @open_transactions_baseline = 0 - end - - private - - def connection - scope.connection - end - end - end -end - -Gitlab::Database::Connection.prepend_mod_with('Gitlab::Database::Connection') diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb new file mode 100644 index 00000000000..7c9e65e6691 --- /dev/null +++ b/lib/gitlab/database/each_database.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module EachDatabase + class << self + def each_database_connection + Gitlab::Database.database_base_models.each_pair do |connection_name, model| + connection = model.connection + + with_shared_connection(connection, connection_name) do + yield connection, connection_name + end + end + end + + def each_model_connection(models) + models.each do |model| + connection_name = model.connection.pool.db_config.name + + with_shared_connection(model.connection, connection_name) do + yield model, connection_name + end + end + end + + private + + def with_shared_connection(connection, connection_name) + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::AppLogger.debug(message: 'Switched database connection', connection_name: connection_name) + + yield + end + end + end + end + end +end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb new file mode 100644 index 00000000000..14807494a79 --- /dev/null +++ b/lib/gitlab/database/gitlab_schema.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +# This module gathers information about table to schema mapping +# to understand table affinity +# +# Each table / view needs to have assigned gitlab_schema. Names supported today: +# +# - gitlab_shared - defines a set of tables that are found on all databases (data accessed is dependent on connection) +# - gitlab_main / gitlab_ci - defines a set of tables that can only exist on a given database +# +# Tables for the purpose of tests should be prefixed with `_test_my_table_name` + +module Gitlab + module Database + module GitlabSchema + # These tables are deleted/renamed, but still referenced by migrations. + # This is needed for now, but should be removed in the future + DELETED_TABLES = { + # main tables + 'alerts_service_data' => :gitlab_main, + 'analytics_devops_adoption_segment_selections' => :gitlab_main, + 'analytics_repository_file_commits' => :gitlab_main, + 'analytics_repository_file_edits' => :gitlab_main, + 'analytics_repository_files' => :gitlab_main, + 'audit_events_archived' => :gitlab_main, + 'backup_labels' => :gitlab_main, + 'clusters_applications_fluentd' => :gitlab_main, + 'forked_project_links' => :gitlab_main, + 'issue_milestones' => :gitlab_main, + 'merge_request_milestones' => :gitlab_main, + 'namespace_onboarding_actions' => :gitlab_main, + 'services' => :gitlab_main, + 'terraform_state_registry' => :gitlab_main, + 'tmp_fingerprint_sha256_migration' => :gitlab_main, # used by lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb + 'web_hook_logs_archived' => :gitlab_main, + 'vulnerability_export_registry' => :gitlab_main, + 'vulnerability_finding_fingerprints' => :gitlab_main, + 'vulnerability_export_verification_status' => :gitlab_main, + + # CI tables + 'ci_build_trace_sections' => :gitlab_ci, + 'ci_build_trace_section_names' => :gitlab_ci, + 'ci_daily_report_results' => :gitlab_ci, + 'ci_test_cases' => :gitlab_ci, + 'ci_test_case_failures' => :gitlab_ci, + + # leftovers from early implementation of partitioning + 'audit_events_part_5fc467ac26' => :gitlab_main, + 'web_hook_logs_part_0c5294f417' => :gitlab_main + }.freeze + + def self.table_schemas(tables) + tables.map { |table| table_schema(table) }.to_set + end + + def self.table_schema(name) + schema_name, table_name = name.split('.', 2) # Strip schema name like: `public.` + + # Most of names do not have schemas, ensure that this is table + unless table_name + table_name = schema_name + schema_name = nil + end + + # strip partition number of a form `loose_foreign_keys_deleted_records_1` + table_name.gsub!(/_[0-9]+$/, '') + + # Tables that are properly mapped + if gitlab_schema = tables_to_schema[table_name] + return gitlab_schema + end + + # Tables that are deleted, but we still need to reference them + if gitlab_schema = DELETED_TABLES[table_name] + return gitlab_schema + end + + # All tables from `information_schema.` are `:gitlab_shared` + return :gitlab_shared if schema_name == 'information_schema' + + # All tables that start with `_test_` are shared and ignored + return :gitlab_shared if table_name.start_with?('_test_') + + # All `pg_` tables are marked as `shared` + return :gitlab_shared if table_name.start_with?('pg_') + + # When undefined it's best to return a unique name so that we don't incorrectly assume that 2 undefined schemas belong on the same database + :"undefined_#{table_name}" + end + + def self.tables_to_schema + @tables_to_schema ||= YAML.load_file(Rails.root.join('lib/gitlab/database/gitlab_schemas.yml')) + end + end + end +end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml new file mode 100644 index 00000000000..66157e998a0 --- /dev/null +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -0,0 +1,543 @@ +abuse_reports: :gitlab_main +agent_group_authorizations: :gitlab_main +agent_project_authorizations: :gitlab_main +alert_management_alert_assignees: :gitlab_main +alert_management_alerts: :gitlab_main +alert_management_alert_user_mentions: :gitlab_main +alert_management_http_integrations: :gitlab_main +allowed_email_domains: :gitlab_main +analytics_cycle_analytics_group_stages: :gitlab_main +analytics_cycle_analytics_group_value_streams: :gitlab_main +analytics_cycle_analytics_issue_stage_events: :gitlab_main +analytics_cycle_analytics_merge_request_stage_events: :gitlab_main +analytics_cycle_analytics_project_stages: :gitlab_main +analytics_cycle_analytics_project_value_streams: :gitlab_main +analytics_cycle_analytics_stage_event_hashes: :gitlab_main +analytics_devops_adoption_segments: :gitlab_main +analytics_devops_adoption_snapshots: :gitlab_main +analytics_language_trend_repository_languages: :gitlab_main +analytics_usage_trends_measurements: :gitlab_main +appearances: :gitlab_main +application_settings: :gitlab_main +application_setting_terms: :gitlab_main +approval_merge_request_rules_approved_approvers: :gitlab_main +approval_merge_request_rules: :gitlab_main +approval_merge_request_rules_groups: :gitlab_main +approval_merge_request_rule_sources: :gitlab_main +approval_merge_request_rules_users: :gitlab_main +approval_project_rules: :gitlab_main +approval_project_rules_groups: :gitlab_main +approval_project_rules_protected_branches: :gitlab_main +approval_project_rules_users: :gitlab_main +approvals: :gitlab_main +approver_groups: :gitlab_main +approvers: :gitlab_main +ar_internal_metadata: :gitlab_shared +atlassian_identities: :gitlab_main +audit_events_external_audit_event_destinations: :gitlab_main +audit_events: :gitlab_main +authentication_events: :gitlab_main +award_emoji: :gitlab_main +aws_roles: :gitlab_main +background_migration_jobs: :gitlab_main +badges: :gitlab_main +banned_users: :gitlab_main +batched_background_migration_jobs: :gitlab_main +batched_background_migrations: :gitlab_main +board_assignees: :gitlab_main +board_group_recent_visits: :gitlab_main +board_labels: :gitlab_main +board_project_recent_visits: :gitlab_main +boards_epic_board_labels: :gitlab_main +boards_epic_board_positions: :gitlab_main +boards_epic_board_recent_visits: :gitlab_main +boards_epic_boards: :gitlab_main +boards_epic_lists: :gitlab_main +boards_epic_list_user_preferences: :gitlab_main +boards_epic_user_preferences: :gitlab_main +boards: :gitlab_main +board_user_preferences: :gitlab_main +broadcast_messages: :gitlab_main +bulk_import_configurations: :gitlab_main +bulk_import_entities: :gitlab_main +bulk_import_exports: :gitlab_main +bulk_import_export_uploads: :gitlab_main +bulk_import_failures: :gitlab_main +bulk_imports: :gitlab_main +bulk_import_trackers: :gitlab_main +chat_names: :gitlab_main +chat_teams: :gitlab_main +ci_build_needs: :gitlab_ci +ci_build_pending_states: :gitlab_ci +ci_build_report_results: :gitlab_ci +ci_builds: :gitlab_ci +ci_builds_metadata: :gitlab_ci +ci_builds_runner_session: :gitlab_ci +ci_build_trace_chunks: :gitlab_ci +ci_build_trace_metadata: :gitlab_ci +ci_daily_build_group_report_results: :gitlab_ci +ci_deleted_objects: :gitlab_ci +ci_freeze_periods: :gitlab_ci +ci_group_variables: :gitlab_ci +ci_instance_variables: :gitlab_ci +ci_job_artifacts: :gitlab_ci +ci_job_token_project_scope_links: :gitlab_ci +ci_job_variables: :gitlab_ci +ci_minutes_additional_packs: :gitlab_ci +ci_namespace_monthly_usages: :gitlab_ci +ci_pending_builds: :gitlab_ci +ci_pipeline_artifacts: :gitlab_ci +ci_pipeline_chat_data: :gitlab_ci +ci_pipeline_messages: :gitlab_ci +ci_pipeline_schedules: :gitlab_ci +ci_pipeline_schedule_variables: :gitlab_ci +ci_pipelines_config: :gitlab_ci +ci_pipelines: :gitlab_ci +ci_pipeline_variables: :gitlab_ci +ci_platform_metrics: :gitlab_ci +ci_project_monthly_usages: :gitlab_ci +ci_refs: :gitlab_ci +ci_resource_groups: :gitlab_ci +ci_resources: :gitlab_ci +ci_runner_namespaces: :gitlab_ci +ci_runner_projects: :gitlab_ci +ci_runners: :gitlab_ci +ci_running_builds: :gitlab_ci +ci_sources_pipelines: :gitlab_ci +ci_sources_projects: :gitlab_ci +ci_stages: :gitlab_ci +ci_subscriptions_projects: :gitlab_ci +ci_trigger_requests: :gitlab_ci +ci_triggers: :gitlab_ci +ci_unit_test_failures: :gitlab_ci +ci_unit_tests: :gitlab_ci +ci_variables: :gitlab_ci +cluster_agents: :gitlab_main +cluster_agent_tokens: :gitlab_main +cluster_groups: :gitlab_main +cluster_platforms_kubernetes: :gitlab_main +cluster_projects: :gitlab_main +cluster_providers_aws: :gitlab_main +cluster_providers_gcp: :gitlab_main +clusters_applications_cert_managers: :gitlab_main +clusters_applications_cilium: :gitlab_main +clusters_applications_crossplane: :gitlab_main +clusters_applications_elastic_stacks: :gitlab_main +clusters_applications_helm: :gitlab_main +clusters_applications_ingress: :gitlab_main +clusters_applications_jupyter: :gitlab_main +clusters_applications_knative: :gitlab_main +clusters_applications_prometheus: :gitlab_main +clusters_applications_runners: :gitlab_main +clusters: :gitlab_main +clusters_integration_elasticstack: :gitlab_main +clusters_integration_prometheus: :gitlab_main +clusters_kubernetes_namespaces: :gitlab_main +commit_user_mentions: :gitlab_main +compliance_management_frameworks: :gitlab_main +container_expiration_policies: :gitlab_main +container_repositories: :gitlab_main +content_blocked_states: :gitlab_main +conversational_development_index_metrics: :gitlab_main +coverage_fuzzing_corpuses: :gitlab_main +csv_issue_imports: :gitlab_main +custom_emoji: :gitlab_main +customer_relations_contacts: :gitlab_main +customer_relations_organizations: :gitlab_main +dast_profile_schedules: :gitlab_main +dast_profiles: :gitlab_main +dast_profiles_pipelines: :gitlab_main +dast_scanner_profiles_builds: :gitlab_main +dast_scanner_profiles: :gitlab_main +dast_site_profiles_builds: :gitlab_main +dast_site_profile_secret_variables: :gitlab_main +dast_site_profiles: :gitlab_main +dast_site_profiles_pipelines: :gitlab_main +dast_sites: :gitlab_main +dast_site_tokens: :gitlab_main +dast_site_validations: :gitlab_main +dependency_proxy_blobs: :gitlab_main +dependency_proxy_group_settings: :gitlab_main +dependency_proxy_image_ttl_group_policies: :gitlab_main +dependency_proxy_manifests: :gitlab_main +deploy_keys_projects: :gitlab_main +deployment_clusters: :gitlab_main +deployment_merge_requests: :gitlab_main +deployments: :gitlab_main +deploy_tokens: :gitlab_main +description_versions: :gitlab_main +design_management_designs: :gitlab_main +design_management_designs_versions: :gitlab_main +design_management_versions: :gitlab_main +design_user_mentions: :gitlab_main +detached_partitions: :gitlab_shared +diff_note_positions: :gitlab_main +dora_daily_metrics: :gitlab_main +draft_notes: :gitlab_main +elastic_index_settings: :gitlab_main +elastic_reindexing_slices: :gitlab_main +elastic_reindexing_subtasks: :gitlab_main +elastic_reindexing_tasks: :gitlab_main +elasticsearch_indexed_namespaces: :gitlab_main +elasticsearch_indexed_projects: :gitlab_main +emails: :gitlab_main +environments: :gitlab_main +epic_issues: :gitlab_main +epic_metrics: :gitlab_main +epics: :gitlab_main +epic_user_mentions: :gitlab_main +error_tracking_client_keys: :gitlab_main +error_tracking_error_events: :gitlab_main +error_tracking_errors: :gitlab_main +events: :gitlab_main +evidences: :gitlab_main +experiments: :gitlab_main +experiment_subjects: :gitlab_main +experiment_users: :gitlab_main +external_approval_rules: :gitlab_main +external_approval_rules_protected_branches: :gitlab_main +external_pull_requests: :gitlab_main +external_status_checks: :gitlab_main +external_status_checks_protected_branches: :gitlab_main +feature_gates: :gitlab_main +features: :gitlab_main +fork_network_members: :gitlab_main +fork_networks: :gitlab_main +geo_cache_invalidation_events: :gitlab_main +geo_container_repository_updated_events: :gitlab_main +geo_event_log: :gitlab_main +geo_events: :gitlab_main +geo_hashed_storage_attachments_events: :gitlab_main +geo_hashed_storage_migrated_events: :gitlab_main +geo_job_artifact_deleted_events: :gitlab_main +geo_lfs_object_deleted_events: :gitlab_main +geo_node_namespace_links: :gitlab_main +geo_nodes: :gitlab_main +geo_node_statuses: :gitlab_main +geo_repositories_changed_events: :gitlab_main +geo_repository_created_events: :gitlab_main +geo_repository_deleted_events: :gitlab_main +geo_repository_renamed_events: :gitlab_main +geo_repository_updated_events: :gitlab_main +geo_reset_checksum_events: :gitlab_main +gitlab_subscription_histories: :gitlab_main +gitlab_subscriptions: :gitlab_main +gpg_keys: :gitlab_main +gpg_key_subkeys: :gitlab_main +gpg_signatures: :gitlab_main +grafana_integrations: :gitlab_main +group_custom_attributes: :gitlab_main +group_deletion_schedules: :gitlab_main +group_deploy_keys: :gitlab_main +group_deploy_keys_groups: :gitlab_main +group_deploy_tokens: :gitlab_main +group_group_links: :gitlab_main +group_import_states: :gitlab_main +group_merge_request_approval_settings: :gitlab_main +group_repository_storage_moves: :gitlab_main +group_wiki_repositories: :gitlab_main +historical_data: :gitlab_main +identities: :gitlab_main +import_export_uploads: :gitlab_main +import_failures: :gitlab_main +incident_management_escalation_policies: :gitlab_main +incident_management_escalation_rules: :gitlab_main +incident_management_issuable_escalation_statuses: :gitlab_main +incident_management_oncall_participants: :gitlab_main +incident_management_oncall_rotations: :gitlab_main +incident_management_oncall_schedules: :gitlab_main +incident_management_oncall_shifts: :gitlab_main +incident_management_pending_alert_escalations: :gitlab_main +incident_management_pending_issue_escalations: :gitlab_main +index_statuses: :gitlab_main +in_product_marketing_emails: :gitlab_main +insights: :gitlab_main +integrations: :gitlab_main +internal_ids: :gitlab_main +ip_restrictions: :gitlab_main +issuable_metric_images: :gitlab_main +issuable_severities: :gitlab_main +issuable_slas: :gitlab_main +issue_assignees: :gitlab_main +issue_customer_relations_contacts: :gitlab_main +issue_email_participants: :gitlab_main +issue_links: :gitlab_main +issue_metrics: :gitlab_main +issues: :gitlab_main +issues_prometheus_alert_events: :gitlab_main +issues_self_managed_prometheus_alert_events: :gitlab_main +issue_tracker_data: :gitlab_main +issue_user_mentions: :gitlab_main +iterations_cadences: :gitlab_main +jira_connect_installations: :gitlab_main +jira_connect_subscriptions: :gitlab_main +jira_imports: :gitlab_main +jira_tracker_data: :gitlab_main +keys: :gitlab_main +label_links: :gitlab_main +label_priorities: :gitlab_main +labels: :gitlab_main +ldap_group_links: :gitlab_main +lfs_file_locks: :gitlab_main +lfs_objects: :gitlab_main +lfs_objects_projects: :gitlab_main +licenses: :gitlab_main +lists: :gitlab_main +list_user_preferences: :gitlab_main +loose_foreign_keys_deleted_records: :gitlab_shared +member_tasks: :gitlab_main +members: :gitlab_main +merge_request_assignees: :gitlab_main +merge_request_blocks: :gitlab_main +merge_request_cleanup_schedules: :gitlab_main +merge_request_context_commit_diff_files: :gitlab_main +merge_request_context_commits: :gitlab_main +merge_request_diff_commits: :gitlab_main +merge_request_diff_commit_users: :gitlab_main +merge_request_diff_details: :gitlab_main +merge_request_diff_files: :gitlab_main +merge_request_diffs: :gitlab_main +merge_request_metrics: :gitlab_main +merge_request_reviewers: :gitlab_main +merge_requests_closing_issues: :gitlab_main +merge_requests: :gitlab_main +merge_request_user_mentions: :gitlab_main +merge_trains: :gitlab_main +metrics_dashboard_annotations: :gitlab_main +metrics_users_starred_dashboards: :gitlab_main +milestone_releases: :gitlab_main +milestones: :gitlab_main +namespace_admin_notes: :gitlab_main +namespace_aggregation_schedules: :gitlab_main +namespace_limits: :gitlab_main +namespace_package_settings: :gitlab_main +namespace_root_storage_statistics: :gitlab_main +namespace_settings: :gitlab_main +namespaces: :gitlab_main +namespace_statistics: :gitlab_main +note_diff_files: :gitlab_main +notes: :gitlab_main +notification_settings: :gitlab_main +oauth_access_grants: :gitlab_main +oauth_access_tokens: :gitlab_main +oauth_applications: :gitlab_main +oauth_openid_requests: :gitlab_main +onboarding_progresses: :gitlab_main +operations_feature_flags_clients: :gitlab_main +operations_feature_flag_scopes: :gitlab_main +operations_feature_flags: :gitlab_main +operations_feature_flags_issues: :gitlab_main +operations_scopes: :gitlab_main +operations_strategies: :gitlab_main +operations_strategies_user_lists: :gitlab_main +operations_user_lists: :gitlab_main +packages_build_infos: :gitlab_main +packages_composer_cache_files: :gitlab_main +packages_composer_metadata: :gitlab_main +packages_conan_file_metadata: :gitlab_main +packages_conan_metadata: :gitlab_main +packages_debian_file_metadata: :gitlab_main +packages_debian_group_architectures: :gitlab_main +packages_debian_group_component_files: :gitlab_main +packages_debian_group_components: :gitlab_main +packages_debian_group_distribution_keys: :gitlab_main +packages_debian_group_distributions: :gitlab_main +packages_debian_project_architectures: :gitlab_main +packages_debian_project_component_files: :gitlab_main +packages_debian_project_components: :gitlab_main +packages_debian_project_distribution_keys: :gitlab_main +packages_debian_project_distributions: :gitlab_main +packages_debian_publications: :gitlab_main +packages_dependencies: :gitlab_main +packages_dependency_links: :gitlab_main +packages_events: :gitlab_main +packages_helm_file_metadata: :gitlab_main +packages_maven_metadata: :gitlab_main +packages_npm_metadata: :gitlab_main +packages_nuget_dependency_link_metadata: :gitlab_main +packages_nuget_metadata: :gitlab_main +packages_package_file_build_infos: :gitlab_main +packages_package_files: :gitlab_main +packages_packages: :gitlab_main +packages_pypi_metadata: :gitlab_main +packages_rubygems_metadata: :gitlab_main +packages_tags: :gitlab_main +pages_deployments: :gitlab_main +pages_domain_acme_orders: :gitlab_main +pages_domains: :gitlab_main +partitioned_foreign_keys: :gitlab_main +path_locks: :gitlab_main +personal_access_tokens: :gitlab_main +plan_limits: :gitlab_main +plans: :gitlab_main +pool_repositories: :gitlab_main +postgres_async_indexes: :gitlab_shared +postgres_foreign_keys: :gitlab_shared +postgres_index_bloat_estimates: :gitlab_shared +postgres_indexes: :gitlab_shared +postgres_partitioned_tables: :gitlab_shared +postgres_partitions: :gitlab_shared +postgres_reindex_actions: :gitlab_shared +postgres_reindex_queued_actions: :gitlab_main +product_analytics_events_experimental: :gitlab_main +programming_languages: :gitlab_main +project_access_tokens: :gitlab_main +project_alerting_settings: :gitlab_main +project_aliases: :gitlab_main +project_authorizations: :gitlab_main +project_auto_devops: :gitlab_main +project_ci_cd_settings: :gitlab_main +project_ci_feature_usages: :gitlab_main +project_compliance_framework_settings: :gitlab_main +project_custom_attributes: :gitlab_main +project_daily_statistics: :gitlab_main +project_deploy_tokens: :gitlab_main +project_error_tracking_settings: :gitlab_main +project_export_jobs: :gitlab_main +project_features: :gitlab_main +project_feature_usages: :gitlab_main +project_group_links: :gitlab_main +project_import_data: :gitlab_main +project_incident_management_settings: :gitlab_main +project_metrics_settings: :gitlab_main +project_mirror_data: :gitlab_main +project_pages_metadata: :gitlab_main +project_repositories: :gitlab_main +project_repository_states: :gitlab_main +project_repository_storage_moves: :gitlab_main +project_security_settings: :gitlab_main +project_settings: :gitlab_main +projects: :gitlab_main +project_statistics: :gitlab_main +project_topics: :gitlab_main +project_tracing_settings: :gitlab_main +prometheus_alert_events: :gitlab_main +prometheus_alerts: :gitlab_main +prometheus_metrics: :gitlab_main +protected_branches: :gitlab_main +protected_branch_merge_access_levels: :gitlab_main +protected_branch_push_access_levels: :gitlab_main +protected_branch_unprotect_access_levels: :gitlab_main +protected_environment_deploy_access_levels: :gitlab_main +protected_environments: :gitlab_main +protected_tag_create_access_levels: :gitlab_main +protected_tags: :gitlab_main +push_event_payloads: :gitlab_main +push_rules: :gitlab_main +raw_usage_data: :gitlab_main +redirect_routes: :gitlab_main +release_links: :gitlab_main +releases: :gitlab_main +remote_mirrors: :gitlab_main +repository_languages: :gitlab_main +required_code_owners_sections: :gitlab_main +requirements: :gitlab_main +requirements_management_test_reports: :gitlab_main +resource_iteration_events: :gitlab_main +resource_label_events: :gitlab_main +resource_milestone_events: :gitlab_main +resource_state_events: :gitlab_main +resource_weight_events: :gitlab_main +reviews: :gitlab_main +routes: :gitlab_main +saml_group_links: :gitlab_main +saml_providers: :gitlab_main +schema_migrations: :gitlab_shared +scim_identities: :gitlab_main +scim_oauth_access_tokens: :gitlab_main +security_findings: :gitlab_main +security_orchestration_policy_configurations: :gitlab_main +security_orchestration_policy_rule_schedules: :gitlab_main +security_scans: :gitlab_main +self_managed_prometheus_alert_events: :gitlab_main +sent_notifications: :gitlab_main +sentry_issues: :gitlab_main +serverless_domain_cluster: :gitlab_main +service_desk_settings: :gitlab_main +shards: :gitlab_main +slack_integrations: :gitlab_main +smartcard_identities: :gitlab_main +snippet_repositories: :gitlab_main +snippet_repository_storage_moves: :gitlab_main +snippets: :gitlab_main +snippet_statistics: :gitlab_main +snippet_user_mentions: :gitlab_main +software_license_policies: :gitlab_main +software_licenses: :gitlab_main +spam_logs: :gitlab_main +sprints: :gitlab_main +status_check_responses: :gitlab_main +status_page_published_incidents: :gitlab_main +status_page_settings: :gitlab_main +subscriptions: :gitlab_main +suggestions: :gitlab_main +system_note_metadata: :gitlab_main +taggings: :gitlab_ci +tags: :gitlab_ci +term_agreements: :gitlab_main +terraform_states: :gitlab_main +terraform_state_versions: :gitlab_main +timelogs: :gitlab_main +todos: :gitlab_main +token_with_ivs: :gitlab_main +topics: :gitlab_main +trending_projects: :gitlab_main +u2f_registrations: :gitlab_main +upcoming_reconciliations: :gitlab_main +uploads: :gitlab_main +user_agent_details: :gitlab_main +user_callouts: :gitlab_main +user_canonical_emails: :gitlab_main +user_credit_card_validations: :gitlab_main +user_custom_attributes: :gitlab_main +user_details: :gitlab_main +user_follow_users: :gitlab_main +user_group_callouts: :gitlab_main +user_highest_roles: :gitlab_main +user_interacted_projects: :gitlab_main +user_permission_export_uploads: :gitlab_main +user_preferences: :gitlab_main +users: :gitlab_main +users_ops_dashboard_projects: :gitlab_main +users_security_dashboard_projects: :gitlab_main +users_star_projects: :gitlab_main +users_statistics: :gitlab_main +user_statuses: :gitlab_main +user_synced_attributes_metadata: :gitlab_main +verification_codes: :gitlab_main +vulnerabilities: :gitlab_main +vulnerability_exports: :gitlab_main +vulnerability_external_issue_links: :gitlab_main +vulnerability_feedback: :gitlab_main +vulnerability_finding_evidence_assets: :gitlab_main +vulnerability_finding_evidence_headers: :gitlab_main +vulnerability_finding_evidence_requests: :gitlab_main +vulnerability_finding_evidence_responses: :gitlab_main +vulnerability_finding_evidences: :gitlab_main +vulnerability_finding_evidence_sources: :gitlab_main +vulnerability_finding_evidence_supporting_messages: :gitlab_main +vulnerability_finding_links: :gitlab_main +vulnerability_finding_signatures: :gitlab_main +vulnerability_findings_remediations: :gitlab_main +vulnerability_flags: :gitlab_main +vulnerability_historical_statistics: :gitlab_main +vulnerability_identifiers: :gitlab_main +vulnerability_issue_links: :gitlab_main +vulnerability_occurrence_identifiers: :gitlab_main +vulnerability_occurrence_pipelines: :gitlab_main +vulnerability_occurrences: :gitlab_main +vulnerability_remediations: :gitlab_main +vulnerability_scanners: :gitlab_main +vulnerability_statistics: :gitlab_main +vulnerability_user_mentions: :gitlab_main +webauthn_registrations: :gitlab_main +web_hook_logs: :gitlab_main +web_hooks: :gitlab_main +wiki_page_meta: :gitlab_main +wiki_page_slugs: :gitlab_main +work_item_types: :gitlab_main +x509_certificates: :gitlab_main +x509_commit_signatures: :gitlab_main +x509_issuers: :gitlab_main +zentao_tracker_data: :gitlab_main +zoom_meetings: :gitlab_main diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index 3e322e752b7..52eb0764ae3 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -26,7 +26,7 @@ module Gitlab return to_enum(__method__) unless block_given? base_models.each do |model| - yield model.connection.load_balancer + yield model.load_balancer end end diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index 6156515bd73..da313361073 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -7,7 +7,7 @@ module Gitlab class Configuration attr_accessor :hosts, :max_replication_difference, :max_replication_lag_time, :replica_check_interval, - :service_discovery, :model + :service_discovery # Creates a configuration object for the given ActiveRecord model. def self.for_model(model) @@ -41,6 +41,8 @@ module Gitlab end end + config.reuse_primary_connection! + config end @@ -59,6 +61,28 @@ module Gitlab disconnect_timeout: 120, use_tcp: false } + + # Temporary model for GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ + # To be removed with FF + @primary_model = nil + end + + def db_config_name + @model.connection_db_config.name.to_sym + end + + # With connection re-use the primary connection can be overwritten + # to be used from different model + def primary_connection_specification_name + (@primary_model || @model).connection_specification_name + end + + def primary_db_config + (@primary_model || @model).connection_db_config + end + + def replica_db_config + @model.connection_db_config end def pool_size @@ -86,6 +110,30 @@ module Gitlab def service_discovery_enabled? service_discovery[:record].present? end + + # TODO: This is temporary code to allow re-use of primary connection + # if the two connections are pointing to the same host. This is needed + # to properly support transaction visibility. + # + # This behavior is required to support [Phase 3](https://gitlab.com/groups/gitlab-org/-/epics/6160#progress). + # This method is meant to be removed as soon as it is finished. + # + # The remapping is done as-is: + # export GITLAB_LOAD_BALANCING_REUSE_PRIMARY_<name-of-connection>=<new-name-of-connection> + # + # Ex.: + # export GITLAB_LOAD_BALANCING_REUSE_PRIMARY_ci=main + # + def reuse_primary_connection! + new_connection = ENV["GITLAB_LOAD_BALANCING_REUSE_PRIMARY_#{db_config_name}"] + return unless new_connection.present? + + @primary_model = Gitlab::Database.database_base_models[new_connection.to_sym] + + unless @primary_model + raise "Invalid value for 'GITLAB_LOAD_BALANCING_REUSE_PRIMARY_#{db_config_name}=#{new_connection}'" + end + end end end end diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index 1be63da8896..a91df2eccdd 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -13,6 +13,13 @@ module Gitlab WriteInsideReadOnlyTransactionError = Class.new(StandardError) READ_ONLY_TRANSACTION_KEY = :load_balacing_read_only_transaction + # The load balancer returned by connection might be different + # between `model.connection.load_balancer` vs `model.load_balancer` + # + # The used `model.connection` is dependent on `use_model_load_balancing`. + # See more in: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73949. + # + # Always use `model.load_balancer` or `model.sticking`. attr_reader :load_balancer # These methods perform writes after which we need to stick to the diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 2be7f0baa60..1e27bcfc55d 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -12,7 +12,7 @@ module Gitlab REPLICA_SUFFIX = '_replica' - attr_reader :name, :host_list, :configuration + attr_reader :host_list, :configuration # configuration - An instance of `LoadBalancing::Configuration` that # contains the configuration details (such as the hosts) @@ -26,8 +26,10 @@ module Gitlab else HostList.new(configuration.hosts.map { |addr| Host.new(addr, self) }) end + end - @name = @configuration.model.connection_db_config.name.to_sym + def name + @configuration.db_config_name end def primary_only? @@ -64,7 +66,7 @@ module Gitlab # times before using the primary instead. will_retry = conflict_retried < @host_list.length * 3 - LoadBalancing::Logger.warn( + ::Gitlab::Database::LoadBalancing::Logger.warn( event: :host_query_conflict, message: 'Query conflict on host', conflict_retried: conflict_retried, @@ -89,7 +91,7 @@ module Gitlab end end - LoadBalancing::Logger.warn( + ::Gitlab::Database::LoadBalancing::Logger.warn( event: :no_secondaries_available, message: 'No secondaries were available, using primary instead', conflict_retried: conflict_retried, @@ -136,7 +138,7 @@ module Gitlab # Returns the transaction write location of the primary. def primary_write_location location = read_write do |connection| - ::Gitlab::Database.main.get_write_location(connection) + get_write_location(connection) end return location if location @@ -230,7 +232,7 @@ module Gitlab # host - An optional host name to use instead of the default one. # port - An optional port to connect to. def create_replica_connection_pool(pool_size, host = nil, port = nil) - db_config = pool.db_config + db_config = @configuration.replica_db_config env_config = db_config.configuration_hash.dup env_config[:pool] = pool_size @@ -255,22 +257,67 @@ module Gitlab # leverage that. def pool ActiveRecord::Base.connection_handler.retrieve_connection_pool( - @configuration.model.connection_specification_name, + @configuration.primary_connection_specification_name, role: ActiveRecord::Base.writing_role, shard: ActiveRecord::Base.default_shard ) || raise(::ActiveRecord::ConnectionNotEstablished) end + def wal_diff(location1, location2) + read_write do |connection| + lsn1 = connection.quote(location1) + lsn2 = connection.quote(location2) + + query = <<-SQL.squish + SELECT pg_wal_lsn_diff(#{lsn1}, #{lsn2}) + AS result + SQL + + row = connection.select_all(query).first + row['result'] if row + end + end + private def ensure_caching! - host.enable_query_cache! unless host.query_cache_enabled + return unless Rails.application.executor.active? + return if host.query_cache_enabled + + host.enable_query_cache! end def request_cache base = SafeRequestStore[:gitlab_load_balancer] ||= {} base[self] ||= {} end + + # @param [ActiveRecord::Connection] ar_connection + # @return [String] + def get_write_location(ar_connection) + use_new_load_balancer_query = Gitlab::Utils + .to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: true) + + sql = + if use_new_load_balancer_query + <<~NEWSQL + SELECT CASE + WHEN pg_is_in_recovery() = true AND EXISTS (SELECT 1 FROM pg_stat_get_wal_senders()) + THEN pg_last_wal_replay_lsn()::text + WHEN pg_is_in_recovery() = false + THEN pg_current_wal_insert_lsn()::text + ELSE NULL + END AS location; + NEWSQL + else + <<~SQL + SELECT pg_current_wal_insert_lsn()::text AS location + SQL + end + + row = ar_connection.select_all(sql).first + row['location'] if row + end end end end diff --git a/lib/gitlab/database/load_balancing/primary_host.rb b/lib/gitlab/database/load_balancing/primary_host.rb index 7070cc54d4b..fb52b384ddb 100644 --- a/lib/gitlab/database/load_balancing/primary_host.rb +++ b/lib/gitlab/database/load_balancing/primary_host.rb @@ -49,6 +49,11 @@ module Gitlab end def offline! + ::Gitlab::Database::LoadBalancing::Logger.warn( + event: :host_offline, + message: 'Marking primary host as offline' + ) + nil end diff --git a/lib/gitlab/database/load_balancing/rack_middleware.rb b/lib/gitlab/database/load_balancing/rack_middleware.rb index 7ce7649cc22..99b1c31b04b 100644 --- a/lib/gitlab/database/load_balancing/rack_middleware.rb +++ b/lib/gitlab/database/load_balancing/rack_middleware.rb @@ -38,8 +38,8 @@ module Gitlab def unstick_or_continue_sticking(env) namespaces_and_ids = sticking_namespaces(env) - namespaces_and_ids.each do |(model, namespace, id)| - model.sticking.unstick_or_continue_sticking(namespace, id) + namespaces_and_ids.each do |(sticking, namespace, id)| + sticking.unstick_or_continue_sticking(namespace, id) end end @@ -47,8 +47,8 @@ module Gitlab def stick_if_necessary(env) namespaces_and_ids = sticking_namespaces(env) - namespaces_and_ids.each do |model, namespace, id| - model.sticking.stick_if_necessary(namespace, id) + namespaces_and_ids.each do |sticking, namespace, id| + sticking.stick_if_necessary(namespace, id) end end @@ -74,7 +74,7 @@ module Gitlab # models that support load balancing. In the future (if we # determined this to be OK) we may be able to relax this. ::Gitlab::Database::LoadBalancing.base_models.map do |model| - [model, :user, warden.user.id] + [model.sticking, :user, warden.user.id] end elsif env[STICK_OBJECT].present? env[STICK_OBJECT].to_a diff --git a/lib/gitlab/database/load_balancing/setup.rb b/lib/gitlab/database/load_balancing/setup.rb index 3cce839a960..ef38f42f50b 100644 --- a/lib/gitlab/database/load_balancing/setup.rb +++ b/lib/gitlab/database/load_balancing/setup.rb @@ -5,7 +5,7 @@ module Gitlab module LoadBalancing # Class for setting up load balancing of a specific model. class Setup - attr_reader :configuration + attr_reader :model, :configuration def initialize(model, start_service_discovery: false) @model = model @@ -14,47 +14,102 @@ module Gitlab end def setup - disable_prepared_statements - setup_load_balancer + configure_connection + setup_connection_proxy setup_service_discovery + setup_feature_flag_to_model_load_balancing end - def disable_prepared_statements + def configure_connection db_config_object = @model.connection_db_config - config = - db_config_object.configuration_hash.merge(prepared_statements: false) + + hash = db_config_object.configuration_hash.merge( + prepared_statements: false, + pool: Gitlab::Database.default_pool_size + ) hash_config = ActiveRecord::DatabaseConfigurations::HashConfig.new( db_config_object.env_name, db_config_object.name, - config + hash ) @model.establish_connection(hash_config) end - def setup_load_balancer - lb = LoadBalancer.new(configuration) - + def setup_connection_proxy # We just use a simple `class_attribute` here so we don't need to # inject any modules and/or expose unnecessary methods. - @model.class_attribute(:connection) - @model.class_attribute(:sticking) + setup_class_attribute(:load_balancer, load_balancer) + setup_class_attribute(:connection, ConnectionProxy.new(load_balancer)) + setup_class_attribute(:sticking, Sticking.new(load_balancer)) + end + + # TODO: This is temporary code to gradually redirect traffic to use + # a dedicated DB replicas, or DB primaries (depending on configuration) + # This implements a sticky behavior for the current request if enabled. + # + # This is needed for Phase 3 and Phase 4 of application rollout + # https://gitlab.com/groups/gitlab-org/-/epics/6160#progress + # + # If `GITLAB_USE_MODEL_LOAD_BALANCING` is set, its value is preferred + # Otherwise, a `use_model_load_balancing` FF value is used + def setup_feature_flag_to_model_load_balancing + return if active_record_base? - @model.connection = ConnectionProxy.new(lb) - @model.sticking = Sticking.new(lb) + @model.singleton_class.prepend(ModelLoadBalancingFeatureFlagMixin) end def setup_service_discovery return unless configuration.service_discovery_enabled? - lb = @model.connection.load_balancer - sv = ServiceDiscovery.new(lb, **configuration.service_discovery) + sv = ServiceDiscovery.new(load_balancer, **configuration.service_discovery) sv.perform_service_discovery sv.start if @start_service_discovery end + + def load_balancer + @load_balancer ||= LoadBalancer.new(configuration) + end + + private + + def setup_class_attribute(attribute, value) + @model.class_attribute(attribute) + @model.public_send("#{attribute}=", value) # rubocop:disable GitlabSecurity/PublicSend + end + + def active_record_base? + @model == ActiveRecord::Base + end + + module ModelLoadBalancingFeatureFlagMixin + extend ActiveSupport::Concern + + def use_model_load_balancing? + # Cache environment variable and return env variable first if defined + use_model_load_balancing_env = Gitlab::Utils.to_boolean(ENV["GITLAB_USE_MODEL_LOAD_BALANCING"]) + + unless use_model_load_balancing_env.nil? + return use_model_load_balancing_env + end + + # Check a feature flag using RequestStore (if active) + return false unless Gitlab::SafeRequestStore.active? + + Gitlab::SafeRequestStore.fetch(:use_model_load_balancing) do + Feature.enabled?(:use_model_load_balancing, default_enabled: :yaml) + end + end + + # rubocop:disable Database/MultipleDatabases + def connection + use_model_load_balancing? ? super : ActiveRecord::Base.connection + end + # rubocop:enable Database/MultipleDatabases + end end end end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index f0c7016032b..b9acc36b4cc 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -13,7 +13,7 @@ module Gitlab job['load_balancing_strategy'] = strategy.to_s if use_primary?(strategy) - Session.current.use_primary! + ::Gitlab::Database::LoadBalancing::Session.current.use_primary! elsif strategy == :retry raise JobReplicaNotUpToDate, "Sidekiq job #{worker_class} JID-#{job['jid']} couldn't use the replica."\ " Replica was not up to date." @@ -29,8 +29,8 @@ module Gitlab private def clear - LoadBalancing.release_hosts - Session.clear_session + ::Gitlab::Database::LoadBalancing.release_hosts + ::Gitlab::Database::LoadBalancing::Session.clear_session end def use_primary?(strategy) @@ -66,7 +66,7 @@ module Gitlab def legacy_wal_location(job) wal_location = job['database_write_location'] || job['database_replica_location'] - { Gitlab::Database::MAIN_DATABASE_NAME.to_sym => wal_location } if wal_location + { ::Gitlab::Database::MAIN_DATABASE_NAME.to_sym => wal_location } if wal_location end def load_balancing_available?(worker_class) @@ -90,7 +90,7 @@ module Gitlab end def databases_in_sync?(wal_locations) - LoadBalancing.each_load_balancer.all? do |lb| + ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb| if (location = wal_locations[lb.name]) lb.select_up_to_date_host(location) else diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb index df4ad18581f..834e9c6d3c6 100644 --- a/lib/gitlab/database/load_balancing/sticking.rb +++ b/lib/gitlab/database/load_balancing/sticking.rb @@ -12,7 +12,6 @@ module Gitlab def initialize(load_balancer) @load_balancer = load_balancer - @model = load_balancer.configuration.model end # Unsticks or continues sticking the current request. @@ -27,8 +26,8 @@ module Gitlab def stick_or_unstick_request(env, namespace, id) unstick_or_continue_sticking(namespace, id) - env[RackMiddleware::STICK_OBJECT] ||= Set.new - env[RackMiddleware::STICK_OBJECT] << [@model, namespace, id] + env[::Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT] ||= Set.new + env[::Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT] << [self, namespace, id] end # Sticks to the primary if a write was performed. diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 9968096b1f6..7dce4fa0ce2 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -10,8 +10,6 @@ module Gitlab # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 - - PERMITTED_TIMESTAMP_COLUMNS = %i[created_at updated_at deleted_at].to_set.freeze DEFAULT_TIMESTAMP_COLUMNS = %i[created_at updated_at].freeze # Adds `created_at` and `updated_at` columns with timezone information. @@ -28,33 +26,23 @@ module Gitlab # :default - The default value for the column. # :null - When set to `true` the column will allow NULL values. # The default is to not allow NULL values. - # :columns - the column names to create. Must be one - # of `Gitlab::Database::MigrationHelpers::PERMITTED_TIMESTAMP_COLUMNS`. + # :columns - the column names to create. Must end with `_at`. # Default value: `DEFAULT_TIMESTAMP_COLUMNS` # # All options are optional. def add_timestamps_with_timezone(table_name, options = {}) - options[:null] = false if options[:null].nil? columns = options.fetch(:columns, DEFAULT_TIMESTAMP_COLUMNS) - default_value = options[:default] - - validate_not_in_transaction!(:add_timestamps_with_timezone, 'with default value') if default_value columns.each do |column_name| validate_timestamp_column_name!(column_name) - # If default value is presented, use `add_column_with_default` method instead. - if default_value - add_column_with_default( - table_name, - column_name, - :datetime_with_timezone, - default: default_value, - allow_null: options[:null] - ) - else - add_column(table_name, column_name, :datetime_with_timezone, **options) - end + add_column( + table_name, + column_name, + :datetime_with_timezone, + default: options[:default], + null: options[:null] || false + ) end end @@ -147,8 +135,18 @@ module Gitlab options = options.merge({ algorithm: :concurrently }) if index_exists?(table_name, column_name, **options) - Gitlab::AppLogger.warn "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" - return + name = options[:name] || index_name(table_name, column_name) + _, schema = table_name.to_s.split('.').reverse + + if index_invalid?(name, schema: schema) + say "Index being recreated because the existing version was INVALID: table_name: #{table_name}, column_name: #{column_name}" + + remove_concurrent_index_by_name(table_name, name) + else + say "Index not created because it already exists (this may be due to an aborted migration or similar): table_name: #{table_name}, column_name: #{column_name}" + + return + end end disable_statement_timeout do @@ -159,6 +157,23 @@ module Gitlab unprepare_async_index(table_name, column_name, **options) end + def index_invalid?(index_name, schema: nil) + index_name = connection.quote(index_name) + schema = connection.quote(schema) if schema + schema ||= 'current_schema()' + + connection.select_value(<<~SQL) + select not i.indisvalid + from pg_class c + inner join pg_index i + on c.oid = i.indexrelid + inner join pg_namespace n + on n.oid = c.relnamespace + where n.nspname = #{schema} + and c.relname = #{index_name} + SQL + end + # Removes an existed index, concurrently # # Example: @@ -1245,8 +1260,8 @@ module Gitlab def check_trigger_permissions!(table) unless Grant.create_and_execute_trigger?(table) - dbname = Database.main.database_name - user = Database.main.username + dbname = ApplicationRecord.database.database_name + user = ApplicationRecord.database.username raise <<-EOF Your database user is not allowed to create, drop, or execute triggers on the @@ -1568,8 +1583,8 @@ into similar problems in the future (e.g. when new tables are created). def create_extension(extension) execute('CREATE EXTENSION IF NOT EXISTS %s' % extension) rescue ActiveRecord::StatementInvalid => e - dbname = Database.main.database_name - user = Database.main.username + dbname = ApplicationRecord.database.database_name + user = ApplicationRecord.database.username warn(<<~MSG) if e.to_s =~ /permission denied/ GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but @@ -1596,8 +1611,8 @@ into similar problems in the future (e.g. when new tables are created). def drop_extension(extension) execute('DROP EXTENSION IF EXISTS %s' % extension) rescue ActiveRecord::StatementInvalid => e - dbname = Database.main.database_name - user = Database.main.username + dbname = ApplicationRecord.database.database_name + user = ApplicationRecord.database.username warn(<<~MSG) if e.to_s =~ /permission denied/ This migration attempts to drop the PostgreSQL extension '#{extension}' @@ -1791,11 +1806,11 @@ into similar problems in the future (e.g. when new tables are created). end def validate_timestamp_column_name!(column_name) - return if PERMITTED_TIMESTAMP_COLUMNS.member?(column_name) + return if column_name.to_s.end_with?('_at') raise <<~MESSAGE Illegal timestamp column name! Got #{column_name}. - Must be one of: #{PERMITTED_TIMESTAMP_COLUMNS.to_a} + Must end with `_at`} MESSAGE end diff --git a/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb b/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb index d9ef5ab462e..8a37e619285 100644 --- a/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb +++ b/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb @@ -31,10 +31,10 @@ module Gitlab namespace_options = options.merge(null: true, default: nil) - add_column(:namespace_settings, setting_name, type, namespace_options) + add_column(:namespace_settings, setting_name, type, **namespace_options) add_column(:namespace_settings, lock_column_name, :boolean, default: false, null: false) - add_column(:application_settings, setting_name, type, options) + add_column(:application_settings, setting_name, type, **options) add_column(:application_settings, lock_column_name, :boolean, default: false, null: false) end diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb index 54eedec3c7b..a494c357950 100644 --- a/lib/gitlab/database/migrations/observation.rb +++ b/lib/gitlab/database/migrations/observation.rb @@ -1,3 +1,4 @@ +# rubocop:disable Naming/FileName # frozen_string_literal: true module Gitlab @@ -14,3 +15,5 @@ module Gitlab end end end + +# rubocop:enable Naming/FileName diff --git a/lib/gitlab/database/migrations/observers.rb b/lib/gitlab/database/migrations/observers.rb index 140b3feed64..b890e62c2d0 100644 --- a/lib/gitlab/database/migrations/observers.rb +++ b/lib/gitlab/database/migrations/observers.rb @@ -9,7 +9,8 @@ module Gitlab TotalDatabaseSizeChange, QueryStatistics, QueryLog, - QueryDetails + QueryDetails, + TransactionDuration ] end end diff --git a/lib/gitlab/database/migrations/observers/transaction_duration.rb b/lib/gitlab/database/migrations/observers/transaction_duration.rb new file mode 100644 index 00000000000..a96b94334cf --- /dev/null +++ b/lib/gitlab/database/migrations/observers/transaction_duration.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + class TransactionDuration < MigrationObserver + def before + file_path = File.join(output_dir, "#{observation.version}_#{observation.name}-transaction-duration.json") + @file = File.open(file_path, 'wb') + @writer = Oj::StreamWriter.new(@file, {}) + @writer.push_array + @subscriber = ActiveSupport::Notifications.subscribe('transaction.active_record') do |*args| + record_sql_event(*args) + end + end + + def after + ActiveSupport::Notifications.unsubscribe(@subscriber) + @writer.pop_all + @writer.flush + @file.close + end + + def record + # no-op + end + + def record_sql_event(_name, started, finished, _unique_id, payload) + return if payload[:transaction_type] == :fake_transaction + + @writer.push_value({ + start_time: started.iso8601(6), + end_time: finished.iso8601(6), + transaction_type: payload[:transaction_type] + }) + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index 71fb995577a..1343354715a 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -3,20 +3,83 @@ module Gitlab module Database module Partitioning - def self.register_models(models) - registered_models.merge(models) - end + class TableWithoutModel + include PartitionedTable::ClassMethods - def self.registered_models - @registered_models ||= Set.new - end + attr_reader :table_name + + def initialize(table_name:, partitioned_column:, strategy:) + @table_name = table_name + partitioned_by(partitioned_column, strategy: strategy) + end - def self.sync_partitions(models_to_sync = registered_models) - MultiDatabasePartitionManager.new(models_to_sync).sync_partitions + def connection + Gitlab::Database::SharedModel.connection + end end - def self.drop_detached_partitions - MultiDatabasePartitionDropper.new.drop_detached_partitions + class << self + def register_models(models) + models.each do |model| + raise "#{model} should have partitioning strategy defined" unless model.respond_to?(:partitioning_strategy) + + registered_models << model + end + end + + def register_tables(tables) + registered_tables.merge(tables) + end + + def sync_partitions_ignore_db_error + sync_partitions unless ENV['DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP'] + rescue ActiveRecord::ActiveRecordError, PG::Error + # ignore - happens when Rake tasks yet have to create a database, e.g. for testing + end + + def sync_partitions(models_to_sync = registered_for_sync) + Gitlab::AppLogger.info(message: 'Syncing dynamic postgres partitions') + + Gitlab::Database::EachDatabase.each_model_connection(models_to_sync) do |model| + PartitionManager.new(model).sync_partitions + end + + Gitlab::AppLogger.info(message: 'Finished sync of dynamic postgres partitions') + end + + def report_metrics(models_to_monitor = registered_models) + partition_monitoring = PartitionMonitoring.new + + Gitlab::Database::EachDatabase.each_model_connection(models_to_monitor) do |model| + partition_monitoring.report_metrics_for_model(model) + end + end + + def drop_detached_partitions + Gitlab::AppLogger.info(message: 'Dropping detached postgres partitions') + + Gitlab::Database::EachDatabase.each_database_connection do + DetachedPartitionDropper.new.perform + end + + Gitlab::AppLogger.info(message: 'Finished dropping detached postgres partitions') + end + + def registered_models + @registered_models ||= Set.new + end + + def registered_tables + @registered_tables ||= Set.new + end + + private + + def registered_for_sync + registered_models + registered_tables.map do |table| + TableWithoutModel.new(**table) + end + end end end end diff --git a/lib/gitlab/database/partitioning/detached_partition_dropper.rb b/lib/gitlab/database/partitioning/detached_partition_dropper.rb index 3e7ddece20b..593824384b5 100644 --- a/lib/gitlab/database/partitioning/detached_partition_dropper.rb +++ b/lib/gitlab/database/partitioning/detached_partition_dropper.rb @@ -9,13 +9,10 @@ module Gitlab Gitlab::AppLogger.info(message: "Checking for previously detached partitions to drop") Postgresql::DetachedPartition.ready_to_drop.find_each do |detached_partition| - connection.transaction do - # Another process may have already dropped the table and deleted this entry - next unless (detached_partition = Postgresql::DetachedPartition.lock.find_by(id: detached_partition.id)) - - drop_detached_partition(detached_partition.table_name) - - detached_partition.destroy! + if partition_attached?(qualify_partition_name(detached_partition.table_name)) + unmark_partition(detached_partition) + else + drop_partition(detached_partition) end rescue StandardError => e Gitlab::AppLogger.error(message: "Failed to drop previously detached partition", @@ -27,31 +24,100 @@ module Gitlab private + def unmark_partition(detached_partition) + connection.transaction do + # Another process may have already encountered this case and deleted this entry + next unless try_lock_detached_partition(detached_partition.id) + + # The current partition was scheduled for deletion incorrectly + # Dropping it now could delete in-use data and take locks that interrupt other database activity + Gitlab::AppLogger.error(message: "Prevented an attempt to drop an attached database partition", partition_name: detached_partition.table_name) + detached_partition.destroy! + end + end + + def drop_partition(detached_partition) + remove_foreign_keys(detached_partition) + + connection.transaction do + # Another process may have already dropped the table and deleted this entry + next unless try_lock_detached_partition(detached_partition.id) + + drop_detached_partition(detached_partition.table_name) + + detached_partition.destroy! + end + end + + def remove_foreign_keys(detached_partition) + partition_identifier = qualify_partition_name(detached_partition.table_name) + + # We want to load all of these into memory at once to get a consistent view to loop over, + # since we'll be deleting from this list as we go + fks_to_drop = PostgresForeignKey.by_constrained_table_identifier(partition_identifier).to_a + fks_to_drop.each do |foreign_key| + drop_foreign_key_if_present(detached_partition, foreign_key) + end + end + + # Drops the given foreign key for the given detached partition, but only if another process has not already + # detached the partition first. This method must be safe to call even if the associated partition table has already + # been detached, as it could be called by multiple processes at once. + def drop_foreign_key_if_present(detached_partition, foreign_key) + # It is important to only drop one foreign key per transaction. + # Dropping a foreign key takes an ACCESS EXCLUSIVE lock on both tables participating in the foreign key. + + partition_identifier = qualify_partition_name(detached_partition.table_name) + with_lock_retries do + connection.transaction(requires_new: false) do + next unless try_lock_detached_partition(detached_partition.id) + + # Another process may have already dropped this foreign key + next unless PostgresForeignKey.by_constrained_table_identifier(partition_identifier).where(name: foreign_key.name).exists? + + connection.execute("ALTER TABLE #{connection.quote_table_name(partition_identifier)} DROP CONSTRAINT #{connection.quote_table_name(foreign_key.name)}") + + Gitlab::AppLogger.info(message: "Dropped foreign key for previously detached partition", + partition_name: detached_partition.table_name, + referenced_table_name: foreign_key.referenced_table_identifier, + foreign_key_name: foreign_key.name) + end + end + end + def drop_detached_partition(partition_name) partition_identifier = qualify_partition_name(partition_name) - if partition_detached?(partition_identifier) - connection.drop_table(partition_identifier, if_exists: true) + connection.drop_table(partition_identifier, if_exists: true) - Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: partition_name) - else - Gitlab::AppLogger.error(message: "Attempt to drop attached database partition", partition_name: partition_name) - end + Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: partition_name) end def qualify_partition_name(table_name) "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{table_name}" end - def partition_detached?(partition_identifier) + def partition_attached?(partition_identifier) # PostgresPartition checks the pg_inherits view, so our partition will only show here if it's still attached # and thus should not be dropped - !Gitlab::Database::PostgresPartition.for_identifier(partition_identifier).exists? + Gitlab::Database::PostgresPartition.for_identifier(partition_identifier).exists? + end + + def try_lock_detached_partition(id) + Postgresql::DetachedPartition.lock.find_by(id: id).present? end def connection Postgresql::DetachedPartition.connection end + + def with_lock_retries(&block) + Gitlab::Database::WithLockRetries.new( + klass: self.class, + logger: Gitlab::AppLogger, + connection: connection + ).run(raise_on_exhaustion: true, &block) + end end end end diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index 4cdde5bf2f1..c93e775d7ed 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -96,10 +96,6 @@ module Gitlab def oldest_active_date (Date.today - retain_for).beginning_of_month end - - def connection - ActiveRecord::Base.connection - end end end end diff --git a/lib/gitlab/database/partitioning/multi_database_partition_dropper.rb b/lib/gitlab/database/partitioning/multi_database_partition_dropper.rb deleted file mode 100644 index 769b658bae4..00000000000 --- a/lib/gitlab/database/partitioning/multi_database_partition_dropper.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module Partitioning - class MultiDatabasePartitionDropper - def drop_detached_partitions - Gitlab::AppLogger.info(message: "Dropping detached postgres partitions") - - each_database_connection do |name, connection| - Gitlab::Database::SharedModel.using_connection(connection) do - Gitlab::AppLogger.debug(message: "Switched database connection", connection_name: name) - - DetachedPartitionDropper.new.perform - end - end - - Gitlab::AppLogger.info(message: "Finished dropping detached postgres partitions") - end - - private - - def each_database_connection - databases.each_pair do |name, connection_wrapper| - yield name, connection_wrapper.scope.connection - end - end - - def databases - Gitlab::Database.databases - end - end - end - end -end diff --git a/lib/gitlab/database/partitioning/multi_database_partition_manager.rb b/lib/gitlab/database/partitioning/multi_database_partition_manager.rb deleted file mode 100644 index 5a93e3fb1fb..00000000000 --- a/lib/gitlab/database/partitioning/multi_database_partition_manager.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module Partitioning - class MultiDatabasePartitionManager - def initialize(models) - @models = models - end - - def sync_partitions - Gitlab::AppLogger.info(message: "Syncing dynamic postgres partitions") - - models.each do |model| - Gitlab::Database::SharedModel.using_connection(model.connection) do - Gitlab::AppLogger.debug(message: "Switched database connection", - connection_name: connection_name, - table_name: model.table_name) - - PartitionManager.new(model).sync_partitions - end - end - - Gitlab::AppLogger.info(message: "Finished sync of dynamic postgres partitions") - end - - private - - attr_reader :models - - def connection_name - Gitlab::Database::SharedModel.connection.pool.db_config.name - end - end - end - end -end diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb index e5b561fc447..1a23f58285d 100644 --- a/lib/gitlab/database/partitioning/partition_monitoring.rb +++ b/lib/gitlab/database/partitioning/partition_monitoring.rb @@ -4,20 +4,12 @@ module Gitlab module Database module Partitioning class PartitionMonitoring - attr_reader :models + def report_metrics_for_model(model) + strategy = model.partitioning_strategy - def initialize(models = Gitlab::Database::Partitioning.registered_models) - @models = models - end - - def report_metrics - models.each do |model| - strategy = model.partitioning_strategy - - gauge_present.set({ table: model.table_name }, strategy.current_partitions.size) - gauge_missing.set({ table: model.table_name }, strategy.missing_partitions.size) - gauge_extra.set({ table: model.table_name }, strategy.extra_partitions.size) - end + gauge_present.set({ table: model.table_name }, strategy.current_partitions.size) + gauge_missing.set({ table: model.table_name }, strategy.missing_partitions.size) + gauge_extra.set({ table: model.table_name }, strategy.extra_partitions.size) end private diff --git a/lib/gitlab/database/partitioning/replace_table.rb b/lib/gitlab/database/partitioning/replace_table.rb index 6f6af223fa2..a7686e97553 100644 --- a/lib/gitlab/database/partitioning/replace_table.rb +++ b/lib/gitlab/database/partitioning/replace_table.rb @@ -9,7 +9,8 @@ module Gitlab attr_reader :original_table, :replacement_table, :replaced_table, :primary_key_column, :sequence, :original_primary_key, :replacement_primary_key, :replaced_primary_key - def initialize(original_table, replacement_table, replaced_table, primary_key_column) + def initialize(connection, original_table, replacement_table, replaced_table, primary_key_column) + @connection = connection @original_table = original_table @replacement_table = replacement_table @replaced_table = replaced_table @@ -29,10 +30,8 @@ module Gitlab private + attr_reader :connection delegate :execute, :quote_table_name, :quote_column_name, to: :connection - def connection - @connection ||= ActiveRecord::Base.connection - end def default_sequence(table, column) "#{table}_#{column}_seq" diff --git a/lib/gitlab/database/partitioning/time_partition.rb b/lib/gitlab/database/partitioning/time_partition.rb index e09ca483549..649687bdd12 100644 --- a/lib/gitlab/database/partitioning/time_partition.rb +++ b/lib/gitlab/database/partitioning/time_partition.rb @@ -87,7 +87,7 @@ module Gitlab end def conn - @conn ||= ActiveRecord::Base.connection + @conn ||= Gitlab::Database::SharedModel.connection end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 0dc9f92e4c8..c382d2f0715 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -428,8 +428,8 @@ module Gitlab end def replace_table(original_table_name, replacement_table_name, replaced_table_name, primary_key_name) - replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(original_table_name.to_s, - replacement_table_name, replaced_table_name, primary_key_name) + replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(connection, + original_table_name.to_s, replacement_table_name, replaced_table_name, primary_key_name) transaction do drop_sync_trigger(original_table_name) diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb index 72640f8785d..241b6f009f7 100644 --- a/lib/gitlab/database/postgres_foreign_key.rb +++ b/lib/gitlab/database/postgres_foreign_key.rb @@ -10,6 +10,12 @@ module Gitlab where(referenced_table_identifier: identifier) end + + scope :by_constrained_table_identifier, ->(identifier) do + raise ArgumentError, "Constrained table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + + where(constrained_table_identifier: identifier) + end end end end diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb index 2e3f674cf82..4e973efebca 100644 --- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb +++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb @@ -57,7 +57,7 @@ module Gitlab # @param finish final pkey range # @return [Gitlab::Database::PostgresHll::Buckets] HyperLogLog data structure instance that can estimate number of unique elements def execute(batch_size: nil, start: nil, finish: nil) - raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? # rubocop: disable Database/MultipleDatabases + raise 'BatchCount can not be run inside a transaction' if transaction_open? batch_size ||= DEFAULT_BATCH_SIZE start = actual_start(start) @@ -79,6 +79,10 @@ module Gitlab private + def transaction_open? + @relation.connection.transaction_open? + end + def unwanted_configuration?(start, finish, batch_size) batch_size <= MIN_REQUIRED_BATCH_SIZE || (finish - start) >= MAX_DATA_VOLUME || diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index 1079bfdeda3..4a9d8728c83 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -2,7 +2,7 @@ module Gitlab module Database - class PostgresIndex < ActiveRecord::Base + class PostgresIndex < SharedModel include Gitlab::Utils::StrongMemoize self.table_name = 'postgres_indexes' @@ -11,6 +11,7 @@ module Gitlab has_one :bloat_estimate, class_name: 'Gitlab::Database::PostgresIndexBloatEstimate', foreign_key: :identifier has_many :reindexing_actions, class_name: 'Gitlab::Database::Reindexing::ReindexAction', foreign_key: :index_identifier + has_many :queued_reindexing_actions, class_name: 'Gitlab::Database::Reindexing::QueuedAction', foreign_key: :index_identifier scope :by_identifier, ->(identifier) do raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ diff --git a/lib/gitlab/database/postgres_index_bloat_estimate.rb b/lib/gitlab/database/postgres_index_bloat_estimate.rb index 379227bf87c..5c9b5777b74 100644 --- a/lib/gitlab/database/postgres_index_bloat_estimate.rb +++ b/lib/gitlab/database/postgres_index_bloat_estimate.rb @@ -6,7 +6,7 @@ module Gitlab # for all indexes can be expensive in a large database. # # Best used on a per-index basis. - class PostgresIndexBloatEstimate < ActiveRecord::Base + class PostgresIndexBloatEstimate < SharedModel self.table_name = 'postgres_index_bloat_estimates' self.primary_key = 'identifier' diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb new file mode 100644 index 00000000000..0f285688876 --- /dev/null +++ b/lib/gitlab/database/query_analyzer.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # The purpose of this class is to implement a various query analyzers based on `pg_query` + # And process them all via `Gitlab::Database::QueryAnalyzers::*` + # + # Sometimes this might cause errors in specs. + # This is best to be disable with `describe '...', query_analyzers: false do` + class QueryAnalyzer + include ::Singleton + + Parsed = Struct.new( + :sql, :connection, :pg + ) + + attr_reader :all_analyzers + + def initialize + @all_analyzers = [] + end + + def hook! + @subscriber = ActiveSupport::Notifications.subscribe('sql.active_record') do |event| + # In some cases analyzer code might trigger another SQL call + # to avoid stack too deep this detects recursive call of subscriber + with_ignored_recursive_calls do + process_sql(event.payload[:sql], event.payload[:connection]) + end + end + end + + def within + # Due to singleton nature of analyzers + # only an outer invocation of the `.within` + # is allowed to initialize them + return yield if already_within? + + begin! + + begin + yield + ensure + end! + end + end + + def already_within? + # If analyzers are set they are already configured + !enabled_analyzers.nil? + end + + def process_sql(sql, connection) + analyzers = enabled_analyzers + return unless analyzers&.any? + + parsed = parse(sql, connection) + return unless parsed + + analyzers.each do |analyzer| + next if analyzer.suppressed? + + analyzer.analyze(parsed) + rescue StandardError => e + # We catch all standard errors to prevent validation errors to introduce fatal errors in production + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + end + + private + + # Enable query analyzers + def begin! + analyzers = all_analyzers.select do |analyzer| + if analyzer.enabled? + analyzer.begin! + + true + end + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + + false + end + + Thread.current[:query_analyzer_enabled_analyzers] = analyzers + end + + # Disable enabled query analyzers + def end! + enabled_analyzers.select do |analyzer| + analyzer.end! + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + + Thread.current[:query_analyzer_enabled_analyzers] = nil + end + + def enabled_analyzers + Thread.current[:query_analyzer_enabled_analyzers] + end + + def parse(sql, connection) + parsed = PgQuery.parse(sql) + return unless parsed + + normalized = PgQuery.normalize(sql) + Parsed.new(normalized, connection, parsed) + rescue PgQuery::ParseError => e + # Ignore PgQuery parse errors (due to depth limit or other reasons) + Gitlab::ErrorTracking.track_exception(e) + + nil + end + + def with_ignored_recursive_calls + return if Thread.current[:query_analyzer_recursive] + + begin + Thread.current[:query_analyzer_recursive] = true + yield + ensure + Thread.current[:query_analyzer_recursive] = nil + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/base.rb b/lib/gitlab/database/query_analyzers/base.rb new file mode 100644 index 00000000000..e8066f7a706 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/base.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class Base + def self.suppressed? + Thread.current[self.suppress_key] + end + + def self.suppress=(value) + Thread.current[self.suppress_key] = value + end + + def self.with_suppressed(value = true, &blk) + previous = self.suppressed? + self.suppress = value + yield + ensure + self.suppress = previous + end + + def self.begin! + Thread.current[self.context_key] = {} + end + + def self.end! + Thread.current[self.context_key] = nil + end + + def self.context + Thread.current[self.context_key] + end + + def self.enabled? + raise NotImplementedError + end + + def self.analyze(parsed) + raise NotImplementedError + end + + def self.context_key + "#{self.class.name}_context" + end + + def self.suppress_key + "#{self.class.name}_suppressed" + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb new file mode 100644 index 00000000000..06e2b114c91 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + # The purpose of this analyzer is to observe via prometheus metrics + # all unique schemas observed on a given connection + # + # This effectively allows to do sample 1% or 0.01% of queries hitting + # system and observe if on a given connection we observe queries that + # are misaligned (`ci_replica` sees queries doing accessing only `gitlab_main`) + # + class GitlabSchemasMetrics < Base + class << self + def enabled? + ::Feature::FlipperFeature.table_exists? && + Feature.enabled?(:query_analyzer_gitlab_schema_metrics) + end + + def analyze(parsed) + db_config_name = ::Gitlab::Database.db_config_name(parsed.connection) + return unless db_config_name + + gitlab_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(parsed.pg.tables) + return if gitlab_schemas.empty? + + # to reduce amount of labels sort schemas used + gitlab_schemas = gitlab_schemas.to_a.sort.join(",") + + schemas_metrics.increment({ + gitlab_schemas: gitlab_schemas, + db_config_name: db_config_name + }) + end + + def schemas_metrics + @schemas_metrics ||= ::Gitlab::Metrics.counter( + :gitlab_database_decomposition_gitlab_schemas_used, + 'The number of observed schemas dependent on connection' + ) + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb new file mode 100644 index 00000000000..2233f3c4646 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventCrossDatabaseModification < Database::QueryAnalyzers::Base + CrossDatabaseModificationAcrossUnsupportedTablesError = Class.new(StandardError) + + # This method will allow cross database modifications within the block + # Example: + # + # allow_cross_database_modification_within_transaction(url: 'url-to-an-issue') do + # create(:build) # inserts ci_build and project record in one transaction + # end + def self.allow_cross_database_modification_within_transaction(url:, &blk) + self.with_suppressed(true, &blk) + end + + # This method will prevent cross database modifications within the block + # if it was allowed previously + def self.with_cross_database_modification_prevented(&blk) + self.with_suppressed(false, &blk) + end + + def self.begin! + super + + context.merge!({ + transaction_depth_by_db: Hash.new { |h, k| h[k] = 0 }, + modified_tables_by_db: Hash.new { |h, k| h[k] = Set.new } + }) + end + + def self.enabled? + ::Feature::FlipperFeature.table_exists? && + Feature.enabled?(:detect_cross_database_modification, default_enabled: :yaml) + end + + # rubocop:disable Metrics/AbcSize + def self.analyze(parsed) + return if in_factory_bot_create? + + database = ::Gitlab::Database.db_config_name(parsed.connection) + sql = parsed.sql + + # We ignore BEGIN in tests as this is the outer transaction for + # DatabaseCleaner + if sql.start_with?('SAVEPOINT') || (!Rails.env.test? && sql.start_with?('BEGIN')) + context[:transaction_depth_by_db][database] += 1 + + return + elsif sql.start_with?('RELEASE SAVEPOINT', 'ROLLBACK TO SAVEPOINT') || (!Rails.env.test? && sql.start_with?('ROLLBACK', 'COMMIT')) + context[:transaction_depth_by_db][database] -= 1 + if context[:transaction_depth_by_db][database] <= 0 + context[:modified_tables_by_db][database].clear + end + + return + end + + return if context[:transaction_depth_by_db].values.all?(&:zero?) + + # PgQuery might fail in some cases due to limited nesting: + # https://github.com/pganalyze/pg_query/issues/209 + tables = sql.downcase.include?(' for update') ? parsed.pg.tables : parsed.pg.dml_tables + + # We have some code where plans and gitlab_subscriptions are lazily + # created and this causes lots of spec failures + # https://gitlab.com/gitlab-org/gitlab/-/issues/343394 + tables -= %w[plans gitlab_subscriptions] + + return if tables.empty? + + # All migrations will write to schema_migrations in the same transaction. + # It's safe to ignore this since schema_migrations exists in all + # databases + return if tables == ['schema_migrations'] + + context[:modified_tables_by_db][database].merge(tables) + all_tables = context[:modified_tables_by_db].values.map(&:to_a).flatten + schemas = ::Gitlab::Database::GitlabSchema.table_schemas(all_tables) + + if schemas.many? + message = "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ + "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables." \ + "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception." + + if schemas.any? { |s| s.to_s.start_with?("undefined") } + message += " The gitlab_schema was undefined for one or more of the tables in this transaction. Any new tables must be added to lib/gitlab/database/gitlab_schemas.yml ." + end + + raise CrossDatabaseModificationAcrossUnsupportedTablesError, message + end + rescue CrossDatabaseModificationAcrossUnsupportedTablesError => e + ::Gitlab::ErrorTracking.track_exception(e, { gitlab_schemas: schemas, tables: all_tables, query: parsed.sql }) + raise if raise_exception? + end + # rubocop:enable Metrics/AbcSize + + # We only raise in tests for now otherwise some features will be broken + # in development. For now we've mostly only added allowlist based on + # spec names. Until we have allowed all the violations inline we don't + # want to raise in development. + def self.raise_exception? + Rails.env.test? + end + + # We ignore execution in the #create method from FactoryBot + # because it is not representative of real code we run in + # production. There are far too many false positives caused + # by instantiating objects in different `gitlab_schema` in a + # FactoryBot `create`. + def self.in_factory_bot_create? + Rails.env.test? && caller_locations.any? { |l| l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' } + end + end + end + end +end diff --git a/lib/gitlab/database/reflection.rb b/lib/gitlab/database/reflection.rb new file mode 100644 index 00000000000..48a4de28541 --- /dev/null +++ b/lib/gitlab/database/reflection.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # A class for reflecting upon a database and its settings, such as the + # adapter name, PostgreSQL version, and the presence of tables or columns. + class Reflection + attr_reader :model + + def initialize(model) + @model = model + @version = nil + end + + def config + # The result of this method must not be cached, as other methods may use + # it after making configuration changes and expect those changes to be + # present. For example, `disable_prepared_statements` expects the + # configuration settings to always be up to date. + # + # See the following for more information: + # + # - https://gitlab.com/gitlab-org/release/retrospectives/-/issues/39 + # - https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5238 + model.connection_db_config.configuration_hash.with_indifferent_access + end + + def username + config[:username] || ENV['USER'] + end + + def database_name + config[:database] + end + + def adapter_name + config[:adapter] + end + + def human_adapter_name + if postgresql? + 'PostgreSQL' + else + 'Unknown' + end + end + + def postgresql? + adapter_name.casecmp('postgresql') == 0 + end + + # Check whether the underlying database is in read-only mode + def db_read_only? + pg_is_in_recovery = + connection + .execute('SELECT pg_is_in_recovery()') + .first + .fetch('pg_is_in_recovery') + + Gitlab::Utils.to_boolean(pg_is_in_recovery) + end + + def db_read_write? + !db_read_only? + end + + def version + @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] + end + + def database_version + connection.execute("SELECT VERSION()").first['version'] + end + + def postgresql_minimum_supported_version? + version.to_f >= MINIMUM_POSTGRES_VERSION + end + + def cached_column_exists?(column_name) + connection + .schema_cache.columns_hash(model.table_name) + .has_key?(column_name.to_s) + end + + def cached_table_exists? + exists? && connection.schema_cache.data_source_exists?(model.table_name) + end + + def exists? + # We can't _just_ check if `connection` raises an error, as it will + # point to a `ConnectionProxy`, and obtaining those doesn't involve any + # database queries. So instead we obtain the database version, which is + # cached after the first call. + connection.schema_cache.database_version + true + rescue StandardError + false + end + + def system_id + row = connection + .execute('SELECT system_identifier FROM pg_control_system()') + .first + + row['system_identifier'] + end + + private + + def connection + model.connection + end + end + end +end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 04b409a9306..7a22e324bdb 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -15,25 +15,58 @@ module Gitlab # on e.g. vacuum. REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 - # candidate_indexes: Array of Gitlab::Database::PostgresIndex - def self.perform(candidate_indexes, how_many: DEFAULT_INDEXES_PER_INVOCATION) - IndexSelection.new(candidate_indexes).take(how_many).each do |index| + # Performs automatic reindexing for a limited number of indexes per call + # 1. Consume from the explicit reindexing queue + # 2. Apply bloat heuristic to find most bloated indexes and reindex those + def self.automatic_reindexing(maximum_records: DEFAULT_INDEXES_PER_INVOCATION) + # Cleanup leftover temporary indexes from previous, possibly aborted runs (if any) + cleanup_leftovers! + + # Consume from the explicit reindexing queue first + done_counter = perform_from_queue(maximum_records: maximum_records) + + return if done_counter >= maximum_records + + # Execute reindexing based on bloat heuristic + perform_with_heuristic(maximum_records: maximum_records - done_counter) + end + + # Reindex based on bloat heuristic for a limited number of indexes per call + # + # We use a bloat heuristic to estimate the index bloat and pick the + # most bloated indexes for reindexing. + def self.perform_with_heuristic(candidate_indexes = Gitlab::Database::PostgresIndex.reindexing_support, maximum_records: DEFAULT_INDEXES_PER_INVOCATION) + IndexSelection.new(candidate_indexes).take(maximum_records).each do |index| Coordinator.new(index).perform end end + # Reindex indexes that have been explicitly enqueued (for a limited number of indexes per call) + def self.perform_from_queue(maximum_records: DEFAULT_INDEXES_PER_INVOCATION) + QueuedAction.in_queue_order.limit(maximum_records).each do |queued_entry| + Coordinator.new(queued_entry.index).perform + + queued_entry.done! + rescue StandardError => e + queued_entry.failed! + + Gitlab::AppLogger.error("Failed to perform reindexing action on queued entry #{queued_entry}: #{e}") + end.size + end + def self.cleanup_leftovers! PostgresIndex.reindexing_leftovers.each do |index| Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity") retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( + connection: index.connection, timing_configuration: REMOVE_INDEX_RETRY_CONFIG, klass: self.class, logger: Gitlab::AppLogger ) retries.run(raise_on_exhaustion: false) do - ApplicationRecord.connection.tap do |conn| + index.connection.tap do |conn| conn.execute("DROP INDEX CONCURRENTLY IF EXISTS #{conn.quote_table_name(index.schema)}.#{conn.quote_table_name(index.name)}") end end diff --git a/lib/gitlab/database/reindexing/index_selection.rb b/lib/gitlab/database/reindexing/index_selection.rb index 2186384e7d7..2d384f2f9e2 100644 --- a/lib/gitlab/database/reindexing/index_selection.rb +++ b/lib/gitlab/database/reindexing/index_selection.rb @@ -9,8 +9,8 @@ module Gitlab # Only reindex indexes with a relative bloat level (bloat estimate / size) higher than this MINIMUM_RELATIVE_BLOAT = 0.2 - # Only consider indexes with a total ondisk size in this range (before reindexing) - INDEX_SIZE_RANGE = (1.gigabyte..100.gigabyte).freeze + # Only consider indexes beyond this size (before reindexing) + INDEX_SIZE_MINIMUM = 1.gigabyte delegate :each, to: :indexes @@ -32,7 +32,7 @@ module Gitlab @indexes ||= candidates .not_recently_reindexed - .where(ondisk_size_bytes: INDEX_SIZE_RANGE) + .where('ondisk_size_bytes >= ?', INDEX_SIZE_MINIMUM) .sort_by(&:relative_bloat_level) # forced N+1 .reverse .select { |candidate| candidate.relative_bloat_level >= MINIMUM_RELATIVE_BLOAT } diff --git a/lib/gitlab/database/reindexing/queued_action.rb b/lib/gitlab/database/reindexing/queued_action.rb new file mode 100644 index 00000000000..c2039a289da --- /dev/null +++ b/lib/gitlab/database/reindexing/queued_action.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + class QueuedAction < SharedModel + self.table_name = 'postgres_reindex_queued_actions' + + enum state: { queued: 0, done: 1, failed: 2 } + + belongs_to :index, foreign_key: :index_identifier, class_name: 'Gitlab::Database::PostgresIndex' + + scope :in_queue_order, -> { queued.order(:created_at) } + + def to_s + "queued action [ id = #{id}, index: #{index_identifier} ]" + end + end + end + end +end diff --git a/lib/gitlab/database/reindexing/reindex_action.rb b/lib/gitlab/database/reindexing/reindex_action.rb index ff465fffb74..73424a76cfe 100644 --- a/lib/gitlab/database/reindexing/reindex_action.rb +++ b/lib/gitlab/database/reindexing/reindex_action.rb @@ -3,7 +3,7 @@ module Gitlab module Database module Reindexing - class ReindexAction < ActiveRecord::Base + class ReindexAction < SharedModel self.table_name = 'postgres_reindex_actions' belongs_to :index, foreign_key: :index_identifier, class_name: 'Gitlab::Database::PostgresIndex' diff --git a/lib/gitlab/database/reindexing/reindex_concurrently.rb b/lib/gitlab/database/reindexing/reindex_concurrently.rb index 7a720f7c539..152935bd734 100644 --- a/lib/gitlab/database/reindexing/reindex_concurrently.rb +++ b/lib/gitlab/database/reindexing/reindex_concurrently.rb @@ -8,7 +8,7 @@ module Gitlab ReindexError = Class.new(StandardError) TEMPORARY_INDEX_PATTERN = '\_ccnew[0-9]*' - STATEMENT_TIMEOUT = 9.hours + STATEMENT_TIMEOUT = 24.hours PG_MAX_INDEX_NAME_LENGTH = 63 attr_reader :index, :logger @@ -99,6 +99,7 @@ module Gitlab logger.info("Removing dangling index #{index.identifier}") retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( + connection: connection, timing_configuration: REMOVE_INDEX_RETRY_CONFIG, klass: self.class, logger: logger @@ -109,11 +110,6 @@ module Gitlab end end - def with_lock_retries(&block) - arguments = { klass: self.class, logger: logger } - Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block) - end - def set_statement_timeout execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT) yield @@ -123,7 +119,7 @@ module Gitlab delegate :execute, :quote_table_name, to: :connection def connection - @connection ||= ActiveRecord::Base.connection + @connection ||= index.connection end end end diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb index f304c32d731..f31dbc01907 100644 --- a/lib/gitlab/database/shared_model.rb +++ b/lib/gitlab/database/shared_model.rb @@ -8,13 +8,17 @@ module Gitlab class << self def using_connection(connection) - raise 'cannot nest connection overrides for shared models' unless overriding_connection.nil? + previous_connection = self.overriding_connection + + unless previous_connection.nil? || previous_connection.equal?(connection) + raise 'cannot nest connection overrides for shared models with different connections' + end self.overriding_connection = connection yield ensure - self.overriding_connection = nil + self.overriding_connection = nil unless previous_connection.equal?(self.overriding_connection) end def connection diff --git a/lib/gitlab/database/unidirectional_copy_trigger.rb b/lib/gitlab/database/unidirectional_copy_trigger.rb index 029c894a5ff..146b5cacd9e 100644 --- a/lib/gitlab/database/unidirectional_copy_trigger.rb +++ b/lib/gitlab/database/unidirectional_copy_trigger.rb @@ -3,7 +3,7 @@ module Gitlab module Database class UnidirectionalCopyTrigger - def self.on_table(table_name, connection: ActiveRecord::Base.connection) + def self.on_table(table_name, connection:) new(table_name, connection) end |