diff options
Diffstat (limited to 'lib/gitlab/database')
26 files changed, 445 insertions, 723 deletions
diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb index d68be19047e..4b4a9b38fd8 100644 --- a/lib/gitlab/database/bulk_update.rb +++ b/lib/gitlab/database/bulk_update.rb @@ -157,7 +157,7 @@ module Gitlab def self.execute(columns, mapping, &to_class) raise ArgumentError if mapping.blank? - entries_by_class = mapping.group_by { |k, v| to_class ? to_class.call(k) : k.class } + entries_by_class = mapping.group_by { |k, v| to_class ? yield(k) : k.class } entries_by_class.each do |model, entries| Setter.new(model, columns, entries).update! diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb index 345c7e44b05..11c83786aa4 100644 --- a/lib/gitlab/database/count/exact_count_strategy.rb +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -18,9 +18,7 @@ module Gitlab end def count - models.each_with_object({}) do |model, data| - data[model] = model.count - end + models.index_with(&:count) rescue *CONNECTION_ERRORS {} end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 365a4283d4c..0f848ed40fb 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -6,14 +6,16 @@ # Each table / view needs to have assigned gitlab_schema. Names supported today: # # - gitlab_shared - defines a set of tables that are found on all databases (data accessed is dependent on connection) -# - gitlab_main / gitlab_ci - defines a set of tables that can only exist on a given database +# - gitlab_main / gitlab_ci - defines a set of tables that can only exist on a given application database +# - gitlab_geo - defines a set of tables that can only exist on the geo database +# - gitlab_internal - defines all internal tables of Rails and PostgreSQL # # Tables for the purpose of tests should be prefixed with `_test_my_table_name` module Gitlab module Database module GitlabSchema - GITLAB_SCHEMAS_FILE = 'lib/gitlab/database/gitlab_schemas.yml' + DICTIONARY_PATH = 'db/docs/' # These tables are deleted/renamed, but still referenced by migrations. # This is needed for now, but should be removed in the future @@ -55,7 +57,7 @@ module Gitlab tables.map { |table| table_schema(table) }.to_set end - def self.table_schema(name) + def self.table_schema(name, undefined: true) schema_name, table_name = name.split('.', 2) # Strip schema name like: `public.` # Most of names do not have schemas, ensure that this is table @@ -68,7 +70,7 @@ module Gitlab table_name.gsub!(/_[0-9]+$/, '') # Tables that are properly mapped - if gitlab_schema = tables_to_schema[table_name] + if gitlab_schema = views_and_tables_to_schema[table_name] return gitlab_schema end @@ -84,6 +86,8 @@ module Gitlab return :gitlab_ci if table_name.start_with?('_test_gitlab_ci_') + return :gitlab_geo if table_name.start_with?('_test_gitlab_geo_') + # All tables that start with `_test_` without a following schema are shared and ignored return :gitlab_shared if table_name.start_with?('_test_') @@ -91,15 +95,39 @@ module Gitlab return :gitlab_internal if table_name.start_with?('pg_') # When undefined it's best to return a unique name so that we don't incorrectly assume that 2 undefined schemas belong on the same database - :"undefined_#{table_name}" + undefined ? :"undefined_#{table_name}" : nil + end + + def self.dictionary_path_globs + [Rails.root.join(DICTIONARY_PATH, '*.yml')] + end + + def self.view_path_globs + [Rails.root.join(DICTIONARY_PATH, 'views', '*.yml')] + end + + def self.views_and_tables_to_schema + @views_and_tables_to_schema ||= self.tables_to_schema.merge(self.views_to_schema) end def self.tables_to_schema - @tables_to_schema ||= YAML.load_file(Rails.root.join(GITLAB_SCHEMAS_FILE)) + @tables_to_schema ||= Dir.glob(self.dictionary_path_globs).each_with_object({}) do |file_path, dic| + data = YAML.load_file(file_path) + + dic[data['table_name']] = data['gitlab_schema'].to_sym + end + end + + def self.views_to_schema + @views_to_schema ||= Dir.glob(self.view_path_globs).each_with_object({}) do |file_path, dic| + data = YAML.load_file(file_path) + + dic[data['view_name']] = data['gitlab_schema'].to_sym + end end def self.schema_names - @schema_names ||= self.tables_to_schema.values.to_set + @schema_names ||= self.views_and_tables_to_schema.values.to_set end end end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml deleted file mode 100644 index bf6ebb21f7d..00000000000 --- a/lib/gitlab/database/gitlab_schemas.yml +++ /dev/null @@ -1,606 +0,0 @@ -abuse_reports: :gitlab_main -agent_activity_events: :gitlab_main -agent_group_authorizations: :gitlab_main -agent_project_authorizations: :gitlab_main -alert_management_alert_assignees: :gitlab_main -alert_management_alerts: :gitlab_main -alert_management_alert_metric_images: :gitlab_main -alert_management_alert_user_mentions: :gitlab_main -alert_management_http_integrations: :gitlab_main -allowed_email_domains: :gitlab_main -analytics_cycle_analytics_aggregations: :gitlab_main -analytics_cycle_analytics_group_stages: :gitlab_main -analytics_cycle_analytics_group_value_streams: :gitlab_main -analytics_cycle_analytics_issue_stage_events: :gitlab_main -analytics_cycle_analytics_merge_request_stage_events: :gitlab_main -analytics_cycle_analytics_project_stages: :gitlab_main -analytics_cycle_analytics_project_value_streams: :gitlab_main -analytics_cycle_analytics_stage_event_hashes: :gitlab_main -analytics_devops_adoption_segments: :gitlab_main -analytics_devops_adoption_snapshots: :gitlab_main -analytics_language_trend_repository_languages: :gitlab_main -analytics_usage_trends_measurements: :gitlab_main -appearances: :gitlab_main -application_settings: :gitlab_main -application_setting_terms: :gitlab_main -approval_merge_request_rules_approved_approvers: :gitlab_main -approval_merge_request_rules: :gitlab_main -approval_merge_request_rules_groups: :gitlab_main -approval_merge_request_rule_sources: :gitlab_main -approval_merge_request_rules_users: :gitlab_main -approval_project_rules: :gitlab_main -approval_project_rules_groups: :gitlab_main -approval_project_rules_protected_branches: :gitlab_main -approval_project_rules_users: :gitlab_main -approvals: :gitlab_main -approver_groups: :gitlab_main -approvers: :gitlab_main -ar_internal_metadata: :gitlab_internal -atlassian_identities: :gitlab_main -audit_events_external_audit_event_destinations: :gitlab_main -audit_events: :gitlab_main -audit_events_streaming_headers: :gitlab_main -audit_events_streaming_event_type_filters: :gitlab_main -authentication_events: :gitlab_main -award_emoji: :gitlab_main -aws_roles: :gitlab_main -background_migration_jobs: :gitlab_shared -badges: :gitlab_main -banned_users: :gitlab_main -batched_background_migration_jobs: :gitlab_shared -batched_background_migrations: :gitlab_shared -board_assignees: :gitlab_main -board_group_recent_visits: :gitlab_main -board_labels: :gitlab_main -board_project_recent_visits: :gitlab_main -boards_epic_board_labels: :gitlab_main -boards_epic_board_positions: :gitlab_main -boards_epic_board_recent_visits: :gitlab_main -boards_epic_boards: :gitlab_main -boards_epic_lists: :gitlab_main -boards_epic_list_user_preferences: :gitlab_main -boards_epic_user_preferences: :gitlab_main -boards: :gitlab_main -board_user_preferences: :gitlab_main -broadcast_messages: :gitlab_main -bulk_import_configurations: :gitlab_main -bulk_import_entities: :gitlab_main -bulk_import_exports: :gitlab_main -bulk_import_export_uploads: :gitlab_main -bulk_import_failures: :gitlab_main -bulk_imports: :gitlab_main -bulk_import_trackers: :gitlab_main -chat_names: :gitlab_main -chat_teams: :gitlab_main -ci_build_needs: :gitlab_ci -ci_build_pending_states: :gitlab_ci -ci_build_report_results: :gitlab_ci -ci_builds: :gitlab_ci -ci_builds_metadata: :gitlab_ci -ci_builds_runner_session: :gitlab_ci -ci_build_trace_chunks: :gitlab_ci -ci_build_trace_metadata: :gitlab_ci -ci_daily_build_group_report_results: :gitlab_ci -ci_deleted_objects: :gitlab_ci -ci_freeze_periods: :gitlab_ci -ci_group_variables: :gitlab_ci -ci_instance_variables: :gitlab_ci -ci_job_artifacts: :gitlab_ci -ci_job_token_project_scope_links: :gitlab_ci -ci_job_variables: :gitlab_ci -ci_job_artifact_states: :gitlab_ci -ci_minutes_additional_packs: :gitlab_ci -ci_namespace_monthly_usages: :gitlab_ci -ci_namespace_mirrors: :gitlab_ci -ci_partitions: :gitlab_ci -ci_pending_builds: :gitlab_ci -ci_pipeline_artifacts: :gitlab_ci -ci_pipeline_chat_data: :gitlab_ci -ci_pipeline_messages: :gitlab_ci -ci_pipeline_schedules: :gitlab_ci -ci_pipeline_schedule_variables: :gitlab_ci -ci_pipelines_config: :gitlab_ci -ci_pipeline_metadata: :gitlab_ci -ci_pipelines: :gitlab_ci -ci_pipeline_variables: :gitlab_ci -ci_platform_metrics: :gitlab_ci -ci_project_monthly_usages: :gitlab_ci -ci_project_mirrors: :gitlab_ci -ci_refs: :gitlab_ci -ci_resource_groups: :gitlab_ci -ci_resources: :gitlab_ci -ci_runner_namespaces: :gitlab_ci -ci_runner_projects: :gitlab_ci -ci_runner_versions: :gitlab_ci -ci_runners: :gitlab_ci -ci_running_builds: :gitlab_ci -ci_sources_pipelines: :gitlab_ci -ci_secure_files: :gitlab_ci -ci_secure_file_states: :gitlab_ci -ci_sources_projects: :gitlab_ci -ci_stages: :gitlab_ci -ci_subscriptions_projects: :gitlab_ci -ci_trigger_requests: :gitlab_ci -ci_triggers: :gitlab_ci -ci_unit_test_failures: :gitlab_ci -ci_unit_tests: :gitlab_ci -ci_variables: :gitlab_ci -cluster_agents: :gitlab_main -cluster_agent_tokens: :gitlab_main -cluster_enabled_grants: :gitlab_main -cluster_groups: :gitlab_main -cluster_platforms_kubernetes: :gitlab_main -cluster_projects: :gitlab_main -cluster_providers_aws: :gitlab_main -cluster_providers_gcp: :gitlab_main -clusters_applications_cert_managers: :gitlab_main -clusters_applications_cilium: :gitlab_main -clusters_applications_crossplane: :gitlab_main -clusters_applications_helm: :gitlab_main -clusters_applications_ingress: :gitlab_main -clusters_applications_jupyter: :gitlab_main -clusters_applications_knative: :gitlab_main -clusters_applications_prometheus: :gitlab_main -clusters_applications_runners: :gitlab_main -clusters: :gitlab_main -clusters_integration_prometheus: :gitlab_main -clusters_kubernetes_namespaces: :gitlab_main -commit_user_mentions: :gitlab_main -compliance_management_frameworks: :gitlab_main -container_expiration_policies: :gitlab_main -container_repositories: :gitlab_main -content_blocked_states: :gitlab_main -conversational_development_index_metrics: :gitlab_main -coverage_fuzzing_corpuses: :gitlab_main -csv_issue_imports: :gitlab_main -custom_emoji: :gitlab_main -customer_relations_contacts: :gitlab_main -customer_relations_organizations: :gitlab_main -dast_profile_schedules: :gitlab_main -dast_profiles: :gitlab_main -dast_profiles_pipelines: :gitlab_main -dast_scanner_profiles_builds: :gitlab_main -dast_scanner_profiles: :gitlab_main -dast_site_profiles_builds: :gitlab_main -dast_site_profile_secret_variables: :gitlab_main -dast_site_profiles: :gitlab_main -dast_site_profiles_pipelines: :gitlab_main -dast_sites: :gitlab_main -dast_site_tokens: :gitlab_main -dast_site_validations: :gitlab_main -dependency_proxy_blob_states: :gitlab_main -dependency_proxy_blobs: :gitlab_main -dependency_proxy_group_settings: :gitlab_main -dependency_proxy_image_ttl_group_policies: :gitlab_main -dependency_proxy_manifests: :gitlab_main -deploy_keys_projects: :gitlab_main -deployment_approvals: :gitlab_main -deployment_clusters: :gitlab_main -deployment_merge_requests: :gitlab_main -deployments: :gitlab_main -deploy_tokens: :gitlab_main -description_versions: :gitlab_main -design_management_designs: :gitlab_main -design_management_designs_versions: :gitlab_main -design_management_versions: :gitlab_main -design_user_mentions: :gitlab_main -detached_partitions: :gitlab_shared -diff_note_positions: :gitlab_main -dora_configurations: :gitlab_main -dora_daily_metrics: :gitlab_main -draft_notes: :gitlab_main -elastic_index_settings: :gitlab_main -elastic_reindexing_slices: :gitlab_main -elastic_reindexing_subtasks: :gitlab_main -elastic_reindexing_tasks: :gitlab_main -elasticsearch_indexed_namespaces: :gitlab_main -elasticsearch_indexed_projects: :gitlab_main -emails: :gitlab_main -environments: :gitlab_main -epic_issues: :gitlab_main -epic_metrics: :gitlab_main -epics: :gitlab_main -epic_user_mentions: :gitlab_main -error_tracking_client_keys: :gitlab_main -error_tracking_error_events: :gitlab_main -error_tracking_errors: :gitlab_main -events: :gitlab_main -evidences: :gitlab_main -experiments: :gitlab_main -experiment_subjects: :gitlab_main -external_approval_rules: :gitlab_main -external_approval_rules_protected_branches: :gitlab_main -external_pull_requests: :gitlab_ci -external_status_checks: :gitlab_main -external_status_checks_protected_branches: :gitlab_main -feature_gates: :gitlab_main -features: :gitlab_main -fork_network_members: :gitlab_main -fork_networks: :gitlab_main -geo_cache_invalidation_events: :gitlab_main -geo_container_repository_updated_events: :gitlab_main -geo_event_log: :gitlab_main -geo_events: :gitlab_main -geo_hashed_storage_attachments_events: :gitlab_main -geo_hashed_storage_migrated_events: :gitlab_main -geo_node_namespace_links: :gitlab_main -geo_nodes: :gitlab_main -geo_node_statuses: :gitlab_main -geo_repositories_changed_events: :gitlab_main -geo_repository_created_events: :gitlab_main -geo_repository_deleted_events: :gitlab_main -geo_repository_renamed_events: :gitlab_main -geo_repository_updated_events: :gitlab_main -geo_reset_checksum_events: :gitlab_main -ghost_user_migrations: :gitlab_main -gitlab_subscription_histories: :gitlab_main -gitlab_subscriptions: :gitlab_main -gpg_keys: :gitlab_main -gpg_key_subkeys: :gitlab_main -gpg_signatures: :gitlab_main -grafana_integrations: :gitlab_main -group_custom_attributes: :gitlab_main -group_crm_settings: :gitlab_main -group_deletion_schedules: :gitlab_main -group_deploy_keys: :gitlab_main -group_deploy_keys_groups: :gitlab_main -group_deploy_tokens: :gitlab_main -group_features: :gitlab_main -group_group_links: :gitlab_main -group_import_states: :gitlab_main -group_merge_request_approval_settings: :gitlab_main -group_repository_storage_moves: :gitlab_main -group_wiki_repositories: :gitlab_main -historical_data: :gitlab_main -identities: :gitlab_main -import_export_uploads: :gitlab_main -import_failures: :gitlab_main -incident_management_escalation_policies: :gitlab_main -incident_management_escalation_rules: :gitlab_main -incident_management_issuable_escalation_statuses: :gitlab_main -incident_management_oncall_participants: :gitlab_main -incident_management_oncall_rotations: :gitlab_main -incident_management_oncall_schedules: :gitlab_main -incident_management_oncall_shifts: :gitlab_main -incident_management_pending_alert_escalations: :gitlab_main -incident_management_pending_issue_escalations: :gitlab_main -incident_management_timeline_events: :gitlab_main -incident_management_timeline_event_tags: :gitlab_main -incident_management_timeline_event_tag_links: :gitlab_main -index_statuses: :gitlab_main -in_product_marketing_emails: :gitlab_main -insights: :gitlab_main -integrations: :gitlab_main -internal_ids: :gitlab_main -ip_restrictions: :gitlab_main -issuable_metric_images: :gitlab_main -issuable_resource_links: :gitlab_main -issuable_severities: :gitlab_main -issuable_slas: :gitlab_main -issue_assignees: :gitlab_main -issue_customer_relations_contacts: :gitlab_main -issue_emails: :gitlab_main -issue_email_participants: :gitlab_main -issue_links: :gitlab_main -issue_metrics: :gitlab_main -issue_search_data: :gitlab_main -issues: :gitlab_main -issues_prometheus_alert_events: :gitlab_main -issues_self_managed_prometheus_alert_events: :gitlab_main -issue_tracker_data: :gitlab_main -issue_user_mentions: :gitlab_main -iterations_cadences: :gitlab_main -jira_connect_installations: :gitlab_main -jira_connect_subscriptions: :gitlab_main -jira_imports: :gitlab_main -jira_tracker_data: :gitlab_main -keys: :gitlab_main -label_links: :gitlab_main -label_priorities: :gitlab_main -labels: :gitlab_main -ldap_group_links: :gitlab_main -lfs_file_locks: :gitlab_main -lfs_objects: :gitlab_main -lfs_objects_projects: :gitlab_main -lfs_object_states: :gitlab_main -licenses: :gitlab_main -lists: :gitlab_main -list_user_preferences: :gitlab_main -loose_foreign_keys_deleted_records: :gitlab_shared -member_roles: :gitlab_main -member_tasks: :gitlab_main -members: :gitlab_main -merge_request_assignees: :gitlab_main -merge_request_blocks: :gitlab_main -merge_request_cleanup_schedules: :gitlab_main -merge_requests_compliance_violations: :gitlab_main -merge_request_context_commit_diff_files: :gitlab_main -merge_request_context_commits: :gitlab_main -merge_request_diff_commits: :gitlab_main -merge_request_diff_commit_users: :gitlab_main -merge_request_diff_details: :gitlab_main -merge_request_diff_files: :gitlab_main -merge_request_diffs: :gitlab_main -merge_request_metrics: :gitlab_main -merge_request_predictions: :gitlab_main -merge_request_reviewers: :gitlab_main -merge_requests_closing_issues: :gitlab_main -merge_requests: :gitlab_main -merge_request_user_mentions: :gitlab_main -merge_trains: :gitlab_main -metrics_dashboard_annotations: :gitlab_main -metrics_users_starred_dashboards: :gitlab_main -milestone_releases: :gitlab_main -milestones: :gitlab_main -ml_candidates: :gitlab_main -ml_experiments: :gitlab_main -ml_candidate_metrics: :gitlab_main -ml_candidate_params: :gitlab_main -namespace_admin_notes: :gitlab_main -namespace_aggregation_schedules: :gitlab_main -namespace_bans: :gitlab_main -namespace_limits: :gitlab_main -namespace_package_settings: :gitlab_main -namespace_root_storage_statistics: :gitlab_main -namespace_ci_cd_settings: :gitlab_main -namespace_commit_emails: :gitlab_main -namespace_settings: :gitlab_main -namespace_details: :gitlab_main -namespaces: :gitlab_main -namespaces_sync_events: :gitlab_main -namespace_statistics: :gitlab_main -note_diff_files: :gitlab_main -notes: :gitlab_main -notification_settings: :gitlab_main -oauth_access_grants: :gitlab_main -oauth_access_tokens: :gitlab_main -oauth_applications: :gitlab_main -oauth_openid_requests: :gitlab_main -onboarding_progresses: :gitlab_main -operations_feature_flags_clients: :gitlab_main -operations_feature_flag_scopes: :gitlab_main -operations_feature_flags: :gitlab_main -operations_feature_flags_issues: :gitlab_main -operations_scopes: :gitlab_main -operations_strategies: :gitlab_main -operations_strategies_user_lists: :gitlab_main -operations_user_lists: :gitlab_main -p_ci_builds_metadata: :gitlab_ci -packages_build_infos: :gitlab_main -packages_cleanup_policies: :gitlab_main -packages_composer_cache_files: :gitlab_main -packages_composer_metadata: :gitlab_main -packages_conan_file_metadata: :gitlab_main -packages_conan_metadata: :gitlab_main -packages_debian_file_metadata: :gitlab_main -packages_debian_group_architectures: :gitlab_main -packages_debian_group_component_files: :gitlab_main -packages_debian_group_components: :gitlab_main -packages_debian_group_distribution_keys: :gitlab_main -packages_debian_group_distributions: :gitlab_main -packages_debian_project_architectures: :gitlab_main -packages_debian_project_component_files: :gitlab_main -packages_debian_project_components: :gitlab_main -packages_debian_project_distribution_keys: :gitlab_main -packages_debian_project_distributions: :gitlab_main -packages_debian_publications: :gitlab_main -packages_dependencies: :gitlab_main -packages_dependency_links: :gitlab_main -packages_events: :gitlab_main -packages_helm_file_metadata: :gitlab_main -packages_maven_metadata: :gitlab_main -packages_npm_metadata: :gitlab_main -packages_rpm_metadata: :gitlab_main -packages_nuget_dependency_link_metadata: :gitlab_main -packages_nuget_metadata: :gitlab_main -packages_package_file_build_infos: :gitlab_main -packages_package_files: :gitlab_main -packages_rpm_repository_files: :gitlab_main -packages_packages: :gitlab_main -packages_pypi_metadata: :gitlab_main -packages_rubygems_metadata: :gitlab_main -packages_tags: :gitlab_main -pages_deployments: :gitlab_main -pages_deployment_states: :gitlab_main -pages_domain_acme_orders: :gitlab_main -pages_domains: :gitlab_main -path_locks: :gitlab_main -personal_access_tokens: :gitlab_main -plan_limits: :gitlab_main -plans: :gitlab_main -pool_repositories: :gitlab_main -postgres_async_indexes: :gitlab_shared -postgres_autovacuum_activity: :gitlab_shared -postgres_constraints: :gitlab_shared -postgres_foreign_keys: :gitlab_shared -postgres_index_bloat_estimates: :gitlab_shared -postgres_indexes: :gitlab_shared -postgres_partitioned_tables: :gitlab_shared -postgres_partitions: :gitlab_shared -postgres_reindex_actions: :gitlab_shared -postgres_reindex_queued_actions: :gitlab_shared -product_analytics_events_experimental: :gitlab_main -programming_languages: :gitlab_main -project_access_tokens: :gitlab_main -project_alerting_settings: :gitlab_main -project_aliases: :gitlab_main -project_authorizations: :gitlab_main -project_auto_devops: :gitlab_main -project_build_artifacts_size_refreshes: :gitlab_main -project_ci_cd_settings: :gitlab_main -project_ci_feature_usages: :gitlab_main -project_compliance_framework_settings: :gitlab_main -project_custom_attributes: :gitlab_main -project_daily_statistics: :gitlab_main -project_deploy_tokens: :gitlab_main -project_error_tracking_settings: :gitlab_main -project_export_jobs: :gitlab_main -project_features: :gitlab_main -project_feature_usages: :gitlab_main -project_group_links: :gitlab_main -project_import_data: :gitlab_main -project_incident_management_settings: :gitlab_main -project_metrics_settings: :gitlab_main -project_mirror_data: :gitlab_main -project_pages_metadata: :gitlab_main -project_relation_export_uploads: :gitlab_main -project_relation_exports: :gitlab_main -project_repositories: :gitlab_main -project_repository_states: :gitlab_main -project_repository_storage_moves: :gitlab_main -project_security_settings: :gitlab_main -project_settings: :gitlab_main -projects: :gitlab_main -projects_sync_events: :gitlab_main -project_statistics: :gitlab_main -project_topics: :gitlab_main -project_wiki_repositories: :gitlab_main -project_wiki_repository_states: :gitlab_main -prometheus_alert_events: :gitlab_main -prometheus_alerts: :gitlab_main -prometheus_metrics: :gitlab_main -protected_branches: :gitlab_main -protected_branch_merge_access_levels: :gitlab_main -protected_branch_push_access_levels: :gitlab_main -protected_branch_unprotect_access_levels: :gitlab_main -protected_environment_approval_rules: :gitlab_main -protected_environment_deploy_access_levels: :gitlab_main -protected_environments: :gitlab_main -protected_tag_create_access_levels: :gitlab_main -protected_tags: :gitlab_main -push_event_payloads: :gitlab_main -push_rules: :gitlab_main -raw_usage_data: :gitlab_main -redirect_routes: :gitlab_main -related_epic_links: :gitlab_main -release_links: :gitlab_main -releases: :gitlab_main -remote_mirrors: :gitlab_main -repository_languages: :gitlab_main -required_code_owners_sections: :gitlab_main -requirements: :gitlab_main -requirements_management_test_reports: :gitlab_main -resource_iteration_events: :gitlab_main -resource_label_events: :gitlab_main -resource_milestone_events: :gitlab_main -resource_state_events: :gitlab_main -resource_weight_events: :gitlab_main -reviews: :gitlab_main -routes: :gitlab_main -saml_group_links: :gitlab_main -saml_providers: :gitlab_main -saved_replies: :gitlab_main -sbom_components: :gitlab_main -sbom_occurrences: :gitlab_main -sbom_component_versions: :gitlab_main -sbom_sources: :gitlab_main -sbom_vulnerable_component_versions: :gitlab_main -schema_migrations: :gitlab_internal -scim_identities: :gitlab_main -scim_oauth_access_tokens: :gitlab_main -security_findings: :gitlab_main -security_orchestration_policy_configurations: :gitlab_main -security_orchestration_policy_rule_schedules: :gitlab_main -security_scans: :gitlab_main -security_training_providers: :gitlab_main -security_trainings: :gitlab_main -self_managed_prometheus_alert_events: :gitlab_main -sent_notifications: :gitlab_main -sentry_issues: :gitlab_main -serverless_domain_cluster: :gitlab_main -service_desk_settings: :gitlab_main -shards: :gitlab_main -slack_integrations: :gitlab_main -smartcard_identities: :gitlab_main -snippet_repositories: :gitlab_main -snippet_repository_storage_moves: :gitlab_main -snippets: :gitlab_main -snippet_statistics: :gitlab_main -snippet_user_mentions: :gitlab_main -software_license_policies: :gitlab_main -software_licenses: :gitlab_main -spam_logs: :gitlab_main -sprints: :gitlab_main -ssh_signatures: :gitlab_main -status_check_responses: :gitlab_main -status_page_published_incidents: :gitlab_main -status_page_settings: :gitlab_main -subscriptions: :gitlab_main -suggestions: :gitlab_main -system_note_metadata: :gitlab_main -taggings: :gitlab_ci -tags: :gitlab_ci -term_agreements: :gitlab_main -terraform_states: :gitlab_main -terraform_state_versions: :gitlab_main -timelogs: :gitlab_main -timelog_categories: :gitlab_main -todos: :gitlab_main -token_with_ivs: :gitlab_main -topics: :gitlab_main -trending_projects: :gitlab_main -u2f_registrations: :gitlab_main -upcoming_reconciliations: :gitlab_main -uploads: :gitlab_main -upload_states: :gitlab_main -user_agent_details: :gitlab_main -user_callouts: :gitlab_main -user_canonical_emails: :gitlab_main -user_credit_card_validations: :gitlab_main -user_custom_attributes: :gitlab_main -user_details: :gitlab_main -user_follow_users: :gitlab_main -user_group_callouts: :gitlab_main -user_project_callouts: :gitlab_main -user_highest_roles: :gitlab_main -user_interacted_projects: :gitlab_main -user_phone_number_validations: :gitlab_main -user_permission_export_uploads: :gitlab_main -user_preferences: :gitlab_main -users: :gitlab_main -users_ops_dashboard_projects: :gitlab_main -users_security_dashboard_projects: :gitlab_main -users_star_projects: :gitlab_main -users_statistics: :gitlab_main -user_statuses: :gitlab_main -user_synced_attributes_metadata: :gitlab_main -verification_codes: :gitlab_main -vulnerabilities: :gitlab_main -vulnerability_advisories: :gitlab_main -vulnerability_exports: :gitlab_main -vulnerability_external_issue_links: :gitlab_main -vulnerability_feedback: :gitlab_main -vulnerability_finding_evidences: :gitlab_main -vulnerability_finding_links: :gitlab_main -vulnerability_finding_signatures: :gitlab_main -vulnerability_findings_remediations: :gitlab_main -vulnerability_flags: :gitlab_main -vulnerability_historical_statistics: :gitlab_main -vulnerability_identifiers: :gitlab_main -vulnerability_issue_links: :gitlab_main -vulnerability_merge_request_links: :gitlab_main -vulnerability_occurrence_identifiers: :gitlab_main -vulnerability_occurrence_pipelines: :gitlab_main -vulnerability_occurrences: :gitlab_main -vulnerability_reads: :gitlab_main -vulnerability_remediations: :gitlab_main -vulnerability_scanners: :gitlab_main -vulnerability_state_transitions: :gitlab_main -vulnerability_statistics: :gitlab_main -vulnerability_user_mentions: :gitlab_main -webauthn_registrations: :gitlab_main -web_hook_logs: :gitlab_main -web_hooks: :gitlab_main -wiki_page_meta: :gitlab_main -wiki_page_slugs: :gitlab_main -work_item_parent_links: :gitlab_main -work_item_types: :gitlab_main -x509_certificates: :gitlab_main -x509_commit_signatures: :gitlab_main -x509_issuers: :gitlab_main -zentao_tracker_data: :gitlab_main -# dingtalk_tracker_data JiHu-specific, see https://jihulab.com/gitlab-cn/gitlab/-/merge_requests/417 -dingtalk_tracker_data: :gitlab_main -zoom_meetings: :gitlab_main -batched_background_migration_job_transition_logs: :gitlab_shared -user_namespace_callouts: :gitlab_main diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index 8799f8d8af8..f0343f9d8b5 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -95,7 +95,7 @@ module Gitlab # name - The name of the method to call on a connection object. def read_using_load_balancer(...) if current_session.use_primary? && - !current_session.use_replicas_for_read_queries? + !current_session.use_replicas_for_read_queries? @load_balancer.read_write do |connection| connection.send(...) end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index 52a9e8798d4..3295301a2d7 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -125,13 +125,6 @@ module Gitlab old_host_list_length: current.length ) replace_hosts(from_dns) - else - ::Gitlab::Database::LoadBalancing::Logger.info( - event: :host_list_unchanged, - message: "Unchanged host list for service discovery", - host_list_length: from_dns.length, - old_host_list_length: current.length - ) end interval diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb index 13afbd8fd37..619f11ae890 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb @@ -57,7 +57,7 @@ module Gitlab if uses_primary? load_balancer.primary_write_location else - load_balancer.host.database_replica_location + load_balancer.host&.database_replica_location || load_balancer.primary_write_location end end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 737852d5ccb..f7b8d2514ba 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -97,14 +97,8 @@ module Gitlab end def databases_in_sync?(wal_locations) - locations = if Feature.enabled?(:indifferent_wal_location_keys) - wal_locations.with_indifferent_access - else - wal_locations - end - ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb| - if (location = locations[lb.name]) + if (location = wal_locations.with_indifferent_access[lb.name]) lb.select_up_to_date_host(location) else # If there's no entry for a load balancer it means the Sidekiq diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index 2594ee04b35..e3ae2892668 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -10,18 +10,34 @@ module Gitlab # See https://www.postgresql.org/message-id/16934.1568989957%40sss.pgh.pa.us EXPECTED_TRIGGER_RECORD_COUNT = 3 + def self.tables_to_lock(connection) + Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| + yield table_name, schema_name + end + + Gitlab::Database::SharedModel.using_connection(connection) do + Postgresql::DetachedPartition.find_each do |detached_partition| + yield detached_partition.fully_qualified_table_name, detached_partition.table_schema + end + end + end + def initialize(table_name:, connection:, database_name:, logger: nil, dry_run: false) @table_name = table_name @connection = connection @database_name = database_name @logger = logger @dry_run = dry_run + + @table_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils + .extract_schema_qualified_name(table_name) + .identifier end def table_locked_for_writes?(table_name) query = <<~SQL SELECT COUNT(*) from information_schema.triggers - WHERE event_object_table = '#{table_name}' + WHERE event_object_table = '#{table_name_without_schema}' AND trigger_name = '#{write_trigger_name(table_name)}' SQL @@ -56,7 +72,7 @@ module Gitlab private - attr_reader :table_name, :connection, :database_name, :logger, :dry_run + attr_reader :table_name, :connection, :database_name, :logger, :dry_run, :table_name_without_schema def execute_sql_statement(sql) if dry_run @@ -99,7 +115,7 @@ module Gitlab end def write_trigger_name(table_name) - "gitlab_schema_write_trigger_for_#{table_name}" + "gitlab_schema_write_trigger_for_#{table_name_without_schema}" end end end diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb index ab8b6988c3d..4d38920f571 100644 --- a/lib/gitlab/database/migration.rb +++ b/lib/gitlab/database/migration.rb @@ -51,6 +51,10 @@ module Gitlab include Gitlab::Database::MigrationHelpers::RestrictGitlabSchema end + class V2_1 < V2_0 # rubocop:disable Naming/ClassAndModuleCamelCase + include Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables + end + def self.[](version) version = version.to_s name = "V#{version.tr('.', '_')}" @@ -61,7 +65,7 @@ module Gitlab # The current version to be used in new migrations def self.current_version - 2.0 + 2.1 end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 16416dd2507..4858a96c173 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -10,6 +10,7 @@ module Gitlab include Migrations::TimeoutHelpers include Migrations::ConstraintsHelpers include Migrations::ExtensionHelpers + include Migrations::SidekiqHelpers include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers @@ -497,17 +498,6 @@ module Gitlab end end - # Adds a column with a default value without locking an entire table. - # - # @deprecated With PostgreSQL 11, adding columns with a default does not lead to a table rewrite anymore. - # As such, this method is not needed anymore and the default `add_column` helper should be used. - # This helper is subject to be removed in a >13.0 release. - def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false) - raise 'Deprecated: add_column_with_default does not support being passed blocks anymore' if block_given? - - add_column(table, column, type, default: default, limit: limit, null: allow_null) - end - # Renames a column without requiring downtime. # # Concurrent renames work by using database triggers to ensure both the @@ -1027,38 +1017,6 @@ module Gitlab rescue ArgumentError end - # Remove any instances of deprecated job classes lingering in queues. - # - # rubocop:disable Cop/SidekiqApiUsage - def sidekiq_remove_jobs(job_klass:) - Sidekiq::Queue.new(job_klass.queue).each do |job| - job.delete if job.klass == job_klass.to_s - end - - Sidekiq::RetrySet.new.each do |retri| - retri.delete if retri.klass == job_klass.to_s - end - - Sidekiq::ScheduledSet.new.each do |scheduled| - scheduled.delete if scheduled.klass == job_klass.to_s - end - end - # rubocop:enable Cop/SidekiqApiUsage - - def sidekiq_queue_migrate(queue_from, to:) - while sidekiq_queue_length(queue_from) > 0 - Sidekiq.redis do |conn| - conn.rpoplpush "queue:#{queue_from}", "queue:#{to}" - end - end - end - - def sidekiq_queue_length(queue_name) - Sidekiq.redis do |conn| - conn.llen("queue:#{queue_name}") - end - end - def check_trigger_permissions!(table) unless Grant.create_and_execute_trigger?(table) dbname = ApplicationRecord.database.database_name diff --git a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb new file mode 100644 index 00000000000..0aa4b0d01c4 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module AutomaticLockWritesOnTables + extend ActiveSupport::Concern + + included do + class_attribute :skip_automatic_lock_on_writes + end + + def exec_migration(connection, direction) + return super if %w[main ci].exclude?(Gitlab::Database.db_config_name(connection)) + return super if automatic_lock_on_writes_disabled? + + # This compares the tables only on the `public` schema. Partitions are not affected + tables = connection.tables + super + new_tables = connection.tables - tables + + new_tables.each do |table_name| + lock_writes_on_table(connection, table_name) if should_lock_writes_on_table?(table_name) + end + end + + private + + def automatic_lock_on_writes_disabled? + # Feature flags are set on the main database, see tables features/feature_gates. + # That is why we switch the ActiveRecord::Base.connection temporarily here back to the 'main' database + # for the cases when the migration is targeting another database, like the 'ci' database. + with_restored_connection_stack do |_| + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + skip_automatic_lock_on_writes || + Gitlab::Utils.to_boolean(ENV['SKIP_AUTOMATIC_LOCK_ON_WRITES']) || + Feature.disabled?(:automatic_lock_writes_on_table, type: :ops) + end + end + end + + def should_lock_writes_on_table?(table_name) + # currently gitlab_schema represents only present existing tables, this is workaround for deleted tables + # that should be skipped as they will be removed in a future migration. + return false if Gitlab::Database::GitlabSchema::DELETED_TABLES[table_name] + + table_schema = Gitlab::Database::GitlabSchema.table_schema(table_name.to_s, undefined: false) + + if table_schema.nil? + error_message = <<~ERROR + No gitlab_schema is defined for the table #{table_name}. Please consider + adding it to the database dictionary. + More info: https://docs.gitlab.com/ee/development/database/database_dictionary.html + ERROR + raise error_message + end + + return false unless %i[gitlab_main gitlab_ci].include?(table_schema) + + Gitlab::Database.gitlab_schemas_for_connection(connection).exclude?(table_schema) + end + + def lock_writes_on_table(connection, table_name) + database_name = Gitlab::Database.db_config_name(connection) + LockWritesManager.new( + table_name: table_name, + connection: connection, + database_name: database_name, + logger: Logger.new($stdout) + ).lock_writes + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/batched_migration_last_id.rb b/lib/gitlab/database/migrations/batched_migration_last_id.rb new file mode 100644 index 00000000000..c77a2e9a375 --- /dev/null +++ b/lib/gitlab/database/migrations/batched_migration_last_id.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class BatchedMigrationLastId + FILE_NAME = 'last-batched-background-migration-id.txt' + + def initialize(connection, base_dir) + @connection = connection + @base_dir = base_dir + end + + def store + File.open(file_path, 'wb') { |file| file.write(last_background_migration_id) } + end + + # Reads the last id from the file + # + # @info casts the file content into an +Integer+. + # Casts any unexpected content to +nil+ + # + # @example + # Integer('4', exception: false) # => 4 + # Integer('', exception: false) # => nil + # + # @return [Integer, nil] + def read + return unless File.exist?(file_path) + + Integer(File.read(file_path).presence, exception: false) + end + + private + + attr_reader :connection, :base_dir + + def file_path + @file_path ||= base_dir.join(FILE_NAME) + end + + def last_background_migration_id + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::BackgroundMigration::BatchedMigration.maximum(:id) + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 27b161419b2..ed55081c9ab 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -29,16 +29,14 @@ module Gitlab def batched_background_migrations(for_database:, legacy_mode: false) runner = nil - result_dir = if legacy_mode - BASE_RESULT_DIR.join('background_migrations') - else - BASE_RESULT_DIR.join(for_database.to_s, 'background_migrations') - end + result_dir = background_migrations_dir(for_database, legacy_mode) # Only one loop iteration since we pass `only:` here Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + from_id = batched_migrations_last_id(for_database).read + runner = Gitlab::Database::Migrations::TestBatchedBackgroundRunner - .new(result_dir: result_dir, connection: connection) + .new(result_dir: result_dir, connection: connection, from_id: from_id) end runner @@ -66,6 +64,18 @@ module Gitlab end # rubocop:enable Database/MultipleDatabases + def batched_migrations_last_id(for_database) + runner = nil + base_dir = background_migrations_dir(for_database, false) + + Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + runner = Gitlab::Database::Migrations::BatchedMigrationLastId + .new(connection, base_dir) + end + + runner + end + private def migrations_for_up(database) @@ -90,6 +100,12 @@ module Gitlab existing_versions.include?(migration.version) && versions_this_branch.include?(migration.version) end end + + def background_migrations_dir(db, legacy_mode) + return BASE_RESULT_DIR.join('background_migrations') if legacy_mode + + BASE_RESULT_DIR.join(db.to_s, 'background_migrations') + end end attr_reader :direction, :result_dir, :migrations diff --git a/lib/gitlab/database/migrations/sidekiq_helpers.rb b/lib/gitlab/database/migrations/sidekiq_helpers.rb new file mode 100644 index 00000000000..c536b33bbdf --- /dev/null +++ b/lib/gitlab/database/migrations/sidekiq_helpers.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + # rubocop:disable Cop/SidekiqApiUsage + # rubocop:disable Cop/SidekiqRedisCall + module SidekiqHelpers + # Constants for default sidekiq_remove_jobs values + DEFAULT_MAX_ATTEMPTS = 5 + DEFAULT_TIMES_IN_A_ROW = 2 + + # Probabilistically removes job_klasses from their specific queues, the + # retry set and the scheduled set. + # + # If jobs are still being processed at the same time, then there is a + # small chance it will not remove all instances of job_klass. To + # minimize this risk, it repeatedly removes matching jobs from each + # until nothing is removed twice in a row. + # + # Before calling this method, you should make sure that job_klass is no + # longer being scheduled within the running application. + def sidekiq_remove_jobs( + job_klasses:, + times_in_a_row: DEFAULT_TIMES_IN_A_ROW, + max_attempts: DEFAULT_MAX_ATTEMPTS + ) + + kwargs = { times_in_a_row: times_in_a_row, max_attempts: max_attempts } + + job_klasses_queues = job_klasses + .select { |job_klass| job_klass.to_s.safe_constantize.present? } + .map { |job_klass| job_klass.safe_constantize.queue } + .uniq + + job_klasses_queues.each do |queue| + delete_jobs_for( + set: Sidekiq::Queue.new(queue), + job_klasses: job_klasses, + kwargs: kwargs + ) + end + + delete_jobs_for( + set: Sidekiq::RetrySet.new, + kwargs: kwargs, + job_klasses: job_klasses + ) + + delete_jobs_for( + set: Sidekiq::ScheduledSet.new, + kwargs: kwargs, + job_klasses: job_klasses + ) + end + + def sidekiq_queue_migrate(queue_from, to:) + while sidekiq_queue_length(queue_from) > 0 + Sidekiq.redis do |conn| + conn.rpoplpush "queue:#{queue_from}", "queue:#{to}" + end + end + end + + def sidekiq_queue_length(queue_name) + Sidekiq.redis do |conn| + conn.llen("queue:#{queue_name}") + end + end + + private + + # Handle the "jobs deleted" tracking that is needed in order to track + # whether a job was deleted or not. + def delete_jobs_for(set:, kwargs:, job_klasses:) + until_equal_to(0, **kwargs) do + set.count do |job| + job_klasses.include?(job.klass) && job.delete + end + end + end + + # Control how many times in a row you want to see a job deleted 0 + # times. The idea is that if you see 0 jobs deleted x number of times + # in a row you've *likely* covered the case in which the queue was + # mutating while this was running. + def until_equal_to(target, times_in_a_row:, max_attempts:) + streak = 0 + + result = { attempts: 0, success: false } + + 1.upto(max_attempts) do |current_attempt| + # yield's return value is a count of "jobs_deleted" + if yield == target + streak += 1 + elsif streak > 0 + streak = 0 + end + + result[:attempts] = current_attempt + result[:success] = streak == times_in_a_row + + break if result[:success] + end + result + end + end + # rubocop:enable Cop/SidekiqApiUsage + # rubocop:enable Cop/SidekiqRedisCall + end + end +end diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index 46855ca1921..a16103f452c 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -6,16 +6,17 @@ module Gitlab class TestBatchedBackgroundRunner < BaseBackgroundRunner include Gitlab::Database::DynamicModelHelpers - def initialize(result_dir:, connection:) + def initialize(result_dir:, connection:, from_id:) super(result_dir: result_dir, connection: connection) @connection = connection + @from_id = from_id end def jobs_by_migration_name Gitlab::Database::SharedModel.using_connection(connection) do Gitlab::Database::BackgroundMigration::BatchedMigration .executable - .created_after(3.hours.ago) # Simple way to exclude migrations already running before migration testing + .where('id > ?', from_id) .to_h do |migration| batching_strategy = migration.batch_class.new(connection: connection) @@ -102,6 +103,10 @@ module Gitlab end end end + + private + + attr_reader :from_id end end end diff --git a/lib/gitlab/database/obsolete_ignored_columns.rb b/lib/gitlab/database/obsolete_ignored_columns.rb index ad5473f1b74..2b88ab12380 100644 --- a/lib/gitlab/database/obsolete_ignored_columns.rb +++ b/lib/gitlab/database/obsolete_ignored_columns.rb @@ -23,8 +23,8 @@ module Gitlab private def ignored_columns_safe_to_remove_for(klass) - ignores = ignored_and_not_present(klass).each_with_object({}) do |col, h| - h[col] = klass.ignored_columns_details[col.to_sym] + ignores = ignored_and_not_present(klass).index_with do |col| + klass.ignored_columns_details[col.to_sym] end ignores.select { |_, i| i&.safe_to_remove? } diff --git a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb index 4e38eea963b..fd99062974c 100644 --- a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb +++ b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb @@ -19,7 +19,7 @@ module Gitlab attr_reader :table, :value - def initialize(table, value, partition_name: nil ) + def initialize(table, value, partition_name: nil) @table = table @value = value @partition_name = partition_name diff --git a/lib/gitlab/database/postgres_hll/buckets.rb b/lib/gitlab/database/postgres_hll/buckets.rb index cbc9544d905..3f64eee030e 100644 --- a/lib/gitlab/database/postgres_hll/buckets.rb +++ b/lib/gitlab/database/postgres_hll/buckets.rb @@ -61,7 +61,7 @@ module Gitlab num_uniques = ( ((TOTAL_BUCKETS**2) * (0.7213 / (1 + 1.079 / TOTAL_BUCKETS))) / - (num_zero_buckets + buckets.values.sum { |bucket_hash| 2**(-1 * bucket_hash) } ) + (num_zero_buckets + buckets.values.sum { |bucket_hash| 2**(-1 * bucket_hash) }) ).to_i if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index 3b1751c863d..dd10e0d7992 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -165,8 +165,8 @@ module Gitlab def self.in_factory_bot_create? Rails.env.test? && caller_locations.any? do |l| l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' || - l.path.end_with?('lib/factory_bot/strategy/create.rb') || - l.path.end_with?('shoulda/matchers/active_record/validate_uniqueness_of_matcher.rb') && l.label == 'create_existing_record' + l.path.end_with?('lib/factory_bot/strategy/create.rb') || + l.path.end_with?('shoulda/matchers/active_record/validate_uniqueness_of_matcher.rb') && l.label == 'create_existing_record' end end end diff --git a/lib/gitlab/database/query_analyzers/query_recorder.rb b/lib/gitlab/database/query_analyzers/query_recorder.rb index 88fe829c3d2..b54f3442512 100644 --- a/lib/gitlab/database/query_analyzers/query_recorder.rb +++ b/lib/gitlab/database/query_analyzers/query_recorder.rb @@ -4,7 +4,7 @@ module Gitlab module Database module QueryAnalyzers class QueryRecorder < Base - LOG_FILE = 'rspec/query_recorder.ndjson' + LOG_PATH = 'query_recorder/' class << self def raw? @@ -12,8 +12,9 @@ module Gitlab end def enabled? - # Only enable QueryRecorder in CI - ENV['CI'].present? + # Only enable QueryRecorder in CI on database MRs or default branch + ENV['CI_MERGE_REQUEST_LABELS']&.include?('database') || + (ENV['CI_COMMIT_REF_NAME'].present? && ENV['CI_COMMIT_REF_NAME'] == ENV['CI_DEFAULT_BRANCH']) end def analyze(sql) @@ -24,11 +25,14 @@ module Gitlab log_query(payload) end + def log_file + Rails.root.join(LOG_PATH, "#{ENV.fetch('CI_JOB_NAME_SLUG', 'rspec')}.ndjson") + end + private def log_query(payload) - log_path = Rails.root.join(LOG_FILE) - log_dir = File.dirname(log_path) + log_dir = Rails.root.join(LOG_PATH) # Create log directory if it does not exist since it is only created # ahead of time by certain CI jobs @@ -36,7 +40,7 @@ module Gitlab log_line = "#{Gitlab::Json.dump(payload)}\n" - File.write(log_path, log_line, mode: 'a') + File.write(log_file, log_line, mode: 'a') end end end diff --git a/lib/gitlab/database/schema_cache_with_renamed_table.rb b/lib/gitlab/database/schema_cache_with_renamed_table.rb index 74900dc0d26..6da76803f7c 100644 --- a/lib/gitlab/database/schema_cache_with_renamed_table.rb +++ b/lib/gitlab/database/schema_cache_with_renamed_table.rb @@ -40,10 +40,8 @@ module Gitlab end def renamed_tables_cache - @renamed_tables ||= begin - Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name| - connection.view_exists?(old_name) - end + @renamed_tables ||= Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name| + connection.view_exists?(old_name) end end diff --git a/lib/gitlab/database/schema_cleaner.rb b/lib/gitlab/database/schema_cleaner.rb index c3cdcf1450d..2c8d0a4eb6d 100644 --- a/lib/gitlab/database/schema_cleaner.rb +++ b/lib/gitlab/database/schema_cleaner.rb @@ -25,7 +25,23 @@ module Gitlab # The intention here is to not introduce an assumption about the standard schema, # unless we have a good reason to do so. structure.gsub!(/public\.(\w+)/, '\1') - structure.gsub!(/CREATE EXTENSION IF NOT EXISTS (\w+) WITH SCHEMA public;/, 'CREATE EXTENSION IF NOT EXISTS \1;') + structure.gsub!( + /CREATE EXTENSION IF NOT EXISTS (\w+) WITH SCHEMA public;/, + 'CREATE EXTENSION IF NOT EXISTS \1;' + ) + + # Table lock-writes triggers should not be added to the schema + # These triggers are added by the rake task gitlab:db:lock_writes for a decomposed database. + structure.gsub!( + %r{ + ^CREATE.TRIGGER.gitlab_schema_write_trigger_\w+ + \s + BEFORE.INSERT.OR.DELETE.OR.UPDATE.OR.TRUNCATE.ON.\w+ + \s + FOR.EACH.STATEMENT.EXECUTE.FUNCTION.gitlab_schema_prevent_write\(\);$ + }x, + '' + ) structure.gsub!(/\n{3,}/, "\n\n") diff --git a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb index 9f096904d31..b2a7f5442e9 100644 --- a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb +++ b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb @@ -26,15 +26,32 @@ module Gitlab # it maps the tables to the tables that depend on it def tables_dependencies - @tables.to_h do |table_name| - [table_name, all_foreign_keys[table_name]&.map(&:from_table).to_a] + @tables.index_with do |table_name| + all_foreign_keys[table_name] end end def all_foreign_keys - @all_foreign_keys ||= @tables.flat_map do |table_name| - @connection.foreign_keys(table_name) - end.group_by(&:to_table) + @all_foreign_keys ||= @tables.each_with_object(Hash.new { |h, k| h[k] = [] }) do |table, hash| + foreign_keys_for(table).each do |fk| + hash[fk.to_table] << table + end + end + end + + def foreign_keys_for(table) + # Detached partitions like gitlab_partitions_dynamic._test_gitlab_partition_20220101 + # store their foreign keys in the public schema. + # + # See spec/lib/gitlab/database/tables_sorted_by_foreign_keys_spec.rb + # for an example + name = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils.extract_schema_qualified_name(table) + + if name.schema == ::Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA.to_s + @connection.foreign_keys(name.identifier) + else + @connection.foreign_keys(table) + end end end end diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index 8380bf23899..807ecdb862a 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -19,24 +19,32 @@ module Gitlab logger&.info "DRY RUN:" if dry_run - connection = Gitlab::Database.database_base_models[database_name].connection - schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) tables_to_truncate = Gitlab::Database::GitlabSchema.tables_to_schema.reject do |_, schema_name| - (GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection)).include?(schema_name) + GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection).include?(schema_name) end.keys + Gitlab::Database::SharedModel.using_connection(connection) do + Postgresql::DetachedPartition.find_each do |detached_partition| + next if GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection).include?(detached_partition.table_schema) + + tables_to_truncate << detached_partition.fully_qualified_table_name + end + end + tables_sorted = Gitlab::Database::TablesSortedByForeignKeys.new(connection, tables_to_truncate).execute # Checking if all the tables have the write-lock triggers # to make sure we are deleting the right tables on the right database. tables_sorted.flatten.each do |table_name| - query = <<~SQL - SELECT COUNT(*) from information_schema.triggers - WHERE event_object_table = '#{table_name}' - AND trigger_name = 'gitlab_schema_write_trigger_for_#{table_name}' - SQL - - if connection.select_value(query) == 0 + lock_writes_manager = Gitlab::Database::LockWritesManager.new( + table_name: table_name, + connection: connection, + database_name: database_name, + logger: logger, + dry_run: dry_run + ) + + unless lock_writes_manager.table_locked_for_writes?(table_name) raise "Table '#{table_name}' is not locked for writes. Run the rake task gitlab:db:lock_writes first" end end @@ -51,18 +59,26 @@ module Gitlab # min_batch_size is the minimum number of new tables to truncate at each stage. # But in each stage we have also have to truncate the already truncated tables in the previous stages logger&.info "Truncating legacy tables for the database #{database_name}" - truncate_tables_in_batches(connection, tables_sorted, min_batch_size) + truncate_tables_in_batches(tables_sorted) end private attr_accessor :database_name, :min_batch_size, :logger, :dry_run, :until_table - def truncate_tables_in_batches(connection, tables_sorted, min_batch_size) + def connection + @connection ||= Gitlab::Database.database_base_models[database_name].connection + end + + def truncate_tables_in_batches(tables_sorted) truncated_tables = [] tables_sorted.flatten.each do |table| - sql_statement = "SELECT set_config('lock_writes.#{table}', 'false', false)" + table_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils + .extract_schema_qualified_name(table) + .identifier + + sql_statement = "SELECT set_config('lock_writes.#{table_name_without_schema}', 'false', false)" logger&.info(sql_statement) connection.execute(sql_statement) unless dry_run end diff --git a/lib/gitlab/database/type/indifferent_jsonb.rb b/lib/gitlab/database/type/indifferent_jsonb.rb new file mode 100644 index 00000000000..69bbcb383ba --- /dev/null +++ b/lib/gitlab/database/type/indifferent_jsonb.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Type + # Extends Rails' Jsonb data type to deserialize it into indifferent access Hash. + # + # Example: + # + # class SomeModel < ApplicationRecord + # # some_model.a_field is of type `jsonb` + # attribute :a_field, :ind_jsonb + # end + class IndifferentJsonb < ::ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Jsonb + def type + :ind_jsonb + end + + def deserialize(value) + data = super + return unless data + + ::Gitlab::Utils.deep_indifferent_access(data) + end + end + end + end +end |