diff options
Diffstat (limited to 'lib/gitlab')
6 files changed, 90 insertions, 76 deletions
diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 5c56594da78..b95b36fd555 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -156,6 +156,9 @@ gemnasium-python-dependency_scanning: bundler-audit-dependency_scanning: extends: .ds-analyzer + variables: + DS_ANALYZER_NAME: "bundler-audit" + DS_MAJOR_VERSION: 2 script: - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.0" - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/347491" @@ -165,6 +168,9 @@ bundler-audit-dependency_scanning: retire-js-dependency_scanning: extends: .ds-analyzer + variables: + DS_ANALYZER_NAME: "retire.js" + DS_MAJOR_VERSION: 2 script: - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.0" - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/289830" diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index a90cae7aea2..d052d5adc4c 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -30,9 +30,23 @@ module Gitlab scope :created_after, ->(time) { where('created_at > ?', time) } - scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do - where(job_class_name: job_class_name, table_name: table_name, column_name: column_name) + scope :for_configuration, ->(gitlab_schema, job_class_name, table_name, column_name, job_arguments) do + relation = where(job_class_name: job_class_name, table_name: table_name, column_name: column_name) .where("job_arguments = ?", job_arguments.to_json) # rubocop:disable Rails/WhereEquals + + # This method is called from migrations older than the gitlab_schema column, + # check and add this filter only if the column exists. + relation = relation.for_gitlab_schema(gitlab_schema) if gitlab_schema_column_exists? + + relation + end + + def self.gitlab_schema_column_exists? + column_names.include?('gitlab_schema') + end + + scope :for_gitlab_schema, ->(gitlab_schema) do + where(gitlab_schema: gitlab_schema) end state_machine :status, initial: :paused do @@ -73,12 +87,13 @@ module Gitlab state_machine.states.map(&:name) end - def self.find_for_configuration(job_class_name, table_name, column_name, job_arguments) - for_configuration(job_class_name, table_name, column_name, job_arguments).first + def self.find_for_configuration(gitlab_schema, job_class_name, table_name, column_name, job_arguments) + for_configuration(gitlab_schema, job_class_name, table_name, column_name, job_arguments).first end - def self.active_migration - executable.queue_order.first + def self.active_migration(connection:) + for_gitlab_schema(Gitlab::Database.gitlab_schemas_for_connection(connection)) + .executable.queue_order.first end def self.successful_rows_counts(migrations) diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 59ff9a9744f..388eb596ce2 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -54,7 +54,10 @@ module Gitlab # in order to prevent it being picked up by the background worker. Perform all pending jobs, # then keep running until migration is finished. def finalize(job_class_name, table_name, column_name, job_arguments) - migration = BatchedMigration.find_for_configuration(job_class_name, table_name, column_name, job_arguments) + migration = BatchedMigration.find_for_configuration( + Gitlab::Database.gitlab_schemas_for_connection(connection), + job_class_name, table_name, column_name, job_arguments + ) configuration = { job_class_name: job_class_name, diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 0453b81d67d..17ada9395d5 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -945,8 +945,13 @@ module Gitlab end def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true) - migration = Gitlab::Database::BackgroundMigration::BatchedMigration - .for_configuration(job_class_name, table_name, column_name, job_arguments).first + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! + + Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration( + Gitlab::Database.gitlab_schemas_for_connection(connection), + job_class_name, table_name, column_name, job_arguments + ) configuration = { job_class_name: job_class_name, diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 7113c3686f1..72b94964c5b 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -67,10 +67,16 @@ module Gitlab batch_class_name: BATCH_CLASS_NAME, batch_size: BATCH_SIZE, max_batch_size: nil, - sub_batch_size: SUB_BATCH_SIZE + sub_batch_size: SUB_BATCH_SIZE, + gitlab_schema: nil ) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! - if Gitlab::Database::BackgroundMigration::BatchedMigration.for_configuration(job_class_name, batch_table_name, batch_column_name, job_arguments).exists? + gitlab_schema ||= gitlab_schema_from_context + + Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information + + if Gitlab::Database::BackgroundMigration::BatchedMigration.for_configuration(gitlab_schema, job_class_name, batch_table_name, batch_column_name, job_arguments).exists? Gitlab::AppLogger.warn "Batched background migration not enqueued because it already exists: " \ "job_class_name: #{job_class_name}, table_name: #{batch_table_name}, column_name: #{batch_column_name}, " \ "job_arguments: #{job_arguments.inspect}" @@ -119,11 +125,17 @@ module Gitlab end end + if migration.respond_to?(:gitlab_schema) + migration.gitlab_schema = gitlab_schema + end + migration.save! migration end def finalize_batched_background_migration(job_class_name:, table_name:, column_name:, job_arguments:) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! + database_name = Gitlab::Database.db_config_name(connection) unless ActiveRecord::Base.configurations.primary?(database_name) @@ -132,12 +144,47 @@ module Gitlab 'https://docs.gitlab.com/ee/development/database/migrations_for_multiple_databases.html' end - migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration(job_class_name, table_name, column_name, job_arguments) + Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information + + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration( + gitlab_schema_from_context, job_class_name, table_name, column_name, job_arguments) raise 'Could not find batched background migration' if migration.nil? Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.finalize(job_class_name, table_name, column_name, job_arguments, connection: connection) end + + # Deletes batched background migration for the given configuration. + # + # job_class_name - The background migration job class as a string + # table_name - The name of the table the migration iterates over + # column_name - The name of the column the migration will batch over + # job_arguments - Migration arguments + # + # Example: + # + # delete_batched_background_migration( + # 'CopyColumnUsingBackgroundMigrationJob', + # :events, + # :id, + # ['column1', 'column2']) + def delete_batched_background_migration(job_class_name, table_name, column_name, job_arguments) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! + Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information + + Gitlab::Database::BackgroundMigration::BatchedMigration + .for_configuration( + gitlab_schema_from_context, job_class_name, table_name, column_name, job_arguments + ).delete_all + end + + def gitlab_schema_from_context + if respond_to?(:allowed_gitlab_schemas) # Gitlab::Database::Migration::V2_0 + Array(allowed_gitlab_schemas).first + else # Gitlab::Database::Migration::V1_0 + :gitlab_main + end + end end end end diff --git a/lib/gitlab/patch/database_config.rb b/lib/gitlab/patch/database_config.rb index c5c73d50518..20d8f7be8fd 100644 --- a/lib/gitlab/patch/database_config.rb +++ b/lib/gitlab/patch/database_config.rb @@ -1,77 +1,15 @@ # frozen_string_literal: true -# The purpose of this code is to transform legacy `database.yml` -# into a `database.yml` containing `main:` as a name of a first database -# -# This should be removed once all places using legacy `database.yml` -# are fixed. The likely moment to remove this check is the %14.0. -# -# This converts the following syntax: -# -# production: -# adapter: postgresql -# database: gitlabhq_production -# username: git -# password: "secure password" -# host: localhost -# -# Into: -# -# production: -# main: -# adapter: postgresql -# database: gitlabhq_production -# username: git -# password: "secure password" -# host: localhost -# - +# The purpose of this code is to set the migrations path +# for the Geo tracking database. module Gitlab module Patch module DatabaseConfig extend ActiveSupport::Concern - def load_database_yaml - return super unless Gitlab.ee? - - super.deep_merge(load_geo_database_yaml) - end - - # This method is taken from Rails to load a database YAML file without - # evaluating ERB. This allows us to create the rake tasks for the Geo - # tracking database without filling in the configuration values or - # loading the environment. To be removed when we start configure Geo - # tracking database in database.yml instead of custom database_geo.yml - # - # https://github.com/rails/rails/blob/v6.1.4/railties/lib/rails/application/configuration.rb#L255 - def load_geo_database_yaml - path = Rails.root.join("config/database_geo.yml") - return {} unless File.exist?(path) - - require "rails/application/dummy_erb_compiler" - - yaml = DummyERB.new(Pathname.new(path).read).result - config = YAML.load(yaml) || {} # rubocop:disable Security/YAMLLoad - - config.to_h do |env, configs| - # This check is taken from Rails where the transformation - # of a flat database.yml is done into `primary:` - # https://github.com/rails/rails/blob/v6.1.4/activerecord/lib/active_record/database_configurations.rb#L169 - if configs.is_a?(Hash) && !configs.all? { |_, v| v.is_a?(Hash) } - configs = { "geo" => configs } - end - - [env, configs] - end - end - def database_configuration super.to_h do |env, configs| if Gitlab.ee? - if !configs.key?("geo") && File.exist?(Rails.root.join("config/database_geo.yml")) - configs["geo"] = Rails.application.config_for(:database_geo).stringify_keys - end - if configs.key?("geo") migrations_paths = Array(configs["geo"]["migrations_paths"]) migrations_paths << "ee/db/geo/migrate" if migrations_paths.empty? |