diff options
Diffstat (limited to 'lib/tasks')
| -rw-r--r-- | lib/tasks/contracts.rake | 50 | ||||
| -rw-r--r-- | lib/tasks/gitlab/db.rake | 16 | ||||
| -rw-r--r-- | lib/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences.rake | 68 | ||||
| -rw-r--r-- | lib/tasks/gitlab/db/lock_writes.rake | 119 | ||||
| -rw-r--r-- | lib/tasks/gitlab/db/validate_config.rake | 77 | ||||
| -rw-r--r-- | lib/tasks/gitlab/pages.rake | 54 | ||||
| -rw-r--r-- | lib/tasks/gitlab/tw/codeowners.rake | 27 | ||||
| -rw-r--r-- | lib/tasks/migrate/composite_primary_keys.rake | 17 | ||||
| -rw-r--r-- | lib/tasks/rubocop.rake | 2 |
9 files changed, 334 insertions, 96 deletions
diff --git a/lib/tasks/contracts.rake b/lib/tasks/contracts.rake new file mode 100644 index 00000000000..6bb7f30ad57 --- /dev/null +++ b/lib/tasks/contracts.rake @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +require 'pact/tasks/verification_task' + +contracts = File.expand_path('../../spec/contracts', __dir__) +provider = File.expand_path('provider', contracts) + +# rubocop:disable Rails/RakeEnvironment +namespace :contracts do + namespace :mr do + Pact::VerificationTask.new(:diffs_batch) do |pact| + pact.uri( + "#{contracts}/contracts/project/merge_request/show/mergerequest#show-merge_request_diffs_batch_endpoint.json", + pact_helper: "#{provider}/pact_helpers/project/merge_request/diffs_batch_helper.rb" + ) + end + + Pact::VerificationTask.new(:diffs_metadata) do |pact| + pact.uri( + "#{contracts}/contracts/project/merge_request/show/" \ + "mergerequest#show-merge_request_diffs_metadata_endpoint.json", + pact_helper: "#{provider}/pact_helpers/project/merge_request/diffs_metadata_helper.rb" + ) + end + + Pact::VerificationTask.new(:discussions) do |pact| + pact.uri( + "#{contracts}/contracts/project/merge_request/show/mergerequest#show-merge_request_discussions_endpoint.json", + pact_helper: "#{provider}/pact_helpers/project/merge_request/discussions_helper.rb" + ) + end + + desc 'Run all merge request contract tests' + task 'test:merge_request', :contract_mr do |_t, arg| + raise(ArgumentError, 'Merge request contract tests require contract_mr to be set') unless arg[:contract_mr] + + ENV['CONTRACT_MR'] = arg[:contract_mr] + errors = %w[metadata discussions diffs].each_with_object([]) do |task, err| + Rake::Task["contracts:mr:pact:verify:#{task}"].execute + rescue StandardError, SystemExit + err << "contracts:mr:pact:verify:#{task}" + end + + raise StandardError, "Errors in tasks #{errors.join(', ')}" unless errors.empty? + end + end +end +# rubocop:enable Rails/RakeEnvironment diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 068dc463d16..a446a17dfc3 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -131,14 +131,6 @@ namespace :gitlab do end end - desc 'GitLab | DB | Sets up EE specific database functionality' - - if Gitlab.ee? - task setup_ee: %w[db:drop:geo db:create:geo db:schema:load:geo db:migrate:geo] - else - task :setup_ee - end - desc 'This adjusts and cleans db/structure.sql - it runs after db:structure:dump' task :clean_structure_sql do |task_name| ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config| @@ -356,7 +348,13 @@ namespace :gitlab do Rake::Task['db:drop'].invoke Rake::Task['db:create'].invoke ActiveRecord::Base.configurations.configs_for(env_name: ActiveRecord::Tasks::DatabaseTasks.env).each do |db_config| - ActiveRecord::Base.establish_connection(db_config.configuration_hash.merge(username: username)) # rubocop: disable Database/EstablishConnection + config = ActiveRecord::DatabaseConfigurations::HashConfig.new( + db_config.env_name, + db_config.name, + db_config.configuration_hash.merge(username: username) + ) + + ActiveRecord::Base.establish_connection(config) # rubocop: disable Database/EstablishConnection Gitlab::Database.check_for_non_superuser Rake::Task['db:migrate'].invoke end diff --git a/lib/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences.rake b/lib/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences.rake new file mode 100644 index 00000000000..4d78acb3011 --- /dev/null +++ b/lib/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences.rake @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :db do + namespace :decomposition do + namespace :rollback do + SEQUENCE_NAME_MATCHER = /nextval\('([a-z_]+)'::regclass\)/.freeze + + desc 'Bump all the CI tables sequences on the Main Database' + task :bump_ci_sequences, [:increase_by] => :environment do |_t, args| + increase_by = args.increase_by.to_i + if increase_by < 1 + puts 'Please specify a positive integer `increase_by` value'.color(:red) + puts 'Example: rake gitlab:db:decomposition:rollback:bump_ci_sequences[100000]'.color(:green) + exit 1 + end + + sequences_by_gitlab_schema(ApplicationRecord, :gitlab_ci).each do |sequence_name| + increment_sequence_by(ApplicationRecord.connection, sequence_name, increase_by) + end + end + end + end + end +end + +# base_model is to choose which connection to use to query the tables +# gitlab_schema, can be 'gitlab_main', 'gitlab_ci', 'gitlab_shared' +def sequences_by_gitlab_schema(base_model, gitlab_schema) + tables = Gitlab::Database::GitlabSchema.tables_to_schema.select do |_table_name, schema_name| + schema_name == gitlab_schema + end.keys + + models = tables.map do |table| + model = Class.new(base_model) + model.table_name = table + model + end + + sequences = [] + models.each do |model| + model.columns.each do |column| + match_result = column.default_function&.match(SEQUENCE_NAME_MATCHER) + next unless match_result + + sequences << match_result[1] + end + end + + sequences +end + +# This method is going to increase the sequence next_value by: +# - increment_by + 1 if the sequence has the attribute is_called = True (which is the common case) +# - increment_by if the sequence has the attribute is_called = False (for example, a newly created sequence) +# It uses ALTER SEQUENCE as a safety mechanism to avoid that no concurrent insertions +# will cause conflicts on the sequence. +# This is because ALTER SEQUENCE blocks concurrent nextval, currval, lastval, and setval calls. +def increment_sequence_by(connection, sequence_name, increment_by) + connection.transaction do + # The first call is to make sure that the sequence's is_called value is set to `true` + # This guarantees that the next call to `nextval` will increase the sequence by `increment_by` + connection.select_value("SELECT nextval($1)", nil, [sequence_name]) + connection.execute("ALTER SEQUENCE #{sequence_name} INCREMENT BY #{increment_by}") + connection.select_value("select nextval($1)", nil, [sequence_name]) + connection.execute("ALTER SEQUENCE #{sequence_name} INCREMENT BY 1") + end +end diff --git a/lib/tasks/gitlab/db/lock_writes.rake b/lib/tasks/gitlab/db/lock_writes.rake new file mode 100644 index 00000000000..b57c2860fe3 --- /dev/null +++ b/lib/tasks/gitlab/db/lock_writes.rake @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :db do + TRIGGER_FUNCTION_NAME = 'gitlab_schema_prevent_write' + + desc "GitLab | DB | Install prevent write triggers on all databases" + task lock_writes: [:environment, 'gitlab:db:validate_config'] do + Gitlab::Database::EachDatabase.each_database_connection do |connection, database_name| + create_write_trigger_function(connection) + + schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) + Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| + if schemas_for_connection.include?(schema_name.to_sym) + drop_write_trigger(database_name, connection, table_name) + else + create_write_trigger(database_name, connection, table_name) + end + end + end + end + + desc "GitLab | DB | Remove all triggers that prevents writes from all databases" + task unlock_writes: :environment do + Gitlab::Database::EachDatabase.each_database_connection do |connection, database_name| + Gitlab::Database::GitlabSchema.tables_to_schema.each do |table_name, schema_name| + drop_write_trigger(database_name, connection, table_name) + end + drop_write_trigger_function(connection) + end + end + + def create_write_trigger_function(connection) + sql = <<-SQL + CREATE OR REPLACE FUNCTION #{TRIGGER_FUNCTION_NAME}() + RETURNS TRIGGER AS + $$ + BEGIN + RAISE EXCEPTION 'Table: "%" is write protected within this Gitlab database.', TG_TABLE_NAME + USING ERRCODE = 'modifying_sql_data_not_permitted', + HINT = 'Make sure you are using the right database connection'; + END + $$ LANGUAGE PLPGSQL + SQL + + connection.execute(sql) + end + + def drop_write_trigger_function(connection) + sql = <<-SQL + DROP FUNCTION IF EXISTS #{TRIGGER_FUNCTION_NAME}() + SQL + + connection.execute(sql) + end + + def create_write_trigger(database_name, connection, table_name) + puts "#{database_name}: '#{table_name}'... Lock Writes".color(:yellow) + sql = <<-SQL + DROP TRIGGER IF EXISTS #{write_trigger_name(table_name)} ON #{table_name}; + CREATE TRIGGER #{write_trigger_name(table_name)} + BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE + ON #{table_name} + FOR EACH STATEMENT EXECUTE FUNCTION #{TRIGGER_FUNCTION_NAME}(); + SQL + + with_retries(connection) do + connection.execute(sql) + end + end + + def drop_write_trigger(database_name, connection, table_name) + puts "#{database_name}: '#{table_name}'... Allow Writes".color(:green) + sql = <<-SQL + DROP TRIGGER IF EXISTS #{write_trigger_name(table_name)} ON #{table_name} + SQL + + with_retries(connection) do + connection.execute(sql) + end + end + + def with_retries(connection, &block) + with_statement_timeout_retries do + with_lock_retries(connection) do + yield + end + end + end + + def with_statement_timeout_retries(times = 5) + current_iteration = 1 + begin + yield + rescue ActiveRecord::QueryCanceled => err + puts "Retrying after #{err.message}" + + if current_iteration <= times + current_iteration += 1 + retry + else + raise err + end + end + end + + def with_lock_retries(connection, &block) + Gitlab::Database::WithLockRetries.new( + klass: "gitlab:db:lock_writes", + logger: Gitlab::AppLogger, + connection: connection + ).run(&block) + end + + def write_trigger_name(table_name) + "gitlab_schema_write_trigger_for_#{table_name}" + end + end +end diff --git a/lib/tasks/gitlab/db/validate_config.rake b/lib/tasks/gitlab/db/validate_config.rake index 66aa949cc94..2a3a54b5351 100644 --- a/lib/tasks/gitlab/db/validate_config.rake +++ b/lib/tasks/gitlab/db/validate_config.rake @@ -4,6 +4,23 @@ databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml namespace :gitlab do namespace :db do + DB_CONFIG_NAME_KEY = 'gitlab_db_config_name' + + DB_IDENTIFIER_SQL = <<-SQL + SELECT system_identifier, current_database() + FROM pg_control_system() + SQL + + # We fetch timestamp as a way to properly handle race conditions + # fail in such cases, which should not really happen in production environment + DB_IDENTIFIER_WITH_DB_CONFIG_NAME_SQL = <<-SQL + SELECT + system_identifier, current_database(), + value as db_config_name, created_at as timestamp + FROM pg_control_system() + LEFT JOIN ar_internal_metadata ON ar_internal_metadata.key=$1 + SQL + desc 'Validates `config/database.yml` to ensure a correct behavior is configured' task validate_config: :environment do original_db_config = ActiveRecord::Base.connection_db_config # rubocop:disable Database/MultipleDatabases @@ -14,26 +31,22 @@ namespace :gitlab do db_configs = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, include_replicas: true) db_configs = db_configs.reject(&:replica?) + # The `pg_control_system()` is not enough to properly discover matching database systems + # since in case of cluster promotion it will return the same identifier as main cluster + # We instead set an `ar_internal_metadata` information with configured database name + db_configs.reverse_each do |db_config| + insert_db_identifier(db_config) + end + # Map each database connection into unique identifier of system+database - # rubocop:disable Database/MultipleDatabases all_connections = db_configs.map do |db_config| - identifier = - begin - ActiveRecord::Base.establish_connection(db_config) # rubocop: disable Database/EstablishConnection - ActiveRecord::Base.connection.select_one("SELECT system_identifier, current_database() FROM pg_control_system()") - rescue ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad => err - warn "WARNING: Could not establish database connection for #{db_config.name}: #{err.message}" - rescue ActiveRecord::NoDatabaseError - end - { name: db_config.name, config: db_config, database_tasks?: db_config.database_tasks?, - identifier: identifier + identifier: get_db_identifier(db_config) } - end.compact - # rubocop:enable Database/MultipleDatabases + end unique_connections = all_connections.group_by { |connection| connection[:identifier] } primary_connection = all_connections.find { |connection| ActiveRecord::Base.configurations.primary?(connection[:name]) } @@ -111,5 +124,43 @@ namespace :gitlab do Rake::Task["db:schema:load:#{name}"].enhance(['gitlab:db:validate_config']) Rake::Task["db:schema:dump:#{name}"].enhance(['gitlab:db:validate_config']) end + + def insert_db_identifier(db_config) + ActiveRecord::Base.establish_connection(db_config) # rubocop: disable Database/EstablishConnection + + if ActiveRecord::InternalMetadata.table_exists? + ts = Time.zone.now + + ActiveRecord::InternalMetadata.upsert( + { key: DB_CONFIG_NAME_KEY, + value: db_config.name, + created_at: ts, + updated_at: ts } + ) + end + rescue ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad => err + warn "WARNING: Could not establish database connection for #{db_config.name}: #{err.message}" + rescue ActiveRecord::NoDatabaseError + rescue ActiveRecord::StatementInvalid => err + raise unless err.cause.is_a?(PG::ReadOnlySqlTransaction) + + warn "WARNING: Could not write to the database #{db_config.name}: #{err.message}" + end + + def get_db_identifier(db_config) + ActiveRecord::Base.establish_connection(db_config) # rubocop: disable Database/EstablishConnection + + # rubocop:disable Database/MultipleDatabases + if ActiveRecord::InternalMetadata.table_exists? + ActiveRecord::Base.connection.select_one( + DB_IDENTIFIER_WITH_DB_CONFIG_NAME_SQL, nil, [DB_CONFIG_NAME_KEY]) + else + ActiveRecord::Base.connection.select_one(DB_IDENTIFIER_SQL) + end + # rubocop:enable Database/MultipleDatabases + rescue ActiveRecord::ConnectionNotEstablished, PG::ConnectionBad => err + warn "WARNING: Could not establish database connection for #{db_config.name}: #{err.message}" + rescue ActiveRecord::NoDatabaseError + end end end diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake index c3828e7eba4..e6fde28e38f 100644 --- a/lib/tasks/gitlab/pages.rake +++ b/lib/tasks/gitlab/pages.rake @@ -4,60 +4,6 @@ require 'logger' namespace :gitlab do namespace :pages do - desc "GitLab | Pages | Migrate legacy storage to zip format" - task migrate_legacy_storage: :gitlab_environment do - logger.info('Starting to migrate legacy pages storage to zip deployments') - - result = ::Pages::MigrateFromLegacyStorageService.new(logger, - ignore_invalid_entries: ignore_invalid_entries, - mark_projects_as_not_deployed: mark_projects_as_not_deployed) - .execute_with_threads(threads: migration_threads, batch_size: batch_size) - - logger.info("A total of #{result[:migrated] + result[:errored]} projects were processed.") - logger.info("- The #{result[:migrated]} projects migrated successfully") - logger.info("- The #{result[:errored]} projects failed to be migrated") - end - - desc "GitLab | Pages | DANGER: Removes data which was migrated from legacy storage on zip storage. Can be used if some bugs in migration are discovered and migration needs to be restarted from scratch." - task clean_migrated_zip_storage: :gitlab_environment do - destroyed_deployments = 0 - - logger.info("Starting to delete migrated pages deployments") - - ::PagesDeployment.migrated_from_legacy_storage.each_batch(of: batch_size) do |batch| - destroyed_deployments += batch.count - - # we need to destroy associated files, so can't use delete_all - batch.destroy_all # rubocop: disable Cop/DestroyAll - - logger.info("#{destroyed_deployments} deployments were deleted") - end - end - - def logger - @logger ||= Logger.new($stdout) - end - - def migration_threads - ENV.fetch('PAGES_MIGRATION_THREADS', '3').to_i - end - - def batch_size - ENV.fetch('PAGES_MIGRATION_BATCH_SIZE', '10').to_i - end - - def ignore_invalid_entries - Gitlab::Utils.to_boolean( - ENV.fetch('PAGES_MIGRATION_IGNORE_INVALID_ENTRIES', 'false') - ) - end - - def mark_projects_as_not_deployed - Gitlab::Utils.to_boolean( - ENV.fetch('PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED', 'false') - ) - end - namespace :deployments do task migrate_to_object_storage: :gitlab_environment do logger = Logger.new($stdout) diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index 6574bfd2549..40d88ea8a5b 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -6,6 +6,17 @@ namespace :tw do desc 'Generates a list of codeowners for documentation pages.' task :codeowners do CodeOwnerRule = Struct.new(:category, :writer) + DocumentOwnerMapping = Struct.new(:path, :writer) do + def writer_owns_all_pages?(mappings) + mappings + .select { |mapping| mapping.directory == directory } + .all? { |mapping| mapping.writer == writer } + end + + def directory + @directory ||= File.dirname(path) + end + end CODE_OWNER_RULES = [ CodeOwnerRule.new('Activation', '@kpaizee'), @@ -61,7 +72,6 @@ namespace :tw do CodeOwnerRule.new('Sharding', '@sselhorn'), CodeOwnerRule.new('Source Code', '@aqualls'), CodeOwnerRule.new('Static Analysis', '@rdickenson'), - CodeOwnerRule.new('Static Site Editor', '@aqualls'), CodeOwnerRule.new('Style Guide', '@sselhorn'), CodeOwnerRule.new('Testing', '@eread'), CodeOwnerRule.new('Threat Insights', '@claytoncornell'), @@ -85,6 +95,7 @@ namespace :tw do end errors = [] + mappings = [] path = Rails.root.join("doc/**/*.md") Dir.glob(path) do |file| @@ -99,9 +110,21 @@ namespace :tw do writer = writer_for_group(document.group) next unless writer - puts "#{file.gsub(Dir.pwd, ".")} #{writer}" if document.has_a_valid_group? + mappings << DocumentOwnerMapping.new(file.delete_prefix(Dir.pwd), writer) if document.has_a_valid_group? end + deduplicated_mappings = Set.new + + mappings.each do |mapping| + if mapping.writer_owns_all_pages?(mappings) + deduplicated_mappings.add("#{mapping.directory}/ #{mapping.writer}") + else + deduplicated_mappings.add("#{mapping.path} #{mapping.writer}") + end + end + + deduplicated_mappings.each { |mapping| puts mapping } + if errors.present? puts "-----" puts "ERRORS - the following files are missing the correct metadata:" diff --git a/lib/tasks/migrate/composite_primary_keys.rake b/lib/tasks/migrate/composite_primary_keys.rake deleted file mode 100644 index 68f7c4d6c4a..00000000000 --- a/lib/tasks/migrate/composite_primary_keys.rake +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -namespace :gitlab do - namespace :db do - desc 'GitLab | DB | Adds primary keys to tables that only have composite unique keys' - task composite_primary_keys_add: :environment do - require Rails.root.join('db/optional_migrations/composite_primary_keys') - CompositePrimaryKeysMigration.new.up - end - - desc 'GitLab | DB | Removes previously added composite primary keys' - task composite_primary_keys_drop: :environment do - require Rails.root.join('db/optional_migrations/composite_primary_keys') - CompositePrimaryKeysMigration.new.down - end - end -end diff --git a/lib/tasks/rubocop.rake b/lib/tasks/rubocop.rake index 6eabdf51dcd..28c370e5ca6 100644 --- a/lib/tasks/rubocop.rake +++ b/lib/tasks/rubocop.rake @@ -36,7 +36,7 @@ unless Rails.env.production? # expected. cop_names = args.to_a - todo_dir = RuboCop::TodoDir.new(RuboCop::TodoDir::DEFAULT_TODO_DIR) + todo_dir = RuboCop::TodoDir.new(RuboCop::Formatter::TodoFormatter.base_directory) if cop_names.any? # We are sorting the cop names to benefit from RuboCop cache which |