diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-01-18 22:00:14 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-01-18 22:00:14 +0300 |
commit | 05f0ebba3a2c8ddf39e436f412dc2ab5bf1353b2 (patch) | |
tree | 11d0f2a6ec31c7793c184106cedc2ded3d9a2cc5 /lib/tasks | |
parent | ec73467c23693d0db63a797d10194da9e72a74af (diff) |
Add latest changes from gitlab-org/gitlab@15-8-stable-eev15.8.0-rc42
Diffstat (limited to 'lib/tasks')
-rw-r--r-- | lib/tasks/contracts/merge_requests.rake | 9 | ||||
-rw-r--r-- | lib/tasks/contracts/pipeline_schedules.rake | 2 | ||||
-rw-r--r-- | lib/tasks/contracts/pipelines.rake | 8 | ||||
-rw-r--r-- | lib/tasks/gitlab/db.rake | 36 | ||||
-rw-r--r-- | lib/tasks/gitlab/db/lock_writes.rake | 2 | ||||
-rw-r--r-- | lib/tasks/gitlab/security/update_banned_ssh_keys.rake | 72 | ||||
-rw-r--r-- | lib/tasks/gitlab/seed/runner_fleet.rake | 40 | ||||
-rw-r--r-- | lib/tasks/gitlab/tw/codeowners.rake | 2 |
8 files changed, 153 insertions, 18 deletions
diff --git a/lib/tasks/contracts/merge_requests.rake b/lib/tasks/contracts/merge_requests.rake index 61823f0cf1a..5a6186d393d 100644 --- a/lib/tasks/contracts/merge_requests.rake +++ b/lib/tasks/contracts/merge_requests.rake @@ -14,15 +14,16 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/merge_requests/show/get_diffs_batch_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end Pact::VerificationTask.new(:get_diffs_metadata) do |pact| pact_helper_location = "pact_helpers/project/merge_requests/show/get_diffs_metadata_helper.rb" + pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end @@ -31,14 +32,14 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/merge_requests/show/get_discussions_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end desc 'Run all merge request contract tests' task 'test:merge_requests', :contract_merge_requests do |_t, arg| - errors = %w[diffs_batch diffs_metadata discussions].each_with_object([]) do |task, err| + errors = %w[get_diffs_batch get_diffs_metadata get_discussions].each_with_object([]) do |task, err| Rake::Task["contracts:merge_requests:pact:verify:#{task}"].execute rescue StandardError, SystemExit err << "contracts:merge_requests:pact:verify:#{task}" diff --git a/lib/tasks/contracts/pipeline_schedules.rake b/lib/tasks/contracts/pipeline_schedules.rake index b4c87d2e3c9..f3e65b94940 100644 --- a/lib/tasks/contracts/pipeline_schedules.rake +++ b/lib/tasks/contracts/pipeline_schedules.rake @@ -14,7 +14,7 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/pipeline_schedules/edit/put_edit_a_pipeline_schedule_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end diff --git a/lib/tasks/contracts/pipelines.rake b/lib/tasks/contracts/pipelines.rake index 55a7baa4539..13c973f1358 100644 --- a/lib/tasks/contracts/pipelines.rake +++ b/lib/tasks/contracts/pipelines.rake @@ -14,7 +14,7 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/pipelines/new/post_create_a_new_pipeline_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end @@ -23,7 +23,7 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/pipelines/index/get_list_project_pipelines_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end @@ -32,7 +32,7 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/pipelines/show/get_pipeline_header_data_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end @@ -41,7 +41,7 @@ namespace :contracts do pact_helper_location = "pact_helpers/project/pipelines/show/delete_pipeline_helper.rb" pact.uri( - Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location), + Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location), pact_helper: "#{provider}/#{pact_helper_location}" ) end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index f0264456201..9c92aa5eb28 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -407,7 +407,12 @@ namespace :gitlab do Rails.application.eager_load! tables = Gitlab::Database.database_base_models.flat_map { |_, m| m.connection.tables } - classes = tables.index_with { [] } + + views = Gitlab::Database.database_base_models.flat_map { |_, m| m.connection.views } + + sources = tables + views + + classes = sources.index_with { [] } Gitlab::Database.database_base_models.each do |_, model_class| model_class @@ -421,12 +426,13 @@ namespace :gitlab do version = Gem::Version.new(File.read('VERSION')) milestone = version.release.segments[0..1].join('.') - tables.each do |table_name| - file = File.join(DB_DOCS_PATH, "#{table_name}.yml") + sources.each do |source_name| + file = dictionary_file_path(source_name, views) + key_name = "#{data_source_type(source_name, views)}_name" table_metadata = { - 'table_name' => table_name, - 'classes' => classes[table_name]&.sort&.uniq, + key_name => source_name, + 'classes' => classes[source_name]&.sort&.uniq, 'feature_categories' => [], 'description' => nil, 'introduced_by_url' => nil, @@ -438,12 +444,12 @@ namespace :gitlab do existing_metadata = YAML.safe_load(File.read(file)) - if existing_metadata['table_name'] != table_metadata['table_name'] - existing_metadata['table_name'] = table_metadata['table_name'] + if existing_metadata[key_name] != table_metadata[key_name] + existing_metadata[key_name] = table_metadata[key_name] outdated = true end - if existing_metadata['classes'].difference(table_metadata['classes']).any? + if existing_metadata['classes'].sort != table_metadata['classes'].sort existing_metadata['classes'] = table_metadata['classes'] outdated = true end @@ -455,6 +461,20 @@ namespace :gitlab do end end + private + + def data_source_type(source_name, views) + return 'view' if views.include?(source_name) + + 'table' + end + + def dictionary_file_path(source_name, views) + sub_directory = views.include?(source_name) ? 'views' : '' + + File.join(DB_DOCS_PATH, sub_directory, "#{source_name}.yml") + end + # Temporary disable this, see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85760#note_998452069 # Rake::Task['db:migrate'].enhance do # Rake::Task['gitlab:db:dictionary:generate'].invoke if Rails.env.development? diff --git a/lib/tasks/gitlab/db/lock_writes.rake b/lib/tasks/gitlab/db/lock_writes.rake index a856aa77abc..212d60a7231 100644 --- a/lib/tasks/gitlab/db/lock_writes.rake +++ b/lib/tasks/gitlab/db/lock_writes.rake @@ -15,6 +15,7 @@ namespace :gitlab do table_name: table_name, connection: connection, database_name: database_name, + with_retries: true, logger: Logger.new($stdout), dry_run: ENV['DRY_RUN'] == 'true' ) @@ -39,6 +40,7 @@ namespace :gitlab do table_name: table_name, connection: connection, database_name: database_name, + with_retries: true, logger: Logger.new($stdout) ) diff --git a/lib/tasks/gitlab/security/update_banned_ssh_keys.rake b/lib/tasks/gitlab/security/update_banned_ssh_keys.rake new file mode 100644 index 00000000000..b3f8bb16ef9 --- /dev/null +++ b/lib/tasks/gitlab/security/update_banned_ssh_keys.rake @@ -0,0 +1,72 @@ +# frozen_string_literal: true +# Update banned SSH keys from a Git repository +# +# This task: +# - Reads banned SSH keys from a Git repository, and updates default key set at config/security/banned_ssh_keys.yml +# - Stops uploading new keys if YAML file size is greater than 2 MB. +# - Caution: The task adds all the files with suffix of .pub, and does NOT check the key's contents. +# +# @param git_url - Remote Git URL. +# @param output_file - Update keys to an output file. Default is config/security/banned_ssh_keys.yml. +# +# @example +# bundle exec rake "gitlab:security:update_banned_ssh_keys[https://github.com/rapid7/ssh-badkeys]" +# +MAX_CONFIG_SIZE = 2.megabytes.freeze + +namespace :gitlab do + namespace :security do + desc 'GitLab | Security | Update banned_ssh_keys config file from a remote Git repository' + task :update_banned_ssh_keys, [:git_url, :output_file] => :gitlab_environment do |_t, args| + require 'yaml' + require 'git' + require 'find' + require_relative '../../../../config/environment' + logger = Logger.new($stdout) + begin + exit 0 unless Rails.env.test? || Rails.env.development? + name = args.git_url.rpartition('/').last.delete_suffix('.git') + tmp_path = Dir.mktmpdir + logger.info "start to clone the git repository at #{tmp_path}/#{name}" + Git.clone(args.git_url, name, path: tmp_path) + logger.info "Git clone finished. Next, add bad keys to config/security/banned_ssh_keys.yml." + + path = args.output_file || Rails.root.join('config/security/banned_ssh_keys.yml') + config_size = File.size?(path) || 0 + exit 0 if config_size > MAX_CONFIG_SIZE + + config = (YAML.load_file(path) if File.exist?(path)) || {} + + Find.find("#{tmp_path}/#{name}") do |path| + next unless path.end_with?('.pub') + + if config_size > MAX_CONFIG_SIZE + logger.info "banned_ssh_keys.yml has grown too large - halting execution" + break + end + + logger.info "update bad SSH keys in #{path}" + keys = File.readlines(path, chomp: true) + keys.each do |key| + pub = Gitlab::SSHPublicKey.new(key) + + type = pub.type.to_s + config[type] = [] unless config.key?(type) + + next if config[type].include?(pub.fingerprint_sha256) + + config[type].append(pub.fingerprint_sha256) + config_size += pub.fingerprint_sha256.size + end + end + rescue StandardError => e + logger.error "Exception: #{e.message}" + logger.debug e.backtrace + exit 1 + end + + logger.info "finish writing." + File.open(path, 'w') { |file| file.write(config.to_yaml) } + end + end +end diff --git a/lib/tasks/gitlab/seed/runner_fleet.rake b/lib/tasks/gitlab/seed/runner_fleet.rake new file mode 100644 index 00000000000..c0b79269c75 --- /dev/null +++ b/lib/tasks/gitlab/seed/runner_fleet.rake @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# Seed database with: +# 1. 2 root groups, one with 2 sub-groups and another with 1 sub-group +# 1. 1 project in each of the sub-groups +# 1. 1 instance runner, 1 shared project runner, and group/project runners in some groups/projects +# 1. Successful and failed pipelines assigned to the first 5 available runners of each group/project +# 1. 1 pipeline on one group runner with the remaining jobs +# +# @param username - user creating subgroups (i.e. GitLab admin) +# @param registration_prefix - prefix used for the group, project, and runner names +# @param runner_count - total number of runners to create (default: 40) +# @param job_count - total number of jobs to create and assign to runners (default: 400) +# +# @example +# bundle exec rake "gitlab:seed:runner_fleet[root, rf-]" +# +namespace :gitlab do + namespace :seed do + desc 'Seed groups with sub-groups/projects/runners/jobs for Runner Fleet testing' + task :runner_fleet, + [:username, :registration_prefix, :runner_count, :job_count] => :gitlab_environment do |_t, args| + timings = Benchmark.measure do + projects_to_runners = Gitlab::Seeders::Ci::Runner::RunnerFleetSeeder.new( + Gitlab::AppLogger, + username: args.username, + registration_prefix: args.registration_prefix, + runner_count: args.runner_count&.to_i + ).seed + + Gitlab::Seeders::Ci::Runner::RunnerFleetPipelineSeeder.new( + projects_to_runners: projects_to_runners, + job_count: args.job_count&.to_i + ).seed + end + + puts "Seed finished. Timings: #{timings}" + end + end +end diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index ec2ea623e02..b3559bde988 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -78,7 +78,7 @@ namespace :tw do CodeOwnerRule.new('Tutorials', '@kpaizee'), CodeOwnerRule.new('Utilization', '@fneill'), CodeOwnerRule.new('Vulnerability Research', '@claytoncornell'), - CodeOwnerRule.new('Workspace', '@lciutacu') + CodeOwnerRule.new('Organization', '@lciutacu') ].freeze ERRORS_EXCLUDED_FILES = [ |