Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/lib/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'lib/tasks')
-rw-r--r--lib/tasks/contracts/merge_requests.rake9
-rw-r--r--lib/tasks/contracts/pipeline_schedules.rake2
-rw-r--r--lib/tasks/contracts/pipelines.rake8
-rw-r--r--lib/tasks/gitlab/db.rake36
-rw-r--r--lib/tasks/gitlab/db/lock_writes.rake2
-rw-r--r--lib/tasks/gitlab/security/update_banned_ssh_keys.rake72
-rw-r--r--lib/tasks/gitlab/seed/runner_fleet.rake40
-rw-r--r--lib/tasks/gitlab/tw/codeowners.rake2
8 files changed, 153 insertions, 18 deletions
diff --git a/lib/tasks/contracts/merge_requests.rake b/lib/tasks/contracts/merge_requests.rake
index 61823f0cf1a..5a6186d393d 100644
--- a/lib/tasks/contracts/merge_requests.rake
+++ b/lib/tasks/contracts/merge_requests.rake
@@ -14,15 +14,16 @@ namespace :contracts do
pact_helper_location = "pact_helpers/project/merge_requests/show/get_diffs_batch_helper.rb"
pact.uri(
- Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location),
+ Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location),
pact_helper: "#{provider}/#{pact_helper_location}"
)
end
Pact::VerificationTask.new(:get_diffs_metadata) do |pact|
pact_helper_location = "pact_helpers/project/merge_requests/show/get_diffs_metadata_helper.rb"
+
pact.uri(
- Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location),
+ Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location),
pact_helper: "#{provider}/#{pact_helper_location}"
)
end
@@ -31,14 +32,14 @@ namespace :contracts do
pact_helper_location = "pact_helpers/project/merge_requests/show/get_discussions_helper.rb"
pact.uri(
- Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location),
+ Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location),
pact_helper: "#{provider}/#{pact_helper_location}"
)
end
desc 'Run all merge request contract tests'
task 'test:merge_requests', :contract_merge_requests do |_t, arg|
- errors = %w[diffs_batch diffs_metadata discussions].each_with_object([]) do |task, err|
+ errors = %w[get_diffs_batch get_diffs_metadata get_discussions].each_with_object([]) do |task, err|
Rake::Task["contracts:merge_requests:pact:verify:#{task}"].execute
rescue StandardError, SystemExit
err << "contracts:merge_requests:pact:verify:#{task}"
diff --git a/lib/tasks/contracts/pipeline_schedules.rake b/lib/tasks/contracts/pipeline_schedules.rake
index b4c87d2e3c9..f3e65b94940 100644
--- a/lib/tasks/contracts/pipeline_schedules.rake
+++ b/lib/tasks/contracts/pipeline_schedules.rake
@@ -14,7 +14,7 @@ namespace :contracts do
pact_helper_location = "pact_helpers/project/pipeline_schedules/edit/put_edit_a_pipeline_schedule_helper.rb"
pact.uri(
- Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location),
+ Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location),
pact_helper: "#{provider}/#{pact_helper_location}"
)
end
diff --git a/lib/tasks/contracts/pipelines.rake b/lib/tasks/contracts/pipelines.rake
index 55a7baa4539..13c973f1358 100644
--- a/lib/tasks/contracts/pipelines.rake
+++ b/lib/tasks/contracts/pipelines.rake
@@ -14,7 +14,7 @@ namespace :contracts do
pact_helper_location = "pact_helpers/project/pipelines/new/post_create_a_new_pipeline_helper.rb"
pact.uri(
- Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location),
+ Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location),
pact_helper: "#{provider}/#{pact_helper_location}"
)
end
@@ -23,7 +23,7 @@ namespace :contracts do
pact_helper_location = "pact_helpers/project/pipelines/index/get_list_project_pipelines_helper.rb"
pact.uri(
- Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location),
+ Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location),
pact_helper: "#{provider}/#{pact_helper_location}"
)
end
@@ -32,7 +32,7 @@ namespace :contracts do
pact_helper_location = "pact_helpers/project/pipelines/show/get_pipeline_header_data_helper.rb"
pact.uri(
- Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location),
+ Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location),
pact_helper: "#{provider}/#{pact_helper_location}"
)
end
@@ -41,7 +41,7 @@ namespace :contracts do
pact_helper_location = "pact_helpers/project/pipelines/show/delete_pipeline_helper.rb"
pact.uri(
- Provider::ContractSourceHelper.contract_location(:rake, pact_helper_location),
+ Provider::ContractSourceHelper.contract_location(requester: :rake, file_path: pact_helper_location),
pact_helper: "#{provider}/#{pact_helper_location}"
)
end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index f0264456201..9c92aa5eb28 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -407,7 +407,12 @@ namespace :gitlab do
Rails.application.eager_load!
tables = Gitlab::Database.database_base_models.flat_map { |_, m| m.connection.tables }
- classes = tables.index_with { [] }
+
+ views = Gitlab::Database.database_base_models.flat_map { |_, m| m.connection.views }
+
+ sources = tables + views
+
+ classes = sources.index_with { [] }
Gitlab::Database.database_base_models.each do |_, model_class|
model_class
@@ -421,12 +426,13 @@ namespace :gitlab do
version = Gem::Version.new(File.read('VERSION'))
milestone = version.release.segments[0..1].join('.')
- tables.each do |table_name|
- file = File.join(DB_DOCS_PATH, "#{table_name}.yml")
+ sources.each do |source_name|
+ file = dictionary_file_path(source_name, views)
+ key_name = "#{data_source_type(source_name, views)}_name"
table_metadata = {
- 'table_name' => table_name,
- 'classes' => classes[table_name]&.sort&.uniq,
+ key_name => source_name,
+ 'classes' => classes[source_name]&.sort&.uniq,
'feature_categories' => [],
'description' => nil,
'introduced_by_url' => nil,
@@ -438,12 +444,12 @@ namespace :gitlab do
existing_metadata = YAML.safe_load(File.read(file))
- if existing_metadata['table_name'] != table_metadata['table_name']
- existing_metadata['table_name'] = table_metadata['table_name']
+ if existing_metadata[key_name] != table_metadata[key_name]
+ existing_metadata[key_name] = table_metadata[key_name]
outdated = true
end
- if existing_metadata['classes'].difference(table_metadata['classes']).any?
+ if existing_metadata['classes'].sort != table_metadata['classes'].sort
existing_metadata['classes'] = table_metadata['classes']
outdated = true
end
@@ -455,6 +461,20 @@ namespace :gitlab do
end
end
+ private
+
+ def data_source_type(source_name, views)
+ return 'view' if views.include?(source_name)
+
+ 'table'
+ end
+
+ def dictionary_file_path(source_name, views)
+ sub_directory = views.include?(source_name) ? 'views' : ''
+
+ File.join(DB_DOCS_PATH, sub_directory, "#{source_name}.yml")
+ end
+
# Temporary disable this, see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85760#note_998452069
# Rake::Task['db:migrate'].enhance do
# Rake::Task['gitlab:db:dictionary:generate'].invoke if Rails.env.development?
diff --git a/lib/tasks/gitlab/db/lock_writes.rake b/lib/tasks/gitlab/db/lock_writes.rake
index a856aa77abc..212d60a7231 100644
--- a/lib/tasks/gitlab/db/lock_writes.rake
+++ b/lib/tasks/gitlab/db/lock_writes.rake
@@ -15,6 +15,7 @@ namespace :gitlab do
table_name: table_name,
connection: connection,
database_name: database_name,
+ with_retries: true,
logger: Logger.new($stdout),
dry_run: ENV['DRY_RUN'] == 'true'
)
@@ -39,6 +40,7 @@ namespace :gitlab do
table_name: table_name,
connection: connection,
database_name: database_name,
+ with_retries: true,
logger: Logger.new($stdout)
)
diff --git a/lib/tasks/gitlab/security/update_banned_ssh_keys.rake b/lib/tasks/gitlab/security/update_banned_ssh_keys.rake
new file mode 100644
index 00000000000..b3f8bb16ef9
--- /dev/null
+++ b/lib/tasks/gitlab/security/update_banned_ssh_keys.rake
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+# Update banned SSH keys from a Git repository
+#
+# This task:
+# - Reads banned SSH keys from a Git repository, and updates default key set at config/security/banned_ssh_keys.yml
+# - Stops uploading new keys if YAML file size is greater than 2 MB.
+# - Caution: The task adds all the files with suffix of .pub, and does NOT check the key's contents.
+#
+# @param git_url - Remote Git URL.
+# @param output_file - Update keys to an output file. Default is config/security/banned_ssh_keys.yml.
+#
+# @example
+# bundle exec rake "gitlab:security:update_banned_ssh_keys[https://github.com/rapid7/ssh-badkeys]"
+#
+MAX_CONFIG_SIZE = 2.megabytes.freeze
+
+namespace :gitlab do
+ namespace :security do
+ desc 'GitLab | Security | Update banned_ssh_keys config file from a remote Git repository'
+ task :update_banned_ssh_keys, [:git_url, :output_file] => :gitlab_environment do |_t, args|
+ require 'yaml'
+ require 'git'
+ require 'find'
+ require_relative '../../../../config/environment'
+ logger = Logger.new($stdout)
+ begin
+ exit 0 unless Rails.env.test? || Rails.env.development?
+ name = args.git_url.rpartition('/').last.delete_suffix('.git')
+ tmp_path = Dir.mktmpdir
+ logger.info "start to clone the git repository at #{tmp_path}/#{name}"
+ Git.clone(args.git_url, name, path: tmp_path)
+ logger.info "Git clone finished. Next, add bad keys to config/security/banned_ssh_keys.yml."
+
+ path = args.output_file || Rails.root.join('config/security/banned_ssh_keys.yml')
+ config_size = File.size?(path) || 0
+ exit 0 if config_size > MAX_CONFIG_SIZE
+
+ config = (YAML.load_file(path) if File.exist?(path)) || {}
+
+ Find.find("#{tmp_path}/#{name}") do |path|
+ next unless path.end_with?('.pub')
+
+ if config_size > MAX_CONFIG_SIZE
+ logger.info "banned_ssh_keys.yml has grown too large - halting execution"
+ break
+ end
+
+ logger.info "update bad SSH keys in #{path}"
+ keys = File.readlines(path, chomp: true)
+ keys.each do |key|
+ pub = Gitlab::SSHPublicKey.new(key)
+
+ type = pub.type.to_s
+ config[type] = [] unless config.key?(type)
+
+ next if config[type].include?(pub.fingerprint_sha256)
+
+ config[type].append(pub.fingerprint_sha256)
+ config_size += pub.fingerprint_sha256.size
+ end
+ end
+ rescue StandardError => e
+ logger.error "Exception: #{e.message}"
+ logger.debug e.backtrace
+ exit 1
+ end
+
+ logger.info "finish writing."
+ File.open(path, 'w') { |file| file.write(config.to_yaml) }
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/seed/runner_fleet.rake b/lib/tasks/gitlab/seed/runner_fleet.rake
new file mode 100644
index 00000000000..c0b79269c75
--- /dev/null
+++ b/lib/tasks/gitlab/seed/runner_fleet.rake
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# Seed database with:
+# 1. 2 root groups, one with 2 sub-groups and another with 1 sub-group
+# 1. 1 project in each of the sub-groups
+# 1. 1 instance runner, 1 shared project runner, and group/project runners in some groups/projects
+# 1. Successful and failed pipelines assigned to the first 5 available runners of each group/project
+# 1. 1 pipeline on one group runner with the remaining jobs
+#
+# @param username - user creating subgroups (i.e. GitLab admin)
+# @param registration_prefix - prefix used for the group, project, and runner names
+# @param runner_count - total number of runners to create (default: 40)
+# @param job_count - total number of jobs to create and assign to runners (default: 400)
+#
+# @example
+# bundle exec rake "gitlab:seed:runner_fleet[root, rf-]"
+#
+namespace :gitlab do
+ namespace :seed do
+ desc 'Seed groups with sub-groups/projects/runners/jobs for Runner Fleet testing'
+ task :runner_fleet,
+ [:username, :registration_prefix, :runner_count, :job_count] => :gitlab_environment do |_t, args|
+ timings = Benchmark.measure do
+ projects_to_runners = Gitlab::Seeders::Ci::Runner::RunnerFleetSeeder.new(
+ Gitlab::AppLogger,
+ username: args.username,
+ registration_prefix: args.registration_prefix,
+ runner_count: args.runner_count&.to_i
+ ).seed
+
+ Gitlab::Seeders::Ci::Runner::RunnerFleetPipelineSeeder.new(
+ projects_to_runners: projects_to_runners,
+ job_count: args.job_count&.to_i
+ ).seed
+ end
+
+ puts "Seed finished. Timings: #{timings}"
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake
index ec2ea623e02..b3559bde988 100644
--- a/lib/tasks/gitlab/tw/codeowners.rake
+++ b/lib/tasks/gitlab/tw/codeowners.rake
@@ -78,7 +78,7 @@ namespace :tw do
CodeOwnerRule.new('Tutorials', '@kpaizee'),
CodeOwnerRule.new('Utilization', '@fneill'),
CodeOwnerRule.new('Vulnerability Research', '@claytoncornell'),
- CodeOwnerRule.new('Workspace', '@lciutacu')
+ CodeOwnerRule.new('Organization', '@lciutacu')
].freeze
ERRORS_EXCLUDED_FILES = [
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-16 17:27:29 +0300