diff options
Diffstat (limited to 'lib/tasks')
| -rw-r--r-- | lib/tasks/gems.rake | 14 | ||||
| -rw-r--r-- | lib/tasks/gitlab/audit_event_types/audit_event_types.rake | 28 | ||||
| -rw-r--r-- | lib/tasks/gitlab/audit_event_types/check_docs_task.rb | 34 | ||||
| -rw-r--r-- | lib/tasks/gitlab/audit_event_types/compile_docs_task.rb | 22 | ||||
| -rw-r--r-- | lib/tasks/gitlab/db/cells/bump_cell_sequences.rake | 25 | ||||
| -rw-r--r-- | lib/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences.rake | 49 | ||||
| -rw-r--r-- | lib/tasks/gitlab/db/migration_squash.rake | 28 | ||||
| -rw-r--r-- | lib/tasks/gitlab/info.rake | 13 | ||||
| -rw-r--r-- | lib/tasks/gitlab/tw/codeowners.rake | 22 | ||||
| -rw-r--r-- | lib/tasks/gitlab/user_management.rake | 15 |
10 files changed, 187 insertions, 63 deletions
diff --git a/lib/tasks/gems.rake b/lib/tasks/gems.rake index fc70048ea6d..0c4cbbfe3f8 100644 --- a/lib/tasks/gems.rake +++ b/lib/tasks/gems.rake @@ -29,7 +29,7 @@ namespace :gems do end def root_directory - File.expand_path('../../vendor/gems', __dir__) + File.expand_path('../../gems', __dir__) end def generate_gem(vendor_gem_dir:, api_url:, gem_name:, module_name:, docker_image:) @@ -53,14 +53,18 @@ namespace :gems do write_file(gem_dir / 'LICENSE', license) write_file(gem_dir / "#{gem_name}.gemspec") do |content| replace_string(content, 'Unlicense', 'MIT') + replace_string(content, /.*add_development_dependency 'rspec'.*/, '') replace_string(content, /(\.files\s*=).*/, '\1 Dir.glob("lib/**/*")') replace_string(content, /(\.test_files\s*=).*/, '\1 []') end + # This is gem is supposed to be generated. No developer should change code. remove_entry_secure(gem_dir / 'Gemfile') + # The generated code doesn't align well with `gitlab-styles` configuration. remove_entry_secure(gem_dir / '.rubocop.yml') remove_entry_secure(gem_dir / '.travis.yml') remove_entry_secure(gem_dir / 'git_push.sh') + # The RSpec examples are stubs and have no value. remove_entry_secure(gem_dir / 'spec') remove_entry_secure(gem_dir / '.rspec') end @@ -78,14 +82,16 @@ namespace :gems do end def readme_banner(task) - # rubocop:disable Rails/TimeZone <<~BANNER - # Generated by `rake #{task.name}` on #{Time.now.strftime('%Y-%m-%d')} + # #{generated_by(task)} See https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/rake_tasks.md#update-openapi-client-for-error-tracking-feature BANNER - # rubocop:enable Rails/TimeZone + end + + def generated_by(task) + "Generated by `rake #{task.name}` on #{Time.now.strftime('%Y-%m-%d')}" # rubocop:disable Rails/TimeZone end def license diff --git a/lib/tasks/gitlab/audit_event_types/audit_event_types.rake b/lib/tasks/gitlab/audit_event_types/audit_event_types.rake new file mode 100644 index 00000000000..289f79568a9 --- /dev/null +++ b/lib/tasks/gitlab/audit_event_types/audit_event_types.rake @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +namespace :gitlab do + namespace :audit_event_types do + event_types_dir = Rails.root.join("doc/administration/audit_event_streaming") + event_types_doc_file = Rails.root.join(event_types_dir, 'audit_event_types.md') + template_directory = 'tooling/audit_events/docs/templates/' + template_erb_file_path = Rails.root.join(template_directory, 'audit_event_types.md.erb') + + desc 'GitLab | Audit event types | Generate audit event types docs' + task compile_docs: :environment do + require_relative './compile_docs_task' + + Tasks::Gitlab::AuditEventTypes::CompileDocsTask + .new(event_types_dir, event_types_doc_file, template_erb_file_path).run + end + + desc 'GitLab | Audit event types | Check if Audit event types docs are up to date' + task check_docs: :environment do + require_relative './check_docs_task' + + Tasks::Gitlab::AuditEventTypes::CheckDocsTask + .new(event_types_dir, event_types_doc_file, template_erb_file_path).run + end + end +end diff --git a/lib/tasks/gitlab/audit_event_types/check_docs_task.rb b/lib/tasks/gitlab/audit_event_types/check_docs_task.rb new file mode 100644 index 00000000000..f62dd116ed1 --- /dev/null +++ b/lib/tasks/gitlab/audit_event_types/check_docs_task.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Tasks + module Gitlab + module AuditEventTypes + class CheckDocsTask + def initialize(docs_dir, docs_path, template_erb_path) + @event_types_dir = docs_dir + @audit_event_types_doc_file = docs_path + @event_type_erb_template = ERB.new(File.read(template_erb_path), trim_mode: '<>') + end + + def run + doc = File.read(@audit_event_types_doc_file) + + if doc == @event_type_erb_template.result + puts "Audit event types documentation is up to date." + else + error_message = "Audit event types documentation is outdated! Please update it by running " \ + "`bundle exec rake gitlab:audit_event_types:compile_docs`." + heading = '#' * 10 + puts heading + puts '#' + puts "# #{error_message}" + puts '#' + puts heading + + abort + end + end + end + end + end +end diff --git a/lib/tasks/gitlab/audit_event_types/compile_docs_task.rb b/lib/tasks/gitlab/audit_event_types/compile_docs_task.rb new file mode 100644 index 00000000000..ffa4f6d3514 --- /dev/null +++ b/lib/tasks/gitlab/audit_event_types/compile_docs_task.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Tasks + module Gitlab + module AuditEventTypes + class CompileDocsTask + def initialize(docs_dir, docs_path, template_erb_path) + @event_types_dir = docs_dir + @audit_event_types_doc_file = docs_path + @event_type_erb_template = ERB.new(File.read(template_erb_path), trim_mode: '<>') + end + + def run + FileUtils.mkdir_p(@event_types_dir) + File.write(@audit_event_types_doc_file, @event_type_erb_template.result) + + puts "Documentation compiled." + end + end + end + end +end diff --git a/lib/tasks/gitlab/db/cells/bump_cell_sequences.rake b/lib/tasks/gitlab/db/cells/bump_cell_sequences.rake new file mode 100644 index 00000000000..04d91c96ebe --- /dev/null +++ b/lib/tasks/gitlab/db/cells/bump_cell_sequences.rake @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :db do + namespace :cells do + desc 'Bump sequences for cell-local tables on the cells database' + task :bump_cell_sequences, [:increase_by] => :environment do |_t, args| + # We do not want to run this on production environment, even accidentally. + unless Gitlab.dev_or_test_env? + puts 'This rake task cannot be run in production environment'.color(:red) + exit 1 + end + + increase_by = args.increase_by.to_i + if increase_by < 1 + puts 'Please specify a positive integer `increase_by` value'.color(:red) + puts 'Example: rake gitlab:db:cells:bump_cell_sequences[100000]'.color(:green) + exit 1 + end + + Gitlab::Database::BumpSequences.new(:gitlab_main_cell, increase_by).execute + end + end + end +end diff --git a/lib/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences.rake b/lib/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences.rake index 4d78acb3011..fac4c68b0a6 100644 --- a/lib/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences.rake +++ b/lib/tasks/gitlab/db/decomposition/rollback/bump_ci_sequences.rake @@ -4,8 +4,6 @@ namespace :gitlab do namespace :db do namespace :decomposition do namespace :rollback do - SEQUENCE_NAME_MATCHER = /nextval\('([a-z_]+)'::regclass\)/.freeze - desc 'Bump all the CI tables sequences on the Main Database' task :bump_ci_sequences, [:increase_by] => :environment do |_t, args| increase_by = args.increase_by.to_i @@ -15,54 +13,9 @@ namespace :gitlab do exit 1 end - sequences_by_gitlab_schema(ApplicationRecord, :gitlab_ci).each do |sequence_name| - increment_sequence_by(ApplicationRecord.connection, sequence_name, increase_by) - end + Gitlab::Database::BumpSequences.new(:gitlab_ci, increase_by).execute end end end end end - -# base_model is to choose which connection to use to query the tables -# gitlab_schema, can be 'gitlab_main', 'gitlab_ci', 'gitlab_shared' -def sequences_by_gitlab_schema(base_model, gitlab_schema) - tables = Gitlab::Database::GitlabSchema.tables_to_schema.select do |_table_name, schema_name| - schema_name == gitlab_schema - end.keys - - models = tables.map do |table| - model = Class.new(base_model) - model.table_name = table - model - end - - sequences = [] - models.each do |model| - model.columns.each do |column| - match_result = column.default_function&.match(SEQUENCE_NAME_MATCHER) - next unless match_result - - sequences << match_result[1] - end - end - - sequences -end - -# This method is going to increase the sequence next_value by: -# - increment_by + 1 if the sequence has the attribute is_called = True (which is the common case) -# - increment_by if the sequence has the attribute is_called = False (for example, a newly created sequence) -# It uses ALTER SEQUENCE as a safety mechanism to avoid that no concurrent insertions -# will cause conflicts on the sequence. -# This is because ALTER SEQUENCE blocks concurrent nextval, currval, lastval, and setval calls. -def increment_sequence_by(connection, sequence_name, increment_by) - connection.transaction do - # The first call is to make sure that the sequence's is_called value is set to `true` - # This guarantees that the next call to `nextval` will increase the sequence by `increment_by` - connection.select_value("SELECT nextval($1)", nil, [sequence_name]) - connection.execute("ALTER SEQUENCE #{sequence_name} INCREMENT BY #{increment_by}") - connection.select_value("select nextval($1)", nil, [sequence_name]) - connection.execute("ALTER SEQUENCE #{sequence_name} INCREMENT BY 1") - end -end diff --git a/lib/tasks/gitlab/db/migration_squash.rake b/lib/tasks/gitlab/db/migration_squash.rake new file mode 100644 index 00000000000..7ddd6c41d12 --- /dev/null +++ b/lib/tasks/gitlab/db/migration_squash.rake @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :db do + desc "GitLab | DB | squash | squash as of a version" + task :squash, [:version] => :environment do |_t, args| + require 'git' + git = ::Git.open(Dir.pwd) + + squasher = Gitlab::Database::Migrations::Squasher.new( + `git ls-tree --name-only -r #{args[:version]} -- db/migrate db/post_migrate` + ) + + new_init_structure_sql = git.show(args[:version], 'db/structure.sql') + # Delete relevant migrations and specs + squasher.files_to_delete.each do |filename| + git.remove filename + puts "\tDeleting #{filename} from repo".red + rescue Git::GitExecuteError + puts "#{filename} is not in the current branch" + end + puts "\tOverwriting init_structure.sql..." + File.write('db/init_structure.sql', new_init_structure_sql) + git.add('db/init_structure.sql') + puts "\tDone!".white + end + end +end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 4f7053b7629..26ffe2c3f7b 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -90,6 +90,19 @@ namespace :gitlab do puts "- #{name}: \t#{repository_storage.gitaly_address}" end puts "GitLab Shell path:\t\t#{Gitlab.config.gitlab_shell.path}" + + # check Gitaly version + puts "" + puts "Gitaly".color(:yellow) + Gitlab.config.repositories.storages.each do |storage_name, storage| + gitaly_server_service = Gitlab::GitalyClient::ServerService.new(storage_name) + gitaly_server_info = gitaly_server_service.info + puts "- #{storage_name} Address: \t#{storage.gitaly_address}" + puts "- #{storage_name} Version: \t#{gitaly_server_info.server_version}" + puts "- #{storage_name} Git Version: \t#{gitaly_server_info.git_version}" + rescue GRPC::DeadlineExceeded + puts "Unable to reach storage #{storage_name}".color(red) + end end end end diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index afe2c564247..cea66125fd0 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -21,7 +21,8 @@ namespace :tw do CODE_OWNER_RULES = [ # CodeOwnerRule.new('Activation', ''), # CodeOwnerRule.new('Acquisition', ''), - # CodeOwnerRule.new('AI Assisted', ''), + CodeOwnerRule.new('AI Framework', '@sselhorn'), + CodeOwnerRule.new('AI Model Validation', '@sselhorn'), CodeOwnerRule.new('Analytics Instrumentation', '@lciutacu'), CodeOwnerRule.new('Anti-Abuse', '@phillipwells'), CodeOwnerRule.new('Application Performance', '@jglassman1'), @@ -34,13 +35,14 @@ namespace :tw do CodeOwnerRule.new('Container Registry', '@marcel.amirault'), CodeOwnerRule.new('Contributor Experience', '@eread'), CodeOwnerRule.new('Database', '@aqualls'), - # CodeOwnerRule.new('DataOps', ''), + CodeOwnerRule.new('DataOps', '@sselhorn'), # CodeOwnerRule.new('Delivery', ''), CodeOwnerRule.new('Development', '@sselhorn'), CodeOwnerRule.new('Distribution', '@axil'), CodeOwnerRule.new('Distribution (Charts)', '@axil'), CodeOwnerRule.new('Distribution (Omnibus)', '@eread'), CodeOwnerRule.new('Documentation Guidelines', '@sselhorn'), + CodeOwnerRule.new('Duo Chat', '@sselhorn'), CodeOwnerRule.new('Dynamic Analysis', '@rdickenson'), CodeOwnerRule.new('IDE', '@ashrafkhamis'), CodeOwnerRule.new('Foundations', '@sselhorn'), @@ -53,7 +55,7 @@ namespace :tw do CodeOwnerRule.new('Import and Integrate', '@eread @ashrafkhamis'), CodeOwnerRule.new('Infrastructure', '@sselhorn'), # CodeOwnerRule.new('Knowledge', ''), - # CodeOwnerRule.new('MLOps', '') + CodeOwnerRule.new('MLOps', '@sselhorn'), # CodeOwnerRule.new('Observability', ''), CodeOwnerRule.new('Optimize', '@lciutacu'), CodeOwnerRule.new('Organization', '@lciutacu'), @@ -71,7 +73,7 @@ namespace :tw do CodeOwnerRule.new('Runner', '@fneill'), CodeOwnerRule.new('Runner SaaS', '@fneill'), CodeOwnerRule.new('Security Policies', '@rdickenson'), - CodeOwnerRule.new('Source Code', '@aqualls @msedlakjakubowski'), + CodeOwnerRule.new('Source Code', ->(path) { path.start_with?('/doc/user') ? '@aqualls' : '@msedlakjakubowski' }), CodeOwnerRule.new('Static Analysis', '@rdickenson'), CodeOwnerRule.new('Style Guide', '@sselhorn'), CodeOwnerRule.new('Tenant Scale', '@lciutacu'), @@ -100,8 +102,14 @@ namespace :tw do end end - def self.writer_for_group(category) - CODE_OWNER_RULES.find { |rule| rule.category == category }&.writer + def self.writer_for_group(category, path) + writer = CODE_OWNER_RULES.find { |rule| rule.category == category }&.writer + + if writer.is_a?(String) || writer.nil? + writer + else + writer.call(path) + end end errors = [] @@ -118,7 +126,7 @@ namespace :tw do next end - writer = writer_for_group(document.group) + writer = writer_for_group(document.group, relative_file) next unless writer mappings << DocumentOwnerMapping.new(relative_file, writer) if document.has_a_valid_group? diff --git a/lib/tasks/gitlab/user_management.rake b/lib/tasks/gitlab/user_management.rake index 29f2360f64a..dbadc7a2f7a 100644 --- a/lib/tasks/gitlab/user_management.rake +++ b/lib/tasks/gitlab/user_management.rake @@ -5,11 +5,18 @@ namespace :gitlab do desc "GitLab | User management | Update all users of a group with personal project limit to 0 and can_create_group to false" task :disable_project_and_group_creation, [:group_id] => :environment do |t, args| group = Group.find(args.group_id) + user_ids = Member.from_union([ + group.hierarchy_members_with_inactive.select(:user_id), + group.descendant_project_members_with_inactive.select(:user_id) + ], remove_duplicates: false).distinct.pluck(:user_id) - result = User.where(id: group.direct_and_indirect_users_with_inactive.select(:id)).update_all(projects_limit: 0, can_create_group: false) - ids_count = group.direct_and_indirect_users_with_inactive.count - puts "Done".color(:green) if result == ids_count - puts "Something went wrong".color(:red) if result != ids_count + result = User.where(id: user_ids).update_all(projects_limit: 0, can_create_group: false) + + if result == user_ids.count + puts "Done".color(:green) + else + puts "Something went wrong".color(:red) + end end end end |