diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 17:34:42 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-05-20 17:34:42 +0300 |
commit | 9f46488805e86b1bc341ea1620b866016c2ce5ed (patch) | |
tree | f9748c7e287041e37d6da49e0a29c9511dc34768 /lib/tasks | |
parent | dfc92d081ea0332d69c8aca2f0e745cb48ae5e6d (diff) |
Add latest changes from gitlab-org/gitlab@13-0-stable-ee
Diffstat (limited to 'lib/tasks')
-rw-r--r-- | lib/tasks/file_hooks.rake | 5 | ||||
-rw-r--r-- | lib/tasks/gemojione.rake | 6 | ||||
-rw-r--r-- | lib/tasks/gitlab/gitaly.rake | 7 | ||||
-rw-r--r-- | lib/tasks/gitlab/import_export/export.rake | 6 | ||||
-rw-r--r-- | lib/tasks/gitlab/import_export/import.rake | 6 | ||||
-rw-r--r-- | lib/tasks/gitlab/shell.rake | 2 | ||||
-rw-r--r-- | lib/tasks/gitlab/snippets.rake | 91 | ||||
-rw-r--r-- | lib/tasks/gitlab/track_deployment.rake | 9 | ||||
-rw-r--r-- | lib/tasks/gitlab/workhorse.rake | 2 | ||||
-rw-r--r-- | lib/tasks/sidekiq.rake | 38 |
10 files changed, 110 insertions, 62 deletions
diff --git a/lib/tasks/file_hooks.rake b/lib/tasks/file_hooks.rake index 66d382db612..f767d63fe0d 100644 --- a/lib/tasks/file_hooks.rake +++ b/lib/tasks/file_hooks.rake @@ -4,6 +4,11 @@ namespace :file_hooks do puts 'Validating file hooks from /file_hooks and /plugins directories' Gitlab::FileHook.files.each do |file| + if File.dirname(file).ends_with?('plugins') + puts 'DEPRECATED: /plugins directory is deprecated and will be removed in 14.0. ' \ + 'Please move your files into /file_hooks directory.' + end + success, message = Gitlab::FileHook.execute(file, Gitlab::DataBuilder::Push::SAMPLE_DATA) if success diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 8cf7c9e89f0..3833689e07e 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -7,7 +7,7 @@ namespace :gemojione do aliases = {} index_file = File.join(Rails.root, 'fixtures', 'emojis', 'index.json') - index = JSON.parse(File.read(index_file)) + index = Gitlab::Json.parse(File.read(index_file)) index.each_pair do |key, data| data['aliases'].each do |a| @@ -19,7 +19,7 @@ namespace :gemojione do out = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') File.open(out, 'w') do |handle| - handle.write(JSON.pretty_generate(aliases, indent: ' ', space: '', space_before: '')) + handle.write(Gitlab::Json.pretty_generate(aliases, indent: ' ', space: '', space_before: '')) end end @@ -58,7 +58,7 @@ namespace :gemojione do out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') File.open(out, 'w') do |handle| - handle.write(JSON.pretty_generate(resultant_emoji_map)) + handle.write(Gitlab::Json.pretty_generate(resultant_emoji_map)) end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index ee47f71af93..fc55d9704d1 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -13,10 +13,9 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") version = Gitlab::GitalyClient.expected_server_version - checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) - - command = %w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE] + checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1]) + command = [] _, status = Gitlab::Popen.popen(%w[which gmake]) command << (status.zero? ? 'gmake' : 'make') @@ -31,7 +30,7 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") Dir.chdir(args.dir) do # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? - Bundler.with_original_env { run_command!(command) } + Bundler.with_original_env { Gitlab::Popen.popen(command, nil, { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil }) } end end end diff --git a/lib/tasks/gitlab/import_export/export.rake b/lib/tasks/gitlab/import_export/export.rake index c9c212fbe4d..4bdc62c9319 100644 --- a/lib/tasks/gitlab/import_export/export.rake +++ b/lib/tasks/gitlab/import_export/export.rake @@ -3,12 +3,12 @@ # Export project to archive # # @example -# bundle exec rake "gitlab:import_export:export[root, root, project_to_export, /path/to/file.tar.gz, true]" +# bundle exec rake "gitlab:import_export:export[root, root, project_to_export, /path/to/file.tar.gz]" # namespace :gitlab do namespace :import_export do desc 'GitLab | Import/Export | EXPERIMENTAL | Export large project archives' - task :export, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args| + task :export, [:username, :namespace_path, :project_path, :archive_path] => :gitlab_environment do |_t, args| # Load it here to avoid polluting Rake tasks with Sidekiq test warnings require 'sidekiq/testing' @@ -18,6 +18,7 @@ namespace :gitlab do warn_user_is_not_gitlab if ENV['EXPORT_DEBUG'].present? + Gitlab::Utils::Measuring.logger = logger ActiveRecord::Base.logger = logger logger.level = Logger::DEBUG else @@ -29,7 +30,6 @@ namespace :gitlab do project_path: args.project_path, username: args.username, file_path: args.archive_path, - measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled), logger: logger ) diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake index 7e2162a7774..2702b530334 100644 --- a/lib/tasks/gitlab/import_export/import.rake +++ b/lib/tasks/gitlab/import_export/import.rake @@ -7,12 +7,12 @@ # 2. Performs Sidekiq job synchronously # # @example -# bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz, true]" +# bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz]" # namespace :gitlab do namespace :import_export do desc 'GitLab | Import/Export | EXPERIMENTAL | Import large project archives' - task :import, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args| + task :import, [:username, :namespace_path, :project_path, :archive_path] => :gitlab_environment do |_t, args| # Load it here to avoid polluting Rake tasks with Sidekiq test warnings require 'sidekiq/testing' @@ -22,6 +22,7 @@ namespace :gitlab do warn_user_is_not_gitlab if ENV['IMPORT_DEBUG'].present? + Gitlab::Utils::Measuring.logger = logger ActiveRecord::Base.logger = logger logger.level = Logger::DEBUG else @@ -33,7 +34,6 @@ namespace :gitlab do project_path: args.project_path, username: args.username, file_path: args.archive_path, - measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled), logger: logger ) diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 6586699f8ba..d6e62a5c550 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -12,7 +12,7 @@ namespace :gitlab do gitlab_url += '/' unless gitlab_url.end_with?('/') target_dir = Gitlab.config.gitlab_shell.path - checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir) + checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir, clone_opts: %w[--depth 1]) # Make sure we're on the right tag Dir.chdir(target_dir) do diff --git a/lib/tasks/gitlab/snippets.rake b/lib/tasks/gitlab/snippets.rake new file mode 100644 index 00000000000..c391cecfdbc --- /dev/null +++ b/lib/tasks/gitlab/snippets.rake @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :snippets do + DEFAULT_LIMIT = 100 + + # @example + # bundle exec rake gitlab:snippets:migrate SNIPPET_IDS=1,2,3,4 + # bundle exec rake gitlab:snippets:migrate SNIPPET_IDS=1,2,3,4 LIMIT=50 + desc 'GitLab | Migrate specific snippets to git' + task :migrate, [:ids] => :environment do |_, args| + unless ENV['SNIPPET_IDS'].presence + raise "Please supply the list of ids through the SNIPPET_IDS env var" + end + + raise "Invalid limit value" if limit.zero? + + if migration_running? + raise "There are already snippet migrations running. Please wait until they are finished." + end + + ids = parse_snippet_ids! + + puts "Starting the migration..." + Gitlab::BackgroundMigration::BackfillSnippetRepositories.new.perform_by_ids(ids) + + list_non_migrated = non_migrated_snippets.where(id: ids) + + if list_non_migrated.exists? + puts "The following snippets couldn't be migrated:" + puts list_non_migrated.pluck(:id).join(',') + else + puts "All snippets were migrated successfully" + end + end + + def parse_snippet_ids! + ids = ENV['SNIPPET_IDS'].delete(' ').split(',').map do |id| + id.to_i.tap do |value| + raise "Invalid id provided" if value.zero? + end + end + + if ids.size > limit + raise "The number of ids provided is higher than #{limit}. You can update this limit by using the env var `LIMIT`" + end + + ids + end + + # @example + # bundle exec rake gitlab:snippets:migration_status + desc 'GitLab | Show whether there are snippet background migrations running' + task migration_status: :environment do + if migration_running? + puts "There are snippet migrations running" + else + puts "There are no snippet migrations running" + end + end + + def migration_running? + Sidekiq::ScheduledSet.new.any? { |r| r.klass == 'BackgroundMigrationWorker' && r.args[0] == 'BackfillSnippetRepositories' } + end + + # @example + # bundle exec rake gitlab:snippets:list_non_migrated + # bundle exec rake gitlab:snippets:list_non_migrated LIMIT=50 + desc 'GitLab | Show non migrated snippets' + task list_non_migrated: :environment do + raise "Invalid limit value" if limit.zero? + + non_migrated_count = non_migrated_snippets.count + if non_migrated_count.zero? + puts "All snippets have been successfully migrated" + else + puts "There are #{non_migrated_count} snippets that haven't been migrated. Showing a batch of ids of those snippets:\n" + puts non_migrated_snippets.limit(limit).pluck(:id).join(',') + end + end + + def non_migrated_snippets + @non_migrated_snippets ||= Snippet.select(:id).where.not(id: SnippetRepository.select(:snippet_id)) + end + + # There are problems with the specs if we memoize this value + def limit + ENV['LIMIT'] ? ENV['LIMIT'].to_i : DEFAULT_LIMIT + end + end +end diff --git a/lib/tasks/gitlab/track_deployment.rake b/lib/tasks/gitlab/track_deployment.rake deleted file mode 100644 index 6f101aea303..00000000000 --- a/lib/tasks/gitlab/track_deployment.rake +++ /dev/null @@ -1,9 +0,0 @@ -namespace :gitlab do - desc 'GitLab | Tracks a deployment in GitLab Performance Monitoring' - task track_deployment: :environment do - metric = Gitlab::Metrics::Metric - .new('deployments', version: Gitlab::VERSION) - - Gitlab::Metrics.submit_metrics([metric.to_hash]) - end -end diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index bae3e4e8001..53343c8f8ff 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -12,7 +12,7 @@ namespace :gitlab do version = Gitlab::Workhorse.version - checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1]) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' diff --git a/lib/tasks/sidekiq.rake b/lib/tasks/sidekiq.rake deleted file mode 100644 index d74878835fd..00000000000 --- a/lib/tasks/sidekiq.rake +++ /dev/null @@ -1,38 +0,0 @@ -namespace :sidekiq do - def deprecation_warning! - warn <<~WARNING - This task is deprecated and will be removed in 13.0 as it is thought to be unused. - - If you are using this task, please comment on the below issue: - https://gitlab.com/gitlab-org/gitlab/issues/196731 - WARNING - end - - desc '[DEPRECATED] GitLab | Sidekiq | Stop sidekiq' - task :stop do - deprecation_warning! - - system(*%w(bin/background_jobs stop)) - end - - desc '[DEPRECATED] GitLab | Sidekiq | Start sidekiq' - task :start do - deprecation_warning! - - system(*%w(bin/background_jobs start)) - end - - desc '[DEPRECATED] GitLab | Sidekiq | Restart sidekiq' - task :restart do - deprecation_warning! - - system(*%w(bin/background_jobs restart)) - end - - desc '[DEPRECATED] GitLab | Sidekiq | Start sidekiq with launchd on Mac OS X' - task :launchd do - deprecation_warning! - - system(*%w(bin/background_jobs start_silent)) - end -end |