diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-11 09:09:46 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-11 09:09:46 +0300 |
commit | 55733b19c526145cceb120e8bb874d476a84383a (patch) | |
tree | dcde3cfb905516cd1f07ab364a94aff5fddff391 /lib/tasks | |
parent | ea99abb145ed193c2ac5d19efbff3b8990a54c9c (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib/tasks')
-rw-r--r-- | lib/tasks/gitlab/import_export/import.rake | 63 |
1 files changed, 59 insertions, 4 deletions
diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake index 8fe61df605a..c832cba0287 100644 --- a/lib/tasks/gitlab/import_export/import.rake +++ b/lib/tasks/gitlab/import_export/import.rake @@ -7,12 +7,12 @@ # 2. Performs Sidekiq job synchronously # # @example -# bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz]" +# bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz, true]" # namespace :gitlab do namespace :import_export do desc 'GitLab | Import/Export | EXPERIMENTAL | Import large project archives' - task :import, [:username, :namespace_path, :project_path, :archive_path] => :gitlab_environment do |_t, args| + task :import, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args| # Load it here to avoid polluting Rake tasks with Sidekiq test warnings require 'sidekiq/testing' @@ -26,7 +26,8 @@ namespace :gitlab do namespace_path: args.namespace_path, project_path: args.project_path, username: args.username, - file_path: args.archive_path + file_path: args.archive_path, + measurement_enabled: args.measurement_enabled == 'true' ).import end end @@ -38,6 +39,7 @@ class GitlabProjectImport @file_path = opts.fetch(:file_path) @namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path)) @current_user = User.find_by_username(opts.fetch(:username)) + @measurement_enabled = opts.fetch(:measurement_enabled) end def import @@ -72,6 +74,54 @@ class GitlabProjectImport RequestStore.clear! end + def with_count_queries(&block) + count = 0 + + counter_f = ->(name, started, finished, unique_id, payload) { + unless payload[:name].in? %w[CACHE SCHEMA] + count += 1 + end + } + + ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block) + + puts "Number of sql calls: #{count}" + end + + def with_gc_counter + gc_counts_before = GC.stat.select { |k, v| k =~ /count/ } + yield + gc_counts_after = GC.stat.select { |k, v| k =~ /count/ } + stats = gc_counts_before.merge(gc_counts_after) { |k, vb, va| va - vb } + puts "Total GC count: #{stats[:count]}" + puts "Minor GC count: #{stats[:minor_gc_count]}" + puts "Major GC count: #{stats[:major_gc_count]}" + end + + def with_measure_time + timing = Benchmark.realtime do + yield + end + + time = Time.at(timing).utc.strftime("%H:%M:%S") + puts "Time to finish: #{time}" + end + + def with_measuring + puts "Measuring enabled..." + with_gc_counter do + with_count_queries do + with_measure_time do + yield + end + end + end + end + + def measurement_enabled? + @measurement_enabled != false + end + # We want to ensure that all Sidekiq jobs are executed # synchronously as part of that process. # This ensures that all expensive operations do not escape @@ -79,8 +129,13 @@ class GitlabProjectImport def with_isolated_sidekiq_job Sidekiq::Testing.fake! do with_request_store do + # If you are attempting to import a large project into a development environment, + # you may see Gitaly throw an error about too many calls or invocations. + # This is due to a n+1 calls limit being set for development setups (not enforced in production) + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635 + # For development setups, this code-path will be excluded from n+1 detection. ::Gitlab::GitalyClient.allow_n_plus_1_calls do - yield + measurement_enabled? ? with_measuring { yield } : yield end end |