diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-25 12:08:11 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-25 12:08:11 +0300 |
commit | 5064bf8c5647d4c4430cbb4d097cf1592416de29 (patch) | |
tree | d051bf2abe2cc7061b3a7facb6669a56ccb9cf54 /lib/gitlab | |
parent | 9c83aadd2604e7e6cb1f84683f951e6b12872618 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib/gitlab')
-rw-r--r-- | lib/gitlab/cleanup/orphan_lfs_file_references.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/git/lfs_changes.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/gitaly_client/blob_service.rb | 7 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_cluster.rb | 13 | ||||
-rw-r--r-- | lib/gitlab/sidekiq_cluster/cli.rb | 32 |
5 files changed, 42 insertions, 14 deletions
diff --git a/lib/gitlab/cleanup/orphan_lfs_file_references.rb b/lib/gitlab/cleanup/orphan_lfs_file_references.rb index 5789fe4f92d..a9961cb8968 100644 --- a/lib/gitlab/cleanup/orphan_lfs_file_references.rb +++ b/lib/gitlab/cleanup/orphan_lfs_file_references.rb @@ -40,7 +40,7 @@ module Gitlab end def lfs_oids_from_repository - project.repository.gitaly_blob_client.get_all_lfs_pointers(nil).map(&:lfs_oid) + project.repository.gitaly_blob_client.get_all_lfs_pointers.map(&:lfs_oid) end def orphan_oids diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index a0fab67e450..a8d1ea08275 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -13,7 +13,7 @@ module Gitlab end def all_pointers - @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) + @repository.gitaly_blob_client.get_all_lfs_pointers end end end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 5cde06bb6aa..8c704c2ceea 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -131,10 +131,9 @@ module Gitlab map_lfs_pointers(response) end - def get_all_lfs_pointers(revision) - request = Gitaly::GetNewLFSPointersRequest.new( - repository: @gitaly_repo, - revision: encode_binary(revision) + def get_all_lfs_pointers + request = Gitaly::GetAllLFSPointersRequest.new( + repository: @gitaly_repo ) response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/sidekiq_cluster.rb b/lib/gitlab/sidekiq_cluster.rb index 70df40fc35d..e74ae8d0f03 100644 --- a/lib/gitlab/sidekiq_cluster.rb +++ b/lib/gitlab/sidekiq_cluster.rb @@ -62,21 +62,28 @@ module Gitlab # directory - The directory of the Rails application. # # Returns an Array containing the PIDs of the started processes. - def self.start(queues, env: :development, directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, dryrun: false) + def self.start(queues, env: :development, directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, timeout: CLI::DEFAULT_SOFT_TIMEOUT_SECONDS, dryrun: false) queues.map.with_index do |pair, index| - start_sidekiq(pair, env: env, directory: directory, max_concurrency: max_concurrency, min_concurrency: min_concurrency, worker_id: index, dryrun: dryrun) + start_sidekiq(pair, env: env, + directory: directory, + max_concurrency: max_concurrency, + min_concurrency: min_concurrency, + worker_id: index, + timeout: timeout, + dryrun: dryrun) end end # Starts a Sidekiq process that processes _only_ the given queues. # # Returns the PID of the started process. - def self.start_sidekiq(queues, env:, directory:, max_concurrency:, min_concurrency:, worker_id:, dryrun:) + def self.start_sidekiq(queues, env:, directory:, max_concurrency:, min_concurrency:, worker_id:, timeout:, dryrun:) counts = count_by_queue(queues) cmd = %w[bundle exec sidekiq] cmd << "-c#{self.concurrency(queues, min_concurrency, max_concurrency)}" cmd << "-e#{env}" + cmd << "-t#{timeout}" cmd << "-gqueues:#{proc_details(counts)}" cmd << "-r#{directory}" diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb index 245d918e382..f1befe4aff1 100644 --- a/lib/gitlab/sidekiq_cluster/cli.rb +++ b/lib/gitlab/sidekiq_cluster/cli.rb @@ -8,9 +8,17 @@ module Gitlab module SidekiqCluster class CLI CHECK_TERMINATE_INTERVAL_SECONDS = 1 - # How long to wait in total when asking for a clean termination - # Sidekiq default to self-terminate is 25s - TERMINATE_TIMEOUT_SECONDS = 30 + + # How long to wait when asking for a clean termination. + # It maps the Sidekiq default timeout: + # https://github.com/mperham/sidekiq/wiki/Signals#term + # + # This value is passed to Sidekiq's `-t` if none + # is given through arguments. + DEFAULT_SOFT_TIMEOUT_SECONDS = 25 + + # After surpassing the soft timeout. + DEFAULT_HARD_TIMEOUT_SECONDS = 5 CommandError = Class.new(StandardError) @@ -74,7 +82,8 @@ module Gitlab directory: @rails_path, max_concurrency: @max_concurrency, min_concurrency: @min_concurrency, - dryrun: @dryrun + dryrun: @dryrun, + timeout: soft_timeout_seconds ) return if @dryrun @@ -88,6 +97,15 @@ module Gitlab SidekiqCluster.write_pid(@pid) if @pid end + def soft_timeout_seconds + @soft_timeout_seconds || DEFAULT_SOFT_TIMEOUT_SECONDS + end + + # The amount of time it'll wait for killing the alive Sidekiq processes. + def hard_timeout_seconds + soft_timeout_seconds + DEFAULT_HARD_TIMEOUT_SECONDS + end + def monotonic_time Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) end @@ -101,7 +119,7 @@ module Gitlab end def wait_for_termination - deadline = monotonic_time + TERMINATE_TIMEOUT_SECONDS + deadline = monotonic_time + hard_timeout_seconds sleep(CHECK_TERMINATE_INTERVAL_SECONDS) while continue_waiting?(deadline) hard_stop_stuck_pids @@ -176,6 +194,10 @@ module Gitlab @interval = int.to_i end + opt.on('-t', '--timeout INT', 'Graceful timeout for all running processes') do |timeout| + @soft_timeout_seconds = timeout.to_i + end + opt.on('-d', '--dryrun', 'Print commands that would be run without this flag, and quit') do |int| @dryrun = true end |