Add latest changes from gitlab-org/gitlab@master

5 files changed, 42 insertions, 14 deletions

diff --git a/lib/gitlab/cleanup/orphan_lfs_file_references.rb b/lib/gitlab/cleanup/orphan_lfs_file_references.rb
index 5789fe4f92d..a9961cb8968 100644
--- a/lib/gitlab/cleanup/orphan_lfs_file_references.rb
+++ b/lib/gitlab/cleanup/orphan_lfs_file_references.rb
@@ -40,7 +40,7 @@ module Gitlab
       end
 
       def lfs_oids_from_repository
-        project.repository.gitaly_blob_client.get_all_lfs_pointers(nil).map(&:lfs_oid)
+        project.repository.gitaly_blob_client.get_all_lfs_pointers.map(&:lfs_oid)
       end
 
       def orphan_oids
diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb
index a0fab67e450..a8d1ea08275 100644
--- a/lib/gitlab/git/lfs_changes.rb
+++ b/lib/gitlab/git/lfs_changes.rb
@@ -13,7 +13,7 @@ module Gitlab
       end
 
       def all_pointers
-        @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev)
+        @repository.gitaly_blob_client.get_all_lfs_pointers
       end
     end
   end
diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb
index 5cde06bb6aa..8c704c2ceea 100644
--- a/lib/gitlab/gitaly_client/blob_service.rb
+++ b/lib/gitlab/gitaly_client/blob_service.rb
@@ -131,10 +131,9 @@ module Gitlab
         map_lfs_pointers(response)
       end
 
-      def get_all_lfs_pointers(revision)
-        request = Gitaly::GetNewLFSPointersRequest.new(
-          repository: @gitaly_repo,
-          revision: encode_binary(revision)
+      def get_all_lfs_pointers
+        request = Gitaly::GetAllLFSPointersRequest.new(
+          repository: @gitaly_repo
         )
 
         response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request, timeout: GitalyClient.medium_timeout)
diff --git a/lib/gitlab/sidekiq_cluster.rb b/lib/gitlab/sidekiq_cluster.rb
index 70df40fc35d..e74ae8d0f03 100644
--- a/lib/gitlab/sidekiq_cluster.rb
+++ b/lib/gitlab/sidekiq_cluster.rb
@@ -62,21 +62,28 @@ module Gitlab
     # directory - The directory of the Rails application.
     #
     # Returns an Array containing the PIDs of the started processes.
-    def self.start(queues, env: :development, directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, dryrun: false)
+    def self.start(queues, env: :development, directory: Dir.pwd, max_concurrency: 50, min_concurrency: 0, timeout: CLI::DEFAULT_SOFT_TIMEOUT_SECONDS, dryrun: false)
       queues.map.with_index do |pair, index|
-        start_sidekiq(pair, env: env, directory: directory, max_concurrency: max_concurrency, min_concurrency: min_concurrency, worker_id: index, dryrun: dryrun)
+        start_sidekiq(pair, env: env,
+                            directory: directory,
+                            max_concurrency: max_concurrency,
+                            min_concurrency: min_concurrency,
+                            worker_id: index,
+                            timeout: timeout,
+                            dryrun: dryrun)
       end
     end
 
     # Starts a Sidekiq process that processes _only_ the given queues.
     #
     # Returns the PID of the started process.
-    def self.start_sidekiq(queues, env:, directory:, max_concurrency:, min_concurrency:, worker_id:, dryrun:)
+    def self.start_sidekiq(queues, env:, directory:, max_concurrency:, min_concurrency:, worker_id:, timeout:, dryrun:)
       counts = count_by_queue(queues)
 
       cmd = %w[bundle exec sidekiq]
       cmd << "-c#{self.concurrency(queues, min_concurrency, max_concurrency)}"
       cmd << "-e#{env}"
+      cmd << "-t#{timeout}"
       cmd << "-gqueues:#{proc_details(counts)}"
       cmd << "-r#{directory}"
 
diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb
index 245d918e382..f1befe4aff1 100644
--- a/lib/gitlab/sidekiq_cluster/cli.rb
+++ b/lib/gitlab/sidekiq_cluster/cli.rb
@@ -8,9 +8,17 @@ module Gitlab
   module SidekiqCluster
     class CLI
       CHECK_TERMINATE_INTERVAL_SECONDS = 1
-      # How long to wait in total when asking for a clean termination
-      # Sidekiq default to self-terminate is 25s
-      TERMINATE_TIMEOUT_SECONDS = 30
+
+      # How long to wait when asking for a clean termination.
+      # It maps the Sidekiq default timeout:
+      # https://github.com/mperham/sidekiq/wiki/Signals#term
+      #
+      # This value is passed to Sidekiq's `-t` if none
+      # is given through arguments.
+      DEFAULT_SOFT_TIMEOUT_SECONDS = 25
+
+      # After surpassing the soft timeout.
+      DEFAULT_HARD_TIMEOUT_SECONDS = 5
 
       CommandError = Class.new(StandardError)
 
@@ -74,7 +82,8 @@ module Gitlab
           directory: @rails_path,
           max_concurrency: @max_concurrency,
           min_concurrency: @min_concurrency,
-          dryrun: @dryrun
+          dryrun: @dryrun,
+          timeout: soft_timeout_seconds
         )
 
         return if @dryrun
@@ -88,6 +97,15 @@ module Gitlab
         SidekiqCluster.write_pid(@pid) if @pid
       end
 
+      def soft_timeout_seconds
+        @soft_timeout_seconds || DEFAULT_SOFT_TIMEOUT_SECONDS
+      end
+
+      # The amount of time it'll wait for killing the alive Sidekiq processes.
+      def hard_timeout_seconds
+        soft_timeout_seconds + DEFAULT_HARD_TIMEOUT_SECONDS
+      end
+
       def monotonic_time
         Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second)
       end
@@ -101,7 +119,7 @@ module Gitlab
       end
 
       def wait_for_termination
-        deadline = monotonic_time + TERMINATE_TIMEOUT_SECONDS
+        deadline = monotonic_time + hard_timeout_seconds
         sleep(CHECK_TERMINATE_INTERVAL_SECONDS) while continue_waiting?(deadline)
 
         hard_stop_stuck_pids
@@ -176,6 +194,10 @@ module Gitlab
             @interval = int.to_i
           end
 
+          opt.on('-t', '--timeout INT', 'Graceful timeout for all running processes') do |timeout|
+            @soft_timeout_seconds = timeout.to_i
+          end
+
           opt.on('-d', '--dryrun', 'Print commands that would be run without this flag, and quit') do |int|
             @dryrun = true
           end