diff options
Diffstat (limited to 'spec/lib/gitlab/process_supervisor_spec.rb')
| -rw-r--r-- | spec/lib/gitlab/process_supervisor_spec.rb | 170 |
1 files changed, 170 insertions, 0 deletions
diff --git a/spec/lib/gitlab/process_supervisor_spec.rb b/spec/lib/gitlab/process_supervisor_spec.rb new file mode 100644 index 00000000000..60b127dadda --- /dev/null +++ b/spec/lib/gitlab/process_supervisor_spec.rb @@ -0,0 +1,170 @@ +# frozen_string_literal: true + +require_relative '../../../lib/gitlab/process_supervisor' + +RSpec.describe Gitlab::ProcessSupervisor do + let(:health_check_interval_seconds) { 0.1 } + let(:check_terminate_interval_seconds) { 1 } + let(:forwarded_signals) { [] } + let(:process_ids) { [spawn_process, spawn_process] } + + def spawn_process + Process.spawn('while true; do sleep 1; done').tap do |pid| + Process.detach(pid) + end + end + + subject(:supervisor) do + described_class.new( + health_check_interval_seconds: health_check_interval_seconds, + check_terminate_interval_seconds: check_terminate_interval_seconds, + terminate_timeout_seconds: 1 + check_terminate_interval_seconds, + forwarded_signals: forwarded_signals + ) + end + + after do + process_ids.each do |pid| + Process.kill('KILL', pid) + rescue Errno::ESRCH + # Ignore if a process wasn't actually alive. + end + end + + describe '#supervise' do + context 'while supervised processes are alive' do + it 'does not invoke callback' do + expect(Gitlab::ProcessManagement.all_alive?(process_ids)).to be(true) + pids_killed = [] + + supervisor.supervise(process_ids) do |dead_pids| + pids_killed = dead_pids + [] + end + + # Wait several times the poll frequency of the supervisor. + sleep health_check_interval_seconds * 10 + + expect(pids_killed).to be_empty + expect(Gitlab::ProcessManagement.all_alive?(process_ids)).to be(true) + end + end + + context 'when a supervised process dies' do + it 'triggers callback with the dead PIDs and adds new PIDs to supervised PIDs' do + expect(Gitlab::ProcessManagement.all_alive?(process_ids)).to be(true) + pids_killed = [] + + supervisor.supervise(process_ids) do |dead_pids| + pids_killed = dead_pids + [42] # Fake starting a new process in place of the terminated one. + end + + # Terminate the supervised process. + Process.kill('TERM', process_ids.first) + + await_condition(sleep_sec: health_check_interval_seconds) do + pids_killed == [process_ids.first] + end + + expect(Gitlab::ProcessManagement.process_alive?(process_ids.first)).to be(false) + expect(Gitlab::ProcessManagement.process_alive?(process_ids.last)).to be(true) + expect(supervisor.supervised_pids).to match_array([process_ids.last, 42]) + end + end + + context 'signal handling' do + before do + allow(supervisor).to receive(:sleep) + allow(Gitlab::ProcessManagement).to receive(:trap_signals) + allow(Gitlab::ProcessManagement).to receive(:all_alive?).and_return(false) + allow(Gitlab::ProcessManagement).to receive(:signal_processes).with(process_ids, anything) + end + + context 'termination signals' do + context 'when TERM results in timely shutdown of processes' do + it 'forwards them to observed processes without waiting for grace period to expire' do + allow(Gitlab::ProcessManagement).to receive(:any_alive?).and_return(false) + + expect(Gitlab::ProcessManagement).to receive(:trap_signals).ordered.with(%i(INT TERM)).and_yield(:TERM) + expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, :TERM) + expect(supervisor).not_to receive(:sleep).with(check_terminate_interval_seconds) + + supervisor.supervise(process_ids) { [] } + end + end + + context 'when TERM does not result in timely shutdown of processes' do + it 'issues a KILL signal after the grace period expires' do + expect(Gitlab::ProcessManagement).to receive(:trap_signals).with(%i(INT TERM)).and_yield(:TERM) + expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, :TERM) + expect(supervisor).to receive(:sleep).ordered.with(check_terminate_interval_seconds).at_least(:once) + expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, '-KILL') + + supervisor.supervise(process_ids) { [] } + end + end + end + + context 'forwarded signals' do + let(:forwarded_signals) { %i(USR1) } + + it 'forwards given signals to the observed processes' do + expect(Gitlab::ProcessManagement).to receive(:trap_signals).with(%i(USR1)).and_yield(:USR1) + expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, :USR1) + + supervisor.supervise(process_ids) { [] } + end + end + end + end + + describe '#shutdown' do + context 'when supervisor is supervising processes' do + before do + supervisor.supervise(process_ids) + end + + context 'when supervisor is alive' do + it 'signals TERM then KILL to all supervised processes' do + expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, :TERM) + expect(Gitlab::ProcessManagement).to receive(:signal_processes).ordered.with(process_ids, '-KILL') + + supervisor.shutdown + end + + it 'stops the supervisor' do + expect { supervisor.shutdown }.to change { supervisor.alive }.from(true).to(false) + end + end + + context 'when supervisor has already shut down' do + before do + supervisor.shutdown + end + + it 'does nothing' do + expect(supervisor.alive).to be(false) + expect(Gitlab::ProcessManagement).not_to receive(:signal_processes) + + supervisor.shutdown + end + end + end + + context 'when supervisor never started' do + it 'does nothing' do + expect(supervisor.alive).to be(false) + expect(Gitlab::ProcessManagement).not_to receive(:signal_processes) + + supervisor.shutdown + end + end + end + + def await_condition(timeout_sec: 5, sleep_sec: 0.1) + Timeout.timeout(timeout_sec) do + sleep sleep_sec until yield + end + end +end |