From 89a7928235ce83f570bf45ca0ee227f05d461a61 Mon Sep 17 00:00:00 2001 From: Mark Lapierre Date: Mon, 24 Dec 2018 17:20:03 -0500 Subject: Allow tests to be quarantined Exclude quarantined tests by default. --- qa/qa/specs/runner.rb | 7 ++++ qa/spec/specs/runner_spec.rb | 91 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 84 insertions(+), 14 deletions(-) (limited to 'qa') diff --git a/qa/qa/specs/runner.rb b/qa/qa/specs/runner.rb index 1bd8101c36d..ebd2ed92df9 100644 --- a/qa/qa/specs/runner.rb +++ b/qa/qa/specs/runner.rb @@ -23,6 +23,9 @@ module QA args.push(%w[--tag ~orchestrated]) unless (%w[-t --tag] & options).any? end + # Skip quarantined tests by default + args.push(%w[--tag ~quarantine]) unless include_quarantine? + args.push(%w[--tag ~skip_signup_disabled]) if QA::Runtime::Env.signup_disabled? QA::Runtime::Env.supported_features.each_key do |key| @@ -38,6 +41,10 @@ module QA abort if status.nonzero? end end + + def include_quarantine? + tags.include?(:quarantine) || options.include?('quarantine') + end end end end diff --git a/qa/spec/specs/runner_spec.rb b/qa/spec/specs/runner_spec.rb index 741821ddf8c..8e2b852510a 100644 --- a/qa/spec/specs/runner_spec.rb +++ b/qa/spec/specs/runner_spec.rb @@ -6,8 +6,12 @@ describe QA::Specs::Runner do allow(QA::Runtime::Browser).to receive(:configure!) end - it 'excludes the orchestrated tag by default' do - expect_rspec_runner_arguments(['--tag', '~orchestrated', *described_class::DEFAULT_TEST_PATH_ARGS]) + it 'excludes the orchestrated and quarantine tags by default' do + expect_rspec_runner_arguments([ + '--tag', '~orchestrated', + '--tag', '~quarantine', + *described_class::DEFAULT_TEST_PATH_ARGS + ]) subject.perform end @@ -16,7 +20,12 @@ describe QA::Specs::Runner do subject { described_class.new.tap { |runner| runner.tty = true } } it 'sets the `--tty` flag' do - expect_rspec_runner_arguments(['--tty', '--tag', '~orchestrated', *described_class::DEFAULT_TEST_PATH_ARGS]) + expect_rspec_runner_arguments([ + '--tty', + '--tag', '~orchestrated', + '--tag', '~quarantine', + *described_class::DEFAULT_TEST_PATH_ARGS + ]) subject.perform end @@ -25,18 +34,54 @@ describe QA::Specs::Runner do context 'when tags are set' do subject { described_class.new.tap { |runner| runner.tags = %i[orchestrated github] } } - it 'focuses on the given tags' do - expect_rspec_runner_arguments(['--tag', 'orchestrated', '--tag', 'github', *described_class::DEFAULT_TEST_PATH_ARGS]) + it 'focuses on the given tags, plus the ~quarantine tag' do + expect_rspec_runner_arguments([ + '--tag', 'orchestrated', + '--tag', 'github', + '--tag', '~quarantine', + *described_class::DEFAULT_TEST_PATH_ARGS + ]) subject.perform end end - context 'when "--tag smoke" is set as options' do + context 'when only "--tag quarantine" is set as options' do + subject { described_class.new.tap { |runner| runner.options = %w[--tag quarantine] } } + + it 'focuses on the given tags without the default args' do + expect_rspec_runner_arguments([ + '--tag', 'quarantine', + *described_class::DEFAULT_TEST_PATH_ARGS + ]) + + subject.perform + end + end + + context 'when only "--tag smoke" is set as options' do subject { described_class.new.tap { |runner| runner.options = %w[--tag smoke] } } - it 'focuses on the given tag without excluded the orchestrated tag' do - expect_rspec_runner_arguments(['--tag', 'smoke', *described_class::DEFAULT_TEST_PATH_ARGS]) + it 'focuses on the given tag without the default args, but with the ~quarantine tag' do + expect_rspec_runner_arguments([ + '--tag', '~quarantine', + '--tag', 'smoke', + *described_class::DEFAULT_TEST_PATH_ARGS + ]) + + subject.perform + end + end + + context 'when "--tag smoke" and "--tag quarantine" are set as options' do + subject { described_class.new.tap { |runner| runner.options = %w[--tag smoke --tag quarantine] } } + + it 'focuses on the given tags without the default args' do + expect_rspec_runner_arguments([ + '--tag', 'smoke', + '--tag', 'quarantine', + *described_class::DEFAULT_TEST_PATH_ARGS + ]) subject.perform end @@ -45,8 +90,12 @@ describe QA::Specs::Runner do context 'when "qa/specs/features/foo" is set as options' do subject { described_class.new.tap { |runner| runner.options = %w[qa/specs/features/foo] } } - it 'passes the given tests path and excludes the orchestrated tag' do - expect_rspec_runner_arguments(['--tag', '~orchestrated', 'qa/specs/features/foo']) + it 'passes the given tests path and excludes the default args' do + expect_rspec_runner_arguments([ + '--tag', '~orchestrated', + '--tag', '~quarantine', + 'qa/specs/features/foo' + ]) subject.perform end @@ -55,8 +104,12 @@ describe QA::Specs::Runner do context 'when "-- qa/specs/features/foo" is set as options' do subject { described_class.new.tap { |runner| runner.options = %w[-- qa/specs/features/foo] } } - it 'passes the given tests path and excludes the orchestrated tag' do - expect_rspec_runner_arguments(['--tag', '~orchestrated', '--', 'qa/specs/features/foo']) + it 'passes the given tests path and excludes the default args' do + expect_rspec_runner_arguments([ + '--tag', '~orchestrated', + '--tag', '~quarantine', + '--', 'qa/specs/features/foo' + ]) subject.perform end @@ -70,7 +123,12 @@ describe QA::Specs::Runner do subject { described_class.new } it 'it includes default args and excludes the skip_signup_disabled tag' do - expect_rspec_runner_arguments(['--tag', '~orchestrated', '--tag', '~skip_signup_disabled', *described_class::DEFAULT_TEST_PATH_ARGS]) + expect_rspec_runner_arguments([ + '--tag', '~orchestrated', + '--tag', '~quarantine', + '--tag', '~skip_signup_disabled', + *described_class::DEFAULT_TEST_PATH_ARGS + ]) subject.perform end @@ -84,7 +142,12 @@ describe QA::Specs::Runner do subject { described_class.new } it 'it includes default args and excludes the requires_git_protocol_v2 tag' do - expect_rspec_runner_arguments(['--tag', '~orchestrated', '--tag', '~requires_git_protocol_v2', *described_class::DEFAULT_TEST_PATH_ARGS]) + expect_rspec_runner_arguments([ + '--tag', '~orchestrated', + '--tag', '~quarantine', + '--tag', '~requires_git_protocol_v2', + *described_class::DEFAULT_TEST_PATH_ARGS + ]) subject.perform end -- cgit v1.2.3