diff options
author | Mark Lapierre <mlapierre@gitlab.com> | 2019-02-14 07:35:20 +0300 |
---|---|---|
committer | Sanad Liaquat <sliaquat@gitlab.com> | 2019-02-14 07:35:20 +0300 |
commit | 893142020207518faf2a4fbb0eaed8a8117adfd9 (patch) | |
tree | 9277b937f443c3beb53bb8773ec32069554a9766 /qa/spec/spec_helper.rb | |
parent | c3c43be39bbd65123f8a58778aa8937dff005d9f (diff) |
Skip contexts in quarantine
This avoids running before/after blocks for tests that are in quarantine
Diffstat (limited to 'qa/spec/spec_helper.rb')
-rw-r--r-- | qa/spec/spec_helper.rb | 67 |
1 files changed, 40 insertions, 27 deletions
diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index 3e7a6dd26ee..f1b4203d422 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -19,26 +19,16 @@ RSpec.configure do |config| end end - config.before do |example| - QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug? - - # If quarantine is tagged, skip tests that have other metadata unless - # they're also tagged. This lets us run quarantined tests in a particular - # category without running tests in other categories. - # E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged - # 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests - # using `--tag quarantine --tag smoke`, without this check we'd end up - # running that ldap test as well. - if config.inclusion_filter[:quarantine] - skip("Running tests tagged with all of #{config.inclusion_filter.rules.keys}") unless quarantine_and_optional_other_tag?(example, config) + config.before(:context) do + if self.class.metadata.keys.include?(:quarantine) + skip_or_run_quarantined_tests(self.class.metadata.keys, config.inclusion_filter.rules.keys) end end - config.before(:each, :quarantine) do |example| - # Skip tests in quarantine unless we explicitly focus on them - # We could use an exclusion filter, but this way the test report will list - # the quarantined tests when they're not run so that we're aware of them - skip('In quarantine') unless config.inclusion_filter[:quarantine] + config.before do |example| + QA::Runtime::Logger.debug("Starting test: #{example.full_description}") if QA::Runtime::Env.debug? + + skip_or_run_quarantined_tests(example.metadata.keys, config.inclusion_filter.rules.keys) end config.expect_with :rspec do |expectations| @@ -57,18 +47,41 @@ RSpec.configure do |config| Kernel.srand config.seed end +# Skip tests in quarantine unless we explicitly focus on them. +# Skip the entire context if a context is tagged. This avoids running before +# blocks unnecessarily. +# If quarantine is focussed, skip tests/contexts that have other metadata +# unless they're also focussed. This lets us run quarantined tests in a +# particular category without running tests in other categories. +# E.g., if a test is tagged 'smoke' and 'quarantine', and another is tagged +# 'ldap' and 'quarantine', if we wanted to run just quarantined smoke tests +# using `--tag quarantine --tag smoke`, without this check we'd end up +# running that ldap test as well. +# We could use an exclusion filter, but this way the test report will list +# the quarantined tests when they're not run so that we're aware of them +def skip_or_run_quarantined_tests(metadata_keys, filter_keys) + included_filters = filters_other_than_quarantine(filter_keys) + + if filter_keys.include?(:quarantine) + skip("Only running tests tagged with :quarantine and any of #{included_filters}") unless quarantine_and_optional_other_tag?(metadata_keys, included_filters) + else + skip('In quarantine') if metadata_keys.include?(:quarantine) + end +end + +def filters_other_than_quarantine(filter_keys) + filter_keys.reject { |key| key == :quarantine } +end + # Checks if a test has the 'quarantine' tag and other tags in the inclusion filter. # # Returns true if -# - the example metadata includes the quarantine tag -# - and the metadata and inclusion filter both have any other tag -# - or no other tags are in the inclusion filter -def quarantine_and_optional_other_tag?(example, config) - return false unless example.metadata.keys.include? :quarantine - - filters_other_than_quarantine = config.inclusion_filter.rules.keys.reject { |key| key == :quarantine } - - return true if filters_other_than_quarantine.empty? +# - the metadata includes the quarantine tag +# - and the metadata and inclusion filter both have any other tag +# - or no other tags are in the inclusion filter +def quarantine_and_optional_other_tag?(metadata_keys, included_filters) + return false unless metadata_keys.include? :quarantine + return true if included_filters.empty? - filters_other_than_quarantine.any? { |key| example.metadata.keys.include? key } + included_filters.any? { |key| metadata_keys.include? key } end |