diff options
author | Robert Speicher <rspeicher@gmail.com> | 2019-01-15 22:35:59 +0300 |
---|---|---|
committer | Robert Speicher <rspeicher@gmail.com> | 2019-01-15 22:35:59 +0300 |
commit | 75b1837da79b5244a8da6bde1e391343fef59431 (patch) | |
tree | 4bb5585cb6cc7df21ff5e37fcce9de42036135b0 | |
parent | 3499da77d31e122e9cdaa70990a5251274b61d84 (diff) | |
parent | d71eb8fc1ad4e722868cd4b9ea54044f17bc46e2 (diff) |
Merge branch 'allow-to-quarantine-tests' into 'master'
Allow to quarantine tests
See merge request gitlab-org/gitlab-ce!24369
-rw-r--r-- | .gitlab-ci.yml | 16 | ||||
-rw-r--r-- | doc/development/testing_guide/flaky_tests.md | 12 | ||||
-rw-r--r-- | spec/features/boards/boards_spec.rb | 6 | ||||
-rw-r--r-- | spec/spec_helper.rb | 7 |
4 files changed, 37 insertions, 4 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6cbc8e21c76..d736e0fa5c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -499,6 +499,22 @@ rspec-mysql: <<: *rspec-metadata-mysql parallel: 50 +.rspec-quarantine: &rspec-quarantine + script: + - export CACHE_CLASSES=true + - scripts/gitaly-test-spawn + - bin/rspec --color --format documentation --tag quarantine spec/ + +rspec-pg-quarantine: + <<: *rspec-metadata-pg + <<: *rspec-quarantine + allow_failure: true + +rspec-mysql-quarantine: + <<: *rspec-metadata-mysql + <<: *rspec-quarantine + allow_failure: true + static-analysis: <<: *dedicated-no-docs-no-db-pull-cache-job dependencies: diff --git a/doc/development/testing_guide/flaky_tests.md b/doc/development/testing_guide/flaky_tests.md index bbb2313ea7b..f6bc9fb0979 100644 --- a/doc/development/testing_guide/flaky_tests.md +++ b/doc/development/testing_guide/flaky_tests.md @@ -5,6 +5,18 @@ It's a test that sometimes fails, but if you retry it enough times, it passes, eventually. +## Quarantined tests + +Tests can be put in quarantine by assigning `:quarantine` metadata. This means +they will be skipped unless run with `--tag quarantine`. This can be used for +tests that are expected to fail while a fix is in progress (similar to how +[`skip` or `pending`](https://relishapp.com/rspec/rspec-core/v/3-8/docs/pending-and-skipped-examples) + can be used). + +``` +bin/rspec --tag quarantine +``` + ## Automatic retries and flaky tests detection On our CI, we use [rspec-retry] to automatically retry a failing example a few diff --git a/spec/features/boards/boards_spec.rb b/spec/features/boards/boards_spec.rb index baa2b1d8af5..08c27354bd2 100644 --- a/spec/features/boards/boards_spec.rb +++ b/spec/features/boards/boards_spec.rb @@ -97,7 +97,7 @@ describe 'Issue Boards', :js do expect(find('.board:nth-child(4)')).to have_selector('.board-card') end - it 'shows description tooltip on list title' do + it 'shows description tooltip on list title', :quarantine do page.within('.board:nth-child(2)') do expect(find('.board-title span.has-tooltip')[:title]).to eq('Test') end @@ -411,7 +411,7 @@ describe 'Issue Boards', :js do wait_for_empty_boards((2..4)) end - it 'filters by label with space after reload' do + it 'filters by label with space after reload', :quarantine do set_filter("label", "\"#{accepting.title}") click_filter_link(accepting.title) submit_filter @@ -477,7 +477,7 @@ describe 'Issue Boards', :js do end end - it 'filters by multiple labels' do + it 'filters by multiple labels', :quarantine do set_filter("label", testing.title) click_filter_link(testing.title) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index ca977effcb6..72684caad32 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -22,7 +22,7 @@ if rspec_profiling_is_configured && (!ENV.key?('CI') || branch_can_be_profiled) require 'rspec_profiling/rspec' end -if ENV['CI'] && !ENV['NO_KNAPSACK'] +if ENV['CI'] && ENV['KNAPSACK_GENERATE_REPORT'] && !ENV['NO_KNAPSACK'] require 'knapsack' Knapsack::Adapters::RSpecAdapter.bind end @@ -132,6 +132,11 @@ RSpec.configure do |config| Gitlab::ReleaseBlogPost.instance.instance_variable_set(:@url, 'https://about.gitlab.com') end + config.before(:example, :quarantine) do + # Skip tests in quarantine unless we explicitly focus on them. + skip('In quarantine') unless config.inclusion_filter[:quarantine] + end + config.before(:example, :request_store) do RequestStore.begin! end |