diff options
author | Grzegorz Bizon <grzegorz@gitlab.com> | 2018-09-05 09:51:19 +0300 |
---|---|---|
committer | Grzegorz Bizon <grzegorz@gitlab.com> | 2018-09-05 09:51:19 +0300 |
commit | c2eff16eca38ea92fb059cc43686fa32e4b40da9 (patch) | |
tree | 5bedd20d1d2b8c28e47977c684744a06fe3dd7bd | |
parent | 91003c6ebb764c9cf3fe01a14d6213b26c09dcc8 (diff) | |
parent | 144b017d77c341849d37927b765c75888569d530 (diff) |
Merge branch '50963-qa-orchestrated-tests-don-t-run-due-to-wrong-tag-filters-qa' into 'master'
Resolve "[QA] Orchestrated tests don't run due to wrong tag filters"
Closes #49654 and #50963
See merge request gitlab-org/gitlab-ce!21468
-rw-r--r-- | qa/qa/scenario/template.rb | 8 | ||||
-rw-r--r-- | qa/qa/scenario/test/instance.rb | 7 | ||||
-rw-r--r-- | qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb | 2 | ||||
-rw-r--r-- | qa/qa/specs/runner.rb | 4 | ||||
-rw-r--r-- | qa/spec/git/repository_spec.rb | 2 | ||||
-rw-r--r-- | qa/spec/scenario/test/instance/all_spec.rb | 44 | ||||
-rw-r--r-- | qa/spec/scenario/test/instance/smoke_spec.rb | 44 | ||||
-rw-r--r-- | qa/spec/scenario/test/integration/github_spec.rb | 21 | ||||
-rw-r--r-- | qa/spec/scenario/test/integration/kubernetes_spec.rb | 9 | ||||
-rw-r--r-- | qa/spec/scenario/test/integration/ldap_spec.rb | 9 | ||||
-rw-r--r-- | qa/spec/scenario/test/integration/mattermost_spec.rb | 18 | ||||
-rw-r--r-- | qa/spec/scenario/test/integration/object_storage_spec.rb | 9 | ||||
-rw-r--r-- | qa/spec/specs/runner_spec.rb | 49 | ||||
-rw-r--r-- | qa/spec/support/shared_examples/scenario_shared_examples.rb | 49 |
14 files changed, 174 insertions, 101 deletions
diff --git a/qa/qa/scenario/template.rb b/qa/qa/scenario/template.rb index a87d925ce32..cb1a1de6b9a 100644 --- a/qa/qa/scenario/template.rb +++ b/qa/qa/scenario/template.rb @@ -28,12 +28,8 @@ module QA Specs::Runner.perform do |specs| specs.tty = true - specs.options = - if rspec_options.any? - rspec_options - else - ['--tag', self.class.focus.join(','), '--', ::File.expand_path('../specs/features', __dir__)] - end + specs.tags = self.class.focus + specs.options = rspec_options if rspec_options.any? end end end diff --git a/qa/qa/scenario/test/instance.rb b/qa/qa/scenario/test/instance.rb index a2d503cc015..b4098619e4e 100644 --- a/qa/qa/scenario/test/instance.rb +++ b/qa/qa/scenario/test/instance.rb @@ -27,12 +27,7 @@ module QA Specs::Runner.perform do |specs| specs.tty = true - specs.options = - if rspec_options.any? - rspec_options - else - ['--', ::File.expand_path('../../specs/features', __dir__)] - end + specs.options = rspec_options if rspec_options.any? end end end diff --git a/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb b/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb index dd24e8ffba5..e558049756d 100644 --- a/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb +++ b/qa/qa/specs/features/browser_ui/7_configure/auto_devops/create_project_with_auto_devops_spec.rb @@ -30,7 +30,7 @@ module QA push.project = project push.directory = Pathname .new(__dir__) - .join('../../../fixtures/auto_devops_rack') + .join('../../../../../fixtures/auto_devops_rack') push.commit_message = 'Create Auto DevOps compatible rack application' end diff --git a/qa/qa/specs/runner.rb b/qa/qa/specs/runner.rb index ccb9d5591de..5b5699d8a93 100644 --- a/qa/qa/specs/runner.rb +++ b/qa/qa/specs/runner.rb @@ -16,9 +16,9 @@ module QA args.push('--tty') if tty if tags.any? - tags.each { |tag| args.push(['-t', tag.to_s]) } + tags.each { |tag| args.push(['--tag', tag.to_s]) } else - args.push(%w[-t ~orchestrated]) + args.push(%w[--tag ~orchestrated]) end args.push(options) diff --git a/qa/spec/git/repository_spec.rb b/qa/spec/git/repository_spec.rb index 5c65128d10c..53bff3bf0b3 100644 --- a/qa/spec/git/repository_spec.rb +++ b/qa/spec/git/repository_spec.rb @@ -29,7 +29,7 @@ describe QA::Git::Repository do def cd_empty_temp_directory tmp_dir = 'tmp/git-repository-spec/' - FileUtils.rm_r(tmp_dir) if ::File.exist?(tmp_dir) + FileUtils.rm_rf(tmp_dir) if ::File.exist?(tmp_dir) FileUtils.mkdir_p tmp_dir FileUtils.cd tmp_dir end diff --git a/qa/spec/scenario/test/instance/all_spec.rb b/qa/spec/scenario/test/instance/all_spec.rb index 1d96352550b..9311d1d8199 100644 --- a/qa/spec/scenario/test/instance/all_spec.rb +++ b/qa/spec/scenario/test/instance/all_spec.rb @@ -1,45 +1,3 @@ describe QA::Scenario::Test::Instance::All do - subject do - Class.new(described_class) do - tags :rspec, :foo - end - end - - context '#perform' do - let(:arguments) { spy('Runtime::Scenario') } - let(:release) { spy('Runtime::Release') } - let(:runner) { spy('Specs::Runner') } - - before do - stub_const('QA::Runtime::Release', release) - stub_const('QA::Runtime::Scenario', arguments) - stub_const('QA::Specs::Runner', runner) - - allow(runner).to receive(:perform).and_yield(runner) - end - - it 'sets an address of the subject' do - subject.perform("hello") - - expect(arguments).to have_received(:define) - .with(:gitlab_address, "hello") - end - - context 'no paths' do - it 'calls runner with default arguments' do - subject.perform("test") - - expect(runner).to have_received(:options=) - .with(['--tag', 'rspec,foo', '--', ::File.expand_path('../../../../qa/specs/features', __dir__)]) - end - end - - context 'specifying paths' do - it 'calls runner with paths' do - subject.perform('test', 'path1', 'path2') - - expect(runner).to have_received(:options=).with(%w[path1 path2]) - end - end - end + it_behaves_like 'a QA scenario class' end diff --git a/qa/spec/scenario/test/instance/smoke_spec.rb b/qa/spec/scenario/test/instance/smoke_spec.rb index 386eefae930..b5db9783af3 100644 --- a/qa/spec/scenario/test/instance/smoke_spec.rb +++ b/qa/spec/scenario/test/instance/smoke_spec.rb @@ -1,45 +1,5 @@ describe QA::Scenario::Test::Instance::Smoke do - subject { Class.new(described_class) { tags :smoke } } - - context '#perform' do - let(:arguments) { spy('Runtime::Scenario') } - let(:release) { spy('Runtime::Release') } - let(:runner) { spy('Specs::Runner') } - - before do - stub_const('QA::Runtime::Release', release) - stub_const('QA::Runtime::Scenario', arguments) - stub_const('QA::Specs::Runner', runner) - - allow(runner).to receive(:perform).and_yield(runner) - end - - it 'sets an address of the subject' do - subject.perform("hello") - - expect(arguments).to have_received(:define) - .with(:gitlab_address, "hello") - end - - it 'has a smoke tag' do - expect(subject.focus).to eq([:smoke]) # rubocop:disable Focus - end - - context 'no paths' do - it 'calls runner with default arguments' do - subject.perform("test") - - expect(runner).to have_received(:options=) - .with(['--tag', 'smoke', '--', ::File.expand_path('../../../../qa/specs/features', __dir__)]) - end - end - - context 'specifying paths' do - it 'calls runner with paths' do - subject.perform('test', 'path1', 'path2') - - expect(runner).to have_received(:options=).with(%w[path1 path2]) - end - end + it_behaves_like 'a QA scenario class' do + let(:tags) { [:smoke] } end end diff --git a/qa/spec/scenario/test/integration/github_spec.rb b/qa/spec/scenario/test/integration/github_spec.rb new file mode 100644 index 00000000000..c2aeb1ded1d --- /dev/null +++ b/qa/spec/scenario/test/integration/github_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +describe QA::Scenario::Test::Integration::Github do + context '#perform' do + let(:env) { spy('Runtime::Env') } + + before do + stub_const('QA::Runtime::Env', env) + end + + it_behaves_like 'a QA scenario class' do + let(:tags) { [:github] } + + it 'requires a GitHub access token' do + subject.perform('gitlab_address') + + expect(env).to have_received(:require_github_access_token!) + end + end + end +end diff --git a/qa/spec/scenario/test/integration/kubernetes_spec.rb b/qa/spec/scenario/test/integration/kubernetes_spec.rb new file mode 100644 index 00000000000..cb43994b229 --- /dev/null +++ b/qa/spec/scenario/test/integration/kubernetes_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +describe QA::Scenario::Test::Integration::Kubernetes do + context '#perform' do + it_behaves_like 'a QA scenario class' do + let(:tags) { [:kubernetes] } + end + end +end diff --git a/qa/spec/scenario/test/integration/ldap_spec.rb b/qa/spec/scenario/test/integration/ldap_spec.rb new file mode 100644 index 00000000000..198856aec3f --- /dev/null +++ b/qa/spec/scenario/test/integration/ldap_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +describe QA::Scenario::Test::Integration::LDAP do + context '#perform' do + it_behaves_like 'a QA scenario class' do + let(:tags) { [:ldap] } + end + end +end diff --git a/qa/spec/scenario/test/integration/mattermost_spec.rb b/qa/spec/scenario/test/integration/mattermost_spec.rb new file mode 100644 index 00000000000..59caf2ba2cd --- /dev/null +++ b/qa/spec/scenario/test/integration/mattermost_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +describe QA::Scenario::Test::Integration::Mattermost do + context '#perform' do + it_behaves_like 'a QA scenario class' do + let(:args) { %w[gitlab_address mattermost_address] } + let(:tags) { [:mattermost] } + let(:options) { ['path1']} + + it 'requires a GitHub access token' do + subject.perform(*args) + + expect(attributes).to have_received(:define) + .with(:mattermost_address, 'mattermost_address') + end + end + end +end diff --git a/qa/spec/scenario/test/integration/object_storage_spec.rb b/qa/spec/scenario/test/integration/object_storage_spec.rb new file mode 100644 index 00000000000..2b7188223e0 --- /dev/null +++ b/qa/spec/scenario/test/integration/object_storage_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +describe QA::Scenario::Test::Integration::ObjectStorage do + context '#perform' do + it_behaves_like 'a QA scenario class' do + let(:tags) { [:object_storage] } + end + end +end diff --git a/qa/spec/specs/runner_spec.rb b/qa/spec/specs/runner_spec.rb new file mode 100644 index 00000000000..b237b954889 --- /dev/null +++ b/qa/spec/specs/runner_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +describe QA::Specs::Runner do + context '#perform' do + before do + allow(QA::Runtime::Browser).to receive(:configure!) + end + + it 'excludes the orchestrated tag by default' do + expect(RSpec::Core::Runner).to receive(:run) + .with(['--tag', '~orchestrated', File.expand_path('../../qa/specs/features', __dir__)], $stderr, $stdout) + .and_return(0) + + subject.perform + end + + context 'when tty is set' do + subject do + described_class.new.tap do |runner| + runner.tty = true + end + end + + it 'sets the `--tty` flag' do + expect(RSpec::Core::Runner).to receive(:run) + .with(['--tty', '--tag', '~orchestrated', File.expand_path('../../qa/specs/features', __dir__)], $stderr, $stdout) + .and_return(0) + + subject.perform + end + end + + context 'when tags are set' do + subject do + described_class.new.tap do |runner| + runner.tags = %i[orchestrated github] + end + end + + it 'focuses on the given tags' do + expect(RSpec::Core::Runner).to receive(:run) + .with(['--tag', 'orchestrated', '--tag', 'github', File.expand_path('../../qa/specs/features', __dir__)], $stderr, $stdout) + .and_return(0) + + subject.perform + end + end + end +end diff --git a/qa/spec/support/shared_examples/scenario_shared_examples.rb b/qa/spec/support/shared_examples/scenario_shared_examples.rb new file mode 100644 index 00000000000..5fd55d7d96b --- /dev/null +++ b/qa/spec/support/shared_examples/scenario_shared_examples.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +shared_examples 'a QA scenario class' do + let(:attributes) { spy('Runtime::Scenario') } + let(:release) { spy('Runtime::Release') } + let(:runner) { spy('Specs::Runner') } + + let(:args) { ['gitlab_address'] } + let(:tags) { [] } + let(:options) { %w[path1 path2] } + + before do + stub_const('QA::Runtime::Release', release) + stub_const('QA::Runtime::Scenario', attributes) + stub_const('QA::Specs::Runner', runner) + + allow(runner).to receive(:perform).and_yield(runner) + end + + it 'responds to perform' do + expect(subject).to respond_to(:perform) + end + + it 'sets an address of the subject' do + subject.perform(*args) + + expect(attributes).to have_received(:define).with(:gitlab_address, 'gitlab_address') + end + + it 'performs before hooks' do + subject.perform(*args) + + expect(release).to have_received(:perform_before_hooks) + end + + it 'sets tags on runner' do + subject.perform(*args) + + expect(runner).to have_received(:tags=).with(tags) + end + + context 'specifying RSpec options' do + it 'sets options on runner' do + subject.perform(*args, *options) + + expect(runner).to have_received(:options=).with(options) + end + end +end |