diff options
Diffstat (limited to 'qa/spec/specs')
| -rw-r--r-- | qa/spec/specs/allure_report_spec.rb | 13 | ||||
| -rw-r--r-- | qa/spec/specs/helpers/feature_flag_spec.rb | 164 | ||||
| -rw-r--r-- | qa/spec/specs/runner_spec.rb | 35 | ||||
| -rw-r--r-- | qa/spec/specs/scenario_shared_examples.rb | 76 | ||||
| -rw-r--r-- | qa/spec/specs/spec_helper.rb | 5 |
5 files changed, 289 insertions, 4 deletions
diff --git a/qa/spec/specs/allure_report_spec.rb b/qa/spec/specs/allure_report_spec.rb index 86ceaf51cbb..85befb2f602 100644 --- a/qa/spec/specs/allure_report_spec.rb +++ b/qa/spec/specs/allure_report_spec.rb @@ -3,7 +3,7 @@ describe QA::Runtime::AllureReport do include QA::Support::Helpers::StubEnv - let(:rspec_config) { double('RSpec::Core::Configuration', 'add_formatter': nil, append_after: nil) } + let(:rspec_config) { instance_double('RSpec::Core::Configuration', 'add_formatter': nil, append_after: nil) } let(:png_path) { 'png_path' } let(:html_path) { 'html_path' } @@ -42,11 +42,14 @@ describe QA::Runtime::AllureReport do context 'with report generation enabled' do let(:generate_report) { 'true' } + let(:session) { instance_double('Capybara::Session') } + let(:attributes) { class_spy('Runtime::Scenario') } + let(:version_response) { instance_double('HTTPResponse', code: 200, body: versions.to_json) } + let(:png_file) { 'png-file' } let(:html_file) { 'html-file' } let(:ci_job) { 'ee:relative 5' } let(:versions) { { version: '14', revision: '6ced31db947' } } - let(:session) { double('session') } let(:browser_log) { ['log message 1', 'log message 2'] } before do @@ -54,11 +57,13 @@ describe QA::Runtime::AllureReport do stub_env('CI_JOB_NAME', ci_job) stub_env('GITLAB_QA_ADMIN_ACCESS_TOKEN', 'token') + stub_const('QA::Runtime::Scenario', attributes) + allow(Allure).to receive(:add_attachment) allow(File).to receive(:open).with(png_path) { png_file } allow(File).to receive(:open).with(html_path) { html_file } - allow(RestClient::Request).to receive(:execute) { double('response', code: 200, body: versions.to_json) } - allow(QA::Runtime::Scenario).to receive(:method_missing).with(:gitlab_address).and_return('gitlab.com') + allow(RestClient::Request).to receive(:execute) { version_response } + allow(attributes).to receive(:gitlab_address).and_return("https://gitlab.com") allow(Capybara).to receive(:current_session).and_return(session) allow(session).to receive_message_chain('driver.browser.logs.get').and_return(browser_log) diff --git a/qa/spec/specs/helpers/feature_flag_spec.rb b/qa/spec/specs/helpers/feature_flag_spec.rb new file mode 100644 index 00000000000..a1300ecf073 --- /dev/null +++ b/qa/spec/specs/helpers/feature_flag_spec.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +require 'rspec/core/sandbox' + +RSpec.describe QA::Specs::Helpers::FeatureFlag do + include QA::Support::Helpers::StubEnv + include QA::Specs::Helpers::RSpec + + around do |ex| + RSpec::Core::Sandbox.sandboxed do |config| + config.add_formatter QA::Support::Formatters::ContextFormatter + config.add_formatter QA::Support::Formatters::QuarantineFormatter + config.add_formatter QA::Support::Formatters::FeatureFlagFormatter + + # If there is an example-within-an-example, we want to make sure the inner example + # does not get a reference to the outer example (the real spec) if it calls + # something like `pending` + config.before(:context) { RSpec.current_example = nil } + + config.color_mode = :off + + ex.run + end + end + + describe '.skip_or_run_feature_flag_tests_or_contexts' do + shared_examples 'runs with given feature flag metadata' do |metadata| + it do + group = describe_successfully 'Feature flag test', feature_flag: metadata do + it('passes') {} + end + + expect(group.examples.first.execution_result.status).to eq(:passed) + end + end + + shared_examples 'skips with given feature flag metadata' do |metadata| + it do + group = describe_successfully 'Feature flag test', feature_flag: metadata do + it('is skipped') {} + end + + expect(group.examples.first.execution_result.status).to eq(:pending) + end + end + + context 'when run on staging' do + before(:context) do + QA::Runtime::Scenario.define(:gitlab_address, 'https://staging.gitlab.com') + end + + context 'when no scope is defined' do + it_behaves_like 'runs with given feature flag metadata', { name: 'no_scope_ff' } + + it 'is skipped if quarantine tag is also applied' do + group = describe_successfully( + 'Feature flag with no scope', + feature_flag: { name: 'quarantine_with_ff' }, + quarantine: { + issue: 'https://gitlab.com/test-group/test/-/issues/123', + type: 'bug' + } + ) do + it('is skipped') {} + end + + expect(group.examples.first.execution_result.status).to eq(:pending) + end + end + + it_behaves_like 'runs with given feature flag metadata', { name: 'actor_ff', scope: :project } + + it_behaves_like 'skips with given feature flag metadata', { name: 'global_ff', scope: :global } + + context 'when should be skipped in a specific job' do + before do + stub_env('CI_JOB_NAME', 'job-to-skip') + end + + it 'is skipped for that job' do + group = describe_successfully( + 'Test should be skipped', + feature_flag: { name: 'skip_job_ff' }, + except: { job: 'job-to-skip' } + ) do + it('does not run on staging in specified job') {} + end + + expect(group.examples.first.execution_result.status).to eq(:pending) + end + end + + context 'when should only run in a specific job' do + before do + stub_env('CI_JOB_NAME', 'job-to-run') + end + + it 'is run for that job' do + group = describe_successfully( + 'Test should run', + feature_flag: { name: 'run_job_ff' }, + only: { job: 'job-to-run' } + ) do + it('runs on staging in specified job') {} + end + + expect(group.examples.first.execution_result.status).to eq(:passed) + end + + it 'skips if test is set to only run in a job different from current CI job' do + group = describe_successfully( + 'Test should be skipped', + feature_flag: { name: 'skip_job_ff' }, + only: { job: 'other-job' } + ) do + it('does not run on staging in specified job') {} + end + + expect(group.examples.first.execution_result.status).to eq(:pending) + end + end + end + + context 'when run on production' do + before(:context) do + QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com') + end + + context 'when no scope is defined' do + it_behaves_like 'skips with given feature flag metadata', { name: 'no_scope_ff' } + + context 'for only one test in the example group' do + it 'only skips specified test and runs all others' do + group = describe_successfully 'Feature flag set for one test' do + it('is skipped', feature_flag: { name: 'single_test_ff' }) {} + it('passes') {} + end + + expect(group.examples[0].execution_result.status).to eq(:pending) + expect(group.examples[1].execution_result.status).to eq(:passed) + end + end + end + + it_behaves_like 'skips with given feature flag metadata', { name: 'actor_ff', scope: :project } + + it_behaves_like 'skips with given feature flag metadata', { name: 'global_ff', scope: :global } + end + + # The nightly package job, for example, does not run against a live environment with + # a defined gitlab_address. In this case, feature_flag tag logic can be safely ignored + context 'when run without a gitlab address specified' do + before(:context) do + QA::Runtime::Scenario.define(:gitlab_address, nil) + end + + it_behaves_like 'runs with given feature flag metadata', { name: 'no_scope_ff' } + + it_behaves_like 'runs with given feature flag metadata', { name: 'actor_ff', scope: :project } + + it_behaves_like 'runs with given feature flag metadata', { name: 'global_ff', scope: :global } + end + end +end diff --git a/qa/spec/specs/runner_spec.rb b/qa/spec/specs/runner_spec.rb index e52ca1fb17c..d5e442acfe7 100644 --- a/qa/spec/specs/runner_spec.rb +++ b/qa/spec/specs/runner_spec.rb @@ -86,6 +86,41 @@ RSpec.describe QA::Specs::Runner do end end + context 'when test_metadata_only is set as an option' do + let(:rspec_config) { instance_double('RSpec::Core::Configuration') } + let(:output_file) { Pathname.new('/root/tmp/test-metadata.json') } + + before do + QA::Runtime::Scenario.define(:test_metadata_only, true) + allow(RSpec).to receive(:configure).and_yield(rspec_config) + allow(rspec_config).to receive(:add_formatter) + allow(rspec_config).to receive(:fail_if_no_examples=) + end + + it 'sets the `--dry-run` flag' do + expect_rspec_runner_arguments(['--dry-run', '--tag', '~orchestrated', '--tag', '~transient', '--tag', '~geo', *described_class::DEFAULT_TEST_PATH_ARGS], [$stderr, anything]) + + subject.perform + end + + it 'configures json formatted output to file' do + allow(QA::Runtime::Path).to receive(:qa_root).and_return('/root') + + expect(rspec_config).to receive(:add_formatter) + .with(QA::Support::JsonFormatter, output_file) + expect(rspec_config).to receive(:fail_if_no_examples=) + .with(true) + + allow(RSpec::Core::Runner).to receive(:run).and_return(0) + + subject.perform + end + + after do + QA::Runtime::Scenario.attributes.delete(:test_metadata_only) + end + end + context 'when tags are set' do subject { described_class.new.tap { |runner| runner.tags = %i[orchestrated github] } } diff --git a/qa/spec/specs/scenario_shared_examples.rb b/qa/spec/specs/scenario_shared_examples.rb new file mode 100644 index 00000000000..7d806d50d21 --- /dev/null +++ b/qa/spec/specs/scenario_shared_examples.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module QA + RSpec.shared_examples 'a QA scenario class' do + let(:attributes) { class_spy('Runtime::Scenario') } + let(:runner) { class_spy('Specs::Runner') } + let(:release) { class_spy('Runtime::Release') } + let(:feature) { class_spy('Runtime::Feature') } + + let(:args) { { gitlab_address: 'http://gitlab_address' } } + let(:named_options) { %w[--address http://gitlab_address] } + let(:tags) { [] } + let(:options) { %w[path1 path2] } + + before do + stub_const('QA::Specs::Runner', runner) + stub_const('QA::Runtime::Release', release) + stub_const('QA::Runtime::Scenario', attributes) + stub_const('QA::Runtime::Feature', feature) + + allow(attributes).to receive(:gitlab_address).and_return(args[:gitlab_address]) + allow(runner).to receive(:perform).and_yield(runner) + allow(QA::Runtime::Address).to receive(:valid?).and_return(true) + end + + it 'responds to perform' do + expect(subject).to respond_to(:perform) + end + + it 'sets an address of the subject' do + subject.perform(args) + + expect(attributes).to have_received(:define).with(:gitlab_address, 'http://gitlab_address').at_least(:once) + end + + it 'performs before hooks only once' do + subject.perform(args) + + expect(release).to have_received(:perform_before_hooks).once + end + + it 'sets tags on runner' do + subject.perform(args) + + expect(runner).to have_received(:tags=).with(tags) + end + + context 'with RSpec options' do + it 'sets options on runner' do + subject.perform(args, *options) + + expect(runner).to have_received(:options=).with(options) + end + end + + context 'with named command-line options' do + it 'converts options to attributes' do + described_class.launch!(named_options) + + args do |k, v| + expect(attributes).to have_received(:define).with(k, v) + end + end + + it 'raises an error if the option is invalid' do + expect { described_class.launch!(['--foo']) }.to raise_error(OptionParser::InvalidOption) + end + + it 'passes on options after --' do + expect(described_class).to receive(:perform).with(attributes, *%w[--tag quarantine]) + + described_class.launch!(named_options.push(*%w[-- --tag quarantine])) + end + end + end +end diff --git a/qa/spec/specs/spec_helper.rb b/qa/spec/specs/spec_helper.rb new file mode 100644 index 00000000000..e4514c6c64f --- /dev/null +++ b/qa/spec/specs/spec_helper.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require_relative '../../qa' + +require_relative 'scenario_shared_examples' |