diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 12:16:11 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-01-20 12:16:11 +0300 |
commit | edaa33dee2ff2f7ea3fac488d41558eb5f86d68c (patch) | |
tree | 11f143effbfeba52329fb7afbd05e6e2a3790241 /qa/spec | |
parent | d8a5691316400a0f7ec4f83832698f1988eb27c1 (diff) |
Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42
Diffstat (limited to 'qa/spec')
-rw-r--r-- | qa/spec/page/logging_spec.rb | 1 | ||||
-rw-r--r-- | qa/spec/resource/base_spec.rb | 13 | ||||
-rw-r--r-- | qa/spec/runtime/env_spec.rb | 96 | ||||
-rw-r--r-- | qa/spec/scenario/test/integration/github_spec.rb | 2 | ||||
-rw-r--r-- | qa/spec/spec_helper.rb | 23 | ||||
-rw-r--r-- | qa/spec/specs/runner_spec.rb | 26 | ||||
-rw-r--r-- | qa/spec/support/formatters/test_stats_formatter_spec.rb | 9 | ||||
-rw-r--r-- | qa/spec/support/page_error_checker_spec.rb | 217 | ||||
-rw-r--r-- | qa/spec/support/wait_for_requests_spec.rb | 16 | ||||
-rw-r--r-- | qa/spec/tools/long_running_spec_reporter_spec.rb | 69 | ||||
-rw-r--r-- | qa/spec/tools/reliable_report_spec.rb | 62 | ||||
-rw-r--r-- | qa/spec/tools/test_resources_data_processor_spec.rb | 33 |
12 files changed, 511 insertions, 56 deletions
diff --git a/qa/spec/page/logging_spec.rb b/qa/spec/page/logging_spec.rb index 7c521f60b84..98326ecd343 100644 --- a/qa/spec/page/logging_spec.rb +++ b/qa/spec/page/logging_spec.rb @@ -14,6 +14,7 @@ RSpec.describe QA::Support::Page::Logging do allow(page).to receive(:find).and_return(page) allow(page).to receive(:current_url).and_return('http://current-url') allow(page).to receive(:has_css?).with(any_args).and_return(true) + allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code).and_return(0) end subject do diff --git a/qa/spec/resource/base_spec.rb b/qa/spec/resource/base_spec.rb index 2a26a479436..2dd25f983bf 100644 --- a/qa/spec/resource/base_spec.rb +++ b/qa/spec/resource/base_spec.rb @@ -277,17 +277,30 @@ RSpec.describe QA::Resource::Base do describe '#visit!' do include_context 'with simple resource' + let(:wait_for_requests_class) { QA::Support::WaitForRequests } + before do allow(resource).to receive(:visit) end it 'calls #visit with the underlying #web_url' do allow(resource).to receive(:current_url).and_return(subject.current_url) + expect(wait_for_requests_class).to receive(:wait_for_requests).with({ skip_resp_code_check: false }).twice resource.web_url = subject.current_url resource.visit! expect(resource).to have_received(:visit).with(subject.current_url) end + + it 'calls #visit with the underlying #web_url with skip_resp_code_check specified as true' do + allow(resource).to receive(:current_url).and_return(subject.current_url) + expect(wait_for_requests_class).to receive(:wait_for_requests).with({ skip_resp_code_check: true }).twice + + resource.web_url = subject.current_url + resource.visit!(skip_resp_code_check: true) + + expect(resource).to have_received(:visit).with(subject.current_url) + end end end diff --git a/qa/spec/runtime/env_spec.rb b/qa/spec/runtime/env_spec.rb index 80d8a9a1892..0f752ad96b7 100644 --- a/qa/spec/runtime/env_spec.rb +++ b/qa/spec/runtime/env_spec.rb @@ -83,6 +83,46 @@ RSpec.describe QA::Runtime::Env do end end + describe '.running_on_dot_com?' do + using RSpec::Parameterized::TableSyntax + + where(:url, :result) do + 'https://www.gitlab.com' | true + 'https://staging.gitlab.com' | true + 'http://www.gitlab.com' | true + 'http://localhost:3000' | false + 'http://localhost' | false + 'http://gdk.test:3000' | false + end + + with_them do + before do + QA::Runtime::Scenario.define(:gitlab_address, url) + end + + it { expect(described_class.running_on_dot_com?).to eq result } + end + end + + describe '.running_on_dev?' do + using RSpec::Parameterized::TableSyntax + + where(:url, :result) do + 'https://www.gitlab.com' | false + 'http://localhost:3000' | true + 'http://localhost' | false + 'http://gdk.test:3000' | true + end + + with_them do + before do + QA::Runtime::Scenario.define(:gitlab_address, url) + end + + it { expect(described_class.running_on_dev?).to eq result } + end + end + describe '.personal_access_token' do around do |example| described_class.instance_variable_set(:@personal_access_token, nil) @@ -173,31 +213,23 @@ RSpec.describe QA::Runtime::Env do stub_env('CI_NODE_TOTAL', '2') end - it 'returns true if KNAPSACK_GENERATE_REPORT is defined' do - stub_env('KNAPSACK_GENERATE_REPORT', 'true') - + it 'returns true if running in parallel CI run' do expect(described_class.knapsack?).to be_truthy end - it 'returns true if KNAPSACK_REPORT_PATH is defined' do - stub_env('KNAPSACK_REPORT_PATH', '/a/path') - - expect(described_class.knapsack?).to be_truthy + it 'returns false if knapsack disabled' do + stub_env('NO_KNAPSACK', 'true') + expect(described_class.knapsack?).to be_falsey end - it 'returns true if KNAPSACK_TEST_FILE_PATTERN is defined' do - stub_env('KNAPSACK_TEST_FILE_PATTERN', '/a/**/pattern') - - expect(described_class.knapsack?).to be_truthy - end + it 'returns false if not running in a parallel job' do + stub_env('CI_NODE_TOTAL', '1') - it 'returns false if neither KNAPSACK_GENERATE_REPORT nor KNAPSACK_REPORT_PATH nor KNAPSACK_TEST_FILE_PATTERN are defined' do expect(described_class.knapsack?).to be_falsey end - it 'returns false if not running in parallel job' do - stub_env('CI_NODE_TOTAL', '1') - stub_env('KNAPSACK_GENERATE_REPORT', 'true') + it 'returns false if not running in ci' do + stub_env('CI_NODE_TOTAL', nil) expect(described_class.knapsack?).to be_falsey end @@ -328,4 +360,36 @@ RSpec.describe QA::Runtime::Env do end end end + + describe '.test_resources_created_filepath' do + context 'when not in CI' do + before do + allow(described_class).to receive(:running_in_ci?).and_return(false) + end + + it 'returns default path if QA_TEST_RESOURCES_CREATED_FILEPATH is not defined' do + stub_env('QA_TEST_RESOURCES_CREATED_FILEPATH', nil) + + expect(described_class.test_resources_created_filepath).to include('tmp/test-resources.json') + end + + it 'returns path if QA_TEST_RESOURCES_CREATED_FILEPATH is defined' do + stub_env('QA_TEST_RESOURCES_CREATED_FILEPATH', 'path/to_file') + + expect(described_class.test_resources_created_filepath).to eq('path/to_file') + end + end + + context 'when in CI' do + before do + allow(described_class).to receive(:running_in_ci?).and_return(true) + allow(SecureRandom).to receive(:hex).with(3).and_return('abc123') + stub_env('QA_TEST_RESOURCES_CREATED_FILEPATH', nil) + end + + it 'returns path with random hex in file name' do + expect(described_class.test_resources_created_filepath).to include('tmp/test-resources-abc123.json') + end + end + end end diff --git a/qa/spec/scenario/test/integration/github_spec.rb b/qa/spec/scenario/test/integration/github_spec.rb index b68b06a7b9f..4ad42b4f5cc 100644 --- a/qa/spec/scenario/test/integration/github_spec.rb +++ b/qa/spec/scenario/test/integration/github_spec.rb @@ -2,7 +2,7 @@ RSpec.describe QA::Scenario::Test::Integration::Github do describe '#perform' do - let(:env) { spy('Runtime::Env') } + let(:env) { spy('Runtime::Env', knapsack?: false, dry_run: false) } before do stub_const('QA::Runtime::Env', env) diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index 4372d9d2728..47791f76970 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -10,9 +10,9 @@ require 'active_support/core_ext/object/blank' require_relative 'qa_deprecation_toolkit_env' QaDeprecationToolkitEnv.configure! -Knapsack::Adapters::RSpecAdapter.bind if ENV['CI'] && QA::Runtime::Env.knapsack? && !ENV['NO_KNAPSACK'] +Knapsack::Adapters::RSpecAdapter.bind if QA::Runtime::Env.knapsack? -QA::Runtime::Browser.configure! +QA::Runtime::Browser.configure! unless QA::Runtime::Env.dry_run QA::Runtime::AllureReport.configure! QA::Runtime::Scenario.from_env(QA::Runtime::Env.runtime_scenario_attributes) @@ -66,9 +66,14 @@ RSpec.configure do |config| config.after(:suite) do |suite| # If any tests failed, leave the resources behind to help troubleshoot - next if suite.reporter.failed_examples.present? + QA::Resource::ReusableProject.remove_all_via_api! unless suite.reporter.failed_examples.present? - QA::Resource::ReusableProject.remove_all_via_api! + # Write all test created resources to JSON file + QA::Tools::TestResourceDataProcessor.write_to_file + end + + config.append_after(:suite) do + QA::Tools::KnapsackReport.move_regenerated_report if QA::Runtime::Env.knapsack? end config.expect_with :rspec do |expectations| @@ -95,8 +100,14 @@ RSpec.configure do |config| if ENV['CI'] && !QA::Runtime::Env.disable_rspec_retry? non_quarantine_retries = QA::Runtime::Env.ci_project_name =~ /staging|canary|production/ ? 3 : 2 config.around do |example| - retry_times = example.metadata.key?(:quarantine) ? 1 : non_quarantine_retries - example.run_with_retry retry: retry_times + quarantine = example.metadata[:quarantine] + different_quarantine_context = QA::Specs::Helpers::Quarantine.quarantined_different_context?(quarantine) + focused_quarantine = QA::Specs::Helpers::Quarantine.filters.key?(:quarantine) + + # Do not disable retry when spec is quarantined but on different environment + next example.run_with_retry(retry: non_quarantine_retries) if different_quarantine_context && !focused_quarantine + + example.run_with_retry(retry: quarantine ? 1 : non_quarantine_retries) end end end diff --git a/qa/spec/specs/runner_spec.rb b/qa/spec/specs/runner_spec.rb index 021ce9091e0..5cc9ff403cd 100644 --- a/qa/spec/specs/runner_spec.rb +++ b/qa/spec/specs/runner_spec.rb @@ -28,6 +28,26 @@ RSpec.describe QA::Specs::Runner do end end + context 'when count_examples_only is set as an option' do + let(:out) { StringIO.new } + + before do + QA::Runtime::Scenario.define(:count_examples_only, true) + out.string = '22 examples,' + allow(StringIO).to receive(:new).and_return(out) + end + + it 'sets the `--dry-run` flag' do + expect_rspec_runner_arguments(['--dry-run', '--tag', '~orchestrated', '--tag', '~transient', '--tag', '~geo', *described_class::DEFAULT_TEST_PATH_ARGS], [$stderr, anything]) + + subject.perform + end + + after do + QA::Runtime::Scenario.attributes.delete(:count_examples_only) + end + end + context 'when tags are set' do subject { described_class.new.tap { |runner| runner.tags = %i[orchestrated github] } } @@ -158,10 +178,10 @@ RSpec.describe QA::Specs::Runner do end end - def expect_rspec_runner_arguments(arguments) + def expect_rspec_runner_arguments(arguments, std_arguments = described_class::DEFAULT_STD_ARGS) expect(RSpec::Core::Runner).to receive(:run) - .with(arguments, $stderr, $stdout) - .and_return(0) + .with(arguments, *std_arguments) + .and_return(0) end end end diff --git a/qa/spec/support/formatters/test_stats_formatter_spec.rb b/qa/spec/support/formatters/test_stats_formatter_spec.rb index 71ab9c1d541..2bfd7863653 100644 --- a/qa/spec/support/formatters/test_stats_formatter_spec.rb +++ b/qa/spec/support/formatters/test_stats_formatter_spec.rb @@ -56,13 +56,14 @@ describe QA::Support::Formatters::TestStatsFormatter do retry_attempts: 0, job_url: ci_job_url, pipeline_url: ci_pipeline_url, - pipeline_id: ci_pipeline_id + pipeline_id: ci_pipeline_id, + testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234' } } end def run_spec(&spec) - spec ||= -> { it('spec') {} } + spec ||= -> { it('spec', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {} } describe_successfully('stats export', &spec).tap do |example_group| example_group.examples.each { |ex| ex.metadata[:file_path] = file_path } @@ -131,7 +132,7 @@ describe QA::Support::Formatters::TestStatsFormatter do it 'exports data to influxdb with correct reliable tag' do run_spec do - it('spec', :reliable) {} + it('spec', :reliable, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {} end expect(influx_write_api).to have_received(:write).with(data: [data]) @@ -143,7 +144,7 @@ describe QA::Support::Formatters::TestStatsFormatter do it 'exports data to influxdb with correct quarantine tag' do run_spec do - it('spec', :quarantine) {} + it('spec', :quarantine, testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {} end expect(influx_write_api).to have_received(:write).with(data: [data]) diff --git a/qa/spec/support/page_error_checker_spec.rb b/qa/spec/support/page_error_checker_spec.rb new file mode 100644 index 00000000000..764b6110e08 --- /dev/null +++ b/qa/spec/support/page_error_checker_spec.rb @@ -0,0 +1,217 @@ +# frozen_string_literal: true + +RSpec.describe QA::Support::PageErrorChecker do + let(:test_path) { '/test/path' } + + let(:page) { double(Capybara.page) } + + describe '.report!' do + context 'reports errors' do + let(:expected_chrome_error) do + "chrome errors\n\n"\ + "Path: #{test_path}" + end + + let(:expected_basic_error) do + "foo status\n\n"\ + "Path: #{test_path}" + end + + it 'reports error message on chrome browser' do + allow(QA::Support::PageErrorChecker).to receive(:return_chrome_errors).and_return('chrome errors') + allow(page).to receive(:current_path).and_return(test_path) + allow(QA::Runtime::Env).to receive(:browser).and_return(:chrome) + + expect { QA::Support::PageErrorChecker.report!(page, 500) }.to raise_error(RuntimeError, expected_chrome_error) + end + + it 'reports basic message on non-chrome browser' do + allow(QA::Support::PageErrorChecker).to receive(:status_code_report).and_return('foo status') + allow(page).to receive(:current_path).and_return(test_path) + allow(QA::Runtime::Env).to receive(:browser).and_return(:firefox) + + expect { QA::Support::PageErrorChecker.report!(page, 500) }.to raise_error(RuntimeError, expected_basic_error) + end + end + end + + describe '.return_chrome_errors' do + context 'returns error message' do + before do + single_log = Class.new do + def level + 'SEVERE' + end + end + stub_const('SingleLog', single_log) + one_error_mocked_logs = Class.new do + def self.select + [SingleLog] + end + end + stub_const('OneErrorMockedLogs', one_error_mocked_logs) + three_errors_mocked_logs = Class.new do + def self.select + [SingleLog, SingleLog, SingleLog] + end + end + stub_const('ThreeErrorsMockedLogs', three_errors_mocked_logs) + no_error_mocked_logs = Class.new do + def self.select + [] + end + end + stub_const('NoErrorMockedLogs', no_error_mocked_logs) + end + + let(:expected_single_error) do + "There was 1 SEVERE level error:\n\n"\ + "bar foo" + end + + let(:expected_multiple_error) do + "There were 3 SEVERE level errors:\n\n"\ + "bar foo\n"\ + "foo\n"\ + "bar" + end + + it 'returns status code report on no severe errors found' do + allow(QA::Support::PageErrorChecker).to receive(:logs).with(page).and_return(NoErrorMockedLogs) + allow(QA::Support::PageErrorChecker).to receive(:status_code_report).with('123').and_return('Test Status Code return 123') + + expect(QA::Support::PageErrorChecker.return_chrome_errors(page, '123')).to eq('Test Status Code return 123') + end + + it 'returns report on 1 severe error found' do + allow(QA::Support::PageErrorChecker).to receive(:error_report_for).with([SingleLog]).and_return('bar foo') + allow(QA::Support::PageErrorChecker).to receive(:logs).with(page).and_return(OneErrorMockedLogs) + allow(page).to receive(:current_path).and_return(test_path) + + expect(QA::Support::PageErrorChecker.return_chrome_errors(page, '123')).to eq(expected_single_error) + end + + it 'returns report on multiple severe errors found' do + allow(QA::Support::PageErrorChecker).to receive(:error_report_for) + .with([SingleLog, SingleLog, SingleLog]).and_return("bar foo\nfoo\nbar") + allow(QA::Support::PageErrorChecker).to receive(:logs).with(page).and_return(ThreeErrorsMockedLogs) + allow(page).to receive(:current_path).and_return(test_path) + + expect(QA::Support::PageErrorChecker.return_chrome_errors(page, '123')).to eq(expected_multiple_error) + end + end + end + + describe '.check_page_for_error_code' do + require 'nokogiri' + before do + nokogiri_parse = Class.new do + def self.parse(str) + Nokogiri::HTML.parse(str) + end + end + stub_const('NokogiriParse', nokogiri_parse) + end + let(:error_404_str) do + "<div class=\"error\">"\ + "<img src=\"404.png\" alt=\"404\" />"\ + "</div>" + end + + let(:error_500_str) { "<h1> 500 </h1>"} + let(:backtrace_str) {"<body><section class=\"backtrace\">foo</section></body>"} + let(:no_error_str) {"<body>no 404 or 500 or backtrace</body>"} + + it 'calls report with 404 if 404 found' do + allow(page).to receive(:html).and_return(error_404_str) + allow(Nokogiri::HTML).to receive(:parse).with(error_404_str).and_return(NokogiriParse.parse(error_404_str)) + + expect(QA::Support::PageErrorChecker).to receive(:report!).with(page, 404) + QA::Support::PageErrorChecker.check_page_for_error_code(page) + end + it 'calls report with 500 if 500 found' do + allow(page).to receive(:html).and_return(error_500_str) + allow(Nokogiri::HTML).to receive(:parse).with(error_500_str).and_return(NokogiriParse.parse(error_500_str)) + + expect(QA::Support::PageErrorChecker).to receive(:report!).with(page, 500) + QA::Support::PageErrorChecker.check_page_for_error_code(page) + end + it 'calls report with 500 if GDK backtrace found' do + allow(page).to receive(:html).and_return(backtrace_str) + allow(Nokogiri::HTML).to receive(:parse).with(backtrace_str).and_return(NokogiriParse.parse(backtrace_str)) + + expect(QA::Support::PageErrorChecker).to receive(:report!).with(page, 500) + QA::Support::PageErrorChecker.check_page_for_error_code(page) + end + it 'does not call report if no 404, 500 or backtrace found' do + allow(page).to receive(:html).and_return(no_error_str) + allow(Nokogiri::HTML).to receive(:parse).with(no_error_str).and_return(NokogiriParse.parse(no_error_str)) + + expect(QA::Support::PageErrorChecker).not_to receive(:report!) + QA::Support::PageErrorChecker.check_page_for_error_code(page) + end + end + + describe '.error_report_for' do + before do + logs_class_one = Class.new do + def self.message + 'foo\\n' + end + end + stub_const('LogOne', logs_class_one) + logs_class_two = Class.new do + def self.message + 'bar' + end + end + stub_const('LogTwo', logs_class_two) + end + + it 'returns error report array of log messages' do + expect(QA::Support::PageErrorChecker.error_report_for([LogOne, LogTwo])) + .to eq(%W(foo\n bar)) + end + end + + describe '.logs' do + before do + logs_class = Class.new do + def self.get(level) + "logs at #{level} level" + end + end + stub_const('Logs', logs_class) + manage_class = Class.new do + def self.logs + Logs + end + end + stub_const('Manage', manage_class) + browser_class = Class.new do + def self.manage + Manage + end + end + stub_const('Browser', browser_class) + driver_class = Class.new do + def self.browser + Browser + end + end + stub_const('Driver', driver_class) + end + + it 'gets driver browser logs' do + allow(page).to receive(:driver).and_return(Driver) + + expect(QA::Support::PageErrorChecker.logs(page)).to eq('logs at browser level') + end + end + + describe '.status_code_report' do + it 'returns a string message containing the status code' do + expect(QA::Support::PageErrorChecker.status_code_report(1234)).to eq('Status code 1234 found') + end + end +end diff --git a/qa/spec/support/wait_for_requests_spec.rb b/qa/spec/support/wait_for_requests_spec.rb index 47c35addd9f..2492820b67f 100644 --- a/qa/spec/support/wait_for_requests_spec.rb +++ b/qa/spec/support/wait_for_requests_spec.rb @@ -22,5 +22,21 @@ RSpec.describe QA::Support::WaitForRequests do subject.wait_for_requests(skip_finished_loading_check: true) end end + + context 'when skip_resp_code_check is defaulted to false' do + it 'call report' do + allow(QA::Support::PageErrorChecker).to receive(:check_page_for_error_code).with(Capybara.page) + + subject.wait_for_requests + end + end + + context 'when skip_resp_code_check is true' do + it 'does not parse for an error code' do + expect(QA::Support::PageErrorChecker).not_to receive(:check_page_for_error_code) + + subject.wait_for_requests(skip_resp_code_check: true) + end + end end end diff --git a/qa/spec/tools/long_running_spec_reporter_spec.rb b/qa/spec/tools/long_running_spec_reporter_spec.rb new file mode 100644 index 00000000000..1bf520c53af --- /dev/null +++ b/qa/spec/tools/long_running_spec_reporter_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +RSpec.describe QA::Tools::LongRunningSpecReporter do + include QA::Support::Helpers::StubEnv + + subject(:reporter) { described_class.execute } + + let(:gcs_client) { double("Fog::Storage::GoogleJSON", get_object: report) } + let(:slack_notifier) { double("Slack::Notifier", post: nil) } + + before do + stub_env("SLACK_WEBHOOK", "slack_url") + stub_env("QA_KNAPSACK_REPORT_GCS_CREDENTIALS", "gcs_json") + + allow(Fog::Storage::Google).to receive(:new) + .with(google_project: "gitlab-qa-resources", google_json_key_string: "gcs_json") + .and_return(gcs_client) + allow(Slack::Notifier).to receive(:new) + .with("slack_url", channel: "#quality-reports", username: "Spec Runtime Report") + .and_return(slack_notifier) + end + + context "without specs exceeding runtime" do + let(:report) do + { + body: <<~JSON + { + "spec.rb": 5, + "spec_2.rb": 10 + } + JSON + } + end + + it "returns all good message" do + expect { reporter }.to output("No long running specs detected, all good!\n").to_stdout + end + end + + context "with specs exceeding runtime" do + let(:report) do + { + body: <<~JSON + { + "spec.rb": 5.0, + "spec_2.rb": 320.0 + } + JSON + } + end + + let(:spec) { "spec_2.rb: 5.33 minutes" } + + let(:message) do + <<~MSG + Following spec files are exceeding 5 minute runtime threshold! + Current average spec runtime: 5 seconds. + MSG + end + + it "notifies on long running specs" do + expect { reporter }.to output("#{message}\n#{spec}\n").to_stdout + expect(slack_notifier).to have_received(:post).with( + icon_emoji: ":time-out:", + text: "#{message}\n```#{spec}```" + ) + end + end +end diff --git a/qa/spec/tools/reliable_report_spec.rb b/qa/spec/tools/reliable_report_spec.rb index a048aa2e6ea..1ff62df34e0 100644 --- a/qa/spec/tools/reliable_report_spec.rb +++ b/qa/spec/tools/reliable_report_spec.rb @@ -13,30 +13,43 @@ describe QA::Tools::ReliableReport do let(:slack_channel) { "#quality-reports" } let(:range) { 14 } let(:issue_url) { "https://gitlab.com/issue/1" } + let(:time) { "2021-12-07T04:05:25.000000000+00:00" } let(:runs) do - values = { "name" => "stable spec", "status" => "passed", "file_path" => "some/spec.rb", "stage" => "manage" } + values = { + "name" => "stable spec", + "status" => "passed", + "file_path" => "some/spec.rb", + "stage" => "manage", + "_time" => time + } { 0 => instance_double( "InfluxDB2::FluxTable", records: [ instance_double("InfluxDB2::FluxRecord", values: values), instance_double("InfluxDB2::FluxRecord", values: values), - instance_double("InfluxDB2::FluxRecord", values: values) + instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s })) ] ) } end let(:reliable_runs) do - values = { "name" => "unstable spec", "status" => "failed", "file_path" => "some/spec.rb", "stage" => "create" } + values = { + "name" => "unstable spec", + "status" => "failed", + "file_path" => "some/spec.rb", + "stage" => "create", + "_time" => time + } { 0 => instance_double( "InfluxDB2::FluxTable", records: [ instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }), instance_double("InfluxDB2::FluxRecord", values: values), - instance_double("InfluxDB2::FluxRecord", values: values) + instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s })) ] ) } @@ -67,41 +80,34 @@ describe QA::Tools::ReliableReport do def markdown_section(summary, result, stage, type) <<~SECTION.strip - ``` - #{summary_table(summary, type)} - ``` + #{summary_table(summary, type, true)} - ## #{stage} + ## #{stage} (1) <details> <summary>Executions table</summary> - ``` - #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days")} - ``` + #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days", true)} </details> SECTION end - def summary_table(summary, type) - table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50)) + def summary_table(summary, type, markdown = false) + table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50), markdown) end - def table(rows, headings, title) + def table(rows, headings, title, markdown = false) Terminal::Table.new( headings: headings, - style: { all_separators: true }, - title: title, - rows: rows + title: markdown ? nil : title, + rows: rows, + style: markdown ? { border: :markdown } : { all_separators: true } ) end def name_column(spec_name) - name = "name: '#{spec_name}'" - file = "file: 'spec.rb'".ljust(160) - - "#{name}\n#{file}" + "**name**: #{spec_name}<br>**file**: spec.rb" end before do @@ -136,11 +142,15 @@ describe QA::Tools::ReliableReport do <<~TXT.strip [[_TOC_]] - # Candidates for promotion to reliable + # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today}) + + Total amount: **1** #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')} - # Reliable specs with failures + # Reliable specs with failures (#{Date.today - range} - #{Date.today}) + + Total amount: **1** #{markdown_section([['create', 1]], [[name_column('unstable spec'), 3, 2, '66.67%']], 'create', 'unstable')} TXT @@ -155,9 +165,9 @@ describe QA::Tools::ReliableReport do verify_ssl: false, headers: { "PRIVATE-TOKEN" => "gitlab_token" }, payload: { - title: "Reliable spec report", + title: "Reliable e2e test report", description: issue_body, - labels: "Quality,test" + labels: "Quality,test,type::maintenance,reliable test report" } ) expect(slack_notifier).to have_received(:post).with( @@ -180,7 +190,7 @@ describe QA::Tools::ReliableReport do end it "notifies failure", :aggregate_failures do - expect { expect { run }.to raise_error(SystemExit) }.to output.to_stdout + expect { expect { run }.to raise_error("Connection error!") }.to output.to_stdout expect(slack_notifier).to have_received(:post).with( icon_emoji: ":sadpanda:", diff --git a/qa/spec/tools/test_resources_data_processor_spec.rb b/qa/spec/tools/test_resources_data_processor_spec.rb new file mode 100644 index 00000000000..6a8c0fd06a4 --- /dev/null +++ b/qa/spec/tools/test_resources_data_processor_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +RSpec.describe QA::Tools::TestResourceDataProcessor do + let(:info) { 'information' } + let(:api_path) { '/foo' } + let(:result) { [{ info: info, api_path: api_path }] } + + describe '.collect' do + context 'when resource is not restricted' do + let(:resource) { instance_double(QA::Resource::Project, api_delete_path: '/foo', api_response: 'foo') } + + it 'collects resource' do + expect(described_class.collect(resource, info)).to eq(result) + end + end + + context 'when resource api response is nil' do + let(:resource) { double(QA::Resource::Project, api_delete_path: '/foo', api_response: nil) } + + it 'does not collect resource' do + expect(described_class.collect(resource, info)).to eq(nil) + end + end + + context 'when resource is restricted' do + let(:resource) { double(QA::Resource::Sandbox, api_delete_path: '/foo', api_response: 'foo') } + + it 'does not collect resource' do + expect(described_class.collect(resource, info)).to eq(nil) + end + end + end +end |