Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/qa/spec
diff options
context:
space:
mode:
Diffstat (limited to 'qa/spec')
-rw-r--r--qa/spec/runtime/feature_spec.rb48
-rw-r--r--qa/spec/scenario/test/instance/reliable_spec.rb7
-rw-r--r--qa/spec/spec_helper.rb12
-rw-r--r--qa/spec/specs/allure_report_spec.rb19
-rw-r--r--qa/spec/specs/helpers/context_selector_spec.rb18
-rw-r--r--qa/spec/support/formatters/test_stats_formatter_spec.rb26
-rw-r--r--qa/spec/support/repeater_spec.rb114
-rw-r--r--qa/spec/support/retrier_spec.rb71
-rw-r--r--qa/spec/support/shared_contexts/packages_registry_shared_context.rb7
-rw-r--r--qa/spec/support/waiter_spec.rb35
-rw-r--r--qa/spec/tools/reliable_report_spec.rb145
11 files changed, 381 insertions, 121 deletions
diff --git a/qa/spec/runtime/feature_spec.rb b/qa/spec/runtime/feature_spec.rb
index 39c20dd3070..88f5cd5be93 100644
--- a/qa/spec/runtime/feature_spec.rb
+++ b/qa/spec/runtime/feature_spec.rb
@@ -175,6 +175,20 @@ RSpec.describe QA::Runtime::Feature do
expect(described_class.enabled?(feature_flag)).to be_truthy
end
+ it 'raises an error when the scope is unknown' do
+ expect(QA::Runtime::API::Request)
+ .to receive(:new)
+ .with(api_client, "/features")
+ .and_return(request)
+ expect(described_class)
+ .to receive(:get)
+ .and_return(
+ Struct.new(:code, :body)
+ .new(200, %([{ "name": "a_flag", "state": "conditional", "gates": { "key": "groups", "value": ["foo"] } }])))
+
+ expect { described_class.enabled?(feature_flag, scope: 'foo') }.to raise_error(QA::Runtime::Feature::UnknownScopeError)
+ end
+
context 'when a project scope is provided' do
it_behaves_like 'checks a feature flag' do
let(:scope) { :project }
@@ -212,4 +226,38 @@ RSpec.describe QA::Runtime::Feature do
end
end
end
+
+ describe '.set' do
+ let(:scope) { { scope: 'actor' } }
+
+ it 'raises an error when the flag state is unknown' do
+ expect(described_class).not_to receive(:enable)
+ expect(described_class).not_to receive(:disable)
+
+ expect { described_class.set({ foo: 'bar' }, **scope) }.to raise_error(QA::Runtime::Feature::UnknownStateError, 'Unknown feature flag state: bar')
+ end
+
+ it 'enables feature flags' do
+ expect(described_class).to receive(:enable).with(:flag1, scope)
+ expect(described_class).to receive(:enable).with(:flag2, scope)
+ expect(described_class).not_to receive(:disable)
+
+ described_class.set({ flag1: 'enabled', flag2: 'enable' }, **scope)
+ end
+
+ it 'disables feature flags' do
+ expect(described_class).to receive(:disable).with(:flag1, scope)
+ expect(described_class).to receive(:disable).with(:flag2, scope)
+ expect(described_class).not_to receive(:enable)
+
+ described_class.set({ flag1: 'disable', flag2: 'disable' }, **scope)
+ end
+
+ it 'enables and disables feature flags' do
+ expect(described_class).to receive(:enable).with(:flag1, scope)
+ expect(described_class).to receive(:disable).with(:flag2, scope)
+
+ described_class.set({ flag1: 'enabled', flag2: 'disabled' }, **scope)
+ end
+ end
end
diff --git a/qa/spec/scenario/test/instance/reliable_spec.rb b/qa/spec/scenario/test/instance/reliable_spec.rb
new file mode 100644
index 00000000000..4001d386bf3
--- /dev/null
+++ b/qa/spec/scenario/test/instance/reliable_spec.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+RSpec.describe QA::Scenario::Test::Instance::Reliable do
+ it_behaves_like 'a QA scenario class' do
+ let(:tags) { [:reliable] }
+ end
+end
diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb
index e25892a008f..640f2de0ca2 100644
--- a/qa/spec/spec_helper.rb
+++ b/qa/spec/spec_helper.rb
@@ -27,8 +27,12 @@ RSpec.configure do |config|
config.add_formatter QA::Support::Formatters::QuarantineFormatter
config.add_formatter QA::Support::Formatters::TestStatsFormatter if QA::Runtime::Env.export_metrics?
- config.before do |example|
+ config.prepend_before do |example|
QA::Runtime::Logger.debug("\nStarting test: #{example.full_description}\n")
+
+ # Reset fabrication counters tracked in resource base
+ Thread.current[:api_fabrication] = 0
+ Thread.current[:browser_ui_fabrication] = 0
end
config.after do
@@ -36,6 +40,12 @@ RSpec.configure do |config|
QA::Git::Repository.new.delete_netrc
end
+ # Add fabrication time to spec metadata
+ config.append_after do |example|
+ example.metadata[:api_fabrication] = Thread.current[:api_fabrication]
+ example.metadata[:browser_ui_fabrication] = Thread.current[:browser_ui_fabrication]
+ end
+
config.after(:context) do
if !QA::Runtime::Browser.blank_page? && QA::Page::Main::Menu.perform(&:signed_in?)
QA::Page::Main::Menu.perform(&:sign_out)
diff --git a/qa/spec/specs/allure_report_spec.rb b/qa/spec/specs/allure_report_spec.rb
index 03bf77039cc..06b09106140 100644
--- a/qa/spec/specs/allure_report_spec.rb
+++ b/qa/spec/specs/allure_report_spec.rb
@@ -3,7 +3,7 @@
describe QA::Runtime::AllureReport do
include QA::Support::Helpers::StubEnv
- let(:rspec_config) { double('RSpec::Core::Configuration', 'add_formatter': nil, after: nil) }
+ let(:rspec_config) { double('RSpec::Core::Configuration', 'add_formatter': nil, append_after: nil) }
let(:png_path) { 'png_path' }
let(:html_path) { 'html_path' }
@@ -46,6 +46,8 @@ describe QA::Runtime::AllureReport do
let(:html_file) { 'html-file' }
let(:ci_job) { 'ee:relative 5' }
let(:versions) { { version: '14', revision: '6ced31db947' } }
+ let(:session) { double('session') }
+ let(:browser_log) { ['log message 1', 'log message 2'] }
before do
stub_env('CI', 'true')
@@ -58,6 +60,9 @@ describe QA::Runtime::AllureReport do
allow(RestClient::Request).to receive(:execute) { double('response', code: 200, body: versions.to_json) }
allow(QA::Runtime::Scenario).to receive(:method_missing).with(:gitlab_address).and_return('gitlab.com')
+ allow(Capybara).to receive(:current_session).and_return(session)
+ allow(session).to receive_message_chain('driver.browser.logs.get').and_return(browser_log)
+
described_class.configure!
end
@@ -76,7 +81,11 @@ describe QA::Runtime::AllureReport do
.with(QA::Support::Formatters::AllureMetadataFormatter).ordered
end
- it 'configures screenshot saving' do
+ it 'configures attachments saving' do
+ expect(rspec_config).to have_received(:append_after) do |&arg|
+ arg.call
+ end
+
aggregate_failures do
expect(Allure).to have_received(:add_attachment).with(
name: 'screenshot',
@@ -90,6 +99,12 @@ describe QA::Runtime::AllureReport do
type: 'text/html',
test_case: true
)
+ expect(Allure).to have_received(:add_attachment).with(
+ name: 'browser.log',
+ source: browser_log.join("\n\n"),
+ type: Allure::ContentType::TXT,
+ test_case: true
+ )
end
end
end
diff --git a/qa/spec/specs/helpers/context_selector_spec.rb b/qa/spec/specs/helpers/context_selector_spec.rb
index 0152fee6f5b..5a320cde71f 100644
--- a/qa/spec/specs/helpers/context_selector_spec.rb
+++ b/qa/spec/specs/helpers/context_selector_spec.rb
@@ -186,6 +186,24 @@ RSpec.describe QA::Specs::Helpers::ContextSelector do
end
end
+ context 'staging-ref' do
+ before do
+ QA::Runtime::Scenario.define(:gitlab_address, 'https://staging-ref.gitlab.com/')
+ end
+
+ it 'runs on staging-ref' do
+ group = describe_successfully do
+ it('does not run in staging', only: { subdomain: :staging }) {}
+ it('runs in staging-ref', only: { subdomain: /^staging-ref./ }) {}
+ end
+
+ aggregate_failures do
+ expect(group.examples[0].execution_result.status).to eq(:pending)
+ expect(group.examples[1].execution_result.status).to eq(:passed)
+ end
+ end
+ end
+
context 'production' do
before do
QA::Runtime::Scenario.define(:gitlab_address, 'https://gitlab.com/')
diff --git a/qa/spec/support/formatters/test_stats_formatter_spec.rb b/qa/spec/support/formatters/test_stats_formatter_spec.rb
index 859d45a660b..f9baf9bd9d9 100644
--- a/qa/spec/support/formatters/test_stats_formatter_spec.rb
+++ b/qa/spec/support/formatters/test_stats_formatter_spec.rb
@@ -20,6 +20,8 @@ describe QA::Support::Formatters::TestStatsFormatter do
let(:influx_write_api) { instance_double('InfluxDB2::WriteApi', write: nil) }
let(:stage) { '1_manage' }
let(:file_path) { "./qa/specs/features/#{stage}/subfolder/some_spec.rb" }
+ let(:ui_fabrication) { 0 }
+ let(:api_fabrication) { 0 }
let(:influx_client_args) do
{
@@ -48,6 +50,9 @@ describe QA::Support::Formatters::TestStatsFormatter do
fields: {
id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]',
run_time: 0,
+ api_fabrication: api_fabrication * 1000,
+ ui_fabrication: ui_fabrication * 1000,
+ total_fabrication: (api_fabrication + ui_fabrication) * 1000,
retry_attempts: 0,
job_url: ci_job_url,
pipeline_url: ci_pipeline_url,
@@ -69,6 +74,11 @@ describe QA::Support::Formatters::TestStatsFormatter do
RSpec::Core::Sandbox.sandboxed do |config|
config.formatter = QA::Support::Formatters::TestStatsFormatter
+ config.append_after do |example|
+ example.metadata[:api_fabrication] = Thread.current[:api_fabrication]
+ example.metadata[:browser_ui_fabrication] = Thread.current[:browser_ui_fabrication]
+ end
+
config.before(:context) { RSpec.current_example = nil }
example.run
@@ -171,5 +181,21 @@ describe QA::Support::Formatters::TestStatsFormatter do
expect(influx_write_api).to have_received(:write).with(data: [data])
end
end
+
+ context 'with fabrication runtimes' do
+ let(:ui_fabrication) { 10 }
+ let(:api_fabrication) { 4 }
+
+ before do
+ Thread.current[:api_fabrication] = api_fabrication
+ Thread.current[:browser_ui_fabrication] = ui_fabrication
+ end
+
+ it 'exports data to influxdb with fabrication times' do
+ run_spec
+
+ expect(influx_write_api).to have_received(:write).with(data: [data])
+ end
+ end
end
end
diff --git a/qa/spec/support/repeater_spec.rb b/qa/spec/support/repeater_spec.rb
index da8d6b18fb0..4fa3bcde5e7 100644
--- a/qa/spec/support/repeater_spec.rb
+++ b/qa/spec/support/repeater_spec.rb
@@ -23,7 +23,7 @@ RSpec.describe QA::Support::Repeater do
context 'when retry_on_exception is not provided (default: false)' do
context 'when max_duration is provided' do
context 'when max duration is reached' do
- it 'raises an exception' do
+ it 'raises an exception with default message' do
expect do
Timecop.freeze do
subject.repeat_until(max_duration: 1) do
@@ -31,7 +31,20 @@ RSpec.describe QA::Support::Repeater do
false
end
end
- end.to raise_error(QA::Support::Repeater::WaitExceededError, "Wait condition not met after 1 second")
+ end.to raise_error(QA::Support::Repeater::WaitExceededError, "Wait failed after 1 second")
+ end
+
+ it 'raises an exception with custom message' do
+ message = 'Some custom action'
+
+ expect do
+ Timecop.freeze do
+ subject.repeat_until(max_duration: 1, message: message) do
+ Timecop.travel(2)
+ false
+ end
+ end
+ end.to raise_error(QA::Support::Repeater::WaitExceededError, "#{message} failed after 1 second")
end
it 'ignores attempts' do
@@ -70,14 +83,26 @@ RSpec.describe QA::Support::Repeater do
context 'when max_attempts is provided' do
context 'when max_attempts is reached' do
- it 'raises an exception' do
+ it 'raises an exception with default message' do
expect do
Timecop.freeze do
subject.repeat_until(max_attempts: 1) do
false
end
end
- end.to raise_error(QA::Support::Repeater::RetriesExceededError, "Retry condition not met after 1 attempt")
+ end.to raise_error(QA::Support::Repeater::RetriesExceededError, "Retry failed after 1 attempt")
+ end
+
+ it 'raises an exception with custom message' do
+ message = 'Some custom action'
+
+ expect do
+ Timecop.freeze do
+ subject.repeat_until(max_attempts: 1, message: message) do
+ false
+ end
+ end
+ end.to raise_error(QA::Support::Repeater::RetriesExceededError, "#{message} failed after 1 attempt")
end
it 'ignores duration' do
@@ -126,7 +151,7 @@ RSpec.describe QA::Support::Repeater do
false
end
end
- end.to raise_error(QA::Support::Repeater::RetriesExceededError, "Retry condition not met after 1 attempt")
+ end.to raise_error(QA::Support::Repeater::RetriesExceededError, "Retry failed after 1 attempt")
end
end
@@ -141,7 +166,7 @@ RSpec.describe QA::Support::Repeater do
false
end
end
- end.to raise_error(QA::Support::Repeater::WaitExceededError, "Wait condition not met after 1 second")
+ end.to raise_error(QA::Support::Repeater::WaitExceededError, "Wait failed after 1 second")
end
end
end
@@ -210,7 +235,7 @@ RSpec.describe QA::Support::Repeater do
false
end
end
- end.to raise_error(QA::Support::Repeater::RetriesExceededError, "Retry condition not met after 1 attempt")
+ end.to raise_error(QA::Support::Repeater::RetriesExceededError, "Retry failed after 1 attempt")
end
end
@@ -225,7 +250,7 @@ RSpec.describe QA::Support::Repeater do
false
end
end
- end.to raise_error(QA::Support::Repeater::WaitExceededError, "Wait condition not met after 1 second")
+ end.to raise_error(QA::Support::Repeater::WaitExceededError, "Wait failed after 1 second")
end
end
end
@@ -380,34 +405,67 @@ RSpec.describe QA::Support::Repeater do
end
end
- it 'logs attempts' do
- attempted = false
+ context 'with logging' do
+ before do
+ allow(QA::Runtime::Logger).to receive(:debug)
+ end
- expect do
- subject.repeat_until(max_attempts: 1) do
- unless attempted
- attempted = true
- break false
- end
+ it 'skips logging single attempt with max_attempts' do
+ subject.repeat_until(max_attempts: 3) do
+ true
+ end
+ expect(QA::Runtime::Logger).not_to have_received(:debug)
+ end
+
+ it 'skips logging single attempt with max_duration' do
+ subject.repeat_until(max_duration: 3) do
true
end
- end.to output(/Attempt number/).to_stdout_from_any_process
- end
- it 'allows logging to be silenced' do
- attempted = false
+ expect(QA::Runtime::Logger).not_to have_received(:debug)
+ end
- expect do
- subject.repeat_until(max_attempts: 1, log: false) do
- unless attempted
- attempted = true
- break false
- end
+ it 'allows logging to be silenced' do
+ subject.repeat_until(max_attempts: 3, log: false, raise_on_failure: false) do
+ false
+ end
- true
+ expect(QA::Runtime::Logger).not_to have_received(:debug)
+ end
+
+ it 'starts logging on subsequent attempts for max_duration' do
+ subject.repeat_until(max_duration: 0.3, sleep_interval: 0.1, raise_on_failure: false) do
+ false
+ end
+
+ aggregate_failures do
+ expect(QA::Runtime::Logger).to have_received(:debug).with(<<~MSG.strip).ordered.once
+ Retrying action with: max_duration: 0.3; sleep_interval: 0.1; raise_on_failure: false; retry_on_exception: false
+ MSG
+ expect(QA::Runtime::Logger).to have_received(:debug).with('ended retry').ordered.once
+ expect(QA::Runtime::Logger).not_to have_received(:debug).with(/Attempt number/)
+ end
+ end
+
+ it 'starts logging subsequent attempts for max_attempts' do
+ attempts = 0
+ subject.repeat_until(max_attempts: 4, raise_on_failure: false) do
+ next true if attempts == 2
+
+ attempts += 1
+ false
end
- end.not_to output.to_stdout_from_any_process
+
+ aggregate_failures do
+ expect(QA::Runtime::Logger).to have_received(:debug).with(<<~MSG.strip).ordered.once
+ Retrying action with: max_attempts: 4; sleep_interval: 0; raise_on_failure: false; retry_on_exception: false
+ MSG
+ expect(QA::Runtime::Logger).to have_received(:debug).with('Attempt number 2').ordered.once
+ expect(QA::Runtime::Logger).to have_received(:debug).with('Attempt number 3').ordered.once
+ expect(QA::Runtime::Logger).to have_received(:debug).with('ended retry').ordered.once
+ end
+ end
end
end
end
diff --git a/qa/spec/support/retrier_spec.rb b/qa/spec/support/retrier_spec.rb
index 9ad3e85fea9..1f303093a00 100644
--- a/qa/spec/support/retrier_spec.rb
+++ b/qa/spec/support/retrier_spec.rb
@@ -1,42 +1,7 @@
# frozen_string_literal: true
RSpec.describe QA::Support::Retrier do
- before do
- logger = ::Logger.new $stdout
- logger.level = ::Logger::DEBUG
- QA::Runtime::Logger.logger = logger
- end
-
describe '.retry_until' do
- context 'when the condition is true' do
- it 'logs max attempts (3 by default)' do
- expect { subject.retry_until { true } }
- .to output(/with retry_until: max_attempts: 3; reload_page: ; sleep_interval: 0; raise_on_failure: true; retry_on_exception: false/).to_stdout_from_any_process
- end
-
- it 'logs max duration' do
- expect { subject.retry_until(max_duration: 1) { true } }
- .to output(/with retry_until: max_duration: 1; reload_page: ; sleep_interval: 0; raise_on_failure: true; retry_on_exception: false/).to_stdout_from_any_process
- end
-
- it 'logs the end' do
- expect { subject.retry_until { true } }
- .to output(/ended retry_until$/).to_stdout_from_any_process
- end
- end
-
- context 'when the condition is false' do
- it 'logs the start' do
- expect { subject.retry_until(max_duration: 0, raise_on_failure: false) { false } }
- .to output(/with retry_until: max_duration: 0; reload_page: ; sleep_interval: 0; raise_on_failure: false; retry_on_exception: false/).to_stdout_from_any_process
- end
-
- it 'logs the end' do
- expect { subject.retry_until(max_duration: 0, raise_on_failure: false) { false } }
- .to output(/ended retry_until$/).to_stdout_from_any_process
- end
- end
-
context 'when max_duration and max_attempts are nil' do
it 'sets max attempts to 3 by default' do
expect(subject).to receive(:repeat_until).with(hash_including(max_attempts: 3))
@@ -62,35 +27,21 @@ RSpec.describe QA::Support::Retrier do
subject.retry_until
end
- end
- describe '.retry_on_exception' do
- context 'when the condition is true' do
- it 'logs max_attempts, reload_page, and sleep_interval parameters' do
- message = /with retry_on_exception: max_attempts: 1; reload_page: true; sleep_interval: 0/
- expect { subject.retry_on_exception(max_attempts: 1, reload_page: true, sleep_interval: 0) { true } }
- .to output(message).to_stdout_from_any_process
- end
+ it 'allows logs to be silenced' do
+ expect(subject).to receive(:repeat_until).with(hash_including(log: false))
- it 'logs the end' do
- expect { subject.retry_on_exception(max_attempts: 1, reload_page: nil, sleep_interval: 0) { true } }
- .to output(/ended retry_on_exception$/).to_stdout_from_any_process
- end
+ subject.retry_until(log: false)
end
- context 'when the condition is false' do
- it 'logs the start' do
- message = /with retry_on_exception: max_attempts: 1; reload_page: true; sleep_interval: 0/
- expect { subject.retry_on_exception(max_attempts: 1, reload_page: true, sleep_interval: 0) { false } }
- .to output(message).to_stdout_from_any_process
- end
+ it 'sets custom error message' do
+ expect(subject).to receive(:repeat_until).with(hash_including(message: 'Custom message'))
- it 'logs the end' do
- expect { subject.retry_on_exception(max_attempts: 1, reload_page: nil, sleep_interval: 0) { false } }
- .to output(/ended retry_on_exception$/).to_stdout_from_any_process
- end
+ subject.retry_until(message: 'Custom message')
end
+ end
+ describe '.retry_on_exception' do
it 'does not repeat if no exception is raised' do
loop_counter = 0
return_value = "test passed"
@@ -121,5 +72,11 @@ RSpec.describe QA::Support::Retrier do
subject.retry_on_exception
end
+
+ it 'allows logs to be silenced' do
+ expect(subject).to receive(:repeat_until).with(hash_including(log: false))
+
+ subject.retry_on_exception(log: false)
+ end
end
end
diff --git a/qa/spec/support/shared_contexts/packages_registry_shared_context.rb b/qa/spec/support/shared_contexts/packages_registry_shared_context.rb
index 6e197015640..e686d254a44 100644
--- a/qa/spec/support/shared_contexts/packages_registry_shared_context.rb
+++ b/qa/spec/support/shared_contexts/packages_registry_shared_context.rb
@@ -43,8 +43,13 @@ module QA
let(:project_deploy_token) do
Resource::DeployToken.fabricate_via_browser_ui! do |deploy_token|
- deploy_token.name = 'helm-package-deploy-token'
+ deploy_token.name = 'package-deploy-token'
deploy_token.project = package_project
+ deploy_token.scopes = [
+ :read_repository,
+ :read_package_registry,
+ :write_package_registry
+ ]
end
end
diff --git a/qa/spec/support/waiter_spec.rb b/qa/spec/support/waiter_spec.rb
index d0b216b5dc1..c575a27bc35 100644
--- a/qa/spec/support/waiter_spec.rb
+++ b/qa/spec/support/waiter_spec.rb
@@ -1,40 +1,11 @@
# frozen_string_literal: true
RSpec.describe QA::Support::Waiter do
- before do
- logger = ::Logger.new $stdout
- logger.level = ::Logger::DEBUG
- QA::Runtime::Logger.logger = logger
- end
-
describe '.wait_until' do
- context 'when the condition is true' do
- it 'logs the start' do
- expect { subject.wait_until(max_duration: 0, raise_on_failure: false) { true } }
- .to output(/with wait_until: max_duration: 0; reload_page: ; sleep_interval: 0.1/).to_stdout_from_any_process
- end
-
- it 'logs the end' do
- expect { subject.wait_until(max_duration: 0, raise_on_failure: false) { true } }
- .to output(/ended wait_until$/).to_stdout_from_any_process
- end
- end
-
- context 'when the condition is false' do
- it 'logs the start' do
- expect { subject.wait_until(max_duration: 0, raise_on_failure: false) { false } }
- .to output(/with wait_until: max_duration: 0; reload_page: ; sleep_interval: 0.1/).to_stdout_from_any_process
- end
-
- it 'logs the end' do
- expect { subject.wait_until(max_duration: 0, raise_on_failure: false) { false } }
- .to output(/ended wait_until$/).to_stdout_from_any_process
- end
- end
-
it 'allows logs to be silenced' do
- expect { subject.wait_until(max_duration: 0, raise_on_failure: false, log: false) { false } }
- .not_to output.to_stdout_from_any_process
+ expect(subject).to receive(:repeat_until).with(hash_including(log: false))
+
+ subject.wait_until(log: false)
end
it 'sets max_duration to 60 by default' do
diff --git a/qa/spec/tools/reliable_report_spec.rb b/qa/spec/tools/reliable_report_spec.rb
new file mode 100644
index 00000000000..c7d4d28fb21
--- /dev/null
+++ b/qa/spec/tools/reliable_report_spec.rb
@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+describe QA::Tools::ReliableReport do
+ include QA::Support::Helpers::StubEnv
+
+ subject(:reporter) { described_class.new(run_type, range) }
+
+ let(:slack_notifier) { instance_double("Slack::Notifier", post: nil) }
+ let(:influx_client) { instance_double("InfluxDB2::Client", create_query_api: query_api) }
+ let(:query_api) { instance_double("InfluxDB2::QueryApi") }
+
+ let(:slack_channel) { "#quality-reports" }
+ let(:run_type) { "package-and-qa" }
+ let(:range) { 30 }
+ let(:results) { 10 }
+
+ let(:runs) { { 0 => stable_spec, 1 => unstable_spec } }
+
+ let(:stable_spec) do
+ spec_values = { "name" => "stable spec", "status" => "passed", "file_path" => "some/spec.rb" }
+ instance_double(
+ "InfluxDB2::FluxTable",
+ records: [
+ instance_double("InfluxDB2::FluxRecord", values: spec_values),
+ instance_double("InfluxDB2::FluxRecord", values: spec_values),
+ instance_double("InfluxDB2::FluxRecord", values: spec_values)
+ ]
+ )
+ end
+
+ let(:unstable_spec) do
+ spec_values = { "name" => "unstable spec", "status" => "failed", "file_path" => "some/spec.rb" }
+ instance_double(
+ "InfluxDB2::FluxTable",
+ records: [
+ instance_double("InfluxDB2::FluxRecord", values: { **spec_values, "status" => "passed" }),
+ instance_double("InfluxDB2::FluxRecord", values: spec_values),
+ instance_double("InfluxDB2::FluxRecord", values: spec_values)
+ ]
+ )
+ end
+
+ def flux_query(reliable)
+ <<~QUERY
+ from(bucket: "e2e-test-stats")
+ |> range(start: -#{range}d)
+ |> filter(fn: (r) => r._measurement == "test-stats" and
+ r.run_type == "#{run_type}" and
+ r.status != "pending" and
+ r.merge_request == "false" and
+ r.quarantined == "false" and
+ r.reliable == "#{reliable}" and
+ r._field == "id"
+ )
+ |> group(columns: ["name"])
+ QUERY
+ end
+
+ def table(rows, title = nil)
+ Terminal::Table.new(
+ headings: ["name", "runs", "failed", "failure rate"],
+ style: { all_separators: true },
+ title: title,
+ rows: rows
+ )
+ end
+
+ def name_column(spec_name)
+ name = "name: '#{spec_name}'"
+ file = "file: 'spec.rb'".ljust(110)
+
+ "#{name}\n#{file}"
+ end
+
+ before do
+ stub_env("QA_INFLUXDB_URL", "url")
+ stub_env("QA_INFLUXDB_TOKEN", "token")
+ stub_env("CI_SLACK_WEBHOOK_URL", "slack_url")
+
+ allow(Slack::Notifier).to receive(:new).and_return(slack_notifier)
+ allow(InfluxDB2::Client).to receive(:new).and_return(influx_client)
+ allow(query_api).to receive(:query).with(query: query).and_return(runs)
+ end
+
+ context "with stable spec report" do
+ let(:query) { flux_query(false) }
+ let(:fetch_message) { "Fetching data on test execution for past #{range} days in '#{run_type}' runs" }
+ let(:slack_send_message) { "Sending top stable spec report to #{slack_channel} slack channel" }
+ let(:title) { "Top #{results} stable specs for past #{range} days in '#{run_type}' runs" }
+ let(:rows) do
+ [
+ [name_column("stable spec"), 3, 0, "0%"],
+ [name_column("unstable spec"), 3, 2, "66.67%"]
+ ]
+ end
+
+ it "prints top stable spec report to console" do
+ expect { reporter.show_top_stable }.to output("#{fetch_message}\n\n#{table(rows, title)}\n").to_stdout
+ end
+
+ it "sends top stable spec report to slack" do
+ slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" }
+
+ expect { reporter.notify_top_stable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout
+ expect(slack_notifier).to have_received(:post).with(text: "*#{title}*", **slack_args)
+ expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows)}```", **slack_args)
+ end
+ end
+
+ context "with unstable spec report" do
+ let(:query) { flux_query(true) }
+ let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" }
+ let(:slack_send_message) { "Sending top unstable reliable spec report to #{slack_channel} slack channel" }
+ let(:title) { "Top #{results} unstable reliable specs for past #{range} days in '#{run_type}' runs" }
+ let(:rows) { [[name_column("unstable spec"), 3, 2, "66.67%"]] }
+
+ it "prints top unstable spec report to console" do
+ expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{table(rows, title)}\n").to_stdout
+ end
+
+ it "sends top unstable reliable spec report to slack" do
+ slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" }
+
+ expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n\n#{slack_send_message}\n").to_stdout
+ expect(slack_notifier).to have_received(:post).with(text: "*#{title}*", **slack_args)
+ expect(slack_notifier).to have_received(:post).with(text: "```#{table(rows)}```", **slack_args)
+ end
+ end
+
+ context "without unstable reliable specs" do
+ let(:query) { flux_query(true) }
+ let(:runs) { { 0 => stable_spec } }
+ let(:fetch_message) { "Fetching data on reliable test execution for past #{range} days in '#{run_type}' runs" }
+ let(:no_result_message) { "No unstable tests present!" }
+
+ it "prints no result message to console" do
+ expect { reporter.show_top_unstable }.to output("#{fetch_message}\n\n#{no_result_message}\n").to_stdout
+ end
+
+ it "skips slack notification" do
+ expect { reporter.notify_top_unstable }.to output("#{fetch_message}\n\n#{no_result_message}\n").to_stdout
+ expect(slack_notifier).not_to have_received(:post)
+ end
+ end
+end
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-10 17:41:08 +0300