diff options
Diffstat (limited to 'qa/spec')
-rw-r--r-- | qa/spec/git/repository_spec.rb | 2 | ||||
-rw-r--r-- | qa/spec/page/view_spec.rb | 4 | ||||
-rw-r--r-- | qa/spec/resource/api_fabricator_spec.rb | 30 | ||||
-rw-r--r-- | qa/spec/support/formatters/test_metrics_formatter_spec.rb | 26 | ||||
-rw-r--r-- | qa/spec/tools/reliable_report_spec.rb | 523 |
5 files changed, 343 insertions, 242 deletions
diff --git a/qa/spec/git/repository_spec.rb b/qa/spec/git/repository_spec.rb index 1b88cba2ba5..513e807dd99 100644 --- a/qa/spec/git/repository_spec.rb +++ b/qa/spec/git/repository_spec.rb @@ -7,7 +7,7 @@ RSpec.describe QA::Git::Repository do let(:logger) { instance_double(Logger, info: nil, debug: nil) } let(:repo_uri) { 'http://foo/bar.git' } let(:repo_uri_with_credentials) { 'http://root@foo/bar.git' } - let(:env_vars) { [%q{HOME="temp"}] } + let(:env_vars) { [%q(HOME="temp")] } let(:extra_env_vars) { [] } let(:run_params) { { env: env_vars + extra_env_vars, log_prefix: "Git: " } } let(:repository) do diff --git a/qa/spec/page/view_spec.rb b/qa/spec/page/view_spec.rb index 3342b387ed1..1f89e8829b4 100644 --- a/qa/spec/page/view_spec.rb +++ b/qa/spec/page/view_spec.rb @@ -56,7 +56,7 @@ RSpec.describe QA::Page::View do it 'returns an array of errors related to missing elements' do expect(subject.errors).not_to be_empty expect(subject.errors.first) - .to match %r(Missing element `.*` in `.*/some/file.html` view) + .to match %r{Missing element `.*` in `.*/some/file.html` view} end end end @@ -65,7 +65,7 @@ RSpec.describe QA::Page::View do it 'returns an error when it is not able to find the partial' do expect(subject.errors).to be_one expect(subject.errors.first) - .to match %r(Missing view partial `.*/some/file.html`!) + .to match %r{Missing view partial `.*/some/file.html`!} end end end diff --git a/qa/spec/resource/api_fabricator_spec.rb b/qa/spec/resource/api_fabricator_spec.rb index b0b84d0b202..27e9e3d22dd 100644 --- a/qa/spec/resource/api_fabricator_spec.rb +++ b/qa/spec/resource/api_fabricator_spec.rb @@ -203,4 +203,34 @@ RSpec.describe QA::Resource::ApiFabricator do end end end + + describe '#exists?' do + let(:resource) { resource_with_api_support } + let(:request) { double('GET request', url: 'new-url') } + let(:args) { { max_redirects: 0 } } + + before do + allow(QA::Runtime::API::Request).to receive(:new).and_return(request) + end + + context 'when request is successful' do + let(:response) { double('GET response', code: 200) } + + it 'returns true' do + expect(subject).to receive(:get).with(request.url, args).and_return(response) + + expect(subject.exists?(**args)).to eq(true) + end + end + + context 'when request is unsuccessful' do + let(:response) { double('GET response', code: 404) } + + it 'returns false' do + expect(subject).to receive(:get).with(request.url, args).and_return(response) + + expect(subject.exists?(**args)).to eq(false) + end + end + end end diff --git a/qa/spec/support/formatters/test_metrics_formatter_spec.rb b/qa/spec/support/formatters/test_metrics_formatter_spec.rb index 5342cfe12e3..0e8673f1c0f 100644 --- a/qa/spec/support/formatters/test_metrics_formatter_spec.rb +++ b/qa/spec/support/formatters/test_metrics_formatter_spec.rb @@ -22,8 +22,8 @@ describe QA::Support::Formatters::TestMetricsFormatter do let(:quarantined) { 'false' } let(:influx_client) { instance_double('InfluxDB2::Client', create_write_api: influx_write_api) } let(:influx_write_api) { instance_double('InfluxDB2::WriteApi', write: nil) } - let(:stage) { '1_manage' } - let(:file_path) { "./qa/specs/features/#{stage}/subfolder/some_spec.rb" } + let(:file_path) { "./qa/specs/features/1_manage/subfolder/some_spec.rb" } + let(:rerun_file_path) { "./qa/specs/features/1_manage/subfolder/some_spec.rb" } let(:ui_fabrication) { 0 } let(:api_fabrication) { 0 } let(:fabrication_resources) { {} } @@ -53,7 +53,7 @@ describe QA::Support::Formatters::TestMetricsFormatter do job_name: 'test-job', merge_request: 'false', run_type: run_type, - stage: stage.match(%r{\d{1,2}_(\w+)}).captures.first, + stage: 'manage', product_group: product_group, testcase: testcase }, @@ -78,7 +78,10 @@ describe QA::Support::Formatters::TestMetricsFormatter do spec ||= -> { it('spec', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') {} } describe_successfully('stats export', &spec).tap do |example_group| - example_group.examples.each { |ex| ex.metadata[:file_path] = file_path } + example_group.examples.each do |ex| + ex.metadata[:file_path] = file_path + ex.metadata[:rerun_file_path] = rerun_file_path + end end send_stop_notification end @@ -192,7 +195,7 @@ describe QA::Support::Formatters::TestMetricsFormatter do context 'with context quarantined spec' do let(:quarantined) { 'false' } - it 'exports data to influxdb with correct qurantine tag' do + it 'exports data to influxdb with correct quarantine tag' do run_spec do it( 'spec', @@ -294,6 +297,19 @@ describe QA::Support::Formatters::TestMetricsFormatter do end end + context 'with a shared example' do + let(:file_path) { './qa/specs/features/shared_examples/merge_with_code_owner_shared_examples.rb' } + let(:rerun_file_path) { './qa/specs/features/3_create/subfolder/another_spec.rb' } + + it 'exports data to influxdb with correct filename' do + run_spec + + data[:tags][:file_path] = '/3_create/subfolder/another_spec.rb' + data[:tags][:stage] = 'create' + expect(influx_write_api).to have_received(:write).with(data: [data]) + end + end + context 'with fabrication resources' do let(:fabrication_resources) do { diff --git a/qa/spec/tools/reliable_report_spec.rb b/qa/spec/tools/reliable_report_spec.rb index caef8f89bd5..cf5c9dea794 100644 --- a/qa/spec/tools/reliable_report_spec.rb +++ b/qa/spec/tools/reliable_report_spec.rb @@ -3,273 +3,328 @@ describe QA::Tools::ReliableReport do include QA::Support::Helpers::StubEnv - subject(:run) { described_class.run(range: range, report_in_issue_and_slack: create_issue) } - - let(:slack_notifier) { instance_double("Slack::Notifier", post: nil) } - let(:influx_client) { instance_double("InfluxDB2::Client", create_query_api: query_api) } - let(:query_api) { instance_double("InfluxDB2::QueryApi") } - - let(:slack_channel) { "#quality-reports" } - let(:range) { 14 } - let(:issue_url) { "https://gitlab.com/issue/1" } - let(:time) { "2021-12-07T04:05:25.000000000+00:00" } - - let(:runs) do - values = { - "name" => "stable spec", - "status" => "passed", - "file_path" => "some/spec.rb", - "stage" => "manage", - "_time" => time - } - [ - instance_double( - "InfluxDB2::FluxTable", - records: [ - instance_double("InfluxDB2::FluxRecord", values: values), - instance_double("InfluxDB2::FluxRecord", values: values), - instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s })) - ] - ) - ] - end - - let(:reliable_runs) do - values = { - "name" => "unstable spec", - "status" => "failed", - "file_path" => "some/spec.rb", - "stage" => "create", - "failure_exception" => "failure message", - "job_url" => "https://job/url", - "_time" => time - } - [ - instance_double( - "InfluxDB2::FluxTable", - records: [ - instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }), - instance_double("InfluxDB2::FluxRecord", values: values), - instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s })) - ] - ) - ] - end - - def flux_query(reliable:) - <<~QUERY - from(bucket: "e2e-test-stats-main") - |> range(start: -#{range}d) - |> filter(fn: (r) => r._measurement == "test-stats") - |> filter(fn: (r) => r.run_type == "staging-full" or - r.run_type == "staging-sanity" or - r.run_type == "production-full" or - r.run_type == "production-sanity" or - r.run_type == "package-and-qa" or - r.run_type == "nightly" - ) - |> filter(fn: (r) => r.job_name != "airgapped" and - r.job_name != "instance-image-slow-network" and - r.job_name != "nplus1-instance-image" - ) - |> filter(fn: (r) => r.status != "pending" and - r.merge_request == "false" and - r.quarantined == "false" and - r.smoke == "false" and - r.reliable == "#{reliable}" - ) - |> filter(fn: (r) => r["_field"] == "job_url" or - r["_field"] == "failure_exception" or - r["_field"] == "id" + describe '.run' do + subject(:run) { described_class.run(range: range, report_in_issue_and_slack: create_issue) } + + let(:slack_notifier) { instance_double("Slack::Notifier", post: nil) } + let(:influx_client) { instance_double("InfluxDB2::Client", create_query_api: query_api) } + let(:query_api) { instance_double("InfluxDB2::QueryApi") } + + let(:slack_channel) { "#quality-reports" } + let(:range) { 14 } + let(:issue_url) { "https://gitlab.com/issue/1" } + let(:time) { "2021-12-07T04:05:25.000000000+00:00" } + let(:failure_message) { 'random failure message' } + + let(:runs) do + values = { + "name" => "stable spec", + "status" => "passed", + "file_path" => "some/spec.rb", + "stage" => "manage", + "_time" => time + } + [ + instance_double( + "InfluxDB2::FluxTable", + records: [ + instance_double("InfluxDB2::FluxRecord", values: values), + instance_double("InfluxDB2::FluxRecord", values: values), + instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s })) + ] ) - |> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value") - |> group(columns: ["name"]) - QUERY - end - - def markdown_section(summary, result, stage, type) - <<~SECTION.strip - #{summary_table(summary, type, true)} - - ## #{stage} (1) - - <details> - <summary>Executions table</summary> - - #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days", true)} - - </details> - SECTION - end - - def summary_table(summary, type, markdown = false) - table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50), markdown) - end - - def table(rows, headings, title, markdown = false) - Terminal::Table.new( - headings: headings, - title: markdown ? nil : title, - rows: rows, - style: markdown ? { border: :markdown } : { all_separators: true } - ) - end - - def name_column(spec_name, exceptions_and_job_urls = {}) - "**name**: #{spec_name}<br>**file**: spec.rb#{exceptions_markdown(exceptions_and_job_urls)}" - end + ] + end - def exceptions_markdown(exceptions_and_job_urls) - exceptions_and_job_urls.empty? ? '' : "<br>**Exceptions**:<br>- [`failure message`](https://job/url)" - end + let(:reliable_runs) do + values = { + "name" => "unstable spec", + "status" => "failed", + "file_path" => "some/spec.rb", + "stage" => "create", + "failure_exception" => failure_message, + "job_url" => "https://job/url", + "_time" => time + } + [ + instance_double( + "InfluxDB2::FluxTable", + records: [ + instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }), + instance_double("InfluxDB2::FluxRecord", values: values), + instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s })) + ] + ) + ] + end - before do - stub_env("QA_INFLUXDB_URL", "url") - stub_env("QA_INFLUXDB_TOKEN", "token") - stub_env("SLACK_WEBHOOK", "slack_url") - stub_env("CI_API_V4_URL", "gitlab_api_url") - stub_env("GITLAB_ACCESS_TOKEN", "gitlab_token") + def flux_query(reliable:) + <<~QUERY + from(bucket: "e2e-test-stats-main") + |> range(start: -#{range}d) + |> filter(fn: (r) => r._measurement == "test-stats") + |> filter(fn: (r) => r.run_type == "staging-full" or + r.run_type == "staging-sanity" or + r.run_type == "production-full" or + r.run_type == "production-sanity" or + r.run_type == "package-and-qa" or + r.run_type == "nightly" + ) + |> filter(fn: (r) => r.job_name != "airgapped" and + r.job_name != "instance-image-slow-network" and + r.job_name != "nplus1-instance-image" + ) + |> filter(fn: (r) => r.status != "pending" and + r.merge_request == "false" and + r.quarantined == "false" and + r.smoke == "false" and + r.reliable == "#{reliable}" + ) + |> filter(fn: (r) => r["_field"] == "job_url" or + r["_field"] == "failure_exception" or + r["_field"] == "id" + ) + |> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value") + |> group(columns: ["name"]) + QUERY + end - allow(RestClient::Request).to receive(:execute) - allow(Slack::Notifier).to receive(:new).and_return(slack_notifier) - allow(InfluxDB2::Client).to receive(:new).and_return(influx_client) + def markdown_section(summary, result, stage, type) + <<~SECTION.strip + #{summary_table(summary, type, true)} - allow(query_api).to receive(:query).with(query: flux_query(reliable: false)).and_return(runs) - allow(query_api).to receive(:query).with(query: flux_query(reliable: true)).and_return(reliable_runs) - end + ## #{stage} (1) - context "without report creation" do - let(:create_issue) { "false" } + <details> + <summary>Executions table</summary> - it "does not create report issue", :aggregate_failures do - expect { run }.to output.to_stdout + #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days", true)} - expect(RestClient::Request).not_to have_received(:execute) - expect(slack_notifier).not_to have_received(:post) + </details> + SECTION end - end - context "with report creation" do - let(:create_issue) { "true" } - let(:iid) { 2 } - let(:old_iid) { 1 } - let(:issue_endpoint) { "gitlab_api_url/projects/278964/issues" } - - let(:common_api_args) do - { - verify_ssl: false, - headers: { "PRIVATE-TOKEN" => "gitlab_token" }, - cookies: {} - } + def summary_table(summary, type, markdown = false) + table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50), markdown) end - let(:create_issue_response) do - instance_double( - "RestClient::Response", - code: 200, - body: { web_url: issue_url, iid: iid }.to_json + def table(rows, headings, title, markdown = false) + Terminal::Table.new( + headings: headings, + title: markdown ? nil : title, + rows: rows, + style: markdown ? { border: :markdown } : { all_separators: true } ) end - let(:open_issues_response) do - instance_double( - "RestClient::Response", - code: 200, - body: [{ web_url: issue_url, iid: iid }, { web_url: issue_url, iid: old_iid }].to_json - ) + def name_column(spec_name, exceptions_and_job_urls = {}) + "**Name**: #{spec_name}<br>**File**: [spec.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/some/spec.rb)#{exceptions_markdown(exceptions_and_job_urls)}" end - let(:success_response) do - instance_double("RestClient::Response", code: 200, body: {}.to_json) + def exceptions_markdown(exceptions_and_job_urls) + exceptions_and_job_urls.empty? ? '' : "<br>**Exceptions**:<br>- [`#{failure_message}`](https://job/url)" end - let(:issue_body) do - <<~TXT.strip - [[_TOC_]] - - # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today}) + before do + stub_env("QA_INFLUXDB_URL", "url") + stub_env("QA_INFLUXDB_TOKEN", "token") + stub_env("SLACK_WEBHOOK", "slack_url") + stub_env("CI_API_V4_URL", "gitlab_api_url") + stub_env("GITLAB_ACCESS_TOKEN", "gitlab_token") + + allow(RestClient::Request).to receive(:execute) + allow(Slack::Notifier).to receive(:new).and_return(slack_notifier) + allow(InfluxDB2::Client).to receive(:new).and_return(influx_client) + + allow(query_api).to receive(:query).with(query: flux_query(reliable: false)).and_return(runs) + allow(query_api).to receive(:query).with(query: flux_query(reliable: true)).and_return(reliable_runs) + end - Total amount: **1** + context "without report creation" do + let(:create_issue) { "false" } - #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')} + it "does not create report issue", :aggregate_failures do + expect { run }.to output.to_stdout - # Reliable specs with failures (#{Date.today - range} - #{Date.today}) + expect(RestClient::Request).not_to have_received(:execute) + expect(slack_notifier).not_to have_received(:post) + end + end - Total amount: **1** + context "with report creation" do + let(:create_issue) { "true" } + let(:iid) { 2 } + let(:old_iid) { 1 } + let(:issue_endpoint) { "gitlab_api_url/projects/278964/issues" } + + let(:common_api_args) do + { + verify_ssl: false, + headers: { "PRIVATE-TOKEN" => "gitlab_token" }, + cookies: {} + } + end + + let(:create_issue_response) do + instance_double( + "RestClient::Response", + code: 200, + body: { web_url: issue_url, iid: iid }.to_json + ) + end - #{markdown_section([['create', 1]], [[name_column('unstable spec', { 'failure message' => 'https://job/url' }), 3, 2, '66.67%']], 'create', 'unstable')} - TXT + let(:open_issues_response) do + instance_double( + "RestClient::Response", + code: 200, + body: [{ web_url: issue_url, iid: iid }, { web_url: issue_url, iid: old_iid }].to_json + ) + end + + let(:success_response) do + instance_double("RestClient::Response", code: 200, body: {}.to_json) + end + + before do + allow(RestClient::Request).to receive(:execute).exactly(4).times.and_return( + create_issue_response, + open_issues_response, + success_response, + success_response + ) + end + + shared_examples 'report creation' do + it "creates report issue" do + expect { run }.to output.to_stdout + + expect(RestClient::Request).to have_received(:execute).with( + method: :post, + url: issue_endpoint, + payload: { + title: "Reliable e2e test report", + description: expected_issue_body, + labels: "reliable test report,Quality,test,type::maintenance,automation:ml" + }, + **common_api_args + ) + expect(RestClient::Request).to have_received(:execute).with( + method: :get, + url: "#{issue_endpoint}?labels=reliable test report&state=opened", + **common_api_args + ) + expect(RestClient::Request).to have_received(:execute).with( + method: :put, + url: "#{issue_endpoint}/#{old_iid}", + payload: { + state_event: "close" + }, + **common_api_args + ) + expect(RestClient::Request).to have_received(:execute).with( + method: :post, + url: "#{issue_endpoint}/#{old_iid}/notes", + payload: { + body: "Closed issue in favor of ##{iid}" + }, + **common_api_args + ) + expect(slack_notifier).to have_received(:post).with( + icon_emoji: ":tanuki-protect:", + text: expected_slack_text + ) + end + end + + context "with disallowed exception" do + let(:failure_message) { 'random failure message' } + + let(:expected_issue_body) do + <<~TXT.strip + [[_TOC_]] + + # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today}) + + Total amount: **1** + + #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')} + + # Reliable specs with failures (#{Date.today - range} - #{Date.today}) + + Total amount: **1** + + #{markdown_section([['create', 1]], [[name_column('unstable spec', { failure_message => 'https://job/url' }), 3, 2, '66.67%']], 'create', 'unstable')} + TXT + end + + let(:expected_slack_text) do + <<~TEXT + ```#{summary_table([['manage', 1]], 'stable')}``` + ```#{summary_table([['create', 1]], 'unstable')}``` + + #{issue_url} + TEXT + end + + it_behaves_like "report creation" + end + + context "with allowed exception" do + let(:failure_message) { 'Ambiguous match' } + + let(:expected_issue_body) do + <<~TXT.strip + [[_TOC_]] + + # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today}) + + Total amount: **1** + + #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')} + TXT + end + + let(:expected_slack_text) do + <<~TEXT + ```#{summary_table([['manage', 1]], 'stable')}``` + ```#{summary_table([], 'unstable')}``` + + #{issue_url} + TEXT + end + + it_behaves_like "report creation" + end end - before do - allow(RestClient::Request).to receive(:execute).exactly(4).times.and_return( - create_issue_response, - open_issues_response, - success_response, - success_response - ) - end + context "with failure" do + let(:create_issue) { "true" } - it "creates report issue" do - expect { run }.to output.to_stdout - - expect(RestClient::Request).to have_received(:execute).with( - method: :post, - url: issue_endpoint, - payload: { - title: "Reliable e2e test report", - description: issue_body, - labels: "reliable test report,Quality,test,type::maintenance,automation:ml" - }, - **common_api_args - ) - expect(RestClient::Request).to have_received(:execute).with( - method: :get, - url: "#{issue_endpoint}?labels=reliable test report&state=opened", - **common_api_args - ) - expect(RestClient::Request).to have_received(:execute).with( - method: :put, - url: "#{issue_endpoint}/#{old_iid}", - payload: { - state_event: "close" - }, - **common_api_args - ) - expect(RestClient::Request).to have_received(:execute).with( - method: :post, - url: "#{issue_endpoint}/#{old_iid}/notes", - payload: { - body: "Closed issue in favor of ##{iid}" - }, - **common_api_args - ) - expect(slack_notifier).to have_received(:post).with( - icon_emoji: ":tanuki-protect:", - text: <<~TEXT - ```#{summary_table([['manage', 1]], 'stable')}``` - ```#{summary_table([['create', 1]], 'unstable')}``` - - #{issue_url} - TEXT - ) + before do + allow(query_api).to receive(:query).and_raise("Connection error!") + end + + it "notifies failure", :aggregate_failures do + expect { expect { run }.to raise_error("Connection error!") }.to output.to_stdout + + expect(slack_notifier).to have_received(:post).with( + icon_emoji: ":sadpanda:", + text: "Reliable reporter failed to create report. Error: ```Connection error!```" + ) + end end end - context "with failure" do - let(:create_issue) { "true" } + describe "#allowed_failure?" do + subject(:reliable_report) { described_class.new(14) } - before do - allow(query_api).to receive(:query).and_raise("Connection error!") + it "returns true for an allowed failure" do + expect(reliable_report.send(:allowed_failure?, "Couldn't find option named abc")).to be true end - it "notifies failure", :aggregate_failures do - expect { expect { run }.to raise_error("Connection error!") }.to output.to_stdout - - expect(slack_notifier).to have_received(:post).with( - icon_emoji: ":sadpanda:", - text: "Reliable reporter failed to create report. Error: ```Connection error!```" - ) + it "returns false for disallowed failure" do + expect(reliable_report.send(:allowed_failure?, + %q([Unable to find css "[data-testid=\"user_action_dropdown\"]"]))).to be false end end end |