Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/qa
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2021-09-07 15:11:26 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2021-09-07 15:11:26 +0300
commit325245c6f5803227b13051883d00da5b3c235ab0 (patch)
treedc30fdeb752d4699a27c2b45ba5992e77bf66ccc /qa
parent378308b6cde44eb1a320b9202ff8946a911f35f6 (diff)
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'qa')
-rw-r--r--qa/Gemfile1
-rw-r--r--qa/Gemfile.lock2
-rw-r--r--qa/qa/resource/group_base.rb16
-rw-r--r--qa/qa/resource/group_milestone.rb67
-rw-r--r--qa/qa/resource/label_base.rb4
-rw-r--r--qa/qa/runtime/allure_report.rb2
-rw-r--r--qa/qa/runtime/env.rb4
-rw-r--r--qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb104
-rw-r--r--qa/qa/specs/helpers/rspec.rb15
-rw-r--r--qa/qa/support/allure_metadata_formatter.rb34
-rw-r--r--qa/qa/support/formatters/allure_metadata_formatter.rb33
-rw-r--r--qa/qa/support/formatters/context_formatter.rb (renamed from qa/qa/specs/helpers/context_formatter.rb)9
-rw-r--r--qa/qa/support/formatters/formatters.rb11
-rw-r--r--qa/qa/support/formatters/quarantine_formatter.rb (renamed from qa/qa/specs/helpers/quarantine_formatter.rb)9
-rw-r--r--qa/qa/support/formatters/test_stats_formatter.rb119
-rw-r--r--qa/spec/spec_helper.rb8
-rw-r--r--qa/spec/specs/allure_report_spec.rb3
-rw-r--r--qa/spec/specs/helpers/context_selector_spec.rb2
-rw-r--r--qa/spec/specs/helpers/quarantine_spec.rb2
-rw-r--r--qa/spec/support/formatters/allure_metadata_formatter_spec.rb (renamed from qa/spec/support/allure_metadata_formatter_spec.rb)2
-rw-r--r--qa/spec/support/formatters/test_stats_formatter_spec.rb138
21 files changed, 471 insertions, 114 deletions
diff --git a/qa/Gemfile b/qa/Gemfile
index d3d0fe95e49..cc2355cdfa3 100644
--- a/qa/Gemfile
+++ b/qa/Gemfile
@@ -24,6 +24,7 @@ gem 'rspec-parameterized', '~> 0.4.2'
gem 'octokit', '~> 4.21'
gem 'webdrivers', '~> 4.6'
gem 'zeitwerk', '~> 2.4'
+gem 'influxdb-client', '~> 1.17'
gem 'chemlab', '~> 0.7'
gem 'chemlab-library-www-gitlab-com', '~> 0.1'
diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock
index 1df4550a2ba..5f33afaa77b 100644
--- a/qa/Gemfile.lock
+++ b/qa/Gemfile.lock
@@ -88,6 +88,7 @@ GEM
i18n (1.8.10)
concurrent-ruby (~> 1.0)
ice_nine (0.11.2)
+ influxdb-client (1.17.0)
knapsack (1.17.1)
rake
launchy (2.4.3)
@@ -220,6 +221,7 @@ DEPENDENCIES
deprecation_toolkit (~> 1.5.1)
faker (~> 2.19, >= 2.19.0)
gitlab-qa
+ influxdb-client (~> 1.17)
knapsack (~> 1.17)
octokit (~> 4.21)
parallel (~> 1.19)
diff --git a/qa/qa/resource/group_base.rb b/qa/qa/resource/group_base.rb
index b937b704613..a1e5b19f409 100644
--- a/qa/qa/resource/group_base.rb
+++ b/qa/qa/resource/group_base.rb
@@ -30,6 +30,22 @@ module QA
end
end
+ # Get group milestones
+ #
+ # @return [Array<QA::Resource::GroupMilestone>]
+ def milestones
+ parse_body(api_get_from("#{api_get_path}/milestones")).map do |milestone|
+ GroupMilestone.init do |resource|
+ resource.api_client = api_client
+ resource.group = self
+ resource.id = milestone[:id]
+ resource.iid = milestone[:iid]
+ resource.title = milestone[:title]
+ resource.description = milestone[:description]
+ end
+ end
+ end
+
# API get path
#
# @return [String]
diff --git a/qa/qa/resource/group_milestone.rb b/qa/qa/resource/group_milestone.rb
index 1fb07fdbd0b..880ca2b9721 100644
--- a/qa/qa/resource/group_milestone.rb
+++ b/qa/qa/resource/group_milestone.rb
@@ -3,11 +3,14 @@
module QA
module Resource
class GroupMilestone < Base
- attr_writer :start_date, :due_date
-
- attribute :id
- attribute :title
- attribute :description
+ attributes :id,
+ :iid,
+ :title,
+ :description,
+ :start_date,
+ :due_date,
+ :updated_at,
+ :created_at
attribute :group do
Group.fabricate_via_api! do |resource|
@@ -20,6 +23,21 @@ module QA
@description = "My awesome group milestone."
end
+ def fabricate!
+ group.visit!
+
+ Page::Group::Menu.perform(&:go_to_milestones)
+ Page::Group::Milestone::Index.perform(&:click_new_milestone_link)
+
+ Page::Group::Milestone::New.perform do |new_milestone|
+ new_milestone.set_title(@title)
+ new_milestone.set_description(@description)
+ new_milestone.set_start_date(@start_date) if @start_date
+ new_milestone.set_due_date(@due_date) if @due_date
+ new_milestone.click_create_milestone_button
+ end
+ end
+
def api_get_path
"/groups/#{group.id}/milestones/#{id}"
end
@@ -38,19 +56,36 @@ module QA
end
end
- def fabricate!
- group.visit!
+ # Object comparison
+ #
+ # @param [QA::Resource::GroupMilestone] other
+ # @return [Boolean]
+ def ==(other)
+ other.is_a?(GroupMilestone) && comparable_milestone == other.comparable_milestone
+ end
- Page::Group::Menu.perform(&:go_to_milestones)
- Page::Group::Milestone::Index.perform(&:click_new_milestone_link)
+ # Override inspect for a better rspec failure diff output
+ #
+ # @return [String]
+ def inspect
+ JSON.pretty_generate(comparable_milestone)
+ end
- Page::Group::Milestone::New.perform do |new_milestone|
- new_milestone.set_title(@title)
- new_milestone.set_description(@description)
- new_milestone.set_start_date(@start_date) if @start_date
- new_milestone.set_due_date(@due_date) if @due_date
- new_milestone.click_create_milestone_button
- end
+ protected
+
+ # Return subset of fields for comparing milestones
+ #
+ # @return [Hash]
+ def comparable_milestone
+ reload! unless api_response
+
+ api_response.slice(
+ :title,
+ :description,
+ :state,
+ :due_date,
+ :start_date
+ )
end
end
end
diff --git a/qa/qa/resource/label_base.rb b/qa/qa/resource/label_base.rb
index 8f6534cb451..b1af0e23561 100644
--- a/qa/qa/resource/label_base.rb
+++ b/qa/qa/resource/label_base.rb
@@ -64,9 +64,9 @@ module QA
JSON.pretty_generate(comparable_label)
end
- # protected
+ protected
- # Return subset of fields for comparing groups
+ # Return subset of fields for comparing labels
#
# @return [Hash]
def comparable_label
diff --git a/qa/qa/runtime/allure_report.rb b/qa/qa/runtime/allure_report.rb
index 0630e9d333c..5b0456dc607 100644
--- a/qa/qa/runtime/allure_report.rb
+++ b/qa/qa/runtime/allure_report.rb
@@ -73,7 +73,7 @@ module QA
def configure_rspec
RSpec.configure do |config|
config.add_formatter(AllureRspecFormatter)
- config.add_formatter(QA::Support::AllureMetadataFormatter)
+ config.add_formatter(QA::Support::Formatters::AllureMetadataFormatter)
end
end
diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb
index 16c8c4aff3e..cdfa95457c7 100644
--- a/qa/qa/runtime/env.rb
+++ b/qa/qa/runtime/env.rb
@@ -403,6 +403,10 @@ module QA
ENV['GITLAB_TLS_CERTIFICATE']
end
+ def export_metrics?
+ running_in_ci? && enabled?(ENV['QA_EXPORT_TEST_METRICS'], default: true)
+ end
+
private
def remote_grid_credentials
diff --git a/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb b/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb
index 925d50a6639..1422dd5a029 100644
--- a/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb
+++ b/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb
@@ -5,6 +5,7 @@ module QA
describe 'Bulk group import' do
let!(:staging?) { Runtime::Scenario.gitlab_address.include?('staging.gitlab.com') }
+ let(:import_wait_duration) { { max_duration: 300, sleep_interval: 2 } }
let(:admin_api_client) { Runtime::API::Client.as_admin }
let(:user) do
Resource::User.fabricate_via_api! do |usr|
@@ -14,7 +15,6 @@ module QA
end
let(:api_client) { Runtime::API::Client.new(user: user) }
- let(:personal_access_token) { api_client.personal_access_token }
let(:sandbox) do
Resource::Sandbox.fabricate_via_api! do |group|
@@ -29,22 +29,6 @@ module QA
end
end
- let(:subgroup) do
- Resource::Group.fabricate_via_api! do |group|
- group.api_client = api_client
- group.sandbox = source_group
- group.path = "subgroup-for-import-#{SecureRandom.hex(4)}"
- end
- end
-
- let(:imported_subgroup) do
- Resource::Group.init do |group|
- group.api_client = api_client
- group.sandbox = imported_group
- group.path = subgroup.path
- end
- end
-
let(:imported_group) do
Resource::BulkImportGroup.fabricate_via_api! do |group|
group.api_client = api_client
@@ -57,33 +41,79 @@ module QA
Runtime::Feature.enable(:top_level_group_creation_enabled) if staging?
sandbox.add_member(user, Resource::Members::AccessLevel::MAINTAINER)
+ end
- Resource::GroupLabel.fabricate_via_api! do |label|
- label.api_client = api_client
- label.group = source_group
- label.title = "source-group-#{SecureRandom.hex(4)}"
+ context 'with subgroups and labels' do
+ let(:subgroup) do
+ Resource::Group.fabricate_via_api! do |group|
+ group.api_client = api_client
+ group.sandbox = source_group
+ group.path = "subgroup-for-import-#{SecureRandom.hex(4)}"
+ end
end
- Resource::GroupLabel.fabricate_via_api! do |label|
- label.api_client = api_client
- label.group = subgroup
- label.title = "subgroup-#{SecureRandom.hex(4)}"
+
+ let(:imported_subgroup) do
+ Resource::Group.init do |group|
+ group.api_client = api_client
+ group.sandbox = imported_group
+ group.path = subgroup.path
+ end
+ end
+
+ before do
+ Resource::GroupLabel.fabricate_via_api! do |label|
+ label.api_client = api_client
+ label.group = source_group
+ label.title = "source-group-#{SecureRandom.hex(4)}"
+ end
+ Resource::GroupLabel.fabricate_via_api! do |label|
+ label.api_client = api_client
+ label.group = subgroup
+ label.title = "subgroup-#{SecureRandom.hex(4)}"
+ end
+ end
+
+ it(
+ 'successfully imports groups and labels',
+ testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1873'
+ ) do
+ expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration)
+
+ aggregate_failures do
+ expect(imported_group.reload!).to eq(source_group)
+ expect(imported_group.labels).to include(*source_group.labels)
+
+ expect(imported_subgroup.reload!).to eq(subgroup)
+ expect(imported_subgroup.labels).to include(*subgroup.labels)
+ end
end
end
- it(
- 'imports group with subgroups and labels',
- testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1873'
- ) do
- expect { imported_group.import_status }.to(
- eventually_eq('finished').within(max_duration: 300, sleep_interval: 2)
- )
+ context 'with milestones' do
+ let(:source_milestone) do
+ Resource::GroupMilestone.fabricate_via_api! do |milestone|
+ milestone.api_client = api_client
+ milestone.group = source_group
+ end
+ end
+
+ before do
+ source_milestone
+ end
- aggregate_failures do
- expect(imported_group.reload!).to eq(source_group)
- expect(imported_group.labels).to include(*source_group.labels)
+ it(
+ 'successfully imports group milestones',
+ testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/2245'
+ ) do
+ expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration)
- expect(imported_subgroup.reload!).to eq(subgroup)
- expect(imported_subgroup.labels).to include(*subgroup.labels)
+ imported_milestone = imported_group.reload!.milestones.find { |ml| ml.title == source_milestone.title }
+ aggregate_failures do
+ expect(imported_milestone).to eq(source_milestone)
+ expect(imported_milestone.iid).to eq(source_milestone.iid)
+ expect(imported_milestone.created_at).to eq(source_milestone.created_at)
+ expect(imported_milestone.updated_at).to eq(source_milestone.updated_at)
+ end
end
end
diff --git a/qa/qa/specs/helpers/rspec.rb b/qa/qa/specs/helpers/rspec.rb
index 853dfbfd1b6..3e97dbd118a 100644
--- a/qa/qa/specs/helpers/rspec.rb
+++ b/qa/qa/specs/helpers/rspec.rb
@@ -19,13 +19,22 @@ module QA
# expanding into the global state
# See: https://github.com/rspec/rspec-core/issues/2603
def describe_successfully(*args, &describe_body)
- reporter = ::RSpec.configuration.reporter
-
- example_group = RSpec.describe(*args, &describe_body)
+ example_group = ::RSpec.describe(*args, &describe_body)
ran_successfully = example_group.run reporter
expect(ran_successfully).to eq true
example_group
end
+
+ def send_stop_notification
+ reporter.notify(
+ :stop,
+ ::RSpec::Core::Notifications::ExamplesNotification.new(reporter)
+ )
+ end
+
+ def reporter
+ ::RSpec.configuration.reporter
+ end
end
end
end
diff --git a/qa/qa/support/allure_metadata_formatter.rb b/qa/qa/support/allure_metadata_formatter.rb
deleted file mode 100644
index 98b7077b0ae..00000000000
--- a/qa/qa/support/allure_metadata_formatter.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-require 'rspec/core'
-require "rspec/core/formatters/base_formatter"
-
-module QA
- module Support
- class AllureMetadataFormatter < ::RSpec::Core::Formatters::BaseFormatter
- ::RSpec::Core::Formatters.register(
- self,
- :example_started
- )
-
- # Starts example
- # @param [RSpec::Core::Notifications::ExampleNotification] example_notification
- # @return [void]
- def example_started(example_notification)
- example = example_notification.example
-
- quarantine_issue = example.metadata.dig(:quarantine, :issue)
- example.issue('Quarantine issue', quarantine_issue) if quarantine_issue
-
- spec_file = example.file_path.split('/').last
- example.issue(
- 'Failure issues',
- "https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&search=#{spec_file}"
- )
- return unless Runtime::Env.running_in_ci?
-
- example.add_link(name: "Job(#{Runtime::Env.ci_job_name})", url: Runtime::Env.ci_job_url)
- end
- end
- end
-end
diff --git a/qa/qa/support/formatters/allure_metadata_formatter.rb b/qa/qa/support/formatters/allure_metadata_formatter.rb
new file mode 100644
index 00000000000..10769ba5c57
--- /dev/null
+++ b/qa/qa/support/formatters/allure_metadata_formatter.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module QA
+ module Support
+ module Formatters
+ class AllureMetadataFormatter < ::RSpec::Core::Formatters::BaseFormatter
+ ::RSpec::Core::Formatters.register(
+ self,
+ :example_started
+ )
+
+ # Starts example
+ # @param [RSpec::Core::Notifications::ExampleNotification] example_notification
+ # @return [void]
+ def example_started(example_notification)
+ example = example_notification.example
+
+ quarantine_issue = example.metadata.dig(:quarantine, :issue)
+ example.issue('Quarantine issue', quarantine_issue) if quarantine_issue
+
+ spec_file = example.file_path.split('/').last
+ example.issue(
+ 'Failure issues',
+ "https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&search=#{spec_file}"
+ )
+ return unless Runtime::Env.running_in_ci?
+
+ example.add_link(name: "Job(#{Runtime::Env.ci_job_name})", url: Runtime::Env.ci_job_url)
+ end
+ end
+ end
+ end
+end
diff --git a/qa/qa/specs/helpers/context_formatter.rb b/qa/qa/support/formatters/context_formatter.rb
index 26db7c3b67e..c8991561f45 100644
--- a/qa/qa/specs/helpers/context_formatter.rb
+++ b/qa/qa/support/formatters/context_formatter.rb
@@ -1,13 +1,10 @@
# frozen_string_literal: true
-require 'rspec/core'
-require "rspec/core/formatters/base_formatter"
-
module QA
- module Specs
- module Helpers
+ module Support
+ module Formatters
class ContextFormatter < ::RSpec::Core::Formatters::BaseFormatter
- include ContextSelector
+ include Specs::Helpers::ContextSelector
::RSpec::Core::Formatters.register(
self,
diff --git a/qa/qa/support/formatters/formatters.rb b/qa/qa/support/formatters/formatters.rb
new file mode 100644
index 00000000000..f0abf98001f
--- /dev/null
+++ b/qa/qa/support/formatters/formatters.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'rspec/core'
+require 'rspec/core/formatters/base_formatter'
+
+module QA
+ module Support
+ module Formatters
+ end
+ end
+end
diff --git a/qa/qa/specs/helpers/quarantine_formatter.rb b/qa/qa/support/formatters/quarantine_formatter.rb
index c42debee07c..c5d16988dbd 100644
--- a/qa/qa/specs/helpers/quarantine_formatter.rb
+++ b/qa/qa/support/formatters/quarantine_formatter.rb
@@ -1,13 +1,10 @@
# frozen_string_literal: true
-require 'rspec/core'
-require "rspec/core/formatters/base_formatter"
-
module QA
- module Specs
- module Helpers
+ module Support
+ module Formatters
class QuarantineFormatter < ::RSpec::Core::Formatters::BaseFormatter
- include Quarantine
+ include Specs::Helpers::Quarantine
::RSpec::Core::Formatters.register(
self,
diff --git a/qa/qa/support/formatters/test_stats_formatter.rb b/qa/qa/support/formatters/test_stats_formatter.rb
new file mode 100644
index 00000000000..91fdc902e2d
--- /dev/null
+++ b/qa/qa/support/formatters/test_stats_formatter.rb
@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module QA
+ module Support
+ module Formatters
+ class TestStatsFormatter < RSpec::Core::Formatters::BaseFormatter
+ RSpec::Core::Formatters.register(self, :stop)
+
+ # Finish test execution
+ #
+ # @param [RSpec::Core::Notifications::ExamplesNotification] notification
+ # @return [void]
+ def stop(notification)
+ return log(:warn, 'Missing QA_INFLUXDB_URL, skipping metrics export!') unless influxdb_url
+ return log(:warn, 'Missing QA_INFLUXDB_TOKEN, skipping metrics export!') unless influxdb_token
+
+ data = notification.examples.map { |example| test_stats(example) }.compact
+ influx_client.create_write_api.write(data: data)
+ log(:info, "Pushed #{data.length} entries to influxdb")
+ rescue StandardError => e
+ log(:error, "Failed to push data to influxdb, error: #{e}")
+ end
+
+ private
+
+ # InfluxDb client
+ #
+ # @return [InfluxDB2::Client]
+ def influx_client
+ @influx_client ||= InfluxDB2::Client.new(
+ influxdb_url,
+ influxdb_token,
+ bucket: 'e2e-test-stats',
+ org: 'gitlab-qa',
+ use_ssl: false,
+ precision: InfluxDB2::WritePrecision::NANOSECOND
+ )
+ end
+
+ # InfluxDb instance url
+ #
+ # @return [String]
+ def influxdb_url
+ @influxdb_url ||= ENV['QA_INFLUXDB_URL']
+ end
+
+ # Influxdb token
+ #
+ # @return [String]
+ def influxdb_token
+ @influxdb_token ||= ENV['QA_INFLUXDB_TOKEN']
+ end
+
+ # Transform example to influxdb compatible metrics data
+ # https://github.com/influxdata/influxdb-client-ruby#data-format
+ #
+ # @param [RSpec::Core::Example] example
+ # @return [Hash]
+ def test_stats(example)
+ {
+ name: 'test-stats',
+ time: time,
+ tags: {
+ name: example.full_description,
+ file_path: example.metadata[:file_path].gsub('./qa/specs/features', ''),
+ status: example.execution_result.status,
+ reliable: example.metadata.key?(:reliable).to_s,
+ quarantined: example.metadata.key?(:quarantine).to_s,
+ retried: ((example.metadata[:retry_attempts] || 0) > 0).to_s,
+ job_name: job_name,
+ merge_request: merge_request,
+ run_type: ENV['QA_RUN_TYPE']
+ },
+ fields: {
+ id: example.id,
+ run_time: (example.execution_result.run_time * 1000).round,
+ retry_attempts: example.metadata[:retry_attempts] || 0,
+ job_url: QA::Runtime::Env.ci_job_url,
+ pipeline_id: ENV['CI_PIPELINE_ID']
+ }
+ }
+ rescue StandardError => e
+ log(:error, "Failed to transform example '#{example.id}', error: #{e}")
+ nil
+ end
+
+ # Single common timestamp for all exported example metrics to keep data points consistently grouped
+ #
+ # @return [Time]
+ def time
+ @time ||= DateTime.strptime(ENV['CI_PIPELINE_CREATED_AT']).to_time
+ end
+
+ # Is a merge request execution
+ #
+ # @return [String]
+ def merge_request
+ @merge_request ||= (!!ENV['CI_MERGE_REQUEST_IID'] || !!ENV['TOP_UPSTREAM_MERGE_REQUEST_IID']).to_s
+ end
+
+ # Base ci job name
+ #
+ # @return [String]
+ def job_name
+ @job_name ||= QA::Runtime::Env.ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '')
+ end
+
+ # Print log message
+ #
+ # @param [Symbol] level
+ # @param [String] message
+ # @return [void]
+ def log(level, message)
+ QA::Runtime::Logger.public_send(level, "influxdb exporter: #{message}")
+ end
+ end
+ end
+ end
+end
diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb
index 6bc889ebc49..4f0f93bf020 100644
--- a/qa/spec/spec_helper.rb
+++ b/qa/spec/spec_helper.rb
@@ -16,17 +16,15 @@ QA::Runtime::Browser.configure!
QA::Runtime::AllureReport.configure!
QA::Runtime::Scenario.from_env(QA::Runtime::Env.runtime_scenario_attributes)
-Dir[::File.join(__dir__, "support/helpers/*.rb")].sort.each { |f| require f }
-Dir[::File.join(__dir__, "support/matchers/*.rb")].sort.each { |f| require f }
-Dir[::File.join(__dir__, "support/shared_contexts/*.rb")].sort.each { |f| require f }
Dir[::File.join(__dir__, "support/shared_examples/*.rb")].sort.each { |f| require f }
RSpec.configure do |config|
config.include QA::Support::Matchers::EventuallyMatcher
config.include QA::Support::Matchers::HaveMatcher
- config.add_formatter QA::Specs::Helpers::ContextFormatter
- config.add_formatter QA::Specs::Helpers::QuarantineFormatter
+ config.add_formatter QA::Support::Formatters::ContextFormatter
+ config.add_formatter QA::Support::Formatters::QuarantineFormatter
+ config.add_formatter QA::Support::Formatters::TestStatsFormatter if QA::Runtime::Env.export_metrics?
config.before do |example|
QA::Runtime::Logger.debug("\nStarting test: #{example.full_description}\n")
diff --git a/qa/spec/specs/allure_report_spec.rb b/qa/spec/specs/allure_report_spec.rb
index d17fb8e41d0..34116ca6cbd 100644
--- a/qa/spec/specs/allure_report_spec.rb
+++ b/qa/spec/specs/allure_report_spec.rb
@@ -68,7 +68,8 @@ describe QA::Runtime::AllureReport do
it 'adds rspec and metadata formatter' do
expect(rspec_config).to have_received(:add_formatter).with(AllureRspecFormatter).ordered
- expect(rspec_config).to have_received(:add_formatter).with(QA::Support::AllureMetadataFormatter).ordered
+ expect(rspec_config).to have_received(:add_formatter)
+ .with(QA::Support::Formatters::AllureMetadataFormatter).ordered
end
it 'configures screenshot saving' do
diff --git a/qa/spec/specs/helpers/context_selector_spec.rb b/qa/spec/specs/helpers/context_selector_spec.rb
index 1492008972d..0152fee6f5b 100644
--- a/qa/spec/specs/helpers/context_selector_spec.rb
+++ b/qa/spec/specs/helpers/context_selector_spec.rb
@@ -10,7 +10,7 @@ RSpec.describe QA::Specs::Helpers::ContextSelector do
QA::Runtime::Scenario.define(:gitlab_address, 'https://staging.gitlab.com')
RSpec::Core::Sandbox.sandboxed do |config|
- config.formatter = QA::Specs::Helpers::ContextFormatter
+ config.formatter = QA::Support::Formatters::ContextFormatter
# If there is an example-within-an-example, we want to make sure the inner example
# does not get a reference to the outer example (the real spec) if it calls
diff --git a/qa/spec/specs/helpers/quarantine_spec.rb b/qa/spec/specs/helpers/quarantine_spec.rb
index f064aa0b9af..8ea375cdb05 100644
--- a/qa/spec/specs/helpers/quarantine_spec.rb
+++ b/qa/spec/specs/helpers/quarantine_spec.rb
@@ -8,7 +8,7 @@ RSpec.describe QA::Specs::Helpers::Quarantine do
around do |ex|
RSpec::Core::Sandbox.sandboxed do |config|
- config.formatter = QA::Specs::Helpers::QuarantineFormatter
+ config.formatter = QA::Support::Formatters::QuarantineFormatter
# If there is an example-within-an-example, we want to make sure the inner example
# does not get a reference to the outer example (the real spec) if it calls
diff --git a/qa/spec/support/allure_metadata_formatter_spec.rb b/qa/spec/support/formatters/allure_metadata_formatter_spec.rb
index f01e5c9f5f8..631d2eda54f 100644
--- a/qa/spec/support/allure_metadata_formatter_spec.rb
+++ b/qa/spec/support/formatters/allure_metadata_formatter_spec.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-describe QA::Support::AllureMetadataFormatter do
+describe QA::Support::Formatters::AllureMetadataFormatter do
include QA::Support::Helpers::StubEnv
let(:formatter) { described_class.new(StringIO.new) }
diff --git a/qa/spec/support/formatters/test_stats_formatter_spec.rb b/qa/spec/support/formatters/test_stats_formatter_spec.rb
new file mode 100644
index 00000000000..30ffa5839e6
--- /dev/null
+++ b/qa/spec/support/formatters/test_stats_formatter_spec.rb
@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require 'rspec/core/sandbox'
+
+describe QA::Support::Formatters::TestStatsFormatter do
+ include QA::Support::Helpers::StubEnv
+ include QA::Specs::Helpers::RSpec
+
+ let(:url) { "http://influxdb.net" }
+ let(:token) { "token" }
+ let(:ci_timestamp) { "2021-02-23T20:58:41Z" }
+ let(:ci_job_name) { "test-job 1/5" }
+ let(:ci_job_url) { "url" }
+ let(:ci_pipeline_id) { "123" }
+ let(:run_type) { 'staging-full' }
+ let(:influx_client) { instance_double('InfluxDB2::Client', create_write_api: influx_write_api) }
+ let(:influx_write_api) { instance_double('InfluxDB2::WriteApi', write: nil) }
+
+ let(:influx_client_args) do
+ {
+ bucket: 'e2e-test-stats',
+ org: 'gitlab-qa',
+ use_ssl: false,
+ precision: InfluxDB2::WritePrecision::NANOSECOND
+ }
+ end
+
+ let(:data) do
+ {
+ name: 'test-stats',
+ time: DateTime.strptime(ci_timestamp).to_time,
+ tags: {
+ name: "stats export #{spec_name}",
+ file_path: './spec/support/formatters/test_stats_formatter_spec.rb',
+ status: :passed,
+ reliable: reliable,
+ quarantined: quarantined,
+ retried: "false",
+ job_name: "test-job",
+ merge_request: "false",
+ run_type: run_type
+ },
+ fields: {
+ id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]',
+ run_time: 0,
+ retry_attempts: 0,
+ job_url: ci_job_url,
+ pipeline_id: ci_pipeline_id
+ }
+ }
+ end
+
+ def run_spec(&spec)
+ describe_successfully('stats export', &spec)
+ send_stop_notification
+ end
+
+ around do |example|
+ RSpec::Core::Sandbox.sandboxed do |config|
+ config.formatter = QA::Support::Formatters::TestStatsFormatter
+
+ config.before(:context) { RSpec.current_example = nil }
+
+ example.run
+ end
+ end
+
+ before do
+ allow(InfluxDB2::Client).to receive(:new).with(url, token, **influx_client_args) { influx_client }
+ end
+
+ context "without influxdb variables configured" do
+ it "skips export without influxdb url" do
+ stub_env('QA_INFLUXDB_URL', nil)
+ stub_env('QA_INFLUXDB_TOKEN', nil)
+
+ run_spec do
+ it('skips export') {}
+ end
+
+ expect(influx_client).not_to have_received(:create_write_api)
+ end
+
+ it "skips export without influxdb token" do
+ stub_env('QA_INFLUXDB_URL', url)
+ stub_env('QA_INFLUXDB_TOKEN', nil)
+
+ run_spec do
+ it('skips export') {}
+ end
+
+ expect(influx_client).not_to have_received(:create_write_api)
+ end
+ end
+
+ context 'with influxdb variables configured' do
+ let(:spec_name) { 'exports data' }
+ let(:run_type) { ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '') }
+
+ before do
+ stub_env('QA_INFLUXDB_URL', url)
+ stub_env('QA_INFLUXDB_TOKEN', token)
+ stub_env('CI_PIPELINE_CREATED_AT', ci_timestamp)
+ stub_env('CI_JOB_URL', ci_job_url)
+ stub_env('CI_JOB_NAME', ci_job_name)
+ stub_env('CI_PIPELINE_ID', ci_pipeline_id)
+ stub_env('CI_MERGE_REQUEST_IID', nil)
+ stub_env('TOP_UPSTREAM_MERGE_REQUEST_IID', nil)
+ stub_env('QA_RUN_TYPE', run_type)
+ end
+
+ context 'with reliable spec' do
+ let(:reliable) { 'true' }
+ let(:quarantined) { 'false' }
+
+ it 'exports data to influxdb' do
+ run_spec do
+ it('exports data', :reliable) {}
+ end
+
+ expect(influx_write_api).to have_received(:write).with(data: [data])
+ end
+ end
+
+ context 'with quarantined spec' do
+ let(:reliable) { 'false' }
+ let(:quarantined) { 'true' }
+
+ it 'exports data to influxdb' do
+ run_spec do
+ it('exports data', :quarantine) {}
+ end
+
+ expect(influx_write_api).to have_received(:write).with(data: [data])
+ end
+ end
+ end
+end