From 325245c6f5803227b13051883d00da5b3c235ab0 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 7 Sep 2021 12:11:26 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- qa/Gemfile | 1 + qa/Gemfile.lock | 2 + qa/qa/resource/group_base.rb | 16 +++ qa/qa/resource/group_milestone.rb | 67 +++++++--- qa/qa/resource/label_base.rb | 4 +- qa/qa/runtime/allure_report.rb | 2 +- qa/qa/runtime/env.rb | 4 + .../api/1_manage/bulk_import_group_spec.rb | 104 ++++++++++------ qa/qa/specs/helpers/context_formatter.rb | 68 ---------- qa/qa/specs/helpers/quarantine_formatter.rb | 45 ------- qa/qa/specs/helpers/rspec.rb | 15 ++- qa/qa/support/allure_metadata_formatter.rb | 34 ----- .../formatters/allure_metadata_formatter.rb | 33 +++++ qa/qa/support/formatters/context_formatter.rb | 65 ++++++++++ qa/qa/support/formatters/formatters.rb | 11 ++ qa/qa/support/formatters/quarantine_formatter.rb | 42 +++++++ qa/qa/support/formatters/test_stats_formatter.rb | 119 ++++++++++++++++++ qa/spec/spec_helper.rb | 8 +- qa/spec/specs/allure_report_spec.rb | 3 +- qa/spec/specs/helpers/context_selector_spec.rb | 2 +- qa/spec/specs/helpers/quarantine_spec.rb | 2 +- qa/spec/support/allure_metadata_formatter_spec.rb | 45 ------- .../formatters/allure_metadata_formatter_spec.rb | 45 +++++++ .../formatters/test_stats_formatter_spec.rb | 138 +++++++++++++++++++++ 24 files changed, 616 insertions(+), 259 deletions(-) delete mode 100644 qa/qa/specs/helpers/context_formatter.rb delete mode 100644 qa/qa/specs/helpers/quarantine_formatter.rb delete mode 100644 qa/qa/support/allure_metadata_formatter.rb create mode 100644 qa/qa/support/formatters/allure_metadata_formatter.rb create mode 100644 qa/qa/support/formatters/context_formatter.rb create mode 100644 qa/qa/support/formatters/formatters.rb create mode 100644 qa/qa/support/formatters/quarantine_formatter.rb create mode 100644 qa/qa/support/formatters/test_stats_formatter.rb delete mode 100644 qa/spec/support/allure_metadata_formatter_spec.rb create mode 100644 qa/spec/support/formatters/allure_metadata_formatter_spec.rb create mode 100644 qa/spec/support/formatters/test_stats_formatter_spec.rb (limited to 'qa') diff --git a/qa/Gemfile b/qa/Gemfile index d3d0fe95e49..cc2355cdfa3 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -24,6 +24,7 @@ gem 'rspec-parameterized', '~> 0.4.2' gem 'octokit', '~> 4.21' gem 'webdrivers', '~> 4.6' gem 'zeitwerk', '~> 2.4' +gem 'influxdb-client', '~> 1.17' gem 'chemlab', '~> 0.7' gem 'chemlab-library-www-gitlab-com', '~> 0.1' diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 1df4550a2ba..5f33afaa77b 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -88,6 +88,7 @@ GEM i18n (1.8.10) concurrent-ruby (~> 1.0) ice_nine (0.11.2) + influxdb-client (1.17.0) knapsack (1.17.1) rake launchy (2.4.3) @@ -220,6 +221,7 @@ DEPENDENCIES deprecation_toolkit (~> 1.5.1) faker (~> 2.19, >= 2.19.0) gitlab-qa + influxdb-client (~> 1.17) knapsack (~> 1.17) octokit (~> 4.21) parallel (~> 1.19) diff --git a/qa/qa/resource/group_base.rb b/qa/qa/resource/group_base.rb index b937b704613..a1e5b19f409 100644 --- a/qa/qa/resource/group_base.rb +++ b/qa/qa/resource/group_base.rb @@ -30,6 +30,22 @@ module QA end end + # Get group milestones + # + # @return [Array] + def milestones + parse_body(api_get_from("#{api_get_path}/milestones")).map do |milestone| + GroupMilestone.init do |resource| + resource.api_client = api_client + resource.group = self + resource.id = milestone[:id] + resource.iid = milestone[:iid] + resource.title = milestone[:title] + resource.description = milestone[:description] + end + end + end + # API get path # # @return [String] diff --git a/qa/qa/resource/group_milestone.rb b/qa/qa/resource/group_milestone.rb index 1fb07fdbd0b..880ca2b9721 100644 --- a/qa/qa/resource/group_milestone.rb +++ b/qa/qa/resource/group_milestone.rb @@ -3,11 +3,14 @@ module QA module Resource class GroupMilestone < Base - attr_writer :start_date, :due_date - - attribute :id - attribute :title - attribute :description + attributes :id, + :iid, + :title, + :description, + :start_date, + :due_date, + :updated_at, + :created_at attribute :group do Group.fabricate_via_api! do |resource| @@ -20,6 +23,21 @@ module QA @description = "My awesome group milestone." end + def fabricate! + group.visit! + + Page::Group::Menu.perform(&:go_to_milestones) + Page::Group::Milestone::Index.perform(&:click_new_milestone_link) + + Page::Group::Milestone::New.perform do |new_milestone| + new_milestone.set_title(@title) + new_milestone.set_description(@description) + new_milestone.set_start_date(@start_date) if @start_date + new_milestone.set_due_date(@due_date) if @due_date + new_milestone.click_create_milestone_button + end + end + def api_get_path "/groups/#{group.id}/milestones/#{id}" end @@ -38,19 +56,36 @@ module QA end end - def fabricate! - group.visit! + # Object comparison + # + # @param [QA::Resource::GroupMilestone] other + # @return [Boolean] + def ==(other) + other.is_a?(GroupMilestone) && comparable_milestone == other.comparable_milestone + end - Page::Group::Menu.perform(&:go_to_milestones) - Page::Group::Milestone::Index.perform(&:click_new_milestone_link) + # Override inspect for a better rspec failure diff output + # + # @return [String] + def inspect + JSON.pretty_generate(comparable_milestone) + end - Page::Group::Milestone::New.perform do |new_milestone| - new_milestone.set_title(@title) - new_milestone.set_description(@description) - new_milestone.set_start_date(@start_date) if @start_date - new_milestone.set_due_date(@due_date) if @due_date - new_milestone.click_create_milestone_button - end + protected + + # Return subset of fields for comparing milestones + # + # @return [Hash] + def comparable_milestone + reload! unless api_response + + api_response.slice( + :title, + :description, + :state, + :due_date, + :start_date + ) end end end diff --git a/qa/qa/resource/label_base.rb b/qa/qa/resource/label_base.rb index 8f6534cb451..b1af0e23561 100644 --- a/qa/qa/resource/label_base.rb +++ b/qa/qa/resource/label_base.rb @@ -64,9 +64,9 @@ module QA JSON.pretty_generate(comparable_label) end - # protected + protected - # Return subset of fields for comparing groups + # Return subset of fields for comparing labels # # @return [Hash] def comparable_label diff --git a/qa/qa/runtime/allure_report.rb b/qa/qa/runtime/allure_report.rb index 0630e9d333c..5b0456dc607 100644 --- a/qa/qa/runtime/allure_report.rb +++ b/qa/qa/runtime/allure_report.rb @@ -73,7 +73,7 @@ module QA def configure_rspec RSpec.configure do |config| config.add_formatter(AllureRspecFormatter) - config.add_formatter(QA::Support::AllureMetadataFormatter) + config.add_formatter(QA::Support::Formatters::AllureMetadataFormatter) end end diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb index 16c8c4aff3e..cdfa95457c7 100644 --- a/qa/qa/runtime/env.rb +++ b/qa/qa/runtime/env.rb @@ -403,6 +403,10 @@ module QA ENV['GITLAB_TLS_CERTIFICATE'] end + def export_metrics? + running_in_ci? && enabled?(ENV['QA_EXPORT_TEST_METRICS'], default: true) + end + private def remote_grid_credentials diff --git a/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb b/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb index 925d50a6639..1422dd5a029 100644 --- a/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb +++ b/qa/qa/specs/features/api/1_manage/bulk_import_group_spec.rb @@ -5,6 +5,7 @@ module QA describe 'Bulk group import' do let!(:staging?) { Runtime::Scenario.gitlab_address.include?('staging.gitlab.com') } + let(:import_wait_duration) { { max_duration: 300, sleep_interval: 2 } } let(:admin_api_client) { Runtime::API::Client.as_admin } let(:user) do Resource::User.fabricate_via_api! do |usr| @@ -14,7 +15,6 @@ module QA end let(:api_client) { Runtime::API::Client.new(user: user) } - let(:personal_access_token) { api_client.personal_access_token } let(:sandbox) do Resource::Sandbox.fabricate_via_api! do |group| @@ -29,22 +29,6 @@ module QA end end - let(:subgroup) do - Resource::Group.fabricate_via_api! do |group| - group.api_client = api_client - group.sandbox = source_group - group.path = "subgroup-for-import-#{SecureRandom.hex(4)}" - end - end - - let(:imported_subgroup) do - Resource::Group.init do |group| - group.api_client = api_client - group.sandbox = imported_group - group.path = subgroup.path - end - end - let(:imported_group) do Resource::BulkImportGroup.fabricate_via_api! do |group| group.api_client = api_client @@ -57,33 +41,79 @@ module QA Runtime::Feature.enable(:top_level_group_creation_enabled) if staging? sandbox.add_member(user, Resource::Members::AccessLevel::MAINTAINER) + end - Resource::GroupLabel.fabricate_via_api! do |label| - label.api_client = api_client - label.group = source_group - label.title = "source-group-#{SecureRandom.hex(4)}" + context 'with subgroups and labels' do + let(:subgroup) do + Resource::Group.fabricate_via_api! do |group| + group.api_client = api_client + group.sandbox = source_group + group.path = "subgroup-for-import-#{SecureRandom.hex(4)}" + end end - Resource::GroupLabel.fabricate_via_api! do |label| - label.api_client = api_client - label.group = subgroup - label.title = "subgroup-#{SecureRandom.hex(4)}" + + let(:imported_subgroup) do + Resource::Group.init do |group| + group.api_client = api_client + group.sandbox = imported_group + group.path = subgroup.path + end + end + + before do + Resource::GroupLabel.fabricate_via_api! do |label| + label.api_client = api_client + label.group = source_group + label.title = "source-group-#{SecureRandom.hex(4)}" + end + Resource::GroupLabel.fabricate_via_api! do |label| + label.api_client = api_client + label.group = subgroup + label.title = "subgroup-#{SecureRandom.hex(4)}" + end + end + + it( + 'successfully imports groups and labels', + testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1873' + ) do + expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration) + + aggregate_failures do + expect(imported_group.reload!).to eq(source_group) + expect(imported_group.labels).to include(*source_group.labels) + + expect(imported_subgroup.reload!).to eq(subgroup) + expect(imported_subgroup.labels).to include(*subgroup.labels) + end end end - it( - 'imports group with subgroups and labels', - testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/1873' - ) do - expect { imported_group.import_status }.to( - eventually_eq('finished').within(max_duration: 300, sleep_interval: 2) - ) + context 'with milestones' do + let(:source_milestone) do + Resource::GroupMilestone.fabricate_via_api! do |milestone| + milestone.api_client = api_client + milestone.group = source_group + end + end + + before do + source_milestone + end - aggregate_failures do - expect(imported_group.reload!).to eq(source_group) - expect(imported_group.labels).to include(*source_group.labels) + it( + 'successfully imports group milestones', + testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/quality/test_cases/2245' + ) do + expect { imported_group.import_status }.to eventually_eq('finished').within(import_wait_duration) - expect(imported_subgroup.reload!).to eq(subgroup) - expect(imported_subgroup.labels).to include(*subgroup.labels) + imported_milestone = imported_group.reload!.milestones.find { |ml| ml.title == source_milestone.title } + aggregate_failures do + expect(imported_milestone).to eq(source_milestone) + expect(imported_milestone.iid).to eq(source_milestone.iid) + expect(imported_milestone.created_at).to eq(source_milestone.created_at) + expect(imported_milestone.updated_at).to eq(source_milestone.updated_at) + end end end diff --git a/qa/qa/specs/helpers/context_formatter.rb b/qa/qa/specs/helpers/context_formatter.rb deleted file mode 100644 index 26db7c3b67e..00000000000 --- a/qa/qa/specs/helpers/context_formatter.rb +++ /dev/null @@ -1,68 +0,0 @@ -# frozen_string_literal: true - -require 'rspec/core' -require "rspec/core/formatters/base_formatter" - -module QA - module Specs - module Helpers - class ContextFormatter < ::RSpec::Core::Formatters::BaseFormatter - include ContextSelector - - ::RSpec::Core::Formatters.register( - self, - :example_group_started, - :example_started - ) - - # Starts example group - # @param [RSpec::Core::Notifications::GroupNotification] example_group_notification - # @return [void] - def example_group_started(example_group_notification) - set_skip_metadata(example_group_notification.group) - end - - # Starts example - # @param [RSpec::Core::Notifications::ExampleNotification] example_notification - # @return [void] - def example_started(example_notification) - example = example_notification.example - - # if skip propagated from example_group, do not reset skip metadata - set_skip_metadata(example_notification.example) unless example.metadata[:skip] - end - - private - - # Skip example_group or example - # - # @param [] example - # @return [void] - def set_skip_metadata(example) - return skip_only(example.metadata) if example.metadata.key?(:only) - return skip_except(example.metadata) if example.metadata.key?(:except) - end - - # Skip based on 'only' condition - # - # @param [Hash] metadata - # @return [void] - def skip_only(metadata) - return if context_matches?(metadata[:only]) - - metadata[:skip] = 'Test is not compatible with this environment or pipeline' - end - - # Skip based on 'except' condition - # - # @param [Hash] metadata - # @return [void] - def skip_except(metadata) - return unless except?(metadata[:except]) - - metadata[:skip] = 'Test is excluded in this job' - end - end - end - end -end diff --git a/qa/qa/specs/helpers/quarantine_formatter.rb b/qa/qa/specs/helpers/quarantine_formatter.rb deleted file mode 100644 index c42debee07c..00000000000 --- a/qa/qa/specs/helpers/quarantine_formatter.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -require 'rspec/core' -require "rspec/core/formatters/base_formatter" - -module QA - module Specs - module Helpers - class QuarantineFormatter < ::RSpec::Core::Formatters::BaseFormatter - include Quarantine - - ::RSpec::Core::Formatters.register( - self, - :example_group_started, - :example_started - ) - - # Starts example group - # @param [RSpec::Core::Notifications::GroupNotification] example_group_notification - # @return [void] - def example_group_started(example_group_notification) - group = example_group_notification.group - - skip_or_run_quarantined_tests_or_contexts(filters, group) - end - - # Starts example - # @param [RSpec::Core::Notifications::ExampleNotification] example_notification - # @return [void] - def example_started(example_notification) - example = example_notification.example - - # if skip propagated from example_group, do not reset skip metadata - skip_or_run_quarantined_tests_or_contexts(filters, example) unless example.metadata[:skip] - end - - private - - def filters - @filters ||= ::RSpec.configuration.inclusion_filter.rules - end - end - end - end -end diff --git a/qa/qa/specs/helpers/rspec.rb b/qa/qa/specs/helpers/rspec.rb index 853dfbfd1b6..3e97dbd118a 100644 --- a/qa/qa/specs/helpers/rspec.rb +++ b/qa/qa/specs/helpers/rspec.rb @@ -19,13 +19,22 @@ module QA # expanding into the global state # See: https://github.com/rspec/rspec-core/issues/2603 def describe_successfully(*args, &describe_body) - reporter = ::RSpec.configuration.reporter - - example_group = RSpec.describe(*args, &describe_body) + example_group = ::RSpec.describe(*args, &describe_body) ran_successfully = example_group.run reporter expect(ran_successfully).to eq true example_group end + + def send_stop_notification + reporter.notify( + :stop, + ::RSpec::Core::Notifications::ExamplesNotification.new(reporter) + ) + end + + def reporter + ::RSpec.configuration.reporter + end end end end diff --git a/qa/qa/support/allure_metadata_formatter.rb b/qa/qa/support/allure_metadata_formatter.rb deleted file mode 100644 index 98b7077b0ae..00000000000 --- a/qa/qa/support/allure_metadata_formatter.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -require 'rspec/core' -require "rspec/core/formatters/base_formatter" - -module QA - module Support - class AllureMetadataFormatter < ::RSpec::Core::Formatters::BaseFormatter - ::RSpec::Core::Formatters.register( - self, - :example_started - ) - - # Starts example - # @param [RSpec::Core::Notifications::ExampleNotification] example_notification - # @return [void] - def example_started(example_notification) - example = example_notification.example - - quarantine_issue = example.metadata.dig(:quarantine, :issue) - example.issue('Quarantine issue', quarantine_issue) if quarantine_issue - - spec_file = example.file_path.split('/').last - example.issue( - 'Failure issues', - "https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&search=#{spec_file}" - ) - return unless Runtime::Env.running_in_ci? - - example.add_link(name: "Job(#{Runtime::Env.ci_job_name})", url: Runtime::Env.ci_job_url) - end - end - end -end diff --git a/qa/qa/support/formatters/allure_metadata_formatter.rb b/qa/qa/support/formatters/allure_metadata_formatter.rb new file mode 100644 index 00000000000..10769ba5c57 --- /dev/null +++ b/qa/qa/support/formatters/allure_metadata_formatter.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module QA + module Support + module Formatters + class AllureMetadataFormatter < ::RSpec::Core::Formatters::BaseFormatter + ::RSpec::Core::Formatters.register( + self, + :example_started + ) + + # Starts example + # @param [RSpec::Core::Notifications::ExampleNotification] example_notification + # @return [void] + def example_started(example_notification) + example = example_notification.example + + quarantine_issue = example.metadata.dig(:quarantine, :issue) + example.issue('Quarantine issue', quarantine_issue) if quarantine_issue + + spec_file = example.file_path.split('/').last + example.issue( + 'Failure issues', + "https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&search=#{spec_file}" + ) + return unless Runtime::Env.running_in_ci? + + example.add_link(name: "Job(#{Runtime::Env.ci_job_name})", url: Runtime::Env.ci_job_url) + end + end + end + end +end diff --git a/qa/qa/support/formatters/context_formatter.rb b/qa/qa/support/formatters/context_formatter.rb new file mode 100644 index 00000000000..c8991561f45 --- /dev/null +++ b/qa/qa/support/formatters/context_formatter.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module QA + module Support + module Formatters + class ContextFormatter < ::RSpec::Core::Formatters::BaseFormatter + include Specs::Helpers::ContextSelector + + ::RSpec::Core::Formatters.register( + self, + :example_group_started, + :example_started + ) + + # Starts example group + # @param [RSpec::Core::Notifications::GroupNotification] example_group_notification + # @return [void] + def example_group_started(example_group_notification) + set_skip_metadata(example_group_notification.group) + end + + # Starts example + # @param [RSpec::Core::Notifications::ExampleNotification] example_notification + # @return [void] + def example_started(example_notification) + example = example_notification.example + + # if skip propagated from example_group, do not reset skip metadata + set_skip_metadata(example_notification.example) unless example.metadata[:skip] + end + + private + + # Skip example_group or example + # + # @param [] example + # @return [void] + def set_skip_metadata(example) + return skip_only(example.metadata) if example.metadata.key?(:only) + return skip_except(example.metadata) if example.metadata.key?(:except) + end + + # Skip based on 'only' condition + # + # @param [Hash] metadata + # @return [void] + def skip_only(metadata) + return if context_matches?(metadata[:only]) + + metadata[:skip] = 'Test is not compatible with this environment or pipeline' + end + + # Skip based on 'except' condition + # + # @param [Hash] metadata + # @return [void] + def skip_except(metadata) + return unless except?(metadata[:except]) + + metadata[:skip] = 'Test is excluded in this job' + end + end + end + end +end diff --git a/qa/qa/support/formatters/formatters.rb b/qa/qa/support/formatters/formatters.rb new file mode 100644 index 00000000000..f0abf98001f --- /dev/null +++ b/qa/qa/support/formatters/formatters.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'rspec/core' +require 'rspec/core/formatters/base_formatter' + +module QA + module Support + module Formatters + end + end +end diff --git a/qa/qa/support/formatters/quarantine_formatter.rb b/qa/qa/support/formatters/quarantine_formatter.rb new file mode 100644 index 00000000000..c5d16988dbd --- /dev/null +++ b/qa/qa/support/formatters/quarantine_formatter.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module QA + module Support + module Formatters + class QuarantineFormatter < ::RSpec::Core::Formatters::BaseFormatter + include Specs::Helpers::Quarantine + + ::RSpec::Core::Formatters.register( + self, + :example_group_started, + :example_started + ) + + # Starts example group + # @param [RSpec::Core::Notifications::GroupNotification] example_group_notification + # @return [void] + def example_group_started(example_group_notification) + group = example_group_notification.group + + skip_or_run_quarantined_tests_or_contexts(filters, group) + end + + # Starts example + # @param [RSpec::Core::Notifications::ExampleNotification] example_notification + # @return [void] + def example_started(example_notification) + example = example_notification.example + + # if skip propagated from example_group, do not reset skip metadata + skip_or_run_quarantined_tests_or_contexts(filters, example) unless example.metadata[:skip] + end + + private + + def filters + @filters ||= ::RSpec.configuration.inclusion_filter.rules + end + end + end + end +end diff --git a/qa/qa/support/formatters/test_stats_formatter.rb b/qa/qa/support/formatters/test_stats_formatter.rb new file mode 100644 index 00000000000..91fdc902e2d --- /dev/null +++ b/qa/qa/support/formatters/test_stats_formatter.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module QA + module Support + module Formatters + class TestStatsFormatter < RSpec::Core::Formatters::BaseFormatter + RSpec::Core::Formatters.register(self, :stop) + + # Finish test execution + # + # @param [RSpec::Core::Notifications::ExamplesNotification] notification + # @return [void] + def stop(notification) + return log(:warn, 'Missing QA_INFLUXDB_URL, skipping metrics export!') unless influxdb_url + return log(:warn, 'Missing QA_INFLUXDB_TOKEN, skipping metrics export!') unless influxdb_token + + data = notification.examples.map { |example| test_stats(example) }.compact + influx_client.create_write_api.write(data: data) + log(:info, "Pushed #{data.length} entries to influxdb") + rescue StandardError => e + log(:error, "Failed to push data to influxdb, error: #{e}") + end + + private + + # InfluxDb client + # + # @return [InfluxDB2::Client] + def influx_client + @influx_client ||= InfluxDB2::Client.new( + influxdb_url, + influxdb_token, + bucket: 'e2e-test-stats', + org: 'gitlab-qa', + use_ssl: false, + precision: InfluxDB2::WritePrecision::NANOSECOND + ) + end + + # InfluxDb instance url + # + # @return [String] + def influxdb_url + @influxdb_url ||= ENV['QA_INFLUXDB_URL'] + end + + # Influxdb token + # + # @return [String] + def influxdb_token + @influxdb_token ||= ENV['QA_INFLUXDB_TOKEN'] + end + + # Transform example to influxdb compatible metrics data + # https://github.com/influxdata/influxdb-client-ruby#data-format + # + # @param [RSpec::Core::Example] example + # @return [Hash] + def test_stats(example) + { + name: 'test-stats', + time: time, + tags: { + name: example.full_description, + file_path: example.metadata[:file_path].gsub('./qa/specs/features', ''), + status: example.execution_result.status, + reliable: example.metadata.key?(:reliable).to_s, + quarantined: example.metadata.key?(:quarantine).to_s, + retried: ((example.metadata[:retry_attempts] || 0) > 0).to_s, + job_name: job_name, + merge_request: merge_request, + run_type: ENV['QA_RUN_TYPE'] + }, + fields: { + id: example.id, + run_time: (example.execution_result.run_time * 1000).round, + retry_attempts: example.metadata[:retry_attempts] || 0, + job_url: QA::Runtime::Env.ci_job_url, + pipeline_id: ENV['CI_PIPELINE_ID'] + } + } + rescue StandardError => e + log(:error, "Failed to transform example '#{example.id}', error: #{e}") + nil + end + + # Single common timestamp for all exported example metrics to keep data points consistently grouped + # + # @return [Time] + def time + @time ||= DateTime.strptime(ENV['CI_PIPELINE_CREATED_AT']).to_time + end + + # Is a merge request execution + # + # @return [String] + def merge_request + @merge_request ||= (!!ENV['CI_MERGE_REQUEST_IID'] || !!ENV['TOP_UPSTREAM_MERGE_REQUEST_IID']).to_s + end + + # Base ci job name + # + # @return [String] + def job_name + @job_name ||= QA::Runtime::Env.ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '') + end + + # Print log message + # + # @param [Symbol] level + # @param [String] message + # @return [void] + def log(level, message) + QA::Runtime::Logger.public_send(level, "influxdb exporter: #{message}") + end + end + end + end +end diff --git a/qa/spec/spec_helper.rb b/qa/spec/spec_helper.rb index 6bc889ebc49..4f0f93bf020 100644 --- a/qa/spec/spec_helper.rb +++ b/qa/spec/spec_helper.rb @@ -16,17 +16,15 @@ QA::Runtime::Browser.configure! QA::Runtime::AllureReport.configure! QA::Runtime::Scenario.from_env(QA::Runtime::Env.runtime_scenario_attributes) -Dir[::File.join(__dir__, "support/helpers/*.rb")].sort.each { |f| require f } -Dir[::File.join(__dir__, "support/matchers/*.rb")].sort.each { |f| require f } -Dir[::File.join(__dir__, "support/shared_contexts/*.rb")].sort.each { |f| require f } Dir[::File.join(__dir__, "support/shared_examples/*.rb")].sort.each { |f| require f } RSpec.configure do |config| config.include QA::Support::Matchers::EventuallyMatcher config.include QA::Support::Matchers::HaveMatcher - config.add_formatter QA::Specs::Helpers::ContextFormatter - config.add_formatter QA::Specs::Helpers::QuarantineFormatter + config.add_formatter QA::Support::Formatters::ContextFormatter + config.add_formatter QA::Support::Formatters::QuarantineFormatter + config.add_formatter QA::Support::Formatters::TestStatsFormatter if QA::Runtime::Env.export_metrics? config.before do |example| QA::Runtime::Logger.debug("\nStarting test: #{example.full_description}\n") diff --git a/qa/spec/specs/allure_report_spec.rb b/qa/spec/specs/allure_report_spec.rb index d17fb8e41d0..34116ca6cbd 100644 --- a/qa/spec/specs/allure_report_spec.rb +++ b/qa/spec/specs/allure_report_spec.rb @@ -68,7 +68,8 @@ describe QA::Runtime::AllureReport do it 'adds rspec and metadata formatter' do expect(rspec_config).to have_received(:add_formatter).with(AllureRspecFormatter).ordered - expect(rspec_config).to have_received(:add_formatter).with(QA::Support::AllureMetadataFormatter).ordered + expect(rspec_config).to have_received(:add_formatter) + .with(QA::Support::Formatters::AllureMetadataFormatter).ordered end it 'configures screenshot saving' do diff --git a/qa/spec/specs/helpers/context_selector_spec.rb b/qa/spec/specs/helpers/context_selector_spec.rb index 1492008972d..0152fee6f5b 100644 --- a/qa/spec/specs/helpers/context_selector_spec.rb +++ b/qa/spec/specs/helpers/context_selector_spec.rb @@ -10,7 +10,7 @@ RSpec.describe QA::Specs::Helpers::ContextSelector do QA::Runtime::Scenario.define(:gitlab_address, 'https://staging.gitlab.com') RSpec::Core::Sandbox.sandboxed do |config| - config.formatter = QA::Specs::Helpers::ContextFormatter + config.formatter = QA::Support::Formatters::ContextFormatter # If there is an example-within-an-example, we want to make sure the inner example # does not get a reference to the outer example (the real spec) if it calls diff --git a/qa/spec/specs/helpers/quarantine_spec.rb b/qa/spec/specs/helpers/quarantine_spec.rb index f064aa0b9af..8ea375cdb05 100644 --- a/qa/spec/specs/helpers/quarantine_spec.rb +++ b/qa/spec/specs/helpers/quarantine_spec.rb @@ -8,7 +8,7 @@ RSpec.describe QA::Specs::Helpers::Quarantine do around do |ex| RSpec::Core::Sandbox.sandboxed do |config| - config.formatter = QA::Specs::Helpers::QuarantineFormatter + config.formatter = QA::Support::Formatters::QuarantineFormatter # If there is an example-within-an-example, we want to make sure the inner example # does not get a reference to the outer example (the real spec) if it calls diff --git a/qa/spec/support/allure_metadata_formatter_spec.rb b/qa/spec/support/allure_metadata_formatter_spec.rb deleted file mode 100644 index f01e5c9f5f8..00000000000 --- a/qa/spec/support/allure_metadata_formatter_spec.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -describe QA::Support::AllureMetadataFormatter do - include QA::Support::Helpers::StubEnv - - let(:formatter) { described_class.new(StringIO.new) } - - let(:rspec_example_notification) { double('RSpec::Core::Notifications::ExampleNotification', example: rspec_example) } - let(:rspec_example) do - double( - 'RSpec::Core::Example', - tms: nil, - issue: nil, - add_link: nil, - attempts: 0, - file_path: 'file/path/spec.rb', - metadata: { - testcase: 'testcase', - quarantine: { issue: 'issue' } - } - ) - end - - let(:ci_job) { 'ee:relative 5' } - let(:ci_job_url) { 'url' } - - before do - stub_env('CI', 'true') - stub_env('CI_JOB_NAME', ci_job) - stub_env('CI_JOB_URL', ci_job_url) - end - - it "adds additional data to report" do - formatter.example_started(rspec_example_notification) - - aggregate_failures do - expect(rspec_example).to have_received(:issue).with('Quarantine issue', 'issue') - expect(rspec_example).to have_received(:add_link).with(name: "Job(#{ci_job})", url: ci_job_url) - expect(rspec_example).to have_received(:issue).with( - 'Failure issues', - 'https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&search=spec.rb' - ) - end - end -end diff --git a/qa/spec/support/formatters/allure_metadata_formatter_spec.rb b/qa/spec/support/formatters/allure_metadata_formatter_spec.rb new file mode 100644 index 00000000000..631d2eda54f --- /dev/null +++ b/qa/spec/support/formatters/allure_metadata_formatter_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +describe QA::Support::Formatters::AllureMetadataFormatter do + include QA::Support::Helpers::StubEnv + + let(:formatter) { described_class.new(StringIO.new) } + + let(:rspec_example_notification) { double('RSpec::Core::Notifications::ExampleNotification', example: rspec_example) } + let(:rspec_example) do + double( + 'RSpec::Core::Example', + tms: nil, + issue: nil, + add_link: nil, + attempts: 0, + file_path: 'file/path/spec.rb', + metadata: { + testcase: 'testcase', + quarantine: { issue: 'issue' } + } + ) + end + + let(:ci_job) { 'ee:relative 5' } + let(:ci_job_url) { 'url' } + + before do + stub_env('CI', 'true') + stub_env('CI_JOB_NAME', ci_job) + stub_env('CI_JOB_URL', ci_job_url) + end + + it "adds additional data to report" do + formatter.example_started(rspec_example_notification) + + aggregate_failures do + expect(rspec_example).to have_received(:issue).with('Quarantine issue', 'issue') + expect(rspec_example).to have_received(:add_link).with(name: "Job(#{ci_job})", url: ci_job_url) + expect(rspec_example).to have_received(:issue).with( + 'Failure issues', + 'https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&search=spec.rb' + ) + end + end +end diff --git a/qa/spec/support/formatters/test_stats_formatter_spec.rb b/qa/spec/support/formatters/test_stats_formatter_spec.rb new file mode 100644 index 00000000000..30ffa5839e6 --- /dev/null +++ b/qa/spec/support/formatters/test_stats_formatter_spec.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +require 'rspec/core/sandbox' + +describe QA::Support::Formatters::TestStatsFormatter do + include QA::Support::Helpers::StubEnv + include QA::Specs::Helpers::RSpec + + let(:url) { "http://influxdb.net" } + let(:token) { "token" } + let(:ci_timestamp) { "2021-02-23T20:58:41Z" } + let(:ci_job_name) { "test-job 1/5" } + let(:ci_job_url) { "url" } + let(:ci_pipeline_id) { "123" } + let(:run_type) { 'staging-full' } + let(:influx_client) { instance_double('InfluxDB2::Client', create_write_api: influx_write_api) } + let(:influx_write_api) { instance_double('InfluxDB2::WriteApi', write: nil) } + + let(:influx_client_args) do + { + bucket: 'e2e-test-stats', + org: 'gitlab-qa', + use_ssl: false, + precision: InfluxDB2::WritePrecision::NANOSECOND + } + end + + let(:data) do + { + name: 'test-stats', + time: DateTime.strptime(ci_timestamp).to_time, + tags: { + name: "stats export #{spec_name}", + file_path: './spec/support/formatters/test_stats_formatter_spec.rb', + status: :passed, + reliable: reliable, + quarantined: quarantined, + retried: "false", + job_name: "test-job", + merge_request: "false", + run_type: run_type + }, + fields: { + id: './spec/support/formatters/test_stats_formatter_spec.rb[1:1]', + run_time: 0, + retry_attempts: 0, + job_url: ci_job_url, + pipeline_id: ci_pipeline_id + } + } + end + + def run_spec(&spec) + describe_successfully('stats export', &spec) + send_stop_notification + end + + around do |example| + RSpec::Core::Sandbox.sandboxed do |config| + config.formatter = QA::Support::Formatters::TestStatsFormatter + + config.before(:context) { RSpec.current_example = nil } + + example.run + end + end + + before do + allow(InfluxDB2::Client).to receive(:new).with(url, token, **influx_client_args) { influx_client } + end + + context "without influxdb variables configured" do + it "skips export without influxdb url" do + stub_env('QA_INFLUXDB_URL', nil) + stub_env('QA_INFLUXDB_TOKEN', nil) + + run_spec do + it('skips export') {} + end + + expect(influx_client).not_to have_received(:create_write_api) + end + + it "skips export without influxdb token" do + stub_env('QA_INFLUXDB_URL', url) + stub_env('QA_INFLUXDB_TOKEN', nil) + + run_spec do + it('skips export') {} + end + + expect(influx_client).not_to have_received(:create_write_api) + end + end + + context 'with influxdb variables configured' do + let(:spec_name) { 'exports data' } + let(:run_type) { ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '') } + + before do + stub_env('QA_INFLUXDB_URL', url) + stub_env('QA_INFLUXDB_TOKEN', token) + stub_env('CI_PIPELINE_CREATED_AT', ci_timestamp) + stub_env('CI_JOB_URL', ci_job_url) + stub_env('CI_JOB_NAME', ci_job_name) + stub_env('CI_PIPELINE_ID', ci_pipeline_id) + stub_env('CI_MERGE_REQUEST_IID', nil) + stub_env('TOP_UPSTREAM_MERGE_REQUEST_IID', nil) + stub_env('QA_RUN_TYPE', run_type) + end + + context 'with reliable spec' do + let(:reliable) { 'true' } + let(:quarantined) { 'false' } + + it 'exports data to influxdb' do + run_spec do + it('exports data', :reliable) {} + end + + expect(influx_write_api).to have_received(:write).with(data: [data]) + end + end + + context 'with quarantined spec' do + let(:reliable) { 'false' } + let(:quarantined) { 'true' } + + it 'exports data to influxdb' do + run_spec do + it('exports data', :quarantine) {} + end + + expect(influx_write_api).to have_received(:write).with(data: [data]) + end + end + end +end -- cgit v1.2.3