diff options
author | Grzegorz Bizon <grzegorz@gitlab.com> | 2017-10-06 14:40:32 +0300 |
---|---|---|
committer | Grzegorz Bizon <grzegorz@gitlab.com> | 2017-10-06 14:40:32 +0300 |
commit | 050548032475458a70005ed3f7ff74211084a423 (patch) | |
tree | eecc704018dbb2d4e0a65643f8a38c3e6d63c42b | |
parent | 05e3526175d04d7205172cabd594f7492a6a2325 (diff) | |
parent | da5aa64f06afd687a97cfcb5136c77d668f8990d (diff) |
Merge branch 'rc/fix-flaky_example' into 'master'
Fix flaky examples tracking
See merge request gitlab-org/gitlab-ce!14681
-rw-r--r-- | .gitlab-ci.yml | 20 | ||||
-rw-r--r-- | lib/rspec_flaky/config.rb | 21 | ||||
-rw-r--r-- | lib/rspec_flaky/flaky_example.rb | 21 | ||||
-rw-r--r-- | lib/rspec_flaky/flaky_examples_collection.rb | 37 | ||||
-rw-r--r-- | lib/rspec_flaky/listener.rb | 63 | ||||
-rw-r--r-- | spec/lib/rspec_flaky/config_spec.rb | 102 | ||||
-rw-r--r-- | spec/lib/rspec_flaky/flaky_example_spec.rb | 129 | ||||
-rw-r--r-- | spec/lib/rspec_flaky/flaky_examples_collection_spec.rb | 79 | ||||
-rw-r--r-- | spec/lib/rspec_flaky/listener_spec.rb | 172 | ||||
-rw-r--r-- | spec/spec_helper.rb | 5 |
10 files changed, 507 insertions, 142 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8501911fde4..fb170515c1d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,7 +27,7 @@ variables: GET_SOURCES_ATTEMPTS: "3" KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json KNAPSACK_SPINACH_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/spinach_report-master.json - FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/${CI_PROJECT_NAME}/report-master.json + FLAKY_RSPEC_SUITE_REPORT_PATH: rspec_flaky/report-suite.json before_script: - bundle --version @@ -87,12 +87,13 @@ stages: - export CI_NODE_TOTAL=${JOB_NAME[-1]} - export KNAPSACK_REPORT_PATH=knapsack/${CI_PROJECT_NAME}/${JOB_NAME[0]}_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export KNAPSACK_GENERATE_REPORT=true - - export ALL_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/${CI_PROJECT_NAME}/all_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - - export NEW_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/${CI_PROJECT_NAME}/new_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json + - export SUITE_FLAKY_RSPEC_REPORT_PATH=${FLAKY_RSPEC_SUITE_REPORT_PATH} + - export FLAKY_RSPEC_REPORT_PATH=rspec_flaky/all_${JOB_NAME[0]}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json + - export NEW_FLAKY_RSPEC_REPORT_PATH=rspec_flaky/new_${JOB_NAME[0]}_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json - export FLAKY_RSPEC_GENERATE_REPORT=true - export CACHE_CLASSES=true - cp ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} ${KNAPSACK_REPORT_PATH} - - cp ${FLAKY_RSPEC_SUITE_REPORT_PATH} ${ALL_FLAKY_RSPEC_REPORT_PATH} + - '[[ -f $FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_REPORT_PATH}' - '[[ -f $NEW_FLAKY_RSPEC_REPORT_PATH ]] || echo "{}" > ${NEW_FLAKY_RSPEC_REPORT_PATH}' - scripts/gitaly-test-spawn - knapsack rspec "--color --format documentation" @@ -233,7 +234,7 @@ retrieve-tests-metadata: - wget -O $KNAPSACK_SPINACH_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$KNAPSACK_SPINACH_SUITE_REPORT_PATH || rm $KNAPSACK_SPINACH_SUITE_REPORT_PATH - '[[ -f $KNAPSACK_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_RSPEC_SUITE_REPORT_PATH}' - '[[ -f $KNAPSACK_SPINACH_SUITE_REPORT_PATH ]] || echo "{}" > ${KNAPSACK_SPINACH_SUITE_REPORT_PATH}' - - mkdir -p rspec_flaky/${CI_PROJECT_NAME}/ + - mkdir -p rspec_flaky/ - wget -O $FLAKY_RSPEC_SUITE_REPORT_PATH http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/$FLAKY_RSPEC_SUITE_REPORT_PATH || rm $FLAKY_RSPEC_SUITE_REPORT_PATH - '[[ -f $FLAKY_RSPEC_SUITE_REPORT_PATH ]] || echo "{}" > ${FLAKY_RSPEC_SUITE_REPORT_PATH}' @@ -252,22 +253,21 @@ update-tests-metadata: - retry gem install fog-aws mime-types - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json - scripts/merge-reports ${KNAPSACK_SPINACH_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/spinach-pg_node_*.json - - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/${CI_PROJECT_NAME}/all_node_*.json + - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $KNAPSACK_RSPEC_SUITE_REPORT_PATH $KNAPSACK_SPINACH_SUITE_REPORT_PATH' - '[[ -z ${TESTS_METADATA_S3_BUCKET} ]] || scripts/sync-reports put $TESTS_METADATA_S3_BUCKET $FLAKY_RSPEC_SUITE_REPORT_PATH' - rm -f knapsack/${CI_PROJECT_NAME}/*_node_*.json - - rm -f rspec_flaky/${CI_PROJECT_NAME}/*_node_*.json + - rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json flaky-examples-check: <<: *dedicated-runner image: ruby:2.3-alpine services: [] before_script: [] - cache: {} variables: SETUP_DB: "false" USE_BUNDLE_INSTALL: "false" - NEW_FLAKY_SPECS_REPORT: rspec_flaky/${CI_PROJECT_NAME}/new_rspec_flaky_examples.json + NEW_FLAKY_SPECS_REPORT: rspec_flaky/report-new.json stage: post-test allow_failure: yes only: @@ -281,7 +281,7 @@ flaky-examples-check: - rspec_flaky/ script: - '[[ -f $NEW_FLAKY_SPECS_REPORT ]] || echo "{}" > ${NEW_FLAKY_SPECS_REPORT}' - - scripts/merge-reports $NEW_FLAKY_SPECS_REPORT rspec_flaky/${CI_PROJECT_NAME}/new_node_*.json + - scripts/merge-reports ${NEW_FLAKY_SPECS_REPORT} rspec_flaky/new_*_*.json - scripts/detect-new-flaky-examples $NEW_FLAKY_SPECS_REPORT setup-test-env: diff --git a/lib/rspec_flaky/config.rb b/lib/rspec_flaky/config.rb new file mode 100644 index 00000000000..a17ae55910e --- /dev/null +++ b/lib/rspec_flaky/config.rb @@ -0,0 +1,21 @@ +require 'json' + +module RspecFlaky + class Config + def self.generate_report? + ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true' + end + + def self.suite_flaky_examples_report_path + ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/suite-report.json") + end + + def self.flaky_examples_report_path + ENV['FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/report.json") + end + + def self.new_flaky_examples_report_path + ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || Rails.root.join("rspec_flaky/new-report.json") + end + end +end diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb index f81fb90e870..6be24014d89 100644 --- a/lib/rspec_flaky/flaky_example.rb +++ b/lib/rspec_flaky/flaky_example.rb @@ -9,24 +9,21 @@ module RspecFlaky line: example.line, description: example.description, last_attempts_count: example.attempts, - flaky_reports: 1) + flaky_reports: 0) else super end end - def first_flaky_at - self[:first_flaky_at] || Time.now - end - - def last_flaky_at - Time.now - end + def update_flakiness!(last_attempts_count: nil) + self.first_flaky_at ||= Time.now + self.last_flaky_at = Time.now + self.flaky_reports += 1 + self.last_attempts_count = last_attempts_count if last_attempts_count - def last_flaky_job - return unless ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID'] - - "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}" + if ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID'] + self.last_flaky_job = "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}" + end end def to_h diff --git a/lib/rspec_flaky/flaky_examples_collection.rb b/lib/rspec_flaky/flaky_examples_collection.rb new file mode 100644 index 00000000000..973c95b0212 --- /dev/null +++ b/lib/rspec_flaky/flaky_examples_collection.rb @@ -0,0 +1,37 @@ +require 'json' + +module RspecFlaky + class FlakyExamplesCollection < SimpleDelegator + def self.from_json(json) + new(JSON.parse(json)) + end + + def initialize(collection = {}) + unless collection.is_a?(Hash) + raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" + end + + collection_of_flaky_examples = + collection.map do |uid, example| + [ + uid, + example.is_a?(RspecFlaky::FlakyExample) ? example : RspecFlaky::FlakyExample.new(example) + ] + end + + super(Hash[collection_of_flaky_examples]) + end + + def to_report + Hash[map { |uid, example| [uid, example.to_h] }].deep_symbolize_keys + end + + def -(other) + unless other.respond_to?(:key) + raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!" + end + + self.class.new(reject { |uid, _| other.key?(uid) }) + end + end +end diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb index ec2fbd9e36c..4a5bfec9967 100644 --- a/lib/rspec_flaky/listener.rb +++ b/lib/rspec_flaky/listener.rb @@ -2,11 +2,15 @@ require 'json' module RspecFlaky class Listener - attr_reader :all_flaky_examples, :new_flaky_examples - - def initialize - @new_flaky_examples = {} - @all_flaky_examples = init_all_flaky_examples + # - suite_flaky_examples: contains all the currently tracked flacky example + # for the whole RSpec suite + # - flaky_examples: contains the examples detected as flaky during the + # current RSpec run + attr_reader :suite_flaky_examples, :flaky_examples + + def initialize(suite_flaky_examples_json = nil) + @flaky_examples = FlakyExamplesCollection.new + @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json) end def example_passed(notification) @@ -14,29 +18,21 @@ module RspecFlaky return unless current_example.attempts > 1 - flaky_example_hash = all_flaky_examples[current_example.uid] - - all_flaky_examples[current_example.uid] = - if flaky_example_hash - FlakyExample.new(flaky_example_hash).tap do |ex| - ex.last_attempts_count = current_example.attempts - ex.flaky_reports += 1 - end - else - FlakyExample.new(current_example).tap do |ex| - new_flaky_examples[current_example.uid] = ex - end - end + flaky_example = suite_flaky_examples.fetch(current_example.uid) { FlakyExample.new(current_example) } + flaky_example.update_flakiness!(last_attempts_count: current_example.attempts) + + flaky_examples[current_example.uid] = flaky_example end def dump_summary(_) - write_report_file(all_flaky_examples, all_flaky_examples_report_path) + write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path) + new_flaky_examples = flaky_examples - suite_flaky_examples if new_flaky_examples.any? Rails.logger.warn "\nNew flaky examples detected:\n" - Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples)) + Rails.logger.warn JSON.pretty_generate(new_flaky_examples.to_report) - write_report_file(new_flaky_examples, new_flaky_examples_report_path) + write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path) end end @@ -46,30 +42,23 @@ module RspecFlaky private - def init_all_flaky_examples - return {} unless File.exist?(all_flaky_examples_report_path) + def init_suite_flaky_examples(suite_flaky_examples_json = nil) + unless suite_flaky_examples_json + return {} unless File.exist?(RspecFlaky::Config.suite_flaky_examples_report_path) - all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path)) + suite_flaky_examples_json = File.read(RspecFlaky::Config.suite_flaky_examples_report_path) + end - Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }] + FlakyExamplesCollection.from_json(suite_flaky_examples_json) end - def write_report_file(examples, file_path) - return unless ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true' + def write_report_file(examples_collection, file_path) + return unless RspecFlaky::Config.generate_report? report_path_dir = File.dirname(file_path) FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir) - File.write(file_path, JSON.pretty_generate(to_report(examples))) - end - - def all_flaky_examples_report_path - @all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] || - Rails.root.join("rspec_flaky/all-report.json") - end - def new_flaky_examples_report_path - @new_flaky_examples_report_path ||= ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || - Rails.root.join("rspec_flaky/new-report.json") + File.write(file_path, JSON.pretty_generate(examples_collection.to_report)) end end end diff --git a/spec/lib/rspec_flaky/config_spec.rb b/spec/lib/rspec_flaky/config_spec.rb new file mode 100644 index 00000000000..83556787e85 --- /dev/null +++ b/spec/lib/rspec_flaky/config_spec.rb @@ -0,0 +1,102 @@ +require 'spec_helper' + +describe RspecFlaky::Config, :aggregate_failures do + before do + # Stub these env variables otherwise specs don't behave the same on the CI + stub_env('FLAKY_RSPEC_GENERATE_REPORT', nil) + stub_env('SUITE_FLAKY_RSPEC_REPORT_PATH', nil) + stub_env('FLAKY_RSPEC_REPORT_PATH', nil) + stub_env('NEW_FLAKY_RSPEC_REPORT_PATH', nil) + end + + describe '.generate_report?' do + context "when ENV['FLAKY_RSPEC_GENERATE_REPORT'] is not set" do + it 'returns false' do + expect(described_class).not_to be_generate_report + end + end + + context "when ENV['FLAKY_RSPEC_GENERATE_REPORT'] is set to 'false'" do + before do + stub_env('FLAKY_RSPEC_GENERATE_REPORT', 'false') + end + + it 'returns false' do + expect(described_class).not_to be_generate_report + end + end + + context "when ENV['FLAKY_RSPEC_GENERATE_REPORT'] is set to 'true'" do + before do + stub_env('FLAKY_RSPEC_GENERATE_REPORT', 'true') + end + + it 'returns true' do + expect(described_class).to be_generate_report + end + end + end + + describe '.suite_flaky_examples_report_path' do + context "when ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] is not set" do + it 'returns the default path' do + expect(Rails.root).to receive(:join).with('rspec_flaky/suite-report.json') + .and_return('root/rspec_flaky/suite-report.json') + + expect(described_class.suite_flaky_examples_report_path).to eq('root/rspec_flaky/suite-report.json') + end + end + + context "when ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] is set" do + before do + stub_env('SUITE_FLAKY_RSPEC_REPORT_PATH', 'foo/suite-report.json') + end + + it 'returns the value of the env variable' do + expect(described_class.suite_flaky_examples_report_path).to eq('foo/suite-report.json') + end + end + end + + describe '.flaky_examples_report_path' do + context "when ENV['FLAKY_RSPEC_REPORT_PATH'] is not set" do + it 'returns the default path' do + expect(Rails.root).to receive(:join).with('rspec_flaky/report.json') + .and_return('root/rspec_flaky/report.json') + + expect(described_class.flaky_examples_report_path).to eq('root/rspec_flaky/report.json') + end + end + + context "when ENV['FLAKY_RSPEC_REPORT_PATH'] is set" do + before do + stub_env('FLAKY_RSPEC_REPORT_PATH', 'foo/report.json') + end + + it 'returns the value of the env variable' do + expect(described_class.flaky_examples_report_path).to eq('foo/report.json') + end + end + end + + describe '.new_flaky_examples_report_path' do + context "when ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] is not set" do + it 'returns the default path' do + expect(Rails.root).to receive(:join).with('rspec_flaky/new-report.json') + .and_return('root/rspec_flaky/new-report.json') + + expect(described_class.new_flaky_examples_report_path).to eq('root/rspec_flaky/new-report.json') + end + end + + context "when ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] is set" do + before do + stub_env('NEW_FLAKY_RSPEC_REPORT_PATH', 'foo/new-report.json') + end + + it 'returns the value of the env variable' do + expect(described_class.new_flaky_examples_report_path).to eq('foo/new-report.json') + end + end + end +end diff --git a/spec/lib/rspec_flaky/flaky_example_spec.rb b/spec/lib/rspec_flaky/flaky_example_spec.rb index cbfc1e538ab..d19c34bebb3 100644 --- a/spec/lib/rspec_flaky/flaky_example_spec.rb +++ b/spec/lib/rspec_flaky/flaky_example_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe RspecFlaky::FlakyExample do +describe RspecFlaky::FlakyExample, :aggregate_failures do let(:flaky_example_attrs) do { example_id: 'spec/foo/bar_spec.rb:2', @@ -9,6 +9,7 @@ describe RspecFlaky::FlakyExample do description: 'hello world', first_flaky_at: 1234, last_flaky_at: 2345, + last_flaky_job: 'https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/12', last_attempts_count: 2, flaky_reports: 1 } @@ -27,57 +28,78 @@ describe RspecFlaky::FlakyExample do end let(:example) { double(example_attrs) } + before do + # Stub these env variables otherwise specs don't behave the same on the CI + stub_env('CI_PROJECT_URL', nil) + stub_env('CI_JOB_ID', nil) + end + describe '#initialize' do shared_examples 'a valid FlakyExample instance' do - it 'returns valid attributes' do - flaky_example = described_class.new(args) + let(:flaky_example) { described_class.new(args) } + it 'returns valid attributes' do expect(flaky_example.uid).to eq(flaky_example_attrs[:uid]) - expect(flaky_example.example_id).to eq(flaky_example_attrs[:example_id]) + expect(flaky_example.file).to eq(flaky_example_attrs[:file]) + expect(flaky_example.line).to eq(flaky_example_attrs[:line]) + expect(flaky_example.description).to eq(flaky_example_attrs[:description]) + expect(flaky_example.first_flaky_at).to eq(expected_first_flaky_at) + expect(flaky_example.last_flaky_at).to eq(expected_last_flaky_at) + expect(flaky_example.last_attempts_count).to eq(flaky_example_attrs[:last_attempts_count]) + expect(flaky_example.flaky_reports).to eq(expected_flaky_reports) end end context 'when given an Rspec::Example' do - let(:args) { example } - - it_behaves_like 'a valid FlakyExample instance' + it_behaves_like 'a valid FlakyExample instance' do + let(:args) { example } + let(:expected_first_flaky_at) { nil } + let(:expected_last_flaky_at) { nil } + let(:expected_flaky_reports) { 0 } + end end context 'when given a hash' do - let(:args) { flaky_example_attrs } - - it_behaves_like 'a valid FlakyExample instance' + it_behaves_like 'a valid FlakyExample instance' do + let(:args) { flaky_example_attrs } + let(:expected_flaky_reports) { flaky_example_attrs[:flaky_reports] } + let(:expected_first_flaky_at) { flaky_example_attrs[:first_flaky_at] } + let(:expected_last_flaky_at) { flaky_example_attrs[:last_flaky_at] } + end end end - describe '#to_h' do - before do - # Stub these env variables otherwise specs don't behave the same on the CI - stub_env('CI_PROJECT_URL', nil) - stub_env('CI_JOB_ID', nil) - end + describe '#update_flakiness!' do + shared_examples 'an up-to-date FlakyExample instance' do + let(:flaky_example) { described_class.new(args) } - shared_examples 'a valid FlakyExample hash' do - let(:additional_attrs) { {} } + it 'updates the first_flaky_at' do + now = Time.now + expected_first_flaky_at = flaky_example.first_flaky_at ? flaky_example.first_flaky_at : now + Timecop.freeze(now) { flaky_example.update_flakiness! } - it 'returns a valid hash' do - flaky_example = described_class.new(args) - final_hash = flaky_example_attrs - .merge(last_flaky_at: instance_of(Time), last_flaky_job: nil) - .merge(additional_attrs) + expect(flaky_example.first_flaky_at).to eq(expected_first_flaky_at) + end + + it 'updates the last_flaky_at' do + now = Time.now + Timecop.freeze(now) { flaky_example.update_flakiness! } - expect(flaky_example.to_h).to match(hash_including(final_hash)) + expect(flaky_example.last_flaky_at).to eq(now) end - end - context 'when given an Rspec::Example' do - let(:args) { example } + it 'updates the flaky_reports' do + expected_flaky_reports = flaky_example.first_flaky_at ? flaky_example.flaky_reports + 1 : 1 + + expect { flaky_example.update_flakiness! }.to change { flaky_example.flaky_reports }.by(1) + expect(flaky_example.flaky_reports).to eq(expected_flaky_reports) + end + + context 'when passed a :last_attempts_count' do + it 'updates the last_attempts_count' do + flaky_example.update_flakiness!(last_attempts_count: 42) - context 'when run locally' do - it_behaves_like 'a valid FlakyExample hash' do - let(:additional_attrs) do - { first_flaky_at: instance_of(Time) } - end + expect(flaky_example.last_attempts_count).to eq(42) end end @@ -87,10 +109,45 @@ describe RspecFlaky::FlakyExample do stub_env('CI_JOB_ID', 42) end - it_behaves_like 'a valid FlakyExample hash' do - let(:additional_attrs) do - { first_flaky_at: instance_of(Time), last_flaky_job: "https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/42" } - end + it 'updates the last_flaky_job' do + flaky_example.update_flakiness! + + expect(flaky_example.last_flaky_job).to eq('https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/42') + end + end + end + + context 'when given an Rspec::Example' do + it_behaves_like 'an up-to-date FlakyExample instance' do + let(:args) { example } + end + end + + context 'when given a hash' do + it_behaves_like 'an up-to-date FlakyExample instance' do + let(:args) { flaky_example_attrs } + end + end + end + + describe '#to_h' do + shared_examples 'a valid FlakyExample hash' do + let(:additional_attrs) { {} } + + it 'returns a valid hash' do + flaky_example = described_class.new(args) + final_hash = flaky_example_attrs.merge(additional_attrs) + + expect(flaky_example.to_h).to eq(final_hash) + end + end + + context 'when given an Rspec::Example' do + let(:args) { example } + + it_behaves_like 'a valid FlakyExample hash' do + let(:additional_attrs) do + { first_flaky_at: nil, last_flaky_at: nil, last_flaky_job: nil, flaky_reports: 0 } end end end diff --git a/spec/lib/rspec_flaky/flaky_examples_collection_spec.rb b/spec/lib/rspec_flaky/flaky_examples_collection_spec.rb new file mode 100644 index 00000000000..06a8ba0d02e --- /dev/null +++ b/spec/lib/rspec_flaky/flaky_examples_collection_spec.rb @@ -0,0 +1,79 @@ +require 'spec_helper' + +describe RspecFlaky::FlakyExamplesCollection, :aggregate_failures do + let(:collection_hash) do + { + a: { example_id: 'spec/foo/bar_spec.rb:2' }, + b: { example_id: 'spec/foo/baz_spec.rb:3' } + } + end + let(:collection_report) do + { + a: { + example_id: 'spec/foo/bar_spec.rb:2', + first_flaky_at: nil, + last_flaky_at: nil, + last_flaky_job: nil + }, + b: { + example_id: 'spec/foo/baz_spec.rb:3', + first_flaky_at: nil, + last_flaky_at: nil, + last_flaky_job: nil + } + } + end + + describe '.from_json' do + it 'accepts a JSON' do + collection = described_class.from_json(JSON.pretty_generate(collection_hash)) + + expect(collection.to_report).to eq(described_class.new(collection_hash).to_report) + end + end + + describe '#initialize' do + it 'accepts no argument' do + expect { described_class.new }.not_to raise_error + end + + it 'accepts a hash' do + expect { described_class.new(collection_hash) }.not_to raise_error + end + + it 'does not accept anything else' do + expect { described_class.new([1, 2, 3]) }.to raise_error(ArgumentError, "`collection` must be a Hash, Array given!") + end + end + + describe '#to_report' do + it 'calls #to_h on the values' do + collection = described_class.new(collection_hash) + + expect(collection.to_report).to eq(collection_report) + end + end + + describe '#-' do + it 'returns only examples that are not present in the given collection' do + collection1 = described_class.new(collection_hash) + collection2 = described_class.new( + a: { example_id: 'spec/foo/bar_spec.rb:2' }, + c: { example_id: 'spec/bar/baz_spec.rb:4' }) + + expect((collection2 - collection1).to_report).to eq( + c: { + example_id: 'spec/bar/baz_spec.rb:4', + first_flaky_at: nil, + last_flaky_at: nil, + last_flaky_job: nil + }) + end + + it 'fails if the given collection does not respond to `#key?`' do + collection = described_class.new(collection_hash) + + expect { collection - [1, 2, 3] }.to raise_error(ArgumentError, "`other` must respond to `#key?`, Array does not!") + end + end +end diff --git a/spec/lib/rspec_flaky/listener_spec.rb b/spec/lib/rspec_flaky/listener_spec.rb index 0e193bf408b..7590ea9576d 100644 --- a/spec/lib/rspec_flaky/listener_spec.rb +++ b/spec/lib/rspec_flaky/listener_spec.rb @@ -1,22 +1,35 @@ require 'spec_helper' -describe RspecFlaky::Listener do - let(:flaky_example_report) do +describe RspecFlaky::Listener, :aggregate_failures do + let(:already_flaky_example_uid) { '6e869794f4cfd2badd93eb68719371d1' } + let(:suite_flaky_example_report) do { - 'abc123' => { + already_flaky_example_uid => { example_id: 'spec/foo/bar_spec.rb:2', file: 'spec/foo/bar_spec.rb', line: 2, description: 'hello world', first_flaky_at: 1234, - last_flaky_at: instance_of(Time), - last_attempts_count: 2, + last_flaky_at: 4321, + last_attempts_count: 3, flaky_reports: 1, last_flaky_job: nil } } end - let(:example_attrs) do + let(:already_flaky_example_attrs) do + { + id: 'spec/foo/bar_spec.rb:2', + metadata: { + file_path: 'spec/foo/bar_spec.rb', + line_number: 2, + full_description: 'hello world' + }, + execution_result: double(status: 'passed', exception: nil) + } + end + let(:already_flaky_example) { RspecFlaky::FlakyExample.new(suite_flaky_example_report[already_flaky_example_uid]) } + let(:new_example_attrs) do { id: 'spec/foo/baz_spec.rb:3', metadata: { @@ -36,14 +49,14 @@ describe RspecFlaky::Listener do describe '#initialize' do shared_examples 'a valid Listener instance' do - let(:expected_all_flaky_examples) { {} } + let(:expected_suite_flaky_examples) { {} } it 'returns a valid Listener instance' do listener = described_class.new - expect(listener.to_report(listener.all_flaky_examples)) - .to match(hash_including(expected_all_flaky_examples)) - expect(listener.new_flaky_examples).to eq({}) + expect(listener.to_report(listener.suite_flaky_examples)) + .to eq(expected_suite_flaky_examples) + expect(listener.flaky_examples).to eq({}) end end @@ -51,16 +64,16 @@ describe RspecFlaky::Listener do it_behaves_like 'a valid Listener instance' end - context 'when a report file exists and set by ALL_FLAKY_RSPEC_REPORT_PATH' do + context 'when a report file exists and set by SUITE_FLAKY_RSPEC_REPORT_PATH' do let(:report_file) do Tempfile.new(%w[rspec_flaky_report .json]).tap do |f| - f.write(JSON.pretty_generate(flaky_example_report)) + f.write(JSON.pretty_generate(suite_flaky_example_report)) f.rewind end end before do - stub_env('ALL_FLAKY_RSPEC_REPORT_PATH', report_file.path) + stub_env('SUITE_FLAKY_RSPEC_REPORT_PATH', report_file.path) end after do @@ -69,74 +82,122 @@ describe RspecFlaky::Listener do end it_behaves_like 'a valid Listener instance' do - let(:expected_all_flaky_examples) { flaky_example_report } + let(:expected_suite_flaky_examples) { suite_flaky_example_report } end end end describe '#example_passed' do - let(:rspec_example) { double(example_attrs) } + let(:rspec_example) { double(new_example_attrs) } let(:notification) { double(example: rspec_example) } + let(:listener) { described_class.new(suite_flaky_example_report.to_json) } shared_examples 'a non-flaky example' do it 'does not change the flaky examples hash' do - expect { subject.example_passed(notification) } - .not_to change { subject.all_flaky_examples } + expect { listener.example_passed(notification) } + .not_to change { listener.flaky_examples } end end - describe 'when the RSpec example does not respond to attempts' do - it_behaves_like 'a non-flaky example' - end + shared_examples 'an existing flaky example' do + let(:expected_flaky_example) do + { + example_id: 'spec/foo/bar_spec.rb:2', + file: 'spec/foo/bar_spec.rb', + line: 2, + description: 'hello world', + first_flaky_at: 1234, + last_attempts_count: 2, + flaky_reports: 2, + last_flaky_job: nil + } + end - describe 'when the RSpec example has 1 attempt' do - let(:rspec_example) { double(example_attrs.merge(attempts: 1)) } + it 'changes the flaky examples hash' do + new_example = RspecFlaky::Example.new(rspec_example) - it_behaves_like 'a non-flaky example' + now = Time.now + Timecop.freeze(now) do + expect { listener.example_passed(notification) } + .to change { listener.flaky_examples[new_example.uid].to_h } + end + + expect(listener.flaky_examples[new_example.uid].to_h) + .to eq(expected_flaky_example.merge(last_flaky_at: now)) + end end - describe 'when the RSpec example has 2 attempts' do - let(:rspec_example) { double(example_attrs.merge(attempts: 2)) } - let(:expected_new_flaky_example) do + shared_examples 'a new flaky example' do + let(:expected_flaky_example) do { example_id: 'spec/foo/baz_spec.rb:3', file: 'spec/foo/baz_spec.rb', line: 3, description: 'hello GitLab', - first_flaky_at: instance_of(Time), - last_flaky_at: instance_of(Time), last_attempts_count: 2, flaky_reports: 1, last_flaky_job: nil } end - it 'does not change the flaky examples hash' do - expect { subject.example_passed(notification) } - .to change { subject.all_flaky_examples } - + it 'changes the all flaky examples hash' do new_example = RspecFlaky::Example.new(rspec_example) - expect(subject.all_flaky_examples[new_example.uid].to_h) - .to match(hash_including(expected_new_flaky_example)) + now = Time.now + Timecop.freeze(now) do + expect { listener.example_passed(notification) } + .to change { listener.flaky_examples[new_example.uid].to_h } + end + + expect(listener.flaky_examples[new_example.uid].to_h) + .to eq(expected_flaky_example.merge(first_flaky_at: now, last_flaky_at: now)) + end + end + + describe 'when the RSpec example does not respond to attempts' do + it_behaves_like 'a non-flaky example' + end + + describe 'when the RSpec example has 1 attempt' do + let(:rspec_example) { double(new_example_attrs.merge(attempts: 1)) } + + it_behaves_like 'a non-flaky example' + end + + describe 'when the RSpec example has 2 attempts' do + let(:rspec_example) { double(new_example_attrs.merge(attempts: 2)) } + + it_behaves_like 'a new flaky example' + + context 'with an existing flaky example' do + let(:rspec_example) { double(already_flaky_example_attrs.merge(attempts: 2)) } + + it_behaves_like 'an existing flaky example' end end end describe '#dump_summary' do - let(:rspec_example) { double(example_attrs) } - let(:notification) { double(example: rspec_example) } + let(:listener) { described_class.new(suite_flaky_example_report.to_json) } + let(:new_flaky_rspec_example) { double(new_example_attrs.merge(attempts: 2)) } + let(:already_flaky_rspec_example) { double(already_flaky_example_attrs.merge(attempts: 2)) } + let(:notification_new_flaky_rspec_example) { double(example: new_flaky_rspec_example) } + let(:notification_already_flaky_rspec_example) { double(example: already_flaky_rspec_example) } - context 'when a report file path is set by ALL_FLAKY_RSPEC_REPORT_PATH' do + context 'when a report file path is set by FLAKY_RSPEC_REPORT_PATH' do let(:report_file_path) { Rails.root.join('tmp', 'rspec_flaky_report.json') } + let(:new_report_file_path) { Rails.root.join('tmp', 'rspec_flaky_new_report.json') } before do - stub_env('ALL_FLAKY_RSPEC_REPORT_PATH', report_file_path) + stub_env('FLAKY_RSPEC_REPORT_PATH', report_file_path) + stub_env('NEW_FLAKY_RSPEC_REPORT_PATH', new_report_file_path) FileUtils.rm(report_file_path) if File.exist?(report_file_path) + FileUtils.rm(new_report_file_path) if File.exist?(new_report_file_path) end after do FileUtils.rm(report_file_path) if File.exist?(report_file_path) + FileUtils.rm(new_report_file_path) if File.exist?(new_report_file_path) end context 'when FLAKY_RSPEC_GENERATE_REPORT == "false"' do @@ -144,12 +205,13 @@ describe RspecFlaky::Listener do stub_env('FLAKY_RSPEC_GENERATE_REPORT', 'false') end - it 'does not write the report file' do - subject.example_passed(notification) + it 'does not write any report file' do + listener.example_passed(notification_new_flaky_rspec_example) - subject.dump_summary(nil) + listener.dump_summary(nil) expect(File.exist?(report_file_path)).to be(false) + expect(File.exist?(new_report_file_path)).to be(false) end end @@ -158,21 +220,39 @@ describe RspecFlaky::Listener do stub_env('FLAKY_RSPEC_GENERATE_REPORT', 'true') end - it 'writes the report file' do - subject.example_passed(notification) + around do |example| + Timecop.freeze { example.run } + end + + it 'writes the report files' do + listener.example_passed(notification_new_flaky_rspec_example) + listener.example_passed(notification_already_flaky_rspec_example) - subject.dump_summary(nil) + listener.dump_summary(nil) expect(File.exist?(report_file_path)).to be(true) + expect(File.exist?(new_report_file_path)).to be(true) + + expect(File.read(report_file_path)) + .to eq(JSON.pretty_generate(listener.to_report(listener.flaky_examples))) + + new_example = RspecFlaky::Example.new(notification_new_flaky_rspec_example) + new_flaky_example = RspecFlaky::FlakyExample.new(new_example) + new_flaky_example.update_flakiness! + + expect(File.read(new_report_file_path)) + .to eq(JSON.pretty_generate(listener.to_report(new_example.uid => new_flaky_example))) end end end end describe '#to_report' do + let(:listener) { described_class.new(suite_flaky_example_report.to_json) } + it 'transforms the internal hash to a JSON-ready hash' do - expect(subject.to_report('abc123' => RspecFlaky::FlakyExample.new(flaky_example_report['abc123']))) - .to match(hash_including(flaky_example_report)) + expect(listener.to_report(already_flaky_example_uid => already_flaky_example)) + .to match(hash_including(suite_flaky_example_report)) end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 576e4ae1d38..48cacba6a8a 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -81,7 +81,10 @@ RSpec.configure do |config| if ENV['CI'] # This includes the first try, i.e. tests will be run 4 times before failing. config.default_retry_count = 4 - config.reporter.register_listener(RspecFlaky::Listener.new, :example_passed, :dump_summary) + config.reporter.register_listener( + RspecFlaky::Listener.new, + :example_passed, + :dump_summary) end config.before(:suite) do |