diff options
author | Rémy Coutable <remy@rymai.me> | 2017-10-04 14:01:48 +0300 |
---|---|---|
committer | Rémy Coutable <remy@rymai.me> | 2017-10-04 16:14:38 +0300 |
commit | ebfb5a50757b65530e9649815619ec206513c65e (patch) | |
tree | c61b31e1f84989a6ef48f17cb428ee9b32a93956 /lib/rspec_flaky/listener.rb | |
parent | 4a0f720a502ac02423cb9db20727ba386de3e1f1 (diff) |
Ensure RSpecFlaky doesn't automatically update flaky examples
Previously, instantiating a RspecFlaky::FlakyExample object would
automatically update its first_flaky_at, last_flaky_at and
last_flaky_job.
That was wrong because we would overwrite every time the suite report
with this false data.
We now:
- Get the suite report and only read from it
- Write only the currently detected flaky examples in the report, so
that the final report is only updated with flaky examples that were
actually detected in each job. Before, job1 could overwrite the legit
report from job2!
- Write the newly detected flaky examples by rejecting the already
tracked flaky specs instead of using another hash.
Signed-off-by: Rémy Coutable <remy@rymai.me>
Diffstat (limited to 'lib/rspec_flaky/listener.rb')
-rw-r--r-- | lib/rspec_flaky/listener.rb | 65 |
1 files changed, 39 insertions, 26 deletions
diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb index ec2fbd9e36c..4752ebe6410 100644 --- a/lib/rspec_flaky/listener.rb +++ b/lib/rspec_flaky/listener.rb @@ -2,11 +2,15 @@ require 'json' module RspecFlaky class Listener - attr_reader :all_flaky_examples, :new_flaky_examples - - def initialize - @new_flaky_examples = {} - @all_flaky_examples = init_all_flaky_examples + # - suite_flaky_examples: contains all the currently tracked flacky example + # for the whole RSpec suite + # - flaky_examples: contains the examples detected as flaky during the + # current RSpec run + attr_reader :suite_flaky_examples, :flaky_examples + + def initialize(suite_flaky_examples_json = nil) + @flaky_examples = {} + @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json) end def example_passed(notification) @@ -14,24 +18,16 @@ module RspecFlaky return unless current_example.attempts > 1 - flaky_example_hash = all_flaky_examples[current_example.uid] - - all_flaky_examples[current_example.uid] = - if flaky_example_hash - FlakyExample.new(flaky_example_hash).tap do |ex| - ex.last_attempts_count = current_example.attempts - ex.flaky_reports += 1 - end - else - FlakyExample.new(current_example).tap do |ex| - new_flaky_examples[current_example.uid] = ex - end - end + flaky_example = suite_flaky_examples.fetch(current_example.uid) { FlakyExample.new(current_example) } + flaky_example.update_flakiness!(last_attempts_count: current_example.attempts) + + flaky_examples[current_example.uid] = flaky_example end def dump_summary(_) - write_report_file(all_flaky_examples, all_flaky_examples_report_path) + write_report_file(flaky_examples, flaky_examples_report_path) + new_flaky_examples = _new_flaky_examples if new_flaky_examples.any? Rails.logger.warn "\nNew flaky examples detected:\n" Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples)) @@ -46,12 +42,24 @@ module RspecFlaky private - def init_all_flaky_examples - return {} unless File.exist?(all_flaky_examples_report_path) + def init_suite_flaky_examples(suite_flaky_examples_json = nil) + unless suite_flaky_examples_json + return {} unless File.exist?(suite_flaky_examples_report_path) + + suite_flaky_examples_json = File.read(suite_flaky_examples_report_path) + end + + suite_flaky_examples = JSON.parse(suite_flaky_examples_json) - all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path)) + Hash[(suite_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }].freeze + end + + def _new_flaky_examples + flaky_examples.reject { |uid, _| already_flaky?(uid) } + end - Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }] + def already_flaky?(example_uid) + suite_flaky_examples.key?(example_uid) end def write_report_file(examples, file_path) @@ -62,9 +70,14 @@ module RspecFlaky File.write(file_path, JSON.pretty_generate(to_report(examples))) end - def all_flaky_examples_report_path - @all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] || - Rails.root.join("rspec_flaky/all-report.json") + def suite_flaky_examples_report_path + @suite_flaky_examples_report_path ||= ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] || + Rails.root.join("rspec_flaky/suite-report.json") + end + + def flaky_examples_report_path + @flaky_examples_report_path ||= ENV['FLAKY_RSPEC_REPORT_PATH'] || + Rails.root.join("rspec_flaky/report.json") end def new_flaky_examples_report_path |