Use a new RspecFlakyListener to detect flaky specs

Signed-off-by: Rémy Coutable <remy@rymai.me>

3 files changed, 160 insertions, 0 deletions

diff --git a/lib/rspec_flaky/example.rb b/lib/rspec_flaky/example.rb
new file mode 100644
index 00000000000..b6e790cbbab
--- /dev/null
+++ b/lib/rspec_flaky/example.rb
@@ -0,0 +1,46 @@
+module RspecFlaky
+  # This is a wrapper class for RSpec::Core::Example
+  class Example
+    delegate :status, :exception, to: :execution_result
+
+    def initialize(rspec_example)
+      @rspec_example = rspec_example.try(:example) || rspec_example
+    end
+
+    def uid
+      @uid ||= Digest::MD5.hexdigest("#{description}-#{file}")
+    end
+
+    def example_id
+      rspec_example.id
+    end
+
+    def file
+      metadata[:file_path]
+    end
+
+    def line
+      metadata[:line_number]
+    end
+
+    def description
+      metadata[:full_description]
+    end
+
+    def attempts
+      rspec_example.try(:attempts) || 1
+    end
+
+    private
+
+    attr_reader :rspec_example
+
+    def metadata
+      rspec_example.metadata
+    end
+
+    def execution_result
+      rspec_example.execution_result
+    end
+  end
+end
diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb
new file mode 100644
index 00000000000..f81fb90e870
--- /dev/null
+++ b/lib/rspec_flaky/flaky_example.rb
@@ -0,0 +1,39 @@
+module RspecFlaky
+  # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file
+  class FlakyExample < OpenStruct
+    def initialize(example)
+      if example.respond_to?(:example_id)
+        super(
+          example_id: example.example_id,
+          file: example.file,
+          line: example.line,
+          description: example.description,
+          last_attempts_count: example.attempts,
+          flaky_reports: 1)
+      else
+        super
+      end
+    end
+
+    def first_flaky_at
+      self[:first_flaky_at] || Time.now
+    end
+
+    def last_flaky_at
+      Time.now
+    end
+
+    def last_flaky_job
+      return unless ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID']
+
+      "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}"
+    end
+
+    def to_h
+      super.merge(
+        first_flaky_at: first_flaky_at,
+        last_flaky_at: last_flaky_at,
+        last_flaky_job: last_flaky_job)
+    end
+  end
+end
diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb
new file mode 100644
index 00000000000..ec2fbd9e36c
--- /dev/null
+++ b/lib/rspec_flaky/listener.rb
@@ -0,0 +1,75 @@
+require 'json'
+
+module RspecFlaky
+  class Listener
+    attr_reader :all_flaky_examples, :new_flaky_examples
+
+    def initialize
+      @new_flaky_examples = {}
+      @all_flaky_examples = init_all_flaky_examples
+    end
+
+    def example_passed(notification)
+      current_example = RspecFlaky::Example.new(notification.example)
+
+      return unless current_example.attempts > 1
+
+      flaky_example_hash = all_flaky_examples[current_example.uid]
+
+      all_flaky_examples[current_example.uid] =
+        if flaky_example_hash
+          FlakyExample.new(flaky_example_hash).tap do |ex|
+            ex.last_attempts_count = current_example.attempts
+            ex.flaky_reports += 1
+          end
+        else
+          FlakyExample.new(current_example).tap do |ex|
+            new_flaky_examples[current_example.uid] = ex
+          end
+        end
+    end
+
+    def dump_summary(_)
+      write_report_file(all_flaky_examples, all_flaky_examples_report_path)
+
+      if new_flaky_examples.any?
+        Rails.logger.warn "\nNew flaky examples detected:\n"
+        Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples))
+
+        write_report_file(new_flaky_examples, new_flaky_examples_report_path)
+      end
+    end
+
+    def to_report(examples)
+      Hash[examples.map { |k, ex| [k, ex.to_h] }]
+    end
+
+    private
+
+    def init_all_flaky_examples
+      return {} unless File.exist?(all_flaky_examples_report_path)
+
+      all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path))
+
+      Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }]
+    end
+
+    def write_report_file(examples, file_path)
+      return unless ENV['FLAKY_RSPEC_GENERATE_REPORT'] == 'true'
+
+      report_path_dir = File.dirname(file_path)
+      FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir)
+      File.write(file_path, JSON.pretty_generate(to_report(examples)))
+    end
+
+    def all_flaky_examples_report_path
+      @all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] ||
+        Rails.root.join("rspec_flaky/all-report.json")
+    end
+
+    def new_flaky_examples_report_path
+      @new_flaky_examples_report_path ||= ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] ||
+        Rails.root.join("rspec_flaky/new-report.json")
+    end
+  end
+end