blob: 44d329a35909658e57a158eaba7d29da9ef1678d (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
|
#!/usr/bin/env ruby
# frozen_string_literal: true
require 'json'
QUARANTINE_TYPES = %w[stale bug investigating flaky broken test_environment waiting_on].freeze
missing_issues = []
quarantine_type_errors = []
invalid_type_message = %"\n*** The following quarantined tests have invalid types:\n\n%s\n"
missing_issue_message = %"\n*** The following quarantined tests are missing issue links:\n\n%s\n"
test_metadata_file = ARGV.shift
unless test_metadata_file
puts "usage: #{__FILE__} <test_metadata_file>"
exit 1
end
file = File.read(test_metadata_file)
data_hash = JSON.parse(file)
unless data_hash['examples'].count > 1
puts "\nRspec output does not contain examples. Check test-metadata.json file.\n"
exit 1
end
puts "\nAnalyzing quarantined test data...\n"
tests = data_hash['examples']
tests.each do |test|
if test['quarantine']
unless QUARANTINE_TYPES.include?(test['quarantine']['type'])
quarantine_type_errors.push(
<<~TYPE_ERRORS
==> #{test['full_description']}
in file: #{test['id']}
with type: "#{test['quarantine']['type']}"
TYPE_ERRORS
)
end
missing_issues.push(" ==> #{test['id']} - #{test['full_description']}\n") unless test['quarantine']['issue']
end
end
if quarantine_type_errors.empty? && missing_issues.empty?
puts "\nNo errors found."
else
puts "\n*** Quarantine format violations detected! ***\n"
unless quarantine_type_errors.empty?
puts invalid_type_message % quarantine_type_errors.join("\n")
puts "*** Please use one of the following quarantine types for the tests listed above.\n"
puts " #{QUARANTINE_TYPES}\n"
end
puts missing_issue_message % missing_issues unless missing_issues.empty?
puts "See https://about.gitlab.com/handbook/engineering/quality/quality-engineering/debugging-qa-test-failures/#quarantining-tests"
exit 1
end
|
