diff options
Diffstat (limited to 'scripts/qa/quarantine-types-check')
-rwxr-xr-x | scripts/qa/quarantine-types-check | 62 |
1 files changed, 62 insertions, 0 deletions
diff --git a/scripts/qa/quarantine-types-check b/scripts/qa/quarantine-types-check new file mode 100755 index 00000000000..44d329a3590 --- /dev/null +++ b/scripts/qa/quarantine-types-check @@ -0,0 +1,62 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'json' + +QUARANTINE_TYPES = %w[stale bug investigating flaky broken test_environment waiting_on].freeze + +missing_issues = [] +quarantine_type_errors = [] +invalid_type_message = %"\n*** The following quarantined tests have invalid types:\n\n%s\n" +missing_issue_message = %"\n*** The following quarantined tests are missing issue links:\n\n%s\n" + +test_metadata_file = ARGV.shift + +unless test_metadata_file + puts "usage: #{__FILE__} <test_metadata_file>" + exit 1 +end + +file = File.read(test_metadata_file) +data_hash = JSON.parse(file) + +unless data_hash['examples'].count > 1 + puts "\nRspec output does not contain examples. Check test-metadata.json file.\n" + exit 1 +end + +puts "\nAnalyzing quarantined test data...\n" + +tests = data_hash['examples'] + +tests.each do |test| + if test['quarantine'] + unless QUARANTINE_TYPES.include?(test['quarantine']['type']) + quarantine_type_errors.push( + <<~TYPE_ERRORS + ==> #{test['full_description']} + in file: #{test['id']} + with type: "#{test['quarantine']['type']}" + TYPE_ERRORS + ) + end + + missing_issues.push(" ==> #{test['id']} - #{test['full_description']}\n") unless test['quarantine']['issue'] + end +end + +if quarantine_type_errors.empty? && missing_issues.empty? + puts "\nNo errors found." +else + puts "\n*** Quarantine format violations detected! ***\n" + + unless quarantine_type_errors.empty? + puts invalid_type_message % quarantine_type_errors.join("\n") + puts "*** Please use one of the following quarantine types for the tests listed above.\n" + puts " #{QUARANTINE_TYPES}\n" + end + + puts missing_issue_message % missing_issues unless missing_issues.empty? + puts "See https://about.gitlab.com/handbook/engineering/quality/quality-engineering/debugging-qa-test-failures/#quarantining-tests" + exit 1 +end |