diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/api/default_options.rb | 3 | ||||
| -rw-r--r-- | scripts/api/find_issues.rb | 29 | ||||
| -rwxr-xr-x | scripts/failed_tests.rb | 89 | ||||
| -rwxr-xr-x | scripts/pipeline_test_report_builder.rb | 125 | ||||
| -rw-r--r-- | scripts/rspec_helpers.sh | 16 | ||||
| -rw-r--r-- | scripts/utils.sh | 4 |
6 files changed, 179 insertions, 87 deletions
diff --git a/scripts/api/default_options.rb b/scripts/api/default_options.rb index d10666e3a68..3085ef55085 100644 --- a/scripts/api/default_options.rb +++ b/scripts/api/default_options.rb @@ -13,6 +13,7 @@ end module Host DEFAULT_OPTIONS = { instance_base_url: ENV['CI_SERVER_URL'], - mr_id: ENV['CI_MERGE_REQUEST_ID'] + target_project: ENV['CI_MERGE_REQUEST_PROJECT_ID'], + mr_iid: ENV['CI_MERGE_REQUEST_IID'] }.freeze end diff --git a/scripts/api/find_issues.rb b/scripts/api/find_issues.rb new file mode 100644 index 00000000000..a1c37030319 --- /dev/null +++ b/scripts/api/find_issues.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'gitlab' +require_relative 'default_options' + +class FindIssues + def initialize(options) + @project = options.fetch(:project) + + # Force the token to be a string so that if api_token is nil, it's set to '', + # allowing unauthenticated requests (for forks). + api_token = options.delete(:api_token).to_s + + warn "No API token given." if api_token.empty? + + @client = Gitlab.client( + endpoint: options.delete(:endpoint) || API::DEFAULT_OPTIONS[:endpoint], + private_token: api_token + ) + end + + def execute(search_data) + client.issues(project, search_data) + end + + private + + attr_reader :project, :client +end diff --git a/scripts/failed_tests.rb b/scripts/failed_tests.rb index 319961d277c..786d3c24c74 100755 --- a/scripts/failed_tests.rb +++ b/scripts/failed_tests.rb @@ -8,31 +8,47 @@ require 'json' require 'set' class FailedTests + DEFAULT_OPTIONS = { + previous_tests_report_path: 'test_results/previous/test_reports.json', + output_directory: 'tmp/previous_failed_tests/', + format: :oneline, + rspec_pg_regex: /rspec .+ pg12( .+)?/, + rspec_ee_pg_regex: /rspec-ee .+ pg12( .+)?/ + }.freeze + def initialize(options) @filename = options.delete(:previous_tests_report_path) @output_directory = options.delete(:output_directory) + @format = options.delete(:format).to_sym @rspec_pg_regex = options.delete(:rspec_pg_regex) @rspec_ee_pg_regex = options.delete(:rspec_ee_pg_regex) end - def output_failed_test_files + def output_failed_tests create_output_dir - failed_files_for_suite_collection.each do |suite_collection_name, suite_collection_files| - failed_test_files = suite_collection_files.map { |filepath| filepath.delete_prefix('./') }.join(' ') + failed_cases_for_suite_collection.each do |suite_name, suite_tests| + puts "[FailedTests] Detected #{suite_tests.size} failed tests in suite #{suite_name}..." + suite_tests = + case format + when :oneline + suite_tests.map { |test| test['file'] }.join(' ') # rubocop:disable Rails/Pluck + when :json + JSON.pretty_generate(suite_tests.to_a) + end - output_file = File.join(output_directory, "#{suite_collection_name}_failed_files.txt") + output_file = File.join(output_directory, "#{suite_name}_failed_tests.#{output_file_format}") File.open(output_file, 'w') do |file| - file.write(failed_test_files) + file.write(suite_tests) end end end - def failed_files_for_suite_collection - suite_map.each_with_object(Hash.new { |h, k| h[k] = Set.new }) do |(suite_collection_name, suite_collection_regex), hash| + def failed_cases_for_suite_collection + suite_map.each_with_object(Hash.new { |h, k| h[k] = Set.new }) do |(suite_name, suite_collection_regex), hash| failed_suites.each do |suite| - hash[suite_collection_name].merge(failed_files(suite)) if suite['name'] =~ suite_collection_regex + hash[suite_name].merge(failed_cases(suite)) if suite['name'] =~ suite_collection_regex end end end @@ -47,7 +63,7 @@ class FailedTests private - attr_reader :filename, :output_directory, :rspec_pg_regex, :rspec_ee_pg_regex + attr_reader :filename, :output_directory, :format, :rspec_pg_regex, :rspec_ee_pg_regex def file_contents @file_contents ||= begin @@ -65,50 +81,75 @@ class FailedTests end end + def output_file_format + case format + when :oneline + 'txt' + when :json + 'json' + else + raise "[FailedTests] Unsupported format `#{format}` (allowed formats: `oneline` and `json`)!" + end + end + def failed_suites return [] unless file_contents_as_json['suites'] file_contents_as_json['suites'].select { |suite| suite['failed_count'] > 0 } end - def failed_files(suite) + def failed_cases(suite) return [] unless suite - suite['test_cases'].each_with_object([]) do |failure_hash, failed_cases| - failed_cases << failure_hash['file'] if failure_hash['status'] == 'failed' + suite['test_cases'].filter_map do |failure_hash| + next if failure_hash['status'] != 'failed' + + failure_hash['job_url'] = suite['job_url'] + failure_hash['file'] = failure_hash['file'].delete_prefix('./') + + failure_hash end end def create_output_dir return if File.directory?(output_directory) - puts 'Creating output directory...' + puts '[FailedTests] Creating output directory...' FileUtils.mkdir_p(output_directory) end end if $PROGRAM_NAME == __FILE__ - options = { - previous_tests_report_path: 'test_results/previous/test_reports.json', - output_directory: 'tmp/previous_failed_tests/', - rspec_pg_regex: /rspec .+ pg12( .+)?/, - rspec_ee_pg_regex: /rspec-ee .+ pg12( .+)?/ - } + options = FailedTests::DEFAULT_OPTIONS.dup OptionParser.new do |opts| - opts.on("-p", "--previous-tests-report-path PREVIOUS_TESTS_REPORT_PATH", String, "Path of the file listing previous test failures") do |value| + opts.on("-p", "--previous-tests-report-path PREVIOUS_TESTS_REPORT_PATH", String, + "Path of the file listing previous test failures (defaults to " \ + "`#{FailedTests::DEFAULT_OPTIONS[:previous_tests_report_path]}`)") do |value| options[:previous_tests_report_path] = value end - opts.on("-o", "--output-directory OUTPUT_DIRECTORY", String, "Output directory for failed test files") do |value| + opts.on("-o", "--output-directory OUTPUT_DIRECTORY", String, + "Output directory for failed test files (defaults to " \ + "`#{FailedTests::DEFAULT_OPTIONS[:output_directory]}`)") do |value| options[:output_directory] = value end - opts.on("--rspec-pg-regex RSPEC_PG_REGEX", Regexp, "Regex to use when finding matching RSpec jobs") do |value| + opts.on("-f", "--format [oneline|json]", String, + "Format of the output files: oneline (with test filenames) or JSON (defaults to " \ + "`#{FailedTests::DEFAULT_OPTIONS[:format]}`)") do |value| + options[:format] = value + end + + opts.on("--rspec-pg-regex RSPEC_PG_REGEX", Regexp, + "Regex to use when finding matching RSpec jobs (defaults to " \ + "`#{FailedTests::DEFAULT_OPTIONS[:rspec_pg_regex]}`)") do |value| options[:rspec_pg_regex] = value end - opts.on("--rspec-ee-pg-regex RSPEC_EE_PG_REGEX", Regexp, "Regex to use when finding matching RSpec EE jobs") do |value| + opts.on("--rspec-ee-pg-regex RSPEC_EE_PG_REGEX", Regexp, + "Regex to use when finding matching RSpec EE jobs (defaults to " \ + "`#{FailedTests::DEFAULT_OPTIONS[:rspec_ee_pg_regex]}`)") do |value| options[:rspec_ee_pg_regex] = value end @@ -118,5 +159,5 @@ if $PROGRAM_NAME == __FILE__ end end.parse! - FailedTests.new(options).output_failed_test_files + FailedTests.new(options).output_failed_tests end diff --git a/scripts/pipeline_test_report_builder.rb b/scripts/pipeline_test_report_builder.rb index 90af0451864..6f69a5c692f 100755 --- a/scripts/pipeline_test_report_builder.rb +++ b/scripts/pipeline_test_report_builder.rb @@ -5,7 +5,6 @@ require 'optparse' require 'time' require 'fileutils' require 'uri' -require 'cgi' require 'net/http' require 'json' require_relative 'api/default_options' @@ -19,50 +18,79 @@ require_relative 'api/default_options' # https://gitlab.com/gitlab-org/gitlab/-/pipelines/363788864/tests/suite.json?build_ids[]=1555608749 # Push into expected format for failed tests class PipelineTestReportBuilder + DEFAULT_OPTIONS = { + target_project: Host::DEFAULT_OPTIONS[:target_project], + mr_iid: Host::DEFAULT_OPTIONS[:mr_iid], + api_endpoint: API::DEFAULT_OPTIONS[:endpoint], + output_file_path: 'test_results/test_reports.json', + pipeline_index: :previous + }.freeze + def initialize(options) @target_project = options.delete(:target_project) - @mr_id = options.delete(:mr_id) || Host::DEFAULT_OPTIONS[:mr_id] - @instance_base_url = options.delete(:instance_base_url) || Host::DEFAULT_OPTIONS[:instance_base_url] - @output_file_path = options.delete(:output_file_path) - end - - def test_report_for_latest_pipeline - build_test_report_json_for_pipeline(previous_pipeline) + @mr_iid = options.delete(:mr_iid) + @api_endpoint = options.delete(:api_endpoint).to_s + @output_file_path = options.delete(:output_file_path).to_s + @pipeline_index = options.delete(:pipeline_index).to_sym end def execute - if output_file_path - FileUtils.mkdir_p(File.dirname(output_file_path)) - end + FileUtils.mkdir_p(File.dirname(output_file_path)) File.open(output_file_path, 'w') do |file| - file.write(test_report_for_latest_pipeline) + file.write(test_report_for_pipeline) end end + def test_report_for_pipeline + build_test_report_json_for_pipeline + end + + def latest_pipeline + pipelines_sorted_descending[0] + end + def previous_pipeline - # Top of the list will always be the current pipeline + # Top of the list will always be the latest pipeline # Second from top will be the previous pipeline - pipelines_for_mr.sort_by { |a| -Time.parse(a['created_at']).to_i }[1] + pipelines_sorted_descending[1] end private - attr_reader :target_project, :mr_id, :instance_base_url, :output_file_path + def pipeline + @pipeline ||= + case pipeline_index + when :latest + latest_pipeline + when :previous + previous_pipeline + else + raise "[PipelineTestReportBuilder] Unsupported pipeline_index `#{pipeline_index}` (allowed index: `latest` and `previous`!" + end + end + + def pipelines_sorted_descending + # Top of the list will always be the current pipeline + # Second from top will be the previous pipeline + pipelines_for_mr.sort_by { |a| -a['id'] } + end + + attr_reader :target_project, :mr_iid, :api_endpoint, :output_file_path, :pipeline_index def pipeline_project_api_base_url(pipeline) - "#{instance_base_url}/api/v4/projects/#{pipeline['project_id']}" + "#{api_endpoint}/projects/#{pipeline['project_id']}" end def target_project_api_base_url - "#{instance_base_url}/api/v4/projects/#{CGI.escape(target_project)}" + "#{api_endpoint}/projects/#{target_project}" end def pipelines_for_mr - fetch("#{target_project_api_base_url}/merge_requests/#{mr_id}/pipelines") + @pipelines_for_mr ||= fetch("#{target_project_api_base_url}/merge_requests/#{mr_iid}/pipelines") end - def failed_builds_for_pipeline(pipeline) + def failed_builds_for_pipeline fetch("#{pipeline_project_api_base_url(pipeline)}/pipelines/#{pipeline['id']}/jobs?scope=failed&per_page=100") end @@ -70,44 +98,45 @@ class PipelineTestReportBuilder # Here we request individual builds, even though it is possible to supply multiple build IDs. # The reason for this; it is possible to lose the job context and name when requesting multiple builds. # Please see for more info: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69053#note_709939709 - def test_report_for_build(pipeline, build_id) - fetch("#{pipeline['web_url']}/tests/suite.json?build_ids[]=#{build_id}") + def test_report_for_build(pipeline_url, build_id) + fetch("#{pipeline_url}/tests/suite.json?build_ids[]=#{build_id}").tap do |suite| + suite['job_url'] = job_url(pipeline_url, build_id) + end rescue Net::HTTPServerException => e raise e unless e.response.code.to_i == 404 - puts "Artifacts not found. They may have expired. Skipping this build." + puts "[PipelineTestReportBuilder] Artifacts not found. They may have expired. Skipping this build." end - def build_test_report_json_for_pipeline(pipeline) + def build_test_report_json_for_pipeline # empty file if no previous failed pipeline - return {}.to_json if pipeline.nil? || pipeline['status'] != 'failed' + return {}.to_json if pipeline.nil? - test_report = {} + test_report = { 'suites' => [] } - puts "Discovered last failed pipeline (#{pipeline['id']}) for MR!#{mr_id}" + puts "[PipelineTestReportBuilder] Discovered #{pipeline_index} failed pipeline (##{pipeline['id']}) for MR!#{mr_iid}" - failed_builds_for_test_stage = failed_builds_for_pipeline(pipeline).select do |failed_build| - failed_build['stage'] == 'test' - end + failed_builds_for_pipeline.each do |failed_build| + next if failed_build['stage'] != 'test' - puts "#{failed_builds_for_test_stage.length} failed builds in test stage found..." + test_report['suites'] << test_report_for_build(pipeline['web_url'], failed_build['id']) + end - if failed_builds_for_test_stage.any? - test_report['suites'] ||= [] + test_report['suites'].compact! - failed_builds_for_test_stage.each do |failed_build| - suite = test_report_for_build(pipeline, failed_build['id']) - test_report['suites'] << suite if suite - end - end + puts "[PipelineTestReportBuilder] #{test_report['suites'].size} failed builds in test stage found..." test_report.to_json end + def job_url(pipeline_url, build_id) + pipeline_url.sub(%r{/pipelines/.+}, "/jobs/#{build_id}") + end + def fetch(uri_str) uri = URI(uri_str) - puts "URL: #{uri}" + puts "[PipelineTestReportBuilder] URL: #{uri}" request = Net::HTTP::Get.new(uri) @@ -119,7 +148,7 @@ class PipelineTestReportBuilder when Net::HTTPSuccess body = response.read_body else - raise "Unexpected response: #{response.value}" + raise "[PipelineTestReportBuilder] Unexpected response: #{response.value}" end end end @@ -129,25 +158,17 @@ class PipelineTestReportBuilder end if $PROGRAM_NAME == __FILE__ - options = Host::DEFAULT_OPTIONS.dup + options = PipelineTestReportBuilder::DEFAULT_OPTIONS.dup OptionParser.new do |opts| - opts.on("-t", "--target-project TARGET_PROJECT", String, "Project where to find the merge request") do |value| - options[:target_project] = value - end - - opts.on("-m", "--mr-id MR_ID", String, "A merge request ID") do |value| - options[:mr_id] = value - end - - opts.on("-i", "--instance-base-url INSTANCE_BASE_URL", String, "URL of the instance where project and merge request resides") do |value| - options[:instance_base_url] = value - end - opts.on("-o", "--output-file-path OUTPUT_PATH", String, "A path for output file") do |value| options[:output_file_path] = value end + opts.on("-p", "--pipeline-index [latest|previous]", String, "What pipeline to retrieve (defaults to `#{PipelineTestReportBuilder::DEFAULT_OPTIONS[:pipeline_index]}`)") do |value| + options[:pipeline_index] = value + end + opts.on("-h", "--help", "Prints this help") do puts opts exit diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index 5b2c84059ee..de735e03db0 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -75,23 +75,19 @@ function crystalball_rspec_data_exists() { compgen -G "crystalball/rspec*.yml" >/dev/null } -function retrieve_previous_failed_tests() { +function retrieve_failed_tests() { local directory_for_output_reports="${1}" - local rspec_pg_regex="${2}" - local rspec_ee_pg_regex="${3}" - local pipeline_report_path="test_results/previous/test_reports.json" - - # Used to query merge requests. This variable reflects where the merge request has been created - local target_project_path="${CI_MERGE_REQUEST_PROJECT_PATH}" - local instance_url="${CI_SERVER_URL}" + local failed_tests_format="${2}" + local pipeline_index="${3}" + local pipeline_report_path="tmp/test_results/${pipeline_index}/test_reports.json" echo 'Attempting to build pipeline test report...' - scripts/pipeline_test_report_builder.rb --instance-base-url "${instance_url}" --target-project "${target_project_path}" --mr-id "${CI_MERGE_REQUEST_IID}" --output-file-path "${pipeline_report_path}" + scripts/pipeline_test_report_builder.rb --output-file-path "${pipeline_report_path}" --pipeline-index "${pipeline_index}" echo 'Generating failed tests lists...' - scripts/failed_tests.rb --previous-tests-report-path "${pipeline_report_path}" --output-directory "${directory_for_output_reports}" --rspec-pg-regex "${rspec_pg_regex}" --rspec-ee-pg-regex "${rspec_ee_pg_regex}" + scripts/failed_tests.rb --previous-tests-report-path "${pipeline_report_path}" --format "${failed_tests_format}" --output-directory "${directory_for_output_reports}" } function rspec_args() { diff --git a/scripts/utils.sh b/scripts/utils.sh index 44bbabb4c99..6fee1909bb7 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -125,6 +125,10 @@ function install_tff_gem() { run_timed_command "gem install test_file_finder --no-document --version 0.1.4" } +function install_activesupport_gem() { + run_timed_command "gem install activesupport --no-document --version 6.1.7.1" +} + function install_junit_merge_gem() { run_timed_command "gem install junit_merge --no-document --version 0.1.2" } |