diff options
Diffstat (limited to 'scripts/rspec_helpers.sh')
| -rw-r--r-- | scripts/rspec_helpers.sh | 89 |
1 files changed, 59 insertions, 30 deletions
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index 923b633fcc9..de735e03db0 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -75,30 +75,26 @@ function crystalball_rspec_data_exists() { compgen -G "crystalball/rspec*.yml" >/dev/null } -function retrieve_previous_failed_tests() { +function retrieve_failed_tests() { local directory_for_output_reports="${1}" - local rspec_pg_regex="${2}" - local rspec_ee_pg_regex="${3}" - local pipeline_report_path="test_results/previous/test_reports.json" - - # Used to query merge requests. This variable reflects where the merge request has been created - local target_project_path="${CI_MERGE_REQUEST_PROJECT_PATH}" - local instance_url="${CI_SERVER_URL}" + local failed_tests_format="${2}" + local pipeline_index="${3}" + local pipeline_report_path="tmp/test_results/${pipeline_index}/test_reports.json" echo 'Attempting to build pipeline test report...' - scripts/pipeline_test_report_builder.rb --instance-base-url "${instance_url}" --target-project "${target_project_path}" --mr-id "${CI_MERGE_REQUEST_IID}" --output-file-path "${pipeline_report_path}" + scripts/pipeline_test_report_builder.rb --output-file-path "${pipeline_report_path}" --pipeline-index "${pipeline_index}" echo 'Generating failed tests lists...' - scripts/failed_tests.rb --previous-tests-report-path "${pipeline_report_path}" --output-directory "${directory_for_output_reports}" --rspec-pg-regex "${rspec_pg_regex}" --rspec-ee-pg-regex "${rspec_ee_pg_regex}" + scripts/failed_tests.rb --previous-tests-report-path "${pipeline_report_path}" --format "${failed_tests_format}" --output-directory "${directory_for_output_reports}" } function rspec_args() { local rspec_opts="${1}" local junit_report_file="${2:-${JUNIT_RESULT_FILE}}" - echo "-Ispec -rspec_helper --color --format documentation --format RspecJunitFormatter --out ${junit_report_file} ${rspec_opts}" + echo "-Ispec -rspec_helper --color --failure-exit-code 1 --error-exit-code 2 --format documentation --format RspecJunitFormatter --out ${junit_report_file} ${rspec_opts}" } function rspec_simple_job() { @@ -110,10 +106,18 @@ function rspec_simple_job() { eval "${rspec_cmd}" } +function rspec_simple_job_with_retry () { + local rspec_run_status=0 + + rspec_simple_job "${1}" "${2}" || rspec_run_status=$? + + handle_retry_rspec_in_new_process $rspec_run_status +} + function rspec_db_library_code() { local db_files="spec/lib/gitlab/database/" - rspec_simple_job "-- ${db_files}" + rspec_simple_job_with_retry "-- ${db_files}" } function debug_rspec_variables() { @@ -131,11 +135,30 @@ function debug_rspec_variables() { echoinfo "FLAKY_RSPEC_REPORT_PATH: ${FLAKY_RSPEC_REPORT_PATH}" echoinfo "NEW_FLAKY_RSPEC_REPORT_PATH: ${NEW_FLAKY_RSPEC_REPORT_PATH}" echoinfo "SKIPPED_FLAKY_TESTS_REPORT_PATH: ${SKIPPED_FLAKY_TESTS_REPORT_PATH}" - echoinfo "RETRIED_TESTS_REPORT_PATH: ${RETRIED_TESTS_REPORT_PATH}" echoinfo "CRYSTALBALL: ${CRYSTALBALL}" } +function handle_retry_rspec_in_new_process() { + local rspec_run_status="${1}" + + if [[ $rspec_run_status -eq 2 ]]; then + echoerr "Not retrying failing examples since there were errors happening outside of the RSpec examples!" + elif [[ $rspec_run_status -eq 1 ]]; then + # Experiment to retry failed examples in a new RSpec process: https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1148 + if [[ "${RETRY_FAILED_TESTS_IN_NEW_PROCESS}" == "true" ]]; then + retry_failed_rspec_examples + rspec_run_status=$? + else + echoerr "Not retrying failing examples since \$RETRY_FAILED_TESTS_IN_NEW_PROCESS != 'true'!" + fi + else + echosuccess "No examples to retry, congrats!" + fi + + exit $rspec_run_status +} + function rspec_paralellized_job() { read -ra job_name <<< "${CI_JOB_NAME}" local test_tool="${job_name[0]}" @@ -179,7 +202,6 @@ function rspec_paralellized_job() { export FLAKY_RSPEC_REPORT_PATH="${rspec_flaky_folder_path}all_${report_name}_report.json" export NEW_FLAKY_RSPEC_REPORT_PATH="${rspec_flaky_folder_path}new_${report_name}_report.json" export SKIPPED_FLAKY_TESTS_REPORT_PATH="${rspec_flaky_folder_path}skipped_flaky_tests_${report_name}_report.txt" - export RETRIED_TESTS_REPORT_PATH="${rspec_flaky_folder_path}retried_tests_${report_name}_report.txt" if [[ -d "ee/" ]]; then export KNAPSACK_GENERATE_REPORT="true" @@ -204,17 +226,7 @@ function rspec_paralellized_job() { echoinfo "RSpec exited with ${rspec_run_status}." - # Experiment to retry failed examples in a new RSpec process: https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/1148 - if [[ $rspec_run_status -ne 0 ]]; then - if [[ "${RETRY_FAILED_TESTS_IN_NEW_PROCESS}" == "true" ]]; then - retry_failed_rspec_examples - rspec_run_status=$? - fi - else - echosuccess "No examples to retry, congrats!" - fi - - exit $rspec_run_status + handle_retry_rspec_in_new_process $rspec_run_status } function retry_failed_rspec_examples() { @@ -228,6 +240,12 @@ function retry_failed_rspec_examples() { # Keep track of the tests that are retried, later consolidated in a single file by the `rspec:flaky-tests-report` job local failed_examples=$(grep " failed" ${RSPEC_LAST_RUN_RESULTS_FILE}) + local report_name=$(echo "${CI_JOB_NAME}" | sed -E 's|[/ ]|_|g') # e.g. 'rspec unit pg12 1/24' would become 'rspec_unit_pg12_1_24' + local rspec_flaky_folder_path="$(dirname "${FLAKY_RSPEC_SUITE_REPORT_PATH}")/" + + export RETRIED_TESTS_REPORT_PATH="${rspec_flaky_folder_path}retried_tests_${report_name}_report.txt" + echoinfo "RETRIED_TESTS_REPORT_PATH: ${RETRIED_TESTS_REPORT_PATH}" + echo "${CI_JOB_URL}" > "${RETRIED_TESTS_REPORT_PATH}" echo $failed_examples >> "${RETRIED_TESTS_REPORT_PATH}" @@ -241,8 +259,11 @@ function retry_failed_rspec_examples() { # Disable simplecov so retried tests don't override test coverage report export SIMPLECOV=0 + local default_knapsack_pattern="{,ee/,jh/}spec/{,**/}*_spec.rb" + local knapsack_test_file_pattern="${KNAPSACK_TEST_FILE_PATTERN:-$default_knapsack_pattern}" + # Retry only the tests that failed on first try - rspec_simple_job "--only-failures --pattern \"${KNAPSACK_TEST_FILE_PATTERN}\"" "${JUNIT_RETRY_FILE}" + rspec_simple_job "--only-failures --pattern \"${knapsack_test_file_pattern}\"" "${JUNIT_RETRY_FILE}" rspec_run_status=$? # Merge the JUnit report from retry into the first-try report @@ -295,7 +316,7 @@ function rspec_rerun_previous_failed_tests() { fi if [[ -n $test_files ]]; then - rspec_simple_job "${test_files}" + rspec_simple_job_with_retry "${test_files}" else echo "No failed test files to rerun" fi @@ -316,7 +337,7 @@ function rspec_fail_fast() { fi if [[ -n $test_files ]]; then - rspec_simple_job "${rspec_opts} ${test_files}" + rspec_simple_job_with_retry "${rspec_opts} ${test_files}" else echo "No rspec fail-fast tests to run" fi @@ -326,10 +347,18 @@ function filter_rspec_matched_foss_tests() { local matching_tests_file="${1}" local foss_matching_tests_file="${2}" - # Keep only files that exists (i.e. exclude EE speficic files) + # Keep only FOSS files that exists cat ${matching_tests_file} | ruby -e 'puts $stdin.read.split(" ").select { |f| f.start_with?("spec/") && File.exist?(f) }.join(" ")' > "${foss_matching_tests_file}" } +function filter_rspec_matched_ee_tests() { + local matching_tests_file="${1}" + local ee_matching_tests_file="${2}" + + # Keep only EE files that exists + cat ${matching_tests_file} | ruby -e 'puts $stdin.read.split(" ").select { |f| f.start_with?("ee/spec/") && File.exist?(f) }.join(" ")' > "${ee_matching_tests_file}" +} + function generate_frontend_fixtures_mapping() { local pattern="" @@ -351,7 +380,7 @@ function generate_frontend_fixtures_mapping() { mkdir -p $(dirname "$FRONTEND_FIXTURES_MAPPING_PATH") - rspec_simple_job "--pattern \"${pattern}\"" + rspec_simple_job_with_retry "--pattern \"${pattern}\"" } function cleanup_individual_job_reports() { |