diff options
Diffstat (limited to 'scripts/rspec_helpers.sh')
| -rw-r--r-- | scripts/rspec_helpers.sh | 62 |
1 files changed, 44 insertions, 18 deletions
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index 0f14b702de2..5f003d032b7 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -1,39 +1,44 @@ #!/usr/bin/env bash function retrieve_tests_metadata() { - mkdir -p crystalball/ knapsack/ rspec_flaky/ rspec_profiling/ - - local project_path="gitlab-org/gitlab" - local test_metadata_job_id - - # Ruby - test_metadata_job_id=$(scripts/api/get_job_id --project "${project_path}" -q "status=success" -q "ref=master" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata") + mkdir -p knapsack/ rspec_flaky/ rspec_profiling/ if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then - scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" + wget -O "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" fi if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then - scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}" + wget -O "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}" fi - - # FIXME: We will need to find a pipeline where the $RSPEC_PACKED_TESTS_MAPPING_PATH.gz actually exists (Crystalball only runs every two-hours, but the `update-tests-metadata` runs for all `master` pipelines...). - # if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then - # (scripts/api/download_job_artifact --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}" - # fi - # - # scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}" } function update_tests_metadata() { echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" knapsack/rspec*.json + if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then + if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then + scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" + else + echo "Not uplaoding report to S3 as the pipeline is not a scheduled one." + fi + fi + rm -f knapsack/rspec*.json - export FLAKY_RSPEC_GENERATE_REPORT="true" scripts/merge-reports "${FLAKY_RSPEC_SUITE_REPORT_PATH}" rspec_flaky/all_*.json + + export FLAKY_RSPEC_GENERATE_REPORT="true" scripts/flaky_examples/prune-old-flaky-examples "${FLAKY_RSPEC_SUITE_REPORT_PATH}" + + if [[ -n ${TESTS_METADATA_S3_BUCKET} ]]; then + if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then + scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${FLAKY_RSPEC_SUITE_REPORT_PATH}" + else + echo "Not uploading report to S3 as the pipeline is not a scheduled one." + fi + fi + rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then @@ -43,6 +48,16 @@ function update_tests_metadata() { fi } +function retrieve_tests_mapping() { + mkdir -p crystalball/ + + if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then + (wget -O "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" "http://${TESTS_METADATA_S3_BUCKET}.s3.amazonaws.com/${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}" + fi + + scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}" +} + function update_tests_mapping() { if ! crystalball_rspec_data_exists; then echo "No crystalball rspec data found." @@ -50,9 +65,20 @@ function update_tests_mapping() { fi scripts/generate-test-mapping "${RSPEC_TESTS_MAPPING_PATH}" crystalball/rspec*.yml + scripts/pack-test-mapping "${RSPEC_TESTS_MAPPING_PATH}" "${RSPEC_PACKED_TESTS_MAPPING_PATH}" + gzip "${RSPEC_PACKED_TESTS_MAPPING_PATH}" - rm -f crystalball/rspec*.yml "${RSPEC_PACKED_TESTS_MAPPING_PATH}" + + if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then + if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then + scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" + else + echo "Not uploading report to S3 as the pipeline is not a scheduled one." + fi + fi + + rm -f crystalball/rspec*.yml } function crystalball_rspec_data_exists() { |