blob: 7ac6b98938fd0e33784c8e16bc0822832f1f496f (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
|
#!/usr/bin/env bash
function retrieve_tests_metadata() {
mkdir -p $(dirname "$KNAPSACK_RSPEC_SUITE_REPORT_PATH") $(dirname "$FLAKY_RSPEC_SUITE_REPORT_PATH") rspec_profiling/
if [[ -n "${RETRIEVE_TESTS_METADATA_FROM_PAGES}" ]]; then
if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
curl --location -o "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
fi
if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
curl --location -o "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "https://gitlab-org.gitlab.io/gitlab/${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
fi
else
# ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
# always target the canonical project here, so the branch must be hardcoded
local project_path="gitlab-org/gitlab"
local artifact_branch="master"
local test_metadata_job_id
# Ruby
test_metadata_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "status=success" -q "ref=${artifact_branch}" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata")
if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
fi
if [[ ! -f "${FLAKY_RSPEC_SUITE_REPORT_PATH}" ]]; then
scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_job_id}" --artifact-path "${FLAKY_RSPEC_SUITE_REPORT_PATH}" || echo "{}" > "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
fi
fi
}
function update_tests_metadata() {
echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" knapsack/rspec*.json
rm -f knapsack/rspec*.json
export FLAKY_RSPEC_GENERATE_REPORT="true"
scripts/merge-reports "${FLAKY_RSPEC_SUITE_REPORT_PATH}" rspec_flaky/all_*.json
scripts/flaky_examples/prune-old-flaky-examples "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
if [[ "$CI_PIPELINE_SOURCE" == "schedule" ]]; then
scripts/insert-rspec-profiling-data
else
echo "Not inserting profiling data as the pipeline is not a scheduled one."
fi
}
function retrieve_tests_mapping() {
mkdir -p $(dirname "$RSPEC_PACKED_TESTS_MAPPING_PATH")
if [[ -n "${RETRIEVE_TESTS_METADATA_FROM_PAGES}" ]]; then
if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
(curl --location -o "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" "https://gitlab-org.gitlab.io/gitlab/${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
fi
else
# ${CI_DEFAULT_BRANCH} might not be master in other forks but we want to
# always target the canonical project here, so the branch must be hardcoded
local project_path="gitlab-org/gitlab"
local artifact_branch="master"
local test_metadata_with_mapping_job_id
test_metadata_with_mapping_job_id=$(scripts/api/get_job_id.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" -q "status=success" -q "ref=${artifact_branch}" -q "username=gitlab-bot" -Q "scope=success" --job-name "update-tests-metadata" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz")
if [[ ! -f "${RSPEC_PACKED_TESTS_MAPPING_PATH}" ]]; then
(scripts/api/download_job_artifact.rb --endpoint "https://gitlab.com/api/v4" --project "${project_path}" --job-id "${test_metadata_with_mapping_job_id}" --artifact-path "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz" && gzip -d "${RSPEC_PACKED_TESTS_MAPPING_PATH}.gz") || echo "{}" > "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
fi
fi
scripts/unpack-test-mapping "${RSPEC_PACKED_TESTS_MAPPING_PATH}" "${RSPEC_TESTS_MAPPING_PATH}"
}
function update_tests_mapping() {
if ! crystalball_rspec_data_exists; then
echo "No crystalball rspec data found."
return 0
fi
scripts/generate-test-mapping "${RSPEC_TESTS_MAPPING_PATH}" crystalball/rspec*.yml
scripts/pack-test-mapping "${RSPEC_TESTS_MAPPING_PATH}" "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
gzip "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
rm -f crystalball/rspec*.yml "${RSPEC_PACKED_TESTS_MAPPING_PATH}"
}
function crystalball_rspec_data_exists() {
compgen -G "crystalball/rspec*.yml" >/dev/null
}
function retrieve_previous_failed_tests() {
local directory_for_output_reports="${1}"
local rspec_pg_regex="${2}"
local rspec_ee_pg_regex="${3}"
local pipeline_report_path="test_results/previous/test_reports.json"
local project_path="gitlab-org/gitlab"
echo 'Attempting to build pipeline test report...'
scripts/pipeline_test_report_builder.rb --instance-base-url "https://gitlab.com" --project "${project_path}" --mr-id "${CI_MERGE_REQUEST_IID}" --output-file-path "${pipeline_report_path}"
echo 'Generating failed tests lists...'
scripts/failed_tests.rb --previous-tests-report-path "${pipeline_report_path}" --output-directory "${directory_for_output_reports}" --rspec-pg-regex "${rspec_pg_regex}" --rspec-ee-pg-regex "${rspec_ee_pg_regex}"
}
function rspec_simple_job() {
local rspec_opts="${1}"
export NO_KNAPSACK="1"
bin/rspec -Ispec -rspec_helper --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}
}
function rspec_db_library_code() {
local db_files="spec/lib/gitlab/database/"
rspec_simple_job "-- ${db_files}"
}
function rspec_paralellized_job() {
read -ra job_name <<< "${CI_JOB_NAME}"
local test_tool="${job_name[0]}"
local test_level="${job_name[1]}"
local report_name=$(echo "${CI_JOB_NAME}" | sed -E 's|[/ ]|_|g') # e.g. 'rspec unit pg12 1/24' would become 'rspec_unit_pg12_1_24'
local rspec_opts="${1}"
local spec_folder_prefixes=""
if [[ "${test_tool}" =~ "-ee" ]]; then
spec_folder_prefixes="'ee/'"
fi
if [[ "${test_tool}" =~ "-jh" ]]; then
spec_folder_prefixes="'jh/'"
fi
if [[ "${test_tool}" =~ "-all" ]]; then
spec_folder_prefixes="['', 'ee/', 'jh/']"
fi
export KNAPSACK_LOG_LEVEL="debug"
export KNAPSACK_REPORT_PATH="knapsack/${report_name}_report.json"
# There's a bug where artifacts are sometimes not downloaded. Since specs can run without the Knapsack report, we can
# handle the missing artifact gracefully here. See https://gitlab.com/gitlab-org/gitlab/-/issues/212349.
if [[ ! -f "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" ]]; then
echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
fi
cp "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "${KNAPSACK_REPORT_PATH}"
if [[ -z "${KNAPSACK_TEST_FILE_PATTERN}" ]]; then
pattern=$(ruby -r./tooling/quality/test_level.rb -e "puts Quality::TestLevel.new(${spec_folder_prefixes}).pattern(:${test_level})")
export KNAPSACK_TEST_FILE_PATTERN="${pattern}"
fi
echo "KNAPSACK_TEST_FILE_PATTERN: ${KNAPSACK_TEST_FILE_PATTERN}"
if [[ -d "ee/" ]]; then
export KNAPSACK_GENERATE_REPORT="true"
export FLAKY_RSPEC_GENERATE_REPORT="true"
export SUITE_FLAKY_RSPEC_REPORT_PATH="${FLAKY_RSPEC_SUITE_REPORT_PATH}"
export FLAKY_RSPEC_REPORT_PATH="rspec_flaky/all_${report_name}_report.json"
export NEW_FLAKY_RSPEC_REPORT_PATH="rspec_flaky/new_${report_name}_report.json"
if [[ ! -f $FLAKY_RSPEC_REPORT_PATH ]]; then
echo "{}" > "${FLAKY_RSPEC_REPORT_PATH}"
fi
if [[ ! -f $NEW_FLAKY_RSPEC_REPORT_PATH ]]; then
echo "{}" > "${NEW_FLAKY_RSPEC_REPORT_PATH}"
fi
fi
mkdir -p tmp/memory_test
export MEMORY_TEST_PATH="tmp/memory_test/${report_name}_memory.csv"
local rspec_args="-Ispec -rspec_helper --color --format documentation --format RspecJunitFormatter --out junit_rspec.xml ${rspec_opts}"
if [[ -n $RSPEC_TESTS_MAPPING_ENABLED ]]; then
tooling/bin/parallel_rspec --rspec_args "${rspec_args}" --filter "tmp/matching_tests.txt"
else
tooling/bin/parallel_rspec --rspec_args "${rspec_args}"
fi
date
}
function rspec_rerun_previous_failed_tests() {
local test_file_count_threshold=${RSPEC_PREVIOUS_FAILED_TEST_FILE_COUNT_THRESHOLD:-10}
local matching_tests_file=${1}
local rspec_opts=${2}
local test_files="$(cat "${matching_tests_file}")"
local test_file_count=$(wc -w "${matching_tests_file}" | awk {'print $1'})
if [[ "${test_file_count}" -gt "${test_file_count_threshold}" ]]; then
echo "This job is intentionally exited because there are more than ${test_file_count_threshold} test files to rerun."
exit 0
fi
if [[ -n $test_files ]]; then
rspec_simple_job "${test_files}"
else
echo "No failed test files to rerun"
fi
}
function rspec_fail_fast() {
local test_file_count_threshold=${RSPEC_FAIL_FAST_TEST_FILE_COUNT_THRESHOLD:-10}
local matching_tests_file=${1}
local rspec_opts=${2}
local test_files="$(cat "${matching_tests_file}")"
local test_file_count=$(wc -w "${matching_tests_file}" | awk {'print $1'})
if [[ "${test_file_count}" -gt "${test_file_count_threshold}" ]]; then
echo "This job is intentionally skipped because there are more than ${test_file_count_threshold} test files matched,"
echo "which would take too long to run in this job."
echo "All the tests would be run in other rspec jobs."
exit 0
fi
if [[ -n $test_files ]]; then
rspec_simple_job "${rspec_opts} ${test_files}"
else
echo "No rspec fail-fast tests to run"
fi
}
function rspec_matched_foss_tests() {
local test_file_count_threshold=20
local matching_tests_file=${1}
local rspec_opts=${2}
local test_files="$(cat "${matching_tests_file}")"
local test_file_count=$(wc -w "${matching_tests_file}" | awk {'print $1'})
if [[ "${test_file_count}" -gt "${test_file_count_threshold}" ]]; then
echo "This job is intentionally failed because there are more than ${test_file_count_threshold} FOSS test files matched,"
echo "which would take too long to run in this job."
echo "To reduce the likelihood of breaking FOSS pipelines,"
echo "please add ~\"pipeline:run-as-if-foss\" label to the merge request and trigger a new pipeline."
echo "This would run all as-if-foss jobs in this merge request"
echo "and remove this failing job from the pipeline."
exit 1
fi
if [[ -n $test_files ]]; then
rspec_simple_job "${rspec_opts} ${test_files}"
else
echo "No impacted FOSS rspec tests to run"
fi
}
|