1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
|
# Common setup for bulk test runs
: ${ALL:=0}
# Strip away @DEFAULT if it is the default FLAVOR
fix_default_flavor() {
local _originspec="$1"
local var_return="$2"
local _origin _flavor _flavors _default_flavor
originspec_decode "${_originspec}" _origin '' _flavor
[ -z "${_flavor}" ] && return 0
hash_get origin-flavors "${_origin}" _flavors
_default_flavor="${_flavors%% *}"
[ "${_flavor}" = "${FLAVOR_DEFAULT}" ] && _flavor="${_default_flavor}"
if [ "${_flavor}" != "${FLAVOR_ALL}" ]; then
[ "${_default_flavor}" = "${_flavor}" ] || return 0
fi
setvar "${var_return}" "${_origin}"
}
# Cache all pkgnames involved. Being single-threaded this is trivial.
# Return 0 to skip the port
# Return 1 to not skip the port
cache_pkgnames() {
local originspec="$1"
local origin dep_origin flavor flavors pkgname default_flavor ignore
local flavor_originspec ret
if hash_get originspec-pkgname "${originspec}" pkgname; then
hash_get originspec-ignore "${originspec}" ignore
ret=1
[ -n "${ignore}" ] && ret=0
return ${ret}
fi
originspec_decode "${originspec}" origin '' flavor
if [ "${flavor}" = "${FLAVOR_DEFAULT}" ]; then
originspec_encode originspec "${origin}" '' ''
elif [ "${flavor}" = "${FLAVOR_ALL}" ]; then
unset flavor
originspec_encode originspec "${origin}" '' ''
fi
port_var_fetch_originspec "${originspec}" \
PKGNAME pkgname \
FLAVORS flavors \
IGNORE ignore \
_PDEPS='${PKG_DEPENDS} ${EXTRACT_DEPENDS} ${PATCH_DEPENDS} ${FETCH_DEPENDS} ${BUILD_DEPENDS} ${LIB_DEPENDS} ${RUN_DEPENDS}' \
'${_PDEPS:C,([^:]*):([^:]*):?.*,\2,:C,^${PORTSDIR}/,,:O:u}' \
pdeps
hash_set origin-flavors "${origin}" "${flavors}"
fix_default_flavor "${originspec}" originspec
hash_set originspec-pkgname "${originspec}" "${pkgname}"
hash_set originspec-deps "${originspec}" "${pdeps}"
hash_set originspec-ignore "${originspec}" "${ignore}"
# Record all known packages for comparing to the queue later.
ALL_PKGNAMES="${ALL_PKGNAMES}${ALL_PKGNAMES:+ }${pkgname}"
ALL_ORIGINS="${ALL_ORIGINS}${ALL_ORIGINS:+ }${originspec}"
[ -n "${ignore}" ] && list_add IGNOREDPORTS "${originspec}"
for dep_origin in ${pdeps}; do
if cache_pkgnames "${dep_origin}"; then
if ! list_contains SKIPPEDPORTS "${originspec}"; then
list_add SKIPPEDPORTS "${originspec}"
fi
fi
done
# Also cache all of the FLAVOR deps/PKGNAMES
if [ -n "${flavor}" ]; then
default_flavor="${flavors%% *}"
for flavor in ${flavors}; do
# Don't recurse on the first flavor since we are it.
[ "${flavor}" = "${default_flavor}" ] && continue
originspec_encode flavor_originspec "${origin}" '' "${flavor}"
cache_pkgnames "${flavor_originspec}" || :
done
fi
[ -n "${ignore}" ]
}
expand_origin_flavors() {
local origins="$1"
local var_return="$2"
local originspec origin flavor flavors _expanded
for originspec in ${origins}; do
originspec_decode "${originspec}" origin '' flavor
hash_get origin-flavors "${origin}" flavors || flavors=
if [ -n "${flavor}" -a "${flavor}" != "${FLAVOR_ALL}" ] || \
[ -z "${flavors}" ] || \
[ "${FLAVOR_DEFAULT_ALL}" != "yes" -a \
${ALL} -eq 0 -a \
"${flavor}" != "${FLAVOR_ALL}" ]; then
_expanded="${_expanded:+${_expanded} }${originspec}"
continue
fi
# Add all FLAVORS in for this one
for flavor in ${flavors}; do
originspec_encode originspec "${origin}" '' "${flavor}"
_expanded="${_expanded:+${_expanded} }${originspec}"
done
done
setvar "${var_return}" "${_expanded}"
}
list_all_deps() {
local origins="$1"
local var_return="$2"
local originspec origin _out flavors deps
local dep_originspec dep_origin dep_flavor dep_flavors
local dep_default_flavor
# Don't list 'recursed' local or setvar won't work to parent
[ "${var_return}" = recursed ] || _out=
for originspec in ${origins}; do
# If it's already in the list, nothing to do
case " ${_out} " in
*\ ${originspec}\ *) continue ;;
esac
_out="${_out:+${_out} }${originspec}"
originspec_decode "${originspec}" origin '' flavor
flavors=
[ -z "${flavor}" ] && \
hash_get origin-flavors "${origin}" flavors
fix_default_flavor "${originspec}" originspec
# Check all deps
hash_get originspec-deps "${originspec}" deps || deps=
for dep_originspec in ${deps}; do
# If the dependency has flavors and is not
# FLAVOR-specific, it needs to be changed to
# depend on the default FLAVOR instead.
originspec_decode "${dep_originspec}" dep_origin \
'' dep_flavor
if [ -z "${dep_flavor}" ]; then
hash_get origin-flavors \
"${dep_origin}" dep_flavors || \
dep_flavors=
if [ -n "${dep_flavors}" ]; then
# Change to default
dep_default_flavor="${dep_flavors%% *}"
dep_flavor="${dep_default_flavor}"
originspec_encode dep_originspec \
"${dep_origin}" '' "${dep_flavor}"
fi
fi
recursed=
list_all_deps "${dep_originspec}" recursed
_out="${recursed}"
done
# And all FLAVORS if needed
if [ -n "${flavor}" ]; then
orig_originspec="${originspec}"
for flavor in ${flavors}; do
originspec_encode originspec "${origin}" '' \
"${flavor}"
recursed=
list_all_deps "${originspec}" recursed
_out="${recursed}"
done
fi
done
_out="${_out# }"
_out="${_out% }"
setvar "${var_return}" "${_out}"
}
assert_queued() {
local dep="$1"
local origins="$2"
local tmp originspec origins_expanded
tmp="$(mktemp -t queued.${dep})"
awk -v dep="${dep}" '$3 == dep' "${log}/.poudriere.ports.queued" \
> "${tmp}"
# First fix the list to expand main port FLAVORS
expand_origin_flavors "${origins}" origins_expanded
# The queue does remove duplicates - do the same here
origins_expanded="$(echo "${origins_expanded}" | tr ' ' '\n' | sort -u | paste -s -d ' ' -)"
echo "Asserting that only '${origins_expanded}' are in the${dep:+ ${dep}} queue"
for originspec in ${origins_expanded}; do
fix_default_flavor "${originspec}" originspec
hash_get originspec-pkgname "${originspec}" pkgname
assert_not '' "${pkgname}" "PKGNAME needed for ${originspec}"
echo "=> Asserting that ${originspec} | ${pkgname} is${dep:+ dep=${dep}} in queue"
awk -vpkgname="${pkgname}" -voriginspec="${originspec}" -vdep="${dep}" '
$1 == originspec && $2 == pkgname && (dep == "" || $3 == dep) {
print "==> " $0
if (found == 1) {
# A duplicate, no good.
found = 0
exit 1
}
found = 1
next
}
$1 == originspec && $2 == pkgname && dep != "" && $3 != dep {
print "==> " $0
found = 0
exit 1
}
END { if (found != 1) exit 1 }
' ${log}/.poudriere.ports.queued >&2
assert 0 $? "${originspec} | ${pkgname} should be queued${dep:+ with dep=${dep}}"
# Remove the entry so we can assert later that nothing extra
# is in the queue.
cat "${tmp}" | \
awk -vpkgname="${pkgname}" -voriginspec="${originspec}" \
-vdep="${dep}" '
$1 == originspec && $2 == pkgname && $3 == dep { next }
{ print }
' > "${tmp}.new"
mv -f "${tmp}.new" "${tmp}"
done
echo "=> Asserting that nothing else is in the${dep:+ ${dep}} queue"
cat "${tmp}" | sed -e 's,^,==> ,' >&2
! [ -s "${tmp}" ]
assert 0 $? "Queue should be empty"
rm -f "${tmp}"
}
assert_ignored() {
local origins="$1"
local tmp originspec origins_expanded
tmp="$(mktemp -t queued.${dep})"
cp -f "${log}/.poudriere.ports.ignored" "${tmp}"
# First fix the list to expand main port FLAVORS
expand_origin_flavors "${origins}" origins_expanded
# The queue does remove duplicates - do the same here
origins_expanded="$(echo "${origins_expanded}" | tr ' ' '\n' | sort -u | paste -s -d ' ' -)"
echo "Asserting that only '${origins_expanded}' are in the ignored list"
for originspec in ${origins_expanded}; do
fix_default_flavor "${originspec}" originspec
hash_get originspec-pkgname "${originspec}" pkgname
assert_not '' "${pkgname}" "PKGNAME needed for ${originspec}"
echo "=> Asserting that ${originspec} | ${pkgname} is ignored"
awk -vpkgname="${pkgname}" -voriginspec="${originspec}" '
$1 == originspec && $2 == pkgname && ($3 == "ignored") {
print "==> " $0
if (found == 1) {
# A duplicate, no good.
found = 0
exit 1
}
found = 1
next
}
END { if (found != 1) exit 1 }
' ${log}/.poudriere.ports.ignored >&2
assert 0 $? "${originspec} | ${pkgname} should be ignored"
# Remove the entry so we can assert later that nothing extra
# is in the queue.
cat "${tmp}" | \
awk -vpkgname="${pkgname}" -voriginspec="${originspec}" '
$1 == originspec && $2 == pkgname && $3 == "ignored" { next }
{ print }
' > "${tmp}.new"
mv -f "${tmp}.new" "${tmp}"
done
echo "=> Asserting that nothing else is ignored"
cat "${tmp}" | sed -e 's,^,==> ,' >&2
! [ -s "${tmp}" ]
assert 0 $? "Ignore list should be empty"
rm -f "${tmp}"
}
assert_skipped() {
local origins="$1"
local tmp originspec origins_expanded
tmp="$(mktemp -t queued.${dep})"
cp -f "${log}/.poudriere.ports.skipped" "${tmp}"
# First fix the list to expand main port FLAVORS
expand_origin_flavors "${origins}" origins_expanded
# The queue does remove duplicates - do the same here
origins_expanded="$(echo "${origins_expanded}" | tr ' ' '\n' | sort -u | paste -s -d ' ' -)"
echo "Asserting that only '${origins_expanded}' are in the skipped list"
for originspec in ${origins_expanded}; do
fix_default_flavor "${originspec}" originspec
hash_get originspec-pkgname "${originspec}" pkgname
assert_not '' "${pkgname}" "PKGNAME needed for ${originspec}"
echo "=> Asserting that ${originspec} | ${pkgname} is skipped"
awk -vpkgname="${pkgname}" -voriginspec="${originspec}" '
$1 == originspec && $2 == pkgname {
print "==> " $0
if (found == 1) {
# A duplicate, no good.
found = 0
exit 1
}
found = 1
next
}
END { if (found != 1) exit 1 }
' ${log}/.poudriere.ports.skipped >&2
assert 0 $? "${originspec} | ${pkgname} should be skipped"
# Remove the entry so we can assert later that nothing extra
# is in the queue.
cat "${tmp}" | \
awk -vpkgname="${pkgname}" -voriginspec="${originspec}" '
$1 == originspec && $2 == pkgname { next }
{ print }
' > "${tmp}.new"
mv -f "${tmp}.new" "${tmp}"
done
echo "=> Asserting that nothing else is skipped"
cat "${tmp}" | sed -e 's,^,==> ,' >&2
! [ -s "${tmp}" ]
assert 0 $? "Skipped list should be empty"
rm -f "${tmp}"
}
assert_counts() {
local queued expected_queued ignored expected_ignored
local skipped expected_skipped
if [ -z "${ALL_EXPECTED}" ]; then
expected_queued=0
else
expected_queued=$(echo "${ALL_EXPECTED}" | tr ' ' '\n' | wc -l)
expected_queued="${expected_queued##* }"
fi
if [ -z "${IGNOREDPORTS}" ]; then
expected_ignored=0
else
expected_ignored=$(echo "${IGNOREDPORTS}" | tr ' ' '\n' | wc -l)
expected_ignored="${expected_ignored##* }"
fi
if [ -z "${SKIPPEDPORTS}" ]; then
expected_skipped=0
else
expected_skipped=$(echo "${SKIPPEDPORTS}" | tr ' ' '\n' | wc -l)
expected_skipped="${expected_skipped##* }"
fi
expected_queued=$((expected_queued + expected_ignored + expected_skipped))
echo "=> Asserting queued=${expected_queued} ignored=${expected_ignored} skipped=${expected_skipped}"
read queued < "${log}/.poudriere.stats_queued"
assert 0 $? "${log}/.poudriere.stats_queued read should pass"
assert "${expected_queued}" "${queued}" "queued should match"
read ignored < "${log}/.poudriere.stats_ignored"
assert 0 $? "${log}/.poudriere.stats_ignored read should pass"
assert "${expected_ignored}" "${ignored}" "ignored should match"
read skipped < "${log}/.poudriere.stats_skipped"
assert 0 $? "${log}/.poudriere.stats_skipped read should pass"
assert "${expected_skipped}" "${skipped}" "skipped should match"
}
# Avoid injail() for port_var_fetch
INJAIL_HOST=1
. common.sh
SUDO=
if [ $(id -u) -ne 0 ]; then
if ! which sudo >/dev/null 2>&1; then
echo "SKIP: Need root or sudo access for bulk tests" >&2
exit 1
fi
SUDO="sudo"
fi
: ${BUILDNAME:=${0%.sh}}
POUDRIERE="${POUDRIEREPATH} -e ${THISDIR}/etc"
ARCH=$(uname -p)
JAILNAME="poudriere-test-${ARCH}$(echo "${THISDIR}" | tr '/' '_')"
JAIL_VERSION="11.3-RELEASE"
JAILMNT=$(${POUDRIERE} api "jget ${JAILNAME} mnt" || echo)
export UNAME_r=$(freebsd-version)
export UNAME_v="FreeBSD $(freebsd-version)"
if [ -z "${JAILMNT}" ]; then
echo "Setting up jail for testing..." >&2
if ! ${SUDO} ${POUDRIERE} jail -c -j "${JAILNAME}" \
-v "${JAIL_VERSION}" -a ${ARCH}; then
echo "SKIP: Cannot setup jail with Poudriere" >&2
exit 1
fi
JAILMNT=$(${POUDRIERE} api "jget ${JAILNAME} mnt" || echo)
if [ -z "${JAILMNT}" ]; then
echo "SKIP: Failed fetching mnt for new jail in Poudriere" >&2
exit 1
fi
echo "Done setting up test jail" >&2
echo >&2
fi
if [ ${BOOTSTRAP_ONLY:-0} -eq 1 ]; then
exit 0
fi
if [ ${ALL:-0} -eq 0 ]; then
assert_not "" "${LISTPORTS}" "LISTPORTS empty"
fi
. ${SCRIPTPREFIX}/common.sh
: ${PORTSDIR:=${THISDIR}/../test-ports/default}
export PORTSDIR
PTMNT="${PORTSDIR}"
: ${PTNAME:=test}
: ${SETNAME:=}
export PORT_DBDIR=/dev/null
export __MAKE_CONF="${POUDRIERE_ETC}/poudriere.d/make.conf"
export SRCCONF=/dev/null
export SRC_ENV_CONF=/dev/null
set -e
# Import local ports tree
pset "${PTNAME}" mnt "${PTMNT}"
pset "${PTNAME}" method "-"
MASTERNAME=${JAILNAME}-${PTNAME}${SETNAME:+-${SETNAME}}
_mastermnt MASTERMNT
export POUDRIERE_BUILD_TYPE=bulk
_log_path log
# Setup basic overlay to test-ports/overlay/ dir.
for o in ${OVERLAYS}; do
[ -d "${PTMNT%/*}/${o}" ] || continue
pset "${o}" mnt "${PTMNT%/*}/${o}"
pset "${o}" method "-"
# We run port_var_fetch_originspec without a jail so can't use plain
# /overlays. Need to link the host path into our fake MASTERMNT path
# as well as link to the overlay portdir without nullfs.
OVERLAYSDIR="$(mktemp -ut overlays)"
mkdir -p "${MASTERMNT}/${OVERLAYSDIR%/*}"
ln -fs "${MASTERMNT}/${OVERLAYSDIR}" "${OVERLAYSDIR}"
mkdir -p "${MASTERMNT}/${OVERLAYSDIR}"
ln -fs "${PTMNT%/*}/${o}" "${MASTERMNT}/${OVERLAYSDIR}/${o}"
done
echo -n "Pruning previous logs..."
${SUDO} ${POUDRIEREPATH} -e ${POUDRIERE_ETC} logclean -B "${BUILDNAME}" -ay \
>/dev/null
echo " done"
set +e
ALL_PKGNAMES=
ALL_ORIGINS=
if [ ${ALL} -eq 1 ]; then
LISTPORTS="$(listed_ports | paste -s -d ' ' -)"
fi
echo -n "Gathering metadata for requested ports..."
IGNOREDPORTS=""
SKIPPEDPORTS=""
for origin in ${LISTPORTS}; do
cache_pkgnames "${origin}" || :
done
echo " done"
expand_origin_flavors "${LISTPORTS}" LISTPORTS_EXPANDED
LISTPORTS_NOIGNORED="${LISTPORTS_EXPANDED}"
# Separate out IGNORED ports
if [ -n "${IGNOREDPORTS}" ]; then
_IGNOREDPORTS="${IGNOREDPORTS}"
for port in ${_IGNOREDPORTS}; do
list_remove LISTPORTS_NOIGNORED "${port}"
list_remove SKIPPEDPORTS "${port}"
done
fi
# Separate out SKIPPED ports
if [ -n "${SKIPPEDPORTS}" ]; then
_SKIPPEDPORTS="${SKIPPEDPORTS}"
for port in ${_SKIPPEDPORTS}; do
list_remove LISTPORTS_NOIGNORED "${port}"
done
fi
echo "Building: $(echo ${LISTPORTS_EXPANDED})"
|