factorize package building between bulk and testport

both bulk and testport use the same function to build packages.
both calculate dependencies and order them and build them one by one
while here create a injail help to avoir typos in jexec...

3 files changed, 90 insertions, 85 deletions

diff --git a/src/poudriere.d/bulk.sh b/src/poudriere.d/bulk.sh
index 18fecefd..4e9388ec 100755
--- a/src/poudriere.d/bulk.sh
+++ b/src/poudriere.d/bulk.sh
@@ -69,7 +69,7 @@ for JAILNAME in ${JAILNAMES}; do
 
 	if [ ${KEEP} -ne 1 ]; then
 		msg_n "Cleaning previous bulks if any..."
-		rm -rf ${POUDRIERE_DATA}/packages/bulk-${JAILNAME}/*
+		rm -rf ${PKGDIR}/*
 		echo " done"
 	fi
 
@@ -83,74 +83,25 @@ for JAILNAME in ${JAILNAMES}; do
 	tpid=$!
 	exec > ${PIPE} 2>&1
 
-	zfs snapshot ${JAILFS}@bulk
 	msg "Calculating ports order and dependencies"
-	prepare_ports | while read port; do
-		PORTDIRECTORY="/usr/ports/${port}"
-
-		test -d ${JAILBASE}/${PORTDIRECTORY} || {
-			msg "No such port ${port}"
-			continue
-		}
-		LATEST_LINK=$(jexec -U root ${JAILNAME} make -C ${PORTDIRECTORY} -VLATEST_LINK)
-		PKGNAME=$(jexec -U root ${JAILNAME} make -C ${PORTDIRECTORY} -VPKGNAME)
-
-		# delete older one if any
-		if [ -e ${POUDRIERE_DATA}/packages/bulk-${JAILNAME}/Latest/${LATEST_LINK}.${EXT} ]; then
-			PKGNAME_PREV=$(realpath ${POUDRIERE_DATA}/packages/bulk-${JAILNAME}/Latest/${LATEST_LINK}.${EXT})
-			if [ "${PKGNAME_PREV##*/}" = "${PKGNAME}.${EXT}" ]; then
-				msg "$PKGNAME already packaged skipping"
-				continue
-			else
-				msg "Deleting previous version of ${port}"
-				find ${POUDRIERE_DATA}/packages/bulk-${JAILNAME}/ -name ${PKGNAME_PREV##*/} -delete
-				find ${POUDRIERE_DATA}/packages/bulk-${JAILNAME}/ -name ${LATEST_LINK}.${EXT} -delete
-			fi
-		fi
-
-		zfs rollback ${JAILFS}@bulk
-		rm -rf ${JAILBASE}/wrkdirs/*
-		msg "building ${port}"
-		jexec -U root ${JAILNAME} make -C ${PORTDIRECTORY} clean install
-		if [ $? ]; then
-			STATS_BUILT=$((STATS_BUILT+1))
-		else
-			STATS_FAILED=$((STATS_FAILED+1))
-			FAILED_PORTS="$FAILED_PORTS ${PORTDIRECTORY#*/usr/ports/}"
-		fi
-		msg "packaging"
-		if [ $PKGNG -eq 1 ]; then
-			for pkg in `jexec -U root ${JAILNAME} /usr/sbin/pkg info -qa`; do
-				[ -f ${POUDRIERE_DATA}/packages/bulk-${JAILNAME}/All/${pkg}.${EXT} ] && continue
-				msg "packaging ${pkg}"
-				pkgorig=`jexec -U root ${JAILNAME} /usr/sbin/pkg info -q -o ${pkg}`
-				jexec -U root ${JAILNAME} make -C /usr/ports/${pkgorig} package || continue
-			done
-		else
-			for pkg in `jexec -U root ${JAILNAME} /usr/sbin/pkg_info | awk '{ print $1 }'`; do
-				[ -f ${POUDRIERE_DATA}/packages/bulk-${JAILNAME}/All/${pkg}.${EXT} ] && continue
-				msg "packaging ${pkg}"
-				pkgorig=`jexec -U root ${JAILNAME} /usr/sbin/pkg_info -qo ${pkg}`
-				jexec -U root ${JAILNAME} make -C /usr/ports/${pkgorig} package
-			done
-		fi
+	for port in `prepare_ports`; do
+		build_pkg ${port} || :
 	done
-	zfs destroy ${JAILFS}@bulk 2>/dev/null || :
 
 # Package all newly build ports
 	if [ $STATS_BUILT -eq 0 ]; then
 		msg "No package built, no need to update INDEX"
 	elif [ $PKGNG -eq 1 ]; then
 		msg "Packaging all installed ports"
-		jexec -U root ${JAILNAME} /usr/sbin/pkg repo /usr/ports/packages/All/
+		injail /usr/sbin/pkg repo /usr/ports/packages/All/
 	else
 		msg "Preparing index"
-		OSMAJ=`jexec -U root ${JAILNAME} uname -r | awk -F. '{ print $1 }'`
-		INDEXF=${POUDRIERE_DATA}/packages/bulk-${JAILNAME}/INDEX-${OSMAJ}
-		for pkg_file in `ls ${POUDRIERE_DATA}/packages/bulk-${JAILNAME}/All/*.tbz`; do
+		OSMAJ=`injail ${JAILNAME} uname -r | awk -F. '{ print $1 }'`
+		INDEXF=${PKGDIR}/INDEX-${OSMAJ}
+		for pkg_file in `ls ${PKGDIR}/All/*.tbz`; do
 			msg_n "extracting description from `basename ${pkg_file}`"
 			ORIGIN=`/usr/sbin/pkg_info -qo ${pkg_file}`
-			[ -d ${POUDRIERE_PORTSDIR}/${ORIGIN} ] && jexec -U root ${JAILNAME} make -C /usr/ports/${ORIGIN} describe >> ${INDEXF}.1
+			[ -d ${POUDRIERE_PORTSDIR}/${ORIGIN} ] && injail make -C /usr/ports/${ORIGIN} describe >> ${INDEXF}.1
 			echo " done"
 		done
 
diff --git a/src/poudriere.d/common.sh b/src/poudriere.d/common.sh
index 379c7aaa..f2200e0d 100644
--- a/src/poudriere.d/common.sh
+++ b/src/poudriere.d/common.sh
@@ -312,21 +312,64 @@ cleanup() {
 	[ -e ${PIPE} ] && rm -f ${PIPE}
 	FS=`jail_get_fs ${JAILNAME}`
 	zfs destroy ${FS}@bulk 2>/dev/null || :
+	zfs destroy ${FS}@prepkg 2>/dev/null || :
 	zfs destroy ${FS}@prebuild 2>/dev/null || :
 	jail_stop ${JAILNAME}
 }
 
+injail() {
+	jexec -U root ${JAILNAME} $@
+}
+
+build_pkg() {
+	local port=$1
+	local portdir="/usr/ports/${port}"
+	test -d ${JAILBASE}/${portdir} || {
+		msg "No such port ${port}"
+		return 1
+	}
+	local LATEST_LINK=$(injail make -C ${portdir} -VLATEST_LINK)
+	local PKGNAME=$(injail make -C ${portdir} -VPKGNAME)
+
+	# delete older one if any
+	if [ -e ${PKGDIR}/Latest/${LATEST_LINK}.${EXT} ]; then
+		PKGNAME_PREV=$(realpath ${PKGDIR}/Latest/${LATEST_LINK}.${EXT})
+		if [ "${PKGNAME_PREV##*/}" = "${PKGNAME}.${EXT}" ]; then
+			msg "$PKGNAME already packaged skipping"
+			return 2
+		else
+			msg "Deleting previous version of ${port}"
+			find ${PKGDIR}/ -name ${PKGNAME_PREV##*/} -delete
+			find ${PKGDIR}/ -name ${LATEST_LINK}.${EXT} -delete
+		fi
+	fi
+
+	msg "Cleaning up wrkdir"
+	rm -rf ${JAILBASE}/wrkdirs/*
+
+	msg "Building ${port}"
+	injail make -C ${portdir} clean package
+	if [ $? -eq 0 ]; then
+		STATS_BUILT=$(($STATS_BUILT + 1))
+		return 0
+	else
+		STATS_FAILED=$(($STATS_FAILED + 1))
+		FAILED_PORTS="$FAILED_PORTS ${PORTDIRECTORY#*/usr/ports/}"
+		return 1
+	fi
+}
+
 process_deps() {
 	tmplist=$1
 	tmplist2=$2
 	tmplist3=$3
 	local port=$4
-	PORTDIRECTORY="/usr/ports/${port}"
+	local PORTDIRECTORY="/usr/ports/${port}"
 	grep -q "$port" ${tmplist} && return
 	echo $port >> ${tmplist}
 	deps=0
 	local m
-	for m in `jexec -U root ${JAILNAME} make -C ${PORTDIRECTORY} missing`; do
+	for m in `injail make -C ${PORTDIRECTORY} missing`; do
 		process_deps "${tmplist}" "${tmplist2}" "${tmplist3}" "$m"
 		echo $m $port >> ${tmplist2}
 		deps=1
@@ -341,9 +384,15 @@ prepare_ports() {
 	tmplist2=`mktemp /tmp/orderport2.XXXXX`
 	tmplist3=`mktemp /tmp/orderport3.XXXXX`
 	touch ${tmplist}
-	for port in `grep -v -E '(^[[:space:]]*#|^[[:space:]]*$)' ${LISTPKGS}`; do
-		process_deps "${tmplist}" "${tmplist2}" "$tmplist3" "${port}"
-	done
+	if [ -z "${LISTPORTS}" ]; then
+		for port in `grep -v -E '(^[[:space:]]*#|^[[:space:]]*$)' ${LISTPKGS}`; do
+			process_deps "${tmplist}" "${tmplist2}" "$tmplist3" "${port}"
+		done
+	else
+		for port in ${LISTPORTS}; do
+			process_deps "${tmplist}" "${tmplist2}" "$tmplist3" "${port}"
+		done
+	fi
 	tsort ${tmplist2} | while read port; do
 		grep -q ${port} ${tmplist3} || echo $port >> ${tmplist3}
 	done
@@ -383,7 +432,7 @@ prepare_jail() {
 	fi
 
 	msg "Populating LOCALBASE"
-	jexec -U root ${JAILNAME} /usr/sbin/mtree -q -U -f /usr/ports/Templates/BSD.local.dist -d -e -p /usr/local >/dev/null
+	injail /usr/sbin/mtree -q -U -f /usr/ports/Templates/BSD.local.dist -d -e -p /usr/local >/dev/null
 }
 
 RESOLV_CONF=""
diff --git a/src/poudriere.d/test_ports.sh b/src/poudriere.d/test_ports.sh
index 3815f7b9..78d65774 100755
--- a/src/poudriere.d/test_ports.sh
+++ b/src/poudriere.d/test_ports.sh
@@ -20,26 +20,26 @@ EOF
 
 build_port() {
 	msg "Building ${PKGNAME}"
-	jexec -U root ${JAILNAME} mkdir -p /tmp/pkgs
+	injail mkdir -p /tmp/pkgs
 	for PHASE in build install package deinstall
 	do
 		if [ "${PHASE}" = "deinstall" ]; then
 			msg "Checking pkg_info"
-			PKG_DBDIR=${PKG_DBDIR} jexec -U root ${JAILNAME} /usr/sbin/pkg_info ${PKGNAME}
+			PKG_DBDIR=${PKG_DBDIR} injail /usr/sbin/pkg_info ${PKGNAME}
 			PLIST="${PKG_DBDIR}/${PKGNAME}/+CONTENTS"
 			if [ -r ${JAILBASE}${PLIST} ]; then
 				echo "===>> Checking shared library dependencies"
 				grep -v "^@" ${JAILBASE}${PLIST} | \
 				sed -e "s,^,${PREFIX}/," | \
-				xargs jexec -U root ${JAILNAME} ldd 2>&1 | \
+				xargs injail ldd 2>&1 | \
 				grep -v "not a dynamic executable" | \
 				grep '=>' | awk '{ print $3;}' | sort -u
 			fi
 		fi
-		jexec -U root ${JAILNAME} env PACKAGES=/tmp/pkgs PKGREPOSITORY=/tmp/pkgs make -C ${PORTDIRECTORY} ${PORT_FLAGS} ${PHASE}
+		injail env PACKAGES=/tmp/pkgs PKGREPOSITORY=/tmp/pkgs make -C ${PORTDIRECTORY} ${PORT_FLAGS} ${PHASE}
 		if [ "${PHASE}" = "build" ]; then
 			msg "Installing run dependencies"
-			jexec -U root ${JAILNAME} make -C ${PORTDIRECTORY} run-depends || \
+			injail make -C ${PORTDIRECTORY} run-depends || \
 				(create_pkg "Packaging what is installed so far"; exit 1)
 			create_pkg "Packaging all run dependencies"
 			[ $ZVERSION -ge 28 ] && zfs snapshot ${JAILFS}@prebuild
@@ -50,8 +50,8 @@ build_port() {
 
 create_pkg() {
 	msg "$1" | tee -a ${LOGS}/${PKGNAME}-${JAILNAME}.depends.log
-	for pkg in `jexec -U root ${JAILNAME} /usr/sbin/pkg_info -Ea`; do
-		[ -f ${PKGDIR}/All/${pkg}.tbz ] || jexec -U root ${JAILNAME} /usr/sbin/pkg_create -b ${pkg} /usr/ports/packages/All/${pkg}.tbz
+	for pkg in `injail /usr/sbin/pkg_info -Ea`; do
+		[ -f ${PKGDIR}/All/${pkg}.${EXT} ] || injail /usr/sbin/pkg_create -b ${pkg} /usr/ports/packages/All/${pkg}.tbz
 	done
 }
 
@@ -61,6 +61,7 @@ CONFIGSTR=0
 . ${SCRIPTPREFIX}/common.sh
 NOPREFIX=0
 PTNAME="default"
+EXT="tbz"
 
 while getopts "d:o:cnj:p:" FLAG; do
 	case "${FLAG}" in
@@ -134,12 +135,16 @@ for JAILNAME in ${JAILNAMES}; do
 			err 2 "First install portlint if you want USE_PORTLINT to work as expected"
 		fi
 	fi
-	jexec -U root ${JAILNAME} make -C ${PORTDIRECTORY} clean
-	jexec -U root ${JAILNAME} make -C ${PORTDIRECTORY} extract-depends \
-		fetch-depends patch-depends build-depends lib-depends || \
-		(create_pkg "Packaging what is installed so far" && exit 1)
-
-	create_pkg "Packaging all dependencies"
+	LISTPORTS=$(injail make -C ${PORTDIRECTORY} missing)
+	zfs snapshot ${JAILFS}@prepkg
+	msg "Calculating ports order and dependencies"
+	for port in `prepare_ports`; do
+		build_pkg ${port}
+		zfs rollback ${JAILFS}@prepkg
+	done
+	zfs destroy ${JAILFS}@prepkg
+	injail make -C ${PORTDIRECTORY} extract-depends \
+		fetch-depends patch-depends build-depends lib-depends
 
 	exec 1>&3 3>&- 2>&4 4>&-
 	wait $tpid
@@ -148,9 +153,9 @@ for JAILNAME in ${JAILNAMES}; do
 	tee ${LOGS}/${PKGNAME}-${JAILNAME}.build.log < ${PIPE} >&3 &
 	tpid=$!
 	exec > ${PIPE} 2>&1
-	PKGNAME=`jexec -U root ${JAILNAME} make -C ${PORTDIRECTORY} -VPKGNAME`
-	PKG_DBDIR=`jexec -U root ${JAILNAME} mktemp -d -t pkg_db`
-	LOCALBASE=`jexec -U root ${JAILNAME} make -C ${PORTDIRECTORY} -VLOCALBASE`
+	PKGNAME=`injail make -C ${PORTDIRECTORY} -VPKGNAME`
+	PKG_DBDIR=`injail mktemp -d -t pkg_db`
+	LOCALBASE=`injail make -C ${PORTDIRECTORY} -VLOCALBASE`
 	if [ ${NOPREFIX} -eq 1 ]; then
 		PREFIX=${LOCALBASE}
 	else
@@ -159,8 +164,8 @@ for JAILNAME in ${JAILNAMES}; do
 	PORT_FLAGS="NO_DEPENDS=yes PREFIX=${PREFIX} PKG_DBDIR=${PKG_DBDIR}"
 	msg "Building with flags: ${PORT_FLAGS}"
 	msg "Cleaning workspace"
-	jexec -U root ${JAILNAME} make -C ${PORTDIRECTORY} clean
-	[ $CONFIGSTR -eq 1 ] && jexec -U root ${JAILNAME} make -C ${PORTDIRECTORY} config
+	injail make -C ${PORTDIRECTORY} clean
+	[ $CONFIGSTR -eq 1 ] && injail make -C ${PORTDIRECTORY} config
 
 	if [ -d ${JAILBASE}${PREFIX} ]; then
 		msg "Removing existing ${PREFIX}"
@@ -169,7 +174,7 @@ for JAILNAME in ${JAILNAMES}; do
 
 	msg "Populating PREFIX"
 	mkdir -p ${JAILBASE}${PREFIX}
-	jexec -U root ${JAILNAME} /usr/sbin/mtree -q -U -f /usr/ports/Templates/BSD.local.dist -d -e -p ${PREFIX} >/dev/null
+	injail /usr/sbin/mtree -q -U -f /usr/ports/Templates/BSD.local.dist -d -e -p ${PREFIX} >/dev/null
 
 	if [ $ZVERSION -lt 28 ]; then
 		find ${JAILBASE}${LOCALBASE}/ -type d | sed "s,^${JAILBASE}${LOCALBASE}/,," | sort > ${JAILBASE}${PREFIX}.PLIST_DIRS.before
@@ -218,12 +223,12 @@ for JAILNAME in ${JAILNAMES}; do
 	fi
 
 	msg "Installing from package"
-	PKG_DBDIR=${PKG_DBDIR} jexec -U root ${JAILNAME} pkg_add /tmp/pkgs/${PKGNAME}.tbz
+	PKG_DBDIR=${PKG_DBDIR} injail pkg_add /tmp/pkgs/${PKGNAME}.tbz
 	msg "Deinstalling package"
-	PKG_DBDIR=${PKG_DBDIR} jexec -U root ${JAILNAME} pkg_delete ${PKGNAME}
+	PKG_DBDIR=${PKG_DBDIR} injail pkg_delete ${PKGNAME}
 
 	msg "Cleaning up"
-	jexec -U root ${JAILNAME} make -C ${PORTDIRECTORY} clean
+	injail make -C ${PORTDIRECTORY} clean
 
 	msg "Removing existing ${PREFIX} dir"
 	[ "${PREFIX}" != "${LOCALBASE}" ] && rm -rf ${JAILBASE}${PREFIX} ${JAILBASE}${PREFIX}.PLIST_DIRS.before ${JAILBASE}${PREFIX}.PLIST_DIRS.after