Add support for zfs image types, including send streams

Adds the ability to create a ZFS disk image in the same style as bsdinstall(8).
The default is to create a full GPT disk image, but is also supports just the raw pool.

Also adds `zfs send` streams as an output format, includes support for sending the
entire pool, just the boot environment (for upgrades), or both.
Supports specifying both zfs+send targets (+full and +be) in a single run.

Extends the existing -i support, to be able to modify existing images.

Adds the -R flag to allow user to specify ZFS send feature flags

Sponsored by: Modirum MDPay
Sponsored by: Klara Inc.

4 files changed, 248 insertions, 12 deletions

diff --git a/src/man/poudriere-image.8 b/src/man/poudriere-image.8
index 1d458141..5e79a79f 100644
--- a/src/man/poudriere-image.8
+++ b/src/man/poudriere-image.8
@@ -1,7 +1,7 @@
 .\" Copyright (c) 2012 Baptiste Daroussin <bapt@FreeBSD.org>
 .\" Copyright (c) 2012-2014 Bryan Drewery <bdrewery@FreeBSD.org>
 .\" Copyright (c) 2018 SRI International
-.\" Copyright (c) 2018 Allan Jude <allanjude@FreeBSD.org>
+.\" Copyright (c) 2018-2021 Allan Jude <allanjude@FreeBSD.org>
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -29,7 +29,7 @@
 .\"
 .\" Note: The date here should be updated whenever a non-trivial
 .\" change is made to the manual page.
-.Dd August 3, 2021
+.Dd October 31, 2021
 .Dt POUDRIERE-IMAGE 8
 .Os
 .Sh NAME
@@ -82,7 +82,15 @@ This specifies the hostname used for the image.
 Defaults to
 .Ar poudriere-image .
 .It Fl i Ar originimage
-Path to a previously built full.img.gz.
+Path to a previously built image.
+For
+.Fl t Cm zsnapshot ,
+this should be
+.Pa full.img.gz .
+For
+.Fl t Cm zfs
+this should be
+.Pa image.full.zfs .
 .It Fl j Ar name
 This argument specifies the name of the jail that is used.
 .It Fl m Ar overlaydir
@@ -94,6 +102,13 @@ This specifies the name of the resulting image.
 This argument specifies directory where the resulting image will be created.
 .It Fl p Ar tree
 This argument specifies the name of the ports tree that is used.
+.It Fl R Ar flags
+The flags to pass to
+.Ql Cm zfs send
+when creating the replication stream.
+These will control which features are enabled in the stream.
+Default:
+.Fl Rec
 .It Fl S Ar snapshotname
 Name of the snapshot for zsnapshot type.
 .It Fl s Ar size
@@ -120,6 +135,26 @@ filesystem is LZ77 compressed and is MFS mounted.
 A raw UFS2, softupdates-enabled, disk image.
 .It zrawdisk
 A raw ZFS disk image.
+.It zfs
+Create a ZFS image in the same style as
+.Xr bsdinstall 8 .
+Supports the following sub-types, defaulting to gpt if not subtype is specified:
+.It zfs+gpt
+Creates a complete disk image with a GPT partition table.
+Includes both UEFI and Legacy boot code.
+.It zfs+raw
+Created an image of only the ZFS pool with no partitions.
+Not bootable.
+.It zfs+send
+Creates a full ZFS replication stream of the whole pool, including the boot
+environment, to be received using the
+.Xr zfs-recv 8
+command.
+Alias for zfs+send+full.
+You may create both send streams by specifying both sub-types
+(example: zfs+send+full+be).
+.It zfs+send+be
+Creates a ZFS replication stream of only the boot environment.
 .It tar
 An XZ-compressed tarball.
 .It firmware
@@ -153,10 +188,16 @@ format filesystems, and then mount them to
 /world .
 .It Ev WORLDDIR
 The path to the directory that is the root of the image.
-.It Ev zroot
+.It Ev ZFS_BEROOT_NAME
+The name of the dataset that contains all boot environments.
+Default: ROOT.
+.It Ev ZFS_BOOTFS_NAME
+The name of the default boot environment dataset.
+DEfault: default.
+.It Ev ZFS_POOL_NAME
 The name of the ZFS pool.
-The pre-script should populated this variable with the name of the ZFS pool
-that is created, if any.
+Defaults to
+.Ev ${IMAGENAME}root .
 .It Ev md
 The name of the
 .Xr mdconfig 8
diff --git a/src/share/poudriere/common.sh b/src/share/poudriere/common.sh
index de70e425..de70e425 100644..100755
--- a/src/share/poudriere/common.sh
+++ b/src/share/poudriere/common.sh
diff --git a/src/share/poudriere/image.sh b/src/share/poudriere/image.sh
index 01b24c0b..bfa5b0da 100644
--- a/src/share/poudriere/image.sh
+++ b/src/share/poudriere/image.sh
@@ -33,6 +33,7 @@
 . ${SCRIPTPREFIX}/image_rawdisk.sh
 . ${SCRIPTPREFIX}/image_tar.sh
 . ${SCRIPTPREFIX}/image_usb.sh
+. ${SCRIPTPREFIX}/image_zfs.sh
 . ${SCRIPTPREFIX}/image_zsnapshot.sh
 
 usage() {
@@ -45,7 +46,7 @@ Parameters:
     -t type         -- Type of image can be one of
                     -- iso, iso+mfs, iso+zmfs, usb, usb+mfs, usb+zmfs,
                        rawdisk, zrawdisk, tar, firmware, rawfirmware,
-                       dump, zsnapshot
+                       dump, zfs+[raw|gpt|send[+full[+be]]], zsnapshot
 
 Options:
     -A post-script  -- Source this script after populating the \$WRKDIR/world
@@ -66,6 +67,7 @@ Options:
     -o outputdir    -- Image destination directory
     -p portstree    -- Ports tree
     -P pkgbase      -- List of pkgbase packages to install
+    -R flags        -- ZFS Replication Flags
     -s size         -- Set the image size
     -S snapshotname -- Snapshot name
     -w size         -- Set the size of the swap partition
@@ -271,7 +273,7 @@ PKG_QUIET="-q"
 : ${PRE_BUILD_SCRIPT:=""}
 : ${POST_BUILD_SCRIPT:=""}
 
-while getopts "A:bB:c:f:h:i:j:m:n:o:p:P:s:S:t:vw:X:z:" FLAG; do
+while getopts "A:bB:c:f:h:i:j:m:n:o:p:P:R:s:S:t:vw:X:z:" FLAG; do
 	case "${FLAG}" in
 		A)
 			[ "${OPTARG#/}" = "${OPTARG}" ] && \
@@ -339,6 +341,9 @@ while getopts "A:bB:c:f:h:i:j:m:n:o:p:P:s:S:t:vw:X:z:" FLAG; do
 			PKGBASELIST=${OPTARG}
 			INSTALLWORLD=install_world_from_pkgbase
 			;;
+		R)
+			ZFS_SEND_FLAGS="-${OPTARG}"
+			;;
 		s)
 			IMAGESIZE="${OPTARG}"
 			;;
@@ -351,6 +356,8 @@ while getopts "A:bB:c:f:h:i:j:m:n:o:p:P:s:S:t:vw:X:z:" FLAG; do
 			iso|iso+mfs|iso+zmfs|usb|usb+mfs|usb+zmfs) ;;
 			rawdisk|zrawdisk|tar|firmware|rawfirmware) ;;
 			dump|zsnapshot) ;;
+			zfs|zfs+gpt|zfs+raw) ;;
+			zfs+send|zfs+send+full|zfs+send+be|zfs+send+full+be) ;;
 			*) err 1 "invalid mediatype: ${MEDIATYPE}"
 			esac
 			;;
@@ -379,15 +386,19 @@ saved_argv="$@"
 shift $((OPTIND-1))
 post_getopts
 
+[ -n "${JAILNAME}" ] || usage
+
+: ${OUTPUTDIR:=${POUDRIERE_DATA}/images/}
+: ${IMAGENAME:=poudriereimage}
 : ${MEDIATYPE:=none}
 : ${SWAPBEFORE:=0}
 : ${SWAPSIZE:=0}
 : ${PTNAME:=default}
+: ${ZFS_SEND_FLAGS:=-Rec}
+: ${ZFS_POOL_NAME:=${IMAGENAME}root}
+: ${ZFS_BEROOT_NAME:=ROOT}
+: ${ZFS_BOOTFS_NAME:=default}
 
-[ -n "${JAILNAME}" ] || usage
-
-: ${OUTPUTDIR:=${POUDRIERE_DATA}/images/}
-: ${IMAGENAME:=poudriereimage}
 MASTERNAME=${JAILNAME}-${PTNAME}${SETNAME:+-${SETNAME}}
 
 MAINMEDIATYPE=${MEDIATYPE%%+*}
diff --git a/src/share/poudriere/image_zfs.sh b/src/share/poudriere/image_zfs.sh
new file mode 100644
index 00000000..d3b0fa88
--- /dev/null
+++ b/src/share/poudriere/image_zfs.sh
@@ -0,0 +1,184 @@
+#!/bin/sh
+#
+# Copyright (c) 2018-2021 Allan Jude <allanjude@FreeBSD.org>
+# Copyright (c) 2019 Marie Helene Kvello-Aune <freebsd@mhka.no>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+_zfs_writereplicationstream()
+{
+
+	# Arguments:
+	# $1: snapshot to recursively replicate
+	# $2: Image name to write replication stream to
+	[ $# -eq 2 ] || eargs _zfs_writereplicationstream snapshot_from image_to
+	msg "Creating replication stream"
+	zfs send ${ZFS_SEND_FLAGS} "$1" > "${OUTPUTDIR}/$2" ||
+	    err 1 "Failed to save ZFS replication stream"
+}
+
+zfs_check()
+{
+
+	[ -n "${IMAGESIZE}" ] || err 1 "Please specify the imagesize"
+	[ -n "${ZFS_POOL_NAME}" ] || err 1 "Please specify a pool name"
+	zpool list -Ho name ${ZFS_POOL_NAME} >/dev/null 2>&1 || \
+		err 1 "Target pool name already exists"
+	case "${IMAGENAME}" in
+	''|*[!A-Za-z0-9]*)
+		err 1 "Name can only contain alphanumeric characters"
+		;;
+	esac
+	[ -f "${mnt}/boot/kernel/kernel" ] || \
+	    err 1 "The ${MEDIATYPE} media type requires a jail with a kernel"
+	if [ -n "${ORIGIN_IMAGE}" ]; then
+		[ -z "${SNAPSHOT_NAME}" ] || err 1 \
+			"You must specify the snapshot name (-S) when using -i"
+	fi
+}
+
+zfs_prepare()
+{
+
+	truncate -s ${IMAGESIZE} ${WRKDIR}/raw.img
+	md=$(/sbin/mdconfig ${WRKDIR}/raw.img)
+	zroot=${ZFS_POOL_NAME}
+
+	msg "Creating temporary ZFS pool"
+	zpool create \
+		-O mountpoint=/${ZFS_POOL_NAME} \
+		-O canmount=noauto \
+		-O checksum=sha512 \
+		-O compression=on \
+		-O atime=off \
+		-R ${WRKDIR}/world ${zroot} /dev/${md} || exit
+
+	if [ -n "${ORIGIN_IMAGE}" ]; then
+		msg "Importing previous ZFS Datasets"
+		zfs recv -F ${zroot} < "${ORIGIN_IMAGE}"
+	else
+		msg "Creating ZFS Datasets"
+		zfs create -o mountpoint=none ${zroot}/${ZFS_BEROOT_NAME}
+		zfs create -o mountpoint=/ ${zroot}/${ZFS_BEROOT_NAME}/${ZFS_BOOTFS_NAME}
+		zfs create -o mountpoint=/tmp -o exec=on -o setuid=off ${zroot}/tmp
+		zfs create -o mountpoint=/usr -o canmount=off ${zroot}/usr
+		zfs create ${zroot}/usr/home
+		zfs create -o setuid=off ${zroot}/usr/ports
+		zfs create ${zroot}/usr/src
+		zfs create ${zroot}/usr/obj
+		zfs create -o mountpoint=/var -o canmount=off ${zroot}/var
+		zfs create -o exec=off -o setuid=off ${zroot}/var/audit
+		zfs create -o exec=off -o setuid=off ${zroot}/var/crash
+		zfs create -o exec=off -o setuid=off ${zroot}/var/log
+		zfs create -o atime=on ${zroot}/var/mail
+		zfs create -o setuid=off ${zroot}/var/tmp
+		chmod 1777 ${WRKDIR}/world/tmp ${WRKDIR}/world/var/tmp
+	fi
+}
+
+zfs_build()
+{
+	if [ -z "${ORIGIN_IMAGE}" ]; then
+		cat >> ${WRKDIR}/world/boot/loader.conf <<-EOF
+		zfs_load="YES"
+		EOF
+		if [ -n "${SWAPSIZE}" -a "${SWAPSIZE}" != "0" ]; then
+			cat >> ${WRKDIR}/world/etc/fstab <<-EOSWAP
+			/dev/gpt/swapspace none swap sw 0 0
+			EOSWAP
+		fi
+	fi
+}
+
+zfs_generate()
+{
+
+	: ${SNAPSHOT_NAME:=$IMAGENAME}
+	FINALIMAGE=${IMAGENAME}.img
+	zroot="${ZFS_POOL_NAME}"
+	zpool set bootfs=${zroot}/${ZFS_BEROOT_NAME}/${ZFS_BOOTFS_NAME} ${zroot}
+	zpool set autoexpand=on ${zroot}
+	zfs set canmount=noauto ${zroot}/${ZFS_BEROOT_NAME}/${ZFS_BOOTFS_NAME}
+
+	SNAPSPEC="${zroot}@${SNAPSHOT_NAME}"
+
+	msg "Creating snapshot(s) for image generation"
+	zfs snapshot -r "$SNAPSPEC"
+
+	## If we are creating a send stream, we need to do it before we export
+	## the pool. Call the function to export the replication stream(s) here.
+	## We do the inner case twice so we create a +full and a +be in one run.
+	case "$1" in
+	send)
+		FINALIMAGE=${IMAGENAME}.*.zfs
+		case "${MEDIAREMAINDER}" in
+		*full*|send|zfs)
+			_zfs_writereplicationstream "${SNAPSPEC}" "${IMAGENAME}.full.zfs"
+			;;
+		esac
+		case "${MEDIAREMAINDER}" in
+		*be*)
+			BESNAPSPEC="${zroot}/${ZFS_BEROOT_NAME}/${ZFS_BOOTFS_NAME}@${SNAPSHOT_NAME}"
+			_zfs_writereplicationstream "${BESNAPSPEC}" "${IMAGENAME}.be.zfs"
+			;;
+		esac
+		;;
+	esac
+
+	## When generating a disk image, we need to export the pool first.
+	zpool export ${zroot}
+	zroot=
+	/sbin/mdconfig -d -u ${md#md}
+	md=
+
+	case "$1" in
+	raw)
+		mv "${WRKDIR}/raw.img" "${OUTPUTDIR}/${FINALIMAGE}"
+		;;
+	gpt|zfs)
+		espfilename=$(mktemp /tmp/efiboot.XXXXXX)
+		zfsimage=${WRKDIR}/raw.img
+		make_esp_file ${espfilename} 10 ${mnt}/boot/loader.efi
+
+		if [ ${SWAPSIZE} != "0" ]; then
+			SWAPCMD="-p freebsd-swap/swapspace::${SWAPSIZE}"
+			if [ $SWAPBEFORE -eq 1 ]; then
+				SWAPFIRST="$SWAPCMD"
+			else
+				SWAPLAST="$SWAPCMD"
+			fi
+		fi
+		if [ "${arch}" == "amd64" ] || [ "${arch}" == "i386" ]; then
+			pmbr="-b ${mnt}/boot/pmbr"
+			gptboot="-p freebsd-boot:=${mnt}/boot/gptzfsboot:512k"
+		fi
+		mkimg -s gpt ${pmbr} \
+			  -p efi:=${espfilename} \
+			  ${gptboot} \
+			  ${SWAPFIRST} \
+			  -p freebsd-zfs:=${zfsimage} \
+			  ${SWAPLAST} \
+			  -o "${OUTPUTDIR}/${FINALIMAGE}"
+		rm -rf ${espfilename}
+		;;
+	esac
+}