Improve caching

8 files changed, 37 insertions, 32 deletions

diff --git a/.devcontainer/cache/.gitignore b/.devcontainer/cache/.gitignore
deleted file mode 100644
index 4f96ddff402..00000000000
--- a/.devcontainer/cache/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.manifest
diff --git a/.devcontainer/cache/before-cache.sh b/.devcontainer/cache/before-cache.sh
index 9548a154c38..78511d273d1 100755
--- a/.devcontainer/cache/before-cache.sh
+++ b/.devcontainer/cache/before-cache.sh
@@ -4,12 +4,12 @@
 # are run. Its just a find command that filters out a few things we don't need to watch.
 
 set -e
-
-SCRIPT_PATH="$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)"
 SOURCE_FOLDER="${1:-"."}"
+CACHE_FOLDER="${2:-"$HOME/.devcontainer-cache"}"
 
 cd "${SOURCE_FOLDER}"
 echo "[$(date)] Generating ""before"" manifest..."
-find -L . -not -path "*/.git/*" -and -not -path "${SCRIPT_PATH}/*.manifest" -type f >  "${SCRIPT_PATH}/before.manifest"
+mkdir -p "${CACHE_FOLDER}"
+find -L . -not -path "*/.git/*" -and -not -path "${CACHE_FOLDER}/*.manifest" -type f >  "${CACHE_FOLDER}/before.manifest"
 echo "[$(date)] Done!"
 
diff --git a/.devcontainer/cache/build-cache-image.sh b/.devcontainer/cache/build-cache-image.sh
index 865b860898c..6f8b92a0db3 100755
--- a/.devcontainer/cache/build-cache-image.sh
+++ b/.devcontainer/cache/build-cache-image.sh
@@ -19,10 +19,8 @@ TAG="branch-${BRANCH//\//-}"
 echo "[$(date)] ${BRANCH} => ${TAG}"
 cd "${SCRIPT_PATH}/../.."
 
-echo "[$(date)] Starting image build..."
-docker build -t ${CONTAINER_IMAGE_REPOSITORY}:"${TAG}" -f "${SCRIPT_PATH}/cache.Dockerfile" .
-echo "[$(date)] Image build complete."
+echo "[$(date)] Starting image build and push..."
+docker run --privileged --rm tonistiigi/binfmt --install all
+docker buildx build --push --platform linux/amd64,linux/arm64 -t ${CONTAINER_IMAGE_REPOSITORY}:"${TAG}" -f "${SCRIPT_PATH}/cache.Dockerfile" .
 
-echo "[$(date)] Pushing image..."
-docker push ${CONTAINER_IMAGE_REPOSITORY}:"${TAG}"
 echo "[$(date)] Done!"
diff --git a/.devcontainer/cache/cache-diff.sh b/.devcontainer/cache/cache-diff.sh
index 3f8b77e5602..c2444b8fc6b 100755
--- a/.devcontainer/cache/cache-diff.sh
+++ b/.devcontainer/cache/cache-diff.sh
@@ -5,16 +5,19 @@
 
 set -e
 
-SCRIPT_PATH="$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)"
 SOURCE_FOLDER="${1:-"."}"
-CACHE_FOLDER="${2:-"/usr/local/etc/devcontainer-cache"}"
+CACHE_FOLDER="${2:-"$HOME/.devcontainer-cache"}"
+
+if [ ! -d "${CACHE_FOLDER}" ]; then
+	echo "No cache folder found. Be sure to run before-cache.sh to set one up."
+	exit 1
+fi
 
 echo "[$(date)] Starting cache operation..."
 cd "${SOURCE_FOLDER}"
 echo "[$(date)] Determining diffs..."
-find -L . -not -path "*/.git/*" -and -not -path "${SCRIPT_PATH}/*.manifest" -type f > "${SCRIPT_PATH}/after.manifest"
-grep -Fxvf  "${SCRIPT_PATH}/before.manifest" "${SCRIPT_PATH}/after.manifest" > "${SCRIPT_PATH}/cache.manifest"
+find -L . -not -path "*/.git/*" -and -not -path "${CACHE_FOLDER}/*.manifest" -type f > "${CACHE_FOLDER}/after.manifest"
+grep -Fxvf  "${CACHE_FOLDER}/before.manifest" "${CACHE_FOLDER}/after.manifest" > "${CACHE_FOLDER}/cache.manifest"
 echo "[$(date)] Archiving diffs..."
-mkdir -p "${CACHE_FOLDER}"
-tar -cf "${CACHE_FOLDER}/cache.tar" --totals --files-from "${SCRIPT_PATH}/cache.manifest"
+tar -cf "${CACHE_FOLDER}/cache.tar" --totals --files-from "${CACHE_FOLDER}/cache.manifest"
 echo "[$(date)] Done! $(du -h "${CACHE_FOLDER}/cache.tar")"
diff --git a/.devcontainer/cache/cache.Dockerfile b/.devcontainer/cache/cache.Dockerfile
index a2c2866fe23..868685fa4b9 100644
--- a/.devcontainer/cache/cache.Dockerfile
+++ b/.devcontainer/cache/cache.Dockerfile
@@ -4,19 +4,18 @@
 # This first stage generates cache.tar
 FROM mcr.microsoft.com/vscode/devcontainers/repos/microsoft/vscode:dev as cache
 ARG USERNAME=node
+ARG CACHE_FOLDER="$HOME/.devcontainer-cache"
 COPY --chown=${USERNAME}:${USERNAME} . /repo-source-tmp/
-RUN mkdir /usr/local/etc/devcontainer-cache \
-	&& chown ${USERNAME} /usr/local/etc/devcontainer-cache /repo-source-tmp \
+RUN mkdir -p ${CACHE_FOLDER} && chown ${USERNAME} ${CACHE_FOLDER} /repo-source-tmp \
 	&& su ${USERNAME} -c "\
-		cd /repo-source-tmp \
-		&& .devcontainer/cache/before-cache.sh \
-		&& .devcontainer/prepare.sh \
-		&& .devcontainer/cache/cache-diff.sh"
+		.devcontainer/cache/before-cache.sh /repo-source-tmp ${CACHE_FOLDER} \
+		&& .devcontainer/prepare.sh /repo-source-tmp ${CACHE_FOLDER} \
+		&& .devcontainer/cache/cache-diff.sh /repo-source-tmp ${CACHE_FOLDER}"
 
 # This second stage starts fresh and just copies in cache.tar from the previous stage. The related
 # devcontainer.json file is then setup to have postCreateCommand fire restore-diff.sh to expand it.
 FROM mcr.microsoft.com/vscode/devcontainers/repos/microsoft/vscode:dev as dev-container
 ARG USERNAME=node
-ARG CACHE_FOLDER="/usr/local/etc/devcontainer-cache"
+ARG CACHE_FOLDER="$HOME/.devcontainer-cache"
 RUN mkdir -p "${CACHE_FOLDER}" && chown "${USERNAME}:${USERNAME}" "${CACHE_FOLDER}"
 COPY --from=cache ${CACHE_FOLDER}/cache.tar ${CACHE_FOLDER}/
diff --git a/.devcontainer/cache/restore-diff.sh b/.devcontainer/cache/restore-diff.sh
index 827afc45ab1..cec5950fad5 100755
--- a/.devcontainer/cache/restore-diff.sh
+++ b/.devcontainer/cache/restore-diff.sh
@@ -5,9 +5,8 @@
 # is already up where you would typically run a command like "yarn install".
 
 set -e
-
 SOURCE_FOLDER="$(cd "${1:-"."}" && pwd)"
-CACHE_FOLDER="${2:-"/usr/local/etc/devcontainer-cache"}"
+CACHE_FOLDER="${2:-"$HOME/.devcontainer-cache"}"
 
 if [ ! -d "${CACHE_FOLDER}" ]; then
 	echo "No cache folder found."
@@ -16,7 +15,10 @@ fi
 
 echo "[$(date)] Expanding $(du -h "${CACHE_FOLDER}/cache.tar") file to ${SOURCE_FOLDER}..."
 cd "${SOURCE_FOLDER}"
-tar -xf "${CACHE_FOLDER}/cache.tar"
-rm -f "${CACHE_FOLDER}/cache.tar"
+# Ensure user/group is correct if the UID/GID was changed for some reason
+echo "+1000 +$(id -u)" > "${CACHE_FOLDER}/cache-owner-map"
+echo "+1000 +$(id -g)" > "${CACHE_FOLDER}/cache-group-map"
+# Untar to workspace folder, preserving permissions and order, but mapping GID/UID if required
+tar --owner-map="${CACHE_FOLDER}/cache-owner-map" --group-map="${CACHE_FOLDER}/cache-group-map" -xpsf "${CACHE_FOLDER}/cache.tar"
+rm -rf "${CACHE_FOLDER}"
 echo "[$(date)] Done!"
-
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 08fed2de1f3..dc66ce17200 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -4,7 +4,7 @@
 	// Image contents: https://github.com/microsoft/vscode-dev-containers/blob/master/repository-containers/images/github.com/microsoft/vscode/.devcontainer/base.Dockerfile
 	"image": "mcr.microsoft.com/vscode/devcontainers/repos/microsoft/vscode:branch-main",
 	"overrideCommand": false,
-	"runArgs": [ "--init", "--security-opt", "seccomp=unconfined", "--shm-size=1g"],
+	"runArgs": [ "--init", "--shm-size=1g"],
 
 	"settings": {
 		"resmon.show.battery": false,
@@ -30,7 +30,7 @@
 	],
 
 	// Optionally loads a cached yarn install for the repo
-	"postCreateCommand": ".devcontainer/cache/restore-diff.sh && sudo chown node:node /workspaces",
+	"postCreateCommand": ".devcontainer/cache/restore-diff.sh",
 
 	"remoteUser": "node",
 
diff --git a/.devcontainer/prepare.sh b/.devcontainer/prepare.sh
index fba27045ca6..ee7e79f5bbf 100755
--- a/.devcontainer/prepare.sh
+++ b/.devcontainer/prepare.sh
@@ -5,9 +5,13 @@
 # running commands like "yarn install" from the ground up. Developers (and should) still run these commands
 # after the actual dev container is created, but only differences will be processed.
 
+# Fix permissions for chrome sandboxing
+mkdir -p .build/electron/chrome-sandbox
+chmod 4755  .build/electron/chrome-sandbox
+chown root .build/electron/chrome-sandbox
+
 yarn install
 yarn electron
 
-cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.."
-sudo chown root .build/electron/chrome-sandbox
-sudo chmod 4755  .build/electron/chrome-sandbox
+# Improve command line lag by disabling git portion of theme
+git config --global codespaces-theme.hide-status 1