Nonzero Exit Code for performance-setup Failures (#59483)

* failed git clone causes script failure

* Failure notification for moves & copies

* Fix robocopy failure detection

* Use functions to verify

* Close shell ifs

* Remove unintended tabs

2 files changed, 77 insertions, 15 deletions

diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1
index ae30e9bac80..bc17873bd58 100644
--- a/eng/testing/performance/performance-setup.ps1
+++ b/eng/testing/performance/performance-setup.ps1
@@ -28,6 +28,41 @@ Param(
     [switch] $iOSLlvmBuild
 )
 
+function Verified-Move-Item {
+	[Parameter(Mandatory)]
+	[string]$Path
+	[Parameter(Mandatory)]
+	[string]$Destination
+	
+	Move-Item -Path $Path -Destination $Destination
+	if (!$?) {
+		Write-Output "Failed to move $Source to $Destination"
+		exit 1
+	}
+}
+
+function Verified-Copy-Item {
+	[Parameter(Mandatory)]
+	[string]$path
+	[Parameter(Mandatory)]
+	[string]$Destination
+	
+	Copy-Item -path $path $Destination
+	if (!$?) {
+		Write-Output "Failed to copy $Source to $Destination"
+		exit 1
+	}
+}
+
+function Verify-Robocopy {
+	[Parameter(Mandatory)]
+	[string]$Source
+	if ($LASTEXITCODE -ne 0 -or !$?) {
+		Write-Output "Failed to copy $Source: exit code $LASTEXITCODE"
+		exit $LASTEXITCODE
+	}
+}
+
 $RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance")
 $UseCoreRun = ($CoreRootDirectory -ne [string]::Empty)
 $UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty)
@@ -122,25 +157,30 @@ if ($RunFromPerformanceRepo) {
     $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments"
     
     robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git
+	Verify-Robocopy $SourceDirectory
 }
 else {
     git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory
+	if ($LASTEXITCODE -ne 0) {
+		Write-Output "git clone failed with code $LASTEXITCODE"
+		exit 1
+	}
 }
 
 if($MonoDotnet -ne "")
 {
     $UsingMono = "true"
     $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono")
-    Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath
+    Verified-Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath
 }
 
 if ($UseCoreRun) {
     $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root")
-    Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot
+    Verified-Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot
 }
 if ($UseBaselineCoreRun) {
     $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root")
-    Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot
+    Verified-Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot
 }
 
 if ($AndroidMono) {
@@ -148,7 +188,7 @@ if ($AndroidMono) {
     {
         mkdir $WorkItemDirectory
     }
-    Copy-Item -path "$SourceDirectory\artifacts\bin\AndroidSampleApp\arm64\Release\android-arm64\publish\apk\bin\HelloAndroid.apk" $PayloadDirectory
+    Verified-Copy-Item -path "$SourceDirectory\artifacts\bin\AndroidSampleApp\arm64\Release\android-arm64\publish\apk\bin\HelloAndroid.apk" $PayloadDirectory
     $SetupArguments = $SetupArguments -replace $Architecture, 'arm64'
 }
 
@@ -158,9 +198,9 @@ if ($iOSMono) {
         mkdir $WorkItemDirectory
     }
     if($iOSLlvmBuild) {
-        Copy-Item -path "$SourceDirectory\iosHelloWorld\llvm" $PayloadDirectory\iosHelloWorld\llvm -Recurse
+        Verified-Copy-Item -path "$SourceDirectory\iosHelloWorld\llvm" $PayloadDirectory\iosHelloWorld\llvm -Recurse
     } else {
-        Copy-Item -path "$SourceDirectory\iosHelloWorld\nollvm" $PayloadDirectory\iosHelloWorld\nollvm -Recurse
+        Verified-Copy-Item -path "$SourceDirectory\iosHelloWorld\nollvm" $PayloadDirectory\iosHelloWorld\nollvm -Recurse
     }
 
     $SetupArguments = $SetupArguments -replace $Architecture, 'arm64'
@@ -168,6 +208,7 @@ if ($iOSMono) {
 
 $DocsDir = (Join-Path $PerformanceDirectory "docs")
 robocopy $DocsDir $WorkItemDirectory
+Verify-Robocopy $DocsDir
 
 # Set variables that we will need to have in future steps
 $ci = $true
diff --git a/eng/testing/performance/performance-setup.sh b/eng/testing/performance/performance-setup.sh
index c99ad1f48f3..ceb9c3245b4 100755
--- a/eng/testing/performance/performance-setup.sh
+++ b/eng/testing/performance/performance-setup.sh
@@ -31,6 +31,24 @@ use_latest_dotnet=false
 logical_machine=
 javascript_engine="v8"
 
+verified_mv()
+{
+	mv $1 $2
+	if [ "$?" -ne "0" ]; then
+		echo "Failed to move $1 to $2"
+		exit 1
+	fi
+}
+
+verified_rsync()
+{
+	rsync -a --progress $1 $2
+	if [ "$?" -ne "0" ]; then
+		echo "Failed to sync $1 to $2"
+		exit 1
+	fi
+}
+
 while (($# > 0)); do
   lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
   case $lowerI in
@@ -267,24 +285,27 @@ else
     git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance.git $performance_directory
     # uncomment to use BenchmarkDotNet sources instead of nuget packages
     # git clone https://github.com/dotnet/BenchmarkDotNet.git $benchmark_directory
+	if [ "$?" -ne "0" ]; then
+		echo "git clone failed with code $?"
+		exit 1
 
     docs_directory=$performance_directory/docs
-    mv $docs_directory $workitem_directory
+    verified_mv $docs_directory $workitem_directory
 fi
 
 if [[ "$wasm_runtime_loc" != "" ]]; then
     using_wasm=true
     wasm_dotnet_path=$payload_directory/dotnet-wasm
-    mv $wasm_runtime_loc $wasm_dotnet_path
+    verified_mv $wasm_runtime_loc $wasm_dotnet_path
     # install emsdk, $source_directory/src/mono/wasm/ has the nuget.config with require feed. EMSDK may be available in the payload in a different directory, should visit this install to avoid deplicated payload.
     pushd $source_directory/src/mono/wasm/
     make provision-wasm
     EMSDK_PATH = $source_directory/src/mono/wasm/emsdk
     popd
     # wasm aot and interpreter need some source code from dotnet\runtime repo
-    rsync -aq --progress $source_directory/* $wasm_dotnet_path --exclude Payload --exclude docs --exclude src/coreclr --exclude src/tests --exclude artifacts/obj --exclude artifacts/log --exclude artifacts/tests --exclude __download__
+    verified_rsync -aq --progress $source_directory/* $wasm_dotnet_path --exclude Payload --exclude docs --exclude src/coreclr --exclude src/tests --exclude artifacts/obj --exclude artifacts/log --exclude artifacts/tests --exclude __download__
     # copy wasm build drop to the location that aot and interpreter build expects
-    rsync -a --progress $wasm_dotnet_path/artifacts/BrowserWasm/artifacts/* $wasm_dotnet_path/artifacts
+    verified_rsync -a --progress $wasm_dotnet_path/artifacts/BrowserWasm/artifacts/* $wasm_dotnet_path/artifacts
     rm -r $wasm_dotnet_path/artifacts/BrowserWasm/artifacts
     if [[ "$wasmaot" == "true" ]]; then
         extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmEngine /home/helixbot/.jsvu/$javascript_engine --runtimeSrcDir \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm --aotcompilermode wasm --buildTimeout 3600" 
@@ -296,23 +317,23 @@ fi
 if [[ "$mono_dotnet" != "" ]] && [[ "$monoaot" == "false" ]]; then
     using_mono=true
     mono_dotnet_path=$payload_directory/dotnet-mono
-    mv $mono_dotnet $mono_dotnet_path
+    verified_mv $mono_dotnet $mono_dotnet_path
 fi
 
 if [[ "$monoaot" == "true" ]]; then
     monoaot_dotnet_path=$payload_directory/monoaot
-    mv $monoaot_path $monoaot_dotnet_path
+    verified_mv $monoaot_path $monoaot_dotnet_path
     extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --runtimes monoaotllvm --aotcompilerpath \$HELIX_CORRELATION_PAYLOAD/monoaot/sgen/mini/mono-sgen --customruntimepack \$HELIX_CORRELATION_PAYLOAD/monoaot/pack --aotcompilermode llvm"
 fi
 
 if [[ "$use_core_run" = true ]]; then
     new_core_root=$payload_directory/Core_Root
-    mv $core_root_directory $new_core_root
+    verified_mv $core_root_directory $new_core_root
 fi
 
 if [[ "$use_baseline_core_run" = true ]]; then
-  new_baseline_core_root=$payload_directory/Baseline_Core_Root
-  mv $baseline_core_root_directory $new_baseline_core_root
+	new_baseline_core_root=$payload_directory/Baseline_Core_Root
+	verified_mv $baseline_core_root_directory $new_baseline_core_root
 fi
 
 ci=true