[sdks] Add Groovy script to build the SDKs archives for the products (#11137)

2 files changed, 178 insertions, 59 deletions

diff --git a/scripts/ci/pipeline/sdks-archive.groovy b/scripts/ci/pipeline/sdks-archive.groovy
new file mode 100644
index 00000000000..d6ac06e8863
--- /dev/null
+++ b/scripts/ci/pipeline/sdks-archive.groovy
@@ -0,0 +1,112 @@
+properties([compressBuildLog()])
+
+parallel (
+    "Archive-android-debug-Darwin": {
+        node ("osx-devices") {
+            archive ("android", "debug", "Darwin")
+        }
+    },
+    "Archive-android-release-Darwin": {
+        node ("osx-devices") {
+            archive ("android", "release", "Darwin")
+        }
+    },
+    // "Archive-android-debug-Linux": {
+    //     node ("debian-9-amd64multiarchi386-preview") {
+    //         archive ("android", "debug", "Linux",)
+    //     }
+    // },
+    // "Archive-android-release-Linux": {
+    //     node ("debian-9-amd64multiarchi386-preview") {
+    //         archive ("android", "release", "Linux",)
+    //     }
+    // },
+    "Archive-ios-release-Darwin": {
+        node ("osx-devices") {
+            archive ("ios", "release", "Darwin")
+        }
+    },
+    "Archive-wasm-release-Linux": {
+        node ("ubuntu-1804-amd64") {
+            archive ("wasm", "release", "Linux", "ubuntu-1804-amd64-preview", "npm dotnet-sdk-2.1 nuget")
+        }
+    }
+)
+
+def archive (product, configuration, platform, chrootname = "", chrootadditionalpackages = "") {
+    def isPr = (env.ghprbPullId && !env.ghprbPullId.empty ? true : false)
+    def monoBranch = (isPr ? "pr" : env.BRANCH_NAME)
+    def jobName = (isPr ? "archive-mono-pullrequest" : "archive-mono")
+    def packageFileName = null
+    def commitHash = null
+    def utils = null
+
+    ws ("workspace/${jobName}/${monoBranch}/${product}/${configuration}") {
+        timestamps {
+            stage('Checkout') {
+                // clone and checkout repo
+                checkout scm
+
+                utils = load "scripts/ci/pipeline/utils.groovy"
+
+                // remove old stuff
+                sh 'git reset --hard HEAD'
+                sh 'git submodule foreach --recursive git reset --hard HEAD'
+                sh 'git clean -xdff'
+                sh 'git submodule foreach --recursive git clean -xdff'
+
+                // get current commit sha
+                commitHash = sh (script: 'git rev-parse HEAD', returnStdout: true).trim()
+                currentBuild.displayName = "${commitHash.substring(0,7)}"
+            }
+            try {
+                stage('Build') {
+                    utils.reportGitHubStatus (isPr ? env.ghprbActualCommit : commitHash, "Archive-${product}-${configuration}-${platform}", env.BUILD_URL, 'PENDING', 'Building...')
+
+                    // build the Archive
+                    timeout (time: 300, unit: 'MINUTES') {
+                        if (platform == "Darwin") {
+                            def brewpackages = "autoconf automake ccache cmake coreutils gdk-pixbuf gettext glib gnu-sed gnu-tar intltool ios-deploy jpeg libffi libidn2 libpng libtiff libtool libunistring ninja openssl p7zip pcre pkg-config scons wget xz"
+                            sh "brew install ${brewpackages} || brew upgrade ${brewpackages}"
+
+                            sh "CI_TAGS=sdks-${product},no-tests,${configuration} scripts/ci/run-jenkins.sh"
+                        } else if (platform == "Linux") {
+                            chroot chrootName: chrootname,
+                                command: "CI_TAGS=sdks-${product},no-tests,${configuration} scripts/ci/run-jenkins.sh",
+                                additionalPackages: "xvfb xauth mono-devel git python wget bc build-essential libtool autoconf automake gettext iputils-ping cmake lsof libkrb5-dev curl p7zip-full ninja-build zip unzip gcc-multilib g++-multilib mingw-w64 binutils-mingw-w64 openjdk-8-jre ${chrootadditionalpackages}"
+                        } else {
+                            throw new Exception("Unknown platform \"${platform}\"")
+                        }
+                    }
+                    // move Archive to the workspace root
+                    packageFileName = findFiles (glob: "${product}-${configuration}-${platform}-${commitHash}.zip")[0].name
+                }
+                stage('Upload Archive to Azure') {
+                    step([
+                        $class: 'WAStoragePublisher',
+                        allowAnonymousAccess: true,
+                        cleanUpContainer: false,
+                        cntPubAccess: true,
+                        containerName: "mono-sdks",
+                        doNotFailIfArchivingReturnsNothing: false,
+                        doNotUploadIndividualFiles: false,
+                        doNotWaitForPreviousBuild: true,
+                        excludeFilesPath: '',
+                        filesPath: "${packageFileName}",
+                        storageAccName: 'credential for xamjenkinsartifact',
+                        storageCredentialId: 'fbd29020e8166fbede5518e038544343',
+                        uploadArtifactsOnlyIfSuccessful: true,
+                        uploadZips: false,
+                        virtualPath: ""
+                    ])
+                }
+
+                utils.reportGitHubStatus (isPr ? env.ghprbActualCommit : commitHash, "Archive-${product}-${configuration}-${platform}", "https://xamjenkinsartifact.azureedge.net/mono-sdks/${packageFileName}", 'SUCCESS', packageFileName)
+            }
+            catch (Exception e) {
+                utils.reportGitHubStatus (isPr ? env.ghprbActualCommit : commitHash, "Archive-${product}-${configuration}-${platform}", env.BUILD_URL, 'FAILURE', "Build failed.")
+                throw e
+            }
+        }
+    }
+}
diff --git a/scripts/ci/run-jenkins.sh b/scripts/ci/run-jenkins.sh
index af98d05a07c..a12c9820a45 100755
--- a/scripts/ci/run-jenkins.sh
+++ b/scripts/ci/run-jenkins.sh
@@ -114,24 +114,26 @@ if [[ ${CI_TAGS} == *'sdks-ios'* ]];
 
 	   ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-ios NINJA=
 
-	   ${TESTCMD} --label=build-tests --timeout=10m --fatal make -C sdks/ios compile-tests
-	   ${TESTCMD} --label=run-sim --timeout=20m make -C sdks/ios run-ios-sim-all
-	   ${TESTCMD} --label=build-ios-dev --timeout=60m make -C sdks/ios build-ios-dev-all
-	   if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
-		   for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
-	   fi
-	   ${TESTCMD} --label=build-ios-dev-llvm --timeout=60m make -C sdks/ios build-ios-dev-llvm-all
-	   if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
-		   for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-llvm-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
-	   fi
-	   ${TESTCMD} --label=build-ios-dev-interp-only --timeout=60m make -C sdks/ios build-ios-dev-interp-only-all
-	   if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
-		   for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-only-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
-	   fi
-	   ${TESTCMD} --label=build-ios-dev-interp-mixed --timeout=60m make -C sdks/ios build-ios-dev-interp-mixed-all
-	   if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
-		   for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-mixed-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
-	   fi
+        if [[ ${CI_TAGS} != *'no-tests'* ]]; then
+            ${TESTCMD} --label=build-tests --timeout=10m --fatal make -C sdks/ios compile-tests
+            ${TESTCMD} --label=run-sim --timeout=20m make -C sdks/ios run-ios-sim-all
+            ${TESTCMD} --label=build-ios-dev --timeout=60m make -C sdks/ios build-ios-dev-all
+            if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
+                for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
+            fi
+            ${TESTCMD} --label=build-ios-dev-llvm --timeout=60m make -C sdks/ios build-ios-dev-llvm-all
+            if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
+                for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-llvm-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
+            fi
+            ${TESTCMD} --label=build-ios-dev-interp-only --timeout=60m make -C sdks/ios build-ios-dev-interp-only-all
+            if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
+                for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-only-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
+            fi
+            ${TESTCMD} --label=build-ios-dev-interp-mixed --timeout=60m make -C sdks/ios build-ios-dev-interp-mixed-all
+            if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then
+                for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-mixed-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done
+            fi
+        fi
 	   exit 0
 fi
 
@@ -149,34 +151,37 @@ if [[ ${CI_TAGS} == *'sdks-android'* ]];
         # but doesn't get stuck when called via the shell, so let's just call it here now.
         ${TESTCMD} --label=provision-android --timeout=120m --fatal make -j ${CI_CPU_COUNT} -C sdks/builds provision-android && make -C sdks/android accept-android-license
         ${TESTCMD} --label=provision-mxe --timeout=240m --fatal make -j ${CI_CPU_COUNT} -C sdks/builds provision-mxe
-        ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1
-        if [[ ${CI_TAGS} != *'pull-request'* ]]; then
-            ${TESTCMD} --label=archive-debug --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1 CONFIGURATION=debug
+        if [[ ${CI_TAGS} != *'debug'* ]]; then
+            ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1
+        else
+            ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1 CONFIGURATION=debug
         fi
 
-        ${TESTCMD} --label=mini --timeout=60m make -C sdks/android check-mini
-        ${TESTCMD} --label=corlib --timeout=60m make -C sdks/android check-corlib
-        ${TESTCMD} --label=System --timeout=60m make -C sdks/android check-System
-        ${TESTCMD} --label=System.Core --timeout=60m make -C sdks/android check-System.Core
-        ${TESTCMD} --label=System.Data --timeout=60m make -C sdks/android check-System.Data
-        ${TESTCMD} --label=System.IO.Compression.FileSystem --timeout=60m make -C sdks/android check-System.IO.Compression.FileSystem
-        ${TESTCMD} --label=System.IO.Compression --timeout=60m make -C sdks/android check-System.IO.Compression
-        ${TESTCMD} --label=System.Json --timeout=60m make -C sdks/android check-System.Json
-        ${TESTCMD} --label=System.Net.Http --timeout=60m make -C sdks/android check-System.Net.Http
-        ${TESTCMD} --label=System.Numerics --timeout=60m make -C sdks/android check-System.Numerics
-        ${TESTCMD} --label=System.Runtime.Serialization --timeout=60m make -C sdks/android check-System.Runtime.Serialization
-        ${TESTCMD} --label=System.ServiceModel.Web --timeout=60m make -C sdks/android check-System.ServiceModel.Web
-        ${TESTCMD} --label=System.Transactions --timeout=60m make -C sdks/android check-System.Transactions
-        ${TESTCMD} --label=System.Xml --timeout=60m make -C sdks/android check-System.Xml
-        ${TESTCMD} --label=System.Xml.Linq --timeout=60m make -C sdks/android check-System.Xml.Linq
-        ${TESTCMD} --label=Mono.CSharp --timeout=60m make -C sdks/android check-Mono.CSharp
-        ${TESTCMD} --label=Mono.Data.Sqlite --timeout=60m make -C sdks/android check-Mono.Data.Sqlite
-        ${TESTCMD} --label=Mono.Data.Tds --timeout=60m make -C sdks/android check-Mono.Data.Tds
-        ${TESTCMD} --label=Mono.Security --timeout=60m make -C sdks/android check-Mono.Security
+        if [[ ${CI_TAGS} != *'no-tests'* ]]; then
+            ${TESTCMD} --label=mini --timeout=60m make -C sdks/android check-mini
+            ${TESTCMD} --label=corlib --timeout=60m make -C sdks/android check-corlib
+            ${TESTCMD} --label=System --timeout=60m make -C sdks/android check-System
+            ${TESTCMD} --label=System.Core --timeout=60m make -C sdks/android check-System.Core
+            ${TESTCMD} --label=System.Data --timeout=60m make -C sdks/android check-System.Data
+            ${TESTCMD} --label=System.IO.Compression.FileSystem --timeout=60m make -C sdks/android check-System.IO.Compression.FileSystem
+            ${TESTCMD} --label=System.IO.Compression --timeout=60m make -C sdks/android check-System.IO.Compression
+            ${TESTCMD} --label=System.Json --timeout=60m make -C sdks/android check-System.Json
+            ${TESTCMD} --label=System.Net.Http --timeout=60m make -C sdks/android check-System.Net.Http
+            ${TESTCMD} --label=System.Numerics --timeout=60m make -C sdks/android check-System.Numerics
+            ${TESTCMD} --label=System.Runtime.Serialization --timeout=60m make -C sdks/android check-System.Runtime.Serialization
+            ${TESTCMD} --label=System.ServiceModel.Web --timeout=60m make -C sdks/android check-System.ServiceModel.Web
+            ${TESTCMD} --label=System.Transactions --timeout=60m make -C sdks/android check-System.Transactions
+            ${TESTCMD} --label=System.Xml --timeout=60m make -C sdks/android check-System.Xml
+            ${TESTCMD} --label=System.Xml.Linq --timeout=60m make -C sdks/android check-System.Xml.Linq
+            ${TESTCMD} --label=Mono.CSharp --timeout=60m make -C sdks/android check-Mono.CSharp
+            ${TESTCMD} --label=Mono.Data.Sqlite --timeout=60m make -C sdks/android check-Mono.Data.Sqlite
+            ${TESTCMD} --label=Mono.Data.Tds --timeout=60m make -C sdks/android check-Mono.Data.Tds
+            ${TESTCMD} --label=Mono.Security --timeout=60m make -C sdks/android check-Mono.Security
+        fi
         exit 0
 fi
 
-if [[ ${CI_TAGS} == *'webassembly'* ]];
+if [[ ${CI_TAGS} == *'webassembly'* ]] || [[ ${CI_TAGS} == *'wasm'* ]];
    then
 	   echo "DISABLE_ANDROID=1" > sdks/Make.config
 	   echo "DISABLE_IOS=1" >> sdks/Make.config
@@ -190,24 +195,26 @@ if [[ ${CI_TAGS} == *'webassembly'* ]];
 
 	   ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-wasm NINJA=
 
-	   ${TESTCMD} --label=wasm-build --timeout=60m --fatal make -j ${CI_CPU_COUNT} -C sdks/wasm build
-	   ${TESTCMD} --label=ch-mini-test --timeout=60m make -C sdks/wasm run-ch-mini
-	   ${TESTCMD} --label=v8-mini-test --timeout=60m make -C sdks/wasm run-v8-mini
-	   ${TESTCMD} --label=sm-mini-test --timeout=60m make -C sdks/wasm run-sm-mini
-	   ${TESTCMD} --label=jsc-mini-test --timeout=60m make -C sdks/wasm run-jsc-mini
-	   #The following tests are not passing yet, so enabling them would make us perma-red
-	   #${TESTCMD} --label=mini-corlib --timeout=60m make -C sdks/wasm run-all-corlib
-	   #${TESTCMD} --label=mini-system --timeout=60m make -C sdks/wasm run-all-system
-	   # Chakra crashes with System.Core. See https://github.com/mono/mono/issues/8345
-	   ${TESTCMD} --label=ch-system-core --timeout=60m make -C sdks/wasm run-ch-system-core
-	   ${TESTCMD} --label=v8-system-core --timeout=60m make -C sdks/wasm run-v8-system-core
-	   ${TESTCMD} --label=sm-system-core --timeout=60m make -C sdks/wasm run-sm-system-core
-	   ${TESTCMD} --label=jsc-system-core --timeout=60m make -C sdks/wasm run-jsc-system-core
-	   ${TESTCMD} --label=aot-mini --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm run-aot-mini
-	   ${TESTCMD} --label=build-aot-all --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm build-aot-all
-	   for suite in ${aot_test_suites}; do ${TESTCMD} --label=run-aot-${suite} --timeout=10m make -C sdks/wasm run-aot-${suite}; done
-	   #${TESTCMD} --label=check-aot --timeout=60m make -C sdks/wasm check-aot
-	   ${TESTCMD} --label=package --timeout=60m make -C sdks/wasm package
+        if [[ ${CI_TAGS} != *'no-tests'* ]]; then
+            ${TESTCMD} --label=wasm-build --timeout=60m --fatal make -j ${CI_CPU_COUNT} -C sdks/wasm build
+            ${TESTCMD} --label=ch-mini-test --timeout=60m make -C sdks/wasm run-ch-mini
+            ${TESTCMD} --label=v8-mini-test --timeout=60m make -C sdks/wasm run-v8-mini
+            ${TESTCMD} --label=sm-mini-test --timeout=60m make -C sdks/wasm run-sm-mini
+            ${TESTCMD} --label=jsc-mini-test --timeout=60m make -C sdks/wasm run-jsc-mini
+            #The following tests are not passing yet, so enabling them would make us perma-red
+            #${TESTCMD} --label=mini-corlib --timeout=60m make -C sdks/wasm run-all-corlib
+            #${TESTCMD} --label=mini-system --timeout=60m make -C sdks/wasm run-all-system
+            # Chakra crashes with System.Core. See https://github.com/mono/mono/issues/8345
+            ${TESTCMD} --label=ch-system-core --timeout=60m make -C sdks/wasm run-ch-system-core
+            ${TESTCMD} --label=v8-system-core --timeout=60m make -C sdks/wasm run-v8-system-core
+            ${TESTCMD} --label=sm-system-core --timeout=60m make -C sdks/wasm run-sm-system-core
+            ${TESTCMD} --label=jsc-system-core --timeout=60m make -C sdks/wasm run-jsc-system-core
+            ${TESTCMD} --label=aot-mini --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm run-aot-mini
+            ${TESTCMD} --label=build-aot-all --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm build-aot-all
+            for suite in ${aot_test_suites}; do ${TESTCMD} --label=run-aot-${suite} --timeout=10m make -C sdks/wasm run-aot-${suite}; done
+            #${TESTCMD} --label=check-aot --timeout=60m make -C sdks/wasm check-aot
+            ${TESTCMD} --label=package --timeout=60m make -C sdks/wasm package
+        fi
 	   exit 0
 fi