From 0fd450ac6b52ce60f8697a451a8c914e1b238ad4 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Tue, 16 Oct 2018 11:03:51 -0400 Subject: [sdks] Add Groovy script to build the SDKs archives for the products (#11137) --- scripts/ci/pipeline/sdks-archive.groovy | 112 ++++++++++++++++++++++++++++ scripts/ci/run-jenkins.sh | 125 +++++++++++++++++--------------- 2 files changed, 178 insertions(+), 59 deletions(-) create mode 100644 scripts/ci/pipeline/sdks-archive.groovy (limited to 'scripts') diff --git a/scripts/ci/pipeline/sdks-archive.groovy b/scripts/ci/pipeline/sdks-archive.groovy new file mode 100644 index 00000000000..d6ac06e8863 --- /dev/null +++ b/scripts/ci/pipeline/sdks-archive.groovy @@ -0,0 +1,112 @@ +properties([compressBuildLog()]) + +parallel ( + "Archive-android-debug-Darwin": { + node ("osx-devices") { + archive ("android", "debug", "Darwin") + } + }, + "Archive-android-release-Darwin": { + node ("osx-devices") { + archive ("android", "release", "Darwin") + } + }, + // "Archive-android-debug-Linux": { + // node ("debian-9-amd64multiarchi386-preview") { + // archive ("android", "debug", "Linux",) + // } + // }, + // "Archive-android-release-Linux": { + // node ("debian-9-amd64multiarchi386-preview") { + // archive ("android", "release", "Linux",) + // } + // }, + "Archive-ios-release-Darwin": { + node ("osx-devices") { + archive ("ios", "release", "Darwin") + } + }, + "Archive-wasm-release-Linux": { + node ("ubuntu-1804-amd64") { + archive ("wasm", "release", "Linux", "ubuntu-1804-amd64-preview", "npm dotnet-sdk-2.1 nuget") + } + } +) + +def archive (product, configuration, platform, chrootname = "", chrootadditionalpackages = "") { + def isPr = (env.ghprbPullId && !env.ghprbPullId.empty ? true : false) + def monoBranch = (isPr ? "pr" : env.BRANCH_NAME) + def jobName = (isPr ? "archive-mono-pullrequest" : "archive-mono") + def packageFileName = null + def commitHash = null + def utils = null + + ws ("workspace/${jobName}/${monoBranch}/${product}/${configuration}") { + timestamps { + stage('Checkout') { + // clone and checkout repo + checkout scm + + utils = load "scripts/ci/pipeline/utils.groovy" + + // remove old stuff + sh 'git reset --hard HEAD' + sh 'git submodule foreach --recursive git reset --hard HEAD' + sh 'git clean -xdff' + sh 'git submodule foreach --recursive git clean -xdff' + + // get current commit sha + commitHash = sh (script: 'git rev-parse HEAD', returnStdout: true).trim() + currentBuild.displayName = "${commitHash.substring(0,7)}" + } + try { + stage('Build') { + utils.reportGitHubStatus (isPr ? env.ghprbActualCommit : commitHash, "Archive-${product}-${configuration}-${platform}", env.BUILD_URL, 'PENDING', 'Building...') + + // build the Archive + timeout (time: 300, unit: 'MINUTES') { + if (platform == "Darwin") { + def brewpackages = "autoconf automake ccache cmake coreutils gdk-pixbuf gettext glib gnu-sed gnu-tar intltool ios-deploy jpeg libffi libidn2 libpng libtiff libtool libunistring ninja openssl p7zip pcre pkg-config scons wget xz" + sh "brew install ${brewpackages} || brew upgrade ${brewpackages}" + + sh "CI_TAGS=sdks-${product},no-tests,${configuration} scripts/ci/run-jenkins.sh" + } else if (platform == "Linux") { + chroot chrootName: chrootname, + command: "CI_TAGS=sdks-${product},no-tests,${configuration} scripts/ci/run-jenkins.sh", + additionalPackages: "xvfb xauth mono-devel git python wget bc build-essential libtool autoconf automake gettext iputils-ping cmake lsof libkrb5-dev curl p7zip-full ninja-build zip unzip gcc-multilib g++-multilib mingw-w64 binutils-mingw-w64 openjdk-8-jre ${chrootadditionalpackages}" + } else { + throw new Exception("Unknown platform \"${platform}\"") + } + } + // move Archive to the workspace root + packageFileName = findFiles (glob: "${product}-${configuration}-${platform}-${commitHash}.zip")[0].name + } + stage('Upload Archive to Azure') { + step([ + $class: 'WAStoragePublisher', + allowAnonymousAccess: true, + cleanUpContainer: false, + cntPubAccess: true, + containerName: "mono-sdks", + doNotFailIfArchivingReturnsNothing: false, + doNotUploadIndividualFiles: false, + doNotWaitForPreviousBuild: true, + excludeFilesPath: '', + filesPath: "${packageFileName}", + storageAccName: 'credential for xamjenkinsartifact', + storageCredentialId: 'fbd29020e8166fbede5518e038544343', + uploadArtifactsOnlyIfSuccessful: true, + uploadZips: false, + virtualPath: "" + ]) + } + + utils.reportGitHubStatus (isPr ? env.ghprbActualCommit : commitHash, "Archive-${product}-${configuration}-${platform}", "https://xamjenkinsartifact.azureedge.net/mono-sdks/${packageFileName}", 'SUCCESS', packageFileName) + } + catch (Exception e) { + utils.reportGitHubStatus (isPr ? env.ghprbActualCommit : commitHash, "Archive-${product}-${configuration}-${platform}", env.BUILD_URL, 'FAILURE', "Build failed.") + throw e + } + } + } +} diff --git a/scripts/ci/run-jenkins.sh b/scripts/ci/run-jenkins.sh index af98d05a07c..a12c9820a45 100755 --- a/scripts/ci/run-jenkins.sh +++ b/scripts/ci/run-jenkins.sh @@ -114,24 +114,26 @@ if [[ ${CI_TAGS} == *'sdks-ios'* ]]; ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-ios NINJA= - ${TESTCMD} --label=build-tests --timeout=10m --fatal make -C sdks/ios compile-tests - ${TESTCMD} --label=run-sim --timeout=20m make -C sdks/ios run-ios-sim-all - ${TESTCMD} --label=build-ios-dev --timeout=60m make -C sdks/ios build-ios-dev-all - if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then - for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done - fi - ${TESTCMD} --label=build-ios-dev-llvm --timeout=60m make -C sdks/ios build-ios-dev-llvm-all - if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then - for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-llvm-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done - fi - ${TESTCMD} --label=build-ios-dev-interp-only --timeout=60m make -C sdks/ios build-ios-dev-interp-only-all - if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then - for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-only-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done - fi - ${TESTCMD} --label=build-ios-dev-interp-mixed --timeout=60m make -C sdks/ios build-ios-dev-interp-mixed-all - if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then - for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-mixed-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done - fi + if [[ ${CI_TAGS} != *'no-tests'* ]]; then + ${TESTCMD} --label=build-tests --timeout=10m --fatal make -C sdks/ios compile-tests + ${TESTCMD} --label=run-sim --timeout=20m make -C sdks/ios run-ios-sim-all + ${TESTCMD} --label=build-ios-dev --timeout=60m make -C sdks/ios build-ios-dev-all + if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then + for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done + fi + ${TESTCMD} --label=build-ios-dev-llvm --timeout=60m make -C sdks/ios build-ios-dev-llvm-all + if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then + for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-llvm-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done + fi + ${TESTCMD} --label=build-ios-dev-interp-only --timeout=60m make -C sdks/ios build-ios-dev-interp-only-all + if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then + for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-only-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done + fi + ${TESTCMD} --label=build-ios-dev-interp-mixed --timeout=60m make -C sdks/ios build-ios-dev-interp-mixed-all + if [[ ${CI_TAGS} == *'run-device-tests'* ]]; then + for suite in ${device_test_suites}; do ${TESTCMD} --label=run-ios-dev-interp-mixed-${suite} --timeout=10m make -C sdks/ios run-ios-dev-${suite}; done + fi + fi exit 0 fi @@ -149,34 +151,37 @@ if [[ ${CI_TAGS} == *'sdks-android'* ]]; # but doesn't get stuck when called via the shell, so let's just call it here now. ${TESTCMD} --label=provision-android --timeout=120m --fatal make -j ${CI_CPU_COUNT} -C sdks/builds provision-android && make -C sdks/android accept-android-license ${TESTCMD} --label=provision-mxe --timeout=240m --fatal make -j ${CI_CPU_COUNT} -C sdks/builds provision-mxe - ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1 - if [[ ${CI_TAGS} != *'pull-request'* ]]; then - ${TESTCMD} --label=archive-debug --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1 CONFIGURATION=debug + if [[ ${CI_TAGS} != *'debug'* ]]; then + ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1 + else + ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-android NINJA= IGNORE_PROVISION_ANDROID=1 IGNORE_PROVISION_MXE=1 CONFIGURATION=debug fi - ${TESTCMD} --label=mini --timeout=60m make -C sdks/android check-mini - ${TESTCMD} --label=corlib --timeout=60m make -C sdks/android check-corlib - ${TESTCMD} --label=System --timeout=60m make -C sdks/android check-System - ${TESTCMD} --label=System.Core --timeout=60m make -C sdks/android check-System.Core - ${TESTCMD} --label=System.Data --timeout=60m make -C sdks/android check-System.Data - ${TESTCMD} --label=System.IO.Compression.FileSystem --timeout=60m make -C sdks/android check-System.IO.Compression.FileSystem - ${TESTCMD} --label=System.IO.Compression --timeout=60m make -C sdks/android check-System.IO.Compression - ${TESTCMD} --label=System.Json --timeout=60m make -C sdks/android check-System.Json - ${TESTCMD} --label=System.Net.Http --timeout=60m make -C sdks/android check-System.Net.Http - ${TESTCMD} --label=System.Numerics --timeout=60m make -C sdks/android check-System.Numerics - ${TESTCMD} --label=System.Runtime.Serialization --timeout=60m make -C sdks/android check-System.Runtime.Serialization - ${TESTCMD} --label=System.ServiceModel.Web --timeout=60m make -C sdks/android check-System.ServiceModel.Web - ${TESTCMD} --label=System.Transactions --timeout=60m make -C sdks/android check-System.Transactions - ${TESTCMD} --label=System.Xml --timeout=60m make -C sdks/android check-System.Xml - ${TESTCMD} --label=System.Xml.Linq --timeout=60m make -C sdks/android check-System.Xml.Linq - ${TESTCMD} --label=Mono.CSharp --timeout=60m make -C sdks/android check-Mono.CSharp - ${TESTCMD} --label=Mono.Data.Sqlite --timeout=60m make -C sdks/android check-Mono.Data.Sqlite - ${TESTCMD} --label=Mono.Data.Tds --timeout=60m make -C sdks/android check-Mono.Data.Tds - ${TESTCMD} --label=Mono.Security --timeout=60m make -C sdks/android check-Mono.Security + if [[ ${CI_TAGS} != *'no-tests'* ]]; then + ${TESTCMD} --label=mini --timeout=60m make -C sdks/android check-mini + ${TESTCMD} --label=corlib --timeout=60m make -C sdks/android check-corlib + ${TESTCMD} --label=System --timeout=60m make -C sdks/android check-System + ${TESTCMD} --label=System.Core --timeout=60m make -C sdks/android check-System.Core + ${TESTCMD} --label=System.Data --timeout=60m make -C sdks/android check-System.Data + ${TESTCMD} --label=System.IO.Compression.FileSystem --timeout=60m make -C sdks/android check-System.IO.Compression.FileSystem + ${TESTCMD} --label=System.IO.Compression --timeout=60m make -C sdks/android check-System.IO.Compression + ${TESTCMD} --label=System.Json --timeout=60m make -C sdks/android check-System.Json + ${TESTCMD} --label=System.Net.Http --timeout=60m make -C sdks/android check-System.Net.Http + ${TESTCMD} --label=System.Numerics --timeout=60m make -C sdks/android check-System.Numerics + ${TESTCMD} --label=System.Runtime.Serialization --timeout=60m make -C sdks/android check-System.Runtime.Serialization + ${TESTCMD} --label=System.ServiceModel.Web --timeout=60m make -C sdks/android check-System.ServiceModel.Web + ${TESTCMD} --label=System.Transactions --timeout=60m make -C sdks/android check-System.Transactions + ${TESTCMD} --label=System.Xml --timeout=60m make -C sdks/android check-System.Xml + ${TESTCMD} --label=System.Xml.Linq --timeout=60m make -C sdks/android check-System.Xml.Linq + ${TESTCMD} --label=Mono.CSharp --timeout=60m make -C sdks/android check-Mono.CSharp + ${TESTCMD} --label=Mono.Data.Sqlite --timeout=60m make -C sdks/android check-Mono.Data.Sqlite + ${TESTCMD} --label=Mono.Data.Tds --timeout=60m make -C sdks/android check-Mono.Data.Tds + ${TESTCMD} --label=Mono.Security --timeout=60m make -C sdks/android check-Mono.Security + fi exit 0 fi -if [[ ${CI_TAGS} == *'webassembly'* ]]; +if [[ ${CI_TAGS} == *'webassembly'* ]] || [[ ${CI_TAGS} == *'wasm'* ]]; then echo "DISABLE_ANDROID=1" > sdks/Make.config echo "DISABLE_IOS=1" >> sdks/Make.config @@ -190,24 +195,26 @@ if [[ ${CI_TAGS} == *'webassembly'* ]]; ${TESTCMD} --label=archive --timeout=180m --fatal make -j ${CI_CPU_COUNT} --output-sync=recurse --trace -C sdks/builds archive-wasm NINJA= - ${TESTCMD} --label=wasm-build --timeout=60m --fatal make -j ${CI_CPU_COUNT} -C sdks/wasm build - ${TESTCMD} --label=ch-mini-test --timeout=60m make -C sdks/wasm run-ch-mini - ${TESTCMD} --label=v8-mini-test --timeout=60m make -C sdks/wasm run-v8-mini - ${TESTCMD} --label=sm-mini-test --timeout=60m make -C sdks/wasm run-sm-mini - ${TESTCMD} --label=jsc-mini-test --timeout=60m make -C sdks/wasm run-jsc-mini - #The following tests are not passing yet, so enabling them would make us perma-red - #${TESTCMD} --label=mini-corlib --timeout=60m make -C sdks/wasm run-all-corlib - #${TESTCMD} --label=mini-system --timeout=60m make -C sdks/wasm run-all-system - # Chakra crashes with System.Core. See https://github.com/mono/mono/issues/8345 - ${TESTCMD} --label=ch-system-core --timeout=60m make -C sdks/wasm run-ch-system-core - ${TESTCMD} --label=v8-system-core --timeout=60m make -C sdks/wasm run-v8-system-core - ${TESTCMD} --label=sm-system-core --timeout=60m make -C sdks/wasm run-sm-system-core - ${TESTCMD} --label=jsc-system-core --timeout=60m make -C sdks/wasm run-jsc-system-core - ${TESTCMD} --label=aot-mini --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm run-aot-mini - ${TESTCMD} --label=build-aot-all --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm build-aot-all - for suite in ${aot_test_suites}; do ${TESTCMD} --label=run-aot-${suite} --timeout=10m make -C sdks/wasm run-aot-${suite}; done - #${TESTCMD} --label=check-aot --timeout=60m make -C sdks/wasm check-aot - ${TESTCMD} --label=package --timeout=60m make -C sdks/wasm package + if [[ ${CI_TAGS} != *'no-tests'* ]]; then + ${TESTCMD} --label=wasm-build --timeout=60m --fatal make -j ${CI_CPU_COUNT} -C sdks/wasm build + ${TESTCMD} --label=ch-mini-test --timeout=60m make -C sdks/wasm run-ch-mini + ${TESTCMD} --label=v8-mini-test --timeout=60m make -C sdks/wasm run-v8-mini + ${TESTCMD} --label=sm-mini-test --timeout=60m make -C sdks/wasm run-sm-mini + ${TESTCMD} --label=jsc-mini-test --timeout=60m make -C sdks/wasm run-jsc-mini + #The following tests are not passing yet, so enabling them would make us perma-red + #${TESTCMD} --label=mini-corlib --timeout=60m make -C sdks/wasm run-all-corlib + #${TESTCMD} --label=mini-system --timeout=60m make -C sdks/wasm run-all-system + # Chakra crashes with System.Core. See https://github.com/mono/mono/issues/8345 + ${TESTCMD} --label=ch-system-core --timeout=60m make -C sdks/wasm run-ch-system-core + ${TESTCMD} --label=v8-system-core --timeout=60m make -C sdks/wasm run-v8-system-core + ${TESTCMD} --label=sm-system-core --timeout=60m make -C sdks/wasm run-sm-system-core + ${TESTCMD} --label=jsc-system-core --timeout=60m make -C sdks/wasm run-jsc-system-core + ${TESTCMD} --label=aot-mini --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm run-aot-mini + ${TESTCMD} --label=build-aot-all --timeout=60m make -j ${CI_CPU_COUNT} -C sdks/wasm build-aot-all + for suite in ${aot_test_suites}; do ${TESTCMD} --label=run-aot-${suite} --timeout=10m make -C sdks/wasm run-aot-${suite}; done + #${TESTCMD} --label=check-aot --timeout=60m make -C sdks/wasm check-aot + ${TESTCMD} --label=package --timeout=60m make -C sdks/wasm package + fi exit 0 fi -- cgit v1.2.3