diff options
Diffstat (limited to 'eng')
-rw-r--r-- | eng/Version.Details.xml | 8 | ||||
-rw-r--r-- | eng/Versions.props | 2 | ||||
-rw-r--r-- | eng/common/SetupNugetSources.ps1 | 14 | ||||
-rwxr-xr-x | eng/common/cross/build-rootfs.sh | 26 | ||||
-rw-r--r-- | eng/common/post-build/check-channel-consistency.ps1 | 5 | ||||
-rw-r--r-- | eng/common/post-build/symbols-validation.ps1 | 18 | ||||
-rw-r--r-- | eng/common/sdk-task.ps1 | 12 | ||||
-rw-r--r-- | eng/common/sdl/extract-artifact-packages.ps1 | 4 | ||||
-rw-r--r-- | eng/common/sdl/packages.config | 2 | ||||
-rw-r--r-- | eng/common/templates/job/execute-sdl.yml | 34 | ||||
-rw-r--r-- | eng/common/templates/job/job.yml | 10 | ||||
-rw-r--r-- | eng/common/templates/post-build/channels/generic-internal-channel.yml | 4 | ||||
-rw-r--r-- | eng/common/templates/post-build/channels/generic-public-channel.yml | 4 | ||||
-rw-r--r-- | eng/common/templates/post-build/common-variables.yml | 2 | ||||
-rw-r--r-- | eng/common/templates/post-build/post-build.yml | 60 | ||||
-rw-r--r-- | eng/common/templates/post-build/setup-maestro-vars.yml | 26 | ||||
-rw-r--r-- | eng/common/tools.ps1 | 7 | ||||
-rwxr-xr-x | eng/common/tools.sh | 23 |
18 files changed, 176 insertions, 85 deletions
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index dd73396db..5ae52576a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,13 +3,13 @@ <ProductDependencies> </ProductDependencies> <ToolsetDependencies> - <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20201.2"> + <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20221.14"> <Uri>https://github.com/dotnet/arcade</Uri> - <Sha>bce0a98620c1c5a110b2bba9912f3d5929069c6b</Sha> + <Sha>e13ae00db986456a5012f2d9d479d893b3f5aff6</Sha> </Dependency> - <Dependency Name="Microsoft.DotNet.ApiCompat" Version="5.0.0-beta.20201.2"> + <Dependency Name="Microsoft.DotNet.ApiCompat" Version="5.0.0-beta.20221.14"> <Uri>https://github.com/dotnet/arcade</Uri> - <Sha>bce0a98620c1c5a110b2bba9912f3d5929069c6b</Sha> + <Sha>e13ae00db986456a5012f2d9d479d893b3f5aff6</Sha> </Dependency> <Dependency Name="Microsoft.NET.Sdk.IL" Version="5.0.0-preview.5.20226.8"> <Uri>https://github.com/dotnet/runtime</Uri> diff --git a/eng/Versions.props b/eng/Versions.props index d57fe4d99..5864fd7b9 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -20,7 +20,7 @@ <SystemReflectionMetadataVersion>1.5.0</SystemReflectionMetadataVersion> <MicrosoftBuildFrameworkVersion>15.4.8</MicrosoftBuildFrameworkVersion> <MicrosoftBuildUtilitiesCoreVersion>15.4.8</MicrosoftBuildUtilitiesCoreVersion> - <MicrosoftDotNetApiCompatVersion>5.0.0-beta.20201.2</MicrosoftDotNetApiCompatVersion> + <MicrosoftDotNetApiCompatVersion>5.0.0-beta.20221.14</MicrosoftDotNetApiCompatVersion> <!-- We depend on (and redistribute) the official Mono.Cecil NuGet package built from https://github.com/jbevain/cecil --> <MonoCecilVersion>0.11.2</MonoCecilVersion> </PropertyGroup> diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index a5a1e711d..c3c473eb8 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -83,7 +83,7 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("value", $Password) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." @@ -123,19 +123,21 @@ if ($creds -eq $null) { $doc.DocumentElement.AppendChild($creds) | Out-Null } +$userName = "dn-bot" + # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password $dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']") if ($dotnet3Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } $doc.Save($filename) diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 2cdd82d30..6e2312a10 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -e + usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]" @@ -15,6 +17,8 @@ __CodeName=xenial __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) __InitialDir=$PWD __BuildArch=arm +__AlpineArch=armv7 +__QEMUArch=arm __UbuntuArch=armhf __UbuntuRepo="http://ports.ubuntu.com/" __LLDB_Package="liblldb-3.9-dev" @@ -26,9 +30,10 @@ __UbuntuPackages="build-essential" __AlpinePackages="alpine-base" __AlpinePackages+=" build-base" __AlpinePackages+=" linux-headers" -__AlpinePackagesEdgeTesting=" lldb-dev" -__AlpinePackagesEdgeMain=" llvm9-libs" +__AlpinePackagesEdgeCommunity=" lldb-dev" +__AlpinePackagesEdgeMain=" llvm10-libs" __AlpinePackagesEdgeMain+=" python3" +__AlpinePackagesEdgeMain+=" libedit" # symlinks fixer __UbuntuPackages+=" symlinks" @@ -55,12 +60,11 @@ __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" __FreeBSDBase="12.1-RELEASE" -__FreeBSDPkg="1.10.5" +__FreeBSDPkg="1.12.0" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" __FreeBSDPackages+=" libinotify" __FreeBSDPackages+=" lttng-ust" -__FreeBSDPackages+=" llvm-90" __FreeBSDPackages+=" krb5" __UnprocessedBuildArgs= @@ -78,7 +82,7 @@ while :; do arm) __BuildArch=arm __UbuntuArch=armhf - __AlpineArch=armhf + __AlpineArch=armv7 __QEMUArch=arm ;; arm64) @@ -205,7 +209,7 @@ fi if [ -d "$__RootfsDir" ]; then if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* + umount $__RootfsDir/* || true fi rm -rf $__RootfsDir fi @@ -231,9 +235,9 @@ if [[ "$__CodeName" == "alpine" ]]; then add $__AlpinePackagesEdgeMain $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/community \ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackagesEdgeTesting + add $__AlpinePackagesEdgeCommunity rm -r $__ApkToolsDir elif [[ "$__CodeName" == "freebsd" ]]; then @@ -246,7 +250,9 @@ elif [[ "$__CodeName" == "freebsd" ]]; then # get and build package manager wget -O - https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz | tar -C $__RootfsDir/tmp -zxf - cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} - ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make install + # needed for install to succeed + mkdir -p $__RootfsDir/host/etc + ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} # install packages we need. $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update @@ -260,7 +266,7 @@ elif [[ -n $__CodeName ]]; then chroot $__RootfsDir symlinks -cr /usr if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* + umount $__RootfsDir/* || true fi if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1 index 7e6618d64..81db7e7fc 100644 --- a/eng/common/post-build/check-channel-consistency.ps1 +++ b/eng/common/post-build/check-channel-consistency.ps1 @@ -6,6 +6,11 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 + if ($PromoteToChannels -eq "") { + Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info." + ExitWithExitCode 0 + } + # Check that every channel that Maestro told to promote the build to # is available in YAML $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ } diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index f7cfe986d..22eeb2656 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -1,7 +1,8 @@ param( [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use + [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use + [Parameter(Mandatory=$false)][switch] $ContinueOnError # If we should keep checking symbols after an error ) function FirstMatchingSymbolDescriptionOrDefault { @@ -125,6 +126,8 @@ function CheckSymbolsAvailable { Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue } + $TotalFailures = 0 + Get-ChildItem "$InputPath\*.nupkg" | ForEach-Object { $FileName = $_.Name @@ -148,11 +151,22 @@ function CheckSymbolsAvailable { if ($Status -ne 0) { Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName" - ExitWithExitCode $exitCode + + if ($ContinueOnError) { + $TotalFailures++ + } + else { + ExitWithExitCode 1 + } } Write-Host } + + if ($TotalFailures -ne 0) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" + ExitWithExitCode 1 + } } function InstallDotnetSymbol { diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 3872af59b..79c25e7f3 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -57,6 +57,18 @@ try { ExitWithExitCode 1 } + if( $msbuildEngine -eq "vs") { + # Ensure desktop MSBuild is available for sdk tasks. + if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "vs" )) { + $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.4`" }") -MemberType NoteProperty + } + if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.4.0-alpha" -MemberType NoteProperty + } + + InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true + } + $taskProject = GetSdkTaskProject $task if (!(Test-Path $taskProject)) { Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 index 9e5f3cb43..7f28d9c59 100644 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ b/eng/common/sdl/extract-artifact-packages.ps1 @@ -63,7 +63,7 @@ try { } } catch { - Write-Host $_.ScriptStackTrace + Write-Host $_ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 } @@ -74,7 +74,7 @@ try { Measure-Command { ExtractArtifacts } } catch { - Write-Host $_.ScriptStackTrace + Write-Host $_ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 256ffbfb9..968b39bef 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ <?xml version="1.0" encoding="utf-8"?> <packages> - <package id="Microsoft.Guardian.Cli" version="0.7.2"/> + <package id="Microsoft.Guardian.Cli.win10-x64" version="0.20.1"/> </packages> diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 640f2b04e..c64c4f568 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -6,6 +6,7 @@ parameters: # 'continueOnError', the parameter value is not correctly picked up. # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; + downloadArtifacts: true # optional: determines if the artifacts should be dowloaded dependsOn: '' # Optional: dependencies of the job artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts # Usage: @@ -31,8 +32,20 @@ jobs: steps: - checkout: self clean: true - - ${{ if ne(parameters.artifactNames, '') }}: - - ${{ each artifactName in parameters.artifactNames }}: + - ${{ if ne(parameters.downloadArtifacts, 'false')}}: + - ${{ if ne(parameters.artifactNames, '') }}: + - ${{ each artifactName in parameters.artifactNames }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: ${{ artifactName }} + downloadPath: $(Build.ArtifactStagingDirectory)\artifacts + - ${{ if eq(parameters.artifactNames, '') }}: - task: DownloadBuildArtifacts@0 displayName: Download Build Artifacts inputs: @@ -41,20 +54,9 @@ jobs: project: $(AzDOProjectName) pipeline: $(AzDOPipelineId) buildId: $(AzDOBuildId) - artifactName: ${{ artifactName }} + downloadType: specific files + itemPattern: "**" downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - - ${{ if eq(parameters.artifactNames, '') }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: specific files - itemPattern: "**" - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts @@ -81,7 +83,7 @@ jobs: continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.7.2 + -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 536c15c46..fc39647f4 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -24,6 +24,7 @@ parameters: enablePublishBuildAssets: false enablePublishTestResults: false enablePublishUsingPipelines: false + useBuildManifest: false mergeTestResults: false testRunTitle: $(AgentOsName)-$(BuildConfiguration)-xunit name: '' @@ -218,3 +219,12 @@ jobs: ArtifactName: AssetManifests continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Build Manifest + inputs: + PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props' + PublishLocation: Container + ArtifactName: BuildManifests + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index dde27800c..258ba4b77 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -23,7 +23,7 @@ stages: - job: publish_symbols displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) variables: - group: DotNet-Symbol-Server-Pats - name: AzDOProjectName @@ -96,7 +96,7 @@ stages: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - name: AzDOBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) pool: vmImage: 'windows-2019' steps: diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index 08853ec45..bf98d990e 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -25,7 +25,7 @@ stages: - job: publish_symbols displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) variables: - group: DotNet-Symbol-Server-Pats - name: AzDOProjectName @@ -99,7 +99,7 @@ stages: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - name: ArtifactsCategory value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }} - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) pool: vmImage: 'windows-2019' steps: diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 867f37cd8..c99fd7503 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -63,7 +63,7 @@ variables: - name: MaestroApiAccessToken value: $(MaestroAccessToken) - name: MaestroApiVersion - value: "2019-01-16" + value: "2020-02-20" - name: SourceLinkCLIVersion value: 3.0.0 diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index fbab4cb5d..2e1415175 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -9,12 +9,14 @@ parameters: continueOnError: false params: '' artifactNames: '' + downloadArtifacts: true # These parameters let the user customize the call to sdk-task.ps1 for publishing # symbols & general artifacts as well as for signing validation symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' signingValidationAdditionalParameters: '' + useBuildManifest: false # Which stages should finish execution before post-build stages start validateDependsOn: @@ -35,8 +37,9 @@ parameters: NETCoreExperimentalChannelId: 562 NetEngServicesIntChannelId: 678 NetEngServicesProdChannelId: 679 - Net5Preview2ChannelId: 738 Net5Preview3ChannelId: 739 + Net5Preview4ChannelId: 856 + Net5Preview5ChannelId: 857 NetCoreSDK313xxChannelId: 759 NetCoreSDK313xxInternalChannelId: 760 @@ -53,10 +56,8 @@ stages: displayName: Post-build Checks dependsOn: setupMaestroVars variables: - - name: InitialChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'] ] - - name: PromoteToMaestroChannelId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'] ] + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] pool: vmImage: 'windows-2019' steps: @@ -64,8 +65,8 @@ stages: displayName: Maestro Channels Consistency inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 - arguments: -PromoteToChannels "$(InitialChannels)[$(PromoteToMaestroChannelId)]" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview2ChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}} + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.Net5Preview4ChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}} - job: displayName: NuGet Validation @@ -113,6 +114,16 @@ stages: pool: vmImage: 'windows-2019' steps: + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download build manifest + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BuildManifests - task: DownloadBuildArtifacts@0 displayName: Download Package Artifacts inputs: @@ -135,11 +146,13 @@ stages: filePath: eng\common\enable-cross-org-publishing.ps1 arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. - task: PowerShell@2 displayName: Validate inputs: filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine dotnet + arguments: -task SigningValidation -restore -msbuildEngine vs /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' ${{ parameters.signingValidationAdditionalParameters }} @@ -192,6 +205,7 @@ stages: additionalParameters: ${{ parameters.SDLValidationParameters.params }} continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} + downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: @@ -213,10 +227,10 @@ stages: dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview2_Publish' - channelName: '.NET 5 Preview 2' - akaMSChannelName: 'net5/preview2' - channelId: ${{ parameters.Net5Preview2ChannelId }} + stageName: 'Net5_Preview3_Publish' + channelName: '.NET 5 Preview 3' + akaMSChannelName: 'net5/preview3' + channelId: ${{ parameters.Net5Preview3ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' @@ -227,10 +241,24 @@ stages: dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview3_Publish' - channelName: '.NET 5 Preview 3' - akaMSChannelName: 'net5/preview3' - channelId: ${{ parameters.Net5Preview3ChannelId }} + stageName: 'Net5_Preview4_Publish' + channelName: '.NET 5 Preview 4' + akaMSChannelName: 'net5/preview4' + channelId: ${{ parameters.Net5Preview4ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview5_Publish' + channelName: '.NET 5 Preview 5' + akaMSChannelName: 'net5/preview5' + channelId: ${{ parameters.Net5Preview5ChannelId }} transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 05e611edb..b3d29d449 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -5,7 +5,9 @@ jobs: - template: common-variables.yml - name: BuildId value: $[ coalesce(variables.BARBuildId, 0) ] - - name: PromoteToChannelId + - name: PromoteToMaestroChannels + value: $[ coalesce(variables.PromoteToChannelIds, 0) ] + - name: PromoteToMaestroChannel value: $[ coalesce(variables.PromoteToMaestroChannelId, 0) ] pool: vmImage: 'windows-2019' @@ -14,7 +16,7 @@ jobs: - task: DownloadBuildArtifacts@0 displayName: Download Release Configs - condition: eq(variables.PromoteToChannelId, 0) + condition: and(eq(variables.PromoteToMaestroChannels, 0), eq(variables.PromoteToMaestroChannel, 0)) inputs: buildType: current artifactName: ReleaseConfigs @@ -26,20 +28,16 @@ jobs: targetType: inline script: | try { - if ($Env:PromoteToChannelId -eq 0) { + if ($Env:PromoteToMaestroChannels -eq 0 -and $Env:PromoteToMaestroChannel -eq 0) { $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt $BarId = $Content | Select -Index 0 - - $Channels = "" - $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } - + $Channels = $Content | Select -Index 1 $IsStableBuild = $Content | Select -Index 2 $AzureDevOpsProject = $Env:System_TeamProject $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId $AzureDevOpsBuildId = $Env:Build_BuildId - $PromoteToMaestroChannelId = 0 } else { $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" @@ -51,25 +49,23 @@ jobs: $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } $BarId = $Env:BARBuildId - $Channels = 'None' - - #TODO: Fix this once this issue is done: https://github.com/dotnet/arcade/issues/3834 - $IsStableBuild = 'False' + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels][$Env:PromoteToMaestroChannel]" + $IsStableBuild = $buildInfo.stable $AzureDevOpsProject = $buildInfo.azureDevOpsProject $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - $PromoteToMaestroChannelId = $Env:PromoteToMaestroChannelId } Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" - Write-Host "##vso[task.setvariable variable=InitialChannels;isOutput=true]$Channels" + Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels" Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild" Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject" Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId" Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId" - Write-Host "##vso[task.setvariable variable=PromoteToMaestroChannelId;isOutput=true]$PromoteToMaestroChannelId" } catch { Write-Host $_ diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 60c1cd897..3721ce881 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -488,10 +488,11 @@ function GetNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { # Use local cache on CI to ensure deterministic build, # use global cache in dev builds to avoid cost of downloading packages. + # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 if ($useGlobalNuGetCache) { - $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages' + $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { - $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages' + $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' } } @@ -632,6 +633,8 @@ function MSBuild-Core() { } } + $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs" + $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index caae1dbdb..a9dff4408 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -201,7 +201,14 @@ function InstallDotNet { local runtimeSourceFeedKey='' if [[ -n "${7:-}" ]]; then - decodedFeedKey=`echo $7 | base64 --decode` + # The 'base64' binary on alpine uses '-d' and doesn't support '--decode' + # '-d'. To work around this, do a simple detection and switch the parameter + # accordingly. + decodeArg="--decode" + if base64 --help 2>&1 | grep -q "BusyBox"; then + decodeArg="-d" + fi + decodedFeedKey=`echo $7 | base64 $decodeArg` runtimeSourceFeedKey="--feed-credential $decodedFeedKey" fi @@ -415,11 +422,17 @@ function MSBuild-Core { warnaserror_switch="/warnaserror" fi - "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'Build' "Build failed (exit code '$exit_code')." - ExitWithExitCode $exit_code + function RunBuildTool { + export ARCADE_BUILD_TOOL_COMMAND="$_InitializeBuildTool $@" + + "$_InitializeBuildTool" "$@" || { + local exit_code=$? + Write-PipelineTaskError "Build failed (exit code '$exit_code')." + ExitWithExitCode $exit_code + } } + + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } ResolvePath "${BASH_SOURCE[0]}" |