diff options
author | dotnet-maestro[bot] <42748379+dotnet-maestro[bot]@users.noreply.github.com> | 2020-02-10 19:03:10 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-02-10 19:03:10 +0300 |
commit | 900e77c86891443314fd66a63f35cb8a918ff826 (patch) | |
tree | dd8d1bb454a38fdf1bab186a810839adc9596828 /eng | |
parent | 8082e87c4e6f63409acf3862387c0db098d8745a (diff) |
[master] Update dependencies from dotnet/arcade (#18674)
* Update dependencies from https://github.com/dotnet/arcade build 20200201.2
- Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20101.2
- Microsoft.DotNet.Helix.Sdk - 5.0.0-beta.20101.2
* Update dependencies from https://github.com/dotnet/arcade build 20200209.1
- Microsoft.DotNet.Arcade.Sdk - 5.0.0-beta.20109.1
- Microsoft.DotNet.Helix.Sdk - 5.0.0-beta.20109.1
Diffstat (limited to 'eng')
-rw-r--r-- | eng/Version.Details.xml | 8 | ||||
-rw-r--r-- | eng/common/darc-init.ps1 | 4 | ||||
-rw-r--r-- | eng/common/generate-graph-files.ps1 | 2 | ||||
-rw-r--r-- | eng/common/performance/performance-setup.ps1 | 6 | ||||
-rw-r--r-- | eng/common/performance/performance-setup.sh | 6 | ||||
-rw-r--r-- | eng/common/post-build/add-build-to-channel.ps1 (renamed from eng/common/post-build/promote-build.ps1) | 2 | ||||
-rw-r--r-- | eng/common/templates/job/execute-sdl.yml | 20 | ||||
-rw-r--r-- | eng/common/templates/post-build/channels/generic-internal-channel.yml | 32 | ||||
-rw-r--r-- | eng/common/templates/post-build/channels/generic-public-channel.yml | 32 | ||||
-rw-r--r-- | eng/common/templates/post-build/post-build.yml | 219 | ||||
-rw-r--r-- | eng/common/templates/post-build/promote-build.yml | 25 | ||||
-rw-r--r-- | eng/common/templates/post-build/setup-maestro-vars.yml | 51 | ||||
-rw-r--r-- | eng/common/templates/steps/add-build-to-channel.yml (renamed from eng/common/templates/steps/promote-build.yml) | 4 | ||||
-rw-r--r-- | eng/common/tools.ps1 | 51 | ||||
-rwxr-xr-x | eng/common/tools.sh | 38 |
15 files changed, 329 insertions, 171 deletions
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c815d0d37aa..240336214b5 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -15,13 +15,13 @@ </Dependency> </ProductDependencies> <ToolsetDependencies> - <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20076.3"> + <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20109.1"> <Uri>https://github.com/dotnet/arcade</Uri> - <Sha>cc8fe69635c7c6e791c87540851aea75946945fa</Sha> + <Sha>b0e8d3944155f94f83deea8afe025debe369e69f</Sha> </Dependency> - <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="5.0.0-beta.20076.3"> + <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="5.0.0-beta.20109.1"> <Uri>https://github.com/dotnet/arcade</Uri> - <Sha>cc8fe69635c7c6e791c87540851aea75946945fa</Sha> + <Sha>b0e8d3944155f94f83deea8afe025debe369e69f</Sha> </Dependency> </ToolsetDependencies> </Dependencies> diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index fc2190365f3..2a30ccfd746 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -24,13 +24,15 @@ function InstallDarcCli ($darcVersion) { $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content } - $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' Write-Host "Installing Darc CLI version $darcVersion..." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.' if (-not $toolpath) { + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g }else { + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g --tool-path '$toolpath'" & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" } } diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 index 7ad26afa692..0728b1a8b57 100644 --- a/eng/common/generate-graph-files.ps1 +++ b/eng/common/generate-graph-files.ps1 @@ -3,7 +3,7 @@ Param( [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created - [string] $darcVersion = '1.1.0-beta.19175.6', # darc's version + [string] $darcVersion, # darc's version [string] $graphvizVersion = '2.38', # GraphViz version [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index f6e878297db..e3376699294 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -9,12 +9,12 @@ Param( [string] $Branch=$env:BUILD_SOURCEBRANCH, [string] $CommitSha=$env:BUILD_SOURCEVERSION, [string] $BuildNumber=$env:BUILD_BUILDNUMBER, - [string] $RunCategories="coreclr corefx", + [string] $RunCategories="Libraries Runtime", [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Kind="micro", [switch] $Internal, [switch] $Compare, - [string] $Configurations="CompilationMode=$CompilationMode" + [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind" ) $RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") @@ -50,7 +50,7 @@ if ($Internal) { } # FIX ME: This is a workaround until we get this from the actual pipeline -$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations" +$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" if ($RunFromPerformanceRepo) { diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 7ef15514e52..94a04e0fe55 100644 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -13,9 +13,9 @@ build_number=$BUILD_BUILDNUMBER internal=false compare=false kind="micro" -run_categories="coreclr corefx" +run_categories="Libraries Runtime" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" -configurations= +configurations="CompliationMode=$compilation_mode RunKind=$kind" run_from_perf_repo=false use_core_run=true use_baseline_core_run=true @@ -164,7 +164,7 @@ if [[ "$internal" == true ]]; then fi fi -common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations" +common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" if [[ "$run_from_perf_repo" = true ]]; then diff --git a/eng/common/post-build/promote-build.ps1 b/eng/common/post-build/add-build-to-channel.ps1 index ce45635fbd2..de2d957922a 100644 --- a/eng/common/post-build/promote-build.ps1 +++ b/eng/common/post-build/add-build-to-channel.ps1 @@ -17,7 +17,7 @@ try { ExitWithExitCode 1 } - # Get info about which channels the build has already been promoted to + # Get info about which channel(s) the build has already been promoted to $buildInfo = Get-MaestroBuild -BuildId $BuildId if (!$buildInfo) { diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 2973bcaf3a4..640f2b04e24 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -1,4 +1,5 @@ parameters: + enable: 'false' # Whether the SDL validation job should execute or not overrideParameters: '' # Optional: to override values for parameters. additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named @@ -16,8 +17,15 @@ jobs: - job: Run_SDL dependsOn: ${{ parameters.dependsOn }} displayName: Run SDL tool + condition: eq( ${{ parameters.enable }}, 'true') variables: - group: DotNet-VSTS-Bot + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: name: Hosted VS2017 steps: @@ -28,14 +36,22 @@ jobs: - task: DownloadBuildArtifacts@0 displayName: Download Build Artifacts inputs: - buildType: current + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) artifactName: ${{ artifactName }} downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - ${{ if eq(parameters.artifactNames, '') }}: - task: DownloadBuildArtifacts@0 displayName: Download Build Artifacts inputs: - buildType: current + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: specific files itemPattern: "**" downloadPath: $(Build.ArtifactStagingDirectory)\artifacts diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 700211049b5..380122901d0 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -23,9 +23,15 @@ stages: - job: publish_symbols displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )) + condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) variables: - group: DotNet-Symbol-Server-Pats + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: vmImage: 'windows-2019' steps: @@ -33,10 +39,14 @@ stages: displayName: Download Build Assets continueOnError: true inputs: - buildType: 'current' + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | - PDBArtifacts/** + PdbArtifacts/** BlobArtifacts/** downloadPath: '$(Build.ArtifactStagingDirectory)' @@ -79,7 +89,13 @@ stages: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - name: IsStableBuild value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }})) + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) pool: vmImage: 'windows-2019' steps: @@ -87,7 +103,11 @@ stages: displayName: Download Build Assets continueOnError: true inputs: - buildType: 'current' + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | PackageArtifacts/** @@ -146,6 +166,6 @@ stages: StageLabel: '${{ parameters.stageName }}' JobLabel: 'AssetsPublishing' - - template: ../../steps/promote-build.yml + - template: ../../steps/add-build-to-channel.yml parameters: ChannelId: ${{ parameters.channelId }} diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index fbb5a19b677..ae8c6349d49 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -23,9 +23,15 @@ stages: - job: publish_symbols displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )) + condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) variables: - group: DotNet-Symbol-Server-Pats + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool: vmImage: 'windows-2019' steps: @@ -33,10 +39,14 @@ stages: displayName: Download Build Assets continueOnError: true inputs: - buildType: 'current' + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | - PDBArtifacts/** + PdbArtifacts/** BlobArtifacts/** downloadPath: '$(Build.ArtifactStagingDirectory)' @@ -78,7 +88,13 @@ stages: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - name: IsStableBuild value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }})) + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) pool: vmImage: 'windows-2019' steps: @@ -86,7 +102,11 @@ stages: displayName: Download Build Assets continueOnError: true inputs: - buildType: 'current' + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) downloadType: 'specific' itemPattern: | PackageArtifacts/** @@ -145,6 +165,6 @@ stages: StageLabel: '${{ parameters.stageName }}' JobLabel: 'AssetsPublishing' - - template: ../../steps/promote-build.yml + - template: ../../steps/add-build-to-channel.yml parameters: ChannelId: ${{ parameters.channelId }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 33295ba125b..70cb65f50b4 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -26,97 +26,136 @@ stages: - stage: Validate dependsOn: ${{ parameters.validateDependsOn }} displayName: Validate + variables: + - template: common-variables.yml jobs: - - ${{ if eq(parameters.enableNugetValidation, 'true') }}: - - job: - displayName: NuGet Validation - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - ${{ if eq(parameters.enableSigningValidation, 'true') }}: - - job: - displayName: Signing Validation - variables: - - template: common-variables.yml - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: current - artifactName: PackageArtifacts - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' - - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: eng\common\enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine dotnet - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - - - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: - - job: - displayName: SourceLink Validation - variables: - - template: common-variables.yml - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: current - artifactName: BlobArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}: - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} + - template: setup-maestro-vars.yml + + - job: + displayName: NuGet Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableNugetValidation }}, 'true') + pool: + vmImage: 'windows-2019' + variables: + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSigningValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine dotnet + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + + - job: + displayName: SourceLink Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + enable: ${{ parameters.SDLValidationParameters.enable }} + dependsOn: setupMaestroVars + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} + artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: diff --git a/eng/common/templates/post-build/promote-build.yml b/eng/common/templates/post-build/promote-build.yml deleted file mode 100644 index 6b479c3b82a..00000000000 --- a/eng/common/templates/post-build/promote-build.yml +++ /dev/null @@ -1,25 +0,0 @@ -parameters: - ChannelId: 0 - -jobs: -- job: - displayName: Promote Build - dependsOn: setupMaestroVars - condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.ChannelId }})) - variables: - - name: BARBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] - - name: ChannelId - value: ${{ parameters.ChannelId }} - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Add Build to Channel - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1 - arguments: -BuildId $(BARBuildId) - -ChannelId $(ChannelId) - -MaestroApiAccessToken $(MaestroApiAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 716b53f7405..05e611edb68 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -1,6 +1,12 @@ jobs: - job: setupMaestroVars displayName: Setup Maestro Vars + variables: + - template: common-variables.yml + - name: BuildId + value: $[ coalesce(variables.BARBuildId, 0) ] + - name: PromoteToChannelId + value: $[ coalesce(variables.PromoteToMaestroChannelId, 0) ] pool: vmImage: 'windows-2019' steps: @@ -8,6 +14,7 @@ jobs: - task: DownloadBuildArtifacts@0 displayName: Download Release Configs + condition: eq(variables.PromoteToChannelId, 0) inputs: buildType: current artifactName: ReleaseConfigs @@ -19,18 +26,50 @@ jobs: targetType: inline script: | try { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + if ($Env:PromoteToChannelId -eq 0) { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - $BarId = $Content | Select -Index 0 + $BarId = $Content | Select -Index 0 - $Channels = "" - $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } + $Channels = "" + $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } + + $IsStableBuild = $Content | Select -Index 2 - $IsStableBuild = $Content | Select -Index 2 + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + $PromoteToMaestroChannelId = 0 + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = 'None' + + #TODO: Fix this once this issue is done: https://github.com/dotnet/arcade/issues/3834 + $IsStableBuild = 'False' + + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + $PromoteToMaestroChannelId = $Env:PromoteToMaestroChannelId + } Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" Write-Host "##vso[task.setvariable variable=InitialChannels;isOutput=true]$Channels" Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId" + Write-Host "##vso[task.setvariable variable=PromoteToMaestroChannelId;isOutput=true]$PromoteToMaestroChannelId" } catch { Write-Host $_ @@ -38,3 +77,5 @@ jobs: Write-Host $_.ScriptStackTrace exit 1 } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) diff --git a/eng/common/templates/steps/promote-build.yml b/eng/common/templates/steps/add-build-to-channel.yml index b90404435dd..f67a210d62f 100644 --- a/eng/common/templates/steps/promote-build.yml +++ b/eng/common/templates/steps/add-build-to-channel.yml @@ -5,9 +5,9 @@ steps: - task: PowerShell@2 displayName: Add Build to Channel inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1 + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 arguments: -BuildId $(BARBuildId) -ChannelId ${{ parameters.ChannelId }} -MaestroApiAccessToken $(MaestroApiAccessToken) -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 23a3fd53d7a..fef7e5a3fdc 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -155,12 +155,12 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # The following code block is protecting against concurrent access so that this function can # be called in parallel. if ($createSdkLocationFile) { - do { + do { $sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName()) - } + } until (!(Test-Path $sdkCacheFileTemp)) Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot - + try { Rename-Item -Force -Path $sdkCacheFileTemp 'sdk.txt' } catch { @@ -188,7 +188,33 @@ function GetDotNetInstallScript([string] $dotnetRoot) { if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript + + $maxRetries = 5 + $retries = 1 + + $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" + + while($true) { + try { + Write-Host "GET $uri" + Invoke-WebRequest $uri -OutFile $installScript + break + } + catch { + Write-Host "Failed to download '$uri'" + Write-Error $_.Exception.Message -ErrorAction Continue + } + + if (++$retries -le $maxRetries) { + $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff + Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)." + Start-Sleep -Seconds $delayInSeconds + } + else { + throw "Unable to download file in $maxRetries attempts." + } + + } } return $installScript @@ -198,12 +224,12 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $arc InstallDotNet $dotnetRoot $version $architecture } -function InstallDotNet([string] $dotnetRoot, - [string] $version, - [string] $architecture = '', - [string] $runtime = '', - [bool] $skipNonVersionedFiles = $false, - [string] $runtimeSourceFeed = '', +function InstallDotNet([string] $dotnetRoot, + [string] $version, + [string] $architecture = '', + [string] $runtime = '', + [bool] $skipNonVersionedFiles = $false, + [string] $runtimeSourceFeed = '', [string] $runtimeSourceFeedKey = '') { $installScript = GetDotNetInstallScript $dotnetRoot @@ -298,7 +324,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $vsMajorVersion = $vsMinVersion.Major $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha" } - + $vsInstallDir = $null if ($xcopyMSBuildVersion.Trim() -ine "none") { $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install @@ -655,9 +681,6 @@ Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir Write-PipelineSetVariable -Name 'TMP' -Value $TempDir -$env:TEMP=$TempDir -$env:TMP=$TempDir - # Import custom tools configuration, if present in the repo. # Note: Import in global scope so that the script set top-level variables without qualification. if (!$disableConfigureToolsetImport) { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index e071af4ee49..c26525e52b5 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -41,7 +41,7 @@ fi # Configures warning treatment in msbuild. warn_as_error=${warn_as_error:-true} -# True to attempt using .NET Core already that meets requirements specified in global.json +# True to attempt using .NET Core already that meets requirements specified in global.json # installed on the machine instead of downloading one. use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} @@ -172,7 +172,7 @@ function InstallDotNetSdk { function InstallDotNet { local root=$1 local version=$2 - + GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript @@ -218,6 +218,28 @@ function InstallDotNet { } } +function with_retries { + local maxRetries=5 + local retries=1 + echo "Trying to run '$@' for maximum of $maxRetries attempts." + while [[ $((retries++)) -le $maxRetries ]]; do + "$@" + + if [[ $? == 0 ]]; then + echo "Ran '$@' successfully." + return 0 + fi + + timeout=$((2**$retries-1)) + echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2 + sleep $timeout + done + + echo "Failed to execute '$@' for $maxRetries times." 1>&2 + + return 1 +} + function GetDotNetInstallScript { local root=$1 local install_script="$root/dotnet-install.sh" @@ -230,13 +252,13 @@ function GetDotNetInstallScript { # Use curl if available, otherwise use wget if command -v curl > /dev/null; then - curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { + with_retries curl "$install_script_url" -isSLv --retry 10 --create-dirs -o "$install_script" || { local exit_code=$? Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." ExitWithExitCode $exit_code } - else - wget -q -O "$install_script" "$install_script_url" || { + else + with_retries wget -v -O "$install_script" "$install_script_url" || { local exit_code=$? Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." ExitWithExitCode $exit_code @@ -251,11 +273,11 @@ function InitializeBuildTool { if [[ -n "${_InitializeBuildTool:-}" ]]; then return fi - + InitializeDotNetCli $restore # return values - _InitializeBuildTool="$_InitializeDotNetCli/dotnet" + _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" _InitializeBuildToolFramework="netcoreapp2.1" } @@ -319,7 +341,7 @@ function InitializeToolset { if [[ "$binary_log" == true ]]; then bl="/bl:$log_dir/ToolsetRestore.binlog" fi - + echo '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' > "$proj" MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" |