diff options
Diffstat (limited to 'eng')
-rw-r--r-- | eng/Version.Details.xml | 8 | ||||
-rw-r--r-- | eng/common/post-build/dotnetsymbol-init.ps1 | 29 | ||||
-rw-r--r-- | eng/common/post-build/sourcelink-cli-init.ps1 | 29 | ||||
-rw-r--r-- | eng/common/post-build/sourcelink-validation.ps1 | 224 | ||||
-rw-r--r-- | eng/common/post-build/symbols-validation.ps1 | 186 | ||||
-rw-r--r-- | eng/common/templates/job/publish-build-assets.yml | 25 | ||||
-rw-r--r-- | eng/common/templates/post-build/channels/public-dev-release.yml | 146 | ||||
-rw-r--r-- | eng/common/templates/post-build/channels/public-validation-release.yml | 92 | ||||
-rw-r--r-- | eng/common/templates/post-build/common-variables.yml | 9 | ||||
-rw-r--r-- | eng/common/templates/post-build/post-build.yml | 59 | ||||
-rw-r--r-- | eng/common/templates/post-build/promote-build.yml | 28 | ||||
-rw-r--r-- | eng/common/templates/post-build/setup-maestro-vars.yml | 26 | ||||
-rw-r--r-- | eng/common/tools.ps1 | 5 | ||||
-rwxr-xr-x | eng/common/tools.sh | 2 |
14 files changed, 861 insertions, 7 deletions
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5b013cbcd3e..821f25b00c6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,13 +3,13 @@ <ProductDependencies> </ProductDependencies> <ToolsetDependencies> - <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19311.2"> + <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19312.21"> <Uri>https://github.com/dotnet/arcade</Uri> - <Sha>dfc41299b9aadb1ca98093d660df81811eca901b</Sha> + <Sha>e5cd71926eaa13aba3138c14cd80def0ccfebb30</Sha> </Dependency> - <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19311.2"> + <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19312.21"> <Uri>https://github.com/dotnet/arcade</Uri> - <Sha>dfc41299b9aadb1ca98093d660df81811eca901b</Sha> + <Sha>e5cd71926eaa13aba3138c14cd80def0ccfebb30</Sha> </Dependency> </ToolsetDependencies> </Dependencies> diff --git a/eng/common/post-build/dotnetsymbol-init.ps1 b/eng/common/post-build/dotnetsymbol-init.ps1 new file mode 100644 index 00000000000..e7659b98c8c --- /dev/null +++ b/eng/common/post-build/dotnetsymbol-init.ps1 @@ -0,0 +1,29 @@ +param ( + $dotnetsymbolVersion = $null +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +$verbosity = "minimal" + +function Installdotnetsymbol ($dotnetsymbolVersion) { + $dotnetsymbolPackageName = "dotnet-symbol" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { + Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." + } + else { + Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity $verbosity --global + } +} + +Installdotnetsymbol $dotnetsymbolVersion diff --git a/eng/common/post-build/sourcelink-cli-init.ps1 b/eng/common/post-build/sourcelink-cli-init.ps1 new file mode 100644 index 00000000000..9eaa25b3b50 --- /dev/null +++ b/eng/common/post-build/sourcelink-cli-init.ps1 @@ -0,0 +1,29 @@ +param ( + $sourcelinkCliVersion = $null +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +$verbosity = "minimal" + +function InstallSourcelinkCli ($sourcelinkCliVersion) { + $sourcelinkCliPackageName = "sourcelink" + + $dotnetRoot = InitializeDotNetCli -install:$true + $dotnet = "$dotnetRoot\dotnet.exe" + $toolList = & "$dotnet" tool list --global + + if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) { + Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed." + } + else { + Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." + Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." + & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity $verbosity --global + } +} + +InstallSourcelinkCli $sourcelinkCliVersion diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 new file mode 100644 index 00000000000..8abd684e9e5 --- /dev/null +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -0,0 +1,224 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade + [Parameter(Mandatory=$true)][string] $GHCommit, # GitHub commit SHA used to build the packages + [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +# Cache/HashMap (File -> Exist flag) used to consult whether a file exist +# in the repository at a specific commit point. This is populated by inserting +# all files present in the repo at a specific commit point. +$global:RepoFiles = @{} + +$ValidatePackage = { + param( + [string] $PackagePath # Full path to a Symbols.NuGet package + ) + + . $using:PSScriptRoot\..\tools.ps1 + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + # Extensions for which we'll look for SourceLink information + # For now we'll only care about Portable & Embedded PDBs + $RelevantExtensions = @(".dll", ".exe", ".pdb") + + Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + $FailedFiles = 0 + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + try { + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $FileName = $_.FullName + $Extension = [System.IO.Path]::GetExtension($_.Name) + $FakeName = -Join((New-Guid), $Extension) + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName + + # We ignore resource DLLs + if ($FileName.EndsWith(".resources.dll")) { + return + } + + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + + $ValidateFile = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $RealPath, + [ref] $FailedFiles + ) + + $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools" + $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe" + $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String + + if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { + $NumFailedLinks = 0 + + # We only care about Http addresses + $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches + + if ($Matches.Count -ne 0) { + $Matches.Value | + ForEach-Object { + $Link = $_ + $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/" + + $FilePath = $Link.Replace($CommitUrl, "") + $Status = 200 + $Cache = $using:RepoFiles + + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + # Only GitHub links are valid + if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match "github" -or $Uri.Host -match "githubusercontent")) { + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + $Status = 0 + } + } + catch { + write-host $_ + $Status = 0 + } + } + + if ($Status -ne 200) { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ + } + } + } + + if ($NumFailedLinks -ne 0) { + $FailedFiles.value++ + $global:LASTEXITCODE = 1 + } + } + } + + &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) + } + } + catch { + + } + finally { + $zip.Dispose() + } + + if ($FailedFiles -eq 0) { + Write-Host "Passed." + } + else { + Write-PipelineTaskError "$PackagePath has broken SourceLink links." + } +} + +function ValidateSourceLinkLinks { + if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { + if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) { + Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>" + ExitWithExitCode 1 + } + else { + $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2'; + } + } + + if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { + Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string" + ExitWithExitCode 1 + } + + $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") + $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") + + try { + # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash + $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree + + foreach ($file in $Data) { + $Extension = [System.IO.Path]::GetExtension($file.path) + + if ($CodeExtensions.Contains($Extension)) { + $RepoFiles[$file.path] = 1 + } + } + } + catch { + Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" + Write-Host $_ + ExitWithExitCode 1 + } + + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + # Process each NuGet package in parallel + $Jobs = @() + Get-ChildItem "$InputPath\*.symbols.nupkg" | + ForEach-Object { + $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName + } + + foreach ($Job in $Jobs) { + Wait-Job -Id $Job.Id | Receive-Job + } +} + +function CheckExitCode ([string]$stage) { + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Write-Host "Installing SourceLink CLI..." + Get-Location + . $PSScriptRoot\sourcelink-cli-init.ps1 -sourcelinkCliVersion $SourcelinkCliVersion + CheckExitCode "Running sourcelink-cli-init" + + Measure-Command { ValidateSourceLinkLinks } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 new file mode 100644 index 00000000000..69456854e04 --- /dev/null +++ b/eng/common/post-build/symbols-validation.ps1 @@ -0,0 +1,186 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 2.0 + +. $PSScriptRoot\..\tools.ps1 + +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function FirstMatchingSymbolDescriptionOrDefault { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $SymbolsPath + ) + + $FileName = [System.IO.Path]::GetFileName($FullPath) + $Extension = [System.IO.Path]::GetExtension($FullPath) + + # Those below are potential symbol files that the `dotnet symbol` might + # return. Which one will be returned depend on the type of file we are + # checking and which type of file was uploaded. + + # The file itself is returned + $SymbolPath = $SymbolsPath + "\" + $FileName + + # PDB file for the module + $PdbPath = $SymbolPath.Replace($Extension, ".pdb") + + # PDB file for R2R module (created by crossgen) + $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") + + # DBG file for a .so library + $SODbg = $SymbolPath.Replace($Extension, ".so.dbg") + + # DWARF file for a .dylib + $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") + + $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe" + + & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + + if (Test-Path $PdbPath) { + return "PDB" + } + elseif (Test-Path $NGenPdb) { + return "NGen PDB" + } + elseif (Test-Path $SODbg) { + return "DBG for SO" + } + elseif (Test-Path $DylibDwarf) { + return "Dwarf for Dylib" + } + elseif (Test-Path $SymbolPath) { + return "Module" + } + else { + return $null + } +} + +function CountMissingSymbols { + param( + [string] $PackagePath # Path to a NuGet package + ) + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + Write-PipelineTaskError "Input file does not exist: $PackagePath" + ExitWithExitCode 1 + } + + # Extensions for which we'll look for symbols + $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") + + # How many files are missing symbol information + $MissingSymbols = 0 + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $PackageGuid = New-Guid + $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid + $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" + + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + + Get-ChildItem -Recurse $ExtractPath | + Where-Object {$RelevantExtensions -contains $_.Extension} | + ForEach-Object { + if ($_.FullName -Match "\\ref\\") { + Write-Host "`t Ignoring reference assembly file" $_.FullName + return + } + + $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath + $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath + + Write-Host -NoNewLine "`t Checking file" $_.FullName "... " + + if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { + Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" + } + else { + $MissingSymbols++ + + if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { + Write-Host "No symbols found on MSDL or SymWeb!" + } + else { + if ($SymbolsOnMSDL -eq $null) { + Write-Host "No symbols found on MSDL!" + } + else { + Write-Host "No symbols found on SymWeb!" + } + } + } + } + + Pop-Location + + return $MissingSymbols +} + +function CheckSymbolsAvailable { + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + Get-ChildItem "$InputPath\*.nupkg" | + ForEach-Object { + $FileName = $_.Name + + # These packages from Arcade-Services include some native libraries that + # our current symbol uploader can't handle. Below is a workaround until + # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. + if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { + Write-Host "Ignoring Arcade-services file: $FileName" + Write-Host + return + } + + Write-Host "Validating $FileName " + $Status = CountMissingSymbols "$InputPath\$FileName" + + if ($Status -ne 0) { + Write-PipelineTaskError "Missing symbols for $Status modules in the package $FileName" + ExitWithExitCode $exitCode + } + + Write-Host + } +} + +function CheckExitCode ([string]$stage) { + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + Write-PipelineTaskError "Something failed while '$stage'. Check for errors above. Exiting now..." + ExitWithExitCode $exitCode + } +} + +try { + Write-Host "Installing dotnet symbol ..." + Get-Location + . $PSScriptRoot\dotnetsymbol-init.ps1 -dotnetsymbolVersion $DotnetSymbolVersion + CheckExitCode "Running dotnetsymbol-init" + + CheckSymbolsAvailable +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 +} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 620bd3c62e7..619ec68aa43 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -57,8 +57,33 @@ jobs: /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:Configuration=$(_BuildConfig) + /v:detailed condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + - task: powershell@2 + displayName: Create BARBuildId Artifact + inputs: + targetType: inline + script: | + Add-Content -Path "$(Build.StagingDirectory)/BARBuildId.txt" -Value $(BARBuildId) + - task: powershell@2 + displayName: Create Channels Artifact + inputs: + targetType: inline + script: | + Add-Content -Path "$(Build.StagingDirectory)/Channels.txt" -Value "$(DefaultChannels)" + - task: PublishBuildArtifacts@1 + displayName: Publish BAR BuildId to VSTS + inputs: + PathtoPublish: '$(Build.StagingDirectory)/BARBuildId.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + - task: PublishBuildArtifacts@1 + displayName: Publish Channels to VSTS + inputs: + PathtoPublish: '$(Build.StagingDirectory)/Channels.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - task: PublishBuildArtifacts@1 displayName: Publish Logs to VSTS diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/public-dev-release.yml new file mode 100644 index 00000000000..b332cb51732 --- /dev/null +++ b/eng/common/templates/post-build/channels/public-dev-release.yml @@ -0,0 +1,146 @@ +parameters: + enableSymbolValidation: true + +stages: +- stage: Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Developer Channel + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + buildType: current + artifactName: PDBArtifacts + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:Configuration=Release + + - job: + displayName: Publish to Static Feed + dependsOn: setupMaestroVars + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToPackageFeed -restore -msbuildEngine dotnet + /p:AccountKeyToStaticFeed='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:ArtifactsCategory='$(_DotNetArtifactsCategory)' + /p:OverrideAssetsWithSameName=true + /p:PassIfExistingItemIdentical=true + /p:Configuration=Release + + +- stage: PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - ${{ if eq(parameters.enableSymbolValidation, 'true') }}: + - job: + displayName: Symbol Availability + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Check Symbol Availability + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion) + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + variables: + BARBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id) + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com/ --password $(MaestroAccessToken) + continueOnError: true + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml new file mode 100644 index 00000000000..0b9719da82b --- /dev/null +++ b/eng/common/templates/post-build/channels/public-validation-release.yml @@ -0,0 +1,92 @@ +stages: +- stage: PVR_Publish + dependsOn: validate + variables: + - template: ../common-variables.yml + displayName: Validation Channel + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Publish to Static Feed + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) + variables: + - group: DotNet-Blob-Feed + - group: Publish-Build-Assets + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToPackageFeed -restore -msbuildEngine dotnet + /p:AccountKeyToStaticFeed='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='https://maestro-prod.westus2.cloudapp.azure.com' + /p:BuildAssetRegistryToken='$(MaestroAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:ArtifactsCategory='$(_DotNetArtifactsCategory)' + /p:OverrideAssetsWithSameName=true + /p:PassIfExistingItemIdentical=true + /p:Configuration=Release + + +- stage: PVR_PublishValidation + displayName: Publish Validation + variables: + - template: ../common-variables.yml + jobs: + - template: ../setup-maestro-vars.yml + + - job: + displayName: Gather Drop + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id) + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - group: Publish-Build-Assets + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Setup Darc CLI + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/eng/common/darc-init.ps1' + + - task: PowerShell@2 + displayName: Run Darc gather-drop + inputs: + targetType: inline + script: | + darc gather-drop --non-shipping --continue-on-error --id $(BARBuildId) --output-dir $(Agent.BuildDirectory)/Temp/Drop/ --bar-uri https://maestro-prod.westus2.cloudapp.azure.com --password $(MaestroAccessToken) + continueOnError: true + + - template: ../promote-build.yml + parameters: + ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml new file mode 100644 index 00000000000..97b48d97fec --- /dev/null +++ b/eng/common/templates/post-build/common-variables.yml @@ -0,0 +1,9 @@ +variables: + # .NET Core 3 Dev + PublicDevRelease_30_Channel_Id: 3 + + # .NET Tools - Validation + PublicValidationRelease_30_Channel_Id: 9 + + SourceLinkCLIVersion: 3.0.0 + SymbolToolVersion: 1.0.1 diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml new file mode 100644 index 00000000000..6b74475a6f8 --- /dev/null +++ b/eng/common/templates/post-build/post-build.yml @@ -0,0 +1,59 @@ +parameters: + enableSourceLinkValidation: true + enableSigningValidation: true + enableSymbolValidation: true + +stages: +- stage: validate + dependsOn: build + displayName: Validate + jobs: + - ${{ if eq(parameters.enableSigningValidation, 'true') }}: + - job: + displayName: Signing Validation + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine dotnet + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:Configuration=Release + + - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: + - job: + displayName: SourceLink Validation + variables: + - template: common-variables.yml + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + +- template: \eng\common\templates\post-build\channels\public-dev-release.yml + parameters: + enableSymbolValidation: ${{ parameters.enableSymbolValidation }} + +- template: \eng\common\templates\post-build\channels\public-validation-release.yml diff --git a/eng/common/templates/post-build/promote-build.yml b/eng/common/templates/post-build/promote-build.yml new file mode 100644 index 00000000000..d00317003b7 --- /dev/null +++ b/eng/common/templates/post-build/promote-build.yml @@ -0,0 +1,28 @@ +parameters: + ChannelId: 0 + +jobs: +- job: + displayName: Promote Build + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }}) + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: ChannelId + value: ${{ parameters.ChannelId }} + - group: Publish-Build-Assets + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Add Build to Channel + inputs: + targetType: inline + script: | + $headers = @{ + "Accept" = "application/json" + "Authorization" = "Bearer $(MaestroAccessToken)" + } + Invoke-RestMethod -Method Post -Headers $headers -Uri https://maestro-prod.westus2.cloudapp.azure.com/api/channels/$(ChannelId)/builds/$(BARBuildId)?api-version=2019-01-16 + enabled: false diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml new file mode 100644 index 00000000000..b40e0260a3a --- /dev/null +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -0,0 +1,26 @@ +jobs: +- job: setupMaestroVars + displayName: Setup Maestro Vars + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + script: | + . "$(Build.SourcesDirectory)/eng/common/tools.ps1" + + $BarId = Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/BARBuildId.txt" + Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId + + $Channels = "" + Get-Content "$(Build.StagingDirectory)/ReleaseConfigs/Channels.txt" | ForEach-Object { $Channels += "$_ ," } + Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels" diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 1b9e2cd8849..9ea85c4f978 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -138,7 +138,8 @@ function Write-PipelineSetVariable { if($ci) { Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ 'variable' = $Name - 'issecret' = $Secret + 'isSecret' = $Secret + 'isOutput' = 'true' } -AsOutput:$AsOutput } } @@ -377,7 +378,7 @@ function LocateVisualStudio([object]$vsRequirements = $null){ } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild") + $args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild", "-products", "*") if (Get-Member -InputObject $vsRequirements -Name "version") { $args += "-version" diff --git a/eng/common/tools.sh b/eng/common/tools.sh index a8dffd34528..86973b3c2f0 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -349,7 +349,7 @@ function InitializeToolset { fi if [[ "$restore" != true ]]; then - EmitError "Toolset version $toolsetVersion has not been restored." + EmitError "Toolset version $toolset_version has not been restored." ExitWithExitCode 2 fi |