Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/mono/mono.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/eng
diff options
context:
space:
mode:
authordotnet-maestro[bot] <dotnet-maestro[bot]@users.noreply.github.com>2019-09-06 20:52:00 +0300
committerMarek Safar <marek.safar@gmail.com>2019-09-09 20:03:51 +0300
commite563d3c76c448935b8c1f83b27a44a91478856ac (patch)
treed5286daaf120eb4e21fa5f7ebe81fcdec23a7a85 /eng
parent8590a25b301ec08ac688e93df37fe3ef3c2ddb11 (diff)
Update dependencies from https://github.com/dotnet/arcade build 20190906.2
- Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19456.2 - Microsoft.DotNet.Helix.Sdk - 2.0.0-beta.19456.2
Diffstat (limited to 'eng')
-rw-r--r--eng/Version.Details.xml8
-rw-r--r--eng/common/darc-init.ps14
-rwxr-xr-xeng/common/darc-init.sh6
-rw-r--r--eng/common/init-tools-native.sh4
-rw-r--r--eng/common/native/CommonLibrary.psm12
-rw-r--r--eng/common/native/install-cmake-test.sh117
-rw-r--r--eng/common/native/install-cmake.sh2
-rw-r--r--eng/common/performance/perfhelixpublish.proj29
-rw-r--r--eng/common/performance/performance-setup.ps121
-rw-r--r--eng/common/performance/performance-setup.sh78
-rw-r--r--eng/common/post-build/darc-gather-drop.ps110
-rw-r--r--eng/common/post-build/sourcelink-validation.ps184
-rw-r--r--eng/common/post-build/symbols-validation.ps120
-rw-r--r--eng/common/sdl/execute-all-sdl-tools.ps117
-rw-r--r--eng/common/sdl/init-sdl.ps13
-rw-r--r--eng/common/sdl/packages.config4
-rw-r--r--eng/common/sdl/run-sdl.ps134
-rw-r--r--eng/common/templates/job/execute-sdl.yml17
-rw-r--r--eng/common/templates/job/job.yml11
-rw-r--r--eng/common/templates/job/publish-build-assets.yml7
-rw-r--r--eng/common/templates/post-build/channels/netcore-dev-30.yml (renamed from eng/common/templates/post-build/channels/public-dev-release.yml)92
-rw-r--r--eng/common/templates/post-build/channels/netcore-dev-31.yml154
-rw-r--r--eng/common/templates/post-build/channels/netcore-dev-5.yml160
-rw-r--r--eng/common/templates/post-build/channels/netcore-internal-30.yml (renamed from eng/common/templates/post-build/channels/internal-servicing.yml)87
-rw-r--r--eng/common/templates/post-build/channels/netcore-release-30.yml (renamed from eng/common/templates/post-build/channels/public-release.yml)86
-rw-r--r--eng/common/templates/post-build/channels/netcore-release-31.yml156
-rw-r--r--eng/common/templates/post-build/channels/netcore-tools-latest.yml160
-rw-r--r--eng/common/templates/post-build/channels/public-validation-release.yml72
-rw-r--r--eng/common/templates/post-build/common-variables.yml47
-rw-r--r--eng/common/templates/post-build/darc-gather-drop.yml2
-rw-r--r--eng/common/templates/post-build/post-build.yml66
-rw-r--r--eng/common/templates/post-build/promote-build.yml2
-rw-r--r--eng/common/tools.ps111
-rwxr-xr-xeng/common/tools.sh21
34 files changed, 1304 insertions, 290 deletions
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index 37379898267..f977163f023 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -15,13 +15,13 @@
</Dependency>
</ProductDependencies>
<ToolsetDependencies>
- <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19379.1">
+ <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19456.2">
<Uri>https://github.com/dotnet/arcade</Uri>
- <Sha>193793488c727c2d12a04b4abe5e6ea63ad20526</Sha>
+ <Sha>397060df67388da56b50de7e6f7292a5dccc2de6</Sha>
</Dependency>
- <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19379.1">
+ <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19456.2">
<Uri>https://github.com/dotnet/arcade</Uri>
- <Sha>193793488c727c2d12a04b4abe5e6ea63ad20526</Sha>
+ <Sha>397060df67388da56b50de7e6f7292a5dccc2de6</Sha>
</Dependency>
</ToolsetDependencies>
</Dependencies>
diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1
index 8854d979f37..46d175fdfdc 100644
--- a/eng/common/darc-init.ps1
+++ b/eng/common/darc-init.ps1
@@ -1,9 +1,9 @@
param (
$darcVersion = $null,
- $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16"
+ $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16",
+ $verbosity = "m"
)
-$verbosity = "m"
. $PSScriptRoot\tools.ps1
function InstallDarcCli ($darcVersion) {
diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh
index abdd0bc05a1..242429bca65 100755
--- a/eng/common/darc-init.sh
+++ b/eng/common/darc-init.sh
@@ -3,6 +3,7 @@
source="${BASH_SOURCE[0]}"
darcVersion=''
versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16"
+verbosity=m
while [[ $# > 0 ]]; do
opt="$(echo "$1" | awk '{print tolower($0)}')"
@@ -15,6 +16,10 @@ while [[ $# > 0 ]]; do
versionEndpoint=$2
shift
;;
+ --verbosity)
+ verbosity=$2
+ shift
+ ;;
*)
echo "Invalid argument: $1"
usage
@@ -34,7 +39,6 @@ while [[ -h "$source" ]]; do
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
-verbosity=m
. "$scriptroot/tools.sh"
diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh
index 5f2e77f448b..4dafaaca130 100644
--- a/eng/common/init-tools-native.sh
+++ b/eng/common/init-tools-native.sh
@@ -70,8 +70,7 @@ function ReadGlobalJsonNativeTools {
# Only extract the contents of the object.
local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
native_tools_list=${native_tools_list//[\" ]/}
- native_tools_list=${native_tools_list//,/$'\n'}
- native_tools_list="$(echo -e "${native_tools_list}" | tr -d '[[:space:]]')"
+ native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' )
local old_IFS=$IFS
while read -r line; do
@@ -108,6 +107,7 @@ else
installer_command+=" --baseuri $base_uri"
installer_command+=" --installpath $install_bin"
installer_command+=" --version $tool_version"
+ echo $installer_command
if [[ $force = true ]]; then
installer_command+=" --force"
diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1
index 2a08d5246e7..41416862d91 100644
--- a/eng/common/native/CommonLibrary.psm1
+++ b/eng/common/native/CommonLibrary.psm1
@@ -152,6 +152,8 @@ function Get-File {
}
else {
Write-Verbose "Downloading $Uri"
+ # Don't display the console progress UI - it's a huge perf hit
+ $ProgressPreference = 'SilentlyContinue'
while($Attempt -Lt $DownloadRetries)
{
try {
diff --git a/eng/common/native/install-cmake-test.sh b/eng/common/native/install-cmake-test.sh
new file mode 100644
index 00000000000..53ddf4e6860
--- /dev/null
+++ b/eng/common/native/install-cmake-test.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. $scriptroot/common-library.sh
+
+base_uri=
+install_path=
+version=
+clean=false
+force=false
+download_retries=5
+retry_wait_time_seconds=30
+
+while (($# > 0)); do
+ lowerI="$(echo $1 | awk '{print tolower($0)}')"
+ case $lowerI in
+ --baseuri)
+ base_uri=$2
+ shift 2
+ ;;
+ --installpath)
+ install_path=$2
+ shift 2
+ ;;
+ --version)
+ version=$2
+ shift 2
+ ;;
+ --clean)
+ clean=true
+ shift 1
+ ;;
+ --force)
+ force=true
+ shift 1
+ ;;
+ --downloadretries)
+ download_retries=$2
+ shift 2
+ ;;
+ --retrywaittimeseconds)
+ retry_wait_time_seconds=$2
+ shift 2
+ ;;
+ --help)
+ echo "Common settings:"
+ echo " --baseuri <value> Base file directory or Url wrom which to acquire tool archives"
+ echo " --installpath <value> Base directory to install native tool to"
+ echo " --clean Don't install the tool, just clean up the current install of the tool"
+ echo " --force Force install of tools even if they previously exist"
+ echo " --help Print help and exit"
+ echo ""
+ echo "Advanced settings:"
+ echo " --downloadretries Total number of retry attempts"
+ echo " --retrywaittimeseconds Wait time between retry attempts in seconds"
+ echo ""
+ exit 0
+ ;;
+ esac
+done
+
+tool_name="cmake-test"
+tool_os=$(GetCurrentOS)
+tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_arch="x86_64"
+tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
+tool_install_directory="$install_path/$tool_name/$version"
+tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
+shim_path="$install_path/$tool_name.sh"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
+
+# Clean up tool and installers
+if [[ $clean = true ]]; then
+ echo "Cleaning $tool_install_directory"
+ if [[ -d $tool_install_directory ]]; then
+ rm -rf $tool_install_directory
+ fi
+
+ echo "Cleaning $shim_path"
+ if [[ -f $shim_path ]]; then
+ rm -rf $shim_path
+ fi
+
+ tool_temp_path=$(GetTempPathFileName $uri)
+ echo "Cleaning $tool_temp_path"
+ if [[ -f $tool_temp_path ]]; then
+ rm -rf $tool_temp_path
+ fi
+
+ exit 0
+fi
+
+# Install tool
+if [[ -f $tool_file_path ]] && [[ $force = false ]]; then
+ echo "$tool_name ($version) already exists, skipping install"
+ exit 0
+fi
+
+DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
+
+if [[ $? != 0 ]]; then
+ echo "Installation failed" >&2
+ exit 1
+fi
+
+# Generate Shim
+# Always rewrite shims so that we are referencing the expected version
+NewScriptShim $shim_path $tool_file_path true
+
+if [[ $? != 0 ]]; then
+ echo "Shim generation failed" >&2
+ exit 1
+fi
+
+exit 0 \ No newline at end of file
diff --git a/eng/common/native/install-cmake.sh b/eng/common/native/install-cmake.sh
index 293af6017df..5f1a182fa9f 100644
--- a/eng/common/native/install-cmake.sh
+++ b/eng/common/native/install-cmake.sh
@@ -69,7 +69,7 @@ tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
tool_install_directory="$install_path/$tool_name/$version"
tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
shim_path="$install_path/$tool_name.sh"
-uri="${base_uri}/$tool_folder/cmake/$tool_name_moniker.tar.gz"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
# Clean up tool and installers
if [[ $clean = true ]]; then
diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj
index 05e5f098915..e5826b53237 100644
--- a/eng/common/performance/perfhelixpublish.proj
+++ b/eng/common/performance/perfhelixpublish.proj
@@ -5,8 +5,14 @@
<CliArguments>--dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP%</CliArguments>
<Python>py -3</Python>
<CoreRun>%HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe</CoreRun>
+ <BaselineCoreRun>%HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe</BaselineCoreRun>
<HelixPreCommands>$(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd</HelixPreCommands>
<ArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts</ArtifactsDirectory>
+ <BaselineArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
+ <ResultsComparer>%HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj</ResultsComparer>
+ <DotnetExe>%HELIX_CORRELATION_PAYLOAD%\performance\tools\dotnet\$(Architecture)\dotnet.exe</DotnetExe>
+ <Percent>%25%25</Percent>
+ <XMLResults>%HELIX_WORKITEM_ROOT%\testResults.xml</XMLResults>
</PropertyGroup>
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT' and '$(RunFromPerfRepo)' == 'false'">
@@ -24,14 +30,24 @@
<CliArguments>--dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP</CliArguments>
<Python>python3</Python>
<CoreRun>$(BaseDirectory)/Core_Root/corerun</CoreRun>
+ <BaselineCoreRun>$(BaseDirectory)/Baseline_Core_Root/corerun</BaselineCoreRun>
<HelixPreCommands>$(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh</HelixPreCommands>
<ArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts</ArtifactsDirectory>
+ <BaselineArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
+ <ResultsComparer>$(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj</ResultsComparer>
+ <DotnetExe>$(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet</DotnetExe>
+ <Percent>%25</Percent>
+ <XMLResults>$HELIX_WORKITEM_ROOT/testResults.xml</XMLResults>
</PropertyGroup>
<PropertyGroup Condition="'$(UseCoreRun)' == 'true'">
<CoreRunArgument>--corerun $(CoreRun)</CoreRunArgument>
</PropertyGroup>
+ <PropertyGroup Condition="'$(UseBaselineCoreRun)' == 'true'">
+ <BaselineCoreRunArgument>--corerun $(BaselineCoreRun)</BaselineCoreRunArgument>
+ </PropertyGroup>
+
<PropertyGroup Condition="'$(WorkItemCommand)' != ''">
<WorkItemCommand>$(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments)</WorkItemCommand>
</PropertyGroup>
@@ -57,20 +73,29 @@
<Partition Include="$(BuildConfig).Partition4" Index="4" />
</ItemGroup>
+ <PropertyGroup Condition="'$(Compare)' == 'true'">
+ <FailOnTestFailure>false</FailOnTestFailure>
+ </PropertyGroup>
+
<!--
Partition the Microbenchmarks project, but nothing else
-->
<ItemGroup Condition="$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
<HelixWorkItem Include="@(Partition)">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
- <Command>$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
+ <PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</PreCommands>
+ <Command>$(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
+ <PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand)</PostCommands>
<Timeout>4:00</Timeout>
</HelixWorkItem>
</ItemGroup>
+
<ItemGroup Condition="!$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
<HelixWorkItem Include="$(BuildConfig).WorkItem">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
- <Command>$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)"</Command>
+ <PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)"</PreCommands>
+ <Command>$(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)"</Command>
+ <PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults)</PostCommands>
<Timeout>4:00</Timeout>
</HelixWorkItem>
</ItemGroup>
diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1
index 7e5441f7974..268986246e5 100644
--- a/eng/common/performance/performance-setup.ps1
+++ b/eng/common/performance/performance-setup.ps1
@@ -1,8 +1,9 @@
Param(
[string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,
[string] $CoreRootDirectory,
+ [string] $BaselineCoreRootDirectory,
[string] $Architecture="x64",
- [string] $Framework="netcoreapp3.0",
+ [string] $Framework="netcoreapp5.0",
[string] $CompilationMode="Tiered",
[string] $Repository=$env:BUILD_REPOSITORY_NAME,
[string] $Branch=$env:BUILD_SOURCEBRANCH,
@@ -12,11 +13,13 @@ Param(
[string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj",
[string] $Kind="micro",
[switch] $Internal,
+ [switch] $Compare,
[string] $Configurations="CompilationMode=$CompilationMode"
)
$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance")
$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty)
+$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty)
$PayloadDirectory = (Join-Path $SourceDirectory "Payload")
$PerformanceDirectory = (Join-Path $PayloadDirectory "performance")
@@ -29,11 +32,17 @@ $HelixSourcePrefix = "pr"
$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
if ($Framework.StartsWith("netcoreapp")) {
- $Queue = "Windows.10.Amd64.ClientRS4.Open"
+ $Queue = "Windows.10.Amd64.ClientRS5.Open"
+}
+
+if ($Compare) {
+ $Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open"
+ $PerfLabArguments = ""
+ $ExtraBenchmarkDotNetArguments = ""
}
if ($Internal) {
- $Queue = "Windows.10.Amd64.ClientRS5.Perf"
+ $Queue = "Windows.10.Amd64.19H1.Tiger.Perf"
$PerfLabArguments = "--upload-to-perflab-container"
$ExtraBenchmarkDotNetArguments = ""
$Creator = ""
@@ -56,6 +65,10 @@ if ($UseCoreRun) {
$NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root")
Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot
}
+if ($UseBaselineCoreRun) {
+ $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root")
+ Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot
+}
$DocsDir = (Join-Path $PerformanceDirectory "docs")
robocopy $DocsDir $WorkItemDirectory
@@ -80,7 +93,9 @@ Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVaria
Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false
# Helix Arguments
Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false
diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh
index 126da5f76d4..550b3ebf18e 100644
--- a/eng/common/performance/performance-setup.sh
+++ b/eng/common/performance/performance-setup.sh
@@ -2,20 +2,23 @@
source_directory=$BUILD_SOURCESDIRECTORY
core_root_directory=
+baseline_core_root_directory=
architecture=x64
-framework=netcoreapp3.0
+framework=netcoreapp5.0
compilation_mode=tiered
repository=$BUILD_REPOSITORY_NAME
branch=$BUILD_SOURCEBRANCH
commit_sha=$BUILD_SOURCEVERSION
build_number=$BUILD_BUILDNUMBER
internal=false
+compare=false
kind="micro"
run_categories="coreclr corefx"
csproj="src\benchmarks\micro\MicroBenchmarks.csproj"
configurations=
run_from_perf_repo=false
use_core_run=true
+use_baseline_core_run=true
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
@@ -28,6 +31,10 @@ while (($# > 0)); do
core_root_directory=$2
shift 2
;;
+ --baselinecorerootdirectory)
+ baseline_core_root_directory=$2
+ shift 2
+ ;;
--architecture)
architecture=$2
shift 2
@@ -72,6 +79,10 @@ while (($# > 0)); do
internal=true
shift 1
;;
+ --compare)
+ compare=true
+ shift 1
+ ;;
--configurations)
configurations=$2
shift 2
@@ -114,6 +125,10 @@ if [ -z "$core_root_directory" ]; then
use_core_run=false
fi
+if [ -z "$baseline_core_root_directory" ]; then
+ use_baseline_core_run=false
+fi
+
payload_directory=$source_directory/Payload
performance_directory=$payload_directory/performance
workitem_directory=$source_directory/workitem
@@ -123,6 +138,19 @@ queue=Ubuntu.1804.Amd64.Open
creator=$BUILD_DEFINITIONNAME
helix_source_prefix="pr"
+if [[ "$compare" == true ]]; then
+ extra_benchmark_dotnet_arguments=
+ perflab_arguments=
+
+ # No open queues for arm64
+ if [[ "$architecture" = "arm64" ]]; then
+ echo "Compare not available for arm64"
+ exit 1
+ fi
+
+ queue=Ubuntu.1804.Amd64.Tiger.Perf.Open
+fi
+
if [[ "$internal" == true ]]; then
perflab_arguments="--upload-to-perflab-container"
helix_source_prefix="official"
@@ -132,7 +160,7 @@ if [[ "$internal" == true ]]; then
if [[ "$architecture" = "arm64" ]]; then
queue=Ubuntu.1804.Arm64.Perf
else
- queue=Ubuntu.1804.Amd64.Perf
+ queue=Ubuntu.1804.Amd64.Tiger.Perf
fi
fi
@@ -156,21 +184,33 @@ if [[ "$use_core_run" = true ]]; then
mv $core_root_directory $new_core_root
fi
+if [[ "$use_baseline_core_run" = true ]]; then
+ new_baseline_core_root=$payload_directory/Baseline_Core_Root
+ mv $baseline_core_root_directory $new_baseline_core_root
+fi
+
+ci=true
+
+_script_dir=$(pwd)/eng/common
+. "$_script_dir/pipeline-logging-functions.sh"
+
# Make sure all of our variables are available for future steps
-echo "##vso[task.setvariable variable=UseCoreRun]$use_core_run"
-echo "##vso[task.setvariable variable=Architecture]$architecture"
-echo "##vso[task.setvariable variable=PayloadDirectory]$payload_directory"
-echo "##vso[task.setvariable variable=PerformanceDirectory]$performance_directory"
-echo "##vso[task.setvariable variable=WorkItemDirectory]$workitem_directory"
-echo "##vso[task.setvariable variable=Queue]$queue"
-echo "##vso[task.setvariable variable=SetupArguments]$setup_arguments"
-echo "##vso[task.setvariable variable=Python]python3"
-echo "##vso[task.setvariable variable=PerfLabArguments]$perflab_arguments"
-echo "##vso[task.setvariable variable=ExtraBenchmarkDotNetArguments]$extra_benchmark_dotnet_arguments"
-echo "##vso[task.setvariable variable=BDNCategories]$run_categories"
-echo "##vso[task.setvariable variable=TargetCsproj]$csproj"
-echo "##vso[task.setvariable variable=RunFromPerfRepo]$run_from_perf_repo"
-echo "##vso[task.setvariable variable=Creator]$creator"
-echo "##vso[task.setvariable variable=HelixSourcePrefix]$helix_source_prefix"
-echo "##vso[task.setvariable variable=Kind]$kind"
-echo "##vso[task.setvariable variable=_BuildConfig]$architecture.$kind.$framework" \ No newline at end of file
+Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false
+Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false
+Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false
+Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false
+Write-PipelineSetVariable -name "WorkItemDirectory" -value "$workitem_directory" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Queue" -value "$queue" -is_multi_job_variable false
+Write-PipelineSetVariable -name "SetupArguments" -value "$setup_arguments" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Python" -value "$python3" -is_multi_job_variable false
+Write-PipelineSetVariable -name "PerfLabArguments" -value "$perflab_arguments" -is_multi_job_variable false
+Write-PipelineSetVariable -name "ExtraBenchmarkDotNetArguments" -value "$extra_benchmark_dotnet_arguments" -is_multi_job_variable false
+Write-PipelineSetVariable -name "BDNCategories" -value "$run_categories" -is_multi_job_variable false
+Write-PipelineSetVariable -name "TargetCsproj" -value "$csproj" -is_multi_job_variable false
+Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false
+Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false
+Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false
diff --git a/eng/common/post-build/darc-gather-drop.ps1 b/eng/common/post-build/darc-gather-drop.ps1
index 93a0bd83285..89854d3c1c2 100644
--- a/eng/common/post-build/darc-gather-drop.ps1
+++ b/eng/common/post-build/darc-gather-drop.ps1
@@ -19,7 +19,17 @@ try {
ExitWithExitCode $exitCode
}
+ # For now, only use a dry run.
+ # Ideally we would change darc to enable a quick request that
+ # would check whether the file exists that you can download it,
+ # and that it won't conflict with other files.
+ # https://github.com/dotnet/arcade/issues/3674
+ # Right now we can't remove continue-on-error because we ocassionally will have
+ # dependencies that have no associated builds (e.g. an old dependency).
+ # We need to add an option to baseline specific dependencies away, or add them manually
+ # to the BAR.
darc gather-drop --non-shipping `
+ --dry-run `
--continue-on-error `
--id $BarBuildId `
--output-dir $DropLocation `
diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1
index 41e01ae6e67..bbfdacca130 100644
--- a/eng/common/post-build/sourcelink-validation.ps1
+++ b/eng/common/post-build/sourcelink-validation.ps1
@@ -1,8 +1,8 @@
param(
[Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored
[Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
- [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade
- [Parameter(Mandatory=$true)][string] $GHCommit, # GitHub commit SHA used to build the packages
+ [Parameter(Mandatory=$false)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade
+ [Parameter(Mandatory=$false)][string] $GHCommit, # GitHub commit SHA used to build the packages
[Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use
)
@@ -13,6 +13,12 @@ param(
# all files present in the repo at a specific commit point.
$global:RepoFiles = @{}
+# Maximum number of jobs to run in parallel
+$MaxParallelJobs = 6
+
+# Wait time between check for system load
+$SecondsBetweenLoadChecks = 10
+
$ValidatePackage = {
param(
[string] $PackagePath # Full path to a Symbols.NuGet package
@@ -22,8 +28,8 @@ $ValidatePackage = {
# Ensure input file exist
if (!(Test-Path $PackagePath)) {
- Write-PipelineTaskError "Input file does not exist: $PackagePath"
- ExitWithExitCode 1
+ Write-Host "Input file does not exist: $PackagePath"
+ return 1
}
# Extensions for which we'll look for SourceLink information
@@ -38,7 +44,7 @@ $ValidatePackage = {
Add-Type -AssemblyName System.IO.Compression.FileSystem
- [System.IO.Directory]::CreateDirectory($ExtractPath);
+ [System.IO.Directory]::CreateDirectory($ExtractPath) | Out-Null
try {
$zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
@@ -138,16 +144,18 @@ $ValidatePackage = {
if ($FailedFiles -eq 0) {
Write-Host "Passed."
+ return 0
}
else {
- Write-PipelineTaskError "$PackagePath has broken SourceLink links."
+ Write-Host "$PackagePath has broken SourceLink links."
+ return 1
}
}
function ValidateSourceLinkLinks {
- if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
+ if ($GHRepoName -ne "" -and !($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) {
- Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>"
+ Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>. '$GHRepoName'"
ExitWithExitCode 1
}
else {
@@ -155,30 +163,33 @@ function ValidateSourceLinkLinks {
}
}
- if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
- Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string"
+ if ($GHCommit -ne "" -and !($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
+ Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'"
ExitWithExitCode 1
}
- $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
- $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
+ if ($GHRepoName -ne "" -and $GHCommit -ne "") {
+ $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
+ $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
- try {
- # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
- $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
+ try {
+ # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
+ $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
- foreach ($file in $Data) {
- $Extension = [System.IO.Path]::GetExtension($file.path)
+ foreach ($file in $Data) {
+ $Extension = [System.IO.Path]::GetExtension($file.path)
- if ($CodeExtensions.Contains($Extension)) {
- $RepoFiles[$file.path] = 1
+ if ($CodeExtensions.Contains($Extension)) {
+ $RepoFiles[$file.path] = 1
+ }
}
}
+ catch {
+ Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching."
+ }
}
- catch {
- Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL"
- Write-Host $_
- ExitWithExitCode 1
+ elseif ($GHRepoName -ne "" -or $GHCommit -ne "") {
+ Write-Host "For using the http caching mechanism both GHRepoName and GHCommit should be informed."
}
if (Test-Path $ExtractPath) {
@@ -186,14 +197,33 @@ function ValidateSourceLinkLinks {
}
# Process each NuGet package in parallel
- $Jobs = @()
Get-ChildItem "$InputPath\*.symbols.nupkg" |
ForEach-Object {
- $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName
+ Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
+ $NumJobs = @(Get-Job -State 'Running').Count
+
+ while ($NumJobs -ge $MaxParallelJobs) {
+ Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
+ sleep $SecondsBetweenLoadChecks
+ $NumJobs = @(Get-Job -State 'Running').Count
+ }
+
+ foreach ($Job in @(Get-Job -State 'Completed')) {
+ Receive-Job -Id $Job.Id
+ Remove-Job -Id $Job.Id
+ }
}
- foreach ($Job in $Jobs) {
- Wait-Job -Id $Job.Id | Receive-Job
+ $ValidationFailures = 0
+ foreach ($Job in @(Get-Job)) {
+ $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+ if ($jobResult -ne "0") {
+ $ValidationFailures++
+ }
+ }
+ if ($ValidationFailures -gt 0) {
+ Write-PipelineTaskError " $ValidationFailures package(s) failed validation."
+ ExitWithExitCode 1
}
}
diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1
index d5ec51b150f..096ac321d12 100644
--- a/eng/common/post-build/symbols-validation.ps1
+++ b/eng/common/post-build/symbols-validation.ps1
@@ -37,10 +37,10 @@ function FirstMatchingSymbolDescriptionOrDefault {
# DWARF file for a .dylib
$DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf")
- $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools"
- $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe"
+ $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
+ $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"
- & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
+ & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
if (Test-Path $PdbPath) {
return "PDB"
@@ -159,25 +159,25 @@ function CheckSymbolsAvailable {
}
}
-function Installdotnetsymbol {
- $dotnetsymbolPackageName = "dotnet-symbol"
+function InstallDotnetSymbol {
+ $dotnetSymbolPackageName = "dotnet-symbol"
$dotnetRoot = InitializeDotNetCli -install:$true
$dotnet = "$dotnetRoot\dotnet.exe"
$toolList = & "$dotnet" tool list --global
- if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) {
- Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed."
+ if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) {
+ Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed."
}
else {
- Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..."
+ Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..."
Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed."
- & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity "minimal" --global
+ & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global
}
}
try {
- Installdotnetsymbol
+ InstallDotnetSymbol
CheckSymbolsAvailable
}
diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1
index aab7589f2cc..01799d63ff3 100644
--- a/eng/common/sdl/execute-all-sdl-tools.ps1
+++ b/eng/common/sdl/execute-all-sdl-tools.ps1
@@ -45,6 +45,7 @@ if ($GuardianPackageName) {
$guardianCliLocation = $GuardianCliLocation
}
+$workingDirectory = (Split-Path $SourceDirectory -Parent)
$ValidPath = Test-Path $guardianCliLocation
if ($ValidPath -eq $False)
@@ -53,13 +54,13 @@ if ($ValidPath -eq $False)
exit 1
}
-& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
-$gdnFolder = Join-Path $ArtifactsDirectory ".gdn"
+& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
+$gdnFolder = Join-Path $workingDirectory ".gdn"
if ($TsaOnboard) {
if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
- Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel"
- & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel
+ Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
+ & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
@@ -71,10 +72,10 @@ if ($TsaOnboard) {
}
if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) {
- & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+ & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
}
if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
- & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+ & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
}
if ($UpdateBaseline) {
@@ -86,8 +87,8 @@ if ($TsaPublish) {
if (-not $TsaRepositoryName) {
$TsaRepositoryName = "$($Repository)-$($BranchName)"
}
- Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $SourceDirectory --logger-level $GuardianLoggerLevel"
- & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel
+ Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
+ & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1
index 26e01c0673c..c737eb0e71c 100644
--- a/eng/common/sdl/init-sdl.ps1
+++ b/eng/common/sdl/init-sdl.ps1
@@ -11,6 +11,9 @@ $ErrorActionPreference = "Stop"
Set-StrictMode -Version 2.0
$LASTEXITCODE = 0
+# Don't display the console progress UI - it's a huge perf hit
+$ProgressPreference = 'SilentlyContinue'
+
# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file
$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken"))
$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn")
diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config
index fb9b7128637..256ffbfb93a 100644
--- a/eng/common/sdl/packages.config
+++ b/eng/common/sdl/packages.config
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
<packages>
- <package id="Microsoft.Guardian.Cli" version="0.6.0"/>
+ <package id="Microsoft.Guardian.Cli" version="0.7.2"/>
</packages>
diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1
index d7b85644581..9bc25314ae2 100644
--- a/eng/common/sdl/run-sdl.ps1
+++ b/eng/common/sdl/run-sdl.ps1
@@ -25,43 +25,35 @@ if ($ValidPath -eq $False)
exit 1
}
+$configParam = @("--config")
+
foreach ($tool in $ToolsList) {
$gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig"
- $config = $False
Write-Host $tool
# We have to manually configure tools that run on source to look at the source directory only
if ($tool -eq "credscan") {
- Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
- & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
+ Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
+ & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
}
- $config = $True
}
if ($tool -eq "policheck") {
- Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
- & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
+ Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
+ & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
}
- $config = $True
}
- Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile $config"
- if ($config) {
- & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel --config $gdnConfigFile
- if ($LASTEXITCODE -ne 0) {
- Write-Host "Guardian run for $tool using $gdnConfigFile failed with exit code $LASTEXITCODE."
- exit $LASTEXITCODE
- }
- } else {
- & $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel
- if ($LASTEXITCODE -ne 0) {
- Write-Host "Guardian run for $tool failed with exit code $LASTEXITCODE."
- exit $LASTEXITCODE
- }
- }
+ $configParam+=$gdnConfigFile
}
+Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam"
+& $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam
+if ($LASTEXITCODE -ne 0) {
+ Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
+ exit $LASTEXITCODE
+}
diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml
index 5837f3d56d3..a7f99641952 100644
--- a/eng/common/templates/job/execute-sdl.yml
+++ b/eng/common/templates/job/execute-sdl.yml
@@ -1,7 +1,10 @@
parameters:
overrideParameters: '' # Optional: to override values for parameters.
additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
- continueOnError: false # optional: determines whether to continue the build if the step errors;
+ # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
+ # 'continueOnError', the parameter value is not correctly picked up.
+ # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
+ sdlContinueOnError: false # optional: determines whether to continue the build if the step errors;
dependsOn: '' # Optional: dependencies of the job
jobs:
@@ -10,6 +13,8 @@ jobs:
displayName: Run SDL tool
variables:
- group: DotNet-VSTS-Bot
+ pool:
+ name: Hosted VS2017
steps:
- checkout: self
clean: true
@@ -24,12 +29,12 @@ jobs:
-InputPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts
-ExtractPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts
displayName: Extract Blob Artifacts
- continueOnError: ${{ parameters.continueOnError }}
+ continueOnError: ${{ parameters.sdlContinueOnError }}
- powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts
-ExtractPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts
displayName: Extract Package Artifacts
- continueOnError: ${{ parameters.continueOnError }}
+ continueOnError: ${{ parameters.sdlContinueOnError }}
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
- task: NuGetCommand@2
@@ -43,12 +48,12 @@ jobs:
- ${{ if ne(parameters.overrideParameters, '') }}:
- powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }}
displayName: Execute SDL
- continueOnError: ${{ parameters.continueOnError }}
+ continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if eq(parameters.overrideParameters, '') }}:
- powershell: eng/common/sdl/execute-all-sdl-tools.ps1
- -GuardianPackageName Microsoft.Guardian.Cli.0.6.0
+ -GuardianPackageName Microsoft.Guardian.Cli.0.7.2
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages
-AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }}
displayName: Execute SDL
- continueOnError: ${{ parameters.continueOnError }}
+ continueOnError: ${{ parameters.sdlContinueOnError }}
diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
index 1814e0ab612..ffda80a197b 100644
--- a/eng/common/templates/job/job.yml
+++ b/eng/common/templates/job/job.yml
@@ -24,7 +24,7 @@ parameters:
workspace: ''
-# Job base template specific parameters
+ # Job base template specific parameters
# Optional: Enable installing Microbuild plugin
# if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
# _TeamName - the name of your team
@@ -37,6 +37,9 @@ parameters:
# Optional: Enable publishing to the build asset registry
enablePublishBuildAssets: false
+ # Optional: Prevent gather/push manifest from executing when using publishing pipelines
+ enablePublishUsingPipelines: false
+
# Optional: Include PublishTestResults task
enablePublishTestResults: false
@@ -148,6 +151,9 @@ jobs:
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: NuGetAuthenticate@0
+
- ${{ each step in parameters.steps }}:
- ${{ step }}
@@ -187,7 +193,7 @@ jobs:
continueOnError: true
condition: always()
- - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(variables['_PublishUsingPipelines'], 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: CopyFiles@2
displayName: Gather Asset Manifests
inputs:
@@ -195,6 +201,7 @@ jobs:
TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
- task: PublishBuildArtifacts@1
displayName: Push Asset Manifests
inputs:
diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml
index 9e77ef1b54b..b722975f9c2 100644
--- a/eng/common/templates/job/publish-build-assets.yml
+++ b/eng/common/templates/job/publish-build-assets.yml
@@ -47,6 +47,10 @@ jobs:
downloadPath: '$(Build.StagingDirectory)/Download'
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: NuGetAuthenticate@0
+
- task: PowerShell@2
displayName: Publish Build Assets
inputs:
@@ -59,6 +63,7 @@ jobs:
/p:Configuration=$(_BuildConfig)
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
+
- task: powershell@2
displayName: Create ReleaseConfigs Artifact
inputs:
@@ -67,12 +72,14 @@ jobs:
Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId)
Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)"
Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild)
+
- task: PublishBuildArtifacts@1
displayName: Publish ReleaseConfigs Artifact
inputs:
PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt'
PublishLocation: Container
ArtifactName: ReleaseConfigs
+
- ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
- task: PublishBuildArtifacts@1
displayName: Publish Logs to VSTS
diff --git a/eng/common/templates/post-build/channels/public-dev-release.yml b/eng/common/templates/post-build/channels/netcore-dev-30.yml
index b0f085b1420..16e4eff7d0e 100644
--- a/eng/common/templates/post-build/channels/public-dev-release.yml
+++ b/eng/common/templates/post-build/channels/netcore-dev-30.yml
@@ -1,42 +1,53 @@
parameters:
enableSymbolValidation: true
+ symbolPublishingAdditionalParameters: ''
+ artifactsPublishingAdditionalParameters: ''
+ publishInstallersAndChecksums: false
stages:
-- stage: Publish
+- stage: NetCore_Dev30_Publish
dependsOn: validate
variables:
- template: ../common-variables.yml
- displayName: Developer Channel
+ displayName: .NET Core 3.0 Dev Publishing
jobs:
- template: ../setup-maestro-vars.yml
- job:
displayName: Symbol Publishing
dependsOn: setupMaestroVars
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id)
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_30_Channel_Id))
variables:
- group: DotNet-Symbol-Server-Pats
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
- displayName: Download Artifacts
+ displayName: Download Blob Artifacts
+ inputs:
+ artifactName: 'BlobArtifacts'
+ continueOnError: true
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download PDB Artifacts
inputs:
- downloadType: specific files
- matchingPattern: "*Artifacts*"
+ artifactName: 'PDBArtifacts'
+ continueOnError: true
- task: PowerShell@2
displayName: Publish
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
- /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
- /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+ /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+ /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release
+ ${{ parameters.symbolPublishingAdditionalParameters }}
- - job:
+ - job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
variables:
@@ -46,7 +57,7 @@ stages:
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id)
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_30_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
@@ -68,22 +79,33 @@ stages:
buildType: current
artifactName: AssetManifests
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
- task: PowerShell@2
- displayName: Add Assets Location
+ displayName: Publish Assets
env:
- AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw)
+ AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw)
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
- /p:ChannelId=$(PublicDevRelease_30_Channel_Id)
/p:ArtifactsCategory=$(_DotNetArtifactsCategory)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
- /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe
- /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)'
- /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)'
+ /p:StaticInternalFeed=$(StaticInternalFeed)
+ /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
+ /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey)
+ /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
+ /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey)
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
@@ -92,30 +114,18 @@ stages:
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
-
- - task: NuGetCommand@2
- displayName: Publish Packages to AzDO Feed
- condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com')
- inputs:
- command: push
- vstsFeed: $(AzDoFeedName)
- packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg
- publishVstsFeed: $(AzDoFeedName)
-
- - task: PowerShell@2
- displayName: Publish Blobs to AzDO Feed
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1
- arguments: -FeedName $(AzDoFeedName)
- -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/
- -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw)
- enabled: false
+ /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+ /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+ /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+ /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+ /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
-- stage: PublishValidation
- displayName: Publish Validation
+- stage: NetCore_Dev30_Publish_Validation
+ displayName: .NET Core 3.0 Dev Publish Validation
variables:
- - template: ../common-variables.yml
+ - template: ../common-variables.yml
jobs:
- template: ../setup-maestro-vars.yml
@@ -123,7 +133,7 @@ stages:
- job:
displayName: Symbol Availability
dependsOn: setupMaestroVars
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicDevRelease_30_Channel_Id)
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_30_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
@@ -139,10 +149,6 @@ stages:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1
arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion)
- - template: ../darc-gather-drop.yml
+ - template: ../promote-build.yml
parameters:
ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }}
-
- - template: ../promote-build.yml
- parameters:
- ChannelId: ${{ variables.PublicDevRelease_30_Channel_Id }}
diff --git a/eng/common/templates/post-build/channels/netcore-dev-31.yml b/eng/common/templates/post-build/channels/netcore-dev-31.yml
new file mode 100644
index 00000000000..52322b128e2
--- /dev/null
+++ b/eng/common/templates/post-build/channels/netcore-dev-31.yml
@@ -0,0 +1,154 @@
+parameters:
+ enableSymbolValidation: true
+ symbolPublishingAdditionalParameters: ''
+ artifactsPublishingAdditionalParameters: ''
+ publishInstallersAndChecksums: false
+
+stages:
+- stage: NetCore_Dev31_Publish
+ dependsOn: validate
+ variables:
+ - template: ../common-variables.yml
+ displayName: .NET Core 3.1 Dev Publishing
+ jobs:
+ - template: ../setup-maestro-vars.yml
+
+ - job:
+ displayName: Symbol Publishing
+ dependsOn: setupMaestroVars
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_31_Channel_Id))
+ variables:
+ - group: DotNet-Symbol-Server-Pats
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ artifactName: 'BlobArtifacts'
+ continueOnError: true
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download PDB Artifacts
+ inputs:
+ artifactName: 'PDBArtifacts'
+ continueOnError: true
+
+ - task: PowerShell@2
+ displayName: Publish
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+ /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+ /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+ /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+ /p:Configuration=Release
+ ${{ parameters.symbolPublishingAdditionalParameters }}
+
+ - job:
+ displayName: Publish Assets
+ dependsOn: setupMaestroVars
+ variables:
+ - group: DotNet-Blob-Feed
+ - group: AzureDevOps-Artifact-Feeds-Pats
+ - name: BARBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+ - name: IsStableBuild
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_31_Channel_Id))
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: current
+ artifactName: PackageArtifacts
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ buildType: current
+ artifactName: BlobArtifacts
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Asset Manifests
+ inputs:
+ buildType: current
+ artifactName: AssetManifests
+
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Publish Assets
+ env:
+ AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw)
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
+ /p:ArtifactsCategory=$(_DotNetArtifactsCategory)
+ /p:IsStableBuild=$(IsStableBuild)
+ /p:IsInternalBuild=$(IsInternalBuild)
+ /p:RepositoryName=$(Build.Repository.Name)
+ /p:CommitSha=$(Build.SourceVersion)
+ /p:StaticInternalFeed=$(StaticInternalFeed)
+ /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
+ /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey)
+ /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
+ /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey)
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
+ /p:BARBuildId=$(BARBuildId)
+ /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
+ /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
+ /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+ /p:Configuration=Release
+ /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+ /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+ /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+ /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+ /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
+
+
+- stage: NetCore_Dev31_Publish_Validation
+ displayName: .NET Core 3.1 Dev Publish Validation
+ variables:
+ - template: ../common-variables.yml
+ jobs:
+ - template: ../setup-maestro-vars.yml
+
+ - ${{ if eq(parameters.enableSymbolValidation, 'true') }}:
+ - job:
+ displayName: Symbol Availability
+ dependsOn: setupMaestroVars
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_31_Channel_Id))
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: current
+ artifactName: PackageArtifacts
+
+ - task: PowerShell@2
+ displayName: Check Symbol Availability
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1
+ arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion)
+
+ - template: ../promote-build.yml
+ parameters:
+ ChannelId: ${{ variables.PublicDevRelease_31_Channel_Id }}
diff --git a/eng/common/templates/post-build/channels/netcore-dev-5.yml b/eng/common/templates/post-build/channels/netcore-dev-5.yml
new file mode 100644
index 00000000000..839121dec6a
--- /dev/null
+++ b/eng/common/templates/post-build/channels/netcore-dev-5.yml
@@ -0,0 +1,160 @@
+parameters:
+ enableSymbolValidation: true
+ symbolPublishingAdditionalParameters: ''
+ artifactsPublishingAdditionalParameters: ''
+ publishInstallersAndChecksums: false
+ publishToAzureDevOpsFeeds: true
+
+stages:
+- stage: NetCore_Dev5_Publish
+ dependsOn: validate
+ variables:
+ - template: ../common-variables.yml
+ displayName: .NET Core 5 Dev Publishing
+ jobs:
+ - template: ../setup-maestro-vars.yml
+
+ - job:
+ displayName: Symbol Publishing
+ dependsOn: setupMaestroVars
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_5_Dev_Channel_Id))
+ variables:
+ - group: DotNet-Symbol-Server-Pats
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ artifactName: 'BlobArtifacts'
+ continueOnError: true
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download PDB Artifacts
+ inputs:
+ artifactName: 'PDBArtifacts'
+ continueOnError: true
+
+ - task: PowerShell@2
+ displayName: Publish
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+ /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+ /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+ /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+ /p:Configuration=Release
+ ${{ parameters.symbolPublishingAdditionalParameters }}
+
+ - job: publish_assets
+ displayName: Publish Assets
+ dependsOn: setupMaestroVars
+ variables:
+ - group: DotNet-Blob-Feed
+ - group: AzureDevOps-Artifact-Feeds-Pats
+ - name: BARBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+ - name: IsStableBuild
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_5_Dev_Channel_Id))
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: current
+ artifactName: PackageArtifacts
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ buildType: current
+ artifactName: BlobArtifacts
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Asset Manifests
+ inputs:
+ buildType: current
+ artifactName: AssetManifests
+
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Publish Assets
+ env:
+ AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw)
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
+ /p:ArtifactsCategory=$(_DotNetArtifactsCategory)
+ /p:IsStableBuild=$(IsStableBuild)
+ /p:IsInternalBuild=$(IsInternalBuild)
+ /p:RepositoryName=$(Build.Repository.Name)
+ /p:CommitSha=$(Build.SourceVersion)
+ /p:StaticInternalFeed=$(StaticInternalFeed)
+ /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
+ /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey)
+ /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
+ /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey)
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
+ /p:BARBuildId=$(BARBuildId)
+ /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
+ /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
+ /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+ /p:Configuration=Release
+ /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+ /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+ /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+ /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+ /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+ /p:PublishToAzureDevOpsNuGetFeeds=${{ parameters.publishToAzureDevOpsFeeds }}
+ /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
+ /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
+ /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
+
+
+- stage: NetCore_Dev5_Publish_Validation
+ displayName: .NET Core 5 Dev Publish Validation
+ variables:
+ - template: ../common-variables.yml
+ jobs:
+ - template: ../setup-maestro-vars.yml
+
+ - ${{ if eq(parameters.enableSymbolValidation, 'true') }}:
+ - job:
+ displayName: Symbol Availability
+ dependsOn: setupMaestroVars
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_5_Dev_Channel_Id))
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: current
+ artifactName: PackageArtifacts
+
+ - task: PowerShell@2
+ displayName: Check Symbol Availability
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1
+ arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion)
+
+ - template: ../promote-build.yml
+ parameters:
+ ChannelId: ${{ variables.NetCore_5_Dev_Channel_Id }}
diff --git a/eng/common/templates/post-build/channels/internal-servicing.yml b/eng/common/templates/post-build/channels/netcore-internal-30.yml
index 12fd2b4653d..ad36501274a 100644
--- a/eng/common/templates/post-build/channels/internal-servicing.yml
+++ b/eng/common/templates/post-build/channels/netcore-internal-30.yml
@@ -1,40 +1,50 @@
parameters:
enableSymbolValidation: true
+ symbolPublishingAdditionalParameters: ''
+ artifactsPublishingAdditionalParameters: ''
stages:
-- stage: IS_Publish
+- stage: NetCore_30_Internal_Servicing_Publishing
dependsOn: validate
variables:
- template: ../common-variables.yml
- displayName: Internal Servicing
+ displayName: .NET Core 3.0 Internal Servicing Publishing
jobs:
- template: ../setup-maestro-vars.yml
- job:
displayName: Symbol Publishing
dependsOn: setupMaestroVars
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id)
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.InternalServicing_30_Channel_Id))
variables:
- group: DotNet-Symbol-Server-Pats
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
- displayName: Download Artifacts
+ displayName: Download Blob Artifacts
+ inputs:
+ artifactName: 'BlobArtifacts'
+ continueOnError: true
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download PDB Artifacts
inputs:
- downloadType: specific files
- matchingPattern: "*Artifacts*"
+ artifactName: 'PDBArtifacts'
+ continueOnError: true
- task: PowerShell@2
displayName: Publish
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
- /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
- /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+ /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+ /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release
+ ${{ parameters.symbolPublishingAdditionalParameters }}
- job: publish_assets
displayName: Publish Assets
@@ -46,7 +56,7 @@ stages:
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id)
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.InternalServicing_30_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
@@ -68,57 +78,50 @@ stages:
buildType: current
artifactName: AssetManifests
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
- task: PowerShell@2
- displayName: Add Assets Location
+ displayName: Publish Assets
env:
- AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw)
+ AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw)
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
- /p:ChannelId=$(InternalServicing_30_Channel_Id)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
- /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName)
- /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1)
- /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url)
- /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url)
- /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe
+ /p:StaticInternalFeed=$(StaticInternalFeed)
+ /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
+ /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey)
+ /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
+ /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey)
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
- /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts'
- /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
-
- - task: NuGetCommand@2
- displayName: Publish Packages to AzDO Feed
- condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com')
- inputs:
- command: push
- vstsFeed: $(AzDoFeedName)
- packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg
- publishVstsFeed: $(AzDoFeedName)
-
- - task: PowerShell@2
- displayName: Publish Blobs to AzDO Feed
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1
- arguments: -FeedName $(AzDoFeedName)
- -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/
- -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw)
- enabled: false
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../trigger-subscription.yml
parameters:
ChannelId: ${{ variables.InternalServicing_30_Channel_Id }}
-
-- stage: IS_PublishValidation
- displayName: Publish Validation
+
+- stage: NetCore_30_Internal_Servicing_Publish_Validation
+ displayName: .NET Core 3.0 Internal Servicing Publish Validation
variables:
- - template: ../common-variables.yml
+ - template: ../common-variables.yml
jobs:
- template: ../setup-maestro-vars.yml
@@ -126,7 +129,7 @@ stages:
- job:
displayName: Symbol Availability
dependsOn: setupMaestroVars
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.InternalServicing_30_Channel_Id)
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.InternalServicing_30_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
diff --git a/eng/common/templates/post-build/channels/public-release.yml b/eng/common/templates/post-build/channels/netcore-release-30.yml
index 4c63fb43f0c..8c4bb6005bc 100644
--- a/eng/common/templates/post-build/channels/public-release.yml
+++ b/eng/common/templates/post-build/channels/netcore-release-30.yml
@@ -1,40 +1,50 @@
parameters:
enableSymbolValidation: true
+ symbolPublishingAdditionalParameters: ''
+ artifactsPublishingAdditionalParameters: ''
stages:
-- stage: PubRel_Publish
+- stage: NetCore_Release30_Publish
dependsOn: validate
variables:
- template: ../common-variables.yml
- displayName: Public Release
+ displayName: .NET Core 3.0 Release Publishing
jobs:
- template: ../setup-maestro-vars.yml
- job:
displayName: Symbol Publishing
dependsOn: setupMaestroVars
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id)
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id))
variables:
- group: DotNet-Symbol-Server-Pats
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
- displayName: Download Artifacts
+ displayName: Download Blob Artifacts
+ inputs:
+ artifactName: 'BlobArtifacts'
+ continueOnError: true
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download PDB Artifacts
inputs:
- downloadType: specific files
- matchingPattern: "*Artifacts*"
+ artifactName: 'PDBArtifacts'
+ continueOnError: true
- task: PowerShell@2
displayName: Publish
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
- /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
- /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+ /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+ /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release
+ ${{ parameters.symbolPublishingAdditionalParameters }}
- job: publish_assets
displayName: Publish Assets
@@ -46,7 +56,7 @@ stages:
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id)
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
@@ -68,57 +78,50 @@ stages:
buildType: current
artifactName: AssetManifests
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
- task: PowerShell@2
- displayName: Publish
+ displayName: Publish Assets
env:
- AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw)
+ AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw)
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
- /p:ChannelId=$(PublicRelease_30_Channel_Id)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
- /p:AzureStorageAccountName=$(ProxyBackedFeedsAccountName)
- /p:AzureStorageAccountKey=$(dotnetfeed-storage-access-key-1)
- /p:AzureDevOpsFeedsBaseUrl=$(dotnetfeed-internal-private-feed-url)
- /p:StaticInternalFeed=$(dotnetfeed-internal-nonstable-feed-url)
- /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe
+ /p:StaticInternalFeed=$(StaticInternalFeed)
+ /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
+ /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey)
+ /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
+ /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey)
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
- /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts'
- /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
-
- - task: NuGetCommand@2
- displayName: Publish Packages to AzDO Feed
- condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com')
- inputs:
- command: push
- vstsFeed: $(AzDoFeedName)
- packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg
- publishVstsFeed: $(AzDoFeedName)
-
- - task: PowerShell@2
- displayName: Publish Blobs to AzDO Feed
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1
- arguments: -FeedName $(AzDoFeedName)
- -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/
- -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw)
- enabled: false
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../trigger-subscription.yml
parameters:
ChannelId: ${{ variables.PublicRelease_30_Channel_Id }}
-- stage: PubRel_PublishValidation
- displayName: Publish Validation
+- stage: NetCore_Release30_Publish_Validation
+ displayName: .NET Core 3.0 Release Publish Validation
variables:
- - template: ../common-variables.yml
+ - template: ../common-variables.yml
jobs:
- template: ../setup-maestro-vars.yml
@@ -126,7 +129,8 @@ stages:
- job:
displayName: Symbol Availability
dependsOn: setupMaestroVars
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicRelease_30_Channel_Id)
+ condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id)),
+ contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id)))
pool:
vmImage: 'windows-2019'
steps:
diff --git a/eng/common/templates/post-build/channels/netcore-release-31.yml b/eng/common/templates/post-build/channels/netcore-release-31.yml
new file mode 100644
index 00000000000..cfe6386daf4
--- /dev/null
+++ b/eng/common/templates/post-build/channels/netcore-release-31.yml
@@ -0,0 +1,156 @@
+parameters:
+ enableSymbolValidation: true
+ symbolPublishingAdditionalParameters: ''
+ artifactsPublishingAdditionalParameters: ''
+
+stages:
+- stage: NetCore_Release31_Publish
+ dependsOn: validate
+ variables:
+ - template: ../common-variables.yml
+ displayName: .NET Core 3.1 Release Publishing
+ jobs:
+ - template: ../setup-maestro-vars.yml
+
+ - job:
+ displayName: Symbol Publishing
+ dependsOn: setupMaestroVars
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id))
+ variables:
+ - group: DotNet-Symbol-Server-Pats
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ artifactName: 'BlobArtifacts'
+ continueOnError: true
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download PDB Artifacts
+ inputs:
+ artifactName: 'PDBArtifacts'
+ continueOnError: true
+
+ - task: PowerShell@2
+ displayName: Publish
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+ /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+ /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+ /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+ /p:Configuration=Release
+ ${{ parameters.symbolPublishingAdditionalParameters }}
+
+ - job: publish_assets
+ displayName: Publish Assets
+ dependsOn: setupMaestroVars
+ variables:
+ - group: DotNet-Blob-Feed
+ - group: AzureDevOps-Artifact-Feeds-Pats
+ - name: BARBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+ - name: IsStableBuild
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id))
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: current
+ artifactName: PackageArtifacts
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ buildType: current
+ artifactName: BlobArtifacts
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Asset Manifests
+ inputs:
+ buildType: current
+ artifactName: AssetManifests
+
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Publish Assets
+ env:
+ AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw)
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
+ /p:ArtifactsCategory=$(_DotNetArtifactsCategory)
+ /p:IsStableBuild=$(IsStableBuild)
+ /p:IsInternalBuild=$(IsInternalBuild)
+ /p:RepositoryName=$(Build.Repository.Name)
+ /p:CommitSha=$(Build.SourceVersion)
+ /p:StaticInternalFeed=$(StaticInternalFeed)
+ /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
+ /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey)
+ /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
+ /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey)
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
+ /p:BARBuildId=$(BARBuildId)
+ /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
+ /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
+ /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+ /p:Configuration=Release
+ /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+ /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+ /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+ /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+ /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
+
+ - template: ../trigger-subscription.yml
+ parameters:
+ ChannelId: ${{ variables.PublicRelease_31_Channel_Id }}
+
+- stage: NetCore_Release31_Publish_Validation
+ displayName: .NET Core 3.1 Release Publish Validation
+ variables:
+ - template: ../common-variables.yml
+ jobs:
+ - template: ../setup-maestro-vars.yml
+
+ - ${{ if eq(parameters.enableSymbolValidation, 'true') }}:
+ - job:
+ displayName: Symbol Availability
+ dependsOn: setupMaestroVars
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id))
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: current
+ artifactName: PackageArtifacts
+
+ - task: PowerShell@2
+ displayName: Check Symbol Availability
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1
+ arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion)
+
+ - template: ../promote-build.yml
+ parameters:
+ ChannelId: ${{ variables.PublicRelease_31_Channel_Id }}
diff --git a/eng/common/templates/post-build/channels/netcore-tools-latest.yml b/eng/common/templates/post-build/channels/netcore-tools-latest.yml
new file mode 100644
index 00000000000..83dbdeea125
--- /dev/null
+++ b/eng/common/templates/post-build/channels/netcore-tools-latest.yml
@@ -0,0 +1,160 @@
+parameters:
+ enableSymbolValidation: true
+ symbolPublishingAdditionalParameters: ''
+ artifactsPublishingAdditionalParameters: ''
+ publishInstallersAndChecksums: false
+ publishToAzureDevOpsFeeds: true
+
+stages:
+- stage: NetCore_Tools_Latest_Publish
+ dependsOn: validate
+ variables:
+ - template: ../common-variables.yml
+ displayName: .NET Tools - Latest Publishing
+ jobs:
+ - template: ../setup-maestro-vars.yml
+
+ - job:
+ displayName: Symbol Publishing
+ dependsOn: setupMaestroVars
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_Tools_Latest_Channel_Id))
+ variables:
+ - group: DotNet-Symbol-Server-Pats
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ artifactName: 'BlobArtifacts'
+ continueOnError: true
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download PDB Artifacts
+ inputs:
+ artifactName: 'PDBArtifacts'
+ continueOnError: true
+
+ - task: PowerShell@2
+ displayName: Publish
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+ /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+ /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+ /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+ /p:Configuration=Release
+ ${{ parameters.symbolPublishingAdditionalParameters }}
+
+ - job: publish_assets
+ displayName: Publish Assets
+ dependsOn: setupMaestroVars
+ variables:
+ - group: DotNet-Blob-Feed
+ - group: AzureDevOps-Artifact-Feeds-Pats
+ - name: BARBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+ - name: IsStableBuild
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_Tools_Latest_Channel_Id))
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: current
+ artifactName: PackageArtifacts
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ buildType: current
+ artifactName: BlobArtifacts
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Asset Manifests
+ inputs:
+ buildType: current
+ artifactName: AssetManifests
+
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Publish Assets
+ env:
+ AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw)
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
+ /p:ArtifactsCategory=$(_DotNetArtifactsCategory)
+ /p:IsStableBuild=$(IsStableBuild)
+ /p:IsInternalBuild=$(IsInternalBuild)
+ /p:RepositoryName=$(Build.Repository.Name)
+ /p:CommitSha=$(Build.SourceVersion)
+ /p:StaticInternalFeed=$(StaticInternalFeed)
+ /p:InternalChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
+ /p:InternalChecksumsTargetStaticFeedKey=$(InternalChecksumsBlobFeedKey)
+ /p:InternalInstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
+ /p:InternalInstallersTargetStaticFeedKey=$(InternalInstallersBlobFeedKey)
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
+ /p:BARBuildId=$(BARBuildId)
+ /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
+ /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
+ /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+ /p:Configuration=Release
+ /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+ /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+ /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+ /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+ /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+ /p:PublishToAzureDevOpsNuGetFeeds=${{ parameters.publishToAzureDevOpsFeeds }}
+ /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
+
+
+- stage: NetCore_Tools_Latest_PublishValidation
+ displayName: .NET Tools - Latest Publish Validation
+ variables:
+ - template: ../common-variables.yml
+ jobs:
+ - template: ../setup-maestro-vars.yml
+
+ - ${{ if eq(parameters.enableSymbolValidation, 'true') }}:
+ - job:
+ displayName: Symbol Availability
+ dependsOn: setupMaestroVars
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_Tools_Latest_Channel_Id))
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: current
+ artifactName: PackageArtifacts
+
+ - task: PowerShell@2
+ displayName: Check Symbol Availability
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/symbols-validation.ps1
+ arguments: -InputPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ -ExtractPath $(Agent.BuildDirectory)/Temp/ -DotnetSymbolVersion $(SymbolToolVersion)
+
+ - template: ../promote-build.yml
+ parameters:
+ ChannelId: ${{ variables.NetCore_Tools_Latest_Channel_Id }}
diff --git a/eng/common/templates/post-build/channels/public-validation-release.yml b/eng/common/templates/post-build/channels/public-validation-release.yml
index 1089ac5fa6b..a5fcdc581ad 100644
--- a/eng/common/templates/post-build/channels/public-validation-release.yml
+++ b/eng/common/templates/post-build/channels/public-validation-release.yml
@@ -1,13 +1,18 @@
+parameters:
+ artifactsPublishingAdditionalParameters: ''
+ publishInstallersAndChecksums: false
+ publishToAzureDevOpsFeeds: true
+
stages:
- stage: PVR_Publish
dependsOn: validate
variables:
- template: ../common-variables.yml
- displayName: Validation Channel
+ displayName: .NET Tools - Validation Publishing
jobs:
- template: ../setup-maestro-vars.yml
- - job:
+ - job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
variables:
@@ -17,7 +22,7 @@ stages:
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], variables.PublicValidationRelease_30_Channel_Id)
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicValidationRelease_30_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
@@ -39,61 +44,56 @@ stages:
buildType: current
artifactName: AssetManifests
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
- task: PowerShell@2
- displayName: Add Assets Location
+ displayName: Publish Assets
env:
- AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-unviersal-packages-rw)
+ AZURE_DEVOPS_EXT_PAT: $(dn-bot-dnceng-universal-packages-rw)
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
- /p:ChannelId=$(PublicValidationRelease_30_Channel_Id)
/p:ArtifactsCategory=$(_DotNetValidationArtifactsCategory)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
- /p:NugetPath=$(Agent.BuildDirectory)\Nuget\NuGet.exe
- /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)'
- /p:TargetFeedPAT='$(dn-bot-dnceng-unviersal-packages-rw)'
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:TargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
- /p:BlobBasePath='$(Build.ArtifactStagingDirectory)\BlobArtifacts'
- /p:PackageBasePath='$(Build.ArtifactStagingDirectory)\PackageArtifacts'
- /p:Configuration=Release
-
- - task: NuGetCommand@2
- displayName: Publish Packages to AzDO Feed
- condition: contains(variables['TargetAzDOFeed'], 'pkgs.visualstudio.com')
- inputs:
- command: push
- vstsFeed: $(AzDoFeedName)
- packagesToPush: $(Build.ArtifactStagingDirectory)\PackageArtifacts\*.nupkg
- publishVstsFeed: $(AzDoFeedName)
-
- - task: PowerShell@2
- displayName: Publish Blobs to AzDO Feed
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-blobs-to-azdo.ps1
- arguments: -FeedName $(AzDoFeedName)
- -SourceFolderCollection $(Build.ArtifactStagingDirectory)/BlobArtifacts/
- -PersonalAccessToken $(dn-bot-dnceng-unviersal-packages-rw)
- enabled: false
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+ /p:Configuration=Release
+ /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+ /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+ /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+ /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+ /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+ /p:PublishToAzureDevOpsNuGetFeeds=${{ parameters.publishToAzureDevOpsFeeds }}
+ /p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
- stage: PVR_PublishValidation
- displayName: Publish Validation
+ displayName: .NET Tools - Validation Publish Validation
variables:
- template: ../common-variables.yml
jobs:
- template: ../setup-maestro-vars.yml
- - template: ../darc-gather-drop.yml
+ - template: ../promote-build.yml
parameters:
ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }}
-
- - template: ../promote-build.yml
- parameters:
- ChannelId: ${{ variables.PublicValidationRelease_30_Channel_Id }}
diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml
index bd0bc5e4daa..c1347644961 100644
--- a/eng/common/templates/post-build/common-variables.yml
+++ b/eng/common/templates/post-build/common-variables.yml
@@ -1,14 +1,28 @@
variables:
- group: Publish-Build-Assets
+ - group: DotNet-DotNetCli-Storage
+ - group: DotNet-MSRC-Storage
# .NET Core 3 Dev
- name: PublicDevRelease_30_Channel_Id
value: 3
+ # .NET Core 3.1 Dev
+ - name: PublicDevRelease_31_Channel_Id
+ value: 128
+
+ # .NET Core 5 Dev
+ - name: NetCore_5_Dev_Channel_Id
+ value: 131
+
# .NET Tools - Validation
- name: PublicValidationRelease_30_Channel_Id
value: 9
+ # .NET Tools - Latest
+ - name: NetCore_Tools_Latest_Channel_Id
+ value: 2
+
# .NET Core 3.0 Internal Servicing
- name: InternalServicing_30_Channel_Id
value: 184
@@ -17,14 +31,14 @@ variables:
- name: PublicRelease_30_Channel_Id
value: 19
+ # .NET Core 3.1 Release
+ - name: PublicRelease_31_Channel_Id
+ value: 129
+
# Whether the build is internal or not
- name: IsInternalBuild
value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }}
- # Storage account name for proxy-backed feeds
- - name: ProxyBackedFeedsAccountName
- value: dotnetfeed
-
# Default Maestro++ API Endpoint and API Version
- name: MaestroApiEndPoint
value: "https://maestro-prod.westus2.cloudapp.azure.com"
@@ -37,3 +51,28 @@ variables:
value: 3.0.0
- name: SymbolToolVersion
value: 1.0.1
+
+ # Feed Configurations
+ # These should include the suffix "/index.json"
+
+ # Configuration for the feed where packages from internal non-stable builds will be published to
+ - name: StaticInternalFeed
+ value: 'https://dnceng.pkgs.visualstudio.com/_packaging/dotnet-core-internal/nuget/v3/index.json'
+
+ # Default locations for Installers and checksums
+ # Public Locations
+ - name: ChecksumsBlobFeedUrl
+ value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
+ - name: InstallersBlobFeedUrl
+ value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
+
+ # Private Locations
+ - name: InternalChecksumsBlobFeedUrl
+ value: https://dotnetclichecksumsmsrc.blob.core.windows.net/dotnet/index.json
+ - name: InternalChecksumsBlobFeedKey
+ value: $(dotnetclichecksumsmsrc-storage-key)
+
+ - name: InternalInstallersBlobFeedUrl
+ value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json
+ - name: InternalInstallersBlobFeedKey
+ value: $(dotnetclimsrc-access-key)
diff --git a/eng/common/templates/post-build/darc-gather-drop.yml b/eng/common/templates/post-build/darc-gather-drop.yml
index f4e3bfcf5cd..3268ccaa551 100644
--- a/eng/common/templates/post-build/darc-gather-drop.yml
+++ b/eng/common/templates/post-build/darc-gather-drop.yml
@@ -5,7 +5,7 @@ jobs:
- job: gatherDrop
displayName: Gather Drop
dependsOn: setupMaestroVars
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }})
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.ChannelId }}))
variables:
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml
index 0872db4ed94..5770bb21cc2 100644
--- a/eng/common/templates/post-build/post-build.yml
+++ b/eng/common/templates/post-build/post-build.yml
@@ -1,12 +1,21 @@
parameters:
- enableSourceLinkValidation: true
+ enableSourceLinkValidation: false
enableSigningValidation: true
- enableSymbolValidation: true
+ enableSymbolValidation: false
enableNugetValidation: true
+ publishInstallersAndChecksums: false
+ enableAzDONuGetFeeds: true
SDLValidationParameters:
enable: false
+ continueOnError: false
params: ''
+ # These parameters let the user customize the call to sdk-task.ps1 for publishing
+ # symbols & general artifacts as well as for signing validation
+ symbolPublishingAdditionalParameters: ''
+ artifactsPublishingAdditionalParameters: ''
+ signingValidationAdditionalParameters: ''
+
# Which stages should finish execution before post-build stages start
dependsOn: [build]
@@ -52,7 +61,9 @@ stages:
filePath: eng\common\sdk-task.ps1
arguments: -task SigningValidation -restore -msbuildEngine dotnet
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
- /p:Configuration=Release
+ /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
+ /p:Configuration=Release
+ ${{ parameters.signingValidationAdditionalParameters }}
- ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}:
- job:
@@ -77,18 +88,61 @@ stages:
-GHRepoName $(Build.Repository.Name)
-GHCommit $(Build.SourceVersion)
-SourcelinkCliVersion $(SourceLinkCLIVersion)
+ continueOnError: true
- ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}:
- template: /eng/common/templates/job/execute-sdl.yml
parameters:
additionalParameters: ${{ parameters.SDLValidationParameters.params }}
+ continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
+
+- template: \eng\common\templates\post-build\channels\netcore-dev-5.yml
+ parameters:
+ enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ enableAzDONuGetFeeds: ${{ parameters.enableAzDONuGetFeeds }}
-- template: \eng\common\templates\post-build\channels\public-dev-release.yml
+- template: \eng\common\templates\post-build\channels\netcore-dev-30.yml
parameters:
enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+
+- template: \eng\common\templates\post-build\channels\netcore-dev-31.yml
+ parameters:
+ enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+
+- template: \eng\common\templates\post-build\channels\netcore-tools-latest.yml
+ parameters:
+ enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ enableAzDONuGetFeeds: ${{ parameters.enableAzDONuGetFeeds }}
- template: \eng\common\templates\post-build\channels\public-validation-release.yml
+ parameters:
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ enableAzDONuGetFeeds: ${{ parameters.enableAzDONuGetFeeds }}
-- template: \eng\common\templates\post-build\channels\public-release.yml
+- template: \eng\common\templates\post-build\channels\netcore-release-30.yml
+ parameters:
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
-- template: \eng\common\templates\post-build\channels\internal-servicing.yml
+- template: \eng\common\templates\post-build\channels\netcore-release-31.yml
+ parameters:
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+
+- template: \eng\common\templates\post-build\channels\netcore-internal-30.yml
+ parameters:
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
diff --git a/eng/common/templates/post-build/promote-build.yml b/eng/common/templates/post-build/promote-build.yml
index 9387c583b31..6b479c3b82a 100644
--- a/eng/common/templates/post-build/promote-build.yml
+++ b/eng/common/templates/post-build/promote-build.yml
@@ -5,7 +5,7 @@ jobs:
- job:
displayName: Promote Build
dependsOn: setupMaestroVars
- condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], ${{ parameters.ChannelId }})
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.ChannelId }}))
variables:
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index 8fe2b11ad21..bb5638930ab 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -39,6 +39,10 @@
# installed on the machine instead of downloading one.
[bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
+# Enable repos to use a particular version of the on-line dotnet-install scripts.
+# default URL: https://dot.net/v1/dotnet-install.ps1
+[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { "v1" }
+
# True to use global NuGet cache instead of restoring packages to repository-local directory.
[bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci }
@@ -159,7 +163,8 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
$installScript = Join-Path $dotnetRoot "dotnet-install.ps1"
if (!(Test-Path $installScript)) {
Create-Directory $dotnetRoot
- Invoke-WebRequest "https://dot.net/v1/dotnet-install.ps1" -OutFile $installScript
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
+ Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript
}
return $installScript
@@ -278,6 +283,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
Create-Directory $packageDir
Write-Host "Downloading $packageName $packageVersion"
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath
Unzip $packagePath $packageDir
}
@@ -518,6 +524,9 @@ function MSBuild-Core() {
if ($warnAsError) {
$cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true"
}
+ else {
+ $cmdArgs += " /p:TreatWarningsAsErrors=false"
+ }
foreach ($arg in $args) {
if ($arg -ne $null -and $arg.Trim() -ne "") {
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
index 738bb5669da..94a1edd7d0b 100755
--- a/eng/common/tools.sh
+++ b/eng/common/tools.sh
@@ -45,6 +45,10 @@ warn_as_error=${warn_as_error:-true}
# installed on the machine instead of downloading one.
use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
+# Enable repos to use a particular version of the on-line dotnet-install scripts.
+# default URL: https://dot.net/v1/dotnet-install.sh
+dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
+
# True to use global NuGet cache instead of restoring packages to repository-local directory.
if [[ "$ci" == true ]]; then
use_global_nuget_cache=${use_global_nuget_cache:-false}
@@ -195,7 +199,7 @@ function InstallDotNet {
function GetDotNetInstallScript {
local root=$1
local install_script="$root/dotnet-install.sh"
- local install_script_url="https://dot.net/v1/dotnet-install.sh"
+ local install_script_url="https://dot.net/$dotnetInstallScriptVersion/dotnet-install.sh"
if [[ ! -a "$install_script" ]]; then
mkdir -p "$root"
@@ -204,12 +208,19 @@ function GetDotNetInstallScript {
# Use curl if available, otherwise use wget
if command -v curl > /dev/null; then
- curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script"
- else
- wget -q -O "$install_script" "$install_script_url"
+ curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
+ else
+ wget -q -O "$install_script" "$install_script_url" || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
fi
fi
-
# return value
_GetDotNetInstallScript="$install_script"
}