diff options
author | dotnet-maestro[bot] <42748379+dotnet-maestro[bot]@users.noreply.github.com> | 2022-01-19 13:02:07 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-01-19 13:02:07 +0300 |
commit | 70dc7f6daaf50e8eb67afb91876b8efc8330103f (patch) | |
tree | 03145d7e7ca17212e54a93cb467175f8d74afc2b /eng/common | |
parent | 67e3d793b82c46e62e767cf0ead3294f9689430f (diff) |
Update dependencies from https://github.com/dotnet/arcade build 20220112.1 (#2514)
Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.ApiCompat
From Version 7.0.0-beta.21524.1 -> To Version 6.0.0-beta.22062.1
Co-authored-by: dotnet-maestro[bot] <dotnet-maestro[bot]@users.noreply.github.com>
Diffstat (limited to 'eng/common')
-rwxr-xr-x | eng/common/build.sh | 3 | ||||
-rwxr-xr-x | eng/common/cross/build-rootfs.sh | 40 | ||||
-rw-r--r-- | eng/common/cross/toolchain.cmake | 5 | ||||
-rw-r--r-- | eng/common/native/find-native-compiler.sh (renamed from eng/common/native/init-compiler.sh) | 102 | ||||
-rw-r--r-- | eng/common/post-build/sourcelink-validation.ps1 | 5 | ||||
-rw-r--r-- | eng/common/post-build/symbols-validation.ps1 | 79 | ||||
-rw-r--r-- | eng/common/sdk-task.ps1 | 3 | ||||
-rw-r--r-- | eng/common/sdl/configure-sdl-tool.ps1 | 6 | ||||
-rw-r--r-- | eng/common/sdl/packages.config | 2 | ||||
-rw-r--r-- | eng/common/templates/job/execute-sdl.yml | 4 | ||||
-rw-r--r-- | eng/common/templates/job/onelocbuild.yml | 5 | ||||
-rw-r--r-- | eng/common/templates/job/source-index-stage1.yml | 2 | ||||
-rw-r--r-- | eng/common/templates/jobs/jobs.yml | 4 | ||||
-rw-r--r-- | eng/common/templates/steps/send-to-helix.yml | 3 | ||||
-rw-r--r-- | eng/common/tools.ps1 | 76 | ||||
-rwxr-xr-x | eng/common/tools.sh | 67 |
16 files changed, 191 insertions, 215 deletions
diff --git a/eng/common/build.sh b/eng/common/build.sh index bc07a1c68..a16e18b17 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -188,9 +188,6 @@ function InitializeCustomToolset { function Build { - if [[ "$ci" == true ]]; then - TryLogClientIpAddress - fi InitializeToolset InitializeCustomToolset diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 6fa2c8aa5..5c05b39f1 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -6,7 +6,7 @@ usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]" echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo " for FreeBSD can be: freebsd11, freebsd12, freebsd13" echo " for illumos can be: illumos." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" @@ -32,9 +32,9 @@ __UbuntuPackages="build-essential" __AlpinePackages="alpine-base" __AlpinePackages+=" build-base" __AlpinePackages+=" linux-headers" -__AlpinePackages+=" lldb-dev" -__AlpinePackages+=" python3" -__AlpinePackages+=" libedit" +__AlpinePackagesEdgeCommunity=" lldb-dev" +__AlpinePackagesEdgeMain+=" python3" +__AlpinePackagesEdgeMain+=" libedit" # symlinks fixer __UbuntuPackages+=" symlinks" @@ -185,17 +185,23 @@ while :; do __UbuntuRepo= __Tizen=tizen ;; - alpine|alpine3.13) + alpine|alpine3.9) __CodeName=alpine __UbuntuRepo= - __AlpineVersion=3.13 - __AlpinePackages+=" llvm10-libs" + __AlpineVersion=3.9 + __AlpinePackagesEdgeMain+=" llvm11-libs" + __AlpinePackagesEdgeMain+=" clang-libs" ;; - alpine3.14) + alpine3.13) __CodeName=alpine __UbuntuRepo= - __AlpineVersion=3.14 - __AlpinePackages+=" llvm11-libs" + __AlpineVersion=3.13 + # Alpine 3.13 has all the packages we need in the 3.13 repository + __AlpinePackages+=$__AlpinePackagesEdgeCommunity + __AlpinePackagesEdgeCommunity= + __AlpinePackages+=$__AlpinePackagesEdgeMain + __AlpinePackagesEdgeMain= + __AlpinePackages+=" llvm10-libs" ;; freebsd11) __FreeBSDBase="11.3-RELEASE" @@ -273,6 +279,20 @@ if [[ "$__CodeName" == "alpine" ]]; then -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ add $__AlpinePackages + if [[ -n "$__AlpinePackagesEdgeMain" ]]; then + $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ + -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ + add $__AlpinePackagesEdgeMain + fi + + if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then + $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/community \ + -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ + add $__AlpinePackagesEdgeCommunity + fi + rm -r $__ApkToolsDir elif [[ "$__CodeName" == "freebsd" ]]; then mkdir -p $__RootfsDir/usr/local/etc diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 6501c3a95..ec8971eb0 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -44,7 +44,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") set(TOOLCHAIN "i686-linux-gnu") elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") set(CMAKE_SYSTEM_PROCESSOR "x86_64") - set(triple "x86_64-unknown-freebsd12") + set(triple "x86_64-unknown-freebsd11") elseif (ILLUMOS) set(CMAKE_SYSTEM_PROCESSOR "x86_64") set(TOOLCHAIN "x86_64-illumos") @@ -91,9 +91,6 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") set(CMAKE_CXX_COMPILER_TARGET ${triple}) set(CMAKE_ASM_COMPILER_TARGET ${triple}) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld") elseif(ILLUMOS) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/find-native-compiler.sh index 8c944f30b..aed19d07d 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/find-native-compiler.sh @@ -1,34 +1,39 @@ #!/usr/bin/env bash # -# This file detects the C/C++ compiler and exports it to the CC/CXX environment variables +# This file locates the native compiler with the given name and version and sets the environment variables to locate it. # -if [[ "$#" -lt 3 ]]; then +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +if [ $# -lt 0 ] +then echo "Usage..." - echo "init-compiler.sh <script directory> <Architecture> <compiler> <compiler major version> <compiler minor version>" - echo "Specify the script directory." - echo "Specify the target architecture." + echo "find-native-compiler.sh <compiler> <compiler major version> <compiler minor version>" echo "Specify the name of compiler (clang or gcc)." echo "Specify the major version of compiler." echo "Specify the minor version of compiler." exit 1 fi -nativescriptroot="$1" -build_arch="$2" -compiler="$3" -cxxCompiler="$compiler++" -majorVersion="$4" -minorVersion="$5" - -. "$nativescriptroot"/../pipeline-logging-functions.sh +. $scriptroot/../pipeline-logging-functions.sh -# clear the existing CC and CXX from environment -CC= -CXX= -LDFLAGS= +compiler="$1" +cxxCompiler="$compiler++" +majorVersion="$2" +minorVersion="$3" -if [[ "$compiler" == "gcc" ]]; then cxxCompiler="g++"; fi +if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi check_version_exists() { desired_version=-1 @@ -45,38 +50,38 @@ check_version_exists() { echo "$desired_version" } -if [[ -z "$CLR_CC" ]]; then +if [ -z "$CLR_CC" ]; then # Set default versions - if [[ -z "$majorVersion" ]]; then + if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [[ "$compiler" == "clang" ]]; then versions=( 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) - elif [[ "$compiler" == "gcc" ]]; then versions=( 12 11 10 9 8 7 6 5 4.9 ); fi + if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) + elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi for version in "${versions[@]}"; do parts=(${version//./ }) desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")" - if [[ "$desired_version" != "-1" ]]; then majorVersion="${parts[0]}"; break; fi + if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi done - if [[ -z "$majorVersion" ]]; then + if [ -z "$majorVersion" ]; then if command -v "$compiler" > /dev/null; then - if [[ "$(uname)" != "Darwin" ]]; then + if [ "$(uname)" != "Darwin" ]; then Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH." fi - CC="$(command -v "$compiler")" - CXX="$(command -v "$cxxCompiler")" + export CC="$(command -v "$compiler")" + export CXX="$(command -v "$cxxCompiler")" else Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found." exit 1 fi else - if [[ "$compiler" == "clang" && "$majorVersion" -lt 5 ]]; then - if [[ "$build_arch" == "arm" || "$build_arch" == "armel" ]]; then + if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then + if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then if command -v "$compiler" > /dev/null; then Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." - CC="$(command -v "$compiler")" - CXX="$(command -v "$cxxCompiler")" + export CC="$(command -v "$compiler")" + export CXX="$(command -v "$cxxCompiler")" else Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." exit 1 @@ -86,40 +91,31 @@ if [[ -z "$CLR_CC" ]]; then fi else desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" - if [[ "$desired_version" == "-1" ]]; then + if [ "$desired_version" = "-1" ]; then Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion." exit 1 fi fi - if [[ -z "$CC" ]]; then - CC="$(command -v "$compiler$desired_version")" - CXX="$(command -v "$cxxCompiler$desired_version")" - if [[ -z "$CXX" ]]; then CXX="$(command -v "$cxxCompiler")"; fi + if [ -z "$CC" ]; then + export CC="$(command -v "$compiler$desired_version")" + export CXX="$(command -v "$cxxCompiler$desired_version")" + if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi fi else - if [[ ! -f "$CLR_CC" ]]; then + if [ ! -f "$CLR_CC" ]; then Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist" exit 1 fi - CC="$CLR_CC" - CXX="$CLR_CXX" + export CC="$CLR_CC" + export CXX="$CLR_CXX" fi -if [[ -z "$CC" ]]; then - Write-PipelineTelemetryError -category "Build" "Unable to find $compiler." +if [ -z "$CC" ]; then + Write-PipelineTelemetryError -category "Build" "Unable to find $compiler." exit 1 fi -if [[ "$compiler" == "clang" ]]; then - if command -v "lld$desired_version" > /dev/null; then - # Only lld version >= 9 can be considered stable - if [[ "$majorVersion" -ge 9 ]]; then - LDFLAGS="-fuse-ld=lld" - fi - fi -fi - -SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" - -export CC CXX LDFLAGS SCAN_BUILD_COMMAND +export CCC_CC="$CC" +export CCC_CXX="$CXX" +export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 4011d324e..e8ab29afe 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -22,11 +22,6 @@ $RetryWaitTimeInSeconds = 30 # Wait time between check for system load $SecondsBetweenLoadChecks = 10 -if (!$InputPath -or !(Test-Path $InputPath)){ - Write-Host "No files to validate." - ExitWithExitCode 0 -} - $ValidatePackage = { param( [string] $PackagePath # Full path to a Symbols.NuGet package diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index a4a92efbe..a5af041ba 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -4,11 +4,9 @@ param( [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error - [Parameter(Mandatory = $false)][switch] $Clean, # Clean extracted symbols directory after checking symbols - [Parameter(Mandatory = $false)][string] $SymbolExclusionFile # Exclude the symbols in the file from publishing to symbol server + [Parameter(Mandatory = $false)][switch] $Clean # Clean extracted symbols directory after checking symbols ) -. $PSScriptRoot\..\tools.ps1 # Maximum number of jobs to run in parallel $MaxParallelJobs = 16 @@ -27,28 +25,14 @@ if ($CheckForWindowsPdbs) { $WindowsPdbVerificationParam = "--windows-pdbs" } -$ExclusionSet = New-Object System.Collections.Generic.HashSet[string]; - -if (!$InputPath -or !(Test-Path $InputPath)){ - Write-Host "No symbols to validate." - ExitWithExitCode 0 -} - -#Check if the path exists -if ($SymbolExclusionFile -and (Test-Path $SymbolExclusionFile)){ - [string[]]$Exclusions = Get-Content "$SymbolExclusionFile" - $Exclusions | foreach { if($_ -and $_.Trim()){$ExclusionSet.Add($_)} } -} -else{ - Write-Host "Symbol Exclusion file does not exists. No symbols to exclude." -} - $CountMissingSymbols = { param( [string] $PackagePath, # Path to a NuGet package [string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs ) + . $using:PSScriptRoot\..\tools.ps1 + Add-Type -AssemblyName System.IO.Compression.FileSystem Write-Host "Validating $PackagePath " @@ -158,44 +142,37 @@ $CountMissingSymbols = { return $null } - $FileRelativePath = $FileName.Replace("$ExtractPath\", "") - if (($($using:ExclusionSet) -ne $null) -and ($($using:ExclusionSet).Contains($FileRelativePath) -or ($($using:ExclusionSet).Contains($FileRelativePath.Replace("\", "/"))))){ - Write-Host "Skipping $FileName from symbol validation" + $FileGuid = New-Guid + $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid + + $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault ` + -FullPath $FileName ` + -TargetServerParam '--microsoft-symbol-server' ` + -SymbolsPath "$ExpandedSymbolsPath-msdl" ` + -WindowsPdbVerificationParam $WindowsPdbVerificationParam + $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault ` + -FullPath $FileName ` + -TargetServerParam '--internal-server' ` + -SymbolsPath "$ExpandedSymbolsPath-symweb" ` + -WindowsPdbVerificationParam $WindowsPdbVerificationParam + + Write-Host -NoNewLine "`t Checking file " $FileName "... " + + if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { + Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" } - else { - $FileGuid = New-Guid - $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid - - $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault ` - -FullPath $FileName ` - -TargetServerParam '--microsoft-symbol-server' ` - -SymbolsPath "$ExpandedSymbolsPath-msdl" ` - -WindowsPdbVerificationParam $WindowsPdbVerificationParam - $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault ` - -FullPath $FileName ` - -TargetServerParam '--internal-server' ` - -SymbolsPath "$ExpandedSymbolsPath-symweb" ` - -WindowsPdbVerificationParam $WindowsPdbVerificationParam - - Write-Host -NoNewLine "`t Checking file " $FileName "... " - - if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { - Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" + $MissingSymbols++ + + if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { + Write-Host 'No symbols found on MSDL or SymWeb!' } else { - $MissingSymbols++ - - if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { - Write-Host 'No symbols found on MSDL or SymWeb!' + if ($SymbolsOnMSDL -eq $null) { + Write-Host 'No symbols found on MSDL!' } else { - if ($SymbolsOnMSDL -eq $null) { - Write-Host 'No symbols found on MSDL!' - } - else { - Write-Host 'No symbols found on SymWeb!' - } + Write-Host 'No symbols found on SymWeb!' } } } diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 7ab9baac5..b1bca63ab 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -83,9 +83,6 @@ try { } if ($restore) { - if ($ci) { - Try-LogClientIpAddress - } Build 'Restore' } diff --git a/eng/common/sdl/configure-sdl-tool.ps1 b/eng/common/sdl/configure-sdl-tool.ps1 index 8a68fc24b..4999c3070 100644 --- a/eng/common/sdl/configure-sdl-tool.ps1 +++ b/eng/common/sdl/configure-sdl-tool.ps1 @@ -69,13 +69,13 @@ try { # For some tools, add default and automatic args. if ($tool.Name -eq 'credscan') { if ($targetDirectory) { - $tool.Args += "`"TargetDirectory < $TargetDirectory`"" + $tool.Args += "TargetDirectory < $TargetDirectory" } - $tool.Args += "`"OutputType < pre`"" + $tool.Args += "OutputType < pre" $tool.Args += $CrScanAdditionalRunConfigParams } elseif ($tool.Name -eq 'policheck') { if ($targetDirectory) { - $tool.Args += "`"Target < $TargetDirectory`"" + $tool.Args += "Target < $TargetDirectory" } $tool.Args += $PoliCheckAdditionalRunConfigParams } diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 3bd8b29eb..b7bcfe38c 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ <?xml version="1.0" encoding="utf-8"?> <packages> - <package id="Microsoft.Guardian.Cli" version="0.53.3"/> + <package id="Microsoft.Guardian.Cli" version="0.110.1"/> </packages> diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 69eb67849..0ca2afe69 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -54,7 +54,7 @@ jobs: # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in # sync with the packages.config file. - name: DefaultGuardianVersion - value: 0.53.3 + value: 0.110.1 - name: GuardianVersion value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - name: GuardianPackagesConfigFile @@ -62,7 +62,7 @@ jobs: pool: # To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019. ${{ if eq(parameters.extractArchiveArtifacts, 'false') }}: - name: Hosted VS2017 + vmImage: windows-2019 ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: vmImage: windows-2019 steps: diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index e8bc77d2e..069098b0a 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -4,7 +4,7 @@ parameters: # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool pool: - vmImage: vs2017-win2016 + vmImage: windows-2019 CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex GithubPat: $(BotAccount-dotnet-bot-repo-PAT) @@ -12,6 +12,7 @@ parameters: SourcesDirectory: $(Build.SourcesDirectory) CreatePr: true AutoCompletePr: false + ReusePr: true UseLfLineEndings: true UseCheckedInLocProjectJson: false LanguageSet: VS_Main_Languages @@ -64,6 +65,8 @@ jobs: ${{ if eq(parameters.CreatePr, true) }}: isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 1cc0c29e4..d8990549e 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -6,7 +6,7 @@ parameters: preSteps: [] binlogPath: artifacts/log/Debug/Build.binlog pool: - vmImage: vs2017-win2016 + vmImage: windows-2019 condition: '' dependsOn: '' diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index a1f8fce96..90015a7e5 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -83,7 +83,7 @@ jobs: - ${{ if eq(parameters.enableSourceBuild, true) }}: - Source_Build_Complete pool: - vmImage: vs2017-win2016 + vmImage: windows-2019 runAsPublic: ${{ parameters.runAsPublic }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} @@ -96,4 +96,4 @@ jobs: dependsOn: - Asset_Registry_Publish pool: - vmImage: vs2017-win2016 + vmImage: windows-2019 diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 09a223989..cd02ae160 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -20,6 +20,7 @@ parameters: IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting int) @@ -53,6 +54,7 @@ steps: IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} HelixBaseUri: ${{ parameters.HelixBaseUri }} Creator: ${{ parameters.Creator }} @@ -83,6 +85,7 @@ steps: IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} HelixBaseUri: ${{ parameters.HelixBaseUri }} Creator: ${{ parameters.Creator }} diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 444842899..f1e1cb539 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -163,9 +163,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Disable telemetry on CI. if ($ci) { $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 - - # In case of network error, try to log the current IP for reference - Try-LogClientIpAddress } # Source Build uses DotNetCoreSdkDir variable @@ -301,32 +298,45 @@ function InstallDotNet([string] $dotnetRoot, if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } if ($noPath) { $installParameters.NoPath = $True } - try { - & $installScript @installParameters - } - catch { - if ($runtimeSourceFeed -or $runtimeSourceFeedKey) { - Write-Host "Failed to install dotnet from public location. Trying from '$runtimeSourceFeed'" - if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed } + $variations = @() + $variations += @($installParameters) - if ($runtimeSourceFeedKey) { - $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey) - $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes) - $installParameters.FeedCredential = $decodedString - } + $dotnetBuilds = $installParameters.Clone() + $dotnetbuilds.AzureFeed = "https://dotnetbuilds.azureedge.net/public" + $variations += @($dotnetBuilds) - try { - & $installScript @installParameters - } - catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from custom location '$runtimeSourceFeed'." - ExitWithExitCode 1 - } + if ($runtimeSourceFeed) { + $runtimeSource = $installParameters.Clone() + $runtimeSource.AzureFeed = $runtimeSourceFeed + if ($runtimeSourceFeedKey) { + $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey) + $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes) + $runtimeSource.FeedCredential = $decodedString + } + $variations += @($runtimeSource) + } + + $installSuccess = $false + foreach ($variation in $variations) { + if ($variation | Get-Member AzureFeed) { + $location = $variation.AzureFeed } else { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from public location." - ExitWithExitCode 1 + $location = "public location"; + } + Write-Host "Attempting to install dotnet from $location." + try { + & $installScript @variation + $installSuccess = $true + break + } + catch { + Write-Host "Failed to install dotnet from $location." } } + if (-not $installSuccess) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations." + ExitWithExitCode 1 + } } # @@ -882,24 +892,6 @@ if (!$disableConfigureToolsetImport) { } } -function Try-LogClientIpAddress() -{ - Write-Host "Attempting to log this client's IP for Azure Package feed telemetry purposes" - try - { - $result = Invoke-WebRequest -Uri "http://co1.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing - $lines = $result.Content.Split([Environment]::NewLine) - $socketIp = $lines | Select-String -Pattern "^Socket IP:.*" - Write-Host $socketIp - $clientIp = $lines | Select-String -Pattern "^Client IP:.*" - Write-Host $clientIp - } - catch - { - Write-Host "Unable to get this machine's effective IP address for logging: $_" - } -} - # # If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic. # diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 6a4871ef7..e555c3426 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -188,28 +188,29 @@ function InstallDotNet { GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript - local archArg='' + local installParameters=(--version $version --install-dir "$root") + if [[ -n "${3:-}" ]] && [ "$3" != 'unset' ]; then - archArg="--architecture $3" + installParameters+=(--architecture $3) fi - local runtimeArg='' if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then - runtimeArg="--runtime $4" + installParameters+=(--runtime $4) fi - local skipNonVersionedFilesArg="" if [[ "$#" -ge "5" ]] && [[ "$5" != 'false' ]]; then - skipNonVersionedFilesArg="--skip-non-versioned-files" + installParameters+=(--skip-non-versioned-files) fi - bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { - local exit_code=$? - echo "Failed to install dotnet SDK from public location (exit code '$exit_code')." - local runtimeSourceFeed='' - if [[ -n "${6:-}" ]]; then - runtimeSourceFeed="--azure-feed $6" - fi + local variations=() # list of variable names with parameter arrays in them + + local public_location=("${installParameters[@]}") + variations+=(public_location) - local runtimeSourceFeedKey='' + local dotnetbuilds=("${installParameters[@]}" --azure-feed "https://dotnetbuilds.azureedge.net/public") + variations+=(dotnetbuilds) + + if [[ -n "${6:-}" ]]; then + variations+=(private_feed) + local private_feed=("${installParameters[@]}" --azure-feed $6) if [[ -n "${7:-}" ]]; then # The 'base64' binary on alpine uses '-d' and doesn't support '--decode' # '-d'. To work around this, do a simple detection and switch the parameter @@ -219,22 +220,27 @@ function InstallDotNet { decodeArg="-d" fi decodedFeedKey=`echo $7 | base64 $decodeArg` - runtimeSourceFeedKey="--feed-credential $decodedFeedKey" + private_feed+=(--feed-credential $decodedFeedKey) fi + fi - if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then - bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')." - ExitWithExitCode $exit_code - } - else - if [[ $exit_code != 0 ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')." - fi - ExitWithExitCode $exit_code + local installSuccess=0 + for variationName in "${variations[@]}"; do + local name="$variationName[@]" + local variation=("${!name}") + echo "Attempting to install dotnet from $variationName." + bash "$install_script" "${variation[@]}" && installSuccess=1 + if [[ "$installSuccess" -eq 1 ]]; then + break fi - } + + echo "Failed to install dotnet from $variationName." + done + + if [[ "$installSuccess" -eq 0 ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from any of the specified locations." + ExitWithExitCode 1 + fi } function with_retries { @@ -399,13 +405,6 @@ function StopProcesses { return 0 } -function TryLogClientIpAddress () { - echo 'Attempting to log this client''s IP for Azure Package feed telemetry purposes' - if command -v curl > /dev/null; then - curl -s 'http://co1.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' || true - fi -} - function MSBuild { local args=$@ if [[ "$pipelines_log" == true ]]; then |