Add arcade infrastructure to dotnet/cecil (#37)

Add arcade infrastructure to dotnet/cecil
Adds eng/common folder from arcade
Adds NuGet.config file with the dotnet eng/tools/public package sources
Adds build scripts at root level to execute the eng/common/build scripts
Adds eng/Versions.props and eng/Version.Details.xml files with minimal dependencies since Cecil doesn't require anything, mostly just setting up the package version
Adds global.json
Adds arcade artifacts to .gitignore file
Add Microsoft.Dotnet.Arcade.Sdk
Make Mono.Cecil a package
Remove frameworks non-compatible with Arcade SDK
Make the package non-shippable so it doesn't publish in Nuget
Suppress license validation since Cecil has a different license than MIT one
Workaround the publickey and publickey token generation from arcade
Modify version to match Cecil version 0.11.4.0
Remove helix SDK since we don't use helix testing
Add pipeline yaml file
Add support for NetCoreAppToolCurrent equals to net7.0 and ToolsFramework netstandard 2.0
Add a variable to find the resources folder given that there is a new structure in the artifacts directory
Use VS test runner to avoid running tests using xunit
Add signing properties for 3rd party libraries

1 files changed, 339 insertions, 0 deletions

diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1
new file mode 100644
index 0000000..cd2181b
--- /dev/null
+++ b/eng/common/post-build/symbols-validation.ps1
@@ -0,0 +1,339 @@
+param(
+  [Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored
+  [Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
+  [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use
+  [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs
+  [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error
+  [Parameter(Mandatory = $false)][switch] $Clean,           # Clean extracted symbols directory after checking symbols
+  [Parameter(Mandatory = $false)][string] $SymbolExclusionFile  # Exclude the symbols in the file from publishing to symbol server
+)
+
+. $PSScriptRoot\..\tools.ps1
+# Maximum number of jobs to run in parallel
+$MaxParallelJobs = 16
+
+# Max number of retries
+$MaxRetry = 5
+
+# Wait time between check for system load
+$SecondsBetweenLoadChecks = 10
+
+# Set error codes
+Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1
+Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2
+
+$WindowsPdbVerificationParam = ""
+if ($CheckForWindowsPdbs) {
+  $WindowsPdbVerificationParam = "--windows-pdbs"
+}
+
+$ExclusionSet = New-Object System.Collections.Generic.HashSet[string];
+
+if (!$InputPath -or !(Test-Path $InputPath)){
+  Write-Host "No symbols to validate."
+  ExitWithExitCode 0
+}
+
+#Check if the path exists
+if ($SymbolExclusionFile -and (Test-Path $SymbolExclusionFile)){
+  [string[]]$Exclusions = Get-Content "$SymbolExclusionFile"
+  $Exclusions | foreach { if($_ -and $_.Trim()){$ExclusionSet.Add($_)} }
+}
+else{
+  Write-Host "Symbol Exclusion file does not exists. No symbols to exclude."
+}
+
+$CountMissingSymbols = {
+  param( 
+    [string] $PackagePath, # Path to a NuGet package
+    [string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs
+  )
+
+  Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+  Write-Host "Validating $PackagePath "
+
+  # Ensure input file exist
+  if (!(Test-Path $PackagePath)) {
+    Write-PipelineTaskError "Input file does not exist: $PackagePath"
+    return [pscustomobject]@{
+      result      = $using:ERROR_FILEDOESNOTEXIST
+      packagePath = $PackagePath
+    }
+  }
+  
+  # Extensions for which we'll look for symbols
+  $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib')
+
+  # How many files are missing symbol information
+  $MissingSymbols = 0
+
+  $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
+  $PackageGuid = New-Guid
+  $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageGuid
+  $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols'
+  
+  try {
+    [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
+  }
+  catch {
+    Write-Host "Something went wrong extracting $PackagePath"
+    Write-Host $_
+    return [pscustomobject]@{
+      result      = $using:ERROR_BADEXTRACT
+      packagePath = $PackagePath
+    }
+  }
+
+  Get-ChildItem -Recurse $ExtractPath |
+  Where-Object { $RelevantExtensions -contains $_.Extension } |
+  ForEach-Object {
+    $FileName = $_.FullName
+    if ($FileName -Match '\\ref\\') {
+      Write-Host "`t Ignoring reference assembly file " $FileName
+      return
+    }
+
+    $FirstMatchingSymbolDescriptionOrDefault = {
+      param( 
+        [string] $FullPath, # Full path to the module that has to be checked
+        [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
+        [string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs.
+        [string] $SymbolsPath
+      )
+
+      $FileName = [System.IO.Path]::GetFileName($FullPath)
+      $Extension = [System.IO.Path]::GetExtension($FullPath)
+
+      # Those below are potential symbol files that the `dotnet symbol` might
+      # return. Which one will be returned depend on the type of file we are
+      # checking and which type of file was uploaded.
+
+      # The file itself is returned
+      $SymbolPath = $SymbolsPath + '\' + $FileName
+
+      # PDB file for the module
+      $PdbPath = $SymbolPath.Replace($Extension, '.pdb')
+
+      # PDB file for R2R module (created by crossgen)
+      $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb')
+
+      # DBG file for a .so library
+      $SODbg = $SymbolPath.Replace($Extension, '.so.dbg')
+
+      # DWARF file for a .dylib
+      $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
+
+      $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
+      $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"
+
+      $totalRetries = 0
+
+      while ($totalRetries -lt $using:MaxRetry) {
+
+        # Save the output and get diagnostic output
+        $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
+
+        if ((Test-Path $PdbPath) -and (Test-path $SymbolPath)) {
+          return 'Module and PDB for Module'
+        }
+        elseif ((Test-Path $NGenPdb) -and (Test-Path $PdbPath) -and (Test-Path $SymbolPath)) {
+          return 'Dll, PDB and NGen PDB'
+        }
+        elseif ((Test-Path $SODbg) -and (Test-Path $SymbolPath)) {
+          return 'So and DBG for SO'
+        }  
+        elseif ((Test-Path $DylibDwarf) -and (Test-Path $SymbolPath)) {
+          return 'Dylib and Dwarf for Dylib'
+        }  
+        elseif (Test-Path $SymbolPath) {
+          return 'Module'
+        }
+        else
+        {
+          $totalRetries++
+        }
+      }
+      
+      return $null
+    }
+
+    $FileRelativePath = $FileName.Replace("$ExtractPath\", "")
+    if (($($using:ExclusionSet) -ne $null) -and ($($using:ExclusionSet).Contains($FileRelativePath) -or ($($using:ExclusionSet).Contains($FileRelativePath.Replace("\", "/"))))){
+      Write-Host "Skipping $FileName from symbol validation"
+    }
+
+    else {
+      $FileGuid = New-Guid
+      $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid
+
+      $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault `
+          -FullPath $FileName `
+          -TargetServerParam '--microsoft-symbol-server' `
+          -SymbolsPath "$ExpandedSymbolsPath-msdl" `
+          -WindowsPdbVerificationParam $WindowsPdbVerificationParam
+      $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault `
+          -FullPath $FileName `
+          -TargetServerParam '--internal-server' `
+          -SymbolsPath "$ExpandedSymbolsPath-symweb" `
+          -WindowsPdbVerificationParam $WindowsPdbVerificationParam
+
+      Write-Host -NoNewLine "`t Checking file " $FileName "... "
+  
+      if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
+        Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
+      }
+      else {
+        $MissingSymbols++
+
+        if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
+          Write-Host 'No symbols found on MSDL or SymWeb!'
+        }
+        else {
+          if ($SymbolsOnMSDL -eq $null) {
+            Write-Host 'No symbols found on MSDL!'
+          }
+          else {
+            Write-Host 'No symbols found on SymWeb!'
+          }
+        }
+      }
+    }
+  }
+  
+  if ($using:Clean) {
+    Remove-Item $ExtractPath -Recurse -Force
+  }
+  
+  Pop-Location
+
+  return [pscustomobject]@{
+    result      = $MissingSymbols
+    packagePath = $PackagePath
+  }
+}
+
+function CheckJobResult(
+  $result, 
+  $packagePath,
+  [ref]$DupedSymbols,
+  [ref]$TotalFailures) {
+  if ($result -eq $ERROR_BADEXTRACT) {
+    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files"
+    $DupedSymbols.Value++
+  } 
+  elseif ($result -eq $ERROR_FILEDOESNOTEXIST) {
+    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist"
+    $TotalFailures.Value++
+  }
+  elseif ($result -gt '0') {
+    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
+    $TotalFailures.Value++
+  }
+  else {
+    Write-Host "All symbols verified for package $packagePath"
+  }
+}
+
+function CheckSymbolsAvailable {
+  if (Test-Path $ExtractPath) {
+    Remove-Item $ExtractPath -Force  -Recurse -ErrorAction SilentlyContinue
+  }
+
+  $TotalPackages = 0
+  $TotalFailures = 0
+  $DupedSymbols = 0
+
+  Get-ChildItem "$InputPath\*.nupkg" |
+    ForEach-Object {
+      $FileName = $_.Name
+      $FullName = $_.FullName
+
+      # These packages from Arcade-Services include some native libraries that
+      # our current symbol uploader can't handle. Below is a workaround until
+      # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
+      if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') {
+        Write-Host "Ignoring Arcade-services file: $FileName"
+        Write-Host
+        return
+      }
+      elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') {
+        Write-Host "Ignoring Arcade-services file: $FileName"
+        Write-Host
+        return
+      }
+
+      $TotalPackages++
+
+      Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null
+
+      $NumJobs = @(Get-Job -State 'Running').Count
+
+      while ($NumJobs -ge $MaxParallelJobs) {
+        Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
+        sleep $SecondsBetweenLoadChecks
+        $NumJobs = @(Get-Job -State 'Running').Count
+      }
+
+      foreach ($Job in @(Get-Job -State 'Completed')) {
+        $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+        CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures)
+        Remove-Job -Id $Job.Id
+      }
+      Write-Host
+    }
+
+  foreach ($Job in @(Get-Job)) {
+    $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+    CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures)
+  }
+
+  if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) {
+    if ($TotalFailures -gt 0) {
+      Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages"
+    }
+
+    if ($DupedSymbols -gt 0) {
+      Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted"
+    }
+    
+    ExitWithExitCode 1
+  }
+  else {
+    Write-Host "All symbols validated!"
+  }
+}
+
+function InstallDotnetSymbol {
+  $dotnetSymbolPackageName = 'dotnet-symbol'
+
+  $dotnetRoot = InitializeDotNetCli -install:$true
+  $dotnet = "$dotnetRoot\dotnet.exe"
+  $toolList = & "$dotnet" tool list --global
+
+  if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) {
+    Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed."
+  }
+  else {
+    Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..."
+    Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
+    & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global
+  }
+}
+
+try {
+  . $PSScriptRoot\post-build-utils.ps1
+  
+  InstallDotnetSymbol
+
+  foreach ($Job in @(Get-Job)) {
+    Remove-Job -Id $Job.Id
+  }
+
+  CheckSymbolsAvailable
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'CheckSymbols' -Message $_
+  ExitWithExitCode 1
+}