ArPow stage 1: local source-build infrastructure (#53294)

* Initial arcade-powered source-build infra

* Add patches, fixup from 5.0

* Rename patches to match 6.0-p1 work

* Add source-build specific build script

* Incorporate build-source-build.sh, into eng/

* Run inner build script, through coreclr

* Initial source-build args based on live build (not props file)

* Cleanup: add RID comments, rm empty/absolute args

* Fix subsets

* Disable sourcelink in env, not args

* Fix CI NZEC

* Revert "Use same code to DetectCiphersuiteConfiguration for portable and non-portable builds"

This reverts commit 464010d9d0241bbdcbfbda25b32e78991ddf6093.

* Fix portability

* Fix AllJits build

* Fix missing crossgen2 for non-portable RID

* Create supplemental intermediate nupkgs

* Tweak category names

* Use centralized supplemental nupkg infra

* Add additional 6.0 patches

* Patch updates after merging in main

* SourceBuild.props cleanup

* Fix issue with incorrect patch merge

* Patch updates

* Edit clr.tools patch

* patch updates

* Revert patch integration

* Patch updates

* Edits per code review feedback

* ILAsm workaround

* patch updates

* Move logic to set ILAsmToolPath for source-build

* Update eng/SourceBuild.props

Co-authored-by: Viktor Hofer <viktor.hofer@microsoft.com>

* Remove libraries specific patches

* Patch updates necessary with latest main merge

* Add back libraries-packages.proj harvesting patch

* Refactor intermediate package split to be chunkier

* Integrate patch 0017

* Subsets update per code review

* Remove obsolete patch

* Removed patches that were integrated into main

* Remove two additional patches

* Remove remaining patches

Co-authored-by: Davis Goodin <dagood@microsoft.com>
Co-authored-by: dseefeld <dseefeld@microsoft.com>
Co-authored-by: Viktor Hofer <viktor.hofer@microsoft.com>

9 files changed, 105 insertions, 27 deletions

diff --git a/Directory.Build.props b/Directory.Build.props
index d1b03d7ef8d..e2c0de947f3 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -83,6 +83,7 @@
     <WasmBuildTasksAssemblyPath>$([MSBuild]::NormalizePath('$(WasmBuildTasksDir)', 'WasmBuildTasks.dll'))</WasmBuildTasksAssemblyPath>
     <MonoAOTCompilerTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll'))</MonoAOTCompilerTasksAssemblyPath>
     <RuntimeConfigParserTasksAssemblyPath>$([MSBuild]::NormalizePath('$(RuntimeConfigParserDir)', 'RuntimeConfigParser.dll'))</RuntimeConfigParserTasksAssemblyPath>
+    <ILAsmToolPath Condition="'$(DotNetBuildFromSource)' == 'true'">$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)'))</ILAsmToolPath>
   </PropertyGroup>
 
   <PropertyGroup Label="CalculateConfiguration">
diff --git a/eng/SourceBuild.props b/eng/SourceBuild.props
new file mode 100644
index 00000000000..7dfc80d41fd
--- /dev/null
+++ b/eng/SourceBuild.props
@@ -0,0 +1,73 @@
+<Project>
+
+  <PropertyGroup>
+    <GitHubRepositoryName>runtime</GitHubRepositoryName>
+  </PropertyGroup>
+
+  <!-- Set up the dotnet/runtime source-build command. -->
+  <PropertyGroup>
+    <BaseInnerSourceBuildCommand>./build.sh</BaseInnerSourceBuildCommand>
+
+    <SourceBuildPortable>true</SourceBuildPortable>
+    <SourceBuildPortable Condition="'$(SourceBuildNonPortable)' == 'true'">false</SourceBuildPortable>
+
+    <!-- If TargetRid not specified, detect RID based on portability. -->
+    <TargetRid Condition="'$(TargetRid)' == '' and '$(SourceBuildNonPortable)' == 'true'">$([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier)</TargetRid>
+    <TargetRid Condition="'$(TargetRid)' == ''">$(__DistroRid)</TargetRid>
+
+    <!-- Split e.g. 'fedora.33-x64' into 'fedora.33' and 'x64'. -->
+    <_targetRidPlatformIndex>$(TargetRid.LastIndexOfAny("-"))</_targetRidPlatformIndex>
+    <TargetRidWithoutPlatform>$(TargetRid.Substring(0, $(_targetRidPlatformIndex)))</TargetRidWithoutPlatform>
+    <TargetRidPlatform>$(TargetRid.Substring($(_targetRidPlatformIndex)).TrimStart('-'))</TargetRidPlatform>
+
+    <LogVerbosity Condition="'$(LogVerbosity)' == ''">minimal</LogVerbosity>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <!-- Work around issue where local clone may cause failure using non-origin remote fallback: https://github.com/dotnet/sourcelink/issues/629 -->
+    <InnerBuildEnv Include="EnableSourceControlManagerQueries=false" />
+    <InnerBuildEnv Include="EnableSourceLink=false" />
+    <InnerBuildEnv Include="DisableSourceLink=true" />
+    <InnerBuildEnv Include="DeterministicSourcePaths=false" />
+  </ItemGroup>
+
+  <Target Name="GetRuntimeSourceBuildCommandConfiguration"
+          BeforeTargets="GetSourceBuildCommandConfiguration">
+    <PropertyGroup>
+      <InnerBuildArgs>$(InnerBuildArgs) --arch $(TargetRidPlatform)</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --configuration $(Configuration)</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --ci</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --allconfigurations</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --verbosity $(LogVerbosity)</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --nodereuse false</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --warnAsError false</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) --cmakeargs -DCLR_CMAKE_USE_SYSTEM_LIBUNWIND=TRUE</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:MicrosoftNetFrameworkReferenceAssembliesVersion=1.0.0</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:ContinuousIntegrationBuild=true</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:PackageRid=$(TargetRid)</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:NoPgoOptimize=true</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:KeepNativeSymbols=true</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:RuntimeOS=$(TargetRidWithoutPlatform)</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:PortableBuild=$(SourceBuildPortable)</InnerBuildArgs>
+      <InnerBuildArgs>$(InnerBuildArgs) /p:BuildDebPackage=false</InnerBuildArgs>
+    </PropertyGroup>
+  </Target>
+
+  <Target Name="CategorizeRuntimeSupplementalArtifacts"
+          BeforeTargets="GetCategorizedIntermediateNupkgContents">
+    <ItemGroup>
+      <!--
+        Runtime artifacts are too large to fit into a single package (Azure DevOps feeds 500 mb constraint).
+        Split large components into separate packages.
+      -->
+      <IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\dotnet-runtime-*.tar.gz" Category="runtime" />
+
+      <IntermediateNupkgArtifactFile
+        Include="
+          $(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\dotnet-crossgen2-*.tar.gz;
+          $(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\Microsoft.NETCore.App.Crossgen2.*.nupkg"
+        Category="Crossgen2Pack" />
+    </ItemGroup>
+  </Target>
+
+</Project>
diff --git a/eng/SourceBuildPrebuiltBaseline.xml b/eng/SourceBuildPrebuiltBaseline.xml
new file mode 100644
index 00000000000..c1b6dfbf053
--- /dev/null
+++ b/eng/SourceBuildPrebuiltBaseline.xml
@@ -0,0 +1,5 @@
+<UsageData>
+  <IgnorePatterns>
+    <UsagePattern IdentityGlob="*/*" />
+  </IgnorePatterns>
+</UsageData>
diff --git a/eng/Subsets.props b/eng/Subsets.props
index bf6b25bffc7..105b34254c6 100644
--- a/eng/Subsets.props
+++ b/eng/Subsets.props
@@ -25,8 +25,8 @@
   <PropertyGroup>
     <DefaultSubsets>clr+mono+libs+host+packs</DefaultSubsets>
     <DefaultSubsets Condition="'$(TargetsMobile)' == 'true'">mono+libs+packs</DefaultSubsets>
-    <!-- On Windows mono is supported for x86/x64 only -->
-    <DefaultSubsets Condition="'$(TargetOS)' == 'windows' and '$(TargetArchitecture)' != 'x86' and '$(TargetArchitecture)' != 'x64'">clr+libs+host+packs</DefaultSubsets>
+    <!-- mono is not supported in source build. On Windows mono is supported for x86/x64 only. -->
+    <DefaultSubsets Condition="'$(DotNetBuildFromSource)' == 'true' or ('$(TargetOS)' == 'windows' and '$(TargetArchitecture)' != 'x86' and '$(TargetArchitecture)' != 'x64')">clr+libs+host+packs</DefaultSubsets>
   </PropertyGroup>
 
   <!-- Init _subset here in to allow RuntimeFlavor to be set as early as possible -->
@@ -49,18 +49,21 @@
     <DefaultMonoSubsets Condition="'$(TargetOS)' == 'Browser'">$(DefaultMonoSubsets)mono.wasmruntime+</DefaultMonoSubsets>
     <DefaultMonoSubsets Condition="'$(MonoCrossAOTTargetOS)' != ''">$(DefaultMonoSubsets)mono.aotcross+</DefaultMonoSubsets>
     <DefaultMonoSubsets>$(DefaultMonoSubsets)mono.runtime+mono.corelib+mono.packages</DefaultMonoSubsets>
-     
+
     <DefaultLibrariesSubsets Condition="'$(BuildTargetFramework)' == '$(NetCoreAppCurrent)' or
                                         '$(BuildTargetFramework)' == '' or
                                         '$(BuildAllConfigurations)' == 'true'">libs.native+</DefaultLibrariesSubsets>
-    <DefaultLibrariesSubsets>$(DefaultLibrariesSubsets)libs.ref+libs.src+libs.pretest+libs.packages</DefaultLibrariesSubsets>
+    <DefaultLibrariesSubsets>$(DefaultLibrariesSubsets)libs.ref+libs.src+libs.packages</DefaultLibrariesSubsets>
+    <DefaultLibrariesSubsets Condition="'$(DotNetBuildFromSource)' != 'true'">$(DefaultLibrariesSubsets)+libs.pretest</DefaultLibrariesSubsets>
 
-    <DefaultHostSubsets>host.native+host.pkg+host.tools+host.tests</DefaultHostSubsets>
+    <DefaultHostSubsets>host.native+host.tools</DefaultHostSubsets>
+    <DefaultHostSubsets Condition="'$(DotNetBuildFromSource)' != 'true'">$(DefaultHostSubsets)+host.pkg+host.tests</DefaultHostSubsets>
     <DefaultHostSubsets Condition="'$(RuntimeFlavor)' == 'Mono'"></DefaultHostSubsets>
     <DefaultHostSubsets Condition="'$(RuntimeFlavor)' == 'Mono' and '$(TargetsMobile)' != 'true'">host.native</DefaultHostSubsets>
 
     <DefaultPacksSubsets>packs.product</DefaultPacksSubsets>
     <DefaultPacksSubsets Condition="'$(BuildMonoAOTCrossCompilerOnly)' != 'true'">$(DefaultPacksSubsets)+packs.tests</DefaultPacksSubsets>
+    <DefaultPacksSubsets Condition="'$(DotNetBuildFromSource)' == 'true'">$(DefaultPacksSubsets)+packs.installers</DefaultPacksSubsets>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -204,12 +207,12 @@
     <ProjectToBuild Include="$(CoreClrProjectRoot)tools\runincontext\runincontext.csproj;
                              $(CoreClrProjectRoot)tools\r2rdump\R2RDump.csproj;
                              $(CoreClrProjectRoot)tools\dotnet-pgo\dotnet-pgo.csproj;
-                             $(CoreClrProjectRoot)tools\r2rtest\R2RTest.csproj;
-                             $(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2.csproj" Category="clr" />
+                             $(CoreClrProjectRoot)tools\r2rtest\R2RTest.csproj" Category="clr" Condition="'$(DotNetBuildFromSource)' != 'true'"/>
+    <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2.csproj" Category="clr" />
 
     <ProjectToBuild Condition="'$(TargetArchitecture)' != 'x64' and '$(BuildArchitecture)' == 'x64'" Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2_crossarch.csproj" Category="clr" />
     <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\ILCompiler.TypeSystem.ReadyToRun.Tests\ILCompiler.TypeSystem.ReadyToRun.Tests.csproj"
-      Test="true" Category="clr" Condition="'$(__DistroRid)' != 'linux-musl-x64'"/>
+      Test="true" Category="clr" Condition="'$(__DistroRid)' != 'linux-musl-x64' and '$(DotNetBuildFromSource)' != 'true'"/>
   </ItemGroup>
 
   <ItemGroup Condition="$(_subset.Contains('+clr.nativecorelib+'))">
@@ -221,7 +224,7 @@
 
   <ItemGroup Condition="$(_subset.Contains('+clr.packages+')) and '$(PgoInstrument)' != 'true'">
     <ProjectToBuild Include="$(CoreClrProjectRoot).nuget\coreclr-packages.proj" Pack="true" Category="clr" />
-    <ProjectToBuild Include="$(CoreClrProjectRoot)tools\dotnet-pgo\dotnet-pgo-pack.proj" Pack="true" BuildInParallel="false" Category="clr" />
+    <ProjectToBuild Include="$(CoreClrProjectRoot)tools\dotnet-pgo\dotnet-pgo-pack.proj" Pack="true" BuildInParallel="false" Category="clr" Condition="'$(DotNetBuildFromSource)' != 'true'"/>
   </ItemGroup>
 
   <!-- Mono sets -->
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index f1ef354b76c..4177e280edf 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -9,6 +9,7 @@
     <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21311.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>4a2b475948d498b89fedef7cf890883f49bc1ea3</Sha>
+      <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21311.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
diff --git a/eng/packaging.props b/eng/packaging.props
index 87cecef871a..534d3ddcf32 100644
--- a/eng/packaging.props
+++ b/eng/packaging.props
@@ -16,10 +16,6 @@
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(DotNetBuildFromSource)' == 'true'">
-    <!-- Don't build referenced projects -->
-    <BuildPackageLibraryReferences>false</BuildPackageLibraryReferences>
-    <!-- Omit any files that were not built -->
-    <AllowPartialPackages>true</AllowPartialPackages>
     <!-- Don't permit harvesting since this requires pre-builts -->
     <HarvestStablePackage>false</HarvestStablePackage>
     <!-- Validation will fail in case we were relying on harvested assets or assets not built to satisfy stated support -->
diff --git a/eng/restore/harvestPackages.targets b/eng/restore/harvestPackages.targets
index a7fc8aa3465..07eb5a91a7a 100644
--- a/eng/restore/harvestPackages.targets
+++ b/eng/restore/harvestPackages.targets
@@ -1,4 +1,14 @@
 <Project InitialTargets="AddPackageDownload">
+
+  <PropertyGroup>
+    <PackagingTaskAssembly>$(NuGetPackageRoot)microsoft.dotnet.build.tasks.packaging\$(MicrosoftDotNetBuildTasksPackagingVersion)\tools\</PackagingTaskAssembly>
+    <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' == 'core'">$(PackagingTaskAssembly)netcoreapp3.1\</PackagingTaskAssembly>
+    <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' != 'core'">$(PackagingTaskAssembly)net472\</PackagingTaskAssembly>
+    <!-- TODO: Remove when https://github.com/dotnet/arcade/issues/7413 is fixed. -->
+    <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' == 'core' and '$(DotNetBuildFromSource)' == 'true'">$(PackagingTaskAssembly)net5.0\</PackagingTaskAssembly>
+    <PackagingTaskAssembly>$(PackagingTaskAssembly)Microsoft.DotNet.Build.Tasks.Packaging.dll</PackagingTaskAssembly>
+  </PropertyGroup>
+
   <UsingTask TaskName="GetLastStablePackage" AssemblyFile="$(PackagingTaskAssembly)"/>
   <Target Name="AddPackageDownload">
     <ItemGroup>
diff --git a/src/libraries/Directory.Build.props b/src/libraries/Directory.Build.props
index b06fc950644..0078c1331d3 100644
--- a/src/libraries/Directory.Build.props
+++ b/src/libraries/Directory.Build.props
@@ -28,7 +28,6 @@
   <PropertyGroup>
     <!-- Build all .NET Framework configurations when net48 is passed in. This is for convenience. -->
     <AdditionalBuildTargetFrameworks Condition="'$(BuildTargetFramework)' == 'net48'">net461;net462;net47;net471;net472</AdditionalBuildTargetFrameworks>
-    <AdditionalBuildTargetFrameworks Condition="'$(DotNetBuildFromSource)' == 'true' and '$(BuildAllProjects)' == 'true'">$(AdditionalBuildTargetFrameworks);netstandard2.0</AdditionalBuildTargetFrameworks>
   </PropertyGroup>
 
   <!-- Define test projects and companions -->
diff --git a/src/libraries/libraries-packages.proj b/src/libraries/libraries-packages.proj
index 2f4d0949db3..383dd088e12 100644
--- a/src/libraries/libraries-packages.proj
+++ b/src/libraries/libraries-packages.proj
@@ -1,20 +1,11 @@
 <Project Sdk="Microsoft.Build.Traversal" DefaultTargets="Pack">
 
-  <PropertyGroup>
-    <AdditionalBuildTargetFrameworks Condition="'$(DotNetBuildFromSource)' == 'true'">$(AdditionalBuildTargetFrameworks);package-$(Configuration)</AdditionalBuildTargetFrameworks>
-  </PropertyGroup>
-
-  <PropertyGroup>
-    <PackagingTaskAssembly>$(NuGetPackageRoot)microsoft.dotnet.build.tasks.packaging\$(MicrosoftDotNetBuildTasksPackagingVersion)\tools\</PackagingTaskAssembly>
-    <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' == 'core'">$(PackagingTaskAssembly)netcoreapp3.1\</PackagingTaskAssembly>
-    <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' != 'core'">$(PackagingTaskAssembly)net472\</PackagingTaskAssembly>
-    <PackagingTaskAssembly>$(PackagingTaskAssembly)Microsoft.DotNet.Build.Tasks.Packaging.dll</PackagingTaskAssembly>
-  </PropertyGroup>
-
   <ItemGroup>
+    <PackageReference Include="Microsoft.DotNet.Build.Tasks.Packaging" Version="$(MicrosoftDotNetBuildTasksPackagingVersion)" />
+
     <ProjectReference Include="$(PkgDir)*\*.proj" Exclude="$(PkgDir)test\*" />
     <ProjectReference Include="$(MSBuildThisFileDirectory)*\src\*.*proj"
-                      Condition="('$(BuildAllConfigurations)' == 'true' or '$(DotNetBuildFromSource)' == 'true')"
+                      Condition="'$(BuildAllConfigurations)' == 'true'"
                       Exclude="$(MSBuildThisFileDirectory)*\src\**\*.shproj"/>
   </ItemGroup>
 
@@ -31,7 +22,6 @@
     ones that might do this. After we ship a stable set of packages this target should be ran and the
     changes to the package index should be commited to the repo.
   -->
-  <UsingTask TaskName="UpdatePackageIndex" AssemblyFile="$(PackagingTaskAssembly)"/>
   <Target Name="UpdatePackageIndexWithStableVersions"
           BeforeTargets="Build;Pack"
           Condition="'$(DotNetFinalVersionKind)' == 'release'">