diff options
author | Michael Simons <msimons@microsoft.com> | 2021-06-22 11:14:07 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-06-22 11:14:07 +0300 |
commit | 849033aea0ac5746e9030961c3eadd51149bcf90 (patch) | |
tree | 41eb5f390e32e7dcaaeb8be8d0e3fd484c023b9e | |
parent | 8a20ae03566e3aabb0c95d2bb206a9ee780db4fd (diff) |
ArPow stage 1: local source-build infrastructure (#53294)
* Initial arcade-powered source-build infra
* Add patches, fixup from 5.0
* Rename patches to match 6.0-p1 work
* Add source-build specific build script
* Incorporate build-source-build.sh, into eng/
* Run inner build script, through coreclr
* Initial source-build args based on live build (not props file)
* Cleanup: add RID comments, rm empty/absolute args
* Fix subsets
* Disable sourcelink in env, not args
* Fix CI NZEC
* Revert "Use same code to DetectCiphersuiteConfiguration for portable and non-portable builds"
This reverts commit 464010d9d0241bbdcbfbda25b32e78991ddf6093.
* Fix portability
* Fix AllJits build
* Fix missing crossgen2 for non-portable RID
* Create supplemental intermediate nupkgs
* Tweak category names
* Use centralized supplemental nupkg infra
* Add additional 6.0 patches
* Patch updates after merging in main
* SourceBuild.props cleanup
* Fix issue with incorrect patch merge
* Patch updates
* Edit clr.tools patch
* patch updates
* Revert patch integration
* Patch updates
* Edits per code review feedback
* ILAsm workaround
* patch updates
* Move logic to set ILAsmToolPath for source-build
* Update eng/SourceBuild.props
Co-authored-by: Viktor Hofer <viktor.hofer@microsoft.com>
* Remove libraries specific patches
* Patch updates necessary with latest main merge
* Add back libraries-packages.proj harvesting patch
* Refactor intermediate package split to be chunkier
* Integrate patch 0017
* Subsets update per code review
* Remove obsolete patch
* Removed patches that were integrated into main
* Remove two additional patches
* Remove remaining patches
Co-authored-by: Davis Goodin <dagood@microsoft.com>
Co-authored-by: dseefeld <dseefeld@microsoft.com>
Co-authored-by: Viktor Hofer <viktor.hofer@microsoft.com>
-rw-r--r-- | Directory.Build.props | 1 | ||||
-rw-r--r-- | eng/SourceBuild.props | 73 | ||||
-rw-r--r-- | eng/SourceBuildPrebuiltBaseline.xml | 5 | ||||
-rw-r--r-- | eng/Subsets.props | 21 | ||||
-rw-r--r-- | eng/Version.Details.xml | 1 | ||||
-rw-r--r-- | eng/packaging.props | 4 | ||||
-rw-r--r-- | eng/restore/harvestPackages.targets | 10 | ||||
-rw-r--r-- | src/libraries/Directory.Build.props | 1 | ||||
-rw-r--r-- | src/libraries/libraries-packages.proj | 16 |
9 files changed, 105 insertions, 27 deletions
diff --git a/Directory.Build.props b/Directory.Build.props index d1b03d7ef8d..e2c0de947f3 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -83,6 +83,7 @@ <WasmBuildTasksAssemblyPath>$([MSBuild]::NormalizePath('$(WasmBuildTasksDir)', 'WasmBuildTasks.dll'))</WasmBuildTasksAssemblyPath> <MonoAOTCompilerTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll'))</MonoAOTCompilerTasksAssemblyPath> <RuntimeConfigParserTasksAssemblyPath>$([MSBuild]::NormalizePath('$(RuntimeConfigParserDir)', 'RuntimeConfigParser.dll'))</RuntimeConfigParserTasksAssemblyPath> + <ILAsmToolPath Condition="'$(DotNetBuildFromSource)' == 'true'">$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)'))</ILAsmToolPath> </PropertyGroup> <PropertyGroup Label="CalculateConfiguration"> diff --git a/eng/SourceBuild.props b/eng/SourceBuild.props new file mode 100644 index 00000000000..7dfc80d41fd --- /dev/null +++ b/eng/SourceBuild.props @@ -0,0 +1,73 @@ +<Project> + + <PropertyGroup> + <GitHubRepositoryName>runtime</GitHubRepositoryName> + </PropertyGroup> + + <!-- Set up the dotnet/runtime source-build command. --> + <PropertyGroup> + <BaseInnerSourceBuildCommand>./build.sh</BaseInnerSourceBuildCommand> + + <SourceBuildPortable>true</SourceBuildPortable> + <SourceBuildPortable Condition="'$(SourceBuildNonPortable)' == 'true'">false</SourceBuildPortable> + + <!-- If TargetRid not specified, detect RID based on portability. --> + <TargetRid Condition="'$(TargetRid)' == '' and '$(SourceBuildNonPortable)' == 'true'">$([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier)</TargetRid> + <TargetRid Condition="'$(TargetRid)' == ''">$(__DistroRid)</TargetRid> + + <!-- Split e.g. 'fedora.33-x64' into 'fedora.33' and 'x64'. --> + <_targetRidPlatformIndex>$(TargetRid.LastIndexOfAny("-"))</_targetRidPlatformIndex> + <TargetRidWithoutPlatform>$(TargetRid.Substring(0, $(_targetRidPlatformIndex)))</TargetRidWithoutPlatform> + <TargetRidPlatform>$(TargetRid.Substring($(_targetRidPlatformIndex)).TrimStart('-'))</TargetRidPlatform> + + <LogVerbosity Condition="'$(LogVerbosity)' == ''">minimal</LogVerbosity> + </PropertyGroup> + + <ItemGroup> + <!-- Work around issue where local clone may cause failure using non-origin remote fallback: https://github.com/dotnet/sourcelink/issues/629 --> + <InnerBuildEnv Include="EnableSourceControlManagerQueries=false" /> + <InnerBuildEnv Include="EnableSourceLink=false" /> + <InnerBuildEnv Include="DisableSourceLink=true" /> + <InnerBuildEnv Include="DeterministicSourcePaths=false" /> + </ItemGroup> + + <Target Name="GetRuntimeSourceBuildCommandConfiguration" + BeforeTargets="GetSourceBuildCommandConfiguration"> + <PropertyGroup> + <InnerBuildArgs>$(InnerBuildArgs) --arch $(TargetRidPlatform)</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) --configuration $(Configuration)</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) --ci</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) --allconfigurations</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) --verbosity $(LogVerbosity)</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) --nodereuse false</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) --warnAsError false</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) --cmakeargs -DCLR_CMAKE_USE_SYSTEM_LIBUNWIND=TRUE</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) /p:MicrosoftNetFrameworkReferenceAssembliesVersion=1.0.0</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) /p:ContinuousIntegrationBuild=true</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) /p:PackageRid=$(TargetRid)</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) /p:NoPgoOptimize=true</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) /p:KeepNativeSymbols=true</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) /p:RuntimeOS=$(TargetRidWithoutPlatform)</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) /p:PortableBuild=$(SourceBuildPortable)</InnerBuildArgs> + <InnerBuildArgs>$(InnerBuildArgs) /p:BuildDebPackage=false</InnerBuildArgs> + </PropertyGroup> + </Target> + + <Target Name="CategorizeRuntimeSupplementalArtifacts" + BeforeTargets="GetCategorizedIntermediateNupkgContents"> + <ItemGroup> + <!-- + Runtime artifacts are too large to fit into a single package (Azure DevOps feeds 500 mb constraint). + Split large components into separate packages. + --> + <IntermediateNupkgArtifactFile Include="$(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\dotnet-runtime-*.tar.gz" Category="runtime" /> + + <IntermediateNupkgArtifactFile + Include=" + $(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\dotnet-crossgen2-*.tar.gz; + $(CurrentRepoSourceBuildArtifactsPackagesDir)Shipping\Microsoft.NETCore.App.Crossgen2.*.nupkg" + Category="Crossgen2Pack" /> + </ItemGroup> + </Target> + +</Project> diff --git a/eng/SourceBuildPrebuiltBaseline.xml b/eng/SourceBuildPrebuiltBaseline.xml new file mode 100644 index 00000000000..c1b6dfbf053 --- /dev/null +++ b/eng/SourceBuildPrebuiltBaseline.xml @@ -0,0 +1,5 @@ +<UsageData> + <IgnorePatterns> + <UsagePattern IdentityGlob="*/*" /> + </IgnorePatterns> +</UsageData> diff --git a/eng/Subsets.props b/eng/Subsets.props index bf6b25bffc7..105b34254c6 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -25,8 +25,8 @@ <PropertyGroup> <DefaultSubsets>clr+mono+libs+host+packs</DefaultSubsets> <DefaultSubsets Condition="'$(TargetsMobile)' == 'true'">mono+libs+packs</DefaultSubsets> - <!-- On Windows mono is supported for x86/x64 only --> - <DefaultSubsets Condition="'$(TargetOS)' == 'windows' and '$(TargetArchitecture)' != 'x86' and '$(TargetArchitecture)' != 'x64'">clr+libs+host+packs</DefaultSubsets> + <!-- mono is not supported in source build. On Windows mono is supported for x86/x64 only. --> + <DefaultSubsets Condition="'$(DotNetBuildFromSource)' == 'true' or ('$(TargetOS)' == 'windows' and '$(TargetArchitecture)' != 'x86' and '$(TargetArchitecture)' != 'x64')">clr+libs+host+packs</DefaultSubsets> </PropertyGroup> <!-- Init _subset here in to allow RuntimeFlavor to be set as early as possible --> @@ -49,18 +49,21 @@ <DefaultMonoSubsets Condition="'$(TargetOS)' == 'Browser'">$(DefaultMonoSubsets)mono.wasmruntime+</DefaultMonoSubsets> <DefaultMonoSubsets Condition="'$(MonoCrossAOTTargetOS)' != ''">$(DefaultMonoSubsets)mono.aotcross+</DefaultMonoSubsets> <DefaultMonoSubsets>$(DefaultMonoSubsets)mono.runtime+mono.corelib+mono.packages</DefaultMonoSubsets> - + <DefaultLibrariesSubsets Condition="'$(BuildTargetFramework)' == '$(NetCoreAppCurrent)' or '$(BuildTargetFramework)' == '' or '$(BuildAllConfigurations)' == 'true'">libs.native+</DefaultLibrariesSubsets> - <DefaultLibrariesSubsets>$(DefaultLibrariesSubsets)libs.ref+libs.src+libs.pretest+libs.packages</DefaultLibrariesSubsets> + <DefaultLibrariesSubsets>$(DefaultLibrariesSubsets)libs.ref+libs.src+libs.packages</DefaultLibrariesSubsets> + <DefaultLibrariesSubsets Condition="'$(DotNetBuildFromSource)' != 'true'">$(DefaultLibrariesSubsets)+libs.pretest</DefaultLibrariesSubsets> - <DefaultHostSubsets>host.native+host.pkg+host.tools+host.tests</DefaultHostSubsets> + <DefaultHostSubsets>host.native+host.tools</DefaultHostSubsets> + <DefaultHostSubsets Condition="'$(DotNetBuildFromSource)' != 'true'">$(DefaultHostSubsets)+host.pkg+host.tests</DefaultHostSubsets> <DefaultHostSubsets Condition="'$(RuntimeFlavor)' == 'Mono'"></DefaultHostSubsets> <DefaultHostSubsets Condition="'$(RuntimeFlavor)' == 'Mono' and '$(TargetsMobile)' != 'true'">host.native</DefaultHostSubsets> <DefaultPacksSubsets>packs.product</DefaultPacksSubsets> <DefaultPacksSubsets Condition="'$(BuildMonoAOTCrossCompilerOnly)' != 'true'">$(DefaultPacksSubsets)+packs.tests</DefaultPacksSubsets> + <DefaultPacksSubsets Condition="'$(DotNetBuildFromSource)' == 'true'">$(DefaultPacksSubsets)+packs.installers</DefaultPacksSubsets> </PropertyGroup> <PropertyGroup> @@ -204,12 +207,12 @@ <ProjectToBuild Include="$(CoreClrProjectRoot)tools\runincontext\runincontext.csproj; $(CoreClrProjectRoot)tools\r2rdump\R2RDump.csproj; $(CoreClrProjectRoot)tools\dotnet-pgo\dotnet-pgo.csproj; - $(CoreClrProjectRoot)tools\r2rtest\R2RTest.csproj; - $(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2.csproj" Category="clr" /> + $(CoreClrProjectRoot)tools\r2rtest\R2RTest.csproj" Category="clr" Condition="'$(DotNetBuildFromSource)' != 'true'"/> + <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2.csproj" Category="clr" /> <ProjectToBuild Condition="'$(TargetArchitecture)' != 'x64' and '$(BuildArchitecture)' == 'x64'" Include="$(CoreClrProjectRoot)tools\aot\crossgen2\crossgen2_crossarch.csproj" Category="clr" /> <ProjectToBuild Include="$(CoreClrProjectRoot)tools\aot\ILCompiler.TypeSystem.ReadyToRun.Tests\ILCompiler.TypeSystem.ReadyToRun.Tests.csproj" - Test="true" Category="clr" Condition="'$(__DistroRid)' != 'linux-musl-x64'"/> + Test="true" Category="clr" Condition="'$(__DistroRid)' != 'linux-musl-x64' and '$(DotNetBuildFromSource)' != 'true'"/> </ItemGroup> <ItemGroup Condition="$(_subset.Contains('+clr.nativecorelib+'))"> @@ -221,7 +224,7 @@ <ItemGroup Condition="$(_subset.Contains('+clr.packages+')) and '$(PgoInstrument)' != 'true'"> <ProjectToBuild Include="$(CoreClrProjectRoot).nuget\coreclr-packages.proj" Pack="true" Category="clr" /> - <ProjectToBuild Include="$(CoreClrProjectRoot)tools\dotnet-pgo\dotnet-pgo-pack.proj" Pack="true" BuildInParallel="false" Category="clr" /> + <ProjectToBuild Include="$(CoreClrProjectRoot)tools\dotnet-pgo\dotnet-pgo-pack.proj" Pack="true" BuildInParallel="false" Category="clr" Condition="'$(DotNetBuildFromSource)' != 'true'"/> </ItemGroup> <!-- Mono sets --> diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f1ef354b76c..4177e280edf 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -9,6 +9,7 @@ <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21311.3"> <Uri>https://github.com/dotnet/arcade</Uri> <Sha>4a2b475948d498b89fedef7cf890883f49bc1ea3</Sha> + <SourceBuild RepoName="arcade" ManagedOnly="true" /> </Dependency> <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21311.3"> <Uri>https://github.com/dotnet/arcade</Uri> diff --git a/eng/packaging.props b/eng/packaging.props index 87cecef871a..534d3ddcf32 100644 --- a/eng/packaging.props +++ b/eng/packaging.props @@ -16,10 +16,6 @@ </PropertyGroup> <PropertyGroup Condition="'$(DotNetBuildFromSource)' == 'true'"> - <!-- Don't build referenced projects --> - <BuildPackageLibraryReferences>false</BuildPackageLibraryReferences> - <!-- Omit any files that were not built --> - <AllowPartialPackages>true</AllowPartialPackages> <!-- Don't permit harvesting since this requires pre-builts --> <HarvestStablePackage>false</HarvestStablePackage> <!-- Validation will fail in case we were relying on harvested assets or assets not built to satisfy stated support --> diff --git a/eng/restore/harvestPackages.targets b/eng/restore/harvestPackages.targets index a7fc8aa3465..07eb5a91a7a 100644 --- a/eng/restore/harvestPackages.targets +++ b/eng/restore/harvestPackages.targets @@ -1,4 +1,14 @@ <Project InitialTargets="AddPackageDownload"> + + <PropertyGroup> + <PackagingTaskAssembly>$(NuGetPackageRoot)microsoft.dotnet.build.tasks.packaging\$(MicrosoftDotNetBuildTasksPackagingVersion)\tools\</PackagingTaskAssembly> + <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' == 'core'">$(PackagingTaskAssembly)netcoreapp3.1\</PackagingTaskAssembly> + <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' != 'core'">$(PackagingTaskAssembly)net472\</PackagingTaskAssembly> + <!-- TODO: Remove when https://github.com/dotnet/arcade/issues/7413 is fixed. --> + <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' == 'core' and '$(DotNetBuildFromSource)' == 'true'">$(PackagingTaskAssembly)net5.0\</PackagingTaskAssembly> + <PackagingTaskAssembly>$(PackagingTaskAssembly)Microsoft.DotNet.Build.Tasks.Packaging.dll</PackagingTaskAssembly> + </PropertyGroup> + <UsingTask TaskName="GetLastStablePackage" AssemblyFile="$(PackagingTaskAssembly)"/> <Target Name="AddPackageDownload"> <ItemGroup> diff --git a/src/libraries/Directory.Build.props b/src/libraries/Directory.Build.props index b06fc950644..0078c1331d3 100644 --- a/src/libraries/Directory.Build.props +++ b/src/libraries/Directory.Build.props @@ -28,7 +28,6 @@ <PropertyGroup> <!-- Build all .NET Framework configurations when net48 is passed in. This is for convenience. --> <AdditionalBuildTargetFrameworks Condition="'$(BuildTargetFramework)' == 'net48'">net461;net462;net47;net471;net472</AdditionalBuildTargetFrameworks> - <AdditionalBuildTargetFrameworks Condition="'$(DotNetBuildFromSource)' == 'true' and '$(BuildAllProjects)' == 'true'">$(AdditionalBuildTargetFrameworks);netstandard2.0</AdditionalBuildTargetFrameworks> </PropertyGroup> <!-- Define test projects and companions --> diff --git a/src/libraries/libraries-packages.proj b/src/libraries/libraries-packages.proj index 2f4d0949db3..383dd088e12 100644 --- a/src/libraries/libraries-packages.proj +++ b/src/libraries/libraries-packages.proj @@ -1,20 +1,11 @@ <Project Sdk="Microsoft.Build.Traversal" DefaultTargets="Pack"> - <PropertyGroup> - <AdditionalBuildTargetFrameworks Condition="'$(DotNetBuildFromSource)' == 'true'">$(AdditionalBuildTargetFrameworks);package-$(Configuration)</AdditionalBuildTargetFrameworks> - </PropertyGroup> - - <PropertyGroup> - <PackagingTaskAssembly>$(NuGetPackageRoot)microsoft.dotnet.build.tasks.packaging\$(MicrosoftDotNetBuildTasksPackagingVersion)\tools\</PackagingTaskAssembly> - <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' == 'core'">$(PackagingTaskAssembly)netcoreapp3.1\</PackagingTaskAssembly> - <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' != 'core'">$(PackagingTaskAssembly)net472\</PackagingTaskAssembly> - <PackagingTaskAssembly>$(PackagingTaskAssembly)Microsoft.DotNet.Build.Tasks.Packaging.dll</PackagingTaskAssembly> - </PropertyGroup> - <ItemGroup> + <PackageReference Include="Microsoft.DotNet.Build.Tasks.Packaging" Version="$(MicrosoftDotNetBuildTasksPackagingVersion)" /> + <ProjectReference Include="$(PkgDir)*\*.proj" Exclude="$(PkgDir)test\*" /> <ProjectReference Include="$(MSBuildThisFileDirectory)*\src\*.*proj" - Condition="('$(BuildAllConfigurations)' == 'true' or '$(DotNetBuildFromSource)' == 'true')" + Condition="'$(BuildAllConfigurations)' == 'true'" Exclude="$(MSBuildThisFileDirectory)*\src\**\*.shproj"/> </ItemGroup> @@ -31,7 +22,6 @@ ones that might do this. After we ship a stable set of packages this target should be ran and the changes to the package index should be commited to the repo. --> - <UsingTask TaskName="UpdatePackageIndex" AssemblyFile="$(PackagingTaskAssembly)"/> <Target Name="UpdatePackageIndexWithStableVersions" BeforeTargets="Build;Pack" Condition="'$(DotNetFinalVersionKind)' == 'release'"> |