Support for dotnet publish (#4870)

Support for the dotnet publish command. Built on top of changes made by @nattress .
NuGet packages that target .NETCore specifically can be published safely, but most others targeting the .NETFramework cannot.

7 files changed, 305 insertions, 17 deletions

diff --git a/Documentation/how-to-build-and-run-ilcompiler-in-console-shell-prompt.md b/Documentation/how-to-build-and-run-ilcompiler-in-console-shell-prompt.md
index baf53f154..fdbc04d84 100644
--- a/Documentation/how-to-build-and-run-ilcompiler-in-console-shell-prompt.md
+++ b/Documentation/how-to-build-and-run-ilcompiler-in-console-shell-prompt.md
@@ -60,10 +60,10 @@ This approach uses the same code-generator (RyuJIT), as [CoreCLR](https://github
 From the shell/command prompt, issue the following commands, from the folder containing your project, to generate the native executable
 
 ``` 
-    dotnet build /t:LinkNative
+    dotnet publish -r win-x64|linux-x64|osx-x64 
 ``` 
 
-Native executable will be dropped in `./bin/[configuration]/native/` folder and will have the same name as the folder in which your source file is present.
+Native executable will be dropped in `./bin/x64/[configuration]/netcoreapp2.0/publish/` folder and will have the same name as the folder in which your source file is present.
 
 ## Using CPP Code Generator ##
 
@@ -72,11 +72,15 @@ This approach uses [transpiler](https://en.wikipedia.org/wiki/Source-to-source_c
 From the shell/command prompt, issue the following commands to generate the native executable:
 
 ``` 
-    dotnet build /t:LinkNative /p:NativeCodeGen=cpp
+    dotnet publish /p:NativeCodeGen=cpp -r win-x64|linux-x64|osx-x64 
 ```
 
 For CoreRT debug build on Windows, add an extra `/p:AdditionalCppCompilerFlags=/MTd` argument.
 
+## Disabling Native Compilation 
+
+Native compilation can be disabled during publishing by adding an extra `/p:NativeCompilationDuringPublish=false` argument.
+
 ## Workarounds for build errors on Windows ##
 
 If you are seeing errors such as:
diff --git a/src/BuildIntegration/BuildFrameworkNativeObjects.proj b/src/BuildIntegration/BuildFrameworkNativeObjects.proj
index 3a1f2db03..806b087f4 100644
--- a/src/BuildIntegration/BuildFrameworkNativeObjects.proj
+++ b/src/BuildIntegration/BuildFrameworkNativeObjects.proj
@@ -1,7 +1,7 @@
 <Project DefaultTargets="CreateLib" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 
   <PropertyGroup>
-    <IlcCompileDependsOn>BuildOneFrameworkLibrary</IlcCompileDependsOn>
+    <IlcCompileDependsOn>ComputeIlcCompileInputs;BuildOneFrameworkLibrary</IlcCompileDependsOn>
     <CreateLibDependsOn>BuildAllFrameworkLibrariesAsSingleLib</CreateLibDependsOn>
     <IlcMultiModule>true</IlcMultiModule>
     <NativeIntermediateOutputPath Condition="'$(FrameworkObjPath)' != ''">$(FrameworkObjPath)\</NativeIntermediateOutputPath>
@@ -11,12 +11,12 @@
   <Import Project="Microsoft.NETCore.Native.targets" />
 
   <Target Name="BuildAllFrameworkLibraries"
-    Inputs="@(IlcReference)"
-    Outputs="@(IlcReference->'$(NativeIntermediateOutputPath)\%(Filename)$(NativeObjectExt)')">
+    Inputs="@(DefaultFrameworkAssemblies)"
+    Outputs="@(DefaultFrameworkAssemblies->'$(NativeIntermediateOutputPath)\%(Filename)$(NativeObjectExt)')">
     <ItemGroup>
       <ProjectToBuild Include="$(MSBuildProjectFullPath)">
         <AdditionalProperties>
-          LibraryToCompile=%(IlcReference.Identity)
+          LibraryToCompile=%(DefaultFrameworkAssemblies.Identity)
         </AdditionalProperties>
       </ProjectToBuild>
     </ItemGroup>
diff --git a/src/BuildIntegration/Microsoft.NETCore.Native.Publish.targets b/src/BuildIntegration/Microsoft.NETCore.Native.Publish.targets
new file mode 100644
index 000000000..755aa647f
--- /dev/null
+++ b/src/BuildIntegration/Microsoft.NETCore.Native.Publish.targets
@@ -0,0 +1,61 @@
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+
+  <PropertyGroup>
+    <BuildTasksPath Condition="'$(BuildTasksPath)' == ''">$(MSBuildThisFileDirectory)..\tools\ILCompiler.Build.Tasks.dll</BuildTasksPath>
+    
+    <!--
+      Prevent dotnet CLI from deploying the CoreCLR shim executable since we produce
+      a native executable.
+    -->
+    <DeployAppHost>false</DeployAppHost>
+  </PropertyGroup>
+
+  <Target Name="_ComputeIlcCompileInputs"
+          BeforeTargets="ComputeIlcCompileInputs">
+    <ItemGroup>
+      <IlcReference Include="@(_ManagedResolvedAssembliesToPublish)" />
+    </ItemGroup>
+  </Target>
+
+  <!--
+    This target hooks into the dotnet CLI publish pipeline. That pipeline has
+    a target called ComputeFilesToPublish which produces the ItemGroup
+    ResolvedFileToPublish based on the inputs of @(IntermediateAssembly)
+    and @(ResolvedAssembliesToPublish). We modify those two item groups
+    to control what gets published after CoreRT optimizes the application.
+  -->
+  <Target Name="ComputeLinkedFilesToPublish"
+          BeforeTargets="ComputeFilesToPublish"
+          DependsOnTargets="_ComputeAssembliesToCompileToNative;LinkNative">
+    
+    <ItemGroup>
+      <ResolvedAssembliesToPublish Remove="@(_AssembliesToSkipPublish)" />
+      <ResolvedAssembliesToPublish Include="@(_LinkedResolvedAssemblies)" />
+    </ItemGroup>
+    
+    <ItemGroup>
+      <_NativeIntermediateAssembly Include="@(IntermediateAssembly->'$(NativeOutputPath)%(Filename)$(NativeBinaryExt)')" />
+      <IntermediateAssembly Remove="@(IntermediateAssembly)" />
+      <IntermediateAssembly Include="@(_NativeIntermediateAssembly)" />
+    </ItemGroup>
+  </Target>
+  
+  <!--
+    Filter the input publish file list selecting managed assemblies for compilation.
+    Also produces _AssembliesToSkipPublish which chops out things from the publish
+    pipeline we don't want to see in the output (native images, CoreCLR artifacts)
+    until we get a proper AOT NetCore app package.
+  -->
+  <UsingTask TaskName="ComputeManagedAssemblies" AssemblyFile="$(BuildTasksPath)" />
+  <Target Name="_ComputeAssembliesToCompileToNative">
+
+    <ComputeManagedAssemblies Assemblies="@(ResolvedAssembliesToPublish)" 
+    DotNetAppHostExecutableName="$(_DotNetAppHostExecutableName)" DotNetHostFxrLibraryName="$(_DotNetHostFxrLibraryName)" DotNetHostPolicyLibraryName="$(_DotNetHostPolicyLibraryName)"
+    SdkAssemblies="@(PrivateSdkAssemblies)" FrameworkAssemblies="@(FrameworkAssemblies)">
+      <Output TaskParameter="ManagedAssemblies" ItemName="_ManagedResolvedAssembliesToPublish" />
+      <Output TaskParameter="AssembliesToSkipPublish" ItemName="_AssembliesToSkipPublish" />
+    </ComputeManagedAssemblies>
+    
+  </Target>
+
+</Project>
diff --git a/src/BuildIntegration/Microsoft.NETCore.Native.targets b/src/BuildIntegration/Microsoft.NETCore.Native.targets
index 6e522fe48..22dd34fb0 100644
--- a/src/BuildIntegration/Microsoft.NETCore.Native.targets
+++ b/src/BuildIntegration/Microsoft.NETCore.Native.targets
@@ -19,6 +19,7 @@ See the LICENSE file in the project root for more information.
   <PropertyGroup>
     <NativeIntermediateOutputPath Condition="'$(NativeIntermediateOutputPath)' == ''">$(IntermediateOutputPath)native\</NativeIntermediateOutputPath>
     <NativeOutputPath Condition="'$(NativeOutputPath)' == ''">$(OutputPath)native\</NativeOutputPath>
+    <NativeCompilationDuringPublish Condition="'$(NativeCompilationDuringPublish)' == ''">true</NativeCompilationDuringPublish>
     <!-- Workaround for lack of current host OS detection - https://github.com/Microsoft/msbuild/issues/539 -->
     <TargetOS Condition="'$(TargetOS)' == '' and '$(OS)' == 'Unix' and Exists('/Applications')">OSX</TargetOS>
     <TargetOS Condition="'$(TargetOS)' == ''">$(OS)</TargetOS>
@@ -60,20 +61,23 @@ See the LICENSE file in the project root for more information.
     <SharedLibrary Condition="'$(OS)' != 'Windows_NT'">$(FrameworkLibPath)\libframework$(LibFileExt)</SharedLibrary>
   </PropertyGroup>
 
-  <ItemGroup Condition="$(BuildingFrameworkLibrary) != 'true'">
-    <ManagedBinary Include="$(IntermediateOutputPath)$(TargetName)$(TargetExt)" />
-  </ItemGroup>
+  <PropertyGroup Condition="'$(IlcCompileDependsOn)'=='' and '$(NativeCompilationDuringPublish)' != 'false'">
+    <IlcCompileDependsOn Condition="'$(BuildingFrameworkLibrary)' != 'true'">Compile;ComputeIlcCompileInputs</IlcCompileDependsOn>
+    <IlcCompileDependsOn Condition="'$(IlcMultiModule)' == 'true' and '$(BuildingFrameworkLibrary)' != 'true'">$(IlcCompileDependsOn);BuildFrameworkLib</IlcCompileDependsOn>
+  </PropertyGroup>
 
   <ItemGroup>
-    <IlcCompileInput Include="@(ManagedBinary)" />
-    <IlcReference Include="$(IlcPath)\sdk\*.dll" />
-    <IlcReference Include="$(IlcPath)\framework\*.dll" />
+    <DefaultFrameworkAssemblies Include="$(IlcPath)\sdk\*.dll" />
+    <DefaultFrameworkAssemblies Include="$(IlcPath)\framework\*.dll" />
   </ItemGroup>
 
-  <PropertyGroup Condition="'$(IlcCompileDependsOn)'==''">
-    <IlcCompileDependsOn Condition="'$(BuildingFrameworkLibrary)' != 'true'">Compile</IlcCompileDependsOn>
-    <IlcCompileDependsOn Condition="'$(IlcMultiModule)' == 'true' and '$(BuildingFrameworkLibrary)' != 'true'">$(IlcCompileDependsOn);BuildFrameworkLib</IlcCompileDependsOn>
-  </PropertyGroup>
+  <Target Name="ComputeIlcCompileInputs">
+    <ItemGroup>
+      <ManagedBinary Condition="$(BuildingFrameworkLibrary) != 'true'" Include="$(IntermediateOutputPath)$(TargetName)$(TargetExt)" />
+      <IlcCompileInput Include="@(ManagedBinary)" />
+      <IlcReference Include="@(DefaultFrameworkAssemblies)" />
+    </ItemGroup>
+  </Target>
 
   <!--
     BuildFrameworkLib is invoked before IlcCompile in multi-module builds to 
@@ -107,6 +111,7 @@ See the LICENSE file in the project root for more information.
       <IlcArg Condition="$(Optimize) == 'true'" Include="-O" />
       <IlcArg Condition="$(DebugSymbols) == 'true'" Include="-g" />
       <IlcArg Condition="$(IlcGenerateMapFile) == 'true'" Include="--map:$(NativeIntermediateOutputPath)%(ManagedBinary.Filename).map.xml" />
+      <IlcArg Condition="$(RdXmlFile) != ''" Include="--rdxml:$(RdXmlFile)" />
     </ItemGroup>
 
     <MakeDir Directories="$(NativeIntermediateOutputPath)" />
@@ -178,4 +183,7 @@ See the LICENSE file in the project root for more information.
     <Exec Command="$(CppLibCreator) @&quot;$(NativeIntermediateOutputPath)lib.rsp&quot;" Condition="'$(OS)' == 'Windows_NT'" />
     <Exec Command="$(CppLibCreator) @(CustomLibArg, ' ')" Condition="'$(OS)' != 'Windows_NT'" />
   </Target>
+
+  <Import Project="$(MSBuildThisFileDirectory)\Microsoft.NETCore.Native.Publish.targets" Condition="'$(NativeCompilationDuringPublish)' != 'false'" />
+
 </Project>
diff --git a/src/ILCompiler.Build.Tasks/src/ComputeManagedAssemblies.cs b/src/ILCompiler.Build.Tasks/src/ComputeManagedAssemblies.cs
new file mode 100644
index 000000000..0e4998ece
--- /dev/null
+++ b/src/ILCompiler.Build.Tasks/src/ComputeManagedAssemblies.cs
@@ -0,0 +1,168 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection.Metadata;
+using System.Reflection.PortableExecutable;
+
+
+
+namespace Build.Tasks
+{
+    public class ComputeManagedAssemblies : Task
+    {
+        [Required]
+        public ITaskItem[] Assemblies
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// The CoreRT-specific System.Private.* assemblies that must be used instead of the netcoreapp2.0 versions.
+        /// </summary>
+        [Required]
+        public ITaskItem[] SdkAssemblies
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// The set of AOT-specific framework assemblies we currently need to use which will replace the same-named ones
+        /// in the app's closure.
+        /// </summary>
+        [Required]
+        public ITaskItem[] FrameworkAssemblies
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// The native apphost (whose name ends up colliding with the CoreRT output binary) 
+        /// </summary>
+        [Required]
+        public string DotNetAppHostExecutableName
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// The CoreCLR dotnet host fixer library that can be skipped during publish
+        /// </summary>
+        [Required]
+        public string DotNetHostFxrLibraryName
+        {
+            get;
+            set;
+        }
+
+        /// <summary>
+        /// The CoreCLR dotnet host policy library that can be skipped during publish
+        /// </summary>
+        [Required]
+        public string DotNetHostPolicyLibraryName
+        {
+            get;
+            set;
+        }
+
+        [Output]
+        public ITaskItem[] ManagedAssemblies
+        {
+            get;
+            set;
+        }
+
+        [Output]
+        public ITaskItem[] AssembliesToSkipPublish
+        {
+            get;
+            set;
+        }
+
+        public override bool Execute()
+        {
+            var list = new List<ITaskItem>();
+            var assembliesToSkipPublish = new List<ITaskItem>();
+
+            var coreRTFrameworkAssembliesToUse = new HashSet<string>();
+
+            foreach (ITaskItem taskItem in SdkAssemblies)
+            {
+                coreRTFrameworkAssembliesToUse.Add(Path.GetFileName(taskItem.ItemSpec));
+            }
+
+            foreach (ITaskItem taskItem in FrameworkAssemblies)
+            {
+                coreRTFrameworkAssembliesToUse.Add(Path.GetFileName(taskItem.ItemSpec));
+            }
+
+            foreach (ITaskItem taskItem in Assemblies)
+            {
+                // In the case of disk-based assemblies, this holds the file path
+                string itemSpec = taskItem.ItemSpec;
+
+                // Skip the native apphost (whose name ends up colliding with the CoreRT output binary) and supporting libraries
+                if (itemSpec.EndsWith(DotNetAppHostExecutableName, StringComparison.OrdinalIgnoreCase) || itemSpec.Contains(DotNetHostFxrLibraryName) || itemSpec.Contains(DotNetHostPolicyLibraryName))
+                {
+                    assembliesToSkipPublish.Add(taskItem);
+                    continue;
+                }
+
+                // Prototype aid - remove the native CoreCLR runtime pieces from the publish folder
+                if (itemSpec.Contains("microsoft.netcore.app") && (itemSpec.Contains("\\native\\") || itemSpec.Contains("/native/")))
+                {
+                    assembliesToSkipPublish.Add(taskItem);
+                    continue;
+                }
+
+                // Remove any assemblies whose implementation we want to come from CoreRT's package.
+                // Currently that's System.Private.* SDK assemblies and a bunch of framework assemblies.
+                if (coreRTFrameworkAssembliesToUse.Contains(Path.GetFileName(itemSpec)))
+                {
+                    assembliesToSkipPublish.Add(taskItem);
+                    continue;
+                }
+
+                try
+                {
+                    using (FileStream moduleStream = File.OpenRead(itemSpec))
+                    using (var module = new PEReader(moduleStream))
+                    {
+                        if (module.HasMetadata)
+                        {
+                            MetadataReader moduleMetadataReader = module.GetMetadataReader();
+                            if (moduleMetadataReader.IsAssembly)
+                            {
+                                string culture = moduleMetadataReader.GetString(moduleMetadataReader.GetAssemblyDefinition().Culture);
+
+                                if (culture == "" || culture.Equals("neutral", StringComparison.OrdinalIgnoreCase))
+                                {
+                                    // CoreRT doesn't consume resource assemblies yet so skip them
+                                    assembliesToSkipPublish.Add(taskItem);
+                                    list.Add(taskItem);
+                                }
+                            }
+                        }
+                    }
+                }
+                catch (BadImageFormatException)
+                {
+                }
+            }
+
+            ManagedAssemblies = list.ToArray();
+            AssembliesToSkipPublish = assembliesToSkipPublish.ToArray();
+
+            return true;
+        }
+    }
+}
diff --git a/src/ILCompiler.Build.Tasks/src/ILCompiler.Build.Tasks.csproj b/src/ILCompiler.Build.Tasks/src/ILCompiler.Build.Tasks.csproj
new file mode 100644
index 000000000..808a4de4c
--- /dev/null
+++ b/src/ILCompiler.Build.Tasks/src/ILCompiler.Build.Tasks.csproj
@@ -0,0 +1,25 @@
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" Condition="'$(IsProjectNLibrary)' != 'true'" />
+  <PropertyGroup>
+    <OutputType>Library</OutputType>
+    <RootNamespace>ILCompiler</RootNamespace>
+    <AssemblyName>ILCompiler.Build.Tasks</AssemblyName>
+    <TargetFramework>netstandard1.3</TargetFramework>
+    <OutputPath>$(BaseOutputPath)$(OSPlatformConfig)/tools</OutputPath>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="ComputeManagedAssemblies.cs" />
+  </ItemGroup>
+  <ItemGroup>
+  <PackageReference Include="Microsoft.Build.Framework">
+    <Version>15.3.409</Version>
+  </PackageReference>
+  <PackageReference Include="Microsoft.Build.Utilities.Core">
+    <Version>15.3.409</Version>
+  </PackageReference>
+  <PackageReference Include="System.Reflection.Metadata">
+    <Version>1.4.2</Version>
+  </PackageReference>
+  </ItemGroup>
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" Condition="'$(IsProjectNLibrary)' != 'true'" />
+</Project>
diff --git a/src/ILCompiler/ILCompiler.sln b/src/ILCompiler/ILCompiler.sln
index 8008033a8..c210088aa 100644
--- a/src/ILCompiler/ILCompiler.sln
+++ b/src/ILCompiler/ILCompiler.sln
@@ -28,6 +28,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ILCompiler.WebAssembly", ".
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "desktop", "desktop\desktop.csproj", "{CE9781B1-0028-4039-A48C-8193F0D28467}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ILCompiler.Build.Tasks", "..\ILCompiler.Build.Tasks\src\ILCompiler.Build.Tasks.csproj", "{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}"
+EndProject
 Global
 	GlobalSection(SharedMSBuildProjectFiles) = preSolution
 		..\System.Private.CoreLib\shared\System.Private.CoreLib.Shared.projitems*{be95c560-b508-4588-8907-f9fc5bc1a0cf}*SharedItemsImports = 4
@@ -253,6 +255,26 @@ Global
 		{CE9781B1-0028-4039-A48C-8193F0D28467}.Release|x64.Build.0 = Release|x64
 		{CE9781B1-0028-4039-A48C-8193F0D28467}.Release|x86.ActiveCfg = Release|Any CPU
 		{CE9781B1-0028-4039-A48C-8193F0D28467}.Release|x86.Build.0 = Release|Any CPU
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Debug|arm.ActiveCfg = Debug|arm
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Debug|arm.Build.0 = Debug|arm
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Debug|arm64.ActiveCfg = Debug|arm64
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Debug|arm64.Build.0 = Debug|arm64
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Debug|x64.ActiveCfg = Debug|x64
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Debug|x64.Build.0 = Debug|x64
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Debug|x86.ActiveCfg = Debug|x86
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Debug|x86.Build.0 = Debug|x86
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Release|arm.ActiveCfg = Release|arm
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Release|arm.Build.0 = Release|arm
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Release|arm64.ActiveCfg = Release|arm64
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Release|arm64.Build.0 = Release|arm64
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Release|x64.ActiveCfg = Release|x64
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Release|x64.Build.0 = Release|x64
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Release|x86.ActiveCfg = Release|x86
+		{44DE7F7B-AD73-40EA-87CC-554F2F57C38A}.Release|x86.Build.0 = Release|x86
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE