Move to using the new Roslyn IIncrementalGenerator API for better in-VS performance (dotnet/runtimelab#1374)

Commit migrated from https://github.com/dotnet/runtimelab/commit/90c617d39a9775e8eb0c5d4aba2f0cc6932bfb87

3 files changed, 243 insertions, 17 deletions

diff --git a/src/libraries/System.Runtime.InteropServices/tests/Ancillary.Interop/Ancillary.Interop.csproj b/src/libraries/System.Runtime.InteropServices/tests/Ancillary.Interop/Ancillary.Interop.csproj
index 7af1a40c40b..e19d9b4e5d9 100644
--- a/src/libraries/System.Runtime.InteropServices/tests/Ancillary.Interop/Ancillary.Interop.csproj
+++ b/src/libraries/System.Runtime.InteropServices/tests/Ancillary.Interop/Ancillary.Interop.csproj
@@ -3,7 +3,6 @@
   <PropertyGroup>
     <AssemblyName>Microsoft.Interop.Ancillary</AssemblyName>
     <TargetFramework>net6.0</TargetFramework>
-    <LangVersion>8.0</LangVersion>
     <RootNamespace>System.Runtime.InteropServices</RootNamespace>
     <Nullable>enable</Nullable>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
diff --git a/src/libraries/System.Runtime.InteropServices/tests/DllImportGenerator.UnitTests/IncrementalGenerationTests.cs b/src/libraries/System.Runtime.InteropServices/tests/DllImportGenerator.UnitTests/IncrementalGenerationTests.cs
new file mode 100644
index 00000000000..e37e6fdd7ab
--- /dev/null
+++ b/src/libraries/System.Runtime.InteropServices/tests/DllImportGenerator.UnitTests/IncrementalGenerationTests.cs
@@ -0,0 +1,203 @@
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Microsoft.CodeAnalysis.Text;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Xunit;
+using static Microsoft.Interop.DllImportGenerator;
+
+namespace DllImportGenerator.UnitTests
+{
+    public class IncrementalGenerationTests
+    {
+        public const string RequiresIncrementalSyntaxTreeModifySupport = "The GeneratorDriver treats all SyntaxTree replace operations on a Compilation as an Add/Remove operation instead of a Modify operation"
+            + ", so all cached results based on that input are thrown out. As a result, we cannot validate that unrelated changes within the same SyntaxTree do not cause regeneration.";
+
+        [Fact]
+        public async Task AddingNewUnrelatedType_DoesNotRegenerateSource()
+        {
+            string source = CodeSnippets.BasicParametersAndModifiers<int>();
+
+            Compilation comp1 = await TestUtils.CreateCompilation(source);
+
+            Microsoft.Interop.DllImportGenerator generator = new();
+            GeneratorDriver driver = TestUtils.CreateDriver(comp1, null, new IIncrementalGenerator[] { generator });
+
+            driver = driver.RunGenerators(comp1);
+
+            generator.IncrementalTracker = new IncrementalityTracker();
+
+            Compilation comp2 = comp1.AddSyntaxTrees(CSharpSyntaxTree.ParseText("struct Foo {}", new CSharpParseOptions(LanguageVersion.Preview)));
+            driver.RunGenerators(comp2);
+
+            Assert.Collection(generator.IncrementalTracker.ExecutedSteps,
+                step =>
+                {
+                    Assert.Equal(IncrementalityTracker.StepName.CalculateStubInformation, step.Step);
+                });
+        }
+
+        [Fact(Skip = RequiresIncrementalSyntaxTreeModifySupport)]
+        public async Task AppendingUnrelatedSource_DoesNotRegenerateSource()
+        {
+            string source = $"namespace NS{{{CodeSnippets.BasicParametersAndModifiers<int>()}}}";
+
+            SyntaxTree syntaxTree = CSharpSyntaxTree.ParseText(source, new CSharpParseOptions(LanguageVersion.Preview));
+
+            Compilation comp1 = await TestUtils.CreateCompilation(new[] { syntaxTree });
+
+            Microsoft.Interop.DllImportGenerator generator = new();
+            GeneratorDriver driver = TestUtils.CreateDriver(comp1, null, new[] { generator });
+
+            driver = driver.RunGenerators(comp1);
+
+            generator.IncrementalTracker = new IncrementalityTracker();
+
+            SyntaxTree newTree = syntaxTree.WithRootAndOptions(syntaxTree.GetCompilationUnitRoot().AddMembers(SyntaxFactory.ParseMemberDeclaration("struct Foo {}")!), syntaxTree.Options);
+
+            Compilation comp2 = comp1.ReplaceSyntaxTree(comp1.SyntaxTrees.First(), newTree);
+            driver.RunGenerators(comp2);
+
+            Assert.Collection(generator.IncrementalTracker.ExecutedSteps,
+                step =>
+                {
+                    Assert.Equal(IncrementalityTracker.StepName.CalculateStubInformation, step.Step);
+                });
+        }
+
+        [Fact]
+        public async Task AddingFileWithNewGeneratedDllImport_DoesNotRegenerateOriginalMethod()
+        {
+            string source = CodeSnippets.BasicParametersAndModifiers<int>();
+
+            Compilation comp1 = await TestUtils.CreateCompilation(source);
+
+            Microsoft.Interop.DllImportGenerator generator = new();
+            GeneratorDriver driver = TestUtils.CreateDriver(comp1, null, new[] { generator });
+
+            driver = driver.RunGenerators(comp1);
+
+            generator.IncrementalTracker = new IncrementalityTracker();
+
+            Compilation comp2 = comp1.AddSyntaxTrees(CSharpSyntaxTree.ParseText(CodeSnippets.BasicParametersAndModifiers<bool>(), new CSharpParseOptions(LanguageVersion.Preview)));
+            driver.RunGenerators(comp2);
+
+            Assert.Equal(2, generator.IncrementalTracker.ExecutedSteps.Count(s => s.Step == IncrementalityTracker.StepName.CalculateStubInformation));
+            Assert.Equal(1, generator.IncrementalTracker.ExecutedSteps.Count(s => s.Step == IncrementalityTracker.StepName.GenerateSingleStub));
+            Assert.Equal(1, generator.IncrementalTracker.ExecutedSteps.Count(s => s.Step == IncrementalityTracker.StepName.NormalizeWhitespace));
+            Assert.Equal(1, generator.IncrementalTracker.ExecutedSteps.Count(s => s.Step == IncrementalityTracker.StepName.ConcatenateStubs));
+            Assert.Equal(1, generator.IncrementalTracker.ExecutedSteps.Count(s => s.Step == IncrementalityTracker.StepName.OutputSourceFile));
+        }
+
+        [Fact]
+        public async Task ReplacingFileWithNewGeneratedDllImport_DoesNotRegenerateStubsInOtherFiles()
+        {
+            string source = CodeSnippets.BasicParametersAndModifiers<int>();
+
+            Compilation comp1 = await TestUtils.CreateCompilation(new string[] { CodeSnippets.BasicParametersAndModifiers<int>(), CodeSnippets.BasicParametersAndModifiers<bool>() });
+
+            Microsoft.Interop.DllImportGenerator generator = new();
+            GeneratorDriver driver = TestUtils.CreateDriver(comp1, null, new[] { generator });
+
+            driver = driver.RunGenerators(comp1);
+
+            generator.IncrementalTracker = new IncrementalityTracker();
+
+            Compilation comp2 = comp1.ReplaceSyntaxTree(comp1.SyntaxTrees.First(), CSharpSyntaxTree.ParseText(CodeSnippets.BasicParametersAndModifiers<ulong>(), new CSharpParseOptions(LanguageVersion.Preview)));
+            driver.RunGenerators(comp2);
+
+            Assert.Equal(2, generator.IncrementalTracker.ExecutedSteps.Count(s => s.Step == IncrementalityTracker.StepName.CalculateStubInformation));
+            Assert.Equal(1, generator.IncrementalTracker.ExecutedSteps.Count(s => s.Step == IncrementalityTracker.StepName.GenerateSingleStub));
+            Assert.Equal(1, generator.IncrementalTracker.ExecutedSteps.Count(s => s.Step == IncrementalityTracker.StepName.NormalizeWhitespace));
+            Assert.Equal(1, generator.IncrementalTracker.ExecutedSteps.Count(s => s.Step == IncrementalityTracker.StepName.ConcatenateStubs));
+            Assert.Equal(1, generator.IncrementalTracker.ExecutedSteps.Count(s => s.Step == IncrementalityTracker.StepName.OutputSourceFile));
+        }
+
+        [Fact]
+        public async Task ChangingMarshallingStrategy_RegeneratesStub()
+        {
+            string stubSource = CodeSnippets.BasicParametersAndModifiers("CustomType");
+
+            string customTypeImpl1 = "struct CustomType { System.IntPtr handle; }";
+
+            string customTypeImpl2 = "class CustomType : Microsoft.Win32.SafeHandles.SafeHandleZeroOrMinusOneIsInvalid { public CustomType():base(true){} protected override bool ReleaseHandle(){return true;} }";
+
+
+            Compilation comp1 = await TestUtils.CreateCompilation(stubSource);
+
+            SyntaxTree customTypeImpl1Tree = CSharpSyntaxTree.ParseText(customTypeImpl1, new CSharpParseOptions(LanguageVersion.Preview));
+            comp1 = comp1.AddSyntaxTrees(customTypeImpl1Tree);
+
+            Microsoft.Interop.DllImportGenerator generator = new();
+            GeneratorDriver driver = TestUtils.CreateDriver(comp1, null, new[] { generator });
+
+            driver = driver.RunGenerators(comp1);
+
+            generator.IncrementalTracker = new IncrementalityTracker();
+
+            Compilation comp2 = comp1.ReplaceSyntaxTree(customTypeImpl1Tree, CSharpSyntaxTree.ParseText(customTypeImpl2, new CSharpParseOptions(LanguageVersion.Preview)));
+            driver.RunGenerators(comp2);
+
+            Assert.Collection(generator.IncrementalTracker.ExecutedSteps,
+                step =>
+                {
+                    Assert.Equal(IncrementalityTracker.StepName.CalculateStubInformation, step.Step);
+                },
+                step =>
+                {
+                    Assert.Equal(IncrementalityTracker.StepName.GenerateSingleStub, step.Step);
+                },
+                step =>
+                {
+                    Assert.Equal(IncrementalityTracker.StepName.NormalizeWhitespace, step.Step);
+                },
+                step =>
+                {
+                    Assert.Equal(IncrementalityTracker.StepName.ConcatenateStubs, step.Step);
+                },
+                step =>
+                {
+                    Assert.Equal(IncrementalityTracker.StepName.OutputSourceFile, step.Step);
+                });
+        }
+
+        [Fact(Skip = RequiresIncrementalSyntaxTreeModifySupport)]
+        public async Task ChangingMarshallingAttributes_SameStrategy_DoesNotRegenerate()
+        {
+            string source = CodeSnippets.BasicParametersAndModifiers<bool>();
+
+            SyntaxTree syntaxTree = CSharpSyntaxTree.ParseText(source, new CSharpParseOptions(LanguageVersion.Preview));
+
+            Compilation comp1 = await TestUtils.CreateCompilation(new[] { syntaxTree });
+
+            Microsoft.Interop.DllImportGenerator generator = new();
+            GeneratorDriver driver = TestUtils.CreateDriver(comp1, null, new[] { generator });
+
+            driver = driver.RunGenerators(comp1);
+
+            generator.IncrementalTracker = new IncrementalityTracker();
+
+            SyntaxTree newTree = syntaxTree.WithRootAndOptions(
+                syntaxTree.GetCompilationUnitRoot().AddMembers(
+                    SyntaxFactory.ParseMemberDeclaration(
+                        CodeSnippets.MarshalAsParametersAndModifiers<bool>(System.Runtime.InteropServices.UnmanagedType.Bool))!),
+                syntaxTree.Options);
+
+            Compilation comp2 = comp1.ReplaceSyntaxTree(comp1.SyntaxTrees.First(), newTree);
+            driver.RunGenerators(comp2);
+
+            Assert.Collection(generator.IncrementalTracker.ExecutedSteps,
+                step =>
+                {
+                    Assert.Equal(IncrementalityTracker.StepName.CalculateStubInformation, step.Step);
+                },
+                step =>
+                {
+                    Assert.Equal(IncrementalityTracker.StepName.GenerateSingleStub, step.Step);
+                });
+        }
+    }
+}
diff --git a/src/libraries/System.Runtime.InteropServices/tests/DllImportGenerator.UnitTests/TestUtils.cs b/src/libraries/System.Runtime.InteropServices/tests/DllImportGenerator.UnitTests/TestUtils.cs
index fd49c82ccf9..db7b0f3fecc 100644
--- a/src/libraries/System.Runtime.InteropServices/tests/DllImportGenerator.UnitTests/TestUtils.cs
+++ b/src/libraries/System.Runtime.InteropServices/tests/DllImportGenerator.UnitTests/TestUtils.cs
@@ -45,14 +45,26 @@ namespace DllImportGenerator.UnitTests
         /// <param name="outputKind">Output type</param>
         /// <param name="allowUnsafe">Whether or not use of the unsafe keyword should be allowed</param>
         /// <returns>The resulting compilation</returns>
-        public static async Task<Compilation> CreateCompilation(string source, OutputKind outputKind = OutputKind.DynamicallyLinkedLibrary, bool allowUnsafe = true, IEnumerable<string>? preprocessorSymbols = null)
+        public static Task<Compilation> CreateCompilation(string source, OutputKind outputKind = OutputKind.DynamicallyLinkedLibrary, bool allowUnsafe = true, IEnumerable<string>? preprocessorSymbols = null)
         {
-            var (mdRefs, ancillary) = GetReferenceAssemblies();
+            return CreateCompilation(new[] { source }, outputKind, allowUnsafe, preprocessorSymbols);
+        }
 
-            return CSharpCompilation.Create("compilation",
-                new[] { CSharpSyntaxTree.ParseText(source, new CSharpParseOptions(LanguageVersion.Preview, preprocessorSymbols: preprocessorSymbols)) },
-                (await mdRefs.ResolveAsync(LanguageNames.CSharp, CancellationToken.None)).Add(ancillary),
-                new CSharpCompilationOptions(outputKind, allowUnsafe: allowUnsafe));
+        /// <summary>
+        /// Create a compilation given sources
+        /// </summary>
+        /// <param name="sources">Sources to compile</param>
+        /// <param name="outputKind">Output type</param>
+        /// <param name="allowUnsafe">Whether or not use of the unsafe keyword should be allowed</param>
+        /// <returns>The resulting compilation</returns>
+        public static Task<Compilation> CreateCompilation(string[] sources, OutputKind outputKind = OutputKind.DynamicallyLinkedLibrary, bool allowUnsafe = true, IEnumerable<string>? preprocessorSymbols = null)
+        {
+            return CreateCompilation(
+                sources.Select(source =>
+                    CSharpSyntaxTree.ParseText(source, new CSharpParseOptions(LanguageVersion.Preview, preprocessorSymbols: preprocessorSymbols))).ToArray(),
+                outputKind,
+                allowUnsafe,
+                preprocessorSymbols);
         }
 
         /// <summary>
@@ -62,13 +74,12 @@ namespace DllImportGenerator.UnitTests
         /// <param name="outputKind">Output type</param>
         /// <param name="allowUnsafe">Whether or not use of the unsafe keyword should be allowed</param>
         /// <returns>The resulting compilation</returns>
-        public static async Task<Compilation> CreateCompilation(string[] sources, OutputKind outputKind = OutputKind.DynamicallyLinkedLibrary, bool allowUnsafe = true, IEnumerable<string>? preprocessorSymbols = null)
+        public static async Task<Compilation> CreateCompilation(SyntaxTree[] sources, OutputKind outputKind = OutputKind.DynamicallyLinkedLibrary, bool allowUnsafe = true, IEnumerable<string>? preprocessorSymbols = null)
         {
             var (mdRefs, ancillary) = GetReferenceAssemblies();
 
             return CSharpCompilation.Create("compilation",
-                sources.Select(source =>
-                    CSharpSyntaxTree.ParseText(source, new CSharpParseOptions(LanguageVersion.Preview, preprocessorSymbols: preprocessorSymbols))).ToArray(),
+                sources,
                 (await mdRefs.ResolveAsync(LanguageNames.CSharp, CancellationToken.None)).Add(ancillary),
                 new CSharpCompilationOptions(outputKind, allowUnsafe: allowUnsafe));
         }
@@ -81,10 +92,23 @@ namespace DllImportGenerator.UnitTests
         /// <param name="outputKind">Output type</param>
         /// <param name="allowUnsafe">Whether or not use of the unsafe keyword should be allowed</param>
         /// <returns>The resulting compilation</returns>
-        public static async Task<Compilation> CreateCompilationWithReferenceAssemblies(string source, ReferenceAssemblies referenceAssemblies, OutputKind outputKind = OutputKind.DynamicallyLinkedLibrary, bool allowUnsafe = true)
+        public static Task<Compilation> CreateCompilationWithReferenceAssemblies(string source, ReferenceAssemblies referenceAssemblies, OutputKind outputKind = OutputKind.DynamicallyLinkedLibrary, bool allowUnsafe = true)
+        {
+            return CreateCompilationWithReferenceAssemblies(new[] { CSharpSyntaxTree.ParseText(source, new CSharpParseOptions(LanguageVersion.Preview)) }, referenceAssemblies, outputKind, allowUnsafe);
+        }
+
+        /// <summary>
+        /// Create a compilation given source and reference assemblies
+        /// </summary>
+        /// <param name="source">Source to compile</param>
+        /// <param name="referenceAssemblies">Reference assemblies to include</param>
+        /// <param name="outputKind">Output type</param>
+        /// <param name="allowUnsafe">Whether or not use of the unsafe keyword should be allowed</param>
+        /// <returns>The resulting compilation</returns>
+        public static async Task<Compilation> CreateCompilationWithReferenceAssemblies(SyntaxTree[] sources, ReferenceAssemblies referenceAssemblies, OutputKind outputKind = OutputKind.DynamicallyLinkedLibrary, bool allowUnsafe = true)
         {
             return CSharpCompilation.Create("compilation",
-                new[] { CSharpSyntaxTree.ParseText(source, new CSharpParseOptions(LanguageVersion.Preview)) },
+                sources,
                 (await referenceAssemblies.ResolveAsync(LanguageNames.CSharp, CancellationToken.None)),
                 new CSharpCompilationOptions(outputKind, allowUnsafe: allowUnsafe));
         }
@@ -96,7 +120,7 @@ namespace DllImportGenerator.UnitTests
                     "net6.0",
                     new PackageIdentity(
                         "Microsoft.NETCore.App.Ref",
-                        "6.0.0-preview.6.21317.4"),
+                        "6.0.0-preview.7.21377.19"),
                     Path.Combine("ref", "net6.0"))
                 .WithNuGetConfigFilePath(Path.Combine(Path.GetDirectoryName(typeof(TestUtils).Assembly.Location)!, "NuGet.config"));
 
@@ -114,7 +138,7 @@ namespace DllImportGenerator.UnitTests
         /// <param name="diagnostics">Resulting diagnostics</param>
         /// <param name="generators">Source generator instances</param>
         /// <returns>The resulting compilation</returns>
-        public static Compilation RunGenerators(Compilation comp, out ImmutableArray<Diagnostic> diagnostics, params ISourceGenerator[] generators)
+        public static Compilation RunGenerators(Compilation comp, out ImmutableArray<Diagnostic> diagnostics, params IIncrementalGenerator[] generators)
         {
             CreateDriver(comp, null, generators).RunGeneratorsAndUpdateCompilation(comp, out var d, out diagnostics);
             return d;
@@ -127,15 +151,15 @@ namespace DllImportGenerator.UnitTests
         /// <param name="diagnostics">Resulting diagnostics</param>
         /// <param name="generators">Source generator instances</param>
         /// <returns>The resulting compilation</returns>
-        public static Compilation RunGenerators(Compilation comp, AnalyzerConfigOptionsProvider options, out ImmutableArray<Diagnostic> diagnostics, params ISourceGenerator[] generators)
+        public static Compilation RunGenerators(Compilation comp, AnalyzerConfigOptionsProvider options, out ImmutableArray<Diagnostic> diagnostics, params IIncrementalGenerator[] generators)
         {
             CreateDriver(comp, options, generators).RunGeneratorsAndUpdateCompilation(comp, out var d, out diagnostics);
             return d;
         }
 
-        private static GeneratorDriver CreateDriver(Compilation c, AnalyzerConfigOptionsProvider? options, ISourceGenerator[] generators)
+        public static GeneratorDriver CreateDriver(Compilation c, AnalyzerConfigOptionsProvider? options, IIncrementalGenerator[] generators)
             => CSharpGeneratorDriver.Create(
-                ImmutableArray.Create(generators),
+                ImmutableArray.Create(generators.Select(gen => gen.AsSourceGenerator()).ToArray()),
                 parseOptions: (CSharpParseOptions)c.SyntaxTrees.First().Options,
                 optionsProvider: options);
     }