Portable IL PDB support

Add support for reading portable IL PDB using System.Reflection.Metadata.dll. We will try to open the IL PDB file using the portable pdb reader first, and fallback to the unmanaged diasymreader for non-portable pdbs.

Fixes #49

8 files changed, 264 insertions, 73 deletions

diff --git a/src/ILCompiler.Compiler/src/Compiler/CompilerTypeSystemContext.cs b/src/ILCompiler.Compiler/src/Compiler/CompilerTypeSystemContext.cs
index 8a86e08f0..fb7bd104b 100644
--- a/src/ILCompiler.Compiler/src/Compiler/CompilerTypeSystemContext.cs
+++ b/src/ILCompiler.Compiler/src/Compiler/CompilerTypeSystemContext.cs
@@ -9,11 +9,11 @@ using System.Collections.Generic;
 using System.Reflection.Metadata;
 using System.Reflection.Metadata.Ecma335;
 using System.Reflection.PortableExecutable;
-using System.Runtime.InteropServices;
+
+using ILCompiler.SymbolReader;
 
 using Internal.TypeSystem;
 using Internal.TypeSystem.Ecma;
-using Internal.IL;
 
 namespace ILCompiler
 {
@@ -68,7 +68,7 @@ namespace ILCompiler
             public EcmaModule Module;
             public MemoryMappedViewAccessor MappedViewAccessor;
 
-            public Microsoft.DiaSymReader.ISymUnmanagedReader PdbReader;
+            public PdbSymbolReader PdbReader;
         }
 
         private class ModuleHashtable : LockFreeReaderHashtable<EcmaModule, ModuleData>
@@ -297,7 +297,7 @@ namespace ILCompiler
             return _metadataRuntimeInterfacesAlgorithm;
         }
 
-        MetadataStringDecoder IMetadataStringDecoderProvider.GetMetadataStringDecoder()
+        public MetadataStringDecoder GetMetadataStringDecoder()
         {
             if (_metadataStringDecoder == null)
                 _metadataStringDecoder = new CachingMetadataStringDecoder(0x10000); // TODO: Tune the size
@@ -308,14 +308,23 @@ namespace ILCompiler
         // Symbols
         //
 
-        private PdbSymbolProvider _pdbSymbolProvider;
-
         private void InitializeSymbolReader(ModuleData moduleData)
         {
-            if (_pdbSymbolProvider == null)
-                _pdbSymbolProvider = new PdbSymbolProvider();
+            // Assume that the .pdb file is next to the binary
+            var pdbFilename = Path.ChangeExtension(moduleData.FilePath, ".pdb");
+
+            if (!File.Exists(pdbFilename))
+                return;
+
+            // Try to open the symbol file as portable pdb first
+            PdbSymbolReader reader = PortablePdbSymbolReader.TryOpen(pdbFilename, GetMetadataStringDecoder());
+            if (reader == null)
+            {
+                // Fallback to the diasymreader for non-portable pdbs
+                reader = UnmanagedPdbSymbolReader.TryOpenSymbolReaderForMetadataFile(moduleData.FilePath);
+            }
 
-            moduleData.PdbReader = _pdbSymbolProvider.GetSymbolReaderForFile(moduleData.FilePath);
+            moduleData.PdbReader = reader;
         }
 
         public IEnumerable<ILSequencePoint> GetSequencePointsForMethod(MethodDesc method)
@@ -331,7 +340,7 @@ namespace ILCompiler
             if (moduleData.PdbReader == null)
                 return null;
 
-            return _pdbSymbolProvider.GetSequencePointsForMethod(moduleData.PdbReader, MetadataTokens.GetToken(ecmaMethod.Handle));
+            return moduleData.PdbReader.GetSequencePointsForMethod(MetadataTokens.GetToken(ecmaMethod.Handle));
         }
 
         public IEnumerable<ILLocalVariable> GetLocalVariableNamesForMethod(MethodDesc method)
@@ -347,7 +356,7 @@ namespace ILCompiler
             if (moduleData.PdbReader == null)
                 return null;
 
-            return _pdbSymbolProvider.GetLocalVariableNamesForMethod(moduleData.PdbReader, MetadataTokens.GetToken(ecmaMethod.Handle));
+            return moduleData.PdbReader.GetLocalVariableNamesForMethod(MetadataTokens.GetToken(ecmaMethod.Handle));
         }
 
         public IEnumerable<string> GetParameterNamesForMethod(MethodDesc method)
diff --git a/src/ILCompiler.Compiler/src/Compiler/SymbolReader/PdbSymbolReader.cs b/src/ILCompiler.Compiler/src/Compiler/SymbolReader/PdbSymbolReader.cs
new file mode 100644
index 000000000..d54d09fd8
--- /dev/null
+++ b/src/ILCompiler.Compiler/src/Compiler/SymbolReader/PdbSymbolReader.cs
@@ -0,0 +1,39 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+
+namespace ILCompiler.SymbolReader
+{
+    /// <summary>
+    ///  IL sequence point record
+    /// </summary>
+    public struct ILSequencePoint
+    {
+        public int Offset;
+        public string Document;
+        public int LineNumber;
+        // TODO: The remaining info
+    }
+
+    /// <summary>
+    ///  IL local variable debug record
+    /// </summary>
+    public struct ILLocalVariable
+    {
+        public int Slot;
+        public string Name;
+        public bool CompilerGenerated;
+    }
+
+    /// <summary>
+    /// Abstraction for reading Pdb files
+    /// </summary>
+    public abstract class PdbSymbolReader : IDisposable
+    {
+        public abstract IEnumerable<ILSequencePoint> GetSequencePointsForMethod(int methodToken);
+        public abstract IEnumerable<ILLocalVariable> GetLocalVariableNamesForMethod(int methodToken);
+        public abstract void Dispose();
+    }
+}
diff --git a/src/ILCompiler.Compiler/src/Compiler/SymbolReader/PortablePdbSymbolReader.cs b/src/ILCompiler.Compiler/src/Compiler/SymbolReader/PortablePdbSymbolReader.cs
new file mode 100644
index 000000000..70d19416d
--- /dev/null
+++ b/src/ILCompiler.Compiler/src/Compiler/SymbolReader/PortablePdbSymbolReader.cs
@@ -0,0 +1,145 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.IO.MemoryMappedFiles;
+using System.Reflection.Metadata;
+using System.Reflection.Metadata.Ecma335;
+
+namespace ILCompiler.SymbolReader
+{
+    /// <summary>
+    ///  Provides PdbSymbolReader for portable PDB via System.Reflection.Metadata
+    /// </summary>
+    internal class PortablePdbSymbolReader : PdbSymbolReader
+    {
+        private static unsafe MetadataReader TryOpenMetadataFile(string filePath, MetadataStringDecoder stringDecoder, out MemoryMappedViewAccessor mappedViewAccessor)
+        {
+            MemoryMappedFile mappedFile = null;
+            MemoryMappedViewAccessor accessor = null;
+
+            try
+            {
+                mappedFile = MemoryMappedFile.CreateFromFile(filePath, FileMode.Open, null, 0, MemoryMappedFileAccess.Read);
+                accessor = mappedFile.CreateViewAccessor(0, 0, MemoryMappedFileAccess.Read);
+
+                var safeBuffer = accessor.SafeMemoryMappedViewHandle;
+
+                // Check whether this is a real metadata file to avoid thrown and caught exceptions
+                // for non-portable .pdbs
+                if (safeBuffer.Read<byte>(0) != 'B' || // COR20MetadataSignature
+                    safeBuffer.Read<byte>(1) != 'S' ||
+                    safeBuffer.Read<byte>(2) != 'J' ||
+                    safeBuffer.Read<byte>(3) != 'B')
+                {
+                    mappedViewAccessor = null;
+                    return null;
+                }
+
+                var metadataReader = new MetadataReader((byte*)safeBuffer.DangerousGetHandle(), (int)safeBuffer.ByteLength, MetadataReaderOptions.Default, stringDecoder);
+
+                // MemoryMappedFile does not need to be kept around. MemoryMappedViewAccessor is enough.
+
+                mappedViewAccessor = accessor;
+                accessor = null;
+
+                return metadataReader;
+            }
+            finally
+            {
+                if (accessor != null)
+                    accessor.Dispose();
+                if (mappedFile != null)
+                    mappedFile.Dispose();
+            }
+        }
+
+        public static PdbSymbolReader TryOpen(string pdbFilename, MetadataStringDecoder stringDecoder)
+        {
+            MemoryMappedViewAccessor mappedViewAccessor;
+            MetadataReader reader = TryOpenMetadataFile(pdbFilename, stringDecoder, out mappedViewAccessor);
+            if (reader == null)
+                return null;
+
+            return new PortablePdbSymbolReader(reader, mappedViewAccessor);
+        }
+
+        private MetadataReader _reader;
+        private MemoryMappedViewAccessor _mappedViewAccessor;
+
+        private PortablePdbSymbolReader(MetadataReader reader, MemoryMappedViewAccessor mappedViewAccessor)
+        {
+            _reader = reader;
+            _mappedViewAccessor = mappedViewAccessor;
+        }
+
+        public override void Dispose()
+        {
+            _mappedViewAccessor.Dispose();
+        }
+
+        public override IEnumerable<ILSequencePoint> GetSequencePointsForMethod(int methodToken)
+        {
+            var debugInformationHandle = ((MethodDefinitionHandle)MetadataTokens.EntityHandle(methodToken)).ToDebugInformationHandle();
+
+            var debugInformation = _reader.GetMethodDebugInformation(debugInformationHandle);
+
+            var sequencePoints = debugInformation.GetSequencePoints();
+
+            foreach (var sequencePoint in sequencePoints)
+            {
+                if (sequencePoint.StartLine == 0xFEEFEE)
+                    continue;
+
+                var url = _reader.GetString(_reader.GetDocument(sequencePoint.Document).Name);
+
+                yield return new ILSequencePoint() { Document = url, LineNumber = sequencePoint.StartLine, Offset = sequencePoint.Offset };
+            }
+        }
+
+        //
+        // Gather the local details in a scope and then recurse to child scopes
+        //
+        private void ProbeScopeForLocals(List<ILLocalVariable> variables, LocalScopeHandle localScopeHandle)
+        {
+            var localScope = _reader.GetLocalScope(localScopeHandle);
+
+            foreach (var localVariableHandle in localScope.GetLocalVariables())
+            {
+                var localVariable = _reader.GetLocalVariable(localVariableHandle);
+
+                var name = _reader.GetString(localVariable.Name);
+                bool compilerGenerated = (localVariable.Attributes & LocalVariableAttributes.DebuggerHidden) != 0;
+
+                variables.Add(new ILLocalVariable() { Slot = localVariable.Index, Name = name, CompilerGenerated = compilerGenerated });
+            }
+
+            var children = localScope.GetChildren();
+            while (children.MoveNext())
+            {
+                ProbeScopeForLocals(variables, children.Current);
+            }
+        }
+
+        //
+        // Recursively scan the scopes for a method stored in a PDB and gather the local slots
+        // and names for all of them.  This assumes a CSC-like compiler that doesn't re-use
+        // local slots in the same method across scopes.
+        //
+        public override IEnumerable<ILLocalVariable> GetLocalVariableNamesForMethod(int methodToken)
+        {
+            var debugInformationHandle = MetadataTokens.MethodDefinitionHandle(methodToken).ToDebugInformationHandle();
+
+            var localScopes = _reader.GetLocalScopes(debugInformationHandle);
+
+            var variables = new List<ILLocalVariable>();
+            foreach (var localScopeHandle in localScopes)
+            {
+                ProbeScopeForLocals(variables, localScopeHandle);
+            }
+            return variables;
+        }
+    }
+}
diff --git a/src/ILCompiler.Compiler/src/Compiler/PdbSymbolProvider.cs b/src/ILCompiler.Compiler/src/Compiler/SymbolReader/UnmanagedPdbSymbolReader.cs
index 0220d99a4..b4e41db9d 100644
--- a/src/ILCompiler.Compiler/src/Compiler/PdbSymbolProvider.cs
+++ b/src/ILCompiler.Compiler/src/Compiler/SymbolReader/UnmanagedPdbSymbolReader.cs
@@ -3,18 +3,16 @@
 
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Runtime.InteropServices;
 
-using Internal.IL;
-
 using Microsoft.DiaSymReader;
 
-namespace ILCompiler
+namespace ILCompiler.SymbolReader
 {
-    // For now, open PDB files using legacy desktop SymBinder
-
-    internal class PdbSymbolProvider
+    /// <summary>
+    ///  Provides PdbSymbolReader via unmanaged SymBinder from full .NET Framework
+    /// </summary>
+    internal sealed class UnmanagedPdbSymbolReader : PdbSymbolReader
     {
         [Guid("809c652e-7396-11d2-9771-00a0c9b4d50c")]
         [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
@@ -60,11 +58,7 @@ namespace ILCompiler
             Marshal.ThrowExceptionForHR(hr, new IntPtr(-1));
         }
 
-        private IMetaDataDispenser _metadataDispenser;
-
-        private ISymUnmanagedBinder _symBinder;
-
-        public PdbSymbolProvider()
+        static UnmanagedPdbSymbolReader()
         {
             try
             {
@@ -74,32 +68,29 @@ namespace ILCompiler
                 object objDispenser;
                 if (MetaDataGetDispenser(ref dispenserClassID, ref dispenserIID, out objDispenser) < 0)
                     return;
-                _metadataDispenser = (IMetaDataDispenser)objDispenser;
+                s_metadataDispenser = (IMetaDataDispenser)objDispenser;
 
                 Guid symBinderClassID = new Guid(0x0A29FF9E, 0x7F9C, 0x4437, 0x8B, 0x11, 0xF4, 0x24, 0x49, 0x1E, 0x39, 0x31); // CLSID_CorSymBinder
                 Guid symBinderIID = new Guid(0xAA544d42, 0x28CB, 0x11d3, 0xbd, 0x22, 0x00, 0x00, 0xf8, 0x08, 0x49, 0xbd); // IID_ISymUnmanagedBinder
                 object objBinder;
                 if (CoCreateInstance(ref symBinderClassID,
-                                 IntPtr.Zero, // pUnkOuter
-                                 1, // CLSCTX_INPROC_SERVER
-                                 ref symBinderIID,
-                                 out objBinder) < 0)
+                                     IntPtr.Zero, // pUnkOuter
+                                     1, // CLSCTX_INPROC_SERVER
+                                     ref symBinderIID,
+                                     out objBinder) < 0)
                     return;
-                _symBinder = (ISymUnmanagedBinder)objBinder;
+                s_symBinder = (ISymUnmanagedBinder)objBinder;
             }
             catch
             {
             }
         }
 
-        public ISymUnmanagedReader GetSymbolReaderForFile(string metadataFileName)
-        {
-            if (!File.Exists(Path.ChangeExtension(metadataFileName, ".pdb")))
-                return null;
-
-            if (_metadataDispenser == null || _symBinder == null)
-                return null;
+        private static IMetaDataDispenser s_metadataDispenser;
+        private static ISymUnmanagedBinder s_symBinder;
 
+        public static PdbSymbolReader TryOpenSymbolReaderForMetadataFile(string metadataFileName)
+        {
             try
             {
                 Guid importerIID = new Guid(0x7dac8207, 0xd3ae, 0x4c75, 0x9b, 0x67, 0x92, 0x80, 0x1a, 0x49, 0x7d, 0x44); // IID_IMetaDataImport
@@ -107,13 +98,14 @@ namespace ILCompiler
                 // Open an metadata importer on the given filename. We'll end up passing this importer straight
                 // through to the Binder.
                 object objImporter;
-                if (_metadataDispenser.OpenScope(metadataFileName, 0x00000010 /* read only */, ref importerIID, out objImporter) < 0)
+                if (s_metadataDispenser.OpenScope(metadataFileName, 0x00000010 /* read only */, ref importerIID, out objImporter) < 0)
                     return null;
 
                 ISymUnmanagedReader reader;
-                if (_symBinder.GetReaderForFile(objImporter, metadataFileName, "", out reader) < 0)
+                if (s_symBinder.GetReaderForFile(objImporter, metadataFileName, "", out reader) < 0)
                     return null;
-                return reader;
+
+                return new UnmanagedPdbSymbolReader(reader);
             }
             catch
             {
@@ -121,30 +113,48 @@ namespace ILCompiler
             }
         }
 
-        private Dictionary<ISymUnmanagedDocument, string> _urlCache = new Dictionary<ISymUnmanagedDocument, string>();
+        private ISymUnmanagedReader _symUnmanagedReader;
+
+        private UnmanagedPdbSymbolReader(ISymUnmanagedReader symUnmanagedReader)
+        {
+            _symUnmanagedReader = symUnmanagedReader;
+        }
+
+        public override void Dispose()
+        {
+            Marshal.ReleaseComObject(_symUnmanagedReader);
+        }
+
+        private Dictionary<ISymUnmanagedDocument, string> _urlCache;
 
         private string GetUrl(ISymUnmanagedDocument doc)
         {
-            string url;
-            if (_urlCache.TryGetValue(doc, out url))
-                return url;
+            lock (this)
+            {
+                if (_urlCache == null)
+                    _urlCache = new Dictionary<ISymUnmanagedDocument, string>();
+
+                string url;
+                if (_urlCache.TryGetValue(doc, out url))
+                    return url;
 
-            int urlLength;
-            ThrowExceptionForHR(doc.GetUrl(0, out urlLength, null));
+                int urlLength;
+                ThrowExceptionForHR(doc.GetUrl(0, out urlLength, null));
 
-            // urlLength includes terminating '\0'
-            char[] urlBuffer = new char[urlLength];
-            ThrowExceptionForHR(doc.GetUrl(urlLength, out urlLength, urlBuffer));
+                // urlLength includes terminating '\0'
+                char[] urlBuffer = new char[urlLength];
+                ThrowExceptionForHR(doc.GetUrl(urlLength, out urlLength, urlBuffer));
 
-            url = new string(urlBuffer, 0, urlLength - 1);
-            _urlCache.Add(doc, url);
-            return url;
+                url = new string(urlBuffer, 0, urlLength - 1);
+                _urlCache.Add(doc, url);
+                return url;
+            }
         }
 
-        public IEnumerable<ILSequencePoint> GetSequencePointsForMethod(ISymUnmanagedReader reader, int methodToken)
+        public override IEnumerable<ILSequencePoint> GetSequencePointsForMethod(int methodToken)
         {
             ISymUnmanagedMethod symbolMethod;
-            if (reader.GetMethod(methodToken, out symbolMethod) < 0)
+            if (_symUnmanagedReader.GetMethod(methodToken, out symbolMethod) < 0)
                 yield break;
 
             int count;
@@ -213,10 +223,10 @@ namespace ILCompiler
         // and names for all of them.  This assumes a CSC-like compiler that doesn't re-use
         // local slots in the same method across scopes.
         //
-        public IEnumerable<ILLocalVariable> GetLocalVariableNamesForMethod(ISymUnmanagedReader reader, int methodToken)
+        public override IEnumerable<ILLocalVariable> GetLocalVariableNamesForMethod(int methodToken)
         {
             ISymUnmanagedMethod symbolMethod;
-            if (reader.GetMethod(methodToken, out symbolMethod) < 0)
+            if (_symUnmanagedReader.GetMethod(methodToken, out symbolMethod) < 0)
                 return null;
 
             ISymUnmanagedScope rootScope;
diff --git a/src/ILCompiler.Compiler/src/CppCodeGen/CppWriter.cs b/src/ILCompiler.Compiler/src/CppCodeGen/CppWriter.cs
index 7790fc05c..bac26c694 100644
--- a/src/ILCompiler.Compiler/src/CppCodeGen/CppWriter.cs
+++ b/src/ILCompiler.Compiler/src/CppCodeGen/CppWriter.cs
@@ -9,6 +9,7 @@ using System.IO;
 using System.Text;
 using System.Linq;
 
+using ILCompiler.SymbolReader;
 using ILCompiler.DependencyAnalysisFramework;
 
 using Internal.TypeSystem;
diff --git a/src/ILCompiler.Compiler/src/CppCodeGen/ILToCppImporter.cs b/src/ILCompiler.Compiler/src/CppCodeGen/ILToCppImporter.cs
index f60729dab..8a61e1cec 100644
--- a/src/ILCompiler.Compiler/src/CppCodeGen/ILToCppImporter.cs
+++ b/src/ILCompiler.Compiler/src/CppCodeGen/ILToCppImporter.cs
@@ -8,30 +8,15 @@ using System.Diagnostics;
 using System.Globalization;
 
 using Internal.TypeSystem;
-using Internal.IL.Stubs;
 
 using ILCompiler;
 using ILCompiler.CppCodeGen;
+using ILCompiler.SymbolReader;
 
 using ILCompiler.DependencyAnalysis;
 
 namespace Internal.IL
 {
-    public struct ILSequencePoint
-    {
-        public int Offset;
-        public string Document;
-        public int LineNumber;
-        // TODO: The remaining info
-    }
-
-    public struct ILLocalVariable
-    {
-        public int Slot;
-        public string Name;
-        public bool CompilerGenerated;
-    }
-
     internal partial class ILImporter
     {
         private Compilation _compilation;
diff --git a/src/ILCompiler.Compiler/src/ILCompiler.Compiler.csproj b/src/ILCompiler.Compiler/src/ILCompiler.Compiler.csproj
index 9e5916457..90c2c6e0e 100644
--- a/src/ILCompiler.Compiler/src/ILCompiler.Compiler.csproj
+++ b/src/ILCompiler.Compiler/src/ILCompiler.Compiler.csproj
@@ -71,7 +71,9 @@
     <Compile Include="Compiler\MemoryHelper.cs" />
     <Compile Include="Compiler\MethodExtensions.cs" />
     <Compile Include="Compiler\NameMangler.cs" />
-    <Compile Include="Compiler\PdbSymbolProvider.cs" />
+    <Compile Include="Compiler\SymbolReader\PdbSymbolReader.cs" />
+    <Compile Include="Compiler\SymbolReader\PortablePdbSymbolReader.cs" />
+    <Compile Include="Compiler\SymbolReader\UnmanagedPdbSymbolReader.cs" />
     <Compile Include="Compiler\VirtualMethodCallHelper.cs" />
     <Compile Include="CppCodeGen\CppWriter.cs" />
   </ItemGroup>
diff --git a/src/ILCompiler.Compiler/src/project.json b/src/ILCompiler.Compiler/src/project.json
index 0ec9967d7..b18114aee 100644
--- a/src/ILCompiler.Compiler/src/project.json
+++ b/src/ILCompiler.Compiler/src/project.json
@@ -17,7 +17,7 @@
     "System.Reflection.Extensions": "4.0.0",
     "System.AppContext": "4.0.0",
     "System.Collections.Immutable": "1.1.37",
-    "System.Reflection.Metadata": "1.0.22",
+    "System.Reflection.Metadata": "1.1.0",
     "Microsoft.DiaSymReader": "1.0.6"
   },
   "frameworks": {