Optimize ObjectWriter to emit series of bytes (#3680)

* Optimize ObjectWriter to emit series of bytes

Writing each byte of an `ObjectNode` out to the object file one by one
is inefficient. We also cannot emit all the bytes of an `ObjectNode` at
once because of the streaming nature of the LLVM API. Relocs, CFI info,
and symbol definitions are all interruptions emitted at specific points
in the stream. Optimize the current implementation by calculating the
longest run of bytes between each required interruption.

Testing on a large app, ASP.NET Core, this yields a 4% reduction in
total compilation time. The total time to compile (collected in Release
mode averaged over 5 runs) decreased from 31611ms to 30345ms. A further
gain may be possible when we get a high performance implementation of
Span<T>, able to efficiently pin the byte arrays we pass to native.

1 files changed, 52 insertions, 3 deletions

diff --git a/src/ILCompiler.Compiler/src/Compiler/DependencyAnalysis/ObjectWriter.cs b/src/ILCompiler.Compiler/src/Compiler/DependencyAnalysis/ObjectWriter.cs
index f223bb8ef..afeb432e2 100644
--- a/src/ILCompiler.Compiler/src/Compiler/DependencyAnalysis/ObjectWriter.cs
+++ b/src/ILCompiler.Compiler/src/Compiler/DependencyAnalysis/ObjectWriter.cs
@@ -30,6 +30,9 @@ namespace ILCompiler.DependencyAnalysis
         // This is a global table across nodes.
         private Dictionary<string, int> _debugFileToId = new Dictionary<string, int>();
 
+        // Track offsets in node data that prevent writing all bytes in one single blob. This includes
+        // relocs, symbol definitions, debug data that must be streamed out using the existing LLVM API
+        private SortedSet<int> _byteInterruptionOffsets = new SortedSet<int>();
         // This is used to look up DebugLocInfo for the given native offset.
         // This is for individual node and should be flushed once node is emitted.
         private Dictionary<int, DebugLocInfo> _offsetToDebugLoc = new Dictionary<int, DebugLocInfo>();
@@ -170,6 +173,13 @@ namespace ILCompiler.DependencyAnalysis
         }
 
         [DllImport(NativeObjectWriterFileName)]
+        private static extern void EmitBlob(IntPtr objWriter, int blobSize, IntPtr blob);
+        public void EmitBytes(IntPtr pArray, int length)
+        {
+            EmitBlob(_nativeObjectWriter, length, pArray);
+        }
+
+        [DllImport(NativeObjectWriterFileName)]
         private static extern void EmitSymbolDef(IntPtr objWriter, byte[] symbolName);
         public void EmitSymbolDef(byte[] symbolName)
         {
@@ -394,6 +404,7 @@ namespace ILCompiler.DependencyAnalysis
                     {
                         Debug.Assert(!_offsetToDebugLoc.ContainsKey(loc.NativeOffset));
                         _offsetToDebugLoc[loc.NativeOffset] = loc;
+                        _byteInterruptionOffsets.Add(loc.NativeOffset);
                     }
                 }
             }
@@ -559,6 +570,8 @@ namespace ILCompiler.DependencyAnalysis
                 // Record start/end of frames which shouldn't be overlapped.
                 _offsetToCfiStart.Add(start);
                 _offsetToCfiEnd.Add(end);
+                _byteInterruptionOffsets.Add(start);
+                _byteInterruptionOffsets.Add(end);
                 _offsetToCfiLsdaBlobName.Add(start, blobSymbolName);
                 for (int j = 0; j < len; j += CfiCodeSize)
                 {
@@ -570,6 +583,7 @@ namespace ILCompiler.DependencyAnalysis
                     {
                         cfis = new List<byte[]>();
                         _offsetToCfis.Add(codeOffset, cfis);
+                        _byteInterruptionOffsets.Add(codeOffset);
                     }
                     byte[] cfi = new byte[CfiCodeSize];
                     Array.Copy(blob, j, cfi, 0, CfiCodeSize);
@@ -639,6 +653,7 @@ namespace ILCompiler.DependencyAnalysis
                 }
 
                 _offsetToDefName[n.Offset].Add(n);
+                _byteInterruptionOffsets.Add(n.Offset);
             }
 
             var symbolNode = node as ISymbolDefinitionNode;
@@ -811,6 +826,16 @@ namespace ILCompiler.DependencyAnalysis
             return new ObjectNodeSection(standardSectionPrefix + section.Name, section.Type, key);
         }
 
+        public void ResetByteRunInterruptionOffsets(Relocation[] relocs)
+        {
+            _byteInterruptionOffsets.Clear();
+
+            for (int i = 0; i < relocs.Length; ++i)
+            {
+                _byteInterruptionOffsets.Add(relocs[i].Offset);
+            }
+        }
+
         public static void EmitObject(string objectFilePath, IEnumerable<DependencyNode> nodes, NodeFactory factory, IObjectDumper dumper)
         {
             ObjectWriter objectWriter = new ObjectWriter(objectFilePath, factory);
@@ -849,6 +874,7 @@ namespace ILCompiler.DependencyAnalysis
                 // Build file info map.
                 objectWriter.BuildFileInfoMap(nodes);
 
+                var listOfOffsets = new List<int>();
                 foreach (DependencyNode depNode in nodes)
                 {
                     ObjectNode node = depNode as ObjectNode;
@@ -890,6 +916,8 @@ namespace ILCompiler.DependencyAnalysis
                     objectWriter.SetSection(section);
                     objectWriter.EmitAlignment(nodeContents.Alignment);
 
+                    objectWriter.ResetByteRunInterruptionOffsets(nodeContents.Relocs);
+
                     // Build symbol definition map.
                     objectWriter.BuildSymbolDefinitionMap(node, nodeContents.DefinedSymbols);
 
@@ -909,6 +937,11 @@ namespace ILCompiler.DependencyAnalysis
                     }
 
                     int i = 0;
+
+                    listOfOffsets.Clear();
+                    listOfOffsets.AddRange(objectWriter._byteInterruptionOffsets);
+
+                    int offsetIndex = 0;
                     while (i < nodeContents.Data.Length)
                     {
                         // Emit symbol definitions if necessary
@@ -951,11 +984,27 @@ namespace ILCompiler.DependencyAnalysis
                         }
                         else
                         {
-                            objectWriter.EmitIntValue(nodeContents.Data[i], 1);
-                            i++;
+                            while (offsetIndex < listOfOffsets.Count && listOfOffsets[offsetIndex] <= i)
+                            {
+                                offsetIndex++;
+                            }
+                            
+                            int nextOffset = offsetIndex == listOfOffsets.Count ? nodeContents.Data.Length : listOfOffsets[offsetIndex];
+                            
+                            unsafe
+                            {
+                                // Todo: Use Span<T> instead once it's available to us in this repo
+                                fixed (byte* pContents = &nodeContents.Data[i])
+                                {
+                                    objectWriter.EmitBytes((IntPtr)(pContents), nextOffset - i);
+                                    i += nextOffset - i;
+                                }
+                            }
+                            
                         }
                     }
-
+                    Debug.Assert(i == nodeContents.Data.Length);
+                    
                     // It is possible to have a symbol just after all of the data.
                     objectWriter.EmitSymbolDefinition(nodeContents.Data.Length);