[profiler] Rework GC roots reporting. (#6144)

This change re-implement GC roots reporting in a way that allows users to correctly track the source of a GC root.

The new design is based on emitting GC root information upfront and only report addresses during heap dumps.

When a GC root is registered we emit an event that is an address range, the root kind, a key and a text description.

A decoder can use this information to match a reported root address with all registration addresses to figure out what they mean.

A GC Root key is used to further its meaning. For example, if kind is thread, key is a tid, if kind is static variables, key is the vtable pointer and so on.

The build of the change is introducing this key argument across all root registration code and rework our root reporting code to encode their addresses.

Some roots use pseudo-addresses when they are not registrable. This is the case of thread stacks/registers and the finalizer queue.

Finally, root reporting was changed to happen only once per collection and at the end, leading to correct and useful data being produced.

6 files changed, 127 insertions, 8 deletions

diff --git a/mcs/class/Mono.Profiler.Log/Makefile b/mcs/class/Mono.Profiler.Log/Makefile
index d8f11017c52..84d4ed93a85 100644
--- a/mcs/class/Mono.Profiler.Log/Makefile
+++ b/mcs/class/Mono.Profiler.Log/Makefile
@@ -6,7 +6,7 @@ LIBRARY_SNK = ../mono.snk
 
 LIB_REFS = System System.Core
 KEYFILE = $(LIBRARY_SNK)
-LIB_MCS_FLAGS = /unsafe /publicsign
+LIB_MCS_FLAGS = /unsafe /publicsign /nowarn:0618
 
 LIBRARY_WARN_AS_ERROR = yes
 
diff --git a/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogEnums.cs b/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogEnums.cs
index c4ffb2bb923..22cd832e869 100644
--- a/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogEnums.cs
+++ b/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogEnums.cs
@@ -54,6 +54,8 @@ namespace Mono.Profiler.Log {
 		HeapEnd = 1 << 4,
 		HeapObject = 2 << 4,
 		HeapRoots = 3 << 4,
+		HeapRootRegister = 4 << 4,
+		HeapRootUnregister = 5 << 4,
 
 		SampleHit = 0 << 4,
 		SampleUnmanagedSymbol = 1 << 4,
@@ -74,6 +76,7 @@ namespace Mono.Profiler.Log {
 		AppDomain = 4,
 		Thread = 5,
 		Context = 6,
+		VTable = 7,
 	}
 
 	// mono/utils/mono-counters.h : MONO_COUNTER_*
@@ -133,6 +136,7 @@ namespace Mono.Profiler.Log {
 
 	// mono/metadata/profiler.h : MonoProfilerGCRootType
 	[Flags]
+	[Obsolete ("The event field using this enum is no longer produced.")]
 	public enum LogHeapRootAttributes {
 		Pinning = 1 << 8,
 		WeakReference = 2 << 8,
@@ -147,6 +151,25 @@ namespace Mono.Profiler.Log {
 		TypeMask = 0xff,
 	}
 
+	// mono/metadata/mono-gc.h : MonoGCRootSource
+	public enum LogHeapRootSource {
+		External = 0,
+		Stack = 1,
+		FinalizerQueue = 2,
+		Static = 3,
+		ThreadStatic = 4,
+		ContextStatic = 5,
+		GCHandle = 6,
+		Jit = 7,
+		Threading = 8,
+		AppDomain = 9,
+		Reflection = 10,
+		Marshal = 11,
+		ThreadPool = 12,
+		Debugger = 13,
+		Handle = 14,
+	}
+
 	// mono/profiler/log.h : MonoProfilerMonitorEvent
 	public enum LogMonitorEvent {
 		Contention = 1,
diff --git a/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogEventVisitor.cs b/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogEventVisitor.cs
index b49416cd2c1..79d61405236 100644
--- a/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogEventVisitor.cs
+++ b/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogEventVisitor.cs
@@ -66,6 +66,10 @@ namespace Mono.Profiler.Log {
 		{
 		}
 
+		public virtual void Visit (VTableLoadEvent ev)
+		{
+		}
+
 		public virtual void Visit (JitEvent ev)
 		{
 		}
@@ -94,6 +98,14 @@ namespace Mono.Profiler.Log {
 		{
 		}
 
+		public virtual void Visit (HeapRootRegisterEvent ev)
+		{
+		}
+
+		public virtual void Visit (HeapRootUnregisterEvent ev)
+		{
+		}
+
 		public virtual void Visit (GCEvent ev)
 		{
 		}
diff --git a/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogEvents.cs b/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogEvents.cs
index 113bf3edc11..faae978cd96 100644
--- a/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogEvents.cs
+++ b/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogEvents.cs
@@ -161,6 +161,20 @@ namespace Mono.Profiler.Log {
 		}
 	}
 
+	public sealed class VTableLoadEvent : LogEvent {
+
+		public long VTablePointer { get; internal set; }
+
+		public long AppDomainId { get; internal set; }
+
+		public long ClassPointer { get; internal set; }
+
+		internal override void Accept (LogEventVisitor visitor)
+		{
+			visitor.Visit (this);
+		}
+	}
+
 	public sealed class JitEvent : LogEvent {
 
 		public long MethodPointer { get; internal set; }
@@ -195,8 +209,11 @@ namespace Mono.Profiler.Log {
 
 	public sealed class AllocationEvent : LogEvent {
 
+		[Obsolete ("This field is no longer produced.")]
 		public long ClassPointer { get; internal set; }
 
+		public long VTablePointer { get; internal set; }
+
 		public long ObjectPointer { get; internal set; }
 
 		public long ObjectSize { get; internal set; }
@@ -236,8 +253,11 @@ namespace Mono.Profiler.Log {
 
 		public long ObjectPointer { get; internal set; }
 
+		[Obsolete ("This field is no longer produced.")]
 		public long ClassPointer { get; internal set; }
 
+		public long VTablePointer { get; internal set; }
+
 		public long ObjectSize { get; internal set; }
 
 		public IReadOnlyList<HeapObjectReference> References { get; internal set; }
@@ -252,13 +272,18 @@ namespace Mono.Profiler.Log {
 
 		public struct HeapRoot {
 
+			public long AddressPointer { get; internal set; }
+
 			public long ObjectPointer { get; internal set; }
 
+			[Obsolete ("This field is no longer produced.")]
 			public LogHeapRootAttributes Attributes { get; internal set; }
 
+			[Obsolete ("This field is no longer produced.")]
 			public long ExtraInfo { get; internal set; }
 		}
 
+		[Obsolete ("This field is no longer produced.")]
 		public long MaxGenerationCollectionCount { get; internal set; }
 
 		public IReadOnlyList<HeapRoot> Roots { get; internal set; }
@@ -269,6 +294,34 @@ namespace Mono.Profiler.Log {
 		}
 	}
 
+	public sealed class HeapRootRegisterEvent : LogEvent {
+
+		public long RootPointer { get; internal set; }
+
+		public long RootSize { get; internal set; }
+
+		public LogHeapRootSource Source { get; internal set; }
+
+		public long Key { get; internal set; }
+
+		public string Name { get; internal set; }
+
+		internal override void Accept (LogEventVisitor visitor)
+		{
+			visitor.Visit (this);
+		}
+	}
+
+	public sealed class HeapRootUnregisterEvent : LogEvent {
+
+		public long RootPointer { get; internal set; }
+
+		internal override void Accept (LogEventVisitor visitor)
+		{
+			visitor.Visit (this);
+		}
+	}
+
 	public sealed class GCEvent : LogEvent {
 
 		public LogGCEvent Type { get; internal set; }
diff --git a/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogProcessor.cs b/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogProcessor.cs
index 82b4a8e78b4..88ea2390dbe 100644
--- a/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogProcessor.cs
+++ b/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogProcessor.cs
@@ -127,7 +127,8 @@ namespace Mono.Profiler.Log {
 				case LogEventType.AllocationBacktrace:
 				case LogEventType.AllocationNoBacktrace:
 					ev = new AllocationEvent {
-						ClassPointer = ReadPointer (),
+						ClassPointer = StreamHeader.FormatVersion < 15 ? ReadPointer () : 0,
+						VTablePointer = StreamHeader.FormatVersion >= 15 ? ReadPointer () : 0,
 						ObjectPointer = ReadObject (),
 						ObjectSize = (long) _reader.ReadULeb128 (),
 						Backtrace = ReadBacktrace (extType == LogEventType.AllocationBacktrace),
@@ -305,6 +306,16 @@ namespace Mono.Profiler.Log {
 					} else
 						throw new LogException ("Invalid context metadata event.");
 					break;
+				case LogMetadataType.VTable:
+					if (load) {
+						ev = new VTableLoadEvent {
+							VTablePointer = ReadPointer (),
+							AppDomainId = ReadPointer (),
+							ClassPointer = ReadPointer (),
+						};
+					} else
+						throw new LogException ("Invalid VTable metadata event.");
+					break;
 				default:
 					throw new LogException ($"Invalid metadata type ({metadataType}).");
 				}
@@ -394,7 +405,8 @@ namespace Mono.Profiler.Log {
 				case LogEventType.HeapObject: {
 					HeapObjectEvent hoe = new HeapObjectEvent {
 						ObjectPointer = ReadObject (),
-						ClassPointer = ReadPointer (),
+						ClassPointer = StreamHeader.FormatVersion < 15 ? ReadPointer () : 0,
+						VTablePointer = StreamHeader.FormatVersion >= 15 ? ReadPointer () : 0,
 						ObjectSize = (long) _reader.ReadULeb128 (),
 					};
 
@@ -414,17 +426,22 @@ namespace Mono.Profiler.Log {
 				}
 
 				case LogEventType.HeapRoots: {
-					// TODO: This entire event makes no sense.
 					var hre = new HeapRootsEvent ();
 					var list = new HeapRootsEvent.HeapRoot [(int) _reader.ReadULeb128 ()];
 
-					hre.MaxGenerationCollectionCount = (long) _reader.ReadULeb128 ();
+					if (StreamHeader.FormatVersion < 15)
+						hre.MaxGenerationCollectionCount = (long) _reader.ReadULeb128 ();
 
 					for (var i = 0; i < list.Length; i++) {
 						list [i] = new HeapRootsEvent.HeapRoot {
+							AddressPointer = StreamHeader.FormatVersion >= 15 ? ReadPointer () : 0,
 							ObjectPointer = ReadObject (),
-							Attributes = StreamHeader.FormatVersion == 13 ? (LogHeapRootAttributes) _reader.ReadByte () : (LogHeapRootAttributes) _reader.ReadULeb128 (),
-							ExtraInfo = (long) _reader.ReadULeb128 (),
+							Attributes = StreamHeader.FormatVersion < 15 ?
+							             (StreamHeader.FormatVersion == 13 ?
+							              (LogHeapRootAttributes) _reader.ReadByte () :
+							              (LogHeapRootAttributes) _reader.ReadULeb128 ()) :
+							             0,
+							ExtraInfo = StreamHeader.FormatVersion < 15 ? (long) _reader.ReadULeb128 () : 0,
 						};
 					}
 
@@ -433,6 +450,20 @@ namespace Mono.Profiler.Log {
 
 					break;
 				}
+				case LogEventType.HeapRootRegister:
+					ev = new HeapRootRegisterEvent {
+						RootPointer = ReadPointer (),
+						RootSize = (long) _reader.ReadULeb128 (),
+						Source = (LogHeapRootSource) _reader.ReadByte (),
+						Key = ReadPointer (),
+						Name = _reader.ReadCString (),
+					};
+					break;
+				case LogEventType.HeapRootUnregister:
+					ev = new HeapRootUnregisterEvent {
+						RootPointer = ReadPointer (),
+					};
+					break;
 				default:
 					throw new LogException ($"Invalid extended event type ({extType}).");
 				}
diff --git a/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogStreamHeader.cs b/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogStreamHeader.cs
index 44575028a7a..aa86b75444a 100644
--- a/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogStreamHeader.cs
+++ b/mcs/class/Mono.Profiler.Log/Mono.Profiler.Log/LogStreamHeader.cs
@@ -9,7 +9,7 @@ namespace Mono.Profiler.Log {
 	public sealed class LogStreamHeader {
 		
 		const int MinVersion = 13;
-		const int MaxVersion = 14;
+		const int MaxVersion = 15;
 
 		const int Id = 0x4d505a01;