[dtrace] GC heap allocation probes for SGen.

These trigger whenever a segment of heap memory is allocated from the
OS, or freed to the OS again.

12 files changed, 56 insertions, 38 deletions

diff --git a/data/mono.d b/data/mono.d
index 3d27128b736..14591aa4838 100644
--- a/data/mono.d
+++ b/data/mono.d
@@ -18,6 +18,9 @@ provider mono {
 	/* Garbage Collector (GC) */	
 	probe gc__begin (int generation);
 	probe gc__end (int generation);
+
+	probe gc__heap__alloc (void *addr, uintptr_t len);
+	probe gc__heap__free (void *addr, uintptr_t len);
 };
 
 #pragma D attributes Evolving/Evolving/Common provider mono provider
diff --git a/mono/metadata/sgen-cardtable.c b/mono/metadata/sgen-cardtable.c
index c83aac3e367..6d9906cc28c 100644
--- a/mono/metadata/sgen-cardtable.c
+++ b/mono/metadata/sgen-cardtable.c
@@ -655,10 +655,10 @@ sgen_card_tables_collect_stats (gboolean begin)
 void
 sgen_card_table_init (SgenRemeberedSet *remset)
 {
-	sgen_cardtable = sgen_alloc_os_memory (CARD_COUNT_IN_BYTES, TRUE, "card table");
+	sgen_cardtable = sgen_alloc_os_memory (CARD_COUNT_IN_BYTES, TRUE, FALSE, "card table");
 
 #ifdef SGEN_HAVE_OVERLAPPING_CARDS
-	sgen_shadow_cardtable = sgen_alloc_os_memory (CARD_COUNT_IN_BYTES, TRUE, "shadow card table");
+	sgen_shadow_cardtable = sgen_alloc_os_memory (CARD_COUNT_IN_BYTES, TRUE, FALSE, "shadow card table");
 #endif
 
 #ifdef HEAVY_STATISTICS
diff --git a/mono/metadata/sgen-debug.c b/mono/metadata/sgen-debug.c
index e9252095111..5496ca97846 100644
--- a/mono/metadata/sgen-debug.c
+++ b/mono/metadata/sgen-debug.c
@@ -364,7 +364,7 @@ sgen_check_whole_heap (void)
 {
 	/*setup valid_nursery_objects*/
 	if (!valid_nursery_objects)
-		valid_nursery_objects = sgen_alloc_os_memory (DEFAULT_NURSERY_SIZE, TRUE, "debugging data");
+		valid_nursery_objects = sgen_alloc_os_memory (DEFAULT_NURSERY_SIZE, TRUE, FALSE, "debugging data");
 	valid_nursery_object_count = 0;
 	sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data, setup_mono_sgen_scan_area_with_callback, NULL, FALSE);
 
diff --git a/mono/metadata/sgen-internal.c b/mono/metadata/sgen-internal.c
index 53d8c62d251..b6b0c3c54f7 100644
--- a/mono/metadata/sgen-internal.c
+++ b/mono/metadata/sgen-internal.c
@@ -121,7 +121,7 @@ sgen_alloc_internal_dynamic (size_t size, int type, gboolean assert_on_failure)
 	void *p;
 
 	if (size > allocator_sizes [NUM_ALLOCATORS - 1]) {
-		p = sgen_alloc_os_memory (size, TRUE, NULL);
+		p = sgen_alloc_os_memory (size, TRUE, FALSE, NULL);
 		if (!p)
 			sgen_assert_memory_alloc (NULL, description_for_type (type));
 		return p;
@@ -139,15 +139,13 @@ sgen_alloc_internal_dynamic (size_t size, int type, gboolean assert_on_failure)
 void
 sgen_free_internal_dynamic (void *addr, size_t size, int type)
 {
-	int index;
-
 	if (!addr)
 		return;
 
-	if (size > allocator_sizes [NUM_ALLOCATORS - 1])
-		return sgen_free_os_memory (addr, size);
-
-	index = index_for_size (size);
+	if (size > allocator_sizes [NUM_ALLOCATORS - 1]) {
+		sgen_free_os_memory (addr, size, FALSE);
+		return;
+	}
 
 	mono_lock_free_free (addr);
 }
diff --git a/mono/metadata/sgen-los.c b/mono/metadata/sgen-los.c
index 516210c20c3..f9f9a4ef9c1 100644
--- a/mono/metadata/sgen-los.c
+++ b/mono/metadata/sgen-los.c
@@ -244,7 +244,7 @@ get_los_section_memory (size_t size)
 	if (!sgen_memgov_try_alloc_space (LOS_SECTION_SIZE, SPACE_LOS))
 		return NULL;
 
-	section = sgen_alloc_os_memory_aligned (LOS_SECTION_SIZE, LOS_SECTION_SIZE, TRUE, NULL);
+	section = sgen_alloc_os_memory_aligned (LOS_SECTION_SIZE, LOS_SECTION_SIZE, TRUE, TRUE, NULL);
 
 	if (!section)
 		return NULL;
@@ -323,7 +323,7 @@ sgen_los_free_object (LOSObject *obj)
 		size += sizeof (LOSObject);
 		size += pagesize - 1;
 		size &= ~(pagesize - 1);
-		sgen_free_os_memory (obj, size);
+		sgen_free_os_memory (obj, size, TRUE);
 		sgen_memgov_release_space (size, SPACE_LOS);
 	} else {
 		free_los_section_memory (obj, size + sizeof (LOSObject));
@@ -351,7 +351,7 @@ sgen_los_alloc_large_inner (MonoVTable *vtable, size_t size)
 
 #ifdef LOS_DUMMY
 	if (!los_segment)
-		los_segment = sgen_alloc_os_memory (LOS_SEGMENT_SIZE, TRUE, NULL);
+		los_segment = sgen_alloc_os_memory (LOS_SEGMENT_SIZE, TRUE, TRUE, NULL);
 	los_segment_index = ALIGN_UP (los_segment_index);
 
 	obj = (LOSObject*)(los_segment + los_segment_index);
@@ -372,7 +372,7 @@ sgen_los_alloc_large_inner (MonoVTable *vtable, size_t size)
 		alloc_size += pagesize - 1;
 		alloc_size &= ~(pagesize - 1);
 		if (sgen_memgov_try_alloc_space (alloc_size, SPACE_LOS)) {
-			obj = sgen_alloc_os_memory (alloc_size, TRUE, NULL);
+			obj = sgen_alloc_os_memory (alloc_size, TRUE, TRUE, NULL);
 			if (obj)
 				obj->huge_object = TRUE;
 		}
@@ -423,7 +423,7 @@ sgen_los_sweep (void)
 				prev->next = next;
 			else
 				los_sections = next;
-			sgen_free_os_memory (section, LOS_SECTION_SIZE);
+			sgen_free_os_memory (section, LOS_SECTION_SIZE, TRUE);
 			sgen_memgov_release_space (LOS_SECTION_SIZE, SPACE_LOS);
 			section = next;
 			--los_num_sections;
diff --git a/mono/metadata/sgen-major-copying.c b/mono/metadata/sgen-major-copying.c
index 6992644d7ee..d1737a6b787 100644
--- a/mono/metadata/sgen-major-copying.c
+++ b/mono/metadata/sgen-major-copying.c
@@ -98,9 +98,9 @@ static void*
 major_alloc_heap (mword nursery_size, mword nursery_align, int the_nursery_bits)
 {
 	if (nursery_align)
-		nursery_start = sgen_alloc_os_memory_aligned (nursery_size, nursery_align, TRUE, "nursery");
+		nursery_start = sgen_alloc_os_memory_aligned (nursery_size, nursery_align, TRUE, TRUE, "nursery");
 	else
-		nursery_start = sgen_alloc_os_memory (nursery_size, TRUE, "nursery");
+		nursery_start = sgen_alloc_os_memory (nursery_size, TRUE, TRUE, "nursery");
 
 	nursery_end = nursery_start + nursery_size;
 	nursery_bits = the_nursery_bits;
@@ -129,7 +129,7 @@ alloc_major_section (void)
 	GCMemSection *section;
 	int scan_starts;
 
-	section = sgen_alloc_os_memory_aligned (MAJOR_SECTION_SIZE, MAJOR_SECTION_SIZE, TRUE, "major heap section");
+	section = sgen_alloc_os_memory_aligned (MAJOR_SECTION_SIZE, MAJOR_SECTION_SIZE, TRUE, TRUE, "major heap section");
 	section->next_data = section->data = (char*)section + SGEN_SIZEOF_GC_MEM_SECTION;
 	g_assert (!((mword)section->data & 7));
 	section->size = MAJOR_SECTION_SIZE - SGEN_SIZEOF_GC_MEM_SECTION;
@@ -157,7 +157,7 @@ free_major_section (GCMemSection *section)
 	DEBUG (3, fprintf (gc_debug_file, "Freed major section %p (%p-%p)\n", section, section->data, section->end_data));
 	sgen_free_internal_dynamic (section->scan_starts,
 			(section->size + SGEN_SCAN_START_SIZE - 1) / SGEN_SCAN_START_SIZE * sizeof (char*), INTERNAL_MEM_SCAN_STARTS);
-	sgen_free_os_memory (section, MAJOR_SECTION_SIZE);
+	sgen_free_os_memory (section, MAJOR_SECTION_SIZE, TRUE);
 
 	--num_major_sections;
 }
diff --git a/mono/metadata/sgen-marksweep.c b/mono/metadata/sgen-marksweep.c
index de1b4f23b03..a3d52f7fd63 100644
--- a/mono/metadata/sgen-marksweep.c
+++ b/mono/metadata/sgen-marksweep.c
@@ -311,7 +311,7 @@ major_alloc_heap (mword nursery_size, mword nursery_align, int the_nursery_bits)
 	if (nursery_align)
 		g_assert (nursery_align % MS_BLOCK_SIZE == 0);
 
-	nursery_start = sgen_alloc_os_memory_aligned (alloc_size, nursery_align ? nursery_align : MS_BLOCK_SIZE, TRUE, "heap");
+	nursery_start = sgen_alloc_os_memory_aligned (alloc_size, nursery_align ? nursery_align : MS_BLOCK_SIZE, TRUE, TRUE, "heap");
 	ms_heap_start = nursery_start + nursery_size;
 	ms_heap_end = ms_heap_start + major_heap_size;
 
@@ -336,9 +336,9 @@ major_alloc_heap (mword nursery_size, mword nursery_align, int the_nursery_bits)
 {
 	char *start;
 	if (nursery_align)
-		start = sgen_alloc_os_memory_aligned (nursery_size, nursery_align, TRUE, "nursery");
+		start = sgen_alloc_os_memory_aligned (nursery_size, nursery_align, TRUE, TRUE, "nursery");
 	else
-		start = sgen_alloc_os_memory (nursery_size, TRUE, "nursery");
+		start = sgen_alloc_os_memory (nursery_size, TRUE, TRUE, "nursery");
 
 	return start;
 }
@@ -389,7 +389,7 @@ ms_get_empty_block (void)
 
  retry:
 	if (!empty_blocks) {
-		p = sgen_alloc_os_memory_aligned (MS_BLOCK_SIZE * MS_BLOCK_ALLOC_NUM, MS_BLOCK_SIZE, TRUE, "major heap section");
+		p = sgen_alloc_os_memory_aligned (MS_BLOCK_SIZE * MS_BLOCK_ALLOC_NUM, MS_BLOCK_SIZE, TRUE, TRUE, "major heap section");
 
 		for (i = 0; i < MS_BLOCK_ALLOC_NUM; ++i) {
 			block = p;
@@ -1697,7 +1697,7 @@ major_have_computer_minor_collection_allowance (void)
 
 	while (num_empty_blocks > section_reserve) {
 		void *next = *(void**)empty_blocks;
-		sgen_free_os_memory (empty_blocks, MS_BLOCK_SIZE);
+		sgen_free_os_memory (empty_blocks, MS_BLOCK_SIZE, TRUE);
 		empty_blocks = next;
 		/*
 		 * Needs not be atomic because this is running
diff --git a/mono/metadata/sgen-memory-governor.c b/mono/metadata/sgen-memory-governor.c
index 99e23061411..dc7fb75d54b 100644
--- a/mono/metadata/sgen-memory-governor.c
+++ b/mono/metadata/sgen-memory-governor.c
@@ -40,6 +40,7 @@
 #include "utils/mono-counters.h"
 #include "utils/mono-mmap.h"
 #include "utils/mono-logger-internal.h"
+#include "utils/dtrace.h"
 
 #define MIN_MINOR_COLLECTION_ALLOWANCE	((mword)(DEFAULT_NURSERY_SIZE * default_allowance_nursery_size_ratio))
 
@@ -301,23 +302,29 @@ sgen_assert_memory_alloc (void *ptr, const char *assert_description)
  * This must not require any lock.
  */
 void*
-sgen_alloc_os_memory (size_t size, int activate, const char *assert_description)
+sgen_alloc_os_memory (size_t size, int activate, gboolean is_heap_memory, const char *assert_description)
 {
 	void *ptr = mono_valloc (0, size, prot_flags_for_activate (activate));
 	sgen_assert_memory_alloc (ptr, assert_description);
-	if (ptr)
+	if (ptr) {
 		SGEN_ATOMIC_ADD_P (total_alloc, size);
+		if (is_heap_memory)
+			MONO_PROBE_GC_HEAP_ALLOC (ptr, size);
+	}
 	return ptr;
 }
 
 /* size must be a power of 2 */
 void*
-sgen_alloc_os_memory_aligned (size_t size, mword alignment, gboolean activate, const char *assert_description)
+sgen_alloc_os_memory_aligned (size_t size, mword alignment, gboolean activate, gboolean is_heap_memory, const char *assert_description)
 {
 	void *ptr = mono_valloc_aligned (size, alignment, prot_flags_for_activate (activate));
 	sgen_assert_memory_alloc (ptr, assert_description);
-	if (ptr)
+	if (ptr) {
 		SGEN_ATOMIC_ADD_P (total_alloc, size);
+		if (is_heap_memory)
+			MONO_PROBE_GC_HEAP_ALLOC (ptr, size);
+	}
 	return ptr;
 }
 
@@ -325,10 +332,12 @@ sgen_alloc_os_memory_aligned (size_t size, mword alignment, gboolean activate, c
  * Free the memory returned by sgen_alloc_os_memory (), returning it to the OS.
  */
 void
-sgen_free_os_memory (void *addr, size_t size)
+sgen_free_os_memory (void *addr, size_t size, gboolean is_heap_memory)
 {
 	mono_vfree (addr, size);
 	SGEN_ATOMIC_ADD_P (total_alloc, -size);
+	if (is_heap_memory)
+		MONO_PROBE_GC_HEAP_FREE (addr, size);
 }
 
 int64_t
diff --git a/mono/metadata/sgen-memory-governor.h b/mono/metadata/sgen-memory-governor.h
index 0a6fc99f5d2..7140bf7313b 100644
--- a/mono/metadata/sgen-memory-governor.h
+++ b/mono/metadata/sgen-memory-governor.h
@@ -45,9 +45,9 @@ gboolean sgen_need_major_collection (mword space_needed) MONO_INTERNAL;
 
 
 /* OS memory allocation */
-void* sgen_alloc_os_memory (size_t size, int activate, const char *assert_description) MONO_INTERNAL;
-void* sgen_alloc_os_memory_aligned (size_t size, mword alignment, gboolean activate, const char *assert_description) MONO_INTERNAL;
-void sgen_free_os_memory (void *addr, size_t size) MONO_INTERNAL;
+void* sgen_alloc_os_memory (size_t size, int activate, gboolean is_heap_memory, const char *assert_description) MONO_INTERNAL;
+void* sgen_alloc_os_memory_aligned (size_t size, mword alignment, gboolean activate, gboolean is_heap_memory, const char *assert_description) MONO_INTERNAL;
+void sgen_free_os_memory (void *addr, size_t size, gboolean is_heap_memory) MONO_INTERNAL;
 
 /* Error handling */
 void sgen_assert_memory_alloc (void *ptr, const char *assert_description) MONO_INTERNAL;
diff --git a/mono/metadata/sgen-pinned-allocator.c b/mono/metadata/sgen-pinned-allocator.c
index 02cf8cb94ff..661f597f8aa 100644
--- a/mono/metadata/sgen-pinned-allocator.c
+++ b/mono/metadata/sgen-pinned-allocator.c
@@ -111,6 +111,8 @@ static const int freelist_sizes [] = {
 
 #define LARGE_PINNED_MEM_HEADER_MAGIC	0x7d289f3a
 
+/* FIXME: Do we even need these anymore?  Large objects are always
+   allocated in the LOS. */
 typedef struct _LargePinnedMemHeader LargePinnedMemHeader;
 struct _LargePinnedMemHeader {
 	guint32 magic;
@@ -217,7 +219,7 @@ alloc_pinned_chunk (SgenPinnedAllocator *alc)
 	int offset;
 	int size = SGEN_PINNED_CHUNK_SIZE;
 
-	chunk = sgen_alloc_os_memory_aligned (size, size, TRUE, "pinned chunk");
+	chunk = sgen_alloc_os_memory_aligned (size, size, TRUE, TRUE, "pinned chunk");
 	chunk->block.role = MEMORY_ROLE_PINNED;
 
 	sgen_update_heap_boundaries ((mword)chunk, ((mword)chunk + size));
@@ -329,7 +331,7 @@ sgen_alloc_pinned (SgenPinnedAllocator *alc, size_t size)
 		LargePinnedMemHeader *mh;
 
 		size += sizeof (LargePinnedMemHeader);
-		mh = sgen_alloc_os_memory (size, TRUE, "large pinned object");
+		mh = sgen_alloc_os_memory (size, TRUE, TRUE, "large pinned object");
 		mh->magic = LARGE_PINNED_MEM_HEADER_MAGIC;
 		mh->size = size;
 		/* FIXME: do a CAS here */
@@ -383,7 +385,7 @@ sgen_free_pinned (SgenPinnedAllocator *alc, void *addr, size_t size)
 	g_assert (mh->size == size + sizeof (LargePinnedMemHeader));
 	/* FIXME: do a CAS */
 	large_pinned_bytes_alloced -= mh->size;
-	sgen_free_os_memory (mh, mh->size);
+	sgen_free_os_memory (mh, mh->size, TRUE);
 }
 
 void
diff --git a/mono/metadata/sgen-protocol.c b/mono/metadata/sgen-protocol.c
index 92b4d9571c0..75ae316704e 100644
--- a/mono/metadata/sgen-protocol.c
+++ b/mono/metadata/sgen-protocol.c
@@ -66,7 +66,7 @@ binary_protocol_flush_buffers_rec (BinaryProtocolBuffer *buffer)
 	g_assert (buffer->index > 0);
 	fwrite (buffer->buffer, 1, buffer->index, binary_protocol_file);
 
-	sgen_free_os_memory (buffer, sizeof (BinaryProtocolBuffer));
+	sgen_free_os_memory (buffer, sizeof (BinaryProtocolBuffer), FALSE);
 }
 
 void
@@ -94,12 +94,12 @@ binary_protocol_get_buffer (int length)
 	if (buffer && buffer->index + length <= BINARY_PROTOCOL_BUFFER_SIZE)
 		return buffer;
 
-	new_buffer = sgen_alloc_os_memory (sizeof (BinaryProtocolBuffer), TRUE, "debugging memory");
+	new_buffer = sgen_alloc_os_memory (sizeof (BinaryProtocolBuffer), TRUE, FALSE, "debugging memory");
 	new_buffer->next = buffer;
 	new_buffer->index = 0;
 
 	if (InterlockedCompareExchangePointer ((void**)&binary_protocol_buffers, new_buffer, buffer) != buffer) {
-		sgen_free_os_memory (new_buffer, sizeof (BinaryProtocolBuffer));
+		sgen_free_os_memory (new_buffer, sizeof (BinaryProtocolBuffer), FALSE);
 		goto retry;
 	}
 
diff --git a/mono/utils/dtrace.h b/mono/utils/dtrace.h
index 1b540f705da..69fe38b390f 100644
--- a/mono/utils/dtrace.h
+++ b/mono/utils/dtrace.h
@@ -46,6 +46,9 @@
 #define MONO_PROBE_GC_END_ENABLED()	MONO_GC_END_ENABLED ()
 
 
+#define MONO_PROBE_GC_HEAP_ALLOC(addr,size)	MONO_GC_HEAP_ALLOC ((addr), (size))
+#define MONO_PROBE_GC_HEAP_FREE(addr,size)	MONO_GC_HEAP_FREE ((addr), (size))
+
 #else
 
 
@@ -70,6 +73,9 @@
 #define MONO_PROBE_GC_END_ENABLED() (0)
 
 
+#define MONO_PROBE_GC_HEAP_ALLOC(addr,size)
+#define MONO_PROBE_GC_HEAP_FREE(addr,size)
+
 #endif
 
 #endif