Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/dotnet/runtime.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorVladimir Sadov <vsadov@microsoft.com>2022-11-10 20:41:51 +0300
committerGitHub <noreply@github.com>2022-11-10 20:41:51 +0300
commitcd7e8718fcf5d5e55f9529c206a8c7205b45af08 (patch)
tree9bf26a024a566a1b3f130934c9602ebbcae7ba86
parent6a676280564ce8e04f10c29fe5ebc346aa462ca1 (diff)
[NativeAOT] Enable software writewatch and card bundles on Windows. (#77934)
* amd64 * arm64 * fix ARM64 build * import conditionally
-rw-r--r--src/coreclr/nativeaot/Runtime/CMakeLists.txt7
-rw-r--r--src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc10
-rw-r--r--src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S17
-rw-r--r--src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm94
-rw-r--r--src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h17
-rw-r--r--src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S267
-rw-r--r--src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm260
-rw-r--r--src/coreclr/nativeaot/Runtime/gcrhenv.cpp4
8 files changed, 343 insertions, 333 deletions
diff --git a/src/coreclr/nativeaot/Runtime/CMakeLists.txt b/src/coreclr/nativeaot/Runtime/CMakeLists.txt
index d4fcf662ec6..5e49efd8f5f 100644
--- a/src/coreclr/nativeaot/Runtime/CMakeLists.txt
+++ b/src/coreclr/nativeaot/Runtime/CMakeLists.txt
@@ -204,11 +204,8 @@ include_directories(${ARCH_SOURCES_DIR})
add_definitions(-DFEATURE_BASICFREEZE)
add_definitions(-DFEATURE_CONSERVATIVE_GC)
-
-if(CLR_CMAKE_TARGET_UNIX)
- add_definitions(-DFEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP)
- add_definitions(-DFEATURE_MANUALLY_MANAGED_CARD_BUNDLES)
-endif()
+add_definitions(-DFEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP)
+add_definitions(-DFEATURE_MANUALLY_MANAGED_CARD_BUNDLES)
add_definitions(-DFEATURE_CUSTOM_IMPORTS)
add_definitions(-DFEATURE_DYNAMIC_CODE)
diff --git a/src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc b/src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc
index d44537b2456..5db25fd9351 100644
--- a/src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc
+++ b/src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc
@@ -406,10 +406,18 @@ EXTERN REDHAWKGCINTERFACE__STRESSGC : PROC
EXTERN THREAD__HIJACKFORGCSTRESS : PROC
endif ;; FEATURE_GC_STRESS
+EXTERN RhpTrapThreads : DWORD
EXTERN g_lowest_address : QWORD
EXTERN g_highest_address : QWORD
EXTERN g_ephemeral_low : QWORD
EXTERN g_ephemeral_high : QWORD
EXTERN g_card_table : QWORD
-EXTERN RhpTrapThreads : DWORD
+
+ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+EXTERN g_card_bundle_table : QWORD
+endif
+
+ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+EXTERN g_write_watch_table : QWORD
+endif
diff --git a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S b/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S
index c31a95c9bec..3888d0a8adf 100644
--- a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S
+++ b/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S
@@ -255,11 +255,10 @@ LEAF_END RhpCheckedXchg, _TEXT
// On entry:
// rdi: address of ref-field (assigned to)
// rsi: address of the data (source)
-// rcx: be trashed
//
// On exit:
// rdi, rsi are incremented by 8,
-// rcx: trashed
+// rcx, r10, r11: trashed
//
LEAF_ENTRY RhpByRefAssignRef, _TEXT
mov rcx, [rsi]
@@ -267,9 +266,9 @@ LEAF_ENTRY RhpByRefAssignRef, _TEXT
// Check whether the writes were even into the heap. If not there's no card update required.
cmp rdi, [C_VAR(g_lowest_address)]
- jb LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
+ jb LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
cmp rdi, [C_VAR(g_highest_address)]
- jae LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
+ jae LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
// Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless
// we're in a debug build and write barrier checking has been enabled).
@@ -293,9 +292,9 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable):
// If the reference is to an object that's not in an ephemeral generation we have no need to track it
// (since the object won't be collected or moved by an ephemeral collection).
cmp rcx, [C_VAR(g_ephemeral_low)]
- jb LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
+ jb LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
cmp rcx, [C_VAR(g_ephemeral_high)]
- jae LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
+ jae LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
// move current rdi value into rcx, we need to keep rdi and eventually increment by 8
mov rcx, rdi
@@ -307,7 +306,7 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable):
shr rcx, 0x0B
mov r10, [C_VAR(g_card_table)]
cmp byte ptr [rcx + r10], 0x0FF
- je LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
+ je LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
// We get here if it's necessary to update the card table.
mov byte ptr [rcx + r10], 0xFF
@@ -317,12 +316,12 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable):
shr rcx, 0x0A
add rcx, [C_VAR(g_card_bundle_table)]
cmp byte ptr [rcx], 0xFF
- je LOCAL_LABEL(RhpByRefAssignRef_NotInHeap)
+ je LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired)
mov byte ptr [rcx], 0xFF
#endif
-LOCAL_LABEL(RhpByRefAssignRef_NotInHeap):
+LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired):
// Increment the pointers before leaving
add rdi, 0x8
add rsi, 0x8
diff --git a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm b/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm
index 5a6dcc666fe..753ca456097 100644
--- a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm
+++ b/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm
@@ -100,6 +100,21 @@ DEFINE_UNCHECKED_WRITE_BARRIER_CORE macro BASENAME, REFREG
;; we're in a debug build and write barrier checking has been enabled).
UPDATE_GC_SHADOW BASENAME, REFREG, rcx
+ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+ mov r11, [g_write_watch_table]
+ cmp r11, 0
+ je &BASENAME&_CheckCardTable_&REFREG&
+
+ mov r10, rcx
+ shr r10, 0Ch ;; SoftwareWriteWatch::AddressToTableByteIndexShift
+ add r10, r11
+ cmp byte ptr [r10], 0
+ jne &BASENAME&_CheckCardTable_&REFREG&
+ mov byte ptr [r10], 0FFh
+endif
+
+&BASENAME&_CheckCardTable_&REFREG&:
+
;; If the reference is to an object that's not in an ephemeral generation we have no need to track it
;; (since the object won't be collected or moved by an ephemeral collection).
cmp REFREG, [g_ephemeral_low]
@@ -111,17 +126,25 @@ DEFINE_UNCHECKED_WRITE_BARRIER_CORE macro BASENAME, REFREG
;; track this write. The location address is translated into an offset in the card table bitmap. We set
;; an entire byte in the card table since it's quicker than messing around with bitmasks and we only write
;; the byte if it hasn't already been done since writes are expensive and impact scaling.
- shr rcx, 11
- add rcx, [g_card_table]
+ shr rcx, 0Bh
+ mov r10, [g_card_table]
+ cmp byte ptr [rcx + r10], 0FFh
+ je &BASENAME&_NoBarrierRequired_&REFREG&
+
+ ;; We get here if it's necessary to update the card table.
+ mov byte ptr [rcx + r10], 0FFh
+
+ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+ ;; Shift rcx by 0Ah more to get the card bundle byte (we shifted by 0x0B already)
+ shr rcx, 0Ah
+ add rcx, [g_card_bundle_table]
cmp byte ptr [rcx], 0FFh
- jne &BASENAME&_UpdateCardTable_&REFREG&
-
-&BASENAME&_NoBarrierRequired_&REFREG&:
- ret
+ je &BASENAME&_NoBarrierRequired_&REFREG&
-;; We get here if it's necessary to update the card table.
-&BASENAME&_UpdateCardTable_&REFREG&:
mov byte ptr [rcx], 0FFh
+endif
+
+&BASENAME&_NoBarrierRequired_&REFREG&:
ret
endm
@@ -248,11 +271,10 @@ LEAF_END RhpCheckedXchg, _TEXT
;; On entry:
;; rdi: address of ref-field (assigned to)
;; rsi: address of the data (source)
-;; rcx: be trashed
;;
;; On exit:
;; rdi, rsi are incremented by 8,
-;; rcx: trashed
+;; rcx, r10, r11: trashed
;;
LEAF_ENTRY RhpByRefAssignRef, _TEXT
mov rcx, [rsi]
@@ -260,43 +282,63 @@ LEAF_ENTRY RhpByRefAssignRef, _TEXT
;; Check whether the writes were even into the heap. If not there's no card update required.
cmp rdi, [g_lowest_address]
- jb RhpByRefAssignRef_NotInHeap
+ jb RhpByRefAssignRef_NoBarrierRequired
cmp rdi, [g_highest_address]
- jae RhpByRefAssignRef_NotInHeap
+ jae RhpByRefAssignRef_NoBarrierRequired
;; Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless
;; we're in a debug build and write barrier checking has been enabled).
UPDATE_GC_SHADOW BASENAME, rcx, rdi
+ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+ mov r11, [g_write_watch_table]
+ cmp r11, 0
+ je RhpByRefAssignRef_CheckCardTable
+
+ mov r10, rdi
+ shr r10, 0Ch ;; SoftwareWriteWatch::AddressToTableByteIndexShift
+ add r10, r11
+ cmp byte ptr [r10], 0
+ jne RhpByRefAssignRef_CheckCardTable
+ mov byte ptr [r10], 0FFh
+endif
+
+RhpByRefAssignRef_CheckCardTable:
+
;; If the reference is to an object that's not in an ephemeral generation we have no need to track it
;; (since the object won't be collected or moved by an ephemeral collection).
cmp rcx, [g_ephemeral_low]
- jb RhpByRefAssignRef_NotInHeap
+ jb RhpByRefAssignRef_NoBarrierRequired
cmp rcx, [g_ephemeral_high]
- jae RhpByRefAssignRef_NotInHeap
+ jae RhpByRefAssignRef_NoBarrierRequired
- ;; move current rdi value into rcx and then increment the pointers
+ ;; move current rdi value into rcx, we need to keep rdi and eventually increment by 8
mov rcx, rdi
- add rsi, 8h
- add rdi, 8h
;; We have a location on the GC heap being updated with a reference to an ephemeral object so we must
;; track this write. The location address is translated into an offset in the card table bitmap. We set
;; an entire byte in the card table since it's quicker than messing around with bitmasks and we only write
;; the byte if it hasn't already been done since writes are expensive and impact scaling.
- shr rcx, 11
- add rcx, [g_card_table]
- cmp byte ptr [rcx], 0FFh
- jne RhpByRefAssignRef_UpdateCardTable
- ret
+ shr rcx, 0Bh
+ mov r10, [g_card_table]
+ cmp byte ptr [rcx + r10], 0FFh
+ je RhpByRefAssignRef_NoBarrierRequired
;; We get here if it's necessary to update the card table.
-RhpByRefAssignRef_UpdateCardTable:
+ mov byte ptr [rcx + r10], 0FFh
+
+ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+ ;; Shift rcx by 0Ah more to get the card bundle byte (we shifted by 0Bh already)
+ shr rcx, 0Ah
+ add rcx, [g_card_bundle_table]
+ cmp byte ptr [rcx], 0FFh
+ je RhpByRefAssignRef_NoBarrierRequired
+
mov byte ptr [rcx], 0FFh
- ret
+endif
-RhpByRefAssignRef_NotInHeap:
- ; Increment the pointers before leaving
+RhpByRefAssignRef_NoBarrierRequired:
+ ;; Increment the pointers before leaving
add rdi, 8h
add rsi, 8h
ret
diff --git a/src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h b/src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h
index d6e3074fbfc..de4aef3d267 100644
--- a/src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h
+++ b/src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h
@@ -108,6 +108,14 @@ OFFSETOF__Thread__m_alloc_context__alloc_limit equ OFFSETOF__Thread__m_rgbA
EXTERN g_ephemeral_high
EXTERN g_card_table
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+ EXTERN g_card_bundle_table
+#endif
+
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+ EXTERN g_write_watch_table
+#endif
+
;; -----------------------------------------------------------------------------
;; Macro used to assign an alternate name to a symbol containing characters normally disallowed in a symbol
@@ -146,6 +154,15 @@ MovInstr SETS "movk"
$MovInstr $Reg, #(($ConstantLo):AND:0xffff)
MEND
+;;-----------------------------------------------------------------------------
+;; Macro for loading a 64bit value of a global variable into a register
+ MACRO
+ PREPARE_EXTERNAL_VAR_INDIRECT $Name, $Reg
+
+ adrp $Reg, $Name
+ ldr $Reg, [$Reg, $Name]
+ MEND
+
;; -----------------------------------------------------------------------------
;;
;; Macro to export a pointer to an address inside a stub as a 64-bit variable
diff --git a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S
index 8d908d993ae..8fbbce7fd81 100644
--- a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S
+++ b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S
@@ -22,29 +22,29 @@
// $refReg: objectref to be stored
//
// On exit:
- // x9,x10: trashed
+ // x12,x17: trashed
// other registers are preserved
//
.macro UPDATE_GC_SHADOW destReg, refReg
// If g_GCShadow is 0, don't perform the check.
- PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, X9
- cbz x9, 1f
+ PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, X12
+ cbz x12, 1f
// Save destReg since we're about to modify it (and we need the original value both within the macro and
// once we exit the macro).
- mov x10, \destReg
+ mov x17, \destReg
// Transform destReg into the equivalent address in the shadow heap.
- PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, X9
- subs \destReg, \destReg, x9
+ PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, X12
+ subs \destReg, \destReg, x12
blo 0f
- PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, X9
- add \destReg, \destReg, x9
+ PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadow, X12
+ add \destReg, \destReg, x12
- PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, X9
- cmp \destReg, x9
+ PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, X12
+ cmp \destReg, x12
bhs 0f
// Update the shadow heap.
@@ -55,20 +55,20 @@
dmb ish
// Now check that the real heap location still contains the value we just wrote into the shadow heap.
- mov x9, x10
- ldr x9, [x9]
- cmp x9, \refReg
+ mov x12, x17
+ ldr x12, [x12]
+ cmp x12, \refReg
beq 0f
// Someone went and updated the real heap. We need to invalidate INVALIDGCVALUE the shadow location since we can not
// guarantee whose shadow update won.
- movz x9, (INVALIDGCVALUE & 0xFFFF) // #0xcccd
- movk x9, ((INVALIDGCVALUE >> 16) & 0xFFFF), LSL #16
- str x9, [\destReg]
+ movz x12, (INVALIDGCVALUE & 0xFFFF) // #0xcccd
+ movk x12, ((INVALIDGCVALUE >> 16) & 0xFFFF), LSL #16
+ str x12, [\destReg]
0:
// Restore original destReg value
- mov \destReg, x10
+ mov \destReg, x17
1:
.endm
@@ -89,14 +89,13 @@
// some interlocked helpers that need an inline barrier.
// On entry:
- // destReg: location to be updated
- // refReg: objectref to be stored
- // trash: register nr than can be trashed
+ // destReg: location to be updated (cannot be x12,x17)
+ // refReg: objectref to be stored (cannot be x12,x17)
//
// On exit:
- // destReg: trashed
+ // x12,x17: trashed
//
- .macro INSERT_UNCHECKED_WRITE_BARRIER_CORE destReg, refReg, trash
+ .macro INSERT_UNCHECKED_WRITE_BARRIER_CORE destReg, refReg
// Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless
// we are in a debug build and write barrier checking has been enabled).
@@ -104,51 +103,50 @@
#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
// Update the write watch table if necessary
- PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x\trash
-
- cbz x\trash, 2f
- add x\trash, x\trash, \destReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift
- ldrb w17, [x\trash]
- cbnz x17, 2f
- mov w17, #0xFF
- strb w17, [x\trash]
+ PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12
+
+ cbz x12, 2f
+ add x12, x12, \destReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift
+ ldrb w17, [x12]
+ cbnz x17, 2f
+ mov w17, #0xFF
+ strb w17, [x12]
#endif
2:
// We can skip the card table write if the reference is to
// an object not on the epehemeral segment.
- PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_low, x\trash
- cmp \refReg, x\trash
+ PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_low, x12
+ cmp \refReg, x12
blo 0f
- PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_high, x\trash
- cmp \refReg, x\trash
+ PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_high, x12
+ cmp \refReg, x12
bhs 0f
// Set this objects card, if it has not already been set.
-
- PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x\trash
- add x17, x\trash, \destReg, lsr #11
+ PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x12
+ add x17, x12, \destReg, lsr #11
// Check that this card has not already been written. Avoiding useless writes is a big win on
// multi-proc systems since it avoids cache thrashing.
- ldrb w\trash, [x17]
- cmp x\trash, 0xFF
+ ldrb w12, [x17]
+ cmp x12, 0xFF
beq 0f
- mov x\trash, 0xFF
- strb w\trash, [x17]
+ mov x12, 0xFF
+ strb w12, [x17]
#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
// Check if we need to update the card bundle table
- PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x\trash
- add x17, x\trash, \destReg, lsr #21
- ldrb w\trash, [x17]
- cmp x\trash, 0xFF
- beq 0f
-
- mov x\trash, 0xFF
- strb w\trash, [x17]
+ PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+ add x17, x12, \destReg, lsr #21
+ ldrb w12, [x17]
+ cmp x12, 0xFF
+ beq 0f
+
+ mov x12, 0xFF
+ strb w12, [x17]
#endif
0:
@@ -158,73 +156,111 @@
// On entry:
// destReg: location to be updated
// refReg: objectref to be stored
- // trash: register nr than can be trashed
//
// On exit:
- // destReg: trashed
+ // x12, x17: trashed
//
- .macro INSERT_CHECKED_WRITE_BARRIER_CORE destReg, refReg, trash
+ .macro INSERT_CHECKED_WRITE_BARRIER_CORE destReg, refReg
// The "check" of this checked write barrier - is destReg
// within the heap? if no, early out.
- PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x\trash
- cmp \destReg, x\trash
+ PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x12
+ cmp \destReg, x12
- PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x\trash
+ PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x12
// If \destReg >= g_lowest_address, compare \destReg to g_highest_address.
// Otherwise, set the C flag (0x2) to take the next branch.
- ccmp \destReg, x\trash, #0x2, hs
+ ccmp \destReg, x12, #0x2, hs
bhs 0f
- INSERT_UNCHECKED_WRITE_BARRIER_CORE \destReg, \refReg, \trash
+ INSERT_UNCHECKED_WRITE_BARRIER_CORE \destReg, \refReg
0:
// Exit label
.endm
-// RhpCheckedAssignRef(Object** dst, Object* src)
+// void JIT_ByRefWriteBarrier
+// On entry:
+// x13 : the source address (points to object reference to write)
+// x14 : the destination address (object reference written here)
+//
+// On exit:
+// x13 : incremented by 8
+// x14 : incremented by 8
+// x15 : trashed
+// x12, x17 : trashed
+//
+LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT
+
+ ldr x15, [x13], 8
+ b C_FUNC(RhpCheckedAssignRefArm64)
+
+LEAF_END RhpByRefAssignRefArm64, _TEXT
+
+// JIT_CheckedWriteBarrier(Object** dst, Object* src)
//
// Write barrier for writes to objects that may reside
// on the managed heap.
//
// On entry:
-// x0 : the destination address (LHS of the assignment).
-// May not be an object reference (hence the checked).
-// x1 : the object reference (RHS of the assignment).
+// x14 : the destination address (LHS of the assignment).
+// May not be a heap location (hence the checked).
+// x15 : the object reference (RHS of the assignment).
+//
// On exit:
-// x1 : trashed
-// x9 : trashed
- LEAF_ENTRY RhpCheckedAssignRef, _TEXT
- ALTERNATE_ENTRY RhpCheckedAssignRefX1
+// x12, x17 : trashed
+// x14 : incremented by 8
+ LEAF_ENTRY RhpCheckedAssignRefArm64, _TEXT
- mov x14, x0 ; x14 = dst
- mov x15, x1 ; x15 = val
- b C_FUNC(RhpCheckedAssignRefArm64)
+ // is destReg within the heap?
+ PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x12
+ cmp x14, x12
-LEAF_END RhpCheckedAssignRef, _TEXT
+ PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x12
+ ccmp x14, x12, #0x2, hs
+ blo C_FUNC(RhpAssignRefArm64)
+
+NotInHeap:
+ ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
+ str x15, [x14], 8
+ ret
-// RhpAssignRef(Object** dst, Object* src)
+LEAF_END RhpCheckedAssignRefArm64, _TEXT
+
+// JIT_WriteBarrier(Object** dst, Object* src)
//
// Write barrier for writes to objects that are known to
// reside on the managed heap.
//
// On entry:
-// x0 : the destination address (LHS of the assignment).
-// x1 : the object reference (RHS of the assignment).
+// x14 : the destination address (LHS of the assignment).
+// x15 : the object reference (RHS of the assignment).
+//
// On exit:
-// x1 : trashed
-// x9 : trashed
-LEAF_ENTRY RhpAssignRef, _TEXT
- ALTERNATE_ENTRY RhpAssignRefX1
+// x12, x17 : trashed
+// x14 : incremented by 8
+LEAF_ENTRY RhpAssignRefArm64, _TEXT
+ ALTERNATE_ENTRY RhpAssignRefAVLocation
+ stlr x15, [x14]
+
+ INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15
+
+ add x14, x14, 8
+ ret
+
+LEAF_END RhpAssignRefArm64, _TEXT
+
+// Same as RhpAssignRefArm64, but with standard ABI.
+LEAF_ENTRY RhpAssignRef, _TEXT
mov x14, x0 ; x14 = dst
mov x15, x1 ; x15 = val
b C_FUNC(RhpAssignRefArm64)
-
LEAF_END RhpAssignRef, _TEXT
+
// Interlocked operation helpers where the location is an objectref, thus requiring a GC write barrier upon
// successful updates.
@@ -243,27 +279,26 @@ LEAF_END RhpAssignRef, _TEXT
//
// On exit:
// x0: original value of objectref
-// x9: trashed
-// x10: trashed
+// x10, x12, x17: trashed
//
LEAF_ENTRY RhpCheckedLockCmpXchg
- ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
CmpXchgRetry:
// Check location value is what we expect.
+ ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
ldaxr x10, [x0]
cmp x10, x2
bne CmpXchgNoUpdate
// Current value matches comparand, attempt to update with the new value.
- stlxr w9, x1, [x0]
- cbnz w9, CmpXchgRetry
+ stlxr w12, x1, [x0]
+ cbnz w12, CmpXchgRetry
// We have successfully updated the value of the objectref so now we need a GC write barrier.
// The following barrier code takes the destination in x0 and the value in x1 so the arguments are
// already correctly set up.
- INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1, 9
+ INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1
CmpXchgNoUpdate:
// x10 still contains the original value.
@@ -287,25 +322,25 @@ CmpXchgNoUpdate:
//
// On exit:
// x0: original value of objectref
-// x9: trashed
// x10: trashed
+// x12, x17: trashed
//
LEAF_ENTRY RhpCheckedXchg, _TEXT
- ALTERNATE_ENTRY RhpCheckedXchgAVLocation
ExchangeRetry:
// Read the existing memory location.
+ ALTERNATE_ENTRY RhpCheckedXchgAVLocation
ldaxr x10, [x0]
// Attempt to update with the new value.
- stlxr w9, x1, [x0]
- cbnz w9, ExchangeRetry
+ stlxr w12, x1, [x0]
+ cbnz w12, ExchangeRetry
// We have successfully updated the value of the objectref so now we need a GC write barrier.
// The following barrier code takes the destination in x0 and the value in x1 so the arguments are
// already correctly set up.
- INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1, 9
+ INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1
// x10 still contains the original value.
mov x0, x10
@@ -313,59 +348,3 @@ ExchangeRetry:
ret
LEAF_END RhpCheckedXchg, _TEXT
-
-LEAF_ENTRY RhpAssignRefArm64, _TEXT
- ALTERNATE_ENTRY RhpAssignRefAVLocation
- ALTERNATE_ENTRY RhpAssignRefX1AVLocation
- stlr x15, [x14]
-
- INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15, 12
-
- ret
-LEAF_END RhpAssignRefArm64, _TEXT
-
-// void JIT_CheckedWriteBarrier(Object** dst, Object* src)
-// On entry:
-// x14 : the destination address (LHS of the assignment)
-// x15 : the object reference (RHS of the assignment)
-//
-// On exit:
-// x12 : trashed
-// x14 : trashed (incremented by 8 to implement JIT_ByRefWriteBarrier contract)
-// x15 : trashed
-// x17 : trashed (ip1) if FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
-//
-LEAF_ENTRY RhpCheckedAssignRefArm64, _TEXT
- ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
- ALTERNATE_ENTRY RhpCheckedAssignRefX1AVLocation
-
- stlr x15, [x14]
-
- INSERT_CHECKED_WRITE_BARRIER_CORE x14, x15, 12
-
- ret
-LEAF_END RhpCheckedAssignRefArm64, _TEXT
-
-// void JIT_ByRefWriteBarrier
-// On entry:
-// x13 : the source address (points to object reference to write)
-// x14 : the destination address (object reference written here)
-//
-// On exit:
-// x12 : trashed
-// x13 : incremented by 8
-// x14 : incremented by 8
-// x15 : trashed
-// x17 : trashed (ip1) if FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
-//
-LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT
- ldr x15, [x13]
- stlr x15, [x14]
-
- INSERT_CHECKED_WRITE_BARRIER_CORE x14, x15, 12
-
- add X13, x13, #8
- add x14, x14, #8
-
- ret
-LEAF_END RhpByRefAssignRefArm64, _TEXT
diff --git a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm
index 005e80a12cd..62639a7a6de 100644
--- a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm
+++ b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm
@@ -42,8 +42,7 @@ INVALIDGCVALUE EQU 0xCCCCCCCD
UPDATE_GC_SHADOW $destReg, $refReg
;; If g_GCShadow is 0, don't perform the check.
- adrp x12, $g_GCShadow
- ldr x12, [x12, $g_GCShadow]
+ PREPARE_EXTERNAL_VAR_INDIRECT $g_GCShadow, x12
cbz x12, %ft1
;; Save $destReg since we're about to modify it (and we need the original value both within the macro and
@@ -51,17 +50,14 @@ INVALIDGCVALUE EQU 0xCCCCCCCD
mov x17, $destReg
;; Transform $destReg into the equivalent address in the shadow heap.
- adrp x12, g_lowest_address
- ldr x12, [x12, g_lowest_address]
+ PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x12
subs $destReg, $destReg, x12
blo %ft0
- adrp x12, $g_GCShadow
- ldr x12, [x12, $g_GCShadow]
+ PREPARE_EXTERNAL_VAR_INDIRECT $g_GCShadow, x12
add $destReg, $destReg, x12
- adrp x12, $g_GCShadowEnd
- ldr x12, [x12, $g_GCShadowEnd]
+ PREPARE_EXTERNAL_VAR_INDIRECT g_GCShadowEnd, x12
cmp $destReg, x12
bhs %ft0
@@ -109,44 +105,63 @@ INVALIDGCVALUE EQU 0xCCCCCCCD
;; On entry:
;; $destReg: location to be updated (cannot be x12,x17)
;; $refReg: objectref to be stored (cannot be x12,x17)
- ;; $trashReg: register that can be trashed (can be $destReg or $refReg)
;;
;; On exit:
- ;; $trashReg: trashed
- ;; x12: trashed
- ;; x17: trashed if WRITE_BARRIER_CHECK
+ ;; x12,x17: trashed
;;
- INSERT_UNCHECKED_WRITE_BARRIER_CORE $destReg, $refReg, $trashReg
+ INSERT_UNCHECKED_WRITE_BARRIER_CORE $destReg, $refReg
;; Update the shadow copy of the heap with the same value just written to the same heap. (A no-op unless
;; we're in a debug build and write barrier checking has been enabled).
UPDATE_GC_SHADOW $destReg, $refReg
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+ // Update the write watch table if necessary
+ PREPARE_EXTERNAL_VAR_INDIRECT g_write_watch_table, x12
+
+ cbz x12, %ft2
+ add x12, x12, $destReg, lsr #0xc // SoftwareWriteWatch::AddressToTableByteIndexShift
+ ldrb w17, [x12]
+ cbnz x17, %ft2
+ mov w17, #0xFF
+ strb w17, [x12]
+#endif
+
+2
;; We can skip the card table write if the reference is to
;; an object not on the epehemeral segment.
- adrp x12, g_ephemeral_low
- ldr x12, [x12, g_ephemeral_low]
+ PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_low, x12
cmp $refReg, x12
blo %ft0
- adrp x12, g_ephemeral_high
- ldr x12, [x12, g_ephemeral_high]
+ PREPARE_EXTERNAL_VAR_INDIRECT g_ephemeral_high, x12
cmp $refReg, x12
bhs %ft0
;; Set this object's card, if it hasn't already been set.
- adrp x12, g_card_table
- ldr x12, [x12, g_card_table]
- add $trashReg, x12, $destReg lsr #11
+ PREPARE_EXTERNAL_VAR_INDIRECT g_card_table, x12
+ add x17, x12, $destReg lsr #11
;; Check that this card hasn't already been written. Avoiding useless writes is a big win on
;; multi-proc systems since it avoids cache trashing.
- ldrb w12, [$trashReg]
+ ldrb w12, [x17]
+ cmp x12, 0xFF
+ beq %ft0
+
+ mov x12, 0xFF
+ strb w12, [x17]
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+ // Check if we need to update the card bundle table
+ PREPARE_EXTERNAL_VAR_INDIRECT g_card_bundle_table, x12
+ add x17, x12, $destReg, lsr #21
+ ldrb w12, [x17]
cmp x12, 0xFF
beq %ft0
mov x12, 0xFF
- strb w12, [$trashReg]
+ strb w12, [x17]
+#endif
0
;; Exit label
@@ -156,82 +171,111 @@ INVALIDGCVALUE EQU 0xCCCCCCCD
;; On entry:
;; $destReg: location to be updated (cannot be x12,x17)
;; $refReg: objectref to be stored (cannot be x12,x17)
- ;; $trashReg: register that can be trashed (can be $destReg or $refReg)
;;
;; On exit:
- ;; $trashReg: trashed
- ;; x12: trashed
- ;; x17: trashed if WRITE_BARRIER_CHECK
+ ;; x12, x17: trashed
;;
- INSERT_CHECKED_WRITE_BARRIER_CORE $destReg, $refReg, $trashReg
+ INSERT_CHECKED_WRITE_BARRIER_CORE $destReg, $refReg
;; The "check" of this checked write barrier - is $destReg
;; within the heap? if no, early out.
- adrp x12, g_lowest_address
- ldr x12, [x12, g_lowest_address]
+ PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x12
cmp $destReg, x12
- adrp x12, g_highest_address
- ldr x12, [x12, g_highest_address]
+ PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x12
;; If $destReg >= g_lowest_address, compare $destReg to g_highest_address.
;; Otherwise, set the C flag (0x2) to take the next branch.
ccmp $destReg, x12, #0x2, hs
bhs %ft0
- INSERT_UNCHECKED_WRITE_BARRIER_CORE $destReg, $refReg, $trashReg
+ INSERT_UNCHECKED_WRITE_BARRIER_CORE $destReg, $refReg
0
;; Exit label
MEND
-;; RhpCheckedAssignRef(Object** dst, Object* src)
+;; void JIT_ByRefWriteBarrier
+;; On entry:
+;; x13 : the source address (points to object reference to write)
+;; x14 : the destination address (object reference written here)
+;;
+;; On exit:
+;; x13 : incremented by 8
+;; x14 : incremented by 8
+;; x15 : trashed
+;; x12, x17 : trashed
+;;
+ LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT
+
+ ldr x15, [x13], 8
+ b RhpCheckedAssignRefArm64
+
+ LEAF_END RhpByRefAssignRefArm64
+
+
+;; JIT_CheckedWriteBarrier(Object** dst, Object* src)
;;
;; Write barrier for writes to objects that may reside
;; on the managed heap.
;;
;; On entry:
-;; x0 : the destination address (LHS of the assignment).
-;; May not be an object reference (hence the checked).
-;; x1 : the object reference (RHS of the assignment)
+;; x14 : the destination address (LHS of the assignment).
+;; May not be a heap location (hence the checked).
+;; x15 : the object reference (RHS of the assignment)
;;
;; On exit:
-;; x12 : trashed
-;; x14 : trashed
-;; x15 : trashed
-;; x17 : trashed if WRITE_BARRIER_CHECK
- LEAF_ENTRY RhpCheckedAssignRef
- ALTERNATE_ENTRY RhpCheckedAssignRefX1
-
- mov x14, x0 ; x14 = dst
- mov x15, x1 ; x15 = val
- b RhpCheckedAssignRefArm64
+;; x12, x17 : trashed
+;; x14 : incremented by 8
+ LEAF_ENTRY RhpCheckedAssignRefArm64
+
+ ;; is destReg within the heap?
+ PREPARE_EXTERNAL_VAR_INDIRECT g_lowest_address, x12
+ cmp x14, x12
+
+ PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x12
+ ccmp x14, x12, #0x2, hs
+ blo RhpAssignRefArm64
+
+NotInHeap
+ ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
+ str x15, [x14], 8
+ ret
- LEAF_END RhpCheckedAssignRef
+ LEAF_END RhpCheckedAssignRefArm64
-;; RhpAssignRef(Object** dst, Object* src)
+;; JIT_WriteBarrier(Object** dst, Object* src)
;;
;; Write barrier for writes to objects that are known to
;; reside on the managed heap.
;;
;; On entry:
-;; x0 : the destination address (LHS of the assignment)
-;; x1 : the object reference (RHS of the assignment)
+;; x14 : the destination address (LHS of the assignment)
+;; x15 : the object reference (RHS of the assignment)
;;
;; On exit:
-;; x12 : trashed
-;; x14 : trashed
-;; x15 : trashed
-;; x17 : trashed if WRITE_BARRIER_CHECK
- LEAF_ENTRY RhpAssignRef
- ALTERNATE_ENTRY RhpAssignRefX1
+;; x12, x17 : trashed
+;; x14 : incremented by 8
+ LEAF_ENTRY RhpAssignRefArm64
- mov x14, x0 ; x14 = dst
- mov x15, x1 ; x15 = val
- b RhpAssignRefArm64
+ ALTERNATE_ENTRY RhpAssignRefAVLocation
+ stlr x15, [x14]
+
+ INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15
+ add x14, x14, 8
+ ret
+
+ LEAF_END RhpAssignRefArm64
+
+;; same as RhpAssignRefArm64, but with standard ABI.
+ LEAF_ENTRY RhpAssignRef
+ mov x14, x0 ; x14 = dst
+ mov x15, x1 ; x15 = val
+ b RhpAssignRefArm64
LEAF_END RhpAssignRef
+
;; Interlocked operation helpers where the location is an objectref, thus requiring a GC write barrier upon
;; successful updates.
@@ -249,30 +293,27 @@ INVALIDGCVALUE EQU 0xCCCCCCCD
;; x2 : comparand
;;
;; On exit:
-;; x0 : original value of objectref
-;; x9 : trashed
-;; x10 : trashed
-;; x12 : trashed
-;; x17 : trashed if WRITE_BARRIER_CHECK
+;; x0: original value of objectref
+;; x10, x12, x17: trashed
;;
LEAF_ENTRY RhpCheckedLockCmpXchg
- ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
CmpXchgRetry
;; Check location value is what we expect.
+ ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation
ldaxr x10, [x0]
cmp x10, x2
bne CmpXchgNoUpdate
;; Current value matches comparand, attempt to update with the new value.
- stlxr w9, x1, [x0]
- cbnz w9, CmpXchgRetry
+ stlxr w12, x1, [x0]
+ cbnz w12, CmpXchgRetry
;; We've successfully updated the value of the objectref so now we need a GC write barrier.
;; The following barrier code takes the destination in x0 and the value in x1 so the arguments are
;; already correctly set up.
- INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1, x0
+ INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1
CmpXchgNoUpdate
;; x10 still contains the original value.
@@ -295,28 +336,26 @@ CmpXchgNoUpdate
;; x1 : exchange value
;;
;; On exit:
-;; x0 : original value of objectref
-;; x9 : trashed
-;; x10 : trashed
-;; x12 : trashed
-;; x17 : trashed if WRITE_BARRIER_CHECK
+;; x0: original value of objectref
+;; x10: trashed
+;; x12, x17: trashed
;;
LEAF_ENTRY RhpCheckedXchg
- ALTERNATE_ENTRY RhpCheckedXchgAVLocation
ExchangeRetry
;; Read the existing memory location.
+ ALTERNATE_ENTRY RhpCheckedXchgAVLocation
ldaxr x10, [x0]
;; Attempt to update with the new value.
- stlxr w9, x1, [x0]
- cbnz w9, ExchangeRetry
+ stlxr w12, x1, [x0]
+ cbnz w12, ExchangeRetry
;; We've successfully updated the value of the objectref so now we need a GC write barrier.
;; The following barrier code takes the destination in x0 and the value in x1 so the arguments are
;; already correctly set up.
- INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1, x0
+ INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1
;; x10 still contains the original value.
mov x0, x10
@@ -325,71 +364,4 @@ ExchangeRetry
LEAF_END RhpCheckedXchg
-;; RhpAssignRefArm64(Object** dst, Object* src)
-;;
-;; On entry:
-;; x14 : the destination address (LHS of the assignment).
-;; x15 : the object reference (RHS of the assignment).
-;;
-;; On exit:
-;; x12 : trashed
-;; x15 : trashed
-;; x17 : trashed if WRITE_BARRIER_CHECK
-;;
- LEAF_ENTRY RhpAssignRefArm64, _TEXT
- ALTERNATE_ENTRY RhpAssignRefAVLocation
- ALTERNATE_ENTRY RhpAssignRefX1AVLocation
-
- stlr x15, [x14]
-
- INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15, x15
-
- ret
- LEAF_END RhpAssignRefArm64
-
-;; void JIT_CheckedWriteBarrier(Object** dst, Object* src)
-;; On entry:
-;; x14 : the destination address (LHS of the assignment)
-;; x15 : the object reference (RHS of the assignment)
-;;
-;; On exit:
-;; x12 : trashed
-;; x15 : trashed
-;; x17 : trashed if WRITE_BARRIER_CHECK
-;;
- LEAF_ENTRY RhpCheckedAssignRefArm64, _TEXT
- ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
- ALTERNATE_ENTRY RhpCheckedAssignRefX1AVLocation
-
- stlr x15, [x14]
-
- INSERT_CHECKED_WRITE_BARRIER_CORE x14, x15, x15
-
- ret
- LEAF_END RhpCheckedAssignRefArm64
-
-;; void JIT_ByRefWriteBarrier
-;; On entry:
-;; x13 : the source address (points to object reference to write)
-;; x14 : the destination address (object reference written here)
-;;
-;; On exit:
-;; x12 : trashed
-;; x13 : incremented by 8
-;; x14 : incremented by 8
-;; x15 : trashed
-;; x17 : trashed if WRITE_BARRIER_CHECK
-;;
- LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT
- ldr x15, [x13]
- stlr x15, [x14]
-
- INSERT_CHECKED_WRITE_BARRIER_CORE x14, x15, x15
-
- add x13, x13, #8
- add x14, x14, #8
-
- ret
- LEAF_END RhpByRefAssignRefArm64
-
end
diff --git a/src/coreclr/nativeaot/Runtime/gcrhenv.cpp b/src/coreclr/nativeaot/Runtime/gcrhenv.cpp
index ead3936f3bd..6d8b819b3a0 100644
--- a/src/coreclr/nativeaot/Runtime/gcrhenv.cpp
+++ b/src/coreclr/nativeaot/Runtime/gcrhenv.cpp
@@ -1029,10 +1029,6 @@ void GCToEEInterface::DiagWalkBGCSurvivors(void* gcContext)
#endif // FEATURE_EVENT_TRACE
}
-#if defined(FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP) && !defined(TARGET_UNIX)
-#error FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP is only implemented for UNIX
-#endif
-
void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args)
{
// NativeAOT doesn't patch the write barrier like CoreCLR does, but it