Add helpers that perform memory copy and gc-write barrier as a single operation

[tfs-changeset: 1558883]

6 files changed, 193 insertions, 14 deletions

diff --git a/src/Native/Runtime/EHHelpers.cpp b/src/Native/Runtime/EHHelpers.cpp
index 9d260b683..e538b1d58 100644
--- a/src/Native/Runtime/EHHelpers.cpp
+++ b/src/Native/Runtime/EHHelpers.cpp
@@ -340,6 +340,8 @@ EXTERN_C void * RhpCopyMultibyteDestAVLocation;
 EXTERN_C void * RhpCopyMultibyteSrcAVLocation;
 EXTERN_C void * RhpCopyMultibyteNoGCRefsDestAVLocation;
 EXTERN_C void * RhpCopyMultibyteNoGCRefsSrcAVLocation;
+EXTERN_C void * RhpCopyMultibyteWithWriteBarrierDestAVLocation;
+EXTERN_C void * RhpCopyMultibyteWithWriteBarrierSrcAVLocation;
 
 static bool InWriteBarrierHelper(UIntNative faultingIP)
 {
@@ -354,6 +356,8 @@ static bool InWriteBarrierHelper(UIntNative faultingIP)
         (UIntNative)&RhpCopyMultibyteSrcAVLocation,
         (UIntNative)&RhpCopyMultibyteNoGCRefsDestAVLocation,
         (UIntNative)&RhpCopyMultibyteNoGCRefsSrcAVLocation,
+        (UIntNative)&RhpCopyMultibyteWithWriteBarrierDestAVLocation,
+        (UIntNative)&RhpCopyMultibyteWithWriteBarrierSrcAVLocation,
     };
 
     // compare the IP against the list of known possible AV locations in the write barrier helpers
diff --git a/src/Native/Runtime/MiscHelpers.cpp b/src/Native/Runtime/MiscHelpers.cpp
index a08d113b7..81475383d 100644
--- a/src/Native/Runtime/MiscHelpers.cpp
+++ b/src/Native/Runtime/MiscHelpers.cpp
@@ -601,6 +601,31 @@ COOP_PINVOKE_CDECL_HELPER(void *, memcpyGCRefs, (void * dest, const void *src, s
     return dest;
 }
 
+EXTERN_C void REDHAWK_CALLCONV RhpBulkWriteBarrier(void* pMemStart, UInt32 cbMemSize);
+
+// This is a GC-safe variant of memcpy.  It guarantees that the object references in the GC heap are updated atomically.
+// This is required for type safety and proper operation of the background GC.
+// Writebarrier is included.
+//
+// USAGE:
+//          1) The caller is responsible for hoisting any null reference exceptions to a place where the hardware 
+//             exception can be properly translated to a managed exception.  This is handled by RhpCopyMultibyte.
+//          2) The caller must ensure that all three parameters are pointer-size-aligned.  This should be the case for
+//             value types which contain GC refs anyway, so if you want to copy structs without GC refs which might be
+//             unaligned, then you must use RhpCopyMultibyteNoGCRefs.
+COOP_PINVOKE_CDECL_HELPER(void *, memcpyGCRefsWithWriteBarrier, (void * dest, const void *src, size_t len))
+{
+    // null pointers are not allowed (they are checked by RhpCopyMultibyteWithWriteBarrier)
+    ASSERT(dest != nullptr);
+    ASSERT(src != nullptr);
+
+    ForwardGCSafeCopy(dest, src, len);
+    RhpBulkWriteBarrier(dest, (UInt32)len);
+
+    // memcpy returns the destination buffer
+    return dest;
+}
+
 // This function clears a piece of memory in a GC safe way.  It makes the guarantee that it will clear memory in at 
 // least pointer sized chunks whenever possible.  Unaligned memory at the beginning and remaining bytes at the end are 
 // written bytewise. We must make this guarantee whenever we clear memory in the GC heap that could contain object 
@@ -632,8 +657,6 @@ COOP_PINVOKE_CDECL_HELPER(void *, RhpInitMultibyte, (void * mem, int c, size_t s
     return mem;
 } 
 
-EXTERN_C void * __cdecl memmove(void *, const void *, size_t);
-
 //
 // Return true if the array slice is valid
 //
@@ -646,6 +669,8 @@ FORCEINLINE bool CheckArraySlice(Array * pArray, Int32 index, Int32 length)
            (length <= arrayLength - index);
 }
 
+EXTERN_C void * __cdecl memmove(void *, const void *, size_t);
+
 //
 // This function handles all cases of Array.Copy that do not require conversions or casting. It returns false if the copy cannot be performed, leaving
 // the handling of the complex cases or throwing appropriate exception to the higher level framework.
diff --git a/src/Native/Runtime/amd64/MiscStubs.asm b/src/Native/Runtime/amd64/MiscStubs.asm
index 915a9bec4..bcd84e626 100644
--- a/src/Native/Runtime/amd64/MiscStubs.asm
+++ b/src/Native/Runtime/amd64/MiscStubs.asm
@@ -9,6 +9,7 @@ EXTERN RhpShutdownHelper            : PROC
 EXTERN GetClasslibCCtorCheck        : PROC
 EXTERN memcpy                       : PROC
 EXTERN memcpyGCRefs                 : PROC
+EXTERN memcpyGCRefsWithWriteBarrier : PROC
 
 ;;
 ;; Currently called only from a managed executable once Main returns, this routine does whatever is needed to
@@ -232,4 +233,39 @@ NothingToCopy:
 
 LEAF_END RhpCopyMultibyte, _TEXT
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;; void* RhpCopyMultibyteWithWriteBarrier(void*, void*, size_t)
+;;
+;; The purpose of this wrapper is to hoist the potential null reference exceptions of copying memory up to a place where
+;; the stack unwinder and exception dispatch can properly transform the exception into a managed exception and dispatch
+;; it to managed code.
+;; Runs a card table update via RhpBulkWriteBarrier after the copy
+;;
+LEAF_ENTRY RhpCopyMultibyteWithWriteBarrier, _TEXT
+
+        ; rcx       dest
+        ; rdx       src
+        ; r8        count
+
+        test        r8, r8              ; check for a zero-length copy
+        jz          NothingToCopy
+
+        ; Now check the dest and src pointers.  If they AV, the EH subsystem will recognize the address of the AV,
+        ; unwind the frame, and fixup the stack to make it look like the (managed) caller AV'ed, which will be 
+        ; translated to a managed exception as usual.
+ALTERNATE_ENTRY RhpCopyMultibyteWithWriteBarrierDestAVLocation
+        cmp         byte ptr [rcx], 0
+ALTERNATE_ENTRY RhpCopyMultibyteWithWriteBarrierSrcAVLocation
+        cmp         byte ptr [rdx], 0
+
+        ; tail-call to the GC-safe memcpy implementation
+        jmp         memcpyGCRefsWithWriteBarrier
+
+NothingToCopy:
+        mov         rax, rcx            ; return dest
+        ret
+
+LEAF_END RhpCopyMultibyteWithWriteBarrier, _TEXT
+
 end
diff --git a/src/Native/Runtime/arm/MiscStubs.asm b/src/Native/Runtime/arm/MiscStubs.asm
index 8934dbf82..82bc0a2cc 100644
--- a/src/Native/Runtime/arm/MiscStubs.asm
+++ b/src/Native/Runtime/arm/MiscStubs.asm
@@ -9,6 +9,7 @@
     EXTERN GetClasslibCCtorCheck
     EXTERN memcpy
     EXTERN memcpyGCRefs
+    EXTERN memcpyGCRefsWithWriteBarrier
 
     TEXTAREA
 
@@ -208,7 +209,7 @@ NothingToCopy_NoGCRefs
         ; r2    count
 
         cmp     r2, #0                      ; check for a zero-length copy
-        beq     NothingToCopy
+        beq     NothingToCopy_RhpCopyMultibyte
 
         ; Now check the dest and src pointers.  If they AV, the EH subsystem will recognize the address of the AV,
         ; unwind the frame, and fixup the stack to make it look like the (managed) caller AV'ed, which will be 
@@ -221,12 +222,47 @@ NothingToCopy_NoGCRefs
         ; tail-call to the GC-safe memcpy implementation
         b       memcpyGCRefs
 
-NothingToCopy
+NothingToCopy_RhpCopyMultibyte
         ; dest is already still in r0
         bx      lr
 
     LEAF_END
 
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;; void* RhpCopyMultibyteWithWriteBarrier(void*, void*, size_t)
+;;
+;; The purpose of this wrapper is to hoist the potential null reference exceptions of copying memory up to a place where
+;; the stack unwinder and exception dispatch can properly transform the exception into a managed exception and dispatch
+;; it to managed code.
+;; Runs a card table update via RhpBulkWriteBarrier after the copy
+;;
+
+    LEAF_ENTRY    RhpCopyMultibyteWithWriteBarrier
+
+        ; r0    dest
+        ; r1    src
+        ; r2    count
+
+        cmp     r2, #0                      ; check for a zero-length copy
+        beq     NothingToCopy_RhpCopyMultibyteWithWriteBarrier
+
+        ; Now check the dest and src pointers.  If they AV, the EH subsystem will recognize the address of the AV,
+        ; unwind the frame, and fixup the stack to make it look like the (managed) caller AV'ed, which will be 
+        ; translated to a managed exception as usual.
+    ALTERNATE_ENTRY RhpCopyMultibyteWithWriteBarrierDestAVLocation
+        ldrb    r3, [r0]
+    ALTERNATE_ENTRY RhpCopyMultibyteWithWriteBarrierSrcAVLocation
+        ldrb    r3, [r1]
+
+        ; tail-call to the GC-safe memcpy implementation
+        b       memcpyGCRefsWithWriteBarrier
+
+NothingToCopy_RhpCopyMultibyteWithWriteBarrier
+        ; dest is already still in r0
+        bx      lr
+
+    LEAF_END
 
 
     end
diff --git a/src/Native/Runtime/gcrhenv.cpp b/src/Native/Runtime/gcrhenv.cpp
index 25b6e160d..db97a3245 100644
--- a/src/Native/Runtime/gcrhenv.cpp
+++ b/src/Native/Runtime/gcrhenv.cpp
@@ -648,6 +648,11 @@ UInt32 RedhawkGCInterface::GetGCDescSize(void * pType)
     return (UInt32)CGCDesc::GetCGCDescFromMT(pMT)->GetSize();
 }
 
+void ForwardGCSafeCopy(void * dest, const void *src, size_t len);
+void BackwardGCSafeCopy(void * dest, const void *src, size_t len);
+EXTERN_C void REDHAWK_CALLCONV RhpBulkWriteBarrier(void* pMemStart, UInt32 cbMemSize);
+
+
 COOP_PINVOKE_HELPER(void, RhpCopyObjectContents, (Object* pobjDest, Object* pobjSrc))
 {
     SIZE_T cbDest = pobjDest->GetSize() - sizeof(ObjHeader);
@@ -655,8 +660,17 @@ COOP_PINVOKE_HELPER(void, RhpCopyObjectContents, (Object* pobjDest, Object* pobj
     if (cbSrc != cbDest)
         return;
 
-    memcpy(pobjDest, pobjSrc, cbDest);
-    GCHeap::GetGCHeap()->SetCardsAfterBulkCopy((Object**) pobjDest, cbDest);
+    ASSERT(pobjDest->get_EEType()->HasReferenceFields() == pobjSrc->get_EEType()->HasReferenceFields());
+
+    if (pobjDest->get_EEType()->HasReferenceFields())
+    {
+        ForwardGCSafeCopy(pobjDest, pobjSrc, cbDest);
+        GCHeap::GetGCHeap()->SetCardsAfterBulkCopy((Object**)pobjDest, cbDest);
+    }
+    else
+    {
+        memcpy(pobjDest, pobjSrc, cbDest);
+    }
 }
 
 // Move memory, in a way that is compatible with a move onto the heap, but
@@ -664,7 +678,11 @@ COOP_PINVOKE_HELPER(void, RhpCopyObjectContents, (Object* pobjDest, Object* pobj
 
 COOP_PINVOKE_HELPER(void, RhBulkMoveWithWriteBarrier, (uint8_t* pDest, uint8_t* pSrc, int cbDest))
 {
-    memmove(pDest, pSrc, cbDest);
+    if (pDest <= pSrc || pSrc + cbDest <= pDest)
+        ForwardGCSafeCopy(pDest, pSrc, cbDest);
+    else
+        BackwardGCSafeCopy(pDest, pSrc, cbDest);
+
     // Use RhpBulkWriteBarrier here instead of SetCardsAfterBulkCopy as RhpBulkWriteBarrier
     // is both faster, and is compatible with a destination that isn't the GC heap.
     InlinedBulkWriteBarrier(pDest, cbDest);
@@ -687,11 +705,16 @@ COOP_PINVOKE_HELPER(void, RhpBox, (Object * pObj, void * pData))
     UInt8 * pbFields = (UInt8*)pObj + sizeof(EEType*);
 
     // Copy the unboxed value type data into the new object.
-    memcpy(pbFields, pData, cbFields);
-
     // Perform any write barriers necessary for embedded reference fields.
     if (pEEType->HasReferenceFields())
+    {
+        ForwardGCSafeCopy(pbFields, pData, cbFields);
         GCHeap::GetGCHeap()->SetCardsAfterBulkCopy((Object**)pbFields, cbFields);
+    }
+    else
+    {
+        memcpy(pbFields, pData, cbFields);
+    }
 }
 
 COOP_PINVOKE_HELPER(void, RhUnbox, (Object * pObj, void * pData, EEType * pUnboxToEEType))
@@ -737,15 +760,25 @@ COOP_PINVOKE_HELPER(void, RhUnbox, (Object * pObj, void * pData, EEType * pUnbox
     SIZE_T cbFields = pEEType->get_BaseSize() - (sizeof(ObjHeader) + sizeof(EEType*) + cbFieldPadding);
     UInt8 * pbFields = (UInt8*)pObj + sizeof(EEType*);
 
-    // Copy the boxed fields into the new location.
-    memcpy(pData, pbFields, cbFields);
-
     // Perform any write barriers necessary for embedded reference fields. SetCardsAfterBulkCopy doesn't range
     // check the address we pass it and in this case we don't know whether pData really points into the GC
     // heap or not. If we call it with an address outside of the GC range we could end up setting a card
     // outside of the allocated range of the card table, i.e. corrupt memory.
-    if (pEEType->HasReferenceFields() && (pData >= g_lowest_address) && (pData < g_highest_address))
-        GCHeap::GetGCHeap()->SetCardsAfterBulkCopy((Object**)pData, cbFields);
+    if (pEEType->HasReferenceFields())
+    {
+        // Copy the boxed fields into the new location in a GC safe manner
+        ForwardGCSafeCopy(pData, pbFields, cbFields);
+
+        if ((pData >= g_lowest_address) && (pData < g_highest_address))
+        {
+            GCHeap::GetGCHeap()->SetCardsAfterBulkCopy((Object**)pData, cbFields);
+        }
+    }
+    else
+    {
+        // Copy the boxed fields into the new location.
+        memcpy(pData, pbFields, cbFields);
+    }
 }
 
 #endif // !DACCESS_COMPILE
diff --git a/src/Native/Runtime/i386/MiscStubs.asm b/src/Native/Runtime/i386/MiscStubs.asm
index e96f7dea0..c7bfdc279 100644
--- a/src/Native/Runtime/i386/MiscStubs.asm
+++ b/src/Native/Runtime/i386/MiscStubs.asm
@@ -14,6 +14,7 @@ EXTERN @RhpShutdownHelper@4     : PROC
 EXTERN @GetClasslibCCtorCheck@4 : PROC
 EXTERN _memcpy                  : PROC
 EXTERN _memcpyGCRefs            : PROC
+EXTERN _memcpyGCRefsWithWriteBarrier  : PROC
 
 ;;
 ;; Currently called only from a managed executable once Main returns, this routine does whatever is needed to
@@ -256,4 +257,48 @@ NothingToCopy:
         ret
 
 _RhpCopyMultibyte ENDP
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;; void* __cdecl RhpCopyMultibyteWithWriteBarrier(void*, void*, size_t)
+;;
+;; The purpose of this wrapper is to hoist the potential null reference exceptions of copying memory up to a place where
+;; the stack unwinder and exception dispatch can properly transform the exception into a managed exception and dispatch
+;; it to managed code.
+;; Runs a card table update via RhpBulkWriteBarrier after the copy
+;;
+_RhpCopyMultibyteWithWriteBarrier PROC PUBLIC
+
+        ;    #locals, num_params, prolog bytes, #regs saved, use ebp, frame type (0 == FRAME_FPO)
+        .FPO(      0,          3,            0,           0,       0,          0)
+
+        ; [esp + 0] return address
+        ; [esp + 4] dest
+        ; [esp + 8] src
+        ; [esp + c] count
+
+        cmp         dword ptr [esp + 0Ch], 0        ; check for a zero-length copy
+        jz          NothingToCopy
+
+        mov         ecx, [esp + 4]  ; ecx <- dest
+        mov         edx, [esp + 8]  ; edx <- src
+
+        ; Now check the dest and src pointers.  If they AV, the EH subsystem will recognize the address of the AV,
+        ; unwind the frame, and fixup the stack to make it look like the (managed) caller AV'ed, which will be 
+        ; translated to a managed exception as usual.
+ALTERNATE_ENTRY RhpCopyMultibyteWithWriteBarrierDestAVLocation
+        cmp         byte ptr [ecx], 0
+ALTERNATE_ENTRY RhpCopyMultibyteWithWriteBarrierSrcAVLocation
+        cmp         byte ptr [edx], 0
+
+        ; tail-call to the GC-safe memcpy implementation
+        ; NOTE: this is also a __cdecl function
+        jmp         _memcpyGCRefsWithWriteBarrier
+
+NothingToCopy:
+        mov         eax, [esp + 4]                  ; return dest
+        ret
+
+_RhpCopyMultibyteWithWriteBarrier ENDP
+
 end