Add conservative upper bound calculation for GC in CoffNativeCodeManager (#5585)

* Add upper bound calculation

* Comment Change

* Correct Stack Pointer calculation

* Fix data flag address adjustment and move reverse pinvoke check

* Move bound one pointer size up

1 files changed, 73 insertions, 1 deletions

diff --git a/src/Native/Runtime/windows/CoffNativeCodeManager.cpp b/src/Native/Runtime/windows/CoffNativeCodeManager.cpp
index 242f86c49..aa3ab5dcb 100644
--- a/src/Native/Runtime/windows/CoffNativeCodeManager.cpp
+++ b/src/Native/Runtime/windows/CoffNativeCodeManager.cpp
@@ -380,9 +380,81 @@ void CoffNativeCodeManager::EnumGcRefs(MethodInfo *    pMethodInfo,
 
 UIntNative CoffNativeCodeManager::GetConservativeUpperBoundForOutgoingArgs(MethodInfo * pMethodInfo, REGDISPLAY * pRegisterSet)
 {
-    // @TODO: CORERT: GetConservativeUpperBoundForOutgoingArgs
+#if defined(_TARGET_AMD64_)
+
+    // Return value
+    UIntNative upperBound;
+    CoffNativeMethodInfo* pNativeMethodInfo = (CoffNativeMethodInfo *) pMethodInfo;
+    
+    size_t unwindDataBlobSize;
+    PTR_VOID pUnwindDataBlob = GetUnwindDataBlob(m_moduleBase, pNativeMethodInfo->runtimeFunction, &unwindDataBlobSize);
+    PTR_UInt8 p = dac_cast<PTR_UInt8>(pUnwindDataBlob) + unwindDataBlobSize;
+    uint8_t unwindBlockFlags = *p++;
+
+    if ((unwindBlockFlags & UBF_FUNC_HAS_ASSOCIATED_DATA) != 0)
+        p += sizeof(int32_t);
+
+    if ((unwindBlockFlags & UBF_FUNC_REVERSE_PINVOKE) != 0)
+    {
+        TADDR basePointer =  dac_cast<TADDR>(pRegisterSet->GetFP());        
+        
+        // Get the method's GC info
+        GcInfoDecoder decoder(GCInfoToken(p), DECODE_REVERSE_PINVOKE_VAR);
+        UINT32 stackBasedRegister = decoder.GetStackBaseRegister();
+        
+        if (stackBasedRegister == NO_STACK_BASE_REGISTER)
+        {
+            basePointer = dac_cast<TADDR>(pRegisterSet->GetSP());
+        }
+        else
+        {
+            basePointer = dac_cast<TADDR>(pRegisterSet->GetFP());
+        }
+        // Reverse PInvoke case.  The embedded reverse PInvoke frame is guaranteed to reside above
+        // all outgoing arguments.
+        INT32 slot = decoder.GetReversePInvokeFrameStackSlot();
+        upperBound =  (UIntNative) dac_cast<TADDR>(basePointer + slot);
+    }
+    else
+    {
+        bool rbp = GetFramePointer(pMethodInfo, pRegisterSet) == NULL;
+        if (!rbp) 
+        {
+            // Unwind the current method context to get the caller's stack pointer
+            // and obtain the upper bound of the callee is the value just below the caller's return address on the stack
+            SIZE_T  EstablisherFrame;
+            PVOID   HandlerData;
+            CONTEXT context;
+            context.Rsp = pRegisterSet->GetSP();
+            context.Rbp = pRegisterSet->GetFP();
+            context.Rip = pRegisterSet->GetIP();
+    
+            RtlVirtualUnwind(NULL,
+                            dac_cast<TADDR>(m_moduleBase),
+                            pRegisterSet->IP,
+                            (PRUNTIME_FUNCTION)pNativeMethodInfo->runtimeFunction,
+                            &context,
+                            &HandlerData,
+                            &EstablisherFrame,
+                            NULL);
+
+            upperBound = dac_cast<TADDR>(context.Rsp - sizeof (PVOID));
+        }
+        else
+        {
+            // In amd64, it is guaranteed that if there is a pushed RBP
+            // value at the top of the frame it resides above all outgoing arguments.  Unlike x86,
+            // the frame pointer generally points to a location that is separated from the pushed RBP
+            // value by an offset that is recorded in the info header.  Recover the address of the
+            // pushed RBP value by subtracting this offset.
+            upperBound = (UIntNative) dac_cast<TADDR>(pRegisterSet->GetFP() - ((PTR_UNWIND_INFO) pUnwindDataBlob)->FrameOffset);
+        }
+    }
+    return upperBound;
+#else
     assert(false);
     return false;
+#endif
 }
 
 bool CoffNativeCodeManager::UnwindStackFrame(MethodInfo *    pMethodInfo,