Add support for reflection invoking ref return methods

This is a port of dotnet/coreclr#17732. Fixes bug 603305.

Per popular demand from people who would like to serialize ref returning properties or use them in data binding, adding support for this in MethodInfo.Invoke. The choice was to make these dereference the result and return it like that. For the corner case of returning a ref to a null, a `NullReferenceException` will be thrown (this behavior has a very unfortunate side effect of slowing down the general Invoke path because we can't throw this exception from a place where we would end up wrapping this in a TargetInvocationException).

I also made tweaks to byref parameters path to fix byref pointer parameters, but this is horribly broken on the CLR so I didn't bother to test it here. Should work.

Also includes a slight size on disk optimization that merges common tails of the static/instance paths of the invoker thunk.

Testing of the calling convention converter portion was done by disabling invoke thunks generation in NUTC and running UnitTests. I also ran it with GCStresss (RH_GCStressThrottleMode=1, RH_GCStress_Mode=1). Let me know if there's more testing needed. This has a huge test matrix for being such a corner case thing.

[tfs-changeset: 1709919]

9 files changed, 170 insertions, 84 deletions

diff --git a/src/Common/src/TypeSystem/IL/Stubs/DynamicInvokeMethodThunk.Sorting.cs b/src/Common/src/TypeSystem/IL/Stubs/DynamicInvokeMethodThunk.Sorting.cs
index 041d3fa3e..d9c937f05 100644
--- a/src/Common/src/TypeSystem/IL/Stubs/DynamicInvokeMethodThunk.Sorting.cs
+++ b/src/Common/src/TypeSystem/IL/Stubs/DynamicInvokeMethodThunk.Sorting.cs
@@ -30,12 +30,9 @@ namespace Internal.IL.Stubs
             if (result != 0)
                 return result;
 
-            if (thisReturnType == DynamicInvokeMethodParameterKind.Pointer)
-            {
-                result = _targetSignature.GetNumerOfReturnTypePointerIndirections() - otherMethod._targetSignature.GetNumerOfReturnTypePointerIndirections();
-                if (result != 0)
-                    return result;
-            }
+            result = _targetSignature.GetNumerOfReturnTypePointerIndirections() - otherMethod._targetSignature.GetNumerOfReturnTypePointerIndirections();
+            if (result != 0)
+                return result;
 
             for (int i = 0; i < _targetSignature.Length; i++)
             {
@@ -44,12 +41,9 @@ namespace Internal.IL.Stubs
                 if (result != 0)
                     return result;
 
-                if (thisParamType == DynamicInvokeMethodParameterKind.Pointer)
-                {
-                    result = _targetSignature.GetNumberOfParameterPointerIndirections(i) - otherMethod._targetSignature.GetNumberOfParameterPointerIndirections(i);
-                    if (result != 0)
-                        return result;
-                }
+                result = _targetSignature.GetNumberOfParameterPointerIndirections(i) - otherMethod._targetSignature.GetNumberOfParameterPointerIndirections(i);
+                if (result != 0)
+                    return result;
             }
 
             Debug.Assert(this == otherMethod);
diff --git a/src/Common/src/TypeSystem/IL/Stubs/DynamicInvokeMethodThunk.cs b/src/Common/src/TypeSystem/IL/Stubs/DynamicInvokeMethodThunk.cs
index e10aac6c2..510b300c5 100644
--- a/src/Common/src/TypeSystem/IL/Stubs/DynamicInvokeMethodThunk.cs
+++ b/src/Common/src/TypeSystem/IL/Stubs/DynamicInvokeMethodThunk.cs
@@ -37,41 +37,37 @@ namespace Internal.IL.Stubs
             return context.SystemModule.GetType("System", "InvokeUtils", false) != null;
         }
 
-        public static bool SupportsSignature(MethodSignature signature)
+        private static TypeDesc UnwrapByRef(TypeDesc type)
         {
-            for (int i = 0; i < signature.Length; i++)
-                if (signature[i].IsByRef && ((ByRefType)signature[i]).ParameterType.IsPointer)
-                    return false;
+            if (type.IsByRef)
+                return ((ByRefType)type).ParameterType;
+            return type;
+        }
 
+        public static bool SupportsSignature(MethodSignature signature)
+        {
             // ----------------------------------------------------------------
             // TODO: function pointer types are odd: https://github.com/dotnet/corert/issues/1929
             // ----------------------------------------------------------------
 
-            if (signature.ReturnType.IsFunctionPointer)
+            if (UnwrapByRef(signature.ReturnType).IsFunctionPointer)
                 return false;
 
             for (int i = 0; i < signature.Length; i++)
-                if (signature[i].IsFunctionPointer)
+                if (UnwrapByRef(signature[i]).IsFunctionPointer)
                     return false;
 
             // ----------------------------------------------------------------
-            // Methods with ByRef returns can't be reflection invoked
-            // ----------------------------------------------------------------
-
-            if (signature.ReturnType.IsByRef)
-                return false;
-
-            // ----------------------------------------------------------------
             // Methods that return ByRef-like types or take them by reference can't be reflection invoked
             // ----------------------------------------------------------------
 
-            if (!signature.ReturnType.IsSignatureVariable && signature.ReturnType.IsByRefLike)
+            if (!signature.ReturnType.IsSignatureVariable && UnwrapByRef(signature.ReturnType).IsByRefLike)
                 return false;
 
             for (int i = 0; i < signature.Length; i++)
             {
                 ByRefType paramType = signature[i] as ByRefType;
-                if (paramType != null && !paramType.ParameterType.IsSignatureVariable && paramType.ParameterType.IsByRefLike)
+                if (paramType != null && !paramType.ParameterType.IsSignatureVariable && UnwrapByRef(paramType.ParameterType).IsByRefLike)
                     return false;
             }
 
@@ -93,7 +89,10 @@ namespace Internal.IL.Stubs
                     // strip ByRefType off the parameter (the method already has ByRef in the signature)
                     parameterType = ((ByRefType)parameterType).ParameterType;
 
-                    Debug.Assert(!parameterType.IsPointer); // TODO: support for methods returning pointer types - https://github.com/dotnet/corert/issues/2113
+                    // Strip off all the pointers. Pointers are not valid instantiation arguments and the thunk compensates for that
+                    // by being specialized for the specific pointer depth.
+                    while (parameterType.IsPointer)
+                        parameterType = ((PointerType)parameterType).ParameterType;
                 }
                 else if (parameterType.IsPointer)
                 {
@@ -131,7 +130,10 @@ namespace Internal.IL.Stubs
             if (!sig.ReturnType.IsVoid)
             {
                 TypeDesc returnType = sig.ReturnType;
-                Debug.Assert(!returnType.IsByRef);
+
+                // strip ByRefType off the return type (the method already has ByRef in the signature)
+                if (returnType.IsByRef)
+                    returnType = ((ByRefType)returnType).ParameterType;
 
                 // If the invoke method return an object reference, we don't need to specialize on the
                 // exact type of the object reference, as the behavior is not different.
@@ -250,10 +252,13 @@ namespace Internal.IL.Stubs
                         break;
                     case DynamicInvokeMethodParameterKind.Pointer:
                         sb.Append('P');
-
                         for (int i = 0; i < _targetSignature.GetNumerOfReturnTypePointerIndirections() - 1; i++)
                             sb.Append('p');
-
+                        break;
+                    case DynamicInvokeMethodParameterKind.Reference:
+                        sb.Append("R");
+                        for (int i = 0; i < _targetSignature.GetNumerOfReturnTypePointerIndirections(); i++)
+                            sb.Append('p');
                         break;
                     case DynamicInvokeMethodParameterKind.Value:
                         sb.Append('O');
@@ -276,6 +281,9 @@ namespace Internal.IL.Stubs
                             break;
                         case DynamicInvokeMethodParameterKind.Reference:
                             sb.Append("R");
+
+                            for (int j = 0; j < _targetSignature.GetNumberOfParameterPointerIndirections(i); j++)
+                                sb.Append('p');
                             break;
                         case DynamicInvokeMethodParameterKind.Value:
                             sb.Append("I");
@@ -296,6 +304,7 @@ namespace Internal.IL.Stubs
             ILCodeStream argSetupStream = emitter.NewCodeStream();
             ILCodeStream thisCallSiteSetupStream = emitter.NewCodeStream();
             ILCodeStream staticCallSiteSetupStream = emitter.NewCodeStream();
+            ILCodeStream returnCodeStream = emitter.NewCodeStream();
 
             // This function will look like
             //
@@ -328,13 +337,7 @@ namespace Internal.IL.Stubs
             //       ldloc localX
             // ldarg.1
             // calli ReturnType thiscall(TypeOfParameter1, ...)
-            // !if ((ReturnType == void)
-            //    ldnull
-            // !elif (ReturnType is pointer)
-            //    System.Reflection.Pointer.Box(ReturnType)
-            // !else
-            //    box ReturnType
-            // ret
+            // br Process_return
 
             // *** Static call instruction stream starts here ***
 
@@ -347,6 +350,14 @@ namespace Internal.IL.Stubs
             //       ldloc localX
             // ldarg.1
             // calli ReturnType (TypeOfParameter1, ...)
+
+            // *** Return code stream starts here ***
+
+            // Process_return:
+            // !if (ReturnType is Byref)
+            //    dup
+            //    brfalse ByRefNull
+            //    ldobj ReturnType
             // !if ((ReturnType == void)
             //    ldnull
             // !elif (ReturnType is pointer)
@@ -354,8 +365,15 @@ namespace Internal.IL.Stubs
             // !else
             //    box ReturnType
             // ret
+            //
+            // !if (ReturnType is ByRef)
+            //   ByRefNull:
+            //   pop
+            //   call InvokeUtils.get_NullByRefValueSentinel
+            //   ret
 
             ILCodeLabel lStaticCall = emitter.NewCodeLabel();
+            ILCodeLabel lProcessReturn = emitter.NewCodeLabel();
             thisCallSiteSetupStream.EmitLdArg(3); // targetIsThisCall
             thisCallSiteSetupStream.Emit(ILOpcode.brfalse, lStaticCall);
             staticCallSiteSetupStream.EmitLabel(lStaticCall);
@@ -374,9 +392,8 @@ namespace Internal.IL.Stubs
                 TypeDesc paramType = Context.GetSignatureVariable(paramIndex, true);
                 DynamicInvokeMethodParameterKind paramKind = _targetSignature[paramIndex];
 
-                if (paramKind == DynamicInvokeMethodParameterKind.Pointer)
-                    for (int i = 0; i < _targetSignature.GetNumberOfParameterPointerIndirections(paramIndex); i++)
-                        paramType = paramType.MakePointerType();
+                for (int i = 0; i < _targetSignature.GetNumberOfParameterPointerIndirections(paramIndex); i++)
+                    paramType = paramType.MakePointerType();
 
                 ILToken tokParamType = emitter.NewToken(paramType);
                 ILLocalVariable local = emitter.NewLocal(paramType.MakeByRefType());
@@ -415,45 +432,69 @@ namespace Internal.IL.Stubs
                 Context.GetSignatureVariable(_targetSignature.Length, true) :
                 Context.GetWellKnownType(WellKnownType.Void);
 
-            if (returnKind == DynamicInvokeMethodParameterKind.Pointer)
-                for (int i = 0; i < _targetSignature.GetNumerOfReturnTypePointerIndirections(); i++)
-                    returnType = returnType.MakePointerType();
+            for (int i = 0; i < _targetSignature.GetNumerOfReturnTypePointerIndirections(); i++)
+                returnType = returnType.MakePointerType();
+
+            if (returnKind == DynamicInvokeMethodParameterKind.Reference)
+                returnType = returnType.MakeByRefType();
 
             MethodSignature thisCallMethodSig = new MethodSignature(0, 0, returnType, targetMethodSignature);
             thisCallSiteSetupStream.Emit(ILOpcode.calli, emitter.NewToken(thisCallMethodSig));
+            thisCallSiteSetupStream.Emit(ILOpcode.br, lProcessReturn);
 
             MethodSignature staticCallMethodSig = new MethodSignature(MethodSignatureFlags.Static, 0, returnType, targetMethodSignature);
             staticCallSiteSetupStream.Emit(ILOpcode.calli, emitter.NewToken(staticCallMethodSig));
 
+            returnCodeStream.EmitLabel(lProcessReturn);
+
+            ILCodeLabel lByRefReturnNull = null;
+
             if (returnKind == DynamicInvokeMethodParameterKind.None)
             {
-                thisCallSiteSetupStream.Emit(ILOpcode.ldnull);
-                staticCallSiteSetupStream.Emit(ILOpcode.ldnull);
-            }
-            else if (returnKind == DynamicInvokeMethodParameterKind.Pointer)
-            {
-                thisCallSiteSetupStream.Emit(ILOpcode.ldtoken, emitter.NewToken(returnType));
-                staticCallSiteSetupStream.Emit(ILOpcode.ldtoken, emitter.NewToken(returnType));
-                MethodDesc getTypeFromHandleMethod =
-                    Context.SystemModule.GetKnownType("System", "Type").GetKnownMethod("GetTypeFromHandle", null);
-                thisCallSiteSetupStream.Emit(ILOpcode.call, emitter.NewToken(getTypeFromHandleMethod));
-                staticCallSiteSetupStream.Emit(ILOpcode.call, emitter.NewToken(getTypeFromHandleMethod));
-
-                MethodDesc pointerBoxMethod =
-                    Context.SystemModule.GetKnownType("System.Reflection", "Pointer").GetKnownMethod("Box", null);
-                thisCallSiteSetupStream.Emit(ILOpcode.call, emitter.NewToken(pointerBoxMethod));
-                staticCallSiteSetupStream.Emit(ILOpcode.call, emitter.NewToken(pointerBoxMethod));
+                returnCodeStream.Emit(ILOpcode.ldnull);
             }
             else
             {
-                Debug.Assert(returnKind == DynamicInvokeMethodParameterKind.Value);
-                ILToken tokReturnType = emitter.NewToken(returnType);
-                thisCallSiteSetupStream.Emit(ILOpcode.box, tokReturnType);
-                staticCallSiteSetupStream.Emit(ILOpcode.box, tokReturnType);
+                TypeDesc returnTypeForBoxing = returnType;
+
+                if (returnType.IsByRef)
+                {
+                    // If this is a byref return, we need to dereference first
+                    returnTypeForBoxing = ((ByRefType)returnType).ParameterType;
+                    lByRefReturnNull = emitter.NewCodeLabel();
+                    returnCodeStream.Emit(ILOpcode.dup);
+                    returnCodeStream.Emit(ILOpcode.brfalse, lByRefReturnNull);
+                    returnCodeStream.Emit(ILOpcode.ldobj, emitter.NewToken(returnTypeForBoxing));
+                }
+
+                if (returnTypeForBoxing.IsPointer)
+                {
+                    // Pointers box differently
+                    returnCodeStream.Emit(ILOpcode.ldtoken, emitter.NewToken(returnTypeForBoxing));
+                    MethodDesc getTypeFromHandleMethod =
+                        Context.SystemModule.GetKnownType("System", "Type").GetKnownMethod("GetTypeFromHandle", null);
+                    returnCodeStream.Emit(ILOpcode.call, emitter.NewToken(getTypeFromHandleMethod));
+
+                    MethodDesc pointerBoxMethod =
+                        Context.SystemModule.GetKnownType("System.Reflection", "Pointer").GetKnownMethod("Box", null);
+                    returnCodeStream.Emit(ILOpcode.call, emitter.NewToken(pointerBoxMethod));
+                }
+                else
+                {
+                    ILToken tokReturnType = emitter.NewToken(returnTypeForBoxing);
+                    returnCodeStream.Emit(ILOpcode.box, tokReturnType);
+                }
             }
 
-            thisCallSiteSetupStream.Emit(ILOpcode.ret);
-            staticCallSiteSetupStream.Emit(ILOpcode.ret);
+            returnCodeStream.Emit(ILOpcode.ret);
+
+            if (lByRefReturnNull != null)
+            {
+                returnCodeStream.EmitLabel(lByRefReturnNull);
+                returnCodeStream.Emit(ILOpcode.pop);
+                returnCodeStream.Emit(ILOpcode.call, emitter.NewToken(InvokeUtilsType.GetKnownMethod("get_NullByRefValueSentinel", null)));
+                returnCodeStream.Emit(ILOpcode.ret);
+            }
 
             return emitter.Link(this);
         }
@@ -541,6 +582,10 @@ namespace Internal.IL.Stubs
 
         public static int GetNumberOfIndirections(TypeDesc type)
         {
+            // Strip byrefness off. This is to support "ref void**"-style signatures.
+            if (type.IsByRef)
+                type = ((ByRefType)type).ParameterType;
+
             int result = 0;
             while (type.IsPointer)
             {
@@ -565,13 +610,13 @@ namespace Internal.IL.Stubs
         {
             get
             {
-                Debug.Assert(!_signature.ReturnType.IsByRef);
-
                 TypeDesc type = _signature.ReturnType;
                 if (type.IsPointer)
                     return DynamicInvokeMethodParameterKind.Pointer;
                 else if (type.IsVoid)
                     return DynamicInvokeMethodParameterKind.None;
+                else if (type.IsByRef)
+                    return DynamicInvokeMethodParameterKind.Reference;
                 else
                     return DynamicInvokeMethodParameterKind.Value;
             }
@@ -607,8 +652,7 @@ namespace Internal.IL.Stubs
             if (thisReturnKind != other.ReturnType)
                 return false;
 
-            if (thisReturnKind == DynamicInvokeMethodParameterKind.Pointer &&
-                GetNumerOfReturnTypePointerIndirections() != other.GetNumerOfReturnTypePointerIndirections())
+            if (GetNumerOfReturnTypePointerIndirections() != other.GetNumerOfReturnTypePointerIndirections())
                 return false;
             
             if (Length != other.Length)
@@ -620,8 +664,7 @@ namespace Internal.IL.Stubs
                 if (thisParamKind != other[i])
                     return false;
 
-                if (thisParamKind == DynamicInvokeMethodParameterKind.Pointer &&
-                    GetNumberOfParameterPointerIndirections(i) != other.GetNumberOfParameterPointerIndirections(i))
+                if (GetNumberOfParameterPointerIndirections(i) != other.GetNumberOfParameterPointerIndirections(i))
                     return false;
             }
 
diff --git a/src/Common/src/TypeSystem/IL/Stubs/ILEmitter.cs b/src/Common/src/TypeSystem/IL/Stubs/ILEmitter.cs
index 042636f41..45627bd1d 100644
--- a/src/Common/src/TypeSystem/IL/Stubs/ILEmitter.cs
+++ b/src/Common/src/TypeSystem/IL/Stubs/ILEmitter.cs
@@ -253,6 +253,8 @@ namespace Internal.IL.Stubs
                     break;
                 case TypeFlags.ValueType:
                 case TypeFlags.Nullable:
+                case TypeFlags.SignatureMethodVariable:
+                case TypeFlags.SignatureTypeVariable:
                     Emit(ILOpcode.ldobj, _emitter.NewToken(type));
                     break;
                 default:
diff --git a/src/System.Private.CoreLib/src/Resources/Strings.resx b/src/System.Private.CoreLib/src/Resources/Strings.resx
index 1cfdaaa76..0fb811bca 100644
--- a/src/System.Private.CoreLib/src/Resources/Strings.resx
+++ b/src/System.Private.CoreLib/src/Resources/Strings.resx
@@ -2755,6 +2755,9 @@
   <data name="Arg_InsufficientNumberOfElements" xml:space="preserve">
     <value>At least {0} element(s) are expected in the parameter "{1}".</value>
   </data>
+  <data name="NullReference_InvokeNullRefReturned" xml:space="preserve">
+    <value>The target method returned a null reference.</value>
+  </data>
   <data name="InvalidOperation_ConcurrentOperationsNotSupported" xml:space="preserve">
     <value>Operations that change non-concurrent collections must have exclusive access. A concurrent update was performed on this collection and corrupted its state. The collection's state is no longer correct.</value>
   </data>
diff --git a/src/System.Private.CoreLib/src/System/InvokeUtils.cs b/src/System.Private.CoreLib/src/System/InvokeUtils.cs
index 865f80fe0..0a66ea6c2 100644
--- a/src/System.Private.CoreLib/src/System/InvokeUtils.cs
+++ b/src/System.Private.CoreLib/src/System/InvokeUtils.cs
@@ -12,6 +12,8 @@ using Internal.Reflection.Core.NonPortable;
 using Internal.Runtime.Augments;
 using Internal.Runtime.CompilerServices;
 
+using Interlocked = System.Threading.Interlocked;
+
 namespace System
 {
     [System.Runtime.CompilerServices.ReflectionBlocked]
@@ -416,9 +418,9 @@ namespace System
                 s_curIndex = 0;
                 s_targetMethodOrDelegate = targetMethodOrDelegate;
 
+                object result;
                 try
                 {
-                    object result = null;
                     if (invokeMethodHelperIsThisCall)
                     {
                         Debug.Assert(methodToCallIsThisCall == true);
@@ -438,8 +440,6 @@ namespace System
                             DebugAnnotations.PreviousCallContainsDebuggerStepInCode();
                         }
                     }
-
-                    return result;
                 }
                 catch (Exception e) when (wrapInTargetInvocationException && argSetupState.fComplete)
                 {
@@ -468,6 +468,11 @@ namespace System
                         }
                     }
                 }
+
+                if (result == NullByRefValueSentinel)
+                    throw new NullReferenceException(SR.NullReference_InvokeNullRefReturned);
+
+                return result;
             }
             finally
             {
@@ -851,6 +856,19 @@ namespace System
                 }
             }
         }
+
+        private static volatile object _nullByRefValueSentinel;
+        public static object NullByRefValueSentinel
+        {
+            get
+            {
+                if (_nullByRefValueSentinel == null)
+                {
+                    Interlocked.CompareExchange(ref _nullByRefValueSentinel, new object(), null);
+                }
+                return _nullByRefValueSentinel;
+            }
+        }
     }
 }
 
diff --git a/src/System.Private.Reflection.Core/src/Resources/Strings.resx b/src/System.Private.Reflection.Core/src/Resources/Strings.resx
index 6488cc434..820a41e26 100644
--- a/src/System.Private.Reflection.Core/src/Resources/Strings.resx
+++ b/src/System.Private.Reflection.Core/src/Resources/Strings.resx
@@ -156,9 +156,6 @@
   <data name="Arg_HTCapacityOverflow" xml:space="preserve">
     <value>Hashtable's capacity overflowed and went negative. Check load factor, capacity and the current size of the table.</value>
   </data>
-  <data name="NotSupported_ByRefReturn" xml:space="preserve">
-    <value>ByRef return value not supported in reflection invocation.</value>
-  </data>
   <data name="Arg_DlgtTargMeth" xml:space="preserve">
     <value>Cannot bind to the target method because its signature is not compatible with that of the delegate type.</value>
   </data>
diff --git a/src/System.Private.Reflection.Core/src/System/Reflection/Runtime/MethodInfos/RuntimeMethodInfo.cs b/src/System.Private.Reflection.Core/src/System/Reflection/Runtime/MethodInfos/RuntimeMethodInfo.cs
index 33d73b16a..4af12bfc9 100644
--- a/src/System.Private.Reflection.Core/src/System/Reflection/Runtime/MethodInfos/RuntimeMethodInfo.cs
+++ b/src/System.Private.Reflection.Core/src/System/Reflection/Runtime/MethodInfos/RuntimeMethodInfo.cs
@@ -354,7 +354,12 @@ namespace System.Reflection.Runtime.MethodInfos
                 if (methodInvoker == null)
                 {
                     if (ReturnType.IsByRef)
-                        throw new NotSupportedException(SR.NotSupported_ByRefReturn);
+                    {
+                        // The invoker is going to dereference and box (for structs) the result of the invocation
+                        // on behalf of the caller. Can't box byref-like types and can't box void.
+                        if (ReturnType.GetElementType().IsByRefLike || ReturnType.GetElementType() == CommonRuntimeTypes.Void)
+                            throw new NotSupportedException();
+                    }
                     methodInvoker = _lazyMethodInvoker = this.UncachedMethodInvoker;
                 }
                 return methodInvoker;
diff --git a/src/System.Private.Reflection.Execution/src/Internal/Reflection/Execution/ExecutionEnvironmentImplementation.MappingTables.cs b/src/System.Private.Reflection.Execution/src/Internal/Reflection/Execution/ExecutionEnvironmentImplementation.MappingTables.cs
index 284e2c25b..8313fbe31 100644
--- a/src/System.Private.Reflection.Execution/src/Internal/Reflection/Execution/ExecutionEnvironmentImplementation.MappingTables.cs
+++ b/src/System.Private.Reflection.Execution/src/Internal/Reflection/Execution/ExecutionEnvironmentImplementation.MappingTables.cs
@@ -1539,6 +1539,8 @@ namespace Internal.Reflection.Execution
 
                     MethodInfo reflectionMethodInfo = _methodBase as MethodInfo;
                     Type returnType = reflectionMethodInfo != null ? reflectionMethodInfo.ReturnType : CommonRuntimeTypes.Void;
+                    if (returnType.IsByRef)
+                        returnType = returnType.GetElementType();
                     result.Insert(0, returnType.TypeHandle);
 
                     return result;
diff --git a/src/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/CallConverterThunk.cs b/src/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/CallConverterThunk.cs
index b89c2ccec..f0f8256c7 100644
--- a/src/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/CallConverterThunk.cs
+++ b/src/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/CallConverterThunk.cs
@@ -1114,11 +1114,12 @@ namespace Internal.Runtime.TypeLoader
 #endif
                     }
                 }
-                else if (conversionParams._conversionInfo.IsAnyDynamicInvokerThunk && (thRetType.IsValueType() || thRetType.IsPointerType()))
+                else if (conversionParams._conversionInfo.IsAnyDynamicInvokerThunk &&
+                    (thRetType.IsValueType() || thRetType.IsPointerType() || returnType == CorElementType.ELEMENT_TYPE_BYREF))
                 {
                     Debug.Assert(returnValueToCopy != null);
 
-                    if (conversionParams._calleeArgs.GetReturnType(out thDummy, out dummyBool) == CorElementType.ELEMENT_TYPE_VOID)
+                    if (returnType == CorElementType.ELEMENT_TYPE_VOID)
                     {
                         // Invokers returning void need to return a null object
                         returnValueToCopy = null;
@@ -1128,15 +1129,36 @@ namespace Internal.Runtime.TypeLoader
                         if (!conversionParams._callerArgs.HasRetBuffArg() && conversionParams._calleeArgs.HasRetBuffArg())
                             returnValueToCopy = (void*)(new IntPtr(*((void**)returnValueToCopy)) + IntPtr.Size);
 
+                        if (returnType == CorElementType.ELEMENT_TYPE_BYREF)
+                        {
+                            // If this is a byref return, we're going to dereference the result
+                            returnValueToCopy = *(void**)returnValueToCopy;
+                        }
+
+                        RuntimeTypeHandle returnTypeRuntimeTypeHandle = thRetType.GetRuntimeTypeHandle();
+
                         // Need to box value type before returning it
                         object returnValue;
-                        if (thRetType.IsPointerType())
+                        if (returnType == CorElementType.ELEMENT_TYPE_BYREF && returnValueToCopy == null)
+                        {
+                            // This is a byref return and dereferencing it would result in a NullReferenceException.
+                            // Set the return value to a sentinel that InvokeUtils will recognize.
+                            // Can't throw from here or we would wrap this in a TargetInvocationException.
+                            returnValue = InvokeUtils.NullByRefValueSentinel;
+                        }
+                        else if (RuntimeAugments.IsUnmanagedPointerType(returnTypeRuntimeTypeHandle))
+                        {
+                            returnValue = System.Reflection.Pointer.Box(*(void**)returnValueToCopy, Type.GetTypeFromHandle(returnTypeRuntimeTypeHandle));
+                        }
+                        else if (RuntimeAugments.IsValueType(returnTypeRuntimeTypeHandle))
                         {
-                            returnValue = System.Reflection.Pointer.Box(*(void**)returnValueToCopy, Type.GetTypeFromHandle(thRetType.GetRuntimeTypeHandle()));
+                            returnValue = RuntimeAugments.Box(returnTypeRuntimeTypeHandle, new IntPtr(returnValueToCopy));
                         }
                         else
                         {
-                            returnValue = RuntimeAugments.Box(thRetType.GetRuntimeTypeHandle(), new IntPtr(returnValueToCopy));
+                            // byref return of a reference type
+                            Debug.Assert(returnType == CorElementType.ELEMENT_TYPE_BYREF);
+                            returnValue = Unsafe.As<byte, object>(ref *(byte*)returnValueToCopy);
                         }
                         CallConversionParameters.s_pinnedGCHandles._returnObjectHandle.Target = returnValue;
                         pinnedResultObject = CallConversionParameters.s_pinnedGCHandles._returnObjectHandle.GetRawTargetAddress();