Inliner: Extend IL limit for profiled call-sites, allow inlining for switches. (#55478)

1 files changed, 56 insertions, 17 deletions

diff --git a/src/coreclr/jit/inlinepolicy.cpp b/src/coreclr/jit/inlinepolicy.cpp
index fdf4ae19341..e84ff2858a0 100644
--- a/src/coreclr/jit/inlinepolicy.cpp
+++ b/src/coreclr/jit/inlinepolicy.cpp
@@ -326,7 +326,6 @@ void DefaultPolicy::NoteBool(InlineObservation obs, bool value)
                 m_ArgFeedsRangeCheck++;
                 break;
 
-            case InlineObservation::CALLEE_HAS_SWITCH:
             case InlineObservation::CALLEE_UNSUPPORTED_OPCODE:
                 propagate = true;
                 break;
@@ -1294,6 +1293,14 @@ void ExtendedDefaultPolicy::NoteBool(InlineObservation obs, bool value)
             m_FoldableBranch++;
             break;
 
+        case InlineObservation::CALLSITE_FOLDABLE_SWITCH:
+            m_FoldableSwitch++;
+            break;
+
+        case InlineObservation::CALLEE_HAS_SWITCH:
+            m_Switch++;
+            break;
+
         case InlineObservation::CALLSITE_DIV_BY_CNS:
             m_DivByCns++;
             break;
@@ -1327,7 +1334,14 @@ void ExtendedDefaultPolicy::NoteInt(InlineObservation obs, int value)
         {
             assert(m_IsForceInlineKnown);
             assert(value != 0);
-            m_CodeSize = static_cast<unsigned>(value);
+            m_CodeSize           = static_cast<unsigned>(value);
+            unsigned maxCodeSize = static_cast<unsigned>(JitConfig.JitExtDefaultPolicyMaxIL());
+
+            // TODO: Enable for PgoSource::Static as well if it's not the generic profile we bundle.
+            if (m_HasProfile && (m_RootCompiler->fgPgoSource == ICorJitInfo::PgoSource::Dynamic))
+            {
+                maxCodeSize = static_cast<unsigned>(JitConfig.JitExtDefaultPolicyMaxILProf());
+            }
 
             if (m_IsForceInline)
             {
@@ -1339,7 +1353,7 @@ void ExtendedDefaultPolicy::NoteInt(InlineObservation obs, int value)
                 // Candidate based on small size
                 SetCandidate(InlineObservation::CALLEE_BELOW_ALWAYS_INLINE_SIZE);
             }
-            else if (m_CodeSize <= (unsigned)JitConfig.JitExtDefaultPolicyMaxIL())
+            else if (m_CodeSize <= maxCodeSize)
             {
                 // Candidate, pending profitability evaluation
                 SetCandidate(InlineObservation::CALLEE_IS_DISCRETIONARY_INLINE);
@@ -1357,16 +1371,16 @@ void ExtendedDefaultPolicy::NoteInt(InlineObservation obs, int value)
             {
                 SetNever(InlineObservation::CALLEE_DOES_NOT_RETURN);
             }
-            else if (!m_IsForceInline)
+            else if (!m_IsForceInline && !m_HasProfile)
             {
                 unsigned bbLimit = (unsigned)JitConfig.JitExtDefaultPolicyMaxBB();
                 if (m_IsPrejitRoot)
                 {
                     // We're not able to recognize arg-specific foldable branches
                     // in prejit-root mode.
-                    bbLimit += 3;
+                    bbLimit += 5 + m_Switch * 10;
                 }
-                bbLimit += m_FoldableBranch;
+                bbLimit += m_FoldableBranch + m_FoldableSwitch * 10;
                 if ((unsigned)value > bbLimit)
                 {
                     SetNever(InlineObservation::CALLEE_TOO_MANY_BASIC_BLOCKS);
@@ -1419,13 +1433,13 @@ double ExtendedDefaultPolicy::DetermineMultiplier()
     if (m_ReturnsStructByValue)
     {
         // For structs-passed-by-value we might avoid expensive copy operations if we inline.
-        multiplier += 1.5;
+        multiplier += 2.0;
         JITDUMP("\nInline candidate returns a struct by value.  Multiplier increased to %g.", multiplier);
     }
     else if (m_ArgIsStructByValue > 0)
     {
         // Same here
-        multiplier += 1.5;
+        multiplier += 2.0;
         JITDUMP("\n%d arguments are structs passed by value.  Multiplier increased to %g.", m_ArgIsStructByValue,
                 multiplier);
     }
@@ -1451,13 +1465,13 @@ double ExtendedDefaultPolicy::DetermineMultiplier()
 
     if (m_ArgFeedsRangeCheck > 0)
     {
-        multiplier += 0.5;
+        multiplier += 1.0;
         JITDUMP("\nInline candidate has arg that feeds range check.  Multiplier increased to %g.", multiplier);
     }
 
     if (m_NonGenericCallsGeneric)
     {
-        multiplier += 1.5;
+        multiplier += 2.0;
         JITDUMP("\nInline candidate is generic and caller is not.  Multiplier increased to %g.", multiplier);
     }
 
@@ -1507,7 +1521,7 @@ double ExtendedDefaultPolicy::DetermineMultiplier()
     if (m_Intrinsic > 0)
     {
         // In most cases such intrinsics are lowered as single CPU instructions
-        multiplier += 1.5;
+        multiplier += 1.0 + m_Intrinsic * 0.3;
         JITDUMP("\nInline has %d intrinsics.  Multiplier increased to %g.", m_Intrinsic, multiplier);
     }
 
@@ -1636,6 +1650,28 @@ double ExtendedDefaultPolicy::DetermineMultiplier()
             break;
     }
 
+    if (m_FoldableSwitch > 0)
+    {
+        multiplier += 6.0;
+        JITDUMP("\nInline candidate has %d foldable switches.  Multiplier increased to %g.", m_FoldableSwitch,
+                multiplier);
+    }
+    else if (m_Switch > 0)
+    {
+        if (m_IsPrejitRoot)
+        {
+            // Assume the switches can be foldable in PrejitRoot mode.
+            multiplier += 6.0;
+            JITDUMP("\nPrejit root candidate has %d switches.  Multiplier increased to %g.", m_Switch, multiplier);
+        }
+        else
+        {
+            // TODO: Investigate cases where it makes sense to inline non-foldable switches
+            multiplier = 0.0;
+            JITDUMP("\nInline candidate has %d switches.  Multiplier limited to %g.", m_Switch, multiplier);
+        }
+    }
+
     if (m_HasProfile)
     {
         // There are cases when Profile Data can be misleading or polluted:
@@ -1657,14 +1693,16 @@ double ExtendedDefaultPolicy::DetermineMultiplier()
         {
             multiplier *= min(m_ProfileFrequency, 1.0) * profileScale;
         }
-        JITDUMP("\nCallsite has profile data: %g.", m_ProfileFrequency);
+        JITDUMP("\nCallsite has profile data: %g.  Multiplier limited to %g.", m_ProfileFrequency, multiplier);
     }
 
-    if (m_RootCompiler->lvaTableCnt > ((unsigned)(JitConfig.JitMaxLocalsToTrack() / 4)))
+    // Slow down if there are already too many locals
+    if (m_RootCompiler->lvaTableCnt > 64)
     {
-        // Slow down inlining if we already have to many locals in the rootCompiler.
-        multiplier /= ((double)m_RootCompiler->lvaTableCnt / ((double)JitConfig.JitMaxLocalsToTrack() / 4.0));
-        JITDUMP("\nCaller %d locals.  Multiplier decreased to %g.", m_RootCompiler->lvaTableCnt, multiplier);
+        // E.g. MaxLocalsToTrack = 1024 and lvaTableCnt = 512 -> multiplier *= 0.5;
+        const double lclFullness = min(1.0, (double)m_RootCompiler->lvaTableCnt / JitConfig.JitMaxLocalsToTrack());
+        multiplier *= (1.0 - lclFullness);
+        JITDUMP("\nCaller has %d locals.  Multiplier decreased to %g.", m_RootCompiler->lvaTableCnt, multiplier);
     }
 
     if (m_BackwardJump)
@@ -1738,6 +1776,8 @@ void ExtendedDefaultPolicy::OnDumpXml(FILE* file, unsigned indent) const
     XATTR_I4(m_FoldableExpr)
     XATTR_I4(m_FoldableExprUn)
     XATTR_I4(m_FoldableBranch)
+    XATTR_I4(m_FoldableSwitch)
+    XATTR_I4(m_Switch)
     XATTR_I4(m_DivByCns)
     XATTR_B(m_ReturnsStructByValue)
     XATTR_B(m_IsFromValueClass)
@@ -1927,7 +1967,6 @@ void DiscretionaryPolicy::NoteDouble(InlineObservation obs, double value)
 {
     assert(obs == InlineObservation::CALLSITE_PROFILE_FREQUENCY);
     assert(value >= 0.0);
-    assert(m_ProfileFrequency == 0.0);
 
     m_ProfileFrequency = value;
 }