Fix card marking stealing for regions (#49704)

Essentially the problem was that for regions, we have several lists of segments, one for each generation. Thus, the card_marking_enumerator logic needs provisions to restart enumerating with another segment list.

The changes fall into 3 categories:

- new method switch_to_segment to start enumerating a new segment list - called for a new generation
- changes to make the enumerator state consistent across one segment list ending and another starting
- added dprintf instrumentation so we can later reconstruct how different threads enumerated chunks.

2 files changed, 26 insertions, 4 deletions

diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp
index 9307b16a9a5..3aa5e6ca80d 100644
--- a/src/coreclr/gc/gc.cpp
+++ b/src/coreclr/gc/gc.cpp
@@ -33533,6 +33533,9 @@ bool card_marking_enumerator::move_next(heap_segment* seg, uint8_t*& low, uint8_
                 low = (chunk_index_within_seg == 0) ? start : (aligned_start + (size_t)chunk_index_within_seg * CARD_MARKING_STEALING_GRANULARITY);
                 high = (chunk_index_within_seg + 1 == chunk_count_within_seg) ? end : (aligned_start + (size_t)(chunk_index_within_seg + 1) * CARD_MARKING_STEALING_GRANULARITY);
                 chunk_high = high;
+
+                dprintf (3, ("cme:mn ci: %u, low: %Ix, high: %Ix", chunk_index, low, high));
+
                 return true;
             }
             else
@@ -33549,16 +33552,24 @@ bool card_marking_enumerator::move_next(heap_segment* seg, uint8_t*& low, uint8_
 
                 // keep the chunk index for later
                 old_chunk_index = chunk_index;
+
+                dprintf (3, ("cme:mn oci: %u, seg mismatch seg: %Ix, segment: %Ix", old_chunk_index, heap_segment_mem (segment), heap_segment_mem (seg)));
+
                 return false;
             }
         }
 
         segment = heap_segment_next_in_range(segment);
+        segment_start_chunk_index += chunk_count_within_seg;
         if (segment == nullptr)
         {
+            // keep the chunk index for later
+            old_chunk_index = chunk_index;
+
+            dprintf (3, ("cme:mn oci: %u no more segments", old_chunk_index));
+
             return false;
         }
-        segment_start_chunk_index += chunk_count_within_seg;
     }
 }
 
@@ -33740,7 +33751,10 @@ void gc_heap::mark_through_cards_for_segments (card_fn fn, BOOL relocating CARD_
                 {
                     // Switch to regions for this generation.
                     seg = generation_start_segment (generation_of (curr_gen_number));
-                    dprintf (REGIONS_LOG, ("h%d switching to gen%d start seg %Ix", 
+#ifdef FEATURE_CARD_MARKING_STEALING
+                    card_mark_enumerator.switch_to_segment(seg);
+#endif // FEATURE_CARD_MARKING_STEALING
+                    dprintf (REGIONS_LOG, ("h%d switching to gen%d start seg %Ix",
                         heap_number, curr_gen_number, (size_t)seg));
                 }
             }
diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h
index 0e9f0a18895..97c2c391db8 100644
--- a/src/coreclr/gc/gcpriv.h
+++ b/src/coreclr/gc/gcpriv.h
@@ -342,12 +342,12 @@ class recursive_gc_sync;
 
 #ifdef MULTIPLE_HEAPS
 // This feature hasn't been enabled for regions yet.
-#ifndef USE_REGIONS
+//#ifndef USE_REGIONS
 // card marking stealing only makes sense in server GC
 // but it works and is easier to debug for workstation GC
 // so turn it on for server GC, turn on for workstation GC if necessary
 #define FEATURE_CARD_MARKING_STEALING
-#endif //!USE_REGIONS
+//#endif //!USE_REGIONS
 #endif //MULTIPLE_HEAPS
 
 #ifdef FEATURE_CARD_MARKING_STEALING
@@ -5664,6 +5664,14 @@ public:
             ;
     }
 
+#ifdef USE_REGIONS
+    void switch_to_segment(heap_segment* seg)
+    {
+        assert(segment == nullptr);
+        segment = seg;
+    }
+#endif
+
     uint8_t* get_chunk_high()
     {
         return chunk_high;