8245240: Shenandoah: support nesting evacuation OOM scope

Reviewed-by: shade, rkennke

src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp

@@ -51,19 +51,23 @@ void ShenandoahEvacOOMHandler::wait_for_no_evac_threads() {
 void ShenandoahEvacOOMHandler::enter_evacuation() {
   jint threads_in_evac = Atomic::load_acquire(&_threads_in_evac);
 
-  assert(!ShenandoahThreadLocalData::is_evac_allowed(Thread::current()), "sanity");
-  assert(!ShenandoahThreadLocalData::is_oom_during_evac(Thread::current()), "TL oom-during-evac must not be set");
-
+  Thread* const thr = Thread::current();
+  uint8_t level = ShenandoahThreadLocalData::push_evac_oom_scope(thr);
   if ((threads_in_evac & OOM_MARKER_MASK) != 0) {
     wait_for_no_evac_threads();
     return;
   }
 
+  // Nesting case, this thread already registered
+  if (level > 0) {
+     return;
+  }
+
+  assert(!ShenandoahThreadLocalData::is_oom_during_evac(Thread::current()), "TL oom-during-evac must not be set");
   while (true) {
     jint other = Atomic::cmpxchg(&_threads_in_evac, threads_in_evac, threads_in_evac + 1);
     if (other == threads_in_evac) {
       // Success: caller may safely enter evacuation
-      DEBUG_ONLY(ShenandoahThreadLocalData::set_evac_allowed(Thread::current(), true));
       return;
     } else {
       // Failure:
@@ -79,7 +83,14 @@ void ShenandoahEvacOOMHandler::enter_evacuation() {
 }
 
 void ShenandoahEvacOOMHandler::leave_evacuation() {
-  if (!ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) {
+  Thread* const thr = Thread::current();
+  uint8_t level = ShenandoahThreadLocalData::pop_evac_oom_scope(thr);
+  // Not top level, just return
+  if (level > 1) {
+    return;
+  }
+
+  if (!ShenandoahThreadLocalData::is_oom_during_evac(thr)) {
     assert((Atomic::load_acquire(&_threads_in_evac) & ~OOM_MARKER_MASK) > 0, "sanity");
     // NOTE: It's ok to simply decrement, even with mask set, because unmasked value is positive.
     Atomic::dec(&_threads_in_evac);
@@ -87,10 +98,9 @@ void ShenandoahEvacOOMHandler::leave_evacuation() {
     // If we get here, the current thread has already gone through the
     // OOM-during-evac protocol and has thus either never entered or successfully left
     // the evacuation region. Simply flip its TL oom-during-evac flag back off.
-    ShenandoahThreadLocalData::set_oom_during_evac(Thread::current(), false);
+    ShenandoahThreadLocalData::set_oom_during_evac(thr, false);
   }
-  DEBUG_ONLY(ShenandoahThreadLocalData::set_evac_allowed(Thread::current(), false));
-  assert(!ShenandoahThreadLocalData::is_oom_during_evac(Thread::current()), "TL oom-during-evac must be turned off");
+  assert(!ShenandoahThreadLocalData::is_oom_during_evac(thr), "TL oom-during-evac must be turned off");
 }
 
 void ShenandoahEvacOOMHandler::handle_out_of_memory_during_evacuation() {

src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp

@@ -26,6 +26,7 @@
 #define SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP
 
 #include "gc/shared/plab.hpp"
+#include "gc/shared/gcThreadLocalData.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
 #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp"
@@ -39,7 +40,9 @@ public:
 
 private:
   char _gc_state;
-  char _oom_during_evac;
+  // Evacuation OOM state
+  uint8_t                 _oom_scope_nesting_level;
+  bool                    _oom_during_evac;
   ShenandoahSATBMarkQueue _satb_mark_queue;
   PLAB* _gclab;
   size_t _gclab_size;
@@ -49,7 +52,8 @@ private:
 
   ShenandoahThreadLocalData() :
     _gc_state(0),
-    _oom_during_evac(0),
+    _oom_scope_nesting_level(0),
+    _oom_during_evac(false),
     _satb_mark_queue(&ShenandoahBarrierSet::satb_mark_queue_set()),
     _gclab(NULL),
     _gclab_size(0),
@@ -90,18 +94,6 @@ public:
     return data(thread)->_satb_mark_queue;
   }
 
-  static bool is_oom_during_evac(Thread* thread) {
-    return (data(thread)->_oom_during_evac & 1) == 1;
-  }
-
-  static void set_oom_during_evac(Thread* thread, bool oom) {
-    if (oom) {
-      data(thread)->_oom_during_evac |= 1;
-    } else {
-      data(thread)->_oom_during_evac &= ~1;
-    }
-  }
-
   static void set_gc_state(Thread* thread, char gc_state) {
     data(thread)->_gc_state = gc_state;
   }
@@ -151,19 +143,38 @@ public:
     data(thread)->_disarmed_value = value;
   }
 
-#ifdef ASSERT
-  static void set_evac_allowed(Thread* thread, bool evac_allowed) {
-    if (evac_allowed) {
-      data(thread)->_oom_during_evac |= 2;
-    } else {
-      data(thread)->_oom_during_evac &= ~2;
-    }
+  // Evacuation OOM handling
+  static bool is_oom_during_evac(Thread* thread) {
+    return data(thread)->_oom_during_evac;
+  }
+
+  static void set_oom_during_evac(Thread* thread, bool oom) {
+    data(thread)->_oom_during_evac = oom;
+  }
+
+  static uint8_t evac_oom_scope_level(Thread* thread) {
+    return data(thread)->_oom_scope_nesting_level;
+  }
+
+  // Push the scope one level deeper, return previous level
+  static uint8_t push_evac_oom_scope(Thread* thread) {
+    uint8_t level = evac_oom_scope_level(thread);
+    assert(level < 254, "Overflow nesting level"); // UINT8_MAX = 255
+    data(thread)->_oom_scope_nesting_level = level + 1;
+    return level;
+  }
+
+  // Pop the scope by one level, return previous level
+  static uint8_t pop_evac_oom_scope(Thread* thread) {
+    uint8_t level = evac_oom_scope_level(thread);
+    assert(level > 0, "Underflow nesting level");
+    data(thread)->_oom_scope_nesting_level = level - 1;
+    return level;
   }
 
   static bool is_evac_allowed(Thread* thread) {
-    return (data(thread)->_oom_during_evac & 2) == 2;
+    return evac_oom_scope_level(thread) > 0;
   }
-#endif
 
   // Offsets
   static ByteSize satb_mark_queue_active_offset() {
@@ -187,4 +198,6 @@ public:
   }
 };
 
+STATIC_ASSERT(sizeof(ShenandoahThreadLocalData) <= sizeof(GCThreadLocalData));
+
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHTHREADLOCALDATA_HPP