8294775: Shenandoah: reduce contention on _threads_in_evac

Reviewed-by: rkennke, shade

src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.cpp

@@ -29,15 +29,112 @@
 #include "runtime/javaThread.hpp"
 #include "runtime/os.hpp"
 
-const jint ShenandoahEvacOOMHandler::OOM_MARKER_MASK = 0x80000000;
+const jint ShenandoahEvacOOMCounter::OOM_MARKER_MASK = 0x80000000;
+
+ShenandoahEvacOOMCounter::ShenandoahEvacOOMCounter() :
+  _bits(0) {
+}
+
+void ShenandoahEvacOOMCounter::decrement() {
+  assert(unmasked_count() > 0, "sanity");
+  // NOTE: It's ok to simply decrement, even with mask set, because unmasked value is positive.
+  Atomic::dec(&_bits);
+}
+
+void ShenandoahEvacOOMCounter::clear() {
+  assert(unmasked_count() == 0, "sanity");
+  Atomic::release_store_fence(&_bits, (jint)0);
+}
+
+void ShenandoahEvacOOMCounter::set_oom_bit(bool decrement) {
+  jint threads_in_evac = Atomic::load_acquire(&_bits);
+  while (true) {
+    jint newval = decrement
+      ? (threads_in_evac - 1) | OOM_MARKER_MASK
+      : threads_in_evac | OOM_MARKER_MASK;
+
+    jint other = Atomic::cmpxchg(&_bits, threads_in_evac, newval);
+    if (other == threads_in_evac) {
+      // Success: wait for other threads to get out of the protocol and return.
+      break;
+    } else {
+      // Failure: try again with updated new value.
+      threads_in_evac = other;
+    }
+  }
+}
+
+bool ShenandoahEvacOOMCounter::try_increment()
+{
+  jint threads_in_evac = Atomic::load_acquire(&_bits);
+
+  while (true) {
+    // Cannot enter evacuation if OOM_MARKER_MASK is set.
+    if ((threads_in_evac & OOM_MARKER_MASK) != 0) {
+      return false;
+    }
+
+    jint other = Atomic::cmpxchg(&_bits, threads_in_evac, threads_in_evac + 1);
+    if (other == threads_in_evac) {
+      // Success: caller may safely enter evacuation
+      return true;
+    } else {
+      threads_in_evac = other;
+    }
+  }
+}
 
 ShenandoahEvacOOMHandler::ShenandoahEvacOOMHandler() :
-  _threads_in_evac(0) {
+  _num_counters(calc_num_counters()) {
+
+  assert(_num_counters > 0, "sanity");
+  assert(is_power_of_2(_num_counters), "must be");
+
+  _threads_in_evac = NEW_C_HEAP_ARRAY(ShenandoahEvacOOMCounter, _num_counters, mtGC);
+  for (int i = 0; i < _num_counters; i++) {
+    new (&_threads_in_evac[i]) ShenandoahEvacOOMCounter();
+  }
+}
+
+int ShenandoahEvacOOMHandler::calc_num_counters() {
+  // Scale the number of counter buckets with the number of CPUs to
+  // minimise contention.  Also make sure the number is a power of two
+  // so we can map hash values to buckets with a simple mask.
+  const int nproc = os::active_processor_count();
+  const int clamped = MAX2(1, MIN2(nproc, 128));
+  return round_up_power_of_2(clamped);
+}
+
+uint64_t ShenandoahEvacOOMHandler::hash_pointer(const void* p) {
+  // Bit mixing function from MurmurHash3
+  uint64_t key = (uintptr_t)p;
+  key ^= (key >> 33);
+  key *= UINT64_C(0xff51afd7ed558ccd);
+  key ^= (key >> 33);
+  key *= UINT64_C(0xc4ceb9fe1a85ec53);
+  key ^= (key >> 33);
+  return key;
+}
+
+ShenandoahEvacOOMCounter* ShenandoahEvacOOMHandler::counter_for_thread(Thread* t) {
+  const uint64_t key = hash_pointer(t);
+  return &_threads_in_evac[key & (_num_counters - 1)];
+}
+
+void ShenandoahEvacOOMHandler::wait_for_one_counter(ShenandoahEvacOOMCounter* ptr) {
+  // We might be racing against handle_out_of_memory_during_evacuation()
+  // setting the OOM_MARKER_MASK bit so we must make sure it is set here
+  // *and* the counter is zero.
+  while (ptr->load_acquire() != ShenandoahEvacOOMCounter::OOM_MARKER_MASK) {
+    os::naked_short_sleep(1);
+  }
 }
 
 void ShenandoahEvacOOMHandler::wait_for_no_evac_threads() {
-  while ((Atomic::load_acquire(&_threads_in_evac) & ~OOM_MARKER_MASK) != 0) {
-    os::naked_short_sleep(1);
+  // Once the OOM_MARKER_MASK bit is set the counter can only decrease
+  // so it's safe to check each bucket in turn.
+  for (int i = 0; i < _num_counters; i++) {
+    wait_for_one_counter(&_threads_in_evac[i]);
   }
   // At this point we are sure that no threads can evacuate anything. Raise
   // the thread-local oom_during_evac flag to indicate that any attempt
@@ -46,32 +143,18 @@ void ShenandoahEvacOOMHandler::wait_for_no_evac_threads() {
 }
 
 void ShenandoahEvacOOMHandler::register_thread(Thread* thr) {
-  jint threads_in_evac = Atomic::load_acquire(&_threads_in_evac);
-
   assert(!ShenandoahThreadLocalData::is_oom_during_evac(Thread::current()), "TL oom-during-evac must not be set");
-  while (true) {
-    // Check for OOM.
-    // If offender has OOM_MARKER_MASK, then loop until no more threads in evac
-    if ((threads_in_evac & OOM_MARKER_MASK) != 0) {
-      wait_for_no_evac_threads();
-      return;
-    }
 
-    jint other = Atomic::cmpxchg(&_threads_in_evac, threads_in_evac, threads_in_evac + 1);
-    if (other == threads_in_evac) {
-      // Success: caller may safely enter evacuation
-      return;
-    } else {
-      threads_in_evac = other;
-    }
+  ShenandoahEvacOOMCounter* counter = counter_for_thread(thr);
+  if (!counter->try_increment()) {
+    // Counter has OOM_MARKER_MASK set, loop until no more threads in evac
+    wait_for_no_evac_threads();
   }
 }
 
 void ShenandoahEvacOOMHandler::unregister_thread(Thread* thr) {
   if (!ShenandoahThreadLocalData::is_oom_during_evac(thr)) {
-    assert((Atomic::load_acquire(&_threads_in_evac) & ~OOM_MARKER_MASK) > 0, "sanity");
-    // NOTE: It's ok to simply decrement, even with mask set, because unmasked value is positive.
-    Atomic::dec(&_threads_in_evac);
+    counter_for_thread(thr)->decrement();
   } else {
     // If we get here, the current thread has already gone through the
     // OOM-during-evac protocol and has thus either never entered or successfully left
@@ -85,22 +168,20 @@ void ShenandoahEvacOOMHandler::handle_out_of_memory_during_evacuation() {
   assert(ShenandoahThreadLocalData::is_evac_allowed(Thread::current()), "sanity");
   assert(!ShenandoahThreadLocalData::is_oom_during_evac(Thread::current()), "TL oom-during-evac must not be set");
 
-  jint threads_in_evac = Atomic::load_acquire(&_threads_in_evac);
-  while (true) {
-    jint other = Atomic::cmpxchg(&_threads_in_evac, threads_in_evac, (threads_in_evac - 1) | OOM_MARKER_MASK);
-    if (other == threads_in_evac) {
-      // Success: wait for other threads to get out of the protocol and return.
-      wait_for_no_evac_threads();
-      return;
-    } else {
-      // Failure: try again with updated new value.
-      threads_in_evac = other;
-    }
+  ShenandoahEvacOOMCounter* self = counter_for_thread(Thread::current());
+  assert(self->unmasked_count() > 0, "sanity");
+
+  for (int i = 0; i < _num_counters; i++) {
+    ShenandoahEvacOOMCounter* counter = &_threads_in_evac[i];
+    counter->set_oom_bit(counter == self);
   }
+
+  wait_for_no_evac_threads();
 }
 
 void ShenandoahEvacOOMHandler::clear() {
   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
-  assert((Atomic::load_acquire(&_threads_in_evac) & ~OOM_MARKER_MASK) == 0, "sanity");
-  Atomic::release_store_fence(&_threads_in_evac, (jint)0);
+  for (int i = 0; i < _num_counters; i++) {
+    _threads_in_evac[i].clear();
+  }
 }

src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.hpp

@@ -30,6 +30,31 @@
 #include "runtime/javaThread.hpp"
 #include "utilities/globalDefinitions.hpp"
 
+/**
+ * Striped counter used to implement the OOM protocol described below.
+ */
+class ShenandoahEvacOOMCounter {
+private:
+  // Combination of a 31-bit counter and 1-bit OOM marker.
+  volatile jint _bits;
+
+  // This class must be at least a cache line in size to prevent false sharing.
+  shenandoah_padding_minus_size(0, sizeof(jint));
+
+public:
+  static const jint OOM_MARKER_MASK;
+
+  ShenandoahEvacOOMCounter();
+
+  void decrement();
+  bool try_increment();
+  void clear();
+  void set_oom_bit(bool decrement);
+
+  inline jint unmasked_count();
+  inline jint load_acquire();
+};
+
 /**
  * Provides safe handling of out-of-memory situations during evacuation.
  *
@@ -57,7 +82,9 @@
  * allocation, copying and CASing of the copy object, and is protected by this
  * OOM-during-evac-handler. The handler allows multiple threads to enter and exit
  * evacuation path, but on OOME it requires all threads that experienced OOME to wait
- * for current threads to leave, and blocks other threads from entering.
+ * for current threads to leave, and blocks other threads from entering. The counter state
+ * is striped across multiple cache lines to reduce contention when many threads attempt
+ * to enter or leave the protocol at the same time.
  *
  * Detailed state change:
  *
@@ -81,14 +108,18 @@
  */
 class ShenandoahEvacOOMHandler {
 private:
-  static const jint OOM_MARKER_MASK;
+  const int _num_counters;
 
   shenandoah_padding(0);
-  volatile jint _threads_in_evac;
-  shenandoah_padding(1);
+  ShenandoahEvacOOMCounter* _threads_in_evac;
+
+  ShenandoahEvacOOMCounter* counter_for_thread(Thread* t);
 
   void wait_for_no_evac_threads();
+  void wait_for_one_counter(ShenandoahEvacOOMCounter* ptr);
 
+  static uint64_t hash_pointer(const void* p);
+  static int calc_num_counters();
 public:
   ShenandoahEvacOOMHandler();
 

src/hotspot/share/gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp

@@ -31,19 +31,25 @@
 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
 #include "runtime/atomic.hpp"
 
-void ShenandoahEvacOOMHandler::enter_evacuation(Thread* thr) {
-  jint threads_in_evac = Atomic::load_acquire(&_threads_in_evac);
+jint ShenandoahEvacOOMCounter::load_acquire() {
+  return Atomic::load_acquire(&_bits);
+}
 
+jint ShenandoahEvacOOMCounter::unmasked_count() {
+  return Atomic::load_acquire(&_bits) & ~OOM_MARKER_MASK;
+}
+
+void ShenandoahEvacOOMHandler::enter_evacuation(Thread* thr) {
   uint8_t level = ShenandoahThreadLocalData::push_evac_oom_scope(thr);
  if (level == 0) {
    // Entering top level scope, register this thread.
    register_thread(thr);
  } else if (!ShenandoahThreadLocalData::is_oom_during_evac(thr)) {
-   jint threads_in_evac = Atomic::load_acquire(&_threads_in_evac);
+   ShenandoahEvacOOMCounter* counter = counter_for_thread(thr);
+   jint threads_in_evac = counter->load_acquire();
    // If OOM is in progress, handle it.
-   if ((threads_in_evac & OOM_MARKER_MASK) != 0) {
-     assert((threads_in_evac & ~OOM_MARKER_MASK) > 0, "sanity");
-     Atomic::dec(&_threads_in_evac);
+   if ((threads_in_evac & ShenandoahEvacOOMCounter::OOM_MARKER_MASK) != 0) {
+     counter->decrement();
      wait_for_no_evac_threads();
    }
  }

src/hotspot/share/gc/shenandoah/shenandoahPadding.hpp

@@ -37,4 +37,7 @@
 #define shenandoah_padding(id) \
   DEFINE_PAD_MINUS_SIZE(id, SHENANDOAH_CACHE_LINE_SIZE, 0)
 
+#define shenandoah_padding_minus_size(id, size) \
+  DEFINE_PAD_MINUS_SIZE(id, SHENANDOAH_CACHE_LINE_SIZE, size)
+
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHPADDING_HPP