8316959: Improve InlineCacheBuffer pending queue management

Reviewed-by: dlong, kvn

src/hotspot/share/code/icBuffer.cpp

@@ -34,6 +34,7 @@
 #include "memory/resourceArea.hpp"
 #include "oops/method.hpp"
 #include "oops/oop.inline.hpp"
+#include "runtime/atomic.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/javaThread.hpp"
 #include "runtime/mutexLocker.hpp"
@@ -44,8 +45,8 @@ DEF_STUB_INTERFACE(ICStub);
 
 StubQueue* InlineCacheBuffer::_buffer    = nullptr;
 
-CompiledICHolder* InlineCacheBuffer::_pending_released = nullptr;
-int InlineCacheBuffer::_pending_count = 0;
+CompiledICHolder* volatile InlineCacheBuffer::_pending_released = nullptr;
+volatile int InlineCacheBuffer::_pending_count = 0;
 
 #ifdef ASSERT
 ICRefillVerifier::ICRefillVerifier()
@@ -247,26 +248,42 @@ void* InlineCacheBuffer::cached_value_for(CompiledIC *ic) {
 // Free CompiledICHolder*s that are no longer in use
 void InlineCacheBuffer::release_pending_icholders() {
   assert(SafepointSynchronize::is_at_safepoint(), "should only be called during a safepoint");
-  CompiledICHolder* holder = _pending_released;
+  CompiledICHolder* holder = Atomic::load(&_pending_released);
   _pending_released = nullptr;
+  int count = 0;
   while (holder != nullptr) {
     CompiledICHolder* next = holder->next();
     delete holder;
     holder = next;
-    _pending_count--;
+    count++;
   }
-  assert(_pending_count == 0, "wrong count");
+  assert(pending_icholder_count() == count, "wrong count");
+  Atomic::store(&_pending_count, 0);
 }
 
 // Enqueue this icholder for release during the next safepoint.  It's
-// not safe to free them until them since they might be visible to
+// not safe to free them until then since they might be visible to
 // another thread.
 void InlineCacheBuffer::queue_for_release(CompiledICHolder* icholder) {
-  MutexLocker mex(InlineCacheBuffer_lock, Mutex::_no_safepoint_check_flag);
-  icholder->set_next(_pending_released);
-  _pending_released = icholder;
-  _pending_count++;
+  assert(icholder->next() == nullptr, "multiple enqueue?");
+
+  CompiledICHolder* old = Atomic::load(&_pending_released);
+  for (;;) {
+    icholder->set_next(old);
+    // The only reader runs at a safepoint serially so there is no need for a more strict atomic.
+    CompiledICHolder* cur = Atomic::cmpxchg(&_pending_released, old, icholder, memory_order_relaxed);
+    if (cur == old) {
+      break;
+    }
+    old = cur;
+  }
+  Atomic::inc(&_pending_count, memory_order_relaxed);
+
   if (TraceICBuffer) {
     tty->print_cr("enqueueing icholder " INTPTR_FORMAT " to be freed", p2i(icholder));
   }
 }
+
+int InlineCacheBuffer::pending_icholder_count() {
+  return Atomic::load(&_pending_count);
+}

src/hotspot/share/code/icBuffer.hpp

@@ -146,8 +146,8 @@ class InlineCacheBuffer: public AllStatic {
 
   static StubQueue* _buffer;
 
-  static CompiledICHolder* _pending_released;
-  static int _pending_count;
+  static CompiledICHolder* volatile _pending_released;
+  static volatile int _pending_count;
 
   static StubQueue* buffer()                         { return _buffer;         }
 
@@ -176,7 +176,7 @@ class InlineCacheBuffer: public AllStatic {
 
   static void release_pending_icholders();
   static void queue_for_release(CompiledICHolder* icholder);
-  static int pending_icholder_count() { return _pending_count; }
+  static int pending_icholder_count();
 
   // New interface
   static bool    create_transition_stub(CompiledIC *ic, void* cached_value, address entry);

src/hotspot/share/oops/compiledICHolder.cpp

@@ -32,7 +32,7 @@ volatile int CompiledICHolder::_live_not_claimed_count;
 #endif
 
 CompiledICHolder::CompiledICHolder(Metadata* metadata, Klass* klass, bool is_method)
-  : _holder_metadata(metadata), _holder_klass(klass), _is_metadata_method(is_method) {
+  : _holder_metadata(metadata), _holder_klass(klass), _next(nullptr), _is_metadata_method(is_method) {
 #ifdef ASSERT
   Atomic::inc(&_live_count);
   Atomic::inc(&_live_not_claimed_count);