8320239: add dynamic switch for JvmtiVTMSTransitionDisabler sync protocol

Reviewed-by: lmesnik, pchilanomate, amenkov

src/hotspot/share/prims/jvmtiThreadState.cpp

@@ -228,6 +228,12 @@ volatile bool JvmtiVTMSTransitionDisabler::_SR_mode = false;
 // Notifications from VirtualThread about VTMS events are enabled.
 bool JvmtiVTMSTransitionDisabler::_VTMS_notify_jvmti_events = false;
 
+// The JvmtiVTMSTransitionDisabler sync protocol is enabled if this count > 0.
+volatile int JvmtiVTMSTransitionDisabler::_sync_protocol_enabled_count = 0;
+
+// JvmtiVTMSTraansitionDisabler sync protocol is enabled permanently after seeing a suspender.
+volatile bool JvmtiVTMSTransitionDisabler::_sync_protocol_enabled_permanently = false;
+
 #ifdef ASSERT
 void
 JvmtiVTMSTransitionDisabler::print_info() {
@@ -255,6 +261,9 @@ JvmtiVTMSTransitionDisabler::JvmtiVTMSTransitionDisabler(jthread thread)
   if (Thread::current_or_null() == nullptr) {
     return;  // Detached thread, can be a call from Agent_OnLoad.
   }
+  if (!sync_protocol_enabled_permanently()) {
+    JvmtiVTMSTransitionDisabler::inc_sync_protocol_enabled_count();
+  }
   if (_thread != nullptr) {
     VTMS_transition_disable_for_one(); // disable VTMS transitions for one virtual thread
   } else {
@@ -272,6 +281,12 @@ JvmtiVTMSTransitionDisabler::JvmtiVTMSTransitionDisabler(bool is_SR)
   if (Thread::current_or_null() == nullptr) {
     return;  // Detached thread, can be a call from Agent_OnLoad.
   }
+  if (!sync_protocol_enabled_permanently()) {
+    JvmtiVTMSTransitionDisabler::inc_sync_protocol_enabled_count();
+    if (is_SR) {
+      Atomic::store(&_sync_protocol_enabled_permanently, true);
+    }
+  }
   VTMS_transition_disable_for_all();
 }
 
@@ -287,6 +302,9 @@ JvmtiVTMSTransitionDisabler::~JvmtiVTMSTransitionDisabler() {
   } else {
     VTMS_transition_enable_for_all(); // enable VTMS transitions for all virtual threads
   }
+  if (!sync_protocol_enabled_permanently()) {
+    JvmtiVTMSTransitionDisabler::dec_sync_protocol_enabled_count();
+  }
 }
 
 // disable VTMS transitions for one virtual thread
@@ -404,14 +422,21 @@ JvmtiVTMSTransitionDisabler::VTMS_transition_enable_for_all() {
 void
 JvmtiVTMSTransitionDisabler::start_VTMS_transition(jthread vthread, bool is_mount) {
   JavaThread* thread = JavaThread::current();
-  HandleMark hm(thread);
-  Handle vth = Handle(thread, JNIHandles::resolve_external_guard(vthread));
-  int attempts = 50000;
+  oop vt = JNIHandles::resolve_external_guard(vthread);
+  assert(!thread->is_in_VTMS_transition(), "VTMS_transition sanity check");
 
   // Avoid using MonitorLocker on performance critical path, use
   // two-level synchronization with lock-free operations on counters.
   Atomic::inc(&_VTMS_transition_count); // Try to enter VTMS transition section optmistically.
-  java_lang_Thread::set_is_in_VTMS_transition(vth(), true);
+  java_lang_Thread::set_is_in_VTMS_transition(vt, true);
+
+  if (!sync_protocol_enabled()) {
+    thread->set_is_in_VTMS_transition(true);
+    return;
+  }
+  HandleMark hm(thread);
+  Handle vth = Handle(thread, vt);
+  int attempts = 50000;
 
   // Do not allow suspends inside VTMS transitions.
   // Block while transitions are disabled or there are suspend requests.
@@ -458,7 +483,6 @@ JvmtiVTMSTransitionDisabler::start_VTMS_transition(jthread vthread, bool is_moun
   }
 #endif
   // Enter VTMS transition section.
-  assert(!thread->is_in_VTMS_transition(), "VTMS_transition sanity check");
   thread->set_is_in_VTMS_transition(true);
 }
 
@@ -469,11 +493,14 @@ JvmtiVTMSTransitionDisabler::finish_VTMS_transition(jthread vthread, bool is_mou
   assert(thread->is_in_VTMS_transition(), "sanity check");
   thread->set_is_in_VTMS_transition(false);
   oop vt = JNIHandles::resolve_external_guard(vthread);
-  int64_t thread_id = java_lang_Thread::thread_id(vt);
   java_lang_Thread::set_is_in_VTMS_transition(vt, false);
-
   Atomic::dec(&_VTMS_transition_count);
 
+  if (!sync_protocol_enabled()) {
+    return;
+  }
+  int64_t thread_id = java_lang_Thread::thread_id(vt);
+
   // Unblock waiting VTMS transition disablers.
   if (_VTMS_transition_disable_for_one_count > 0 ||
       _VTMS_transition_disable_for_all_count > 0) {

src/hotspot/share/prims/jvmtiThreadState.hpp

@@ -83,6 +83,8 @@ class JvmtiVTMSTransitionDisabler {
   static volatile int _VTMS_transition_disable_for_all_count; // transitions for all virtual threads are disabled while it is positive
   static volatile bool _SR_mode;                         // there is an active suspender or resumer
   static volatile int _VTMS_transition_count;            // current number of VTMS transitions
+  static volatile int _sync_protocol_enabled_count;      // current number of JvmtiVTMSTransitionDisablers enabled sync protocol
+  static volatile bool _sync_protocol_enabled_permanently; // seen a suspender: JvmtiVTMSTraansitionDisabler protocol is enabled permanently
 
   bool _is_SR;                                           // is suspender or resumer
   jthread _thread;                                       // virtual thread to disable transitions for, no-op if it is a platform thread
@@ -100,6 +102,13 @@ class JvmtiVTMSTransitionDisabler {
 
   static void set_VTMS_transition_count(bool val)    { _VTMS_transition_count = val; }
 
+  static void inc_sync_protocol_enabled_count()      { Atomic::inc(&_sync_protocol_enabled_count); }
+  static void dec_sync_protocol_enabled_count()      { Atomic::dec(&_sync_protocol_enabled_count); }
+  static int  sync_protocol_enabled_count()          { return Atomic::load(&_sync_protocol_enabled_count); }
+  static bool sync_protocol_enabled_permanently()    { return Atomic::load(&_sync_protocol_enabled_permanently); }
+
+  static bool sync_protocol_enabled()                { return sync_protocol_enabled_permanently() || sync_protocol_enabled_count() > 0; }
+
   // parameter is_SR: suspender or resumer
   JvmtiVTMSTransitionDisabler(bool is_SR = false);
   JvmtiVTMSTransitionDisabler(jthread thread);