8297286: runtime/vthread tests crashing after JDK-8296324

Reviewed-by: lmesnik, pchilanomate, cjplummer

src/hotspot/share/classfile/javaClasses.cpp

@@ -1612,6 +1612,10 @@ void java_lang_Thread::set_is_in_VTMS_transition(oop java_thread, bool val) {
   java_thread->bool_field_put_volatile(_jvmti_is_in_VTMS_transition_offset, val);
 }
 
+int java_lang_Thread::is_in_VTMS_transition_offset() {
+  return _jvmti_is_in_VTMS_transition_offset;
+}
+
 void java_lang_Thread::clear_scopedValueBindings(oop java_thread) {
   assert(java_thread != nullptr, "need a java_lang_Thread pointer here");
   java_thread->obj_field_put(_scopedValueBindings_offset, nullptr);

src/hotspot/share/classfile/javaClasses.hpp

@@ -406,6 +406,7 @@ class java_lang_Thread : AllStatic {
   static void dec_VTMS_transition_disable_count(oop java_thread);
   static bool is_in_VTMS_transition(oop java_thread);
   static void set_is_in_VTMS_transition(oop java_thread, bool val);
+  static int  is_in_VTMS_transition_offset();
 
   // Clear all scoped value bindings on error
   static void clear_scopedValueBindings(oop java_thread);

src/hotspot/share/opto/library_call.cpp

@@ -2876,13 +2876,17 @@ bool LibraryCallKit::inline_native_notify_jvmti_funcs(address funcAddr, const ch
     make_runtime_call(RC_NO_LEAF, tf, funcAddr, funcName, TypePtr::BOTTOM, vt_oop, hide, cond);
     ideal.sync_kit(this);
   } ideal.else_(); {
-    // set hide value to the VTMS transition bit in current JavaThread
+    // set hide value to the VTMS transition bit in current JavaThread and VirtualThread object
+    Node* vt_oop = _gvn.transform(argument(0)); // this argument - VirtualThread oop
     Node* thread = ideal.thread();
-    Node* addr = basic_plus_adr(thread, in_bytes(JavaThread::is_in_VTMS_transition_offset()));
+    Node* jt_addr = basic_plus_adr(thread, in_bytes(JavaThread::is_in_VTMS_transition_offset()));
+    Node* vt_addr = basic_plus_adr(vt_oop, java_lang_Thread::is_in_VTMS_transition_offset());
     const TypePtr *addr_type = _gvn.type(addr)->isa_ptr();
 
     sync_kit(ideal);
-    access_store_at(nullptr, addr, addr_type, hide, _gvn.type(hide), T_BOOLEAN, IN_NATIVE | MO_UNORDERED);
+    access_store_at(nullptr, jt_addr, addr_type, hide, _gvn.type(hide), T_BOOLEAN, IN_NATIVE | MO_UNORDERED);
+    access_store_at(nullptr, vt_addr, addr_type, hide, _gvn.type(hide), T_BOOLEAN, IN_NATIVE | MO_UNORDERED);
+
     ideal.sync_kit(this);
   } ideal.end_if();
   final_sync(ideal);
@@ -2890,19 +2894,15 @@ bool LibraryCallKit::inline_native_notify_jvmti_funcs(address funcAddr, const ch
   return true;
 }
 
-// If notifications are enabled then just update the temporary VTMS transition bit.
+// Always update the temporary VTMS transition bit.
 bool LibraryCallKit::inline_native_notify_jvmti_hide() {
   if (!DoJVMTIVirtualThreadTransitions) {
     return true;
   }
   IdealKit ideal(this);
 
-  Node* ONE = ideal.ConI(1);
-  Node* addr = makecon(TypeRawPtr::make((address)&JvmtiVTMSTransitionDisabler::_VTMS_notify_jvmti_events));
-  Node* notify_jvmti_enabled = ideal.load(ideal.ctrl(), addr, TypeInt::BOOL, T_BOOLEAN, Compile::AliasIdxRaw);
-
-  ideal.if_then(notify_jvmti_enabled, BoolTest::eq, ONE); {
-    // set the VTMS temporary transition bit in current JavaThread
+  {
+    // unconditionally update the temporary VTMS transition bit in current JavaThread
     Node* thread = ideal.thread();
     Node* hide = _gvn.transform(argument(1)); // hide argument for temporary VTMS transition notification
     Node* addr = basic_plus_adr(thread, in_bytes(JavaThread::is_in_tmp_VTMS_transition_offset()));
@@ -2911,7 +2911,7 @@ bool LibraryCallKit::inline_native_notify_jvmti_hide() {
     sync_kit(ideal);
     access_store_at(nullptr, addr, addr_type, hide, _gvn.type(hide), T_BOOLEAN, IN_NATIVE | MO_UNORDERED);
     ideal.sync_kit(this);
-  } ideal.end_if();
+  }
   final_sync(ideal);
 
   return true;

src/hotspot/share/prims/jvm.cpp

@@ -3921,6 +3921,8 @@ JVM_ENTRY(void, JVM_VirtualThreadMount(JNIEnv* env, jobject vthread, jboolean hi
   }
   if (!JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) {
     thread->set_is_in_VTMS_transition(hide);
+    oop vt = JNIHandles::resolve_external_guard(vthread);
+    java_lang_Thread::set_is_in_VTMS_transition(vt, hide);
     return;
   }
   if (hide) {
@@ -3943,6 +3945,8 @@ JVM_ENTRY(void, JVM_VirtualThreadUnmount(JNIEnv* env, jobject vthread, jboolean
   }
   if (!JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) {
     thread->set_is_in_VTMS_transition(hide);
+    oop vt = JNIHandles::resolve_external_guard(vthread);
+    java_lang_Thread::set_is_in_VTMS_transition(vt, hide);
     return;
   }
   if (hide) {
@@ -3955,16 +3959,13 @@ JVM_ENTRY(void, JVM_VirtualThreadUnmount(JNIEnv* env, jobject vthread, jboolean
 #endif
 JVM_END
 
-// If notifications are enabled then just update the temporary VTMS transition bit.
+// Always update the temporary VTMS transition bit.
 JVM_ENTRY(void, JVM_VirtualThreadHideFrames(JNIEnv* env, jobject vthread, jboolean hide))
 #if INCLUDE_JVMTI
   if (!DoJVMTIVirtualThreadTransitions) {
     assert(!JvmtiExport::can_support_virtual_threads(), "sanity check");
     return;
   }
-  if (!JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) {
-    return;
-  }
   assert(!thread->is_in_VTMS_transition(), "sanity check");
   assert(thread->is_in_tmp_VTMS_transition() != (bool)hide, "sanity check");
   thread->toggle_is_in_tmp_VTMS_transition();

src/hotspot/share/prims/jvmtiEnvBase.cpp

@@ -57,7 +57,7 @@
 #include "runtime/signature.hpp"
 #include "runtime/stackWatermarkSet.inline.hpp"
 #include "runtime/threads.hpp"
-#include "runtime/threadSMR.hpp"
+#include "runtime/threadSMR.inline.hpp"
 #include "runtime/vframe.inline.hpp"
 #include "runtime/vframe_hp.hpp"
 #include "runtime/vmThread.hpp"
@@ -1529,6 +1529,104 @@ JvmtiEnvBase::is_in_thread_list(jint count, const jthread* list, oop jt_oop) {
   return false;
 }
 
+class VM_SetNotifyJvmtiEventsMode : public VM_Operation {
+private:
+  static bool _whitebox_used;
+  bool _enable;
+
+  // This function is needed only for testing purposes to support multiple
+  // enable&disable notifyJvmti events. Otherwise, there can be only one call
+  // to enable_virtual_threads_notify_jvmti() for late binding agents. There
+  // have to be no JvmtiThreadState's and need to correct them in such a case.
+  static void correct_jvmti_thread_state(JavaThread* jt) {
+    oop  ct_oop = jt->threadObj();
+    oop  vt_oop = jt->vthread();
+    JvmtiThreadState* jt_state = jt->jvmti_thread_state();
+    JvmtiThreadState* ct_state = java_lang_Thread::jvmti_thread_state(jt->threadObj());
+    JvmtiThreadState* vt_state = vt_oop != nullptr ? java_lang_Thread::jvmti_thread_state(vt_oop) : nullptr;
+    bool virt = vt_oop != nullptr && java_lang_VirtualThread::is_instance(vt_oop);
+
+    // Correct jt->jvmti_thread_state() and jt->jvmti_vthread().
+    // It was not maintained while notifyJvmti was disabled but there can be
+    // a leftover from previous cycle when notification were enabled.
+    if (virt) {
+      jt->set_jvmti_thread_state(nullptr);  // reset jt->jvmti_thread_state()
+      jt->set_jvmti_vthread(vt_oop);        // restore jt->jvmti_vthread()
+    } else {
+      jt->set_jvmti_thread_state(ct_state); // restore jt->jvmti_thread_state()
+      jt->set_jvmti_vthread(ct_oop);        // restore jt->jvmti_vthread()
+    }
+  }
+
+  // This function is called only if _enable == true.
+  // Iterates over all JavaThread's, counts VTMS transitions and restores
+  // jt->jvmti_thread_state() and jt->jvmti_vthread() for VTMS transition protocol.
+  int count_transitions_and_correct_jvmti_thread_states() {
+    int count = 0;
+
+    for (JavaThread* jt : ThreadsListHandle()) {
+      if (jt->is_in_VTMS_transition()) {
+        count++;
+        continue; // no need in JvmtiThreadState correction below if in transition
+      }
+      if (_whitebox_used) {
+        correct_jvmti_thread_state(jt); // needed in testing environment only
+      }
+    }
+    return count;
+  }
+
+public:
+  VMOp_Type type() const { return VMOp_SetNotifyJvmtiEventsMode; }
+  bool allow_nested_vm_operations() const { return false; }
+  VM_SetNotifyJvmtiEventsMode(bool enable) : _enable(enable) {
+    if (!enable) {
+      _whitebox_used = true; // disabling is available via WhiteBox only
+    }
+  }
+
+  void doit() {
+    int count = _enable ? count_transitions_and_correct_jvmti_thread_states() : 0;
+
+    JvmtiVTMSTransitionDisabler::set_VTMS_transition_count(count);
+    JvmtiVTMSTransitionDisabler::set_VTMS_notify_jvmti_events(_enable);
+  }
+};
+
+bool VM_SetNotifyJvmtiEventsMode::_whitebox_used = false;
+
+// This function is to support agents loaded into running VM.
+// Must be called in thread-in-native mode.
+bool
+JvmtiEnvBase::enable_virtual_threads_notify_jvmti() {
+  if (!Continuations::enabled()) {
+    return false;
+  }
+  if (JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) {
+    return false; // already enabled
+  }
+  VM_SetNotifyJvmtiEventsMode op(true);
+  VMThread::execute(&op);
+  return true;
+}
+
+// This function is used in WhiteBox, only needed to test the function above.
+// It is unsafe to use this function when virtual threads are executed.
+// Must be called in thread-in-native mode.
+bool
+JvmtiEnvBase::disable_virtual_threads_notify_jvmti() {
+  if (!Continuations::enabled()) {
+    return false;
+  }
+  if (!JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) {
+    return false; // already disabled
+  }
+  JvmtiVTMSTransitionDisabler disabler(true); // ensure there are no other disablers
+  VM_SetNotifyJvmtiEventsMode op(false);
+  VMThread::execute(&op);
+  return true;
+}
+
 // java_thread - protected by ThreadsListHandle
 jvmtiError
 JvmtiEnvBase::suspend_thread(oop thread_oop, JavaThread* java_thread, bool single_suspend,

src/hotspot/share/prims/jvmtiEnvBase.hpp

@@ -83,6 +83,13 @@ class JvmtiEnvBase : public CHeapObj<mtInternal> {
   static void leaving_dying_thread_env_iteration()  { --_dying_thread_env_iteration_count; }
   static bool is_inside_dying_thread_env_iteration(){ return _dying_thread_env_iteration_count > 0; }
 
+  // This function is to support agents loaded into running VM.
+  static bool enable_virtual_threads_notify_jvmti();
+
+  // This function is used in WhiteBox, only needed to test the function above.
+  // It is unsafe to use this function when virtual threads are executed.
+  static bool disable_virtual_threads_notify_jvmti();
+
   static jvmtiError suspend_thread(oop thread_oop, JavaThread* java_thread, bool single_suspend,
                                    int* need_safepoint_p);
   static jvmtiError resume_thread(oop thread_oop, JavaThread* java_thread, bool single_resume);

src/hotspot/share/prims/jvmtiExport.cpp

@@ -379,7 +379,14 @@ JvmtiExport::get_jvmti_interface(JavaVM *jvm, void **penv, jint version) {
   }
   if (Continuations::enabled()) {
     // Virtual threads support. There is a performance impact when VTMS transitions are enabled.
-    JvmtiVTMSTransitionDisabler::set_VTMS_notify_jvmti_events(true);
+    if (JvmtiEnv::get_phase() == JVMTI_PHASE_LIVE) {
+      if (!JvmtiVTMSTransitionDisabler::VTMS_notify_jvmti_events()) {
+        ThreadInVMfromNative __tiv(JavaThread::current());
+        JvmtiEnvBase::enable_virtual_threads_notify_jvmti();
+      }
+    } else {
+      JvmtiVTMSTransitionDisabler::set_VTMS_notify_jvmti_events(true);
+    }
   }
 
   if (JvmtiEnv::get_phase() == JVMTI_PHASE_LIVE) {

src/hotspot/share/prims/jvmtiThreadState.hpp

@@ -98,6 +98,8 @@ class JvmtiVTMSTransitionDisabler {
   static bool VTMS_notify_jvmti_events()             { return _VTMS_notify_jvmti_events; }
   static void set_VTMS_notify_jvmti_events(bool val) { _VTMS_notify_jvmti_events = val; }
 
+  static void set_VTMS_transition_count(bool val)    { _VTMS_transition_count = val; }
+
   // parameter is_SR: suspender or resumer
   JvmtiVTMSTransitionDisabler(bool is_SR = false);
   JvmtiVTMSTransitionDisabler(jthread thread);

src/hotspot/share/prims/whitebox.cpp

@@ -65,6 +65,7 @@
 #include "oops/objArrayOop.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "oops/typeArrayOop.inline.hpp"
+#include "prims/jvmtiEnvBase.hpp"
 #include "prims/resolvedMethodTable.hpp"
 #include "prims/wbtestmethods/parserTests.hpp"
 #include "prims/whitebox.inline.hpp"
@@ -2537,6 +2538,22 @@ WB_ENTRY(void, WB_UnlockCritical(JNIEnv* env, jobject wb))
   GCLocker::unlock_critical(thread);
 WB_END
 
+WB_ENTRY(jboolean, WB_SetVirtualThreadsNotifyJvmtiMode(JNIEnv* env, jobject wb, jboolean enable))
+  if (!Continuations::enabled()) {
+    tty->print_cr("WB error: must be Continuations::enabled()!");
+    return JNI_FALSE;
+  }
+  jboolean result = false;
+#if INCLUDE_JVMTI
+  if (enable) {
+    result = JvmtiEnvBase::enable_virtual_threads_notify_jvmti();
+  } else {
+    result = JvmtiEnvBase::disable_virtual_threads_notify_jvmti();
+  }
+#endif
+  return result;
+WB_END
+
 #define CC (char*)
 
 static JNINativeMethod methods[] = {
@@ -2816,6 +2833,7 @@ static JNINativeMethod methods[] = {
 
   {CC"lockCritical",    CC"()V",                      (void*)&WB_LockCritical},
   {CC"unlockCritical",  CC"()V",                      (void*)&WB_UnlockCritical},
+  {CC"setVirtualThreadsNotifyJvmtiMode", CC"(Z)Z",    (void*)&WB_SetVirtualThreadsNotifyJvmtiMode},
 };
 
 

src/hotspot/share/runtime/vmOperation.hpp

@@ -78,6 +78,7 @@
   template(VirtualThreadGetOrSetLocal)            \
   template(VirtualThreadGetCurrentLocation)       \
   template(ChangeSingleStep)                      \
+  template(SetNotifyJvmtiEventsMode)              \
   template(HeapWalkOperation)                     \
   template(HeapIterateOperation)                  \
   template(ReportJavaOutOfMemory)                 \

test/hotspot/jtreg/ProblemList.txt

@@ -97,8 +97,6 @@ runtime/os/TestTracePageSizes.java#G1 8267460 linux-aarch64
 runtime/os/TestTracePageSizes.java#Parallel 8267460 linux-aarch64
 runtime/os/TestTracePageSizes.java#Serial 8267460 linux-aarch64
 runtime/ErrorHandling/CreateCoredumpOnCrash.java 8267433 macosx-x64
-runtime/vthread/RedefineClass.java 8297286 generic-all
-runtime/vthread/TestObjectAllocationSampleEvent.java 8297286 generic-all
 runtime/StackGuardPages/TestStackGuardPagesNative.java 8303612 linux-all
 runtime/Thread/TestAlwaysPreTouchStacks.java 8305416 generic-all
 

test/hotspot/jtreg/serviceability/jvmti/vthread/ToggleNotifyJvmtiTest/ToggleNotifyJvmtiTest.java

@@ -0,0 +1,198 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @summary Verifies JVMTI works for agents loaded into running VM
+ * @requires vm.jvmti
+ * @requires vm.continuations
+ * @enablePreview
+ * @library /test/lib /test/hotspot/jtreg
+ * @build jdk.test.whitebox.WhiteBox
+ *
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run main/othervm/native -XX:+WhiteBoxAPI -Xbootclasspath/a:. -agentlib:ToggleNotifyJvmtiTest ToggleNotifyJvmtiTest
+ * @run main/othervm/native -XX:+WhiteBoxAPI -Xbootclasspath/a:. -Djdk.attach.allowAttachSelf=true ToggleNotifyJvmtiTest attach
+ */
+
+import com.sun.tools.attach.VirtualMachine;
+import java.util.concurrent.ThreadFactory;
+import jdk.test.whitebox.WhiteBox;
+
+// The TestTask mimics some thread activity, but it is important
+// to have sleep() calls to provide yielding as some frequency of virtual
+// thread mount state transitions is needed for this test scenario.
+class TestTask implements Runnable {
+    private String name;
+    private volatile boolean threadReady = false;
+    private volatile boolean shouldFinish = false;
+
+    // make thread with specific name
+    public TestTask(String name) {
+        this.name = name;
+    }
+
+    // run thread continuously
+    public void run() {
+        // run in a loop
+        threadReady = true;
+        System.out.println("# Started: " + name);
+
+        int i = 0;
+        int n = 1000;
+        while (!shouldFinish) {
+            if (n <= 0) {
+                n = 1000;
+                ToggleNotifyJvmtiTest.sleep(1);
+            }
+            if (i > n) {
+                i = 0;
+                n = n - 1;
+            }
+            i = i + 1;
+        }
+    }
+
+    // ensure thread is ready
+    public void ensureReady() {
+        while (!threadReady) {
+            ToggleNotifyJvmtiTest.sleep(1);
+        }
+    }
+
+    public void letFinish() {
+        shouldFinish = true;
+    }
+}
+
+/*
+ * The testing scenario consists of a number of serialized test cycles.
+ * Each cycle has initially zero virtual threads and the following steps:
+ *  - disable notifyJvmti events mode
+ *  - start N virtual threads
+ *  - enable notifyJvmti events mode
+ *  - shut the virtual threads down
+ * The JVMTI agent is loaded at a start-up or at a dynamic attach.
+ * It collects events:
+ *  - VirtualThreadStart, VirtualThreadEnd, ThreadStart and ThreadEnd
+ */
+public class ToggleNotifyJvmtiTest {
+    private static final int VTHREADS_CNT = 20;
+    private static final String AGENT_LIB = "ToggleNotifyJvmtiTest";
+    private static final WhiteBox WB = WhiteBox.getWhiteBox();
+
+    private static native boolean IsAgentStarted();
+    private static native int VirtualThreadStartedCount();
+    private static native int VirtualThreadEndedCount();
+    private static native int ThreadStartedCount();
+    private static native int ThreadEndedCount();
+
+    static void log(String str) { System.out.println(str); }
+
+    static public void sleep(long millis) {
+        try {
+            Thread.sleep(millis);
+        } catch (InterruptedException e) {
+            throw new RuntimeException("Interruption in TestTask.sleep: \n\t" + e);
+        }
+    }
+
+    static TestTask[] tasks = new TestTask[VTHREADS_CNT];
+    static Thread vthreads[] = new Thread[VTHREADS_CNT];
+
+    static private void startVirtualThreads() {
+        log("\n# Java: Starting vthreads");
+        for (int i = 0; i < VTHREADS_CNT; i++) {
+            String name = "TestTask" + i;
+            TestTask task = new TestTask(name);
+            vthreads[i] = Thread.ofVirtual().name(name).start(task);
+            tasks[i] = task;
+        }
+    }
+
+    static private void finishVirtualThreads() {
+        try {
+            for (int i = 0; i < VTHREADS_CNT; i++) {
+                tasks[i].ensureReady();
+                tasks[i].letFinish();
+                vthreads[i].join();
+            }
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    static private void setVirtualThreadsNotifyJvmtiMode(int iter, boolean enable) {
+        boolean status = WB.setVirtualThreadsNotifyJvmtiMode(enable);
+        if (!status) {
+            throw new RuntimeException("Java: failed to set VirtualThreadsNotifyJvmtiMode: " + enable);
+        }
+        log("\n# main: SetNotifyJvmtiEvents: #" + iter + " enable: " + enable);
+    }
+
+    // Accumulative results after each finished test cycle.
+    static private void printResults() {
+        log("  VirtualThreadStart events: " + VirtualThreadStartedCount());
+        log("  VirtualThreadEnd events:   " + VirtualThreadEndedCount());
+        log("  ThreadStart events:        " + ThreadStartedCount());
+        log("  ThreadEnd events:          " + ThreadEndedCount());
+    }
+
+    static private void run_test_cycle(int iter) throws Exception {
+        log("\n# Java: Started test cycle #" + iter);
+
+        // Disable notifyJvmti events mode at test cycle start.
+        // It is unsafe to do so if any virtual threads are executed.
+        setVirtualThreadsNotifyJvmtiMode(iter, false);
+
+        startVirtualThreads();
+
+        // We want this somewhere in the middle of virtual threads execution.
+        setVirtualThreadsNotifyJvmtiMode(iter, true);
+
+        finishVirtualThreads();
+
+        log("\n# Java: Finished test cycle #" + iter);
+        printResults();
+    }
+
+    public static void main(String[] args) throws Exception {
+        log("# main: loading " + AGENT_LIB + " lib");
+
+        if (args.length > 0 && args[0].equals("attach")) { // agent loaded into running VM case
+            String arg = args.length == 2 ? args[1] : "";
+            VirtualMachine vm = VirtualMachine.attach(String.valueOf(ProcessHandle.current().pid()));
+            vm.loadAgentLibrary(AGENT_LIB, arg);
+        }
+        int waitCount = 0;
+        while (!IsAgentStarted()) {
+            log("# main: waiting for native agent to start: #" + waitCount++);
+            sleep(20);
+        }
+
+        // The testing scenario consists of a number of sequential testing cycles.
+        for (int iter = 0; iter < 10; iter++) {
+            run_test_cycle(iter);
+        }
+    }
+}

test/hotspot/jtreg/serviceability/jvmti/vthread/ToggleNotifyJvmtiTest/libToggleNotifyJvmtiTest.cpp

@@ -0,0 +1,207 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <cstdlib>
+#include <cstring>
+#include <jvmti.h>
+#include "jvmti_common.h"
+
+extern "C" {
+
+static jvmtiEnv *jvmti;
+static volatile int vthread_started_cnt = 0;
+static volatile int vthread_ended_cnt = 0;
+static volatile int thread_started_cnt = 0;
+static volatile int thread_ended_cnt = 0;
+
+static jrawMonitorID agent_lock = NULL;
+static volatile jboolean agent_started = JNI_FALSE;
+
+static void check_and_print_thread_names(jvmtiEnv *jvmti, JNIEnv* jni, jthread thread,
+                                         bool is_virtual, const char* msg) {
+  jthread cthread = NULL;
+  jthread vthread = NULL;
+
+  if (is_virtual) {
+    vthread = thread;
+    cthread = get_carrier_thread(jvmti, jni, vthread);
+    if (jni->IsVirtualThread(cthread)) {
+      fatal(jni, "Failed: expected to be carrier thread");
+    }
+  } else {
+    cthread = thread;
+  }
+  char* ctname = get_thread_name(jvmti, jni, cthread);
+  char* vtname = vthread == NULL ? NULL : get_thread_name(jvmti, jni, vthread);
+
+  LOG("Event: %s virtual: %d ct: %s vt: %s\n", msg, is_virtual, ctname, vtname);
+
+  deallocate(jvmti, jni, (void*)ctname);
+  deallocate(jvmti, jni, (void*)vtname);
+}
+
+void JNICALL VirtualThreadStart(jvmtiEnv *jvmti, JNIEnv* jni, jthread thread) {
+  if (!jni->IsVirtualThread(thread)) {
+    fatal(jni, "Failed: expected to be virtual thread");
+  }
+  RawMonitorLocker agent_locker(jvmti, jni, agent_lock);
+
+  vthread_started_cnt++;
+  check_and_print_thread_names(jvmti, jni, thread, /* is_virtual */ true, "VirtualThreadStart");
+}
+
+void JNICALL VirtualThreadEnd(jvmtiEnv *jvmti, JNIEnv* jni, jthread thread) {
+  if (!jni->IsVirtualThread(thread)) {
+    fatal(jni, "Failed: expected to be virtual thread");
+  }
+  RawMonitorLocker agent_locker(jvmti, jni, agent_lock);
+
+  vthread_ended_cnt++;
+  check_and_print_thread_names(jvmti, jni, thread, /* is_virtual */ true, "VirtualThreadEnd");
+}
+
+void JNICALL ThreadStart(jvmtiEnv *jvmti, JNIEnv* jni, jthread thread) {
+  if (jni->IsVirtualThread(thread)) {
+    fatal(jni, "Failed: expected to be platform thread");
+  }
+  RawMonitorLocker agent_locker(jvmti, jni, agent_lock);
+
+  thread_started_cnt++;
+  check_and_print_thread_names(jvmti, jni, thread, /*is_virtual*/ false, "ThreadStart");
+}
+
+void JNICALL ThreadEnd(jvmtiEnv *jvmti, JNIEnv* jni, jthread thread) {
+  if (jni->IsVirtualThread(thread)) {
+    fatal(jni, "Failed: expected to be platform thread");
+  }
+  RawMonitorLocker agent_locker(jvmti, jni, agent_lock);
+
+  thread_ended_cnt++;
+  check_and_print_thread_names(jvmti, jni, thread, /*is_virtual*/ false, "ThreadEnd");
+}
+
+JNIEXPORT jboolean JNICALL
+Java_ToggleNotifyJvmtiTest_IsAgentStarted(JNIEnv* jni, jclass clazz) {
+  RawMonitorLocker agent_locker(jvmti, jni, agent_lock);
+
+  return agent_started;
+}
+
+JNIEXPORT jint JNICALL
+Java_ToggleNotifyJvmtiTest_VirtualThreadStartedCount(JNIEnv* jni, jclass clazz) {
+  RawMonitorLocker agent_locker(jvmti, jni, agent_lock);
+
+  return vthread_started_cnt;
+}
+
+JNIEXPORT jint JNICALL
+Java_ToggleNotifyJvmtiTest_VirtualThreadEndedCount(JNIEnv* jni, jclass clazz) {
+  RawMonitorLocker agent_locker(jvmti, jni, agent_lock);
+
+  return vthread_ended_cnt;
+}
+
+JNIEXPORT jint JNICALL
+Java_ToggleNotifyJvmtiTest_ThreadStartedCount(JNIEnv* jni, jclass clazz) {
+  RawMonitorLocker agent_locker(jvmti, jni, agent_lock);
+
+  return thread_started_cnt;
+}
+
+JNIEXPORT jint JNICALL
+Java_ToggleNotifyJvmtiTest_ThreadEndedCount(JNIEnv* jni, jclass clazz) {
+  RawMonitorLocker agent_locker(jvmti, jni, agent_lock);
+
+  return thread_ended_cnt;
+}
+
+jint agent_init(JavaVM *jvm, char *options, void *reserved) {
+  jvmtiCapabilities caps;
+  jvmtiEventCallbacks callbacks;
+  jvmtiError err;
+
+  if (jvm->GetEnv((void **) (&jvmti), JVMTI_VERSION) != JNI_OK) {
+    return JNI_ERR;
+  }
+  memset(&caps, 0, sizeof(caps));
+  memset(&callbacks, 0, sizeof(callbacks));
+  callbacks.VirtualThreadStart = &VirtualThreadStart;
+  callbacks.VirtualThreadEnd = &VirtualThreadEnd;
+  callbacks.ThreadStart = &ThreadStart;
+  callbacks.ThreadEnd = &ThreadEnd;
+
+  {
+    caps.can_support_virtual_threads = 1;
+
+    err = jvmti->AddCapabilities(&caps);
+    if (err != JVMTI_ERROR_NONE) {
+      LOG("Agent init: error in JVMTI AddCapabilities: %s (%d)\n", TranslateError(err), err);
+      return JNI_ERR;
+    }
+    err = jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_VIRTUAL_THREAD_START, NULL);
+    if (err != JVMTI_ERROR_NONE) {
+      LOG("Agent init: error in JVMTI SetEventNotificationMode: %s (%d)\n", TranslateError(err), err);
+      return JNI_ERR;
+    }
+    err = jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_VIRTUAL_THREAD_END, NULL);
+    if (err != JVMTI_ERROR_NONE) {
+      LOG("Agent init: error in JVMTI SetEventNotificationMode: %s (%d)\n", TranslateError(err), err);
+      return JNI_ERR;
+    }
+    err = jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_THREAD_START, NULL);
+    if (err != JVMTI_ERROR_NONE) {
+      LOG("Agent init: error in JVMTI SetEventNotificationMode: %s (%d)\n", TranslateError(err), err);
+      return JNI_ERR;
+    }
+    err = jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_THREAD_END, NULL);
+    if (err != JVMTI_ERROR_NONE) {
+      LOG("Agent init: error in JVMTI SetEventNotificationMode: %s (%d)\n", TranslateError(err), err);
+      return JNI_ERR;
+    }
+  }
+  LOG("Agent init: can_support_virtual_threads capability: %d\n",  caps.can_support_virtual_threads);
+
+  err = jvmti->SetEventCallbacks(&callbacks, (jint)sizeof(callbacks));
+  if (err != JVMTI_ERROR_NONE) {
+    LOG("Agent init: error in JVMTI AddCapabilities: %s (%d)\n", TranslateError(err), err);
+    return JNI_ERR;
+  }
+  agent_lock = create_raw_monitor(jvmti, "agent_lock");
+  agent_started = JNI_TRUE;
+
+  return JNI_OK;
+}
+
+JNIEXPORT jint JNICALL
+Agent_OnLoad(JavaVM *jvm, char *options, void *reserved) {
+  LOG("Agent_OnLoad started\n");
+  return agent_init(jvm, options, reserved);
+}
+
+JNIEXPORT jint JNICALL
+Agent_OnAttach(JavaVM *jvm, char *options, void *reserved) {
+  LOG("Agent_OnAttach started\n");
+  return agent_init(jvm, options, reserved);
+}
+
+} // extern "C"

test/hotspot/jtreg/serviceability/jvmti/vthread/VirtualThreadStartTest/VirtualThreadStartTest.java

@@ -69,7 +69,8 @@ public class VirtualThreadStartTest {
         int startedThreads = getAndResetStartedThreads();
         System.out.println("ThreadStart event count: " + startedThreads + ", expected: " + THREAD_CNT);
         if (startedThreads != THREAD_CNT) {
-            throw new RuntimeException("Failed: wrong ThreadStart event count");
+            throw new RuntimeException("Failed: wrong ThreadStart count: " +
+                                       startedThreads + " expected: " + THREAD_CNT);
         }
     }
 }

test/lib/jdk/test/whitebox/WhiteBox.java

@@ -750,4 +750,6 @@ public class WhiteBox {
   public native void lockCritical();
 
   public native void unlockCritical();
+
+  public native boolean setVirtualThreadsNotifyJvmtiMode(boolean enabled);
 }