8177901: JDWP exit error JVMTI_ERROR_WRONG_PHASE(112): on checking for an interface

Add synchronization between CommandLoop and cbVMDeath callback

Reviewed-by: dholmes, dcubed

src/jdk.jdwp.agent/share/native/libjdwp/debugInit.c

@@ -1301,6 +1301,9 @@ debugInit_exit(jvmtiError error, const char *msg)
 {
     enum exit_codes { EXIT_NO_ERRORS = 0, EXIT_JVMTI_ERROR = 1, EXIT_TRANSPORT_ERROR = 2 };
 
+    // Release commandLoop vmDeathLock if necessary
+    commandLoop_exitVmDeathLockOnError();
+
     // Prepare to exit. Log error and finish logging
     LOG_MISC(("Exiting with error %s(%d): %s", jvmtiErrorText(error), error,
                                                ((msg == NULL) ? "" : msg)));

src/jdk.jdwp.agent/share/native/libjdwp/eventHandler.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2007, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1287,11 +1287,11 @@ cbVMDeath(jvmtiEnv *jvmti_env, JNIEnv *env)
     } debugMonitorExit(callbackBlock);
 
     /*
-     * The VM will die soon after the completion of this callback - we
-     * may need to do a final synchronization with the command loop to
-     * avoid the VM terminating with replying to the final (resume)
-     * command.
+     * The VM will die soon after the completion of this callback -
+     * we synchronize with both the command loop and the debug loop
+     * for a more orderly shutdown.
      */
+    commandLoop_sync();
     debugLoop_sync();
 
     LOG_MISC(("END cbVMDeath"));

src/jdk.jdwp.agent/share/native/libjdwp/eventHelper.c

@@ -29,6 +29,9 @@
 #include "threadControl.h"
 #include "invoker.h"
 
+
+#define COMMAND_LOOP_THREAD_NAME "JDWP Event Helper Thread"
+
 /*
  * Event helper thread command commandKinds
  */
@@ -121,6 +124,9 @@ static CommandQueue commandQueue;
 static jrawMonitorID commandQueueLock;
 static jrawMonitorID commandCompleteLock;
 static jrawMonitorID blockCommandLoopLock;
+static jrawMonitorID vmDeathLock;
+static volatile jboolean commandLoopEnteredVmDeathLock = JNI_FALSE;
+
 static jint maxQueueSize = 50 * 1024; /* TO DO: Make this configurable */
 static jboolean holdEvents;
 static jint currentQueueSize = 0;
@@ -700,9 +706,15 @@ commandLoop(jvmtiEnv* jvmti_env, JNIEnv* jni_env, void* arg)
              * handleCommand() to prevent any races.
              */
             jboolean doBlock = needBlockCommandLoop(command);
-            log_debugee_location("commandLoop(): command being handled", NULL, NULL, 0);
-            handleCommand(jni_env, command);
+            debugMonitorEnter(vmDeathLock);
+            commandLoopEnteredVmDeathLock = JNI_TRUE;
+            if (!gdata->vmDead) {
+                log_debugee_location("commandLoop(): command being handled", NULL, NULL, 0);
+                handleCommand(jni_env, command);
+            }
             completeCommand(command);
+            debugMonitorExit(vmDeathLock);
+            commandLoopEnteredVmDeathLock = JNI_FALSE;
             /* if we just finished a suspend-all cmd, then we block here */
             if (doBlock) {
                 doBlockCommandLoop();
@@ -725,10 +737,11 @@ eventHelper_initialize(jbyte sessionID)
     commandQueueLock = debugMonitorCreate("JDWP Event Helper Queue Monitor");
     commandCompleteLock = debugMonitorCreate("JDWP Event Helper Completion Monitor");
     blockCommandLoopLock = debugMonitorCreate("JDWP Event Block CommandLoop Monitor");
+    vmDeathLock = debugMonitorCreate("JDWP VM_DEATH CommandLoop Monitor");
 
     /* Start the event handler thread */
     func = &commandLoop;
-    (void)spawnNewThread(func, NULL, "JDWP Event Helper Thread");
+    (void)spawnNewThread(func, NULL, COMMAND_LOOP_THREAD_NAME);
 }
 
 void
@@ -759,6 +772,42 @@ eventHelper_unlock(void)
     debugMonitorExit(commandQueueLock);
 }
 
+void commandLoop_exitVmDeathLockOnError()
+{
+    const char* MSG_BASE = "exitVmDeathLockOnError: error in JVMTI %s: %d\n";
+    jthread cur_thread = NULL;
+    jvmtiThreadInfo thread_info;
+    jvmtiError err = JVMTI_ERROR_NONE;
+
+    err = JVMTI_FUNC_PTR(gdata->jvmti, GetCurrentThread)
+              (gdata->jvmti, &cur_thread);
+    if (err != JVMTI_ERROR_NONE) {
+        LOG_ERROR((MSG_BASE, "GetCurrentThread", err));
+        return;
+    }
+
+    err = JVMTI_FUNC_PTR(gdata->jvmti, GetThreadInfo)
+              (gdata->jvmti, cur_thread, &thread_info);
+    if (err != JVMTI_ERROR_NONE) {
+        LOG_ERROR((MSG_BASE, "GetThreadInfo", err));
+        return;
+    }
+    if (strcmp(thread_info.name, COMMAND_LOOP_THREAD_NAME) != 0) {
+        return;
+    }
+    if (commandLoopEnteredVmDeathLock == JNI_TRUE) {
+        debugMonitorExit(vmDeathLock);
+        commandLoopEnteredVmDeathLock = JNI_FALSE;
+    }
+}
+
+void
+commandLoop_sync(void)
+{
+    debugMonitorEnter(vmDeathLock);
+    debugMonitorExit(vmDeathLock);
+}
+
 /* Change all references to global in the EventInfo struct */
 static void
 saveEventInfoRefs(JNIEnv *env, EventInfo *evinfo)

src/jdk.jdwp.agent/share/native/libjdwp/eventHelper.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2004, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -54,6 +54,9 @@ void eventHelper_releaseEvents(void);
 void eventHelper_lock(void);
 void eventHelper_unlock(void);
 
+void commandLoop_sync(void); /* commandLoop sync with cbVMDeath */
+void commandLoop_exitVmDeathLockOnError(void);
+
 /*
  * Private interface for coordinating between eventHelper.c: commandLoop()
  * and ThreadReferenceImpl.c: resume() and VirtualMachineImpl.c: resume().