8303577: [JVMCI] OOME causes crash while translating exceptions

Reviewed-by: kvn, never

src/hotspot/share/classfile/vmSymbols.hpp

@@ -752,6 +752,7 @@
   template(encodeThrowable_name,                       "encodeThrowable")                                         \
   template(encodeThrowable_signature,                  "(Ljava/lang/Throwable;JI)I")                              \
   template(decodeAndThrowThrowable_name,               "decodeAndThrowThrowable")                                 \
+  template(decodeAndThrowThrowable_signature,          "(JZ)V")                                                   \
   template(classRedefinedCount_name,                   "classRedefinedCount")                                     \
   template(classLoader_name,                           "classLoader")                                             \
   template(componentType_name,                         "componentType")                                           \

src/hotspot/share/jvmci/jvmciEnv.cpp

@@ -392,7 +392,7 @@ class HotSpotToSharedLibraryExceptionTranslation : public ExceptionTranslation {
     JNIAccessMark jni(_to_env, THREAD);
     jni()->CallStaticVoidMethod(JNIJVMCI::VMSupport::clazz(),
                                 JNIJVMCI::VMSupport::decodeAndThrowThrowable_method(),
-                                buffer);
+                                buffer, false);
   }
  public:
   HotSpotToSharedLibraryExceptionTranslation(JVMCIEnv* hotspot_env, JVMCIEnv* jni_env, const Handle& throwable) :
@@ -414,11 +414,12 @@ class SharedLibraryToHotSpotExceptionTranslation : public ExceptionTranslation {
   void decode(JavaThread* THREAD, Klass* vmSupport, jlong buffer) {
     JavaCallArguments jargs;
     jargs.push_long(buffer);
+    jargs.push_int(true);
     JavaValue result(T_VOID);
     JavaCalls::call_static(&result,
                             vmSupport,
                             vmSymbols::decodeAndThrowThrowable_name(),
-                            vmSymbols::long_void_signature(), &jargs, THREAD);
+                            vmSymbols::decodeAndThrowThrowable_signature(), &jargs, THREAD);
   }
  public:
   SharedLibraryToHotSpotExceptionTranslation(JVMCIEnv* hotspot_env, JVMCIEnv* jni_env, jthrowable throwable) :

src/hotspot/share/jvmci/jvmciJavaClasses.hpp

@@ -216,7 +216,7 @@
   end_class                                                                                                   \
   start_class(VMSupport, jdk_internal_vm_VMSupport)                                                           \
     jvmci_method(CallStaticIntMethod, GetStaticMethodID, call_static, int, VMSupport, encodeThrowable, encodeThrowable_signature, (JVMCIObject throwable, jlong buffer, int buffer_size)) \
-    jvmci_method(CallStaticVoidMethod, GetStaticMethodID, call_static, void, VMSupport, decodeAndThrowThrowable, long_void_signature, (jlong buffer)) \
+    jvmci_method(CallStaticVoidMethod, GetStaticMethodID, call_static, void, VMSupport, decodeAndThrowThrowable, decodeAndThrowThrowable_signature, (jlong buffer)) \
   end_class                                                                                                   \
   start_class(ArrayIndexOutOfBoundsException, java_lang_ArrayIndexOutOfBoundsException)                       \
     jvmci_constructor(ArrayIndexOutOfBoundsException, "(Ljava/lang/String;)V")                                \

src/java.base/share/classes/jdk/internal/vm/VMSupport.java

@@ -113,15 +113,35 @@ public class VMSupport {
     public static native String getVMTemporaryDirectory();
 
     /**
-     * Decodes the exception encoded in {@code buffer} and throws it.
+     * Decodes the exception encoded in {@code errorOrBuffer} and throws it.
      *
-     * @param buffer a native byte buffer containing an exception encoded by
+     * @param errorOrBuffer an error code or a native byte errorOrBuffer containing an exception encoded by
+     *            {@link #encodeThrowable}. Error code values and their meanings are:
+     *
+     *            <pre>
+     *             0: native memory for the errorOrBuffer could not be allocated
+     *            -1: an OutOfMemoryError was thrown while encoding the exception
+     *            -2: some other throwable was thrown while encoding the exception
+     *            </pre>
+     * @param errorOrBuffer a native byte errorOrBuffer containing an exception encoded by
      *            {@link #encodeThrowable}
+     * @param inJVMHeap [@code true} if executing in the JVM heap, {@code false} otherwise
      */
-    public static void decodeAndThrowThrowable(long buffer) throws Throwable {
-        int encodingLength = U.getInt(buffer);
+    public static void decodeAndThrowThrowable(long errorOrBuffer, boolean inJVMHeap) throws Throwable {
+        if (errorOrBuffer >= -2L && errorOrBuffer <= 0) {
+            String context = String.format("while encoding an exception to translate it %s the JVM heap",
+                    inJVMHeap ? "to" : "from");
+            if (errorOrBuffer == 0) {
+                throw new InternalError("native errorOrBuffer could not be allocated " + context);
+            }
+            if (errorOrBuffer == -1L) {
+                throw new OutOfMemoryError("OutOfMemoryError occurred " + context);
+            }
+            throw new InternalError("unexpected problem occurred " + context);
+        }
+        int encodingLength = U.getInt(errorOrBuffer);
         byte[] encoding = new byte[encodingLength];
-        U.copyMemory(null, buffer + 4, encoding, Unsafe.ARRAY_BYTE_BASE_OFFSET, encodingLength);
+        U.copyMemory(null, errorOrBuffer + 4, encoding, Unsafe.ARRAY_BYTE_BASE_OFFSET, encodingLength);
         throw TranslatedException.decodeThrowable(encoding);
     }
 

test/jdk/jdk/internal/vm/TestTranslatedException.java

@@ -80,7 +80,7 @@ public class TestTranslatedException {
                     bufferSize = -res;
                 } else {
                     try {
-                        VMSupport.decodeAndThrowThrowable(buffer);
+                        VMSupport.decodeAndThrowThrowable(buffer, true);
                         throw new AssertionError("expected decodeAndThrowThrowable to throw an exception");
                     } catch (Throwable decoded) {
                         assertThrowableEquals(throwable, decoded);