8297431: [JVMCI] HotSpotJVMCIRuntime.encodeThrowable should not throw an exception

Reviewed-by: never

src/hotspot/share/jvmci/jvmciEnv.cpp

@@ -316,12 +316,28 @@ class ExceptionTranslation: public StackObj {
     int buffer_size = 2048;
     while (true) {
       ResourceMark rm;
-      jlong buffer = (jlong) NEW_RESOURCE_ARRAY_IN_THREAD(THREAD, jbyte, buffer_size);
+      jlong buffer = (jlong) NEW_RESOURCE_ARRAY_IN_THREAD_RETURN_NULL(THREAD, jbyte, buffer_size);
-      int res = encode(THREAD, runtimeKlass, buffer, buffer_size);
+      if (buffer == 0L) {
-      if ((_from_env != nullptr && _from_env->has_pending_exception()) || HAS_PENDING_EXCEPTION) {
+        decode(THREAD, runtimeKlass, 0L);
-        JVMCIRuntime::fatal_exception(_from_env, "HotSpotJVMCIRuntime.encodeThrowable should not throw an exception");
+        return;
       }
-      if (res < 0) {
+      int res = encode(THREAD, runtimeKlass, buffer, buffer_size);
+      if (_from_env != nullptr && !_from_env->is_hotspot() && _from_env->has_pending_exception()) {
+        // Cannot get name of exception thrown by `encode` as that involves
+        // calling into libjvmci which in turn can raise another exception.
+        _from_env->clear_pending_exception();
+        decode(THREAD, runtimeKlass, -2L);
+        return;
+      } else if (HAS_PENDING_EXCEPTION) {
+        Symbol *ex_name = PENDING_EXCEPTION->klass()->name();
+        CLEAR_PENDING_EXCEPTION;
+        if (ex_name == vmSymbols::java_lang_OutOfMemoryError()) {
+          decode(THREAD, runtimeKlass, -1L);
+        } else {
+          decode(THREAD, runtimeKlass, -2L);
+        }
+        return;
+      } else if (res < 0) {
         int required_buffer_size = -res;
         if (required_buffer_size > buffer_size) {
           buffer_size = required_buffer_size;
@@ -329,7 +345,7 @@ class ExceptionTranslation: public StackObj {
       } else {
         decode(THREAD, runtimeKlass, buffer);
         if (!_to_env->has_pending_exception()) {
-          JVMCIRuntime::fatal_exception(_to_env, "HotSpotJVMCIRuntime.decodeAndThrowThrowable should throw an exception");
+          _to_env->throw_InternalError("HotSpotJVMCIRuntime.decodeAndThrowThrowable should have thrown an exception");
         }
         return;
       }

src/jdk.internal.vm.ci/share/classes/jdk.vm.ci.hotspot/src/jdk/vm/ci/hotspot/HotSpotJVMCIRuntime.java

@@ -209,15 +209,33 @@ public final class HotSpotJVMCIRuntime implements JVMCIRuntime {
     /**
      * Decodes the exception encoded in {@code buffer} and throws it.
      *
-     * @param buffer a native byte buffer containing an exception encoded by
+     * @param errorOrBuffer an error code or a native byte buffer containing an exception encoded by
-     *            {@link #encodeThrowable}
+     *            {@link #encodeThrowable}. Error code values and their meanings are:
+     *
+     *            <pre>
+     *             0: native memory for the buffer could not be allocated
+     *            -1: an OutOfMemoryError was thrown while encoding the exception
+     *            -2: some other throwable was thrown while encoding the exception
+     *            </pre>
      */
     @VMEntryPoint
-    static void decodeAndThrowThrowable(long buffer) throws Throwable {
+    static void decodeAndThrowThrowable(long errorOrBuffer) throws Throwable {
+        if (errorOrBuffer >= -2L && errorOrBuffer <= 0) {
+            String context = String.format("while encoding an exception to translate it from %s to %s",
+                            IS_IN_NATIVE_IMAGE ? "HotSpot" : "libjvmci",
+                            IS_IN_NATIVE_IMAGE ? "libjvmci" : "HotSpot");
+            if (errorOrBuffer == 0) {
+                throw new InternalError("native buffer could not be allocated " + context);
+            }
+            if (errorOrBuffer == -1L) {
+                throw new OutOfMemoryError("OutOfMemoryError occurred " + context);
+            }
+            throw new InternalError("unexpected problem occurred " + context);
+        }
         Unsafe unsafe = UnsafeAccess.UNSAFE;
-        int encodingLength = unsafe.getInt(buffer);
+        int encodingLength = unsafe.getInt(errorOrBuffer);
         byte[] encoding = new byte[encodingLength];
-        unsafe.copyMemory(null, buffer + 4, encoding, Unsafe.ARRAY_BYTE_BASE_OFFSET, encodingLength);
+        unsafe.copyMemory(null, errorOrBuffer + 4, encoding, Unsafe.ARRAY_BYTE_BASE_OFFSET, encodingLength);
         throw TranslatedException.decodeThrowable(encoding);
     }
 

src/jdk.internal.vm.ci/share/classes/jdk.vm.ci.hotspot/src/jdk/vm/ci/hotspot/TranslatedException.java

@@ -149,7 +149,6 @@ final class TranslatedException extends Exception {
      * Encodes {@code throwable} including its stack and causes as a {@linkplain GZIPOutputStream
      * compressed} byte array that can be decoded by {@link #decodeThrowable}.
      */
-    @VMEntryPoint
     static byte[] encodeThrowable(Throwable throwable) throws Throwable {
         try {
             return encodeThrowable(throwable, true);
@@ -223,7 +222,6 @@ final class TranslatedException extends Exception {
      * @param encodedThrowable an encoded exception in the format specified by
      *            {@link #encodeThrowable}
      */
-    @VMEntryPoint
     static Throwable decodeThrowable(byte[] encodedThrowable) {
         try (DataInputStream dis = new DataInputStream(new GZIPInputStream(new ByteArrayInputStream(encodedThrowable)))) {
             Throwable cause = null;