8306992: [JVMCI] mitigate more against JVMCI related OOME causing VM to exit

Reviewed-by: never

src/hotspot/share/jvmci/jvmci.cpp

@@ -233,12 +233,12 @@ void JVMCI::vtrace(int level, const char* format, va_list ap) {
       ResourceMark rm(thread);
       JavaThreadState state = JavaThread::cast(thread)->thread_state();
       if (state == _thread_in_vm || state == _thread_in_Java || state == _thread_new) {
-        tty->print("JVMCITrace-%d[%s]:%*c", level, thread->name(), level, ' ');
+        tty->print("JVMCITrace-%d[" PTR_FORMAT " \"%s\"]:%*c", level, p2i(thread), thread->name(), level, ' ');
       } else {
         // According to check_access_thread_state, it's unsafe to
         // resolve the j.l.Thread object unless the thread is in
         // one of the states above.
-        tty->print("JVMCITrace-%d[%s@" PTR_FORMAT "]:%*c", level, thread->type_name(), p2i(thread), level, ' ');
+        tty->print("JVMCITrace-%d[" PTR_FORMAT " <%s>]:%*c", level, p2i(thread), thread->type_name(), level, ' ');
       }
     } else {
       tty->print("JVMCITrace-%d[?]:%*c", level, level, ' ');

src/hotspot/share/jvmci/jvmciCompiler.cpp

@@ -39,6 +39,9 @@ JVMCICompiler::JVMCICompiler() : AbstractCompiler(compiler_jvmci) {
   _bootstrapping = false;
   _bootstrap_compilation_request_handled = false;
   _methods_compiled = 0;
+  _ok_upcalls = 0;
+  _err_upcalls = 0;
+  _disabled = false;
   _global_compilation_ticks = 0;
   assert(_instance == nullptr, "only one instance allowed");
   _instance = this;
@@ -118,6 +121,9 @@ void JVMCICompiler::bootstrap(TRAPS) {
 }
 
 bool JVMCICompiler::force_comp_at_level_simple(const methodHandle& method) {
+  if (_disabled) {
+    return true;
+  }
   if (_bootstrapping) {
     // When bootstrapping, the JVMCI compiler can compile its own methods.
     return false;
@@ -211,6 +217,39 @@ void JVMCICompiler::inc_methods_compiled() {
   Atomic::inc(&_global_compilation_ticks);
 }
 
+void JVMCICompiler::on_upcall(const char* error, JVMCICompileState* compile_state) {
+  if (error != nullptr) {
+
+    Atomic::inc(&_err_upcalls);
+    int ok = _ok_upcalls;
+    int err = _err_upcalls;
+    // If there have been at least 10 upcalls with an error
+    // and the number of error upcalls is 10% or more of the
+    // number of non-error upcalls, disable JVMCI compilation.
+    if (err > 10 && err * 10 > ok && !_disabled) {
+      _disabled = true;
+      int total = err + ok;
+      const char* disable_msg = err_msg("JVMCI compiler disabled "
+      "after %d of %d upcalls had errors (Last error: \"%s\"). "
+      "Use -Xlog:jit+compilation for more detail.", err, total, error);
+      log_warning(jit,compilation)("%s", disable_msg);
+      if (compile_state != nullptr) {
+        const char* disable_error = os::strdup(disable_msg);
+        if (disable_error != nullptr) {
+          compile_state->set_failure(true, disable_error, true);
+          JVMCI_event_1("%s", disable_error);
+          return;
+        } else {
+          // Leave failure reason as set by caller when strdup fails
+        }
+      }
+    }
+    JVMCI_event_1("JVMCI upcall had an error: %s", error);
+  } else {
+    Atomic::inc(&_ok_upcalls);
+  }
+}
+
 void JVMCICompiler::inc_global_compilation_ticks() {
   Atomic::inc(&_global_compilation_ticks);
 }

src/hotspot/share/jvmci/jvmciCompiler.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,8 @@
 #include "compiler/compiler_globals.hpp"
 #include "runtime/atomic.hpp"
 
+class JVMCICompileState;
+
 class JVMCICompiler : public AbstractCompiler {
  public:
   // Code installation specific statistics.
@@ -62,10 +64,20 @@ class JVMCICompiler : public AbstractCompiler {
 
   /**
    * Number of methods successfully compiled by a call to
-   * JVMCICompiler::compile_method().
+   * JVMCIRuntime::compile_method().
    */
   volatile int _methods_compiled;
 
+  // Tracks upcalls that should only fail under severe conditions (e.g.
+  // memory pressure) and disables JVMCI compilation if too many fail
+  // with an error. A good example is an OOME thrown
+  // when libgraal calls into the HotSpot heap to get a copy
+  // of the system properties or to translate an exception from
+  // the HotSpot heap to the libgraal heap.
+  volatile int _ok_upcalls;
+  volatile int _err_upcalls;
+  bool _disabled;
+
   // Incremented periodically by JVMCI compiler threads
   // to indicate JVMCI compilation activity.
   volatile int _global_compilation_ticks;
@@ -126,6 +138,11 @@ public:
   int methods_compiled() { return _methods_compiled; }
   void inc_methods_compiled();
 
+  // Called after a JVMCI upcall whose success is a measure of the
+  // JVMCI compiler's health. The value of `error` describes
+  // an error during the upcall, null if no error.
+  void on_upcall(const char* error, JVMCICompileState* compile_state=nullptr);
+
   // Gets a value indicating JVMCI compilation activity on any thread.
   // If successive calls to this method return a different value, then
   // some degree of JVMCI compilation occurred between the calls.

src/hotspot/share/jvmci/jvmciCompilerToVM.cpp

@@ -2399,8 +2399,12 @@ C2V_VMENTRY_NULL(jlongArray, registerNativeMethods, (JNIEnv* env, jobject, jclas
   JVMCIRuntime* runtime;
   {
     // Ensure the JVMCI shared library runtime is initialized.
-    JVMCIEnv __peer_jvmci_env__(thread, false, __FILE__, __LINE__);
+    bool jni_enomem_is_fatal = false;
+    JVMCIEnv __peer_jvmci_env__(thread, false, jni_enomem_is_fatal, __FILE__, __LINE__);
     JVMCIEnv* peerEnv = &__peer_jvmci_env__;
+    if (peerEnv->has_jni_enomem()) {
+      JVMCI_THROW_MSG_0(OutOfMemoryError, "JNI_ENOMEM creating or attaching to libjvmci");
+    }
     HandleMark hm(THREAD);
     runtime = JVMCI::compiler_runtime(thread);
     if (peerEnv->has_pending_exception()) {
@@ -2563,8 +2567,13 @@ C2V_VMENTRY_PREFIX(jboolean, attachCurrentThread, (JNIEnv* env, jobject c2vm, jb
 
     {
       // Ensure the JVMCI shared library runtime is initialized.
-      JVMCIEnv __peer_jvmci_env__(thread, false, __FILE__, __LINE__);
+      bool jni_enomem_is_fatal = false;
+      JVMCIEnv __peer_jvmci_env__(thread, false, jni_enomem_is_fatal, __FILE__, __LINE__);
       JVMCIEnv* peerJVMCIEnv = &__peer_jvmci_env__;
+      if (peerJVMCIEnv->has_jni_enomem()) {
+        JVMCI_THROW_MSG_0(OutOfMemoryError, "JNI_ENOMEM creating or attaching to libjvmci");
+      }
+
       HandleMark hm(thread);
       JVMCIObject receiver = runtime->get_HotSpotJVMCIRuntime(peerJVMCIEnv);
       if (peerJVMCIEnv->has_pending_exception()) {
@@ -2658,9 +2667,13 @@ C2V_VMENTRY_0(jlong, translate, (JNIEnv* env, jobject, jobject obj_handle, jbool
   if (obj_handle == nullptr) {
     return 0L;
   }
-  JVMCIEnv __peer_jvmci_env__(thread, !JVMCIENV->is_hotspot(), __FILE__, __LINE__);
+  bool jni_enomem_is_fatal = false;
+  JVMCIEnv __peer_jvmci_env__(thread, !JVMCIENV->is_hotspot(), jni_enomem_is_fatal, __FILE__, __LINE__);
   JVMCIEnv* peerEnv = &__peer_jvmci_env__;
   JVMCIEnv* thisEnv = JVMCIENV;
+  if (peerEnv->has_jni_enomem()) {
+      JVMCI_THROW_MSG_0(OutOfMemoryError, "JNI_ENOMEM creating or attaching to libjvmci");
+  }
 
   JVMCIObject obj = thisEnv->wrap(obj_handle);
   JVMCIObject result;

src/hotspot/share/jvmci/jvmciEnv.cpp

@@ -67,6 +67,21 @@ JVMCICompileState::JVMCICompileState(CompileTask* task, JVMCICompiler* compiler)
   }
 }
 
+void JVMCICompileState::set_failure(bool retryable, const char* reason, bool reason_on_C_heap) {
+  if (_failure_reason != nullptr && _failure_reason_on_C_heap) {
+      os::free((void*) _failure_reason);
+  }
+  _failure_reason = reason;
+  _failure_reason_on_C_heap = reason_on_C_heap;
+  _retryable = retryable;
+}
+
+void JVMCICompileState::notify_libjvmci_oome() {
+  const char* msg = "Out of memory initializing libjvmci or attaching it to the current thread";
+  set_failure(true, msg);
+  _compiler->on_upcall(msg);
+}
+
 // Update global JVMCI compilation ticks after 512 thread-local JVMCI compilation ticks.
 // This mitigates the overhead of the atomic operation used for the global update.
 #define THREAD_TICKS_PER_GLOBAL_TICKS (2 << 9)
@@ -172,7 +187,7 @@ void JVMCIEnv::copy_saved_properties(jbyte* properties, int properties_len, JVMC
   }
 }
 
-void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, bool attach_OOME_is_fatal) {
+void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, bool jni_enomem_is_fatal) {
   assert(thread != nullptr, "npe");
   _env = nullptr;
   _pop_frame_on_close = false;
@@ -204,11 +219,18 @@ void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, boo
   _is_hotspot = false;
 
   _runtime = JVMCI::compiler_runtime(thread);
-  _env = _runtime->init_shared_library_javavm();
+  int create_JavaVM_err = JNI_OK;
-
+  _env = _runtime->init_shared_library_javavm(&create_JavaVM_err);
   if (_env != nullptr) {
     // Creating the JVMCI shared library VM also attaches the current thread
     _detach_on_close = true;
+  } else if (create_JavaVM_err != JNI_OK) {
+    if (!jni_enomem_is_fatal && create_JavaVM_err == JNI_ENOMEM) {
+      _jni_enomem = true;
+      return;
+    } else {
+      fatal("JNI_CreateJavaVM failed with return value %d", create_JavaVM_err);
+    }
   } else {
     _runtime->GetEnv(thread, (void**)&parent_env, JNI_VERSION_1_2);
     if (parent_env != nullptr) {
@@ -227,9 +249,9 @@ void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, boo
       jint attach_result = _runtime->AttachCurrentThread(thread, (void**) &_env, &attach_args);
       if (attach_result == JNI_OK) {
         _detach_on_close = true;
-      } else if (!attach_OOME_is_fatal && attach_result == JNI_ENOMEM) {
+      } else if (!jni_enomem_is_fatal && attach_result == JNI_ENOMEM) {
         _env = nullptr;
-        _attach_threw_OOME = true;
+        _jni_enomem = true;
         return;
       } else {
         fatal("Error attaching current thread (%s) to JVMCI shared library JNI interface", attach_args.name);
@@ -251,32 +273,33 @@ void JVMCIEnv::init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, boo
 }
 
 JVMCIEnv::JVMCIEnv(JavaThread* thread, JVMCICompileState* compile_state, const char* file, int line):
-    _throw_to_caller(false), _file(file), _line(line), _attach_threw_OOME(false), _compile_state(compile_state) {
+    _throw_to_caller(false), _file(file), _line(line), _jni_enomem(false), _compile_state(compile_state) {
-  // In case of OOME, there's a good chance a subsequent attempt to attach might succeed.
+  // In case of JNI_ENOMEM, there's a good chance a subsequent attempt to create libjvmci or attach to it
-  // Other errors most likely indicate a non-recoverable error in the JVMCI runtime.
+  // might succeed. Other errors most likely indicate a non-recoverable error in the JVMCI runtime.
-  init_env_mode_runtime(thread, nullptr, false);
+  bool jni_enomem_is_fatal = false;
-  if (_attach_threw_OOME) {
+  init_env_mode_runtime(thread, nullptr, jni_enomem_is_fatal);
+  if (_jni_enomem) {
     compile_state->set_failure(true, "Out of memory while attaching JVMCI compiler to current thread");
   }
 }
 
 JVMCIEnv::JVMCIEnv(JavaThread* thread, const char* file, int line):
-    _throw_to_caller(false), _file(file), _line(line), _attach_threw_OOME(false), _compile_state(nullptr) {
+    _throw_to_caller(false), _file(file), _line(line), _jni_enomem(false), _compile_state(nullptr) {
   init_env_mode_runtime(thread, nullptr);
 }
 
 JVMCIEnv::JVMCIEnv(JavaThread* thread, JNIEnv* parent_env, const char* file, int line):
-    _throw_to_caller(true), _file(file), _line(line), _attach_threw_OOME(false), _compile_state(nullptr) {
+    _throw_to_caller(true), _file(file), _line(line), _jni_enomem(false), _compile_state(nullptr) {
   init_env_mode_runtime(thread, parent_env);
   assert(_env == nullptr || parent_env == _env, "mismatched JNIEnvironment");
 }
 
-void JVMCIEnv::init(JavaThread* thread, bool is_hotspot, const char* file, int line) {
+void JVMCIEnv::init(JavaThread* thread, bool is_hotspot, bool jni_enomem_is_fatal, const char* file, int line) {
   _compile_state = nullptr;
   _throw_to_caller = false;
   _file = file;
   _line = line;
-  _attach_threw_OOME = false;
+  _jni_enomem = false;
   if (is_hotspot) {
     _env = nullptr;
     _pop_frame_on_close = false;
@@ -284,7 +307,7 @@ void JVMCIEnv::init(JavaThread* thread, bool is_hotspot, const char* file, int l
     _is_hotspot = true;
     _runtime = JVMCI::java_runtime();
   } else {
-    init_env_mode_runtime(thread, nullptr);
+    init_env_mode_runtime(thread, nullptr, jni_enomem_is_fatal);
   }
 }
 
@@ -464,7 +487,7 @@ jboolean JVMCIEnv::transfer_pending_exception(JavaThread* THREAD, JVMCIEnv* peer
 }
 
 JVMCIEnv::~JVMCIEnv() {
-  if (_attach_threw_OOME) {
+  if (_jni_enomem) {
     return;
   }
   if (_throw_to_caller) {
@@ -775,6 +798,7 @@ DO_THROW(IllegalArgumentException)
 DO_THROW(InvalidInstalledCodeException)
 DO_THROW(UnsatisfiedLinkError)
 DO_THROW(UnsupportedOperationException)
+DO_THROW(OutOfMemoryError)
 DO_THROW(ClassNotFoundException)
 
 #undef DO_THROW

src/hotspot/share/jvmci/jvmciEnv.hpp

@@ -137,11 +137,11 @@ class JVMCICompileState : public ResourceObj {
   bool failure_reason_on_C_heap() { return _failure_reason_on_C_heap; }
   bool retryable() { return _retryable; }
 
-  void set_failure(bool retryable, const char* reason, bool reason_on_C_heap = false) {
+  void set_failure(bool retryable, const char* reason, bool reason_on_C_heap = false);
-    _failure_reason = reason;
+
-    _failure_reason_on_C_heap = reason_on_C_heap;
+  // Called when creating or attaching to a libjvmci isolate failed
-    _retryable = retryable;
+  // due to an out of memory condition.
-  }
+  void notify_libjvmci_oome();
 
   jint compilation_ticks() const { return _compilation_ticks; }
   void inc_compilation_ticks();
@@ -157,9 +157,9 @@ class JVMCIEnv : public ResourceObj {
   friend class JNIAccessMark;
 
   // Initializes the _env, _mode and _runtime fields.
-  void init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, bool attach_OOME_is_fatal = true);
+  void init_env_mode_runtime(JavaThread* thread, JNIEnv* parent_env, bool jni_enomem_is_fatal = true);
 
-  void init(JavaThread* thread, bool is_hotspot, const char* file, int line);
+  void init(JavaThread* thread, bool is_hotspot, bool jni_enomem_is_fatal, const char* file, int line);
 
   JNIEnv*                 _env;  // JNI env for calling into shared library
   bool     _pop_frame_on_close;  // Must pop frame on close?
@@ -169,7 +169,9 @@ class JVMCIEnv : public ResourceObj {
   bool        _throw_to_caller;  // Propagate an exception raised in this env to the caller?
   const char*            _file;  // The file and ...
   int                    _line;  // ... line where this JNIEnv was created
-  bool    _attach_threw_OOME;    // Failed to attach thread due to OutOfMemoryError, the JVMCIEnv is invalid
+  bool             _jni_enomem;  // JNI_ENOMEM returned when creating or attaching to a libjvmci isolate.
+                                 // If true, the JVMCIEnv is invalid and should not be used apart from
+                                 // calling has_jni_enomem().
 
   // Translates an exception on the HotSpot heap (i.e., hotspot_env) to an exception on
   // the shared library heap (i.e., jni_env). The translation includes the stack and cause(s) of `throwable`.
@@ -212,18 +214,25 @@ public:
     // on the VM thread.
     assert(for_object.is_hotspot() || !Thread::current()->is_VM_thread(),
         "cannot open JVMCIEnv scope when in the VM thread for accessing a shared library heap object");
-    init(thread, for_object.is_hotspot(), file, line);
+    bool jni_enomem_is_fatal = true;
+    init(thread, for_object.is_hotspot(), jni_enomem_is_fatal, file, line);
   }
 
   // Opens a JNIEnv scope for the HotSpot runtime if `is_hotspot` is true
   // otherwise for the shared library runtime. An exception occurring
   // within the scope must not be propagated back to the caller.
-  JVMCIEnv(JavaThread* thread, bool is_hotspot, const char* file, int line) {
+  JVMCIEnv(JavaThread* thread, bool is_hotspot, bool jni_enomem_is_fatal, const char* file, int line) {
-    init(thread, is_hotspot, file, line);
+    init(thread, is_hotspot, jni_enomem_is_fatal, file, line);
   }
 
   ~JVMCIEnv();
 
+  // Determines if a JNI_ENOMEM occurred while trying to create a libjvmci
+  // isolate or attach to it within the scope of a JVMCIEnv constructor.
+  bool has_jni_enomem() {
+    return _jni_enomem;
+  }
+
   JVMCIRuntime* runtime() {
     return _runtime;
   }
@@ -249,7 +258,9 @@ public:
   // Returns true if a pending exception was transferred, false otherwise.
   static jboolean transfer_pending_exception_to_jni(JavaThread* THREAD, JVMCIEnv* hotspot_env, JVMCIEnv* jni_env);
 
-  // Prints an exception and stack trace of a pending exception.
+  // Prints the toString() and stack trace of a pending exception.
+  // If there is no pending exception, this is a nop.
+  // If `clear` is false, the pending exception will remain pending upon return.
   void describe_pending_exception(bool clear);
 
   int get_length(JVMCIArray array);
@@ -356,6 +367,7 @@ public:
   DO_THROW(InvalidInstalledCodeException)
   DO_THROW(UnsatisfiedLinkError)
   DO_THROW(UnsupportedOperationException)
+  DO_THROW(OutOfMemoryError)
   DO_THROW(ClassNotFoundException)
 
 #undef DO_THROW

src/hotspot/share/jvmci/jvmciJavaClasses.hpp

@@ -244,6 +244,9 @@
   start_class(InternalError, java_lang_InternalError)                                                         \
     jvmci_constructor(InternalError, "(Ljava/lang/String;)V")                                                 \
   end_class                                                                                                   \
+  start_class(OutOfMemoryError, java_lang_OutOfMemoryError)                                                         \
+    jvmci_constructor(OutOfMemoryError, "(Ljava/lang/String;)V")                                                 \
+  end_class                                                                                                   \
   start_class(ClassNotFoundException, java_lang_ClassNotFoundException)                                       \
     jvmci_constructor(ClassNotFoundException, "(Ljava/lang/String;)V")                                        \
   end_class                                                                                                   \

src/hotspot/share/jvmci/jvmciRuntime.cpp

@@ -45,6 +45,7 @@
 #include "oops/typeArrayOop.inline.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "prims/methodHandles.hpp"
+#include "runtime/arguments.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/deoptimization.hpp"
 #include "runtime/fieldDescriptor.inline.hpp"
@@ -1225,10 +1226,16 @@ bool JVMCIRuntime::detach_thread(JavaThread* thread, const char* reason, bool ca
   return destroyed_javavm;
 }
 
-JNIEnv* JVMCIRuntime::init_shared_library_javavm() {
+JNIEnv* JVMCIRuntime::init_shared_library_javavm(int* create_JavaVM_err) {
   MutexLocker locker(_lock);
   JavaVM* javaVM = _shared_library_javavm;
   if (javaVM == nullptr) {
+    const char* val = Arguments::PropertyList_get_value(Arguments::system_properties(), "test.jvmci.forceEnomemOnLibjvmciInit");
+    if (val != nullptr && strcmp(val, "true") == 0) {
+      *create_JavaVM_err = JNI_ENOMEM;
+      return nullptr;
+    }
+
     char* sl_path;
     void* sl_handle = JVMCI::get_shared_library(sl_path, true);
 
@@ -1275,7 +1282,7 @@ JNIEnv* JVMCIRuntime::init_shared_library_javavm() {
       JVMCI_event_1("created JavaVM[%ld]@" PTR_FORMAT " for JVMCI runtime %d", javaVM_id, p2i(javaVM), _id);
       return env;
     } else {
-      fatal("JNI_CreateJavaVM failed with return value %d", result);
+      *create_JavaVM_err = result;
     }
   }
   return nullptr;
@@ -1461,6 +1468,7 @@ void JVMCIRuntime::initialize(JVMCI_TRAPS) {
       Handle properties_exception;
       properties = JVMCIENV->get_serialized_saved_properties(properties_len, THREAD);
       if (JVMCIEnv::transfer_pending_exception_to_jni(THREAD, nullptr, JVMCIENV)) {
+        JVMCI_event_1("error initializing system properties for JVMCI runtime %d", _id);
         return;
       }
       JVMCIENV->copy_saved_properties(properties, properties_len, JVMCI_CHECK);
@@ -1548,8 +1556,9 @@ JVM_END
 
 void JVMCIRuntime::shutdown() {
   if (_HotSpotJVMCIRuntime_instance.is_non_null()) {
+    bool jni_enomem_is_fatal = true;
     JVMCI_event_1("shutting down HotSpotJVMCIRuntime for JVMCI runtime %d", _id);
-    JVMCIEnv __stack_jvmci_env__(JavaThread::current(), _HotSpotJVMCIRuntime_instance.is_hotspot(), __FILE__, __LINE__);
+    JVMCIEnv __stack_jvmci_env__(JavaThread::current(), _HotSpotJVMCIRuntime_instance.is_hotspot(), jni_enomem_is_fatal, __FILE__, __LINE__);
     JVMCIEnv* JVMCIENV = &__stack_jvmci_env__;
     JVMCIENV->call_HotSpotJVMCIRuntime_shutdown(_HotSpotJVMCIRuntime_instance);
     if (_num_attached_threads == cannot_be_attached) {
@@ -1977,6 +1986,34 @@ JVMCI::CodeInstallResult JVMCIRuntime::validate_compile_task_dependencies(Depend
   return JVMCI::dependencies_failed;
 }
 
+// Called after an upcall to `function` while compiling `method`.
+// If an exception occurred, it is cleared, the compilation state
+// is updated with the failure and this method returns true.
+// Otherwise, it returns false.
+static bool after_compiler_upcall(JVMCIEnv* JVMCIENV, JVMCICompiler* compiler, const methodHandle& method, const char* function) {
+  if (JVMCIENV->has_pending_exception()) {
+    bool reason_on_C_heap = true;
+    const char* failure_reason = os::strdup(err_msg("uncaught exception in %s", function), mtJVMCI);
+    if (failure_reason == nullptr) {
+      failure_reason = "uncaught exception";
+      reason_on_C_heap = false;
+    }
+    Log(jit, compilation) log;
+    if (log.is_info()) {
+      ResourceMark rm;
+      log.info("%s while compiling %s", failure_reason, method->name_and_sig_as_C_string());
+      JVMCIENV->describe_pending_exception(true);
+    } else {
+      JVMCIENV->clear_pending_exception();
+    }
+    JVMCICompileState* compile_state = JVMCIENV->compile_state();
+    compile_state->set_failure(true, failure_reason, reason_on_C_heap);
+    compiler->on_upcall(failure_reason, compile_state);
+    return true;
+  }
+  return false;
+}
+
 void JVMCIRuntime::compile_method(JVMCIEnv* JVMCIENV, JVMCICompiler* compiler, const methodHandle& method, int entry_bci) {
   JVMCI_EXCEPTION_CONTEXT
 
@@ -2002,30 +2039,21 @@ void JVMCIRuntime::compile_method(JVMCIEnv* JVMCIENV, JVMCICompiler* compiler, c
 
   HandleMark hm(thread);
   JVMCIObject receiver = get_HotSpotJVMCIRuntime(JVMCIENV);
-  if (JVMCIENV->has_pending_exception()) {
+  if (after_compiler_upcall(JVMCIENV, compiler, method, "get_HotSpotJVMCIRuntime")) {
-    if (PrintWarnings) {
-      ResourceMark rm(thread);
-      warning("HotSpotJVMCIRuntime initialization failed when compiling %s", method->name_and_sig_as_C_string());
-      JVMCIENV->describe_pending_exception(true);
-    }
-    compile_state->set_failure(false, "exception during HotSpotJVMCIRuntime initialization");
     return;
   }
   JVMCIObject jvmci_method = JVMCIENV->get_jvmci_method(method, JVMCIENV);
-  if (JVMCIENV->has_pending_exception()) {
+  if (after_compiler_upcall(JVMCIENV, compiler, method, "get_jvmci_method")) {
-    if (PrintWarnings) {
-      ResourceMark rm(thread);
-      warning("Error creating JVMCI wrapper for %s", method->name_and_sig_as_C_string());
-      JVMCIENV->describe_pending_exception(true);
-    }
-    compile_state->set_failure(false, "exception getting JVMCI wrapper method");
     return;
   }
 
   JVMCIObject result_object = JVMCIENV->call_HotSpotJVMCIRuntime_compileMethod(receiver, jvmci_method, entry_bci,
                                                                      (jlong) compile_state, compile_state->task()->compile_id());
-  if (!JVMCIENV->has_pending_exception()) {
+  if (after_compiler_upcall(JVMCIENV, compiler, method, "call_HotSpotJVMCIRuntime_compileMethod")) {
-    if (result_object.is_non_null()) {
+    return;
+  }
+  compiler->on_upcall(nullptr);
+  guarantee(result_object.is_non_null(), "call_HotSpotJVMCIRuntime_compileMethod returned null");
   JVMCIObject failure_message = JVMCIENV->get_HotSpotCompilationRequestResult_failureMessage(result_object);
   if (failure_message.is_non_null()) {
     // Copy failure reason into resource memory first ...
@@ -2042,14 +2070,6 @@ void JVMCIRuntime::compile_method(JVMCIEnv* JVMCIENV, JVMCICompiler* compiler, c
       compiler->inc_methods_compiled();
     }
   }
-    } else {
-      assert(false, "JVMCICompiler.compileMethod should always return non-null");
-    }
-  } else {
-    // An uncaught exception here implies failure during compiler initialization.
-    // The only sensible thing to do here is to exit the VM.
-    fatal_exception(JVMCIENV, "Exception during JVMCI compiler initialization");
-  }
   if (compiler->is_bootstrapping()) {
     compiler->set_bootstrap_compilation_request_handled();
   }

src/hotspot/share/jvmci/jvmciRuntime.hpp

@@ -279,7 +279,9 @@ class JVMCIRuntime: public CHeapObj<mtJVMCI> {
   // Ensures that a JVMCI shared library JavaVM exists for this runtime.
   // If the JavaVM was created by this call, then the thread-local JNI
   // interface pointer for the JavaVM is returned otherwise null is returned.
-  JNIEnv* init_shared_library_javavm();
+  // If this method tried to create the JavaVM but failed, the error code returned
+  // by JNI_CreateJavaVM is returned in create_JavaVM_err.
+  JNIEnv* init_shared_library_javavm(int* create_JavaVM_err);
 
   // Determines if the JVMCI shared library JavaVM exists for this runtime.
   bool has_shared_library_javavm() { return _shared_library_javavm != nullptr; }

test/hotspot/jtreg/compiler/jvmci/jdk.vm.ci.hotspot.test/src/jdk/vm/ci/hotspot/test/TestHotSpotJVMCIRuntime.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
 /*
  * @test
  * @requires vm.jvmci
+ * @library /test/lib /
  * @modules jdk.internal.vm.ci/jdk.vm.ci.hotspot
  *          jdk.internal.vm.ci/jdk.vm.ci.runtime
  *          jdk.internal.vm.ci/jdk.vm.ci.meta
@@ -46,6 +47,8 @@ import java.util.function.Predicate;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
+import jdk.test.lib.process.ProcessTools;
+import jdk.test.lib.process.OutputAnalyzer;
 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
 import jdk.vm.ci.meta.MetaAccessProvider;
 import jdk.vm.ci.meta.ResolvedJavaType;
@@ -122,4 +125,54 @@ public class TestHotSpotJVMCIRuntime {
             Assert.assertEquals(expected, actual, c + ": cl=" + cl);
         }
     }
+
+    /**
+     * Test program that calls into the VM and expects an {@code OutOfMemoryError} to be
+     * raised when {@code test.jvmci.forceEnomemOnLibjvmciInit == true}.
+     *
+     * For example:
+     * <pre>
+     * Exception in thread "main" java.lang.OutOfMemoryError: JNI_ENOMEM creating or attaching to libjvmci
+     *    at jdk.internal.vm.ci/jdk.vm.ci.hotspot.CompilerToVM.attachCurrentThread(Native Method)
+     *    at jdk.internal.vm.ci/jdk.vm.ci.hotspot.HotSpotJVMCIRuntime.attachCurrentThread(HotSpotJVMCIRuntime.java:1385)
+     *    at jdk.vm.ci.hotspot.test.TestHotSpotJVMCIRuntime$JNIEnomemVMCall.main(TestHotSpotJVMCIRuntime.java:133)
+     * </pre>
+     */
+    public static class JNIEnomemVMCall {
+        public static void main(String[] args) {
+            String name = args[0];
+            HotSpotJVMCIRuntime runtime = HotSpotJVMCIRuntime.runtime();
+            MetaAccessProvider metaAccess = runtime.getHostJVMCIBackend().getMetaAccess();
+            if (name.equals("translate")) {
+                runtime.translate("object");
+            } else if (name.equals("attachCurrentThread")) {
+                runtime.attachCurrentThread(false, null);
+            } else if (name.equals("registerNativeMethods")) {
+                runtime.registerNativeMethods(JNIEnomemVMCall.class);
+            } else {
+                throw new InternalError("Unknown method: " + name);
+            }
+        }
+    }
+
+    @Test
+    public void jniEnomemTest() throws Exception {
+        String[] names = {"translate", "attachCurrentThread", "registerNativeMethods"};
+        for (String name : names) {
+            ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(
+                "-XX:+UnlockExperimentalVMOptions",
+                "-XX:+EnableJVMCI",
+                "-XX:-UseJVMCICompiler",
+                "-XX:+UseJVMCINativeLibrary",
+                "-Dtest.jvmci.forceEnomemOnLibjvmciInit=true",
+                "--add-exports=jdk.internal.vm.ci/jdk.vm.ci.services=ALL-UNNAMED",
+                "--add-exports=jdk.internal.vm.ci/jdk.vm.ci.runtime=ALL-UNNAMED",
+                "--add-exports=jdk.internal.vm.ci/jdk.vm.ci.hotspot=ALL-UNNAMED",
+                "-Xbootclasspath/a:.",
+                JNIEnomemVMCall.class.getName(), name);
+            OutputAnalyzer output = new OutputAnalyzer(pb.start());
+            output.shouldContain("java.lang.OutOfMemoryError: JNI_ENOMEM creating or attaching to libjvmci");
+            output.shouldNotHaveExitValue(0);
+        }
+    }
 }