8331208: Memory stress test that checks OutOfMemoryError stack trace fails

Reviewed-by: dholmes, never

src/hotspot/share/compiler/compileBroker.cpp

@@ -38,6 +38,7 @@
 #include "compiler/compilerEvent.hpp"
 #include "compiler/compilerOracle.hpp"
 #include "compiler/directivesParser.hpp"
+#include "gc/shared/memAllocator.hpp"
 #include "interpreter/linkResolver.hpp"
 #include "jvm.h"
 #include "jfr/jfrEvents.hpp"
@@ -1396,6 +1397,7 @@ nmethod* CompileBroker::compile_method(const methodHandle& method, int osr_bci,
   assert(!HAS_PENDING_EXCEPTION, "No exception should be present");
   // some prerequisites that are compiler specific
   if (comp->is_c2() || comp->is_jvmci()) {
+    InternalOOMEMark iom(THREAD);
     method->constants()->resolve_string_constants(CHECK_AND_CLEAR_NONASYNC_NULL);
     // Resolve all classes seen in the signature of the method
     // we are compiling.

src/hotspot/share/gc/shared/memAllocator.cpp

@@ -123,21 +123,21 @@ bool MemAllocator::Allocation::check_out_of_memory() {
   }
 
   const char* message = _overhead_limit_exceeded ? "GC overhead limit exceeded" : "Java heap space";
-  if (!_thread->in_retryable_allocation()) {
+  if (!_thread->is_in_internal_oome_mark()) {
     // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support
     report_java_out_of_memory(message);
-
     if (JvmtiExport::should_post_resource_exhausted()) {
       JvmtiExport::post_resource_exhausted(
         JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR | JVMTI_RESOURCE_EXHAUSTED_JAVA_HEAP,
         message);
     }
+
     oop exception = _overhead_limit_exceeded ?
         Universe::out_of_memory_error_gc_overhead_limit() :
         Universe::out_of_memory_error_java_heap();
     THROW_OOP_(exception, true);
   } else {
-    THROW_OOP_(Universe::out_of_memory_error_retry(), true);
+    THROW_OOP_(Universe::out_of_memory_error_java_heap_without_backtrace(), true);
   }
 }
 

src/hotspot/share/gc/shared/memAllocator.hpp

@@ -114,4 +114,37 @@ public:
   virtual oop initialize(HeapWord* mem) const;
 };
 
+// Manages a scope where a failed heap allocation results in
+// suppression of JVMTI "resource exhausted" events and
+// throwing a shared, backtrace-less OOME instance.
+// Used for OOMEs that will not be propagated to user code.
+class InternalOOMEMark: public StackObj {
+ private:
+  bool _outer;
+  JavaThread* _thread;
+
+ public:
+  explicit InternalOOMEMark(JavaThread* thread) {
+    if (thread != nullptr) {
+      _outer = thread->is_in_internal_oome_mark();
+      thread->set_is_in_internal_oome_mark(true);
+      _thread = thread;
+    } else {
+      _outer = false;
+      _thread = nullptr;
+    }
+  }
+
+  ~InternalOOMEMark() {
+    if (_thread != nullptr) {
+      // Check that only InternalOOMEMark sets
+      // JavaThread::_is_in_internal_oome_mark
+      assert(_thread->is_in_internal_oome_mark(), "must be");
+      _thread->set_is_in_internal_oome_mark(_outer);
+    }
+  }
+
+  JavaThread* thread() const  { return _thread; }
+};
+
 #endif // SHARE_GC_SHARED_MEMALLOCATOR_HPP

src/hotspot/share/jvmci/jvmciRuntime.cpp

@@ -28,6 +28,7 @@
 #include "classfile/vmClasses.hpp"
 #include "compiler/compileBroker.hpp"
 #include "gc/shared/collectedHeap.hpp"
+#include "gc/shared/memAllocator.hpp"
 #include "gc/shared/oopStorage.inline.hpp"
 #include "jvmci/jniAccessMark.inline.hpp"
 #include "jvmci/jvmciCompilerToVM.hpp"
@@ -92,39 +93,25 @@ static void deopt_caller() {
 }
 
 // Manages a scope for a JVMCI runtime call that attempts a heap allocation.
-// If there is a pending nonasync exception upon closing the scope and the runtime
+// If there is a pending OutOfMemoryError upon closing the scope and the runtime
 // call is of the variety where allocation failure returns null without an
 // exception, the following action is taken:
-//   1. The pending nonasync exception is cleared
+//   1. The pending OutOfMemoryError is cleared
 //   2. null is written to JavaThread::_vm_result
-//   3. Checks that an OutOfMemoryError is Universe::out_of_memory_error_retry().
+class RetryableAllocationMark {
-class RetryableAllocationMark: public StackObj {
  private:
-  JavaThread* _thread;
+   InternalOOMEMark _iom;
  public:
-  RetryableAllocationMark(JavaThread* thread, bool activate) {
+  RetryableAllocationMark(JavaThread* thread, bool activate) : _iom(activate ? thread : nullptr) {}
-    if (activate) {
-      assert(!thread->in_retryable_allocation(), "retryable allocation scope is non-reentrant");
-      _thread = thread;
-      _thread->set_in_retryable_allocation(true);
-    } else {
-      _thread = nullptr;
-    }
-  }
   ~RetryableAllocationMark() {
-    if (_thread != nullptr) {
+    JavaThread* THREAD = _iom.thread(); // For exception macros.
-      _thread->set_in_retryable_allocation(false);
+    if (THREAD != nullptr) {
-      JavaThread* THREAD = _thread; // For exception macros.
       if (HAS_PENDING_EXCEPTION) {
         oop ex = PENDING_EXCEPTION;
-        // Do not clear probable async exceptions.
+        THREAD->set_vm_result(nullptr);
-        CLEAR_PENDING_NONASYNC_EXCEPTION;
+        if (ex->is_a(vmClasses::OutOfMemoryError_klass())) {
-        oop retry_oome = Universe::out_of_memory_error_retry();
+          CLEAR_PENDING_EXCEPTION;
-        if (ex->is_a(retry_oome->klass()) && retry_oome != ex) {
-          ResourceMark rm;
-          fatal("Unexpected exception in scope of retryable allocation: " INTPTR_FORMAT " of type %s", p2i(ex), ex->klass()->external_name());
         }
-        _thread->set_vm_result(nullptr);
       }
     }
   }

src/hotspot/share/memory/universe.cpp

@@ -135,7 +135,6 @@ enum OutOfMemoryInstance { _oom_java_heap,
                            _oom_array_size,
                            _oom_gc_overhead_limit,
                            _oom_realloc_objects,
-                           _oom_retry,
                            _oom_count };
 
 OopHandle Universe::_out_of_memory_errors;
@@ -655,6 +654,10 @@ oop Universe::out_of_memory_error_java_heap() {
   return gen_out_of_memory_error(out_of_memory_errors()->obj_at(_oom_java_heap));
 }
 
+oop Universe::out_of_memory_error_java_heap_without_backtrace() {
+  return out_of_memory_errors()->obj_at(_oom_java_heap);
+}
+
 oop Universe::out_of_memory_error_c_heap() {
   return gen_out_of_memory_error(out_of_memory_errors()->obj_at(_oom_c_heap));
 }
@@ -679,9 +682,6 @@ oop Universe::out_of_memory_error_realloc_objects() {
   return gen_out_of_memory_error(out_of_memory_errors()->obj_at(_oom_realloc_objects));
 }
 
-// Throw default _out_of_memory_error_retry object as it will never propagate out of the VM
-oop Universe::out_of_memory_error_retry()              { return out_of_memory_errors()->obj_at(_oom_retry);  }
-
 oop Universe::class_init_out_of_memory_error()         { return out_of_memory_errors()->obj_at(_oom_java_heap); }
 oop Universe::class_init_stack_overflow_error()        { return _class_init_stack_overflow_error.resolve(); }
 oop Universe::delayed_stack_overflow_error_message()   { return _delayed_stack_overflow_error_message.resolve(); }
@@ -785,9 +785,6 @@ void Universe::create_preallocated_out_of_memory_errors(TRAPS) {
   msg = java_lang_String::create_from_str("Java heap space: failed reallocation of scalar replaced objects", CHECK);
   java_lang_Throwable::set_message(oom_array->obj_at(_oom_realloc_objects), msg());
 
-  msg = java_lang_String::create_from_str("Java heap space: failed retryable allocation", CHECK);
-  java_lang_Throwable::set_message(oom_array->obj_at(_oom_retry), msg());
-
   // Setup the array of errors that have preallocated backtrace
   int len = (StackTraceInThrowable) ? (int)PreallocatedOutOfMemoryErrorCount : 0;
   objArrayOop instance = oopFactory::new_objArray(ik, len, CHECK);

src/hotspot/share/memory/universe.hpp

@@ -272,6 +272,7 @@ class Universe: AllStatic {
   // may or may not have a backtrace. If error has a backtrace then the stack trace is already
   // filled in.
   static oop out_of_memory_error_java_heap();
+  static oop out_of_memory_error_java_heap_without_backtrace();
   static oop out_of_memory_error_c_heap();
   static oop out_of_memory_error_metaspace();
   static oop out_of_memory_error_class_metaspace();
@@ -279,8 +280,6 @@ class Universe: AllStatic {
   static oop out_of_memory_error_gc_overhead_limit();
   static oop out_of_memory_error_realloc_objects();
 
-  // Throw default _out_of_memory_error_retry object as it will never propagate out of the VM
-  static oop out_of_memory_error_retry();
   static oop delayed_stack_overflow_error_message();
 
   // Saved StackOverflowError and OutOfMemoryError for use when

src/hotspot/share/oops/klass.cpp

@@ -873,12 +873,12 @@ void Klass::set_archived_java_mirror(int mirror_index) {
 
 void Klass::check_array_allocation_length(int length, int max_length, TRAPS) {
   if (length > max_length) {
-    if (!THREAD->in_retryable_allocation()) {
+    if (!THREAD->is_in_internal_oome_mark()) {
       report_java_out_of_memory("Requested array size exceeds VM limit");
       JvmtiExport::post_array_size_exhausted();
       THROW_OOP(Universe::out_of_memory_error_array_size());
     } else {
-      THROW_OOP(Universe::out_of_memory_error_retry());
+      THROW_OOP(Universe::out_of_memory_error_java_heap_without_backtrace());
     }
   } else if (length < 0) {
     THROW_MSG(vmSymbols::java_lang_NegativeArraySizeException(), err_msg("%d", length));

src/hotspot/share/runtime/deoptimization.cpp

@@ -35,6 +35,7 @@
 #include "compiler/compilationPolicy.hpp"
 #include "compiler/compilerDefinitions.inline.hpp"
 #include "gc/shared/collectedHeap.hpp"
+#include "gc/shared/memAllocator.hpp"
 #include "interpreter/bytecode.hpp"
 #include "interpreter/bytecodeStream.hpp"
 #include "interpreter/interpreter.hpp"
@@ -1237,6 +1238,7 @@ bool Deoptimization::realloc_objects(JavaThread* thread, frame* fr, RegisterMap*
 
       InstanceKlass* ik = InstanceKlass::cast(k);
       if (obj == nullptr && !cache_init_error) {
+        InternalOOMEMark iom(THREAD);
 #if COMPILER2_OR_JVMCI
         if (EnableVectorSupport && VectorSupport::is_vector(ik)) {
           obj = VectorSupport::allocate_vector(ik, fr, reg_map, sv, THREAD);
@@ -1251,9 +1253,11 @@ bool Deoptimization::realloc_objects(JavaThread* thread, frame* fr, RegisterMap*
       TypeArrayKlass* ak = TypeArrayKlass::cast(k);
       assert(sv->field_size() % type2size[ak->element_type()] == 0, "non-integral array length");
       int len = sv->field_size() / type2size[ak->element_type()];
+      InternalOOMEMark iom(THREAD);
       obj = ak->allocate(len, THREAD);
     } else if (k->is_objArray_klass()) {
       ObjArrayKlass* ak = ObjArrayKlass::cast(k);
+      InternalOOMEMark iom(THREAD);
       obj = ak->allocate(sv->field_size(), THREAD);
     }
 

src/hotspot/share/runtime/javaThread.cpp

@@ -456,11 +456,11 @@ JavaThread::JavaThread() :
 #endif
 #endif
   _jni_attach_state(_not_attaching_via_jni),
+  _is_in_internal_oome_mark(false),
 #if INCLUDE_JVMCI
   _pending_deoptimization(-1),
   _pending_monitorenter(false),
   _pending_transfer_to_interpreter(false),
-  _in_retryable_allocation(false),
   _pending_failed_speculation(0),
   _jvmci{nullptr},
   _libjvmci_runtime(nullptr),

src/hotspot/share/runtime/javaThread.hpp

@@ -52,6 +52,7 @@
 class AsyncExceptionHandshake;
 class ContinuationEntry;
 class DeoptResourceMark;
+class InternalOOMEMark;
 class JNIHandleBlock;
 class JVMCIRuntime;
 
@@ -335,6 +336,8 @@ class JavaThread: public Thread {
   // of _attaching_via_jni and transitions to _attached_via_jni.
   volatile JNIAttachStates _jni_attach_state;
 
+  // In scope of an InternalOOMEMark?
+  bool _is_in_internal_oome_mark;
 
 #if INCLUDE_JVMCI
   // The _pending_* fields below are used to communicate extra information
@@ -350,10 +353,6 @@ class JavaThread: public Thread {
   // Specifies if the DeoptReason for the last uncommon trap was Reason_transfer_to_interpreter
   bool      _pending_transfer_to_interpreter;
 
-  // True if in a runtime call from compiled code that will deoptimize
-  // and re-execute a failed heap allocation in the interpreter.
-  bool      _in_retryable_allocation;
-
   // An id of a speculation that JVMCI compiled code can use to further describe and
   // uniquely identify the speculative optimization guarded by an uncommon trap.
   // See JVMCINMethodData::SPECULATION_LENGTH_BITS for further details.
@@ -718,6 +717,10 @@ private:
   MemRegion deferred_card_mark() const           { return _deferred_card_mark; }
   void set_deferred_card_mark(MemRegion mr)      { _deferred_card_mark = mr;   }
 
+  // Is thread in scope of an InternalOOMEMark?
+  bool is_in_internal_oome_mark() const          { return _is_in_internal_oome_mark; }
+  void set_is_in_internal_oome_mark(bool b)      { _is_in_internal_oome_mark = b;    }
+
 #if INCLUDE_JVMCI
   jlong pending_failed_speculation() const        { return _pending_failed_speculation; }
   void set_pending_monitorenter(bool b)           { _pending_monitorenter = b; }
@@ -727,9 +730,6 @@ private:
   void set_jvmci_alternate_call_target(address a) { assert(_jvmci._alternate_call_target == nullptr, "must be"); _jvmci._alternate_call_target = a; }
   void set_jvmci_implicit_exception_pc(address a) { assert(_jvmci._implicit_exception_pc == nullptr, "must be"); _jvmci._implicit_exception_pc = a; }
 
-  virtual bool in_retryable_allocation() const    { return _in_retryable_allocation; }
-  void set_in_retryable_allocation(bool b)        { _in_retryable_allocation = b; }
-
   JVMCIRuntime* libjvmci_runtime() const          { return _libjvmci_runtime; }
   void set_libjvmci_runtime(JVMCIRuntime* rt) {
     assert((_libjvmci_runtime == nullptr && rt != nullptr) || (_libjvmci_runtime != nullptr && rt == nullptr), "must be");

src/hotspot/share/runtime/thread.hpp

@@ -210,14 +210,6 @@ class Thread: public ThreadShadow {
   DEBUG_ONLY(bool _indirectly_safepoint_thread;)
 
  public:
-  // Determines if a heap allocation failure will be retried
-  // (e.g., by deoptimizing and re-executing in the interpreter).
-  // In this case, the failed allocation must raise
-  // Universe::out_of_memory_error_retry() and omit side effects
-  // such as JVMTI events and handling -XX:+HeapDumpOnOutOfMemoryError
-  // and -XX:OnOutOfMemoryError.
-  virtual bool in_retryable_allocation() const { return false; }
-
 #ifdef ASSERT
   void set_suspendible_thread()   { _suspendible_thread = true; }
   void clear_suspendible_thread() { _suspendible_thread = false; }