8318016: Per-compilation memory ceiling

Reviewed-by: roland, thartmann

src/hotspot/share/c1/c1_Compilation.cpp

@@ -397,6 +397,7 @@ int Compilation::compile_java_method() {
     PhaseTraceTime timeit(_t_buildIR);
     build_hir();
   }
+  CHECK_BAILOUT_(no_frame_size);
   if (BailoutAfterHIR) {
     BAILOUT_("Bailing out because of -XX:+BailoutAfterHIR", no_frame_size);
   }
@@ -446,13 +447,13 @@ void Compilation::install_code(int frame_size) {
 
 void Compilation::compile_method() {
 
-  CompilationMemoryStatisticMark cmsm(env()->task()->directive());
-
   {
     PhaseTraceTime timeit(_t_setup);
 
     // setup compilation
     initialize();
+    CHECK_BAILOUT();
+
   }
 
   if (!method()->can_be_compiled()) {
@@ -605,6 +606,9 @@ Compilation::Compilation(AbstractCompiler* compiler, ciEnv* env, ciMethod* metho
     _cfg_printer_output = new CFGPrinterOutput(this);
   }
 #endif
+
+  CompilationMemoryStatisticMark cmsm(directive);
+
   compile_method();
   if (bailed_out()) {
     _env->record_method_not_compilable(bailout_msg());

src/hotspot/share/c1/c1_Compilation.hpp

@@ -85,6 +85,7 @@ class Compilation: public StackObj {
   bool               _has_monitors; // Fastpath monitors detection for Continuations
   bool               _install_code;
   const char*        _bailout_msg;
+  bool               _oom;
   ExceptionInfoList* _exception_info_list;
   ExceptionHandlerTable _exception_handler_table;
   ImplicitExceptionTable _implicit_exception_table;
@@ -203,6 +204,10 @@ class Compilation: public StackObj {
   }
 #endif // PRODUCT
 
+  // MemLimit handling
+  bool oom() const { return _oom; }
+  void set_oom() { _oom = true; }
+
   // error handling
   void bailout(const char* msg);
   bool bailed_out() const                        { return _bailout_msg != nullptr; }

src/hotspot/share/ci/ciEnv.hpp

@@ -319,10 +319,10 @@ public:
 
   // This is true if the compilation is not going to produce code.
   // (It is reasonable to retry failed compilations.)
-  bool failing() { return _failure_reason != nullptr; }
+  bool failing() const { return _failure_reason != nullptr; }
 
   // Reason this compilation is failing, such as "too many basic blocks".
-  const char* failure_reason() { return _failure_reason; }
+  const char* failure_reason() const { return _failure_reason; }
 
   // Return state of appropriate compatibility
   int compilable() { return _compilable; }

src/hotspot/share/compiler/compilationMemoryStatistic.cpp

@@ -26,6 +26,9 @@
 #include "precompiled.hpp"
 #include "logging/log.hpp"
 #include "logging/logStream.hpp"
+#ifdef COMPILER1
+#include "c1/c1_Compilation.hpp"
+#endif
 #include "compiler/abstractCompiler.hpp"
 #include "compiler/compilationMemoryStatistic.hpp"
 #include "compiler/compilerDirectives.hpp"
@@ -42,15 +45,16 @@
 #endif
 #include "runtime/mutexLocker.hpp"
 #include "runtime/os.hpp"
+#include "utilities/debug.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/ostream.hpp"
 #include "utilities/quickSort.hpp"
 #include "utilities/resourceHash.hpp"
 
-
 ArenaStatCounter::ArenaStatCounter() :
   _current(0), _start(0), _peak(0),
   _na(0), _ra(0),
+  _limit(0), _hit_limit(false),
   _na_at_peak(0), _ra_at_peak(0), _live_nodes_at_peak(0)
 {}
 
@@ -58,8 +62,15 @@ size_t ArenaStatCounter::peak_since_start() const {
   return _peak > _start ? _peak - _start : 0;
 }
 
-void ArenaStatCounter::start() {
+void ArenaStatCounter::start(size_t limit) {
   _peak = _start = _current;
+  _limit = limit;
+  _hit_limit = false;
+}
+
+void ArenaStatCounter::end(){
+  _limit = 0;
+  _hit_limit = false;
 }
 
 void ArenaStatCounter::update_c2_node_count() {
@@ -104,6 +115,10 @@ bool ArenaStatCounter::account(ssize_t delta, int tag) {
     _ra_at_peak = _ra;
     update_c2_node_count();
     rc = true;
+    // Did we hit the memory limit?
+    if (!_hit_limit && _limit > 0 && peak_since_start() > _limit) {
+      _hit_limit = true;
+    }
   }
   return rc;
 }
@@ -125,7 +140,8 @@ class FullMethodName {
 
 public:
 
-  FullMethodName(Symbol* k, Symbol* m, Symbol* s) : _k(k), _m(m), _s(s) {}
+  FullMethodName(const Method* m) :
+    _k(m->klass_name()), _m(m->name()), _s(m->signature()) {};
   FullMethodName(const FullMethodName& o) : _k(o._k), _m(o._m), _s(o._s) {}
 
   void make_permanent() {
@@ -173,13 +189,15 @@ class MemStatEntry : public CHeapObj<mtInternal> {
   size_t _na_at_peak;
   size_t _ra_at_peak;
   unsigned _live_nodes_at_peak;
+  const char* _result;
 
 public:
 
   MemStatEntry(FullMethodName method)
     : _method(method), _comptype(compiler_c1),
       _time(0), _num_recomp(0), _thread(nullptr),
-      _total(0), _na_at_peak(0), _ra_at_peak(0), _live_nodes_at_peak(0) {
+      _total(0), _na_at_peak(0), _ra_at_peak(0), _live_nodes_at_peak(0),
+      _result(nullptr) {
   }
 
   void set_comptype(CompilerType comptype) { _comptype = comptype; }
@@ -192,6 +210,8 @@ public:
   void set_ra_at_peak(size_t n) { _ra_at_peak = n; }
   void set_live_nodes_at_peak(unsigned n) { _live_nodes_at_peak = n; }
 
+  void set_result(const char* s) { _result = s; }
+
   size_t total() const { return _total; }
 
   static void print_legend(outputStream* st) {
@@ -199,7 +219,8 @@ public:
     st->print_cr("  total  : memory allocated via arenas while compiling");
     st->print_cr("  NA     : ...how much in node arenas (if c2)");
     st->print_cr("  RA     : ...how much in resource areas");
-    st->print_cr("  #nodes : ...how many nodes (if c2)");
+    st->print_cr("  result : Result: 'ok' finished successfully, 'oom' hit memory limit, 'err' compilation failed");
+    st->print_cr("  #nodes : ...how many nodes (c2 only)");
     st->print_cr("  time   : time of last compilation (sec)");
     st->print_cr("  type   : compiler type");
     st->print_cr("  #rc    : how often recompiled");
@@ -207,7 +228,7 @@ public:
   }
 
   static void print_header(outputStream* st) {
-    st->print_cr("total     NA        RA        #nodes  time    type  #rc thread              method");
+    st->print_cr("total     NA        RA        result  #nodes  time    type  #rc thread              method");
   }
 
   void print_on(outputStream* st, bool human_readable) const {
@@ -237,6 +258,10 @@ public:
     }
     col += 10; st->fill_to(col);
 
+    // result?
+    st->print("%s ", _result ? _result : "");
+    col += 8; st->fill_to(col);
+
     // Number of Nodes when memory peaked
     st->print("%u ", _live_nodes_at_peak);
     col += 8; st->fill_to(col);
@@ -281,7 +306,7 @@ public:
 
   void add(const FullMethodName& fmn, CompilerType comptype,
            size_t total, size_t na_at_peak, size_t ra_at_peak,
-           unsigned live_nodes_at_peak) {
+           unsigned live_nodes_at_peak, const char* result) {
     assert_lock_strong(NMTCompilationCostHistory_lock);
 
     MemStatEntry** pe = get(fmn);
@@ -302,6 +327,7 @@ public:
     e->set_na_at_peak(na_at_peak);
     e->set_ra_at_peak(ra_at_peak);
     e->set_live_nodes_at_peak(live_nodes_at_peak);
+    e->set_result(result);
   }
 
   // Returns a C-heap-allocated SortMe array containing all entries from the table,
@@ -341,20 +367,21 @@ void CompilationMemoryStatistic::initialize() {
   log_info(compilation, alloc)("Compilation memory statistic enabled");
 }
 
-void CompilationMemoryStatistic::on_start_compilation() {
+void CompilationMemoryStatistic::on_start_compilation(const DirectiveSet* directive) {
   assert(enabled(), "Not enabled?");
-  Thread::current()->as_Compiler_thread()->arena_stat()->start();
+  const size_t limit = directive->mem_limit();
+  Thread::current()->as_Compiler_thread()->arena_stat()->start(limit);
 }
 
 void CompilationMemoryStatistic::on_end_compilation() {
   assert(enabled(), "Not enabled?");
   ResourceMark rm;
   CompilerThread* const th = Thread::current()->as_Compiler_thread();
-  const ArenaStatCounter* const arena_stat = th->arena_stat();
+  ArenaStatCounter* const arena_stat = th->arena_stat();
   const CompilerType ct = th->task()->compiler()->type();
 
   const Method* const m = th->task()->method();
-  FullMethodName fmn(m->klass_name(), m->name(), m->signature());
+  FullMethodName fmn(m);
   fmn.make_permanent();
 
   const DirectiveSet* directive = th->task()->directive();
@@ -368,6 +395,20 @@ void CompilationMemoryStatistic::on_end_compilation() {
     arena_stat->print_on(tty);
     tty->cr();
   }
+
+  // Store result
+  // For this to work, we must call on_end_compilation() at a point where
+  // Compile|Compilation already handed over the failure string to ciEnv,
+  // but ciEnv must still be alive.
+  const char* result = "ok"; // ok
+  const ciEnv* const env = th->env();
+  if (env) {
+    const char* const failure_reason = env->failure_reason();
+    if (failure_reason != nullptr) {
+      result = (failure_reason == failure_reason_memlimit()) ? "oom" : "err";
+    }
+  }
+
   {
     MutexLocker ml(NMTCompilationCostHistory_lock, Mutex::_no_safepoint_check_flag);
     assert(_the_table != nullptr, "not initialized");
@@ -376,14 +417,105 @@ void CompilationMemoryStatistic::on_end_compilation() {
                     arena_stat->peak_since_start(), // total
                     arena_stat->na_at_peak(),
                     arena_stat->ra_at_peak(),
-                    arena_stat->live_nodes_at_peak());
+                    arena_stat->live_nodes_at_peak(),
+                    result);
+  }
+
+  arena_stat->end(); // reset things
+}
+
+static void inform_compilation_about_oom(CompilerType ct) {
+  // Inform C1 or C2 that an OOM happened. They will take delayed action
+  // and abort the compilation in progress. Note that this is not instantaneous,
+  // since the compiler has to actively bailout, which may take a while, during
+  // which memory usage may rise further.
+  //
+  // The mechanism differs slightly between C1 and C2:
+  // - With C1, we directly set the bailout string, which will cause C1 to
+  //   bailout at the typical BAILOUT places.
+  // - With C2, the corresponding mechanism would be the failure string; but
+  //   bailout paths in C2 are not complete and therefore it is dangerous to
+  //   set the failure string at - for C2 - seemingly random places. Instead,
+  //   upon OOM C2 sets the failure string next time it checks the node limit.
+  if (ciEnv::current() != nullptr) {
+    void* compiler_data = ciEnv::current()->compiler_data();
+#ifdef COMPILER1
+    if (ct == compiler_c1) {
+      Compilation* C = static_cast<Compilation*>(compiler_data);
+      if (C != nullptr) {
+        C->bailout(CompilationMemoryStatistic::failure_reason_memlimit());
+        C->set_oom();
+      }
+    }
+#endif
+#ifdef COMPILER2
+    if (ct == compiler_c2) {
+      Compile* C = static_cast<Compile*>(compiler_data);
+      if (C != nullptr) {
+        C->set_oom();
+      }
+    }
+#endif // COMPILER2
   }
 }
 
 void CompilationMemoryStatistic::on_arena_change(ssize_t diff, const Arena* arena) {
   assert(enabled(), "Not enabled?");
   CompilerThread* const th = Thread::current()->as_Compiler_thread();
-  th->arena_stat()->account(diff, (int)arena->get_tag());
+
+  ArenaStatCounter* const arena_stat = th->arena_stat();
+  bool hit_limit_before = arena_stat->hit_limit();
+
+  if (arena_stat->account(diff, (int)arena->get_tag())) { // new peak?
+
+    // Limit handling
+    if (arena_stat->hit_limit()) {
+
+      char name[1024] = "";
+      bool print = false;
+      bool crash = false;
+      CompilerType ct = compiler_none;
+
+      // get some more info
+      const CompileTask* task = th->task();
+      if (task != nullptr) {
+        ct = task->compiler()->type();
+        const DirectiveSet* directive = task->directive();
+        print = directive->should_print_memstat();
+        crash = directive->should_crash_at_mem_limit();
+        const Method* m = th->task()->method();
+        if (m != nullptr) {
+          FullMethodName(m).as_C_string(name, sizeof(name));
+        }
+      }
+
+      char message[1024] = "";
+
+      // build up message if we need it later
+      if (print || crash) {
+        stringStream ss(message, sizeof(message));
+        if (ct != compiler_none && name[0] != '\0') {
+          ss.print("%s %s: ", compilertype2name(ct), name);
+        }
+        ss.print("Hit MemLimit %s (limit: %zu now: %zu)",
+                 (hit_limit_before ? "again" : ""),
+                 arena_stat->limit(), arena_stat->peak_since_start());
+      }
+
+      // log if needed
+      if (print) {
+        tty->print_raw(message);
+        tty->cr();
+      }
+
+      // Crash out if needed
+      if (crash) {
+        report_fatal(OOM_HOTSPOT_ARENA, __FILE__, __LINE__, "%s", message);
+      } else {
+        inform_compilation_about_oom(ct);
+      }
+    }
+  }
 }
 
 static inline ssize_t diff_entries_by_size(const MemStatEntry* e1, const MemStatEntry* e2) {
@@ -438,10 +570,15 @@ void CompilationMemoryStatistic::print_all_by_size(outputStream* st, bool human_
   FREE_C_HEAP_ARRAY(Entry, filtered);
 }
 
+const char* CompilationMemoryStatistic::failure_reason_memlimit() {
+  static const char* const s = "hit memory limit while compiling";
+  return s;
+}
+
 CompilationMemoryStatisticMark::CompilationMemoryStatisticMark(const DirectiveSet* directive)
   : _active(directive->should_collect_memstat()) {
   if (_active) {
-    CompilationMemoryStatistic::on_start_compilation();
+    CompilationMemoryStatistic::on_start_compilation(directive);
   }
 }
 CompilationMemoryStatisticMark::~CompilationMemoryStatisticMark() {

src/hotspot/share/compiler/compilationMemoryStatistic.hpp

@@ -47,6 +47,9 @@ class ArenaStatCounter : public CHeapObj<mtCompiler> {
   size_t _na;
   // Current bytes used for resource areas
   size_t _ra;
+  // MemLimit handling
+  size_t _limit;
+  bool _hit_limit;
 
   // Peak composition:
   // Size of node arena when total peaked (c2 only)
@@ -69,15 +72,20 @@ public:
   size_t ra_at_peak() const { return _ra_at_peak; }
   unsigned live_nodes_at_peak() const { return _live_nodes_at_peak; }
 
-  // Mark the start of a compilation.
+  // Mark the start and end of a compilation.
-  void start();
+  void start(size_t limit);
+  void end();
 
   // Account an arena allocation or de-allocation.
   // Returns true if new peak reached
   bool account(ssize_t delta, int tag);
 
   void set_live_nodes_at_peak(unsigned i) { _live_nodes_at_peak = i; }
+
   void print_on(outputStream* st) const;
+
+  size_t limit() const              { return _limit; }
+  bool   hit_limit() const          { return _hit_limit; }
 };
 
 class CompilationMemoryStatistic : public AllStatic {
@@ -86,10 +94,16 @@ public:
   static void initialize();
   // true if CollectMemStat or PrintMemStat has been enabled for any method
   static bool enabled() { return _enabled; }
-  static void on_start_compilation();
+  static void on_start_compilation(const DirectiveSet* directive);
+
+  // Called at end of compilation. Records the arena usage peak. Also takes over
+  // status information from ciEnv (compilation failed, oom'ed or went okay). ciEnv::_failure_reason
+  // must be set at this point (so place CompilationMemoryStatisticMark correctly).
   static void on_end_compilation();
   static void on_arena_change(ssize_t diff, const Arena* arena);
   static void print_all_by_size(outputStream* st, bool human_readable, size_t minsize);
+  // For compilers
+  static const char* failure_reason_memlimit();
 };
 
 // RAII object to wrap one compilation

src/hotspot/share/compiler/compileTask.cpp

@@ -142,7 +142,7 @@ void CompileTask::initialize(int compile_id,
 /**
  * Returns the compiler for this task.
  */
-AbstractCompiler* CompileTask::compiler() {
+AbstractCompiler* CompileTask::compiler() const {
   return CompileBroker::compiler(_comp_level);
 }
 

src/hotspot/share/compiler/compileTask.hpp

@@ -180,7 +180,7 @@ class CompileTask : public CHeapObj<mtCompiler> {
   int          comp_level()                      { return _comp_level;}
   void         set_comp_level(int comp_level)    { _comp_level = comp_level;}
 
-  AbstractCompiler* compiler();
+  AbstractCompiler* compiler() const;
   CompileTask*      select_for_compilation();
 
   int          num_inlined_bytecodes() const     { return _num_inlined_bytecodes; }

src/hotspot/share/compiler/compilerDirectives.cpp

@@ -203,13 +203,24 @@ bool DirectiveSet::is_c2(CompilerDirectives* directive) const {
 }
 
 bool DirectiveSet::should_collect_memstat() const {
-  return MemStatOption > 0;
+  // MemLimit requires the memory statistic to be active
+  return MemStatOption > 0 || MemLimitOption != 0;
 }
 
 bool DirectiveSet::should_print_memstat() const {
   return MemStatOption == (uintx)MemStatAction::print;
 }
 
+size_t DirectiveSet::mem_limit() const {
+  return MemLimitOption < 0 ? -MemLimitOption : MemLimitOption;
+}
+
+bool DirectiveSet::should_crash_at_mem_limit() const {
+  // The sign encodes the action to be taken when reaching
+  // the memory limit (+ stop - crash)
+  return MemLimitOption < 0;
+}
+
 // In the list of Control/disabled intrinsics, the ID of the control intrinsics can separated:
 // - by ',' (if -XX:Control/DisableIntrinsic is used once when invoking the VM) or
 // - by '\n' (if -XX:Control/DisableIntrinsic is used multiple times when invoking the VM) or

src/hotspot/share/compiler/compilerDirectives.hpp

@@ -41,6 +41,7 @@
     cflags(BreakAtExecute,          bool, false, BreakAtExecute) \
     cflags(BreakAtCompile,          bool, false, BreakAtCompile) \
     cflags(Log,                     bool, LogCompilation, Unknown) \
+    cflags(MemLimit,                intx, 0, MemLimit) \
     cflags(MemStat,                 uintx, 0, MemStat) \
     cflags(PrintAssembly,           bool, PrintAssembly, PrintAssembly) \
     cflags(PrintCompilation,        bool, PrintCompilation, PrintCompilation) \
@@ -150,6 +151,8 @@ public:
   bool is_c2(CompilerDirectives* directive) const;
   bool should_collect_memstat() const;
   bool should_print_memstat() const;
+  size_t mem_limit() const;
+  bool should_crash_at_mem_limit() const; // true: crash false: stop compilation
 
   typedef enum {
 #define enum_of_flags(name, type, dvalue, cc_flag) name##Index,

src/hotspot/share/compiler/compilerOracle.cpp

@@ -39,6 +39,7 @@
 #include "runtime/handles.inline.hpp"
 #include "runtime/jniHandles.hpp"
 #include "runtime/os.hpp"
+#include "utilities/parseInteger.hpp"
 
 static const char* optiontype_names[] = {
 #define enum_of_types(type, name) name,
@@ -459,7 +460,7 @@ bool CompilerOracle::should_print_methods() {
 
 // Tells whether there are any methods to collect memory statistics for
 bool CompilerOracle::should_collect_memstat() {
-  return has_command(CompileCommand::MemStat);
+  return has_command(CompileCommand::MemStat) || has_command(CompileCommand::MemLimit);
 }
 
 bool CompilerOracle::should_print_final_memstat_report() {
@@ -634,6 +635,44 @@ void skip_comma(char* &line) {
   }
 }
 
+static bool parseMemLimit(const char* line, intx& value, int& bytes_read, char* errorbuf, const int buf_size) {
+  // Format:
+  // "<memory size>['~' <suboption>]"
+  // <memory size> can have units, e.g. M
+  // <suboption> one of "crash" "stop", if omitted, "stop" is implied.
+  //
+  // Examples:
+  // -XX:CompileCommand='memlimit,*.*,20m'
+  // -XX:CompileCommand='memlimit,*.*,20m~stop'
+  // -XX:CompileCommand='memlimit,Option::toString,1m~crash'
+  //
+  // The resulting intx carries the size and whether we are to stop or crash:
+  // - neg. value means crash
+  // - pos. value (default) means stop
+  size_t s = 0;
+  char* end;
+  if (!parse_integer<size_t>(line, &end, &s)) {
+    jio_snprintf(errorbuf, buf_size, "MemLimit: invalid value");
+  }
+  bytes_read = (int)(end - line);
+
+  intx v = (intx)s;
+  if ((*end) != '\0') {
+    if (strncasecmp(end, "~crash", 6) == 0) {
+      v = -v;
+      bytes_read += 6;
+    } else if (strncasecmp(end, "~stop", 5) == 0) {
+      // ok, this is the default
+      bytes_read += 5;
+    } else {
+      jio_snprintf(errorbuf, buf_size, "MemLimit: invalid option");
+      return true;
+    }
+  }
+  value = v;
+  return true;
+}
+
 static bool parseEnumValueAsUintx(enum CompileCommand option, const char* line, uintx& value, int& bytes_read, char* errorbuf, const int buf_size) {
   if (option == CompileCommand::MemStat) {
     if (strncasecmp(line, "collect", 7) == 0) {
@@ -659,7 +698,13 @@ static void scan_value(enum OptionType type, char* line, int& total_bytes_read,
   total_bytes_read += skipped;
   if (type == OptionType::Intx) {
     intx value;
-    if (sscanf(line, "" INTX_FORMAT "%n", &value, &bytes_read) == 1) {
+    // Special handling for memlimit
+    bool success = (option == CompileCommand::MemLimit) && parseMemLimit(line, value, bytes_read, errorbuf, buf_size);
+    if (!success) {
+      // Is it a raw number?
+      success = sscanf(line, "" INTX_FORMAT "%n", &value, &bytes_read) == 1;
+    }
+    if (success) {
       total_bytes_read += bytes_read;
       line += bytes_read;
       register_command(matcher, option, value);

src/hotspot/share/compiler/compilerOracle.hpp

@@ -57,6 +57,7 @@ class methodHandle;
   option(Break, "break", Bool) \
   option(BreakAtExecute, "BreakAtExecute", Bool) \
   option(BreakAtCompile, "BreakAtCompile", Bool) \
+  option(MemLimit, "MemLimit", Intx) \
   option(MemStat, "MemStat", Uintx) \
   option(PrintAssembly, "PrintAssembly", Bool) \
   option(PrintCompilation, "PrintCompilation", Bool) \

src/hotspot/share/opto/compile.cpp

@@ -29,6 +29,7 @@
 #include "classfile/javaClasses.hpp"
 #include "code/exceptionHandlerTable.hpp"
 #include "code/nmethod.hpp"
+#include "compiler/compilationMemoryStatistic.hpp"
 #include "compiler/compileBroker.hpp"
 #include "compiler/compileLog.hpp"
 #include "compiler/disassembler.hpp"
@@ -661,6 +662,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci,
                   _vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr),
                   _late_inlines_pos(0),
                   _number_of_mh_late_inlines(0),
+                  _oom(false),
                   _print_inlining_stream(new (mtCompiler) stringStream()),
                   _print_inlining_list(nullptr),
                   _print_inlining_idx(0),
@@ -938,6 +940,7 @@ Compile::Compile( ciEnv* ci_env,
     _types(nullptr),
     _node_hash(nullptr),
     _number_of_mh_late_inlines(0),
+    _oom(false),
     _print_inlining_stream(new (mtCompiler) stringStream()),
     _print_inlining_list(nullptr),
     _print_inlining_idx(0),
@@ -5261,3 +5264,6 @@ Node* Compile::narrow_value(BasicType bt, Node* value, const Type* type, PhaseGV
   return result;
 }
 
+void Compile::record_method_not_compilable_oom() {
+  record_method_not_compilable(CompilationMemoryStatistic::failure_reason_memlimit());
+}

src/hotspot/share/opto/compile.hpp

@@ -460,6 +460,9 @@ private:
   int                           _late_inlines_pos;    // Where in the queue should the next late inlining candidate go (emulate depth first inlining)
   uint                          _number_of_mh_late_inlines; // number of method handle late inlining still pending
 
+  // "MemLimit" directive was specified and the memory limit was hit during compilation
+  bool                          _oom;
+
   // Inlining may not happen in parse order which would make
   // PrintInlining output confusing. Keep track of PrintInlining
   // pieces in order.
@@ -503,6 +506,8 @@ private:
   void log_late_inline_failure(CallGenerator* cg, const char* msg);
   DEBUG_ONLY(bool _exception_backedge;)
 
+  void record_method_not_compilable_oom();
+
  public:
 
   void* barrier_set_state() const { return _barrier_set_state; }
@@ -814,6 +819,10 @@ private:
     record_failure(reason);
   }
   bool check_node_count(uint margin, const char* reason) {
+    if (oom()) {
+      record_method_not_compilable_oom();
+      return true;
+    }
     if (live_nodes() + margin > max_node_limit()) {
       record_method_not_compilable(reason);
       return true;
@@ -821,6 +830,8 @@ private:
       return false;
     }
   }
+  bool oom() const { return _oom; }
+  void set_oom()   { _oom = true; }
 
   // Node management
   uint         unique() const              { return _unique; }

src/hotspot/share/utilities/debug.hpp

@@ -250,7 +250,8 @@ enum VMErrorType : unsigned int {
   OOM_MALLOC_ERROR = 0xe0000001,
   OOM_MMAP_ERROR   = 0xe0000002,
   OOM_MPROTECT_ERROR = 0xe0000003,
-  OOM_JAVA_HEAP_FATAL = 0xe0000004
+  OOM_JAVA_HEAP_FATAL = 0xe0000004,
+  OOM_HOTSPOT_ARENA = 0xe0000005
 };
 
 // error reporting helper functions

test/hotspot/jtreg/compiler/print/CompileCommandMemLimit.java

@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test id=c1_crash
+ * @requires vm.compiler1.enabled
+ * @summary Checks that -XX:CompileCommand=MemLimit,...,crash causes compilation to crash
+ * @library /test/lib
+ * @run driver compiler.print.CompileCommandMemLimit crash false
+ */
+
+/*
+ * @test id=c2_crash
+ * @requires vm.compiler2.enabled
+ * @summary Checks that -XX:CompileCommand=MemLimit,...,crash causes compilation to crash
+ * @library /test/lib
+ * @run driver compiler.print.CompileCommandMemLimit crash true
+ */
+
+/*
+ * @test id=c1_stop
+ * @requires vm.compiler1.enabled
+ * @summary Checks that -XX:CompileCommand=MemLimit,...,stop causes compilation to stop
+ * @library /test/lib
+ * @run driver compiler.print.CompileCommandMemLimit stop false
+ */
+
+/*
+ * @test id=c2_stop
+ * @requires vm.compiler2.enabled
+ * @summary Checks that -XX:CompileCommand=MemLimit,...,stop causes compilation to stop
+ * @library /test/lib
+ * @run driver compiler.print.CompileCommandMemLimit stop true
+ */
+
+package compiler.print;
+
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CompileCommandMemLimit {
+
+    final static String METHOD1 = "method1";
+    final static String METHOD2 = "method2";
+
+    static boolean c2;
+    static boolean test_crash;
+
+    public static void main(String[] args) throws Exception {
+        switch (args[0]) {
+            case "crash" : test_crash = true; break;
+            case "stop" : test_crash = false; break;
+            default: throw new RuntimeException("invalid argument");
+        }
+        c2 = Boolean.parseBoolean(args[1]);
+        test(METHOD1, METHOD2);
+        test(METHOD2, METHOD1);
+    }
+
+    private static void test(String include, String exclude) throws Exception {
+
+        // A method that is known to cost compilers a bit of memory to compile
+
+        List<String> options = new ArrayList<String>();
+        options.add("-Xcomp");
+        options.add("-XX:-Inline");
+        options.add("-XX:CompileCommand=compileonly," + getTestClass() + "::*");
+        // We pass a very small size to guarantee the crash
+        options.add("-XX:CompileCommand=MemStat," + getTestMethod(include) + ",print");
+        options.add("-XX:CompileCommand=MemLimit," + getTestMethod(include) + ",4k" + (test_crash ? "~crash" : ""));
+        if (c2) {
+            options.add("-XX:-TieredCompilation");
+        } else {
+            options.add("-XX:TieredStopAtLevel=1");
+        }
+        options.add(getTestClass());
+
+        OutputAnalyzer oa = ProcessTools.executeTestJvm(options);
+
+        oa.reportDiagnosticSummary();
+
+        String expectedNameIncl = getTestMethod(include)
+                .replace('.', '/')
+                .replace("$", "\\$");
+        String expectedNameExcl = getTestMethod(exclude)
+                .replace('.', '/')
+                .replace("$", "\\$");
+
+        String ct = c2 ? "c2" : "c1";
+
+        if (test_crash) {
+            oa.shouldNotHaveExitValue(0);
+            oa.shouldMatch("# *Internal Error.*");
+            oa.shouldMatch("# *fatal error: " + ct + " *" + expectedNameIncl + ".*: Hit MemLimit .*limit: 4096.*");
+            oa.shouldNotMatch(".*" + expectedNameExcl + ".*");
+        } else {
+            // Should see trace output when methods are compiled
+            oa.shouldHaveExitValue(0)
+                    .shouldMatch(".*" + expectedNameIncl + ".*")
+                    .shouldNotMatch(".*" + expectedNameExcl + ".*");
+
+            // Expect this log line
+            oa.shouldMatch(".*" + expectedNameIncl + ".*Hit MemLimit.*");
+
+            // Expect final output to contain "oom"
+            oa.shouldMatch(".*oom.*" + expectedNameIncl + ".*");
+        }
+    }
+
+    // Test class that is invoked by the sub process
+    public static String getTestClass() {
+        return TestMain.class.getName();
+    }
+
+    public static String getTestMethod(String method) {
+        return getTestClass() + "::" + method;
+    }
+
+    public static class TestMain {
+        public static void main(String[] args) {
+            method1();
+            method2();
+        }
+
+        static long method1() {
+            return System.currentTimeMillis();
+        }
+        static void method2() {}
+    }
+}
+

test/hotspot/jtreg/compiler/print/CompileCommandPrintMemStat.java

@@ -73,11 +73,11 @@ public class CompileCommandPrintMemStat {
 
         // Should see final report
         // Looks like this:
-        // total     NA        RA        #nodes  time    type  #rc thread              method
+        // total     NA        RA        result  #nodes  time    type  #rc thread              method
-        // 621832    0         589104    0       0,025   c1    1   0x00007f5ccc1951a0  java/util/zip/ZipFile$Source::checkAndAddEntry((II)I)
+        // 211488    66440     77624     ok      13      0.057   c2    2   0x00007fb49428db70  compiler/print/CompileCommandPrintMemStat$TestMain::method1(()V)
         oa.shouldMatch("total.*method");
-        oa.shouldMatch("\\d+ +\\d+ +\\d+ +\\d+.*" + expectedNameIncl + ".*");
+        oa.shouldMatch("\\d+ +\\d+ +\\d+ +\\S+ +\\d+.*" + expectedNameIncl + ".*");
-        oa.shouldNotMatch("\\d+ +\\d+ +\\d+ +\\d+.*" + expectedNameExcl + ".*");
+        oa.shouldNotMatch("\\d+ +\\d+ +\\d+ +\\S+ +\\d+.*" + expectedNameExcl + ".*");
     }
 
     // Test class that is invoked by the sub process

test/hotspot/jtreg/serviceability/dcmd/compiler/CompilerMemoryStatisticTest.java

@@ -59,9 +59,9 @@ public class CompilerMemoryStatisticTest {
         out.shouldHaveExitValue(0);
 
         // Looks like this:
-        // total     NA        RA        #nodes  time    type  #rc thread              method
+        // total     NA        RA        result  #nodes  time    type  #rc thread              method
-        // 621832    0         589104    0       0,025   c1    1   0x00007f5ccc1951a0  java/util/zip/ZipFile$Source.checkAndAddEntry((II)I)
+        // 211488    66440     77624     ok      13      0.057   c2    2   0x00007fb49428db70  compiler/print/CompileCommandPrintMemStat$TestMain::method1(()V)
         out.shouldMatch("total.*method");
-        out.shouldMatch("\\d+ +\\d+ +\\d+ +\\d+.*java.*\\(.*\\)");
+        out.shouldMatch("\\d+ +\\d+ +\\d+ +\\S+ +\\d+.*java.*\\(.*\\)");
     }
 }