8331344: No compiler replay file with CompilerCommand MemLimit

Reviewed-by: kvn, asmehra

src/hotspot/share/compiler/compilationMemoryStatistic.cpp

@@ -54,7 +54,7 @@
 ArenaStatCounter::ArenaStatCounter() :
   _current(0), _start(0), _peak(0),
   _na(0), _ra(0),
-  _limit(0), _hit_limit(false),
+  _limit(0), _hit_limit(false), _limit_in_process(false),
   _na_at_peak(0), _ra_at_peak(0), _live_nodes_at_peak(0)
 {}
 
@@ -483,18 +483,23 @@ void CompilationMemoryStatistic::on_arena_change(ssize_t diff, const Arena* aren
   CompilerThread* const th = Thread::current()->as_Compiler_thread();
 
   ArenaStatCounter* const arena_stat = th->arena_stat();
+  if (arena_stat->limit_in_process()) {
+    return; // avoid recursion on limit hit
+  }
+
   bool hit_limit_before = arena_stat->hit_limit();
 
   if (arena_stat->account(diff, (int)arena->get_tag())) { // new peak?
 
     // Limit handling
     if (arena_stat->hit_limit()) {
-
       char name[1024] = "";
       bool print = false;
       bool crash = false;
       CompilerType ct = compiler_none;
 
+      arena_stat->set_limit_in_process(true); // prevent recursive limit hits
+
       // get some more info
       const CompileTask* const task = th->task();
       if (task != nullptr) {
@@ -533,6 +538,8 @@ void CompilationMemoryStatistic::on_arena_change(ssize_t diff, const Arena* aren
       } else {
         inform_compilation_about_oom(ct);
       }
+
+      arena_stat->set_limit_in_process(false);
     }
   }
 }

src/hotspot/share/compiler/compilationMemoryStatistic.hpp

@@ -50,6 +50,7 @@ class ArenaStatCounter : public CHeapObj<mtCompiler> {
   // MemLimit handling
   size_t _limit;
   bool _hit_limit;
+  bool _limit_in_process;
 
   // Peak composition:
   // Size of node arena when total peaked (c2 only)
@@ -86,6 +87,9 @@ public:
 
   size_t limit() const              { return _limit; }
   bool   hit_limit() const          { return _hit_limit; }
+  bool   limit_in_process() const     { return _limit_in_process; }
+  void   set_limit_in_process(bool v) { _limit_in_process = v; }
+
 };
 
 class CompilationMemoryStatistic : public AllStatic {

test/hotspot/jtreg/compiler/print/CompileCommandMemLimit.java

@@ -58,6 +58,7 @@ package compiler.print;
 import jdk.test.lib.process.OutputAnalyzer;
 import jdk.test.lib.process.ProcessTools;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -123,6 +124,20 @@ public class CompileCommandMemLimit {
             oa.shouldMatch("# *Internal Error.*");
             oa.shouldMatch("# *fatal error: " + ct + " *" + expectedNameIncl + ".*: Hit MemLimit .*limit: 4096.*");
             oa.shouldNotMatch(".*" + expectedNameExcl + ".*");
+            // Make sure we get a non-zero-sized replay file (JDK-8331314)
+            oa.shouldContain("# Compiler replay data is saved as:");
+            String replayfile = oa.firstMatch("# (\\S+replay_pid\\d+\\.log)", 1);
+            if (replayfile == null) {
+                throw new RuntimeException("Found no replay file in output");
+            }
+            File f = new File(replayfile);
+            if (!f.exists()) {
+                throw new RuntimeException("Replayfile " + replayfile + " not found");
+            }
+            if (f.length() == 0) {
+                throw new RuntimeException("Replayfile " + replayfile + " has size 0");
+            }
+
         } else {
             // Should see trace output when methods are compiled
             oa.shouldHaveExitValue(0)