8239072: subtype check macro node causes node budget to be exhausted

Reviewed-by: vlivanov, kvn

src/hotspot/share/opto/compile.cpp

@@ -4224,3 +4224,20 @@ void CloneMap::dump(node_idx_t key) const {
     ni.dump();
   }
 }
+
+// Move Allocate nodes to the start of the list
+void Compile::sort_macro_nodes() {
+  int count = macro_count();
+  int allocates = 0;
+  for (int i = 0; i < count; i++) {
+    Node* n = macro_node(i);
+    if (n->is_Allocate()) {
+      if (i != allocates) {
+        Node* tmp = macro_node(allocates);
+        _macro_nodes->at_put(allocates, n);
+        _macro_nodes->at_put(i, tmp);
+      }
+      allocates++;
+    }
+  }
+}

src/hotspot/share/opto/compile.hpp

@@ -718,6 +718,8 @@ class Compile : public Phase {
   int   opaque4_count()       const { return _opaque4_nodes->length(); }
   void  remove_opaque4_nodes(PhaseIterGVN &igvn);
 
+  void sort_macro_nodes();
+
   // remove the opaque nodes that protect the predicates so that the unused checks and
   // uncommon traps will be eliminated from the graph.
   void cleanup_loop_predicates(PhaseIterGVN &igvn);

src/hotspot/share/opto/macro.cpp

@@ -2658,12 +2658,6 @@ bool PhaseMacroExpand::expand_macro_nodes() {
   // Last attempt to eliminate macro nodes.
   eliminate_macro_nodes();
 
-  // Make sure expansion will not cause node limit to be exceeded.
-  // Worst case is a macro node gets expanded into about 200 nodes.
-  // Allow 50% more for optimization.
-  if (C->check_node_count(C->macro_count() * 300, "out of nodes before macro expansion" ) )
-    return true;
-
   // Eliminate Opaque and LoopLimit nodes. Do it after all loop optimizations.
   bool progress = true;
   while (progress) {
@@ -2719,17 +2713,44 @@ bool PhaseMacroExpand::expand_macro_nodes() {
     }
   }
 
+  // Clean up the graph so we're less likely to hit the maximum node
+  // limit
+  _igvn.set_delay_transform(false);
+  _igvn.optimize();
+  if (C->failing())  return true;
+  _igvn.set_delay_transform(true);
+
+
+  // Because we run IGVN after each expansion, some macro nodes may go
+  // dead and be removed from the list as we iterate over it. Move
+  // Allocate nodes (processed in a second pass) at the beginning of
+  // the list and then iterate from the last element of the list until
+  // an Allocate node is seen. This is robust to random deletion in
+  // the list due to nodes going dead.
+  C->sort_macro_nodes();
+
   // expand arraycopy "macro" nodes first
   // For ReduceBulkZeroing, we must first process all arraycopy nodes
   // before the allocate nodes are expanded.
-  for (int i = C->macro_count(); i > 0; i--) {
-    Node* n = C->macro_node(i-1);
+  while (C->macro_count() > 0) {
+    int macro_count = C->macro_count();
+    Node * n = C->macro_node(macro_count-1);
     assert(n->is_macro(), "only macro nodes expected here");
     if (_igvn.type(n) == Type::TOP || (n->in(0) != NULL && n->in(0)->is_top())) {
       // node is unreachable, so don't try to expand it
       C->remove_macro_node(n);
       continue;
     }
+    if (n->is_Allocate()) {
+      break;
+    }
+    // Make sure expansion will not cause node limit to be exceeded.
+    // Worst case is a macro node gets expanded into about 200 nodes.
+    // Allow 50% more for optimization.
+    if (C->check_node_count(300, "out of nodes before macro expansion")) {
+      return true;
+    }
+
     debug_only(int old_macro_count = C->macro_count(););
     switch (n->class_id()) {
     case Node::Class_Lock:
@@ -2748,18 +2769,23 @@ bool PhaseMacroExpand::expand_macro_nodes() {
       expand_subtypecheck_node(n->as_SubTypeCheck());
       assert(C->macro_count() == (old_macro_count - 1), "expansion must have deleted one node from macro list");
       break;
+    default:
+      assert(false, "unknown node type in macro list");
     }
+    assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
     if (C->failing())  return true;
+
+    // Clean up the graph so we're less likely to hit the maximum node
+    // limit
+    _igvn.set_delay_transform(false);
+    _igvn.optimize();
+    if (C->failing())  return true;
+    _igvn.set_delay_transform(true);
   }
 
   // All nodes except Allocate nodes are expanded now. There could be
   // new optimization opportunities (such as folding newly created
   // load from a just allocated object). Run IGVN.
-  _igvn.set_delay_transform(false);
-  _igvn.optimize();
-  if (C->failing())  return true;
-
-  _igvn.set_delay_transform(true);
 
   // expand "macro" nodes
   // nodes are removed from the macro list as they are processed
@@ -2772,6 +2798,12 @@ bool PhaseMacroExpand::expand_macro_nodes() {
       C->remove_macro_node(n);
       continue;
     }
+    // Make sure expansion will not cause node limit to be exceeded.
+    // Worst case is a macro node gets expanded into about 200 nodes.
+    // Allow 50% more for optimization.
+    if (C->check_node_count(300, "out of nodes before macro expansion")) {
+      return true;
+    }
     switch (n->class_id()) {
     case Node::Class_Allocate:
       expand_allocate(n->as_Allocate());
@@ -2784,10 +2816,15 @@ bool PhaseMacroExpand::expand_macro_nodes() {
     }
     assert(C->macro_count() < macro_count, "must have deleted a node from macro list");
     if (C->failing())  return true;
+
+    // Clean up the graph so we're less likely to hit the maximum node
+    // limit
+    _igvn.set_delay_transform(false);
+    _igvn.optimize();
+    if (C->failing())  return true;
+    _igvn.set_delay_transform(true);
   }
 
   _igvn.set_delay_transform(false);
-  _igvn.optimize();
-  if (C->failing())  return true;
   return false;
 }