8148754: C2 loop unrolling fails due to unexpected graph shape

Check if graph shape is appropriate for optimization, bail out optimization if not.

Reviewed-by: kvn, twisti, shade, dnsimon

hotspot/src/share/vm/opto/loopTransform.cpp

@@ -1453,20 +1453,14 @@ void PhaseIdealLoop::do_unroll( IdealLoopTree *loop, Node_List &old_new, bool ad
   Node *opaq = NULL;
   if (adjust_min_trip) {       // If not maximally unrolling, need adjustment
     // Search for zero-trip guard.
-    assert( loop_head->is_main_loop(), "" );
-    assert( ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "" );
-    Node *iff = ctrl->in(0);
-    assert( iff->Opcode() == Op_If, "" );
-    Node *bol = iff->in(1);
-    assert( bol->Opcode() == Op_Bool, "" );
-    Node *cmp = bol->in(1);
-    assert( cmp->Opcode() == Op_CmpI, "" );
-    opaq = cmp->in(2);
-    // Occasionally it's possible for a zero-trip guard Opaque1 node to be
-    // optimized away and then another round of loop opts attempted.
-    // We can not optimize this particular loop in that case.
-    if (opaq->Opcode() != Op_Opaque1)
-      return; // Cannot find zero-trip guard!  Bail out!
+
+    // Check the shape of the graph at the loop entry. If an inappropriate
+    // graph shape is encountered, the compiler bails out loop unrolling;
+    // compilation of the method will still succeed.
+    if (!is_canonical_main_loop_entry(loop_head)) {
+      return;
+    }
+    opaq = ctrl->in(0)->in(1)->in(1)->in(2);
     // Zero-trip test uses an 'opaque' node which is not shared.
     assert(opaq->outcnt() == 1 && opaq->in(1) == limit, "");
   }
@@ -2109,7 +2103,6 @@ void PhaseIdealLoop::do_range_check( IdealLoopTree *loop, Node_List &old_new ) {
 #endif
   assert(RangeCheckElimination, "");
   CountedLoopNode *cl = loop->_head->as_CountedLoop();
-  assert(cl->is_main_loop(), "");
 
   // protect against stride not being a constant
   if (!cl->stride_is_con())
@@ -2121,20 +2114,17 @@ void PhaseIdealLoop::do_range_check( IdealLoopTree *loop, Node_List &old_new ) {
   // to not ever trip end tests
   Node *main_limit = cl->limit();
 
+  // Check graph shape. Cannot optimize a loop if zero-trip
+  // Opaque1 node is optimized away and then another round
+  // of loop opts attempted.
+  if (!is_canonical_main_loop_entry(cl)) {
+    return;
+  }
+
   // Need to find the main-loop zero-trip guard
   Node *ctrl  = cl->in(LoopNode::EntryControl);
-  assert(ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "");
   Node *iffm = ctrl->in(0);
-  assert(iffm->Opcode() == Op_If, "");
-  Node *bolzm = iffm->in(1);
-  assert(bolzm->Opcode() == Op_Bool, "");
-  Node *cmpzm = bolzm->in(1);
-  assert(cmpzm->is_Cmp(), "");
-  Node *opqzm = cmpzm->in(2);
-  // Can not optimize a loop if zero-trip Opaque1 node is optimized
-  // away and then another round of loop opts attempted.
-  if (opqzm->Opcode() != Op_Opaque1)
-    return;
+  Node *opqzm = iffm->in(1)->in(1)->in(2);
   assert(opqzm->in(1) == main_limit, "do not understand situation");
 
   // Find the pre-loop limit; we will expand its iterations to

hotspot/src/share/vm/opto/loopnode.cpp

@@ -3275,6 +3275,41 @@ Node* PhaseIdealLoop::compute_lca_of_uses(Node* n, Node* early, bool verify) {
   return LCA;
 }
 
+// Check the shape of the graph at the loop entry. In some cases,
+// the shape of the graph does not match the shape outlined below.
+// That is caused by the Opaque1 node "protecting" the shape of
+// the graph being removed by, for example, the IGVN performed
+// in PhaseIdealLoop::build_and_optimize().
+//
+// After the Opaque1 node has been removed, optimizations (e.g., split-if,
+// loop unswitching, and IGVN, or a combination of them) can freely change
+// the graph's shape. As a result, the graph shape outlined below cannot
+// be guaranteed anymore.
+bool PhaseIdealLoop::is_canonical_main_loop_entry(CountedLoopNode* cl) {
+  assert(cl->is_main_loop(), "check should be applied to main loops");
+  Node* ctrl = cl->in(LoopNode::EntryControl);
+  if (ctrl == NULL || (!ctrl->is_IfTrue() && !ctrl->is_IfFalse())) {
+    return false;
+  }
+  Node* iffm = ctrl->in(0);
+  if (iffm == NULL || !iffm->is_If()) {
+    return false;
+  }
+  Node* bolzm = iffm->in(1);
+  if (bolzm == NULL || !bolzm->is_Bool()) {
+    return false;
+  }
+  Node* cmpzm = bolzm->in(1);
+  if (cmpzm == NULL || !cmpzm->is_Cmp()) {
+    return false;
+  }
+  Node* opqzm = cmpzm->in(2);
+  if (opqzm == NULL || opqzm->Opcode() != Op_Opaque1) {
+    return false;
+  }
+  return true;
+}
+
 //------------------------------get_late_ctrl----------------------------------
 // Compute latest legal control.
 Node *PhaseIdealLoop::get_late_ctrl( Node *n, Node *early ) {

hotspot/src/share/vm/opto/loopnode.hpp

@@ -656,6 +656,9 @@ class PhaseIdealLoop : public PhaseTransform {
   bool cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop);
 
 public:
+
+  static bool is_canonical_main_loop_entry(CountedLoopNode* cl);
+
   bool has_node( Node* n ) const {
     guarantee(n != NULL, "No Node.");
     return _nodes[n->_idx] != NULL;

hotspot/src/share/vm/opto/superword.cpp

@@ -3074,21 +3074,13 @@ void SuperWord::align_initial_loop_index(MemNode* align_to_ref) {
 //----------------------------get_pre_loop_end---------------------------
 // Find pre loop end from main loop.  Returns null if none.
 CountedLoopEndNode* SuperWord::get_pre_loop_end(CountedLoopNode* cl) {
-  Node* ctrl = cl->in(LoopNode::EntryControl);
-  if (!ctrl->is_IfTrue() && !ctrl->is_IfFalse()) return NULL;
-  Node* iffm = ctrl->in(0);
-  if (!iffm->is_If()) return NULL;
-  Node* bolzm = iffm->in(1);
-  if (!bolzm->is_Bool()) return NULL;
-  Node* cmpzm = bolzm->in(1);
-  if (!cmpzm->is_Cmp()) return NULL;
-  Node* opqzm = cmpzm->in(2);
-  // Can not optimize a loop if zero-trip Opaque1 node is optimized
-  // away and then another round of loop opts attempted.
-  if (opqzm->Opcode() != Op_Opaque1) {
+  // The loop cannot be optimized if the graph shape at
+  // the loop entry is inappropriate.
+  if (!PhaseIdealLoop::is_canonical_main_loop_entry(cl)) {
     return NULL;
   }
-  Node* p_f = iffm->in(0);
+
+  Node* p_f = cl->in(LoopNode::EntryControl)->in(0)->in(0);
   if (!p_f->is_IfFalse()) return NULL;
   if (!p_f->in(0)->is_CountedLoopEnd()) return NULL;
   CountedLoopEndNode* pre_end = p_f->in(0)->as_CountedLoopEnd();