8240227: Loop predicates should be copied to unswitched loops

Copy loop range check predicates to unswitched loops and update their control edges.

Reviewed-by: kvn, neliasso, thartmann, roland

src/hotspot/share/opto/graphKit.cpp

@@ -3920,7 +3920,7 @@ InitializeNode* AllocateNode::initialization() {
 //----------------------------- loop predicates ---------------------------
 
 //------------------------------add_predicate_impl----------------------------
-void GraphKit::add_predicate_impl(Deoptimization::DeoptReason reason, int nargs) {
+void GraphKit::add_empty_predicate_impl(Deoptimization::DeoptReason reason, int nargs) {
   // Too many traps seen?
   if (too_many_traps(reason)) {
 #ifdef ASSERT
@@ -3954,15 +3954,18 @@ void GraphKit::add_predicate_impl(Deoptimization::DeoptReason reason, int nargs)
 }
 
 //------------------------------add_predicate---------------------------------
-void GraphKit::add_predicate(int nargs) {
+void GraphKit::add_empty_predicates(int nargs) {
+  // These loop predicates remain empty. All concrete loop predicates are inserted above the corresponding
+  // empty loop predicate later by 'PhaseIdealLoop::create_new_if_for_predicate'. All concrete loop predicates of
+  // a specific kind (normal, profile or limit check) share the same uncommon trap as the empty loop predicate.
   if (UseLoopPredicate) {
-    add_predicate_impl(Deoptimization::Reason_predicate, nargs);
+    add_empty_predicate_impl(Deoptimization::Reason_predicate, nargs);
   }
   if (UseProfiledLoopPredicate) {
-    add_predicate_impl(Deoptimization::Reason_profile_predicate, nargs);
+    add_empty_predicate_impl(Deoptimization::Reason_profile_predicate, nargs);
   }
   // loop's limit check predicate should be near the loop.
-  add_predicate_impl(Deoptimization::Reason_loop_limit_check, nargs);
+  add_empty_predicate_impl(Deoptimization::Reason_loop_limit_check, nargs);
 }
 
 void GraphKit::sync_kit(IdealKit& ideal) {
@@ -4086,7 +4089,7 @@ void GraphKit::inflate_string_slow(Node* src, Node* dst, Node* start, Node* coun
    *   dst[i_char++] = (char)(src[i_byte] & 0xff);
    * }
    */
-  add_predicate();
+  add_empty_predicates();
   RegionNode* head = new RegionNode(3);
   head->init_req(1, control());
   gvn().set_type(head, Type::CONTROL);

src/hotspot/share/opto/graphKit.hpp

@@ -878,9 +878,8 @@ class GraphKit : public Phase {
     return iff;
   }
 
-  // Insert a loop predicate into the graph
-  void add_predicate(int nargs = 0);
-  void add_predicate_impl(Deoptimization::DeoptReason reason, int nargs);
+  void add_empty_predicates(int nargs = 0);
+  void add_empty_predicate_impl(Deoptimization::DeoptReason reason, int nargs);
 
   Node* make_constant_from_field(ciField* field, Node* obj);
 };

src/hotspot/share/opto/idealKit.cpp

@@ -168,7 +168,7 @@ void IdealKit::loop(GraphKit* gkit, int nargs, IdealVariable& iv, Node* init, Bo
     // Sync IdealKit and graphKit.
     gkit->sync_kit(*this);
     // Add loop predicate.
-    gkit->add_predicate(nargs);
+    gkit->add_empty_predicates(nargs);
     // Update IdealKit memory.
     sync_kit(gkit);
   }

src/hotspot/share/opto/loopPredicate.cpp

@@ -91,11 +91,13 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred)
 //
 //
 // We will create a region to guard the uct call if there is no one there.
-// The true projection (if_cont) of the new_iff is returned.
-// This code is also used to clone predicates to cloned loops.
+// The continuation projection (if_cont) of the new_iff is returned which
+// is by default a true projection if 'if_cont_is_true_proj' is true.
+// Otherwise, the continuation projection is set up to be the false
+// projection. This code is also used to clone predicates to cloned loops.
 ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
                                                       Deoptimization::DeoptReason reason,
-                                                      int opcode) {
+                                                      int opcode, bool if_cont_is_true_proj) {
   assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
   IfNode* iff = cont_proj->in(0)->as_If();
 
@@ -145,8 +147,16 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node*
     new_iff = new RangeCheckNode(entry, iff->in(1), iff->_prob, iff->_fcnt);
   }
   register_control(new_iff, lp, entry);
-  Node *if_cont = new IfTrueNode(new_iff);
-  Node *if_uct  = new IfFalseNode(new_iff);
+  Node* if_cont;
+  Node* if_uct;
+  if (if_cont_is_true_proj) {
+    if_cont = new IfTrueNode(new_iff);
+    if_uct  = new IfFalseNode(new_iff);
+  } else {
+    if_uct  = new IfTrueNode(new_iff);
+    if_cont = new IfFalseNode(new_iff);
+  }
+
   if (cont_proj->is_IfFalse()) {
     // Swap
     Node* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
@@ -189,124 +199,125 @@ ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node*
   return if_cont->as_Proj();
 }
 
-//------------------------------create_new_if_for_predicate------------------------
-// Create a new if below new_entry for the predicate to be cloned (IGVN optimization)
-ProjNode* PhaseIterGVN::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
-                                                    Deoptimization::DeoptReason reason,
-                                                    int opcode) {
-  assert(new_entry != 0, "only used for clone predicate");
-  assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
-  IfNode* iff = cont_proj->in(0)->as_If();
-
-  ProjNode *uncommon_proj = iff->proj_out(1 - cont_proj->_con);
-  Node     *rgn   = uncommon_proj->unique_ctrl_out();
-  assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
-
-  uint proj_index = 1; // region's edge corresponding to uncommon_proj
-  if (!rgn->is_Region()) { // create a region to guard the call
-    assert(rgn->is_Call(), "must be call uct");
-    CallNode* call = rgn->as_Call();
-    rgn = new RegionNode(1);
-    register_new_node_with_optimizer(rgn);
-    rgn->add_req(uncommon_proj);
-    replace_input_of(call, 0, rgn);
-  } else {
-    // Find region's edge corresponding to uncommon_proj
-    for (; proj_index < rgn->req(); proj_index++)
-      if (rgn->in(proj_index) == uncommon_proj) break;
-    assert(proj_index < rgn->req(), "sanity");
-  }
-
-  // Create new_iff in new location.
-  IfNode* new_iff = NULL;
-  if (opcode == Op_If) {
-    new_iff = new IfNode(new_entry, iff->in(1), iff->_prob, iff->_fcnt);
-  } else {
-    assert(opcode == Op_RangeCheck, "no other if variant here");
-    new_iff = new RangeCheckNode(new_entry, iff->in(1), iff->_prob, iff->_fcnt);
-  }
-
-  register_new_node_with_optimizer(new_iff);
-  Node *if_cont = new IfTrueNode(new_iff);
-  Node *if_uct  = new IfFalseNode(new_iff);
-  if (cont_proj->is_IfFalse()) {
-    // Swap
-    Node* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
-  }
-  register_new_node_with_optimizer(if_cont);
-  register_new_node_with_optimizer(if_uct);
-
-  // if_uct to rgn
-  hash_delete(rgn);
-  rgn->add_req(if_uct);
-
-  // If rgn has phis add corresponding new edges which has the same
-  // value as on original uncommon_proj pass.
-  assert(rgn->in(rgn->req() -1) == if_uct, "new edge should be last");
-  bool has_phi = false;
-  for (DUIterator_Fast imax, i = rgn->fast_outs(imax); i < imax; i++) {
-    Node* use = rgn->fast_out(i);
-    if (use->is_Phi() && use->outcnt() > 0) {
-      rehash_node_delayed(use);
-      use->add_req(use->in(proj_index));
-      has_phi = true;
-    }
-  }
-  assert(!has_phi || rgn->req() > 3, "no phis when region is created");
-
-  return if_cont->as_Proj();
-}
-
 //--------------------------clone_predicate-----------------------
-ProjNode* PhaseIdealLoop::clone_predicate(ProjNode* predicate_proj, Node* new_entry,
-                                          Deoptimization::DeoptReason reason,
-                                          PhaseIdealLoop* loop_phase,
-                                          PhaseIterGVN* igvn) {
-  ProjNode* new_predicate_proj;
-  if (loop_phase != NULL) {
-    new_predicate_proj = loop_phase->create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If);
-  } else {
-    new_predicate_proj =       igvn->create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If);
-  }
+ProjNode* PhaseIdealLoop::clone_loop_predicate(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason,
+                                               bool is_slow_loop, uint idx_before_clone, Node_List &old_new) {
+  ProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If);
   IfNode* iff = new_predicate_proj->in(0)->as_If();
   Node* ctrl  = iff->in(0);
 
   // Match original condition since predicate's projections could be swapped.
   assert(predicate_proj->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be");
-  Node* opq = new Opaque1Node(igvn->C, predicate_proj->in(0)->in(1)->in(1)->in(1));
-  igvn->C->add_predicate_opaq(opq);
-
+  Node* opq = new Opaque1Node(C, predicate_proj->in(0)->in(1)->in(1)->in(1));
+  C->add_predicate_opaq(opq);
   Node* bol = new Conv2BNode(opq);
-  if (loop_phase != NULL) {
-    loop_phase->register_new_node(opq, ctrl);
-    loop_phase->register_new_node(bol, ctrl);
-  } else {
-    igvn->register_new_node_with_optimizer(opq);
-    igvn->register_new_node_with_optimizer(bol);
-  }
-  igvn->hash_delete(iff);
+  register_new_node(opq, ctrl);
+  register_new_node(bol, ctrl);
+  _igvn.hash_delete(iff);
   iff->set_req(1, bol);
+  clone_concrete_loop_predicates(reason, predicate_proj, new_predicate_proj, is_slow_loop, idx_before_clone, old_new);
   return new_predicate_proj;
 }
 
+// Clones all non-empty loop predicates (including skeleton predicates) starting at 'old_predicate_proj' to 'new_predicate_proj'
+// and rewires the control edges of data nodes in the loop to the old predicates to the new cloned predicates.
+void PhaseIdealLoop::clone_concrete_loop_predicates(Deoptimization::DeoptReason reason, ProjNode* old_predicate_proj,
+                                                    ProjNode* new_predicate_proj, bool is_slow_loop, uint idx_before_clone,
+                                                    Node_List &old_new) {
+  assert(old_predicate_proj->is_Proj(), "must be projection");
+  IfNode* iff = old_predicate_proj->in(0)->as_If();
+  ProjNode* uncommon_proj = iff->proj_out(1 - old_predicate_proj->as_Proj()->_con);
+  Node* rgn = uncommon_proj->unique_ctrl_out();
+  assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
+  assert(iff->in(1)->in(1)->Opcode() == Op_Opaque1, "unexpected predicate shape");
+  Node* predicate = iff->in(0);
+  Node* current_proj = old_predicate_proj;
+  Node* prev_proj = current_proj;
+  Unique_Node_List list;
+  while (predicate != NULL && predicate->is_Proj() && predicate->in(0)->is_If()) {
+    iff = predicate->in(0)->as_If();
+    uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con);
+    if (uncommon_proj->unique_ctrl_out() != rgn)
+      break;
+    if (iff->is_RangeCheck()) {
+      // Only need to clone range check predicates as those can be changed and duplicated by inserting pre/main/post loops
+      // and doing loop unrolling. Push the original predicates on a list to later process them in reverse order to keep the
+      // original predicate order.
+      list.push(predicate);
+#ifdef ASSERT
+    } else {
+      // All other If predicates should not have a control input to nodes belonging to the original loop
+      for (DUIterator i = predicate->outs(); predicate->has_out(i); i++) {
+        Node* old_node = predicate->out(i);
+        Node* new_node = old_new[old_node->_idx];
+        if (!old_node->is_CFG() && new_node != NULL && old_node->_idx >= idx_before_clone) {
+          assert(false, "should not be part of the original loop");
+        }
+      }
+#endif
+    }
+    predicate = predicate->in(0)->in(0);
+  }
+
+  // Process in reverse order such that 'create_new_if_for_predicate' can be used and the original order is maintained
+  for (int i = list.size()-1; i >= 0; i--) {
+    predicate = list.at(i);
+    assert(predicate->in(0)->is_If(), "must be If node");
+    iff = predicate->in(0)->as_If();
+    assert(predicate->is_Proj() && predicate->as_Proj()->is_IfProj() && iff->is_RangeCheck(), "predicate must be a projection of a range check");
+    IfProjNode* predicate_proj = predicate->as_IfProj();
+
+    // cloned_proj is the same type of projection as the original predicate projection (IfTrue or IfFalse)
+    ProjNode* cloned_proj = create_new_if_for_predicate(new_predicate_proj, NULL, reason, Op_RangeCheck, predicate_proj->is_IfTrue());
+
+    // Replace bool input by input from original predicate
+    _igvn.replace_input_of(cloned_proj->in(0), 1, iff->in(1));
+
+    if (is_slow_loop) {
+      for (DUIterator i = predicate->outs(); predicate->has_out(i); i++) {
+        Node* slow_node = predicate->out(i);
+        Node* fast_node = old_new[slow_node->_idx];
+        if (!slow_node->is_CFG() && fast_node != NULL && slow_node->_idx > idx_before_clone) {
+          // 'slow_node' is a data node and part of the slow loop. This is a clone of the fast loop node
+          // which was temporarily added below in order to verify that 'slow_node' is a clone of 'fast_node'.
+          // Update the control input and reset the mapping for 'slow_node' back to NULL.
+          _igvn.replace_input_of(slow_node, 0, cloned_proj);
+          old_new.map(slow_node->_idx, NULL);
+          --i;
+        }
+        assert(slow_node->_idx <= idx_before_clone || old_new[slow_node->_idx] == NULL, "mapping of cloned nodes must be null");
+      }
+
+      // Let old predicates before unswitched loops which were cloned die if all their control edges were rewired
+      // to the cloned predicates in the unswitched loops.
+      if (predicate->outcnt() == 1) {
+        _igvn.replace_input_of(iff, 1, _igvn.intcon(predicate_proj->_con));
+      }
+    } else {
+      // Fast loop
+      for (DUIterator i = predicate->outs(); predicate->has_out(i); i++) {
+        Node* fast_node = predicate->out(i);
+        Node* slow_node = old_new[fast_node->_idx];
+        if (!fast_node->is_CFG() && slow_node != NULL && slow_node->_idx > idx_before_clone) {
+          // 'fast_node' is a data node and part of the fast loop. Add the clone of the fast loop node
+          // to the 'old_new' mapping in order to verify later when cloning the predicates for the slow loop
+          // that 'slow_node' is a clone of 'fast_node'. Update the control input for 'fast_node'.
+          _igvn.replace_input_of(fast_node, 0, cloned_proj);
+          assert(old_new[slow_node->_idx] == NULL, "mapping must be null for cloned nodes");
+          old_new.map(slow_node->_idx, fast_node);
+          --i;
+        }
+      }
+    }
+  }
+}
 
 //--------------------------clone_loop_predicates-----------------------
-// Interface from IGVN
-Node* PhaseIterGVN::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) {
-  return PhaseIdealLoop::clone_loop_predicates(old_entry, new_entry, clone_limit_check, NULL, this);
-}
-
-// Interface from PhaseIdealLoop
-Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check) {
-  return clone_loop_predicates(old_entry, new_entry, clone_limit_check, this, &this->_igvn);
-}
-
-// Clone loop predicates to cloned loops (peeled, unswitched, split_if).
-Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry,
-                                            bool clone_limit_check,
-                                            PhaseIdealLoop* loop_phase,
-                                            PhaseIterGVN* igvn) {
+// Clone loop predicates to cloned loops when unswitching a loop.
+Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check,
+                                            bool is_slow_loop, uint idx_before_clone, Node_List &old_new) {
 #ifdef ASSERT
+  assert(LoopUnswitching, "sanity - only called when unswitching a loop");
   if (new_entry == NULL || !(new_entry->is_Proj() || new_entry->is_Region() || new_entry->is_SafePoint())) {
     if (new_entry != NULL)
       new_entry->dump();
@@ -333,9 +344,8 @@ Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry,
   }
   if (predicate_proj != NULL) { // right pattern that can be used by loop predication
     // clone predicate
-    new_entry = clone_predicate(predicate_proj, new_entry,
-                                Deoptimization::Reason_predicate,
-                                loop_phase, igvn);
+    new_entry = clone_loop_predicate(predicate_proj, new_entry, Deoptimization::Reason_predicate, is_slow_loop,
+                                     idx_before_clone, old_new);
     assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone predicate");
     if (TraceLoopPredicate) {
       tty->print("Loop Predicate cloned: ");
@@ -344,9 +354,8 @@ Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry,
   }
   if (profile_predicate_proj != NULL) { // right pattern that can be used by loop predication
     // clone predicate
-    new_entry = clone_predicate(profile_predicate_proj, new_entry,
-                                Deoptimization::Reason_profile_predicate,
-                                loop_phase, igvn);
+    new_entry = clone_loop_predicate(profile_predicate_proj, new_entry,Deoptimization::Reason_profile_predicate,
+                                     is_slow_loop, idx_before_clone, old_new);
     assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone predicate");
     if (TraceLoopPredicate) {
       tty->print("Loop Predicate cloned: ");
@@ -357,9 +366,8 @@ Node* PhaseIdealLoop::clone_loop_predicates(Node* old_entry, Node* new_entry,
     // Clone loop limit check last to insert it before loop.
     // Don't clone a limit check which was already finalized
     // for this counted loop (only one limit check is needed).
-    new_entry = clone_predicate(limit_check_proj, new_entry,
-                                Deoptimization::Reason_loop_limit_check,
-                                loop_phase, igvn);
+    new_entry = clone_loop_predicate(limit_check_proj, new_entry, Deoptimization::Reason_loop_limit_check,
+                                     is_slow_loop, idx_before_clone, old_new);
     assert(new_entry != NULL && new_entry->is_Proj(), "IfTrue or IfFalse after clone limit check");
     if (TraceLoopLimitCheck) {
       tty->print("Loop Limit Check cloned: ");

src/hotspot/share/opto/loopUnswitch.cpp

@@ -150,29 +150,32 @@ void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) {
   ProjNode* proj_true = create_slow_version_of_loop(loop, old_new, unswitch_iff->Opcode(), CloneIncludesStripMined);
 
 #ifdef ASSERT
-  Node* uniqc = proj_true->unique_ctrl_out();
+  assert(proj_true->is_IfTrue(), "must be true projection");
   entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
   Node* predicate = find_predicate(entry);
-  if (predicate != NULL) {
-    entry = skip_loop_predicates(entry);
-  }
-  if (predicate != NULL && UseProfiledLoopPredicate) {
-    // We may have two predicates, find first.
-    Node* n = find_predicate(entry);
-    if (n != NULL) {
-      predicate = n;
-      entry = skip_loop_predicates(entry);
+  if (predicate == NULL) {
+    // No empty predicate
+    Node* uniqc = proj_true->unique_ctrl_out();
+    assert((uniqc == head && !head->is_strip_mined()) || (uniqc == head->in(LoopNode::EntryControl)
+           && head->is_strip_mined()), "must hold by construction if no predicates");
+  } else {
+    // There is at least one empty predicate. When calling 'skip_loop_predicates' on each found empty predicate,
+    // we should end up at 'proj_true'.
+    Node* proj_before_first_empty_predicate = skip_loop_predicates(entry);
+    if (UseProfiledLoopPredicate) {
+      predicate = find_predicate(proj_before_first_empty_predicate);
+      if (predicate != NULL) {
+        proj_before_first_empty_predicate = skip_loop_predicates(predicate);
+      }
     }
+    if (UseLoopPredicate) {
+      predicate = find_predicate(proj_before_first_empty_predicate);
+      if (predicate != NULL) {
+        proj_before_first_empty_predicate = skip_loop_predicates(predicate);
+      }
+    }
+    assert(proj_true == proj_before_first_empty_predicate, "must hold by construction if at least one predicate");
   }
-  if (predicate != NULL && UseLoopPredicate) {
-    entry = find_predicate(entry);
-    if (entry != NULL) predicate = entry;
-  }
-  if (predicate != NULL) predicate = predicate->in(0);
-  assert(proj_true->is_IfTrue() &&
-         (predicate == NULL && uniqc == head && !head->is_strip_mined() ||
-          predicate == NULL && uniqc == head->in(LoopNode::EntryControl) && head->is_strip_mined() ||
-          predicate != NULL && uniqc == predicate), "by construction");
 #endif
   // Increment unswitch count
   LoopNode* head_clone = old_new[head->_idx]->as_Loop();
@@ -272,6 +275,7 @@ ProjNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop,
   register_node(iffast, outer_loop, iff, dom_depth(iff));
   ProjNode* ifslow = new IfFalseNode(iff);
   register_node(ifslow, outer_loop, iff, dom_depth(iff));
+  uint idx_before_clone = Compile::current()->unique();
 
   // Clone the loop body.  The clone becomes the slow loop.  The
   // original pre-header will (illegally) have 3 control users
@@ -280,10 +284,10 @@ ProjNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop,
   assert(old_new[head->_idx]->is_Loop(), "" );
 
   // Fast (true) control
-  Node* iffast_pred = clone_loop_predicates(entry, iffast, !counted_loop);
+  Node* iffast_pred = clone_loop_predicates(entry, iffast, !counted_loop, false, idx_before_clone, old_new);
 
   // Slow (false) control
-  Node* ifslow_pred = clone_loop_predicates(entry, ifslow, !counted_loop);
+  Node* ifslow_pred = clone_loop_predicates(entry, ifslow, !counted_loop, true, idx_before_clone, old_new);
 
   Node* l = head->skip_strip_mined();
   _igvn.replace_input_of(l, LoopNode::EntryControl, iffast_pred);

src/hotspot/share/opto/loopnode.hpp

@@ -1127,23 +1127,11 @@ public:
   bool is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset, int depth = 0);
 
   // Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted
-  ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
-                                        Deoptimization::DeoptReason reason,
-                                        int opcode);
+  ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry, Deoptimization::DeoptReason reason,
+                                        int opcode, bool if_cont_is_true_proj = true);
+
   void register_control(Node* n, IdealLoopTree *loop, Node* pred);
 
-  // Clone loop predicates to cloned loops (peeled, unswitched)
-  static ProjNode* clone_predicate(ProjNode* predicate_proj, Node* new_entry,
-                                   Deoptimization::DeoptReason reason,
-                                   PhaseIdealLoop* loop_phase,
-                                   PhaseIterGVN* igvn);
-
-  static Node* clone_loop_predicates(Node* old_entry, Node* new_entry,
-                                         bool clone_limit_check,
-                                         PhaseIdealLoop* loop_phase,
-                                         PhaseIterGVN* igvn);
-  Node* clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check);
-
   static Node* skip_all_loop_predicates(Node* entry);
   static Node* skip_loop_predicates(Node* entry);
 
@@ -1421,6 +1409,15 @@ private:
     _nodes_required = UINT_MAX;
   }
 
+  // Clone loop predicates to slow and fast loop when unswitching a loop
+  Node* clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check, bool is_slow_loop,
+                              uint idx_before_clone, Node_List &old_new);
+  ProjNode* clone_loop_predicate(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason,
+                                 bool is_slow_loop, uint idx_before_clone, Node_List &old_new);
+  void clone_concrete_loop_predicates(Deoptimization::DeoptReason reason, ProjNode* old_predicate_proj,
+                                      ProjNode* new_predicate_proj, bool is_slow_loop,
+                                      uint idx_before_clone, Node_List &old_new);
+
   bool _created_loop_node;
 
 public:

src/hotspot/share/opto/parse1.cpp

@@ -672,7 +672,7 @@ void Parse::do_all_blocks() {
             // Need correct bci for predicate.
             // It is fine to set it here since do_one_block() will set it anyway.
             set_parse_bci(block->start());
-            add_predicate();
+            add_empty_predicates();
           }
           // Add new region for back branches.
           int edges = block->pred_count() - block->preds_parsed() + 1; // +1 for original region
@@ -1654,7 +1654,7 @@ void Parse::merge_common(Parse::Block* target, int pnum) {
         if (target->start() == 0) {
           // Add loop predicate for the special case when
           // there are backbranches to the method entry.
-          add_predicate();
+          add_empty_predicates();
         }
       }
       // Add a Region to start the new basic block.  Phis will be added

src/hotspot/share/opto/parse2.cpp

@@ -1645,7 +1645,7 @@ void Parse::maybe_add_predicate_after_if(Block* path) {
     // Add predicates at bci of if dominating the loop so traps can be
     // recorded on the if's profile data
     int bc_depth = repush_if_args();
-    add_predicate();
+    add_empty_predicates();
     dec_sp(bc_depth);
     path->set_has_predicates();
   }

src/hotspot/share/opto/phaseX.hpp

@@ -553,13 +553,6 @@ public:
     _delay_transform = delay;
   }
 
-  // Clone loop predicates. Defined in loopTransform.cpp.
-  Node* clone_loop_predicates(Node* old_entry, Node* new_entry, bool clone_limit_check);
-  // Create a new if below new_entry for the predicate to be cloned
-  ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
-                                        Deoptimization::DeoptReason reason,
-                                        int opcode);
-
   void remove_speculative_types();
   void check_no_speculative_types() {
     _table.check_no_speculative_types();

src/hotspot/share/opto/stringopts.cpp

@@ -1200,7 +1200,7 @@ Node* PhaseStringOpts::int_stringSize(GraphKit& kit, Node* arg) {
     //     return i+1;
 
     // Add loop predicate first.
-    kit.add_predicate();
+    kit.add_empty_predicates();
 
     RegionNode *loop = new RegionNode(3);
     loop->init_req(1, kit.control());
@@ -1276,7 +1276,7 @@ void PhaseStringOpts::getChars(GraphKit& kit, Node* arg, Node* dst_array, BasicT
   // }
 
   // Add loop predicate first.
-  kit.add_predicate();
+  kit.add_empty_predicates();
 
   RegionNode* head = new RegionNode(3);
   head->init_req(1, kit.control());

test/hotspot/jtreg/compiler/loopopts/TestUnswitchOverunrolling.java

@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8240227
+ * @summary Test different cases of overunrolling the main loop of unswitched loops (pre/main/post) which are then not entered.
+ * @run main/othervm -XX:CompileCommand=compileonly,compiler.loopopts.TestUnswitchOverunrolling::test*
+ *                   -Xcomp -Xbatch -XX:-TieredCompilation compiler.loopopts.TestUnswitchOverunrolling
+ * @run main/othervm -XX:CompileCommand=compileonly,compiler.loopopts.TestUnswitchOverunrolling::*
+ *                   -Xcomp -Xbatch -XX:-TieredCompilation compiler.loopopts.TestUnswitchOverunrolling
+ */
+
+package compiler.loopopts;
+
+public class TestUnswitchOverunrolling {
+
+    public static int v = 1;
+    public static int w = 2;
+    public static int x = 3;
+    public static int y = 4;
+    public static int z = 5;
+
+    // The inner for-loop is unswitched and pre-main-post loops are created for both unswitched loop versions.
+    // Afterwards, both main loops are over-unrolled and vectorized resulting in a crash in the matcher because
+    // the memory input to a vector is top.
+    public static int test(int[] array) {
+        int result = 0;
+        int[] iArr = new int[8];
+        for (int i = 0; i < array.length; i++) {
+            for (int j = 5; j < i; j++) {
+                if (x == 42) {
+                    y = 34;
+                }
+                iArr[j] += array[j];
+                result += array[j];
+            }
+        }
+        return result;
+    }
+
+    // Test with additional null check predicates for objects
+    public static int test2(int[] array, A a, A a2) {
+        int result = 0;
+        int[] iArr = new int[8];
+        for (int i = 0; i < array.length; i++) {
+            for (int j = 5; j < i; j++) {
+                y = a2.iFld;
+                if (x == 42) {
+                    y = 34;
+                }
+                z = a.iFld;
+                iArr[j] += array[j];
+                result += array[j];
+            }
+        }
+        return result;
+    }
+
+    // Test with two conditions (unswitch twice) and additional null check predicates for objects
+    public static int testUnswitchTwice(int[] array, A a, A a2, A a3) {
+        int result = 0;
+        int[] iArr = new int[8];
+        for (int i = 0; i < array.length; i++) {
+            for (int j = 5; j < i; j++) {
+                y = a2.iFld;
+                if (x == 42) {
+                    y = 34;
+                }
+                z = a.iFld;
+                if (w == 22) {
+                    y = 55;
+                }
+                v = a3.iFld;
+                iArr[j] += array[j];
+                result += array[j];
+            }
+        }
+        return result;
+    }
+
+    // Test with three conditions (unswitch three times) and additional null check predicates for objects
+    public static int testUnswitchThreeTimes(int[] array, A a, A a2, A a3) {
+        int result = 0;
+        int[] iArr = new int[8];
+        for (int i = 0; i < array.length; i++) {
+            for (int j = 5; j < i; j++) {
+                y += a2.iFld;
+                if (x == 42) {
+                    y = 34;
+                }
+                if (w == 22) {
+                    y = 55;
+                }
+                v = a3.iFld;
+                if (z == 75) {
+                    y = 66;
+                }
+                y += a.iFld + a2.iFld + a3.iFld;
+                iArr[j] += array[j];
+                result += array[j];
+            }
+        }
+        return result;
+    }
+
+    public static void main(String args[]) {
+        int[] array = new int[8];
+        for (int i = 0; i < array.length; i++) {
+            array[i] = i + 1;
+        }
+
+        check(test(array), 1, 2, 3, 4, 5);
+        check(test2(array, new A(), new A()), 1, 2, 3, 6, 6);
+        check(testUnswitchTwice(array, new A(), new A(), new A()), 6, 2, 3, 6, 6);
+        check(testUnswitchThreeTimes(array, new A(), new A(), new A()), 6, 2, 3, 78, 6);
+
+        for (int i = 0; i < array.length; i++) {
+            if (array[i] != i + 1) {
+                throw new RuntimeException("array values should not change");
+            }
+        }
+    }
+
+    public static void check(int result, int expV, int expW, int expX, int expY, int expZ) {
+        if (result != 19 || expV != v || expW != w || expX != x || expY != y || expZ != z) {
+            throw new RuntimeException("wrong execution");
+        }
+    }
+}
+
+class A {
+    int iFld = 6;
+}
+