8331168: Introduce PredicateEntryIterator to iterate through predicate entries

Reviewed-by: roland, kvn

src/hotspot/share/opto/loopnode.cpp

@@ -169,10 +169,10 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) {
     return earliest;
   }
 
-  while (1) {
-    Node *next = ctl;
-    // Moving the node out of a loop on the projection of a If
-    // confuses loop predication. So once we hit a Loop in a If branch
+  while (true) {
+    Node* next = ctl;
+    // Moving the node out of a loop on the projection of an If
+    // confuses Loop Predication. So, once we hit a loop in an If branch
     // that doesn't branch to an UNC, we stop. The code that process
     // expensive nodes will notice the loop and skip over it to try to
     // move the node further up.
@@ -6081,20 +6081,27 @@ Node* PhaseIdealLoop::get_late_ctrl_with_anti_dep(LoadNode* n, Node* early, Node
   return LCA;
 }
 
-// true if CFG node d dominates CFG node n
-bool PhaseIdealLoop::is_dominator(Node *d, Node *n) {
-  if (d == n)
+// Is CFG node 'dominator' dominating node 'n'?
+bool PhaseIdealLoop::is_dominator(Node* dominator, Node* n) {
+  if (dominator == n) {
     return true;
-  assert(d->is_CFG() && n->is_CFG(), "must have CFG nodes");
-  uint dd = dom_depth(d);
+  }
+  assert(dominator->is_CFG() && n->is_CFG(), "must have CFG nodes");
+  uint dd = dom_depth(dominator);
   while (dom_depth(n) >= dd) {
-    if (n == d)
+    if (n == dominator) {
       return true;
+    }
     n = idom(n);
   }
   return false;
 }
 
+// Is CFG node 'dominator' strictly dominating node 'n'?
+bool PhaseIdealLoop::is_strict_dominator(Node* dominator, Node* n) {
+  return dominator != n && is_dominator(dominator, n);
+}
+
 //------------------------------dom_lca_for_get_late_ctrl_internal-------------
 // Pair-wise LCA with tags.
 // Tag each index with the node 'tag' currently being processed
@@ -6377,31 +6384,16 @@ void PhaseIdealLoop::build_loop_late_post_work(Node *n, bool pinned) {
 
   if (least != early) {
     // Move the node above predicates as far up as possible so a
-    // following pass of loop predication doesn't hoist a predicate
+    // following pass of Loop Predication doesn't hoist a predicate
     // that depends on it above that node.
-    Node* new_ctrl = least;
-    for (;;) {
-      if (!new_ctrl->is_Proj()) {
+    PredicateEntryIterator predicate_iterator(least);
+    while (predicate_iterator.has_next()) {
+      Node* next_predicate_entry = predicate_iterator.next_entry();
+      if (is_strict_dominator(next_predicate_entry, early)) {
         break;
       }
-      CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern();
-      if (call == nullptr) {
-        break;
-      }
-      int req = call->uncommon_trap_request();
-      Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req);
-      if (trap_reason != Deoptimization::Reason_loop_limit_check &&
-          trap_reason != Deoptimization::Reason_predicate &&
-          trap_reason != Deoptimization::Reason_profile_predicate) {
-        break;
-      }
-      Node* c = new_ctrl->in(0)->in(0);
-      if (is_dominator(c, early) && c != early) {
-        break;
-      }
-      new_ctrl = c;
+      least = next_predicate_entry;
     }
-    least = new_ctrl;
   }
   // Try not to place code on a loop entry projection
   // which can inhibit range check elimination.

src/hotspot/share/opto/loopnode.hpp

@@ -1011,8 +1011,10 @@ public:
     assert(n == find_non_split_ctrl(n), "must return legal ctrl" );
     return n;
   }
-  // true if CFG node d dominates CFG node n
-  bool is_dominator(Node *d, Node *n);
+
+  bool is_dominator(Node* dominator, Node* n);
+  bool is_strict_dominator(Node* dominator, Node* n);
+
   // return get_ctrl for a data node and self(n) for a CFG node
   Node* ctrl_or_self(Node* n) {
     if (has_ctrl(n))

src/hotspot/share/opto/loopopts.cpp

@@ -4294,7 +4294,7 @@ bool PhaseIdealLoop::duplicate_loop_backedge(IdealLoopTree *loop, Node_List &old
     } else {
       wq.push(c->in(0));
     }
-    assert(!is_dominator(c, region) || c == region, "shouldn't go above region");
+    assert(!is_strict_dominator(c, region), "shouldn't go above region");
   }
 
   Node* region_dom = idom(region);

src/hotspot/share/opto/predicates.cpp

@@ -32,29 +32,29 @@
 // (i.e. not belonging to an Initialized Assertion Predicate anymore)
 Node* AssertionPredicatesWithHalt::find_entry(Node* start_proj) {
   Node* entry = start_proj;
-  while (is_assertion_predicate_success_proj(entry)) {
+  while (AssertionPredicateWithHalt::is_predicate(entry)) {
     entry = entry->in(0)->in(0);
   }
   return entry;
 }
 
-bool AssertionPredicatesWithHalt::is_assertion_predicate_success_proj(const Node* predicate_proj) {
-  if (predicate_proj == nullptr || !predicate_proj->is_IfProj() || !predicate_proj->in(0)->is_If()) {
+bool AssertionPredicateWithHalt::is_predicate(const Node* maybe_success_proj) {
+  if (maybe_success_proj == nullptr || !maybe_success_proj->is_IfProj() || !maybe_success_proj->in(0)->is_If()) {
     return false;
   }
-  return has_assertion_predicate_opaque(predicate_proj) && has_halt(predicate_proj);
+  return has_assertion_predicate_opaque(maybe_success_proj) && has_halt(maybe_success_proj);
 }
 
 // Check if the If node of `predicate_proj` has an Opaque4 (Template Assertion Predicate) or an
 // OpaqueInitializedAssertionPredicate (Initialized Assertion Predicate) node as input.
-bool AssertionPredicatesWithHalt::has_assertion_predicate_opaque(const Node* predicate_proj) {
+bool AssertionPredicateWithHalt::has_assertion_predicate_opaque(const Node* predicate_proj) {
   IfNode* iff = predicate_proj->in(0)->as_If();
   Node* bol = iff->in(1);
   return bol->is_Opaque4() || bol->is_OpaqueInitializedAssertionPredicate();
 }
 
 // Check if the other projection (UCT projection) of `success_proj` has a Halt node as output.
-bool AssertionPredicatesWithHalt::has_halt(const Node* success_proj) {
+bool AssertionPredicateWithHalt::has_halt(const Node* success_proj) {
   ProjNode* other_proj = success_proj->as_IfProj()->other_if_proj();
   return other_proj->outcnt() == 1 && other_proj->unique_out()->Opcode() == Op_Halt;
 }
@@ -72,7 +72,15 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p
   return nullptr;
 }
 
-Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) {
+bool ParsePredicate::is_predicate(Node* maybe_success_proj) {
+  if (!maybe_success_proj->is_IfProj()) {
+    return false;
+  }
+  IfNode* if_node = maybe_success_proj->in(0)->as_If();
+  return if_node->is_ParsePredicate();
+}
+
+Deoptimization::DeoptReason RegularPredicateWithUCT::uncommon_trap_reason(IfProjNode* if_proj) {
     CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern();
     if (uct_call == nullptr) {
       return Deoptimization::Reason_none;
@@ -80,8 +88,20 @@ Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* i
     return Deoptimization::trap_request_reason(uct_call->uncommon_trap_request());
 }
 
-bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason) {
-  if (may_be_runtime_predicate_if(node)) {
+bool RegularPredicateWithUCT::is_predicate(Node* maybe_success_proj) {
+  if (may_be_predicate_if(maybe_success_proj)) {
+    IfProjNode* success_proj = maybe_success_proj->as_IfProj();
+    const Deoptimization::DeoptReason deopt_reason = uncommon_trap_reason(success_proj);
+    return (deopt_reason == Deoptimization::Reason_loop_limit_check ||
+            deopt_reason == Deoptimization::Reason_predicate ||
+            deopt_reason == Deoptimization::Reason_profile_predicate);
+  } else {
+    return false;
+  }
+}
+
+bool RegularPredicateWithUCT::is_predicate(Node* node, Deoptimization::DeoptReason deopt_reason) {
+  if (may_be_predicate_if(node)) {
     return deopt_reason == uncommon_trap_reason(node->as_IfProj());
   } else {
     return false;
@@ -89,7 +109,7 @@ bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason d
 }
 
 // A Runtime Predicate must have an If or a RangeCheck node, while the If should not be a zero trip guard check.
-bool RuntimePredicate::may_be_runtime_predicate_if(Node* node) {
+bool RegularPredicateWithUCT::may_be_predicate_if(Node* node) {
   if (node->is_IfProj()) {
     const IfNode* if_node = node->in(0)->as_If();
     const int opcode_if = if_node->Opcode();
@@ -101,6 +121,10 @@ bool RuntimePredicate::may_be_runtime_predicate_if(Node* node) {
   return false;
 }
 
+bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason) {
+  return RegularPredicateWithUCT::is_predicate(node, deopt_reason);
+}
+
 ParsePredicateIterator::ParsePredicateIterator(const Predicates& predicates) : _current_index(0) {
   const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block();
   if (loop_limit_check_predicate_block->has_parse_predicate()) {
@@ -356,3 +380,18 @@ bool TemplateAssertionPredicateExpressionNode::is_in_expression(Node* node) {
 bool TemplateAssertionPredicateExpressionNode::is_template_assertion_predicate(Node* node) {
   return node->is_If() && node->in(1)->is_Opaque4();
 }
+
+// Is current node pointed to by iterator a predicate?
+bool PredicateEntryIterator::has_next() const {
+    return ParsePredicate::is_predicate(_current) ||
+           RegularPredicateWithUCT::is_predicate(_current) ||
+           AssertionPredicateWithHalt::is_predicate(_current);
+}
+
+// Skip the current predicate pointed to by iterator by returning the input into the predicate. This could possibly be
+// a non-predicate node.
+Node* PredicateEntryIterator::next_entry() {
+  assert(has_next(), "current must be predicate");
+  _current = _current->in(0)->in(0);
+  return _current;
+}

src/hotspot/share/opto/predicates.hpp

@@ -26,6 +26,7 @@
 #define SHARE_OPTO_PREDICATES_HPP
 
 #include "opto/cfgnode.hpp"
+#include "opto/connode.hpp"
 #include "opto/opaquenode.hpp"
 
 /*
@@ -199,9 +200,6 @@ class AssertionPredicatesWithHalt : public StackObj {
   Node* _entry;
 
   static Node* find_entry(Node* start_proj);
-  static bool has_assertion_predicate_opaque(const Node* predicate_proj);
-  static bool has_halt(const Node* success_proj);
-  static bool is_assertion_predicate_success_proj(const Node* predicate_proj);
 
  public:
   AssertionPredicatesWithHalt(Node* assertion_predicate_proj) : _entry(find_entry(assertion_predicate_proj)) {}
@@ -213,13 +211,37 @@ class AssertionPredicatesWithHalt : public StackObj {
   }
 };
 
+// Class to represent a single Assertion Predicate with a HaltNode. This could either be:
+// - A Template Assertion Predicate.
+// - An Initialized Assertion Predicate.
+// Note that all other Regular Predicates have an UCT node.
+class AssertionPredicateWithHalt : public StackObj {
+  static bool has_assertion_predicate_opaque(const Node* predicate_proj);
+  static bool has_halt(const Node* success_proj);
+ public:
+  static bool is_predicate(const Node* maybe_success_proj);
+};
+
+// Class to represent a single Regular Predicate with an UCT. This could either be:
+// - A Runtime Predicate
+// - A Template Assertion Predicate
+// Note that all other Regular Predicates have a Halt node.
+class RegularPredicateWithUCT : public StackObj {
+  static Deoptimization::DeoptReason uncommon_trap_reason(IfProjNode* if_proj);
+  static bool may_be_predicate_if(Node* node);
+
+ public:
+  static bool is_predicate(Node* maybe_success_proj);
+  static bool is_predicate(Node* node, Deoptimization::DeoptReason deopt_reason);
+};
+
 // Class to represent a Parse Predicate.
 class ParsePredicate : public StackObj {
   ParsePredicateSuccessProj* _success_proj;
   ParsePredicateNode* _parse_predicate_node;
   Node* _entry;
 
-  IfTrueNode* init_success_proj(const Node* parse_predicate_proj) const {
+  static IfTrueNode* init_success_proj(const Node* parse_predicate_proj) {
     assert(parse_predicate_proj != nullptr, "must not be null");
     return parse_predicate_proj->isa_IfTrue();
   }
@@ -253,13 +275,12 @@ class ParsePredicate : public StackObj {
     assert(is_valid(), "must be valid");
     return _success_proj;
   }
+
+  static bool is_predicate(Node* maybe_success_proj);
 };
 
 // Utility class for queries on Runtime Predicates.
 class RuntimePredicate : public StackObj {
-  static Deoptimization::DeoptReason uncommon_trap_reason(IfProjNode* if_proj);
-  static bool may_be_runtime_predicate_if(Node* node);
-
  public:
   static bool is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason);
 };
@@ -473,4 +494,17 @@ class ParsePredicateIterator : public StackObj {
 
   ParsePredicateNode* next();
 };
+
+// Special predicate iterator that can be used to walk through predicate entries, regardless of whether the predicate
+// belongs to the same loop or not (i.e. leftovers from already folded nodes). The iterator returns the next entry
+// to a predicate.
+class PredicateEntryIterator : public StackObj {
+  Node* _current;
+
+ public:
+  explicit PredicateEntryIterator(Node* start) : _current(start) {};
+
+  bool has_next() const;
+  Node* next_entry();
+};
 #endif // SHARE_OPTO_PREDICATES_HPP