From 48621ae193ef70b2fae4dcb7ddc524f349beb131 Mon Sep 17 00:00:00 2001 From: Christian Hagedorn Date: Wed, 19 Jun 2024 06:45:04 +0000 Subject: [PATCH] 8331168: Introduce PredicateEntryIterator to iterate through predicate entries Reviewed-by: roland, kvn --- src/hotspot/share/opto/loopnode.cpp | 54 +++++++++++------------- src/hotspot/share/opto/loopnode.hpp | 6 ++- src/hotspot/share/opto/loopopts.cpp | 2 +- src/hotspot/share/opto/predicates.cpp | 59 ++++++++++++++++++++++----- src/hotspot/share/opto/predicates.hpp | 48 ++++++++++++++++++---- 5 files changed, 118 insertions(+), 51 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index d58be510516..6c16d7cc6a4 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -169,10 +169,10 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) { return earliest; } - while (1) { - Node *next = ctl; - // Moving the node out of a loop on the projection of a If - // confuses loop predication. So once we hit a Loop in a If branch + while (true) { + Node* next = ctl; + // Moving the node out of a loop on the projection of an If + // confuses Loop Predication. So, once we hit a loop in an If branch // that doesn't branch to an UNC, we stop. The code that process // expensive nodes will notice the loop and skip over it to try to // move the node further up. @@ -6081,20 +6081,27 @@ Node* PhaseIdealLoop::get_late_ctrl_with_anti_dep(LoadNode* n, Node* early, Node return LCA; } -// true if CFG node d dominates CFG node n -bool PhaseIdealLoop::is_dominator(Node *d, Node *n) { - if (d == n) +// Is CFG node 'dominator' dominating node 'n'? +bool PhaseIdealLoop::is_dominator(Node* dominator, Node* n) { + if (dominator == n) { return true; - assert(d->is_CFG() && n->is_CFG(), "must have CFG nodes"); - uint dd = dom_depth(d); + } + assert(dominator->is_CFG() && n->is_CFG(), "must have CFG nodes"); + uint dd = dom_depth(dominator); while (dom_depth(n) >= dd) { - if (n == d) + if (n == dominator) { return true; + } n = idom(n); } return false; } +// Is CFG node 'dominator' strictly dominating node 'n'? +bool PhaseIdealLoop::is_strict_dominator(Node* dominator, Node* n) { + return dominator != n && is_dominator(dominator, n); +} + //------------------------------dom_lca_for_get_late_ctrl_internal------------- // Pair-wise LCA with tags. // Tag each index with the node 'tag' currently being processed @@ -6377,31 +6384,16 @@ void PhaseIdealLoop::build_loop_late_post_work(Node *n, bool pinned) { if (least != early) { // Move the node above predicates as far up as possible so a - // following pass of loop predication doesn't hoist a predicate + // following pass of Loop Predication doesn't hoist a predicate // that depends on it above that node. - Node* new_ctrl = least; - for (;;) { - if (!new_ctrl->is_Proj()) { + PredicateEntryIterator predicate_iterator(least); + while (predicate_iterator.has_next()) { + Node* next_predicate_entry = predicate_iterator.next_entry(); + if (is_strict_dominator(next_predicate_entry, early)) { break; } - CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern(); - if (call == nullptr) { - break; - } - int req = call->uncommon_trap_request(); - Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req); - if (trap_reason != Deoptimization::Reason_loop_limit_check && - trap_reason != Deoptimization::Reason_predicate && - trap_reason != Deoptimization::Reason_profile_predicate) { - break; - } - Node* c = new_ctrl->in(0)->in(0); - if (is_dominator(c, early) && c != early) { - break; - } - new_ctrl = c; + least = next_predicate_entry; } - least = new_ctrl; } // Try not to place code on a loop entry projection // which can inhibit range check elimination. diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 90ef4da4f1e..8d9d4b3e0e5 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -1011,8 +1011,10 @@ public: assert(n == find_non_split_ctrl(n), "must return legal ctrl" ); return n; } - // true if CFG node d dominates CFG node n - bool is_dominator(Node *d, Node *n); + + bool is_dominator(Node* dominator, Node* n); + bool is_strict_dominator(Node* dominator, Node* n); + // return get_ctrl for a data node and self(n) for a CFG node Node* ctrl_or_self(Node* n) { if (has_ctrl(n)) diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index b19c71fdd86..23b2edce654 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -4294,7 +4294,7 @@ bool PhaseIdealLoop::duplicate_loop_backedge(IdealLoopTree *loop, Node_List &old } else { wq.push(c->in(0)); } - assert(!is_dominator(c, region) || c == region, "shouldn't go above region"); + assert(!is_strict_dominator(c, region), "shouldn't go above region"); } Node* region_dom = idom(region); diff --git a/src/hotspot/share/opto/predicates.cpp b/src/hotspot/share/opto/predicates.cpp index 9f782f34bdb..5b0de2e02d5 100644 --- a/src/hotspot/share/opto/predicates.cpp +++ b/src/hotspot/share/opto/predicates.cpp @@ -32,29 +32,29 @@ // (i.e. not belonging to an Initialized Assertion Predicate anymore) Node* AssertionPredicatesWithHalt::find_entry(Node* start_proj) { Node* entry = start_proj; - while (is_assertion_predicate_success_proj(entry)) { + while (AssertionPredicateWithHalt::is_predicate(entry)) { entry = entry->in(0)->in(0); } return entry; } -bool AssertionPredicatesWithHalt::is_assertion_predicate_success_proj(const Node* predicate_proj) { - if (predicate_proj == nullptr || !predicate_proj->is_IfProj() || !predicate_proj->in(0)->is_If()) { +bool AssertionPredicateWithHalt::is_predicate(const Node* maybe_success_proj) { + if (maybe_success_proj == nullptr || !maybe_success_proj->is_IfProj() || !maybe_success_proj->in(0)->is_If()) { return false; } - return has_assertion_predicate_opaque(predicate_proj) && has_halt(predicate_proj); + return has_assertion_predicate_opaque(maybe_success_proj) && has_halt(maybe_success_proj); } // Check if the If node of `predicate_proj` has an Opaque4 (Template Assertion Predicate) or an // OpaqueInitializedAssertionPredicate (Initialized Assertion Predicate) node as input. -bool AssertionPredicatesWithHalt::has_assertion_predicate_opaque(const Node* predicate_proj) { +bool AssertionPredicateWithHalt::has_assertion_predicate_opaque(const Node* predicate_proj) { IfNode* iff = predicate_proj->in(0)->as_If(); Node* bol = iff->in(1); return bol->is_Opaque4() || bol->is_OpaqueInitializedAssertionPredicate(); } // Check if the other projection (UCT projection) of `success_proj` has a Halt node as output. -bool AssertionPredicatesWithHalt::has_halt(const Node* success_proj) { +bool AssertionPredicateWithHalt::has_halt(const Node* success_proj) { ProjNode* other_proj = success_proj->as_IfProj()->other_if_proj(); return other_proj->outcnt() == 1 && other_proj->unique_out()->Opcode() == Op_Halt; } @@ -72,7 +72,15 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p return nullptr; } -Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) { +bool ParsePredicate::is_predicate(Node* maybe_success_proj) { + if (!maybe_success_proj->is_IfProj()) { + return false; + } + IfNode* if_node = maybe_success_proj->in(0)->as_If(); + return if_node->is_ParsePredicate(); +} + +Deoptimization::DeoptReason RegularPredicateWithUCT::uncommon_trap_reason(IfProjNode* if_proj) { CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern(); if (uct_call == nullptr) { return Deoptimization::Reason_none; @@ -80,8 +88,20 @@ Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* i return Deoptimization::trap_request_reason(uct_call->uncommon_trap_request()); } -bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason) { - if (may_be_runtime_predicate_if(node)) { +bool RegularPredicateWithUCT::is_predicate(Node* maybe_success_proj) { + if (may_be_predicate_if(maybe_success_proj)) { + IfProjNode* success_proj = maybe_success_proj->as_IfProj(); + const Deoptimization::DeoptReason deopt_reason = uncommon_trap_reason(success_proj); + return (deopt_reason == Deoptimization::Reason_loop_limit_check || + deopt_reason == Deoptimization::Reason_predicate || + deopt_reason == Deoptimization::Reason_profile_predicate); + } else { + return false; + } +} + +bool RegularPredicateWithUCT::is_predicate(Node* node, Deoptimization::DeoptReason deopt_reason) { + if (may_be_predicate_if(node)) { return deopt_reason == uncommon_trap_reason(node->as_IfProj()); } else { return false; @@ -89,7 +109,7 @@ bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason d } // A Runtime Predicate must have an If or a RangeCheck node, while the If should not be a zero trip guard check. -bool RuntimePredicate::may_be_runtime_predicate_if(Node* node) { +bool RegularPredicateWithUCT::may_be_predicate_if(Node* node) { if (node->is_IfProj()) { const IfNode* if_node = node->in(0)->as_If(); const int opcode_if = if_node->Opcode(); @@ -101,6 +121,10 @@ bool RuntimePredicate::may_be_runtime_predicate_if(Node* node) { return false; } +bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason) { + return RegularPredicateWithUCT::is_predicate(node, deopt_reason); +} + ParsePredicateIterator::ParsePredicateIterator(const Predicates& predicates) : _current_index(0) { const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); if (loop_limit_check_predicate_block->has_parse_predicate()) { @@ -356,3 +380,18 @@ bool TemplateAssertionPredicateExpressionNode::is_in_expression(Node* node) { bool TemplateAssertionPredicateExpressionNode::is_template_assertion_predicate(Node* node) { return node->is_If() && node->in(1)->is_Opaque4(); } + +// Is current node pointed to by iterator a predicate? +bool PredicateEntryIterator::has_next() const { + return ParsePredicate::is_predicate(_current) || + RegularPredicateWithUCT::is_predicate(_current) || + AssertionPredicateWithHalt::is_predicate(_current); +} + +// Skip the current predicate pointed to by iterator by returning the input into the predicate. This could possibly be +// a non-predicate node. +Node* PredicateEntryIterator::next_entry() { + assert(has_next(), "current must be predicate"); + _current = _current->in(0)->in(0); + return _current; +} diff --git a/src/hotspot/share/opto/predicates.hpp b/src/hotspot/share/opto/predicates.hpp index bf12aeb6a51..9cac98eb993 100644 --- a/src/hotspot/share/opto/predicates.hpp +++ b/src/hotspot/share/opto/predicates.hpp @@ -26,6 +26,7 @@ #define SHARE_OPTO_PREDICATES_HPP #include "opto/cfgnode.hpp" +#include "opto/connode.hpp" #include "opto/opaquenode.hpp" /* @@ -199,9 +200,6 @@ class AssertionPredicatesWithHalt : public StackObj { Node* _entry; static Node* find_entry(Node* start_proj); - static bool has_assertion_predicate_opaque(const Node* predicate_proj); - static bool has_halt(const Node* success_proj); - static bool is_assertion_predicate_success_proj(const Node* predicate_proj); public: AssertionPredicatesWithHalt(Node* assertion_predicate_proj) : _entry(find_entry(assertion_predicate_proj)) {} @@ -213,13 +211,37 @@ class AssertionPredicatesWithHalt : public StackObj { } }; +// Class to represent a single Assertion Predicate with a HaltNode. This could either be: +// - A Template Assertion Predicate. +// - An Initialized Assertion Predicate. +// Note that all other Regular Predicates have an UCT node. +class AssertionPredicateWithHalt : public StackObj { + static bool has_assertion_predicate_opaque(const Node* predicate_proj); + static bool has_halt(const Node* success_proj); + public: + static bool is_predicate(const Node* maybe_success_proj); +}; + +// Class to represent a single Regular Predicate with an UCT. This could either be: +// - A Runtime Predicate +// - A Template Assertion Predicate +// Note that all other Regular Predicates have a Halt node. +class RegularPredicateWithUCT : public StackObj { + static Deoptimization::DeoptReason uncommon_trap_reason(IfProjNode* if_proj); + static bool may_be_predicate_if(Node* node); + + public: + static bool is_predicate(Node* maybe_success_proj); + static bool is_predicate(Node* node, Deoptimization::DeoptReason deopt_reason); +}; + // Class to represent a Parse Predicate. class ParsePredicate : public StackObj { ParsePredicateSuccessProj* _success_proj; ParsePredicateNode* _parse_predicate_node; Node* _entry; - IfTrueNode* init_success_proj(const Node* parse_predicate_proj) const { + static IfTrueNode* init_success_proj(const Node* parse_predicate_proj) { assert(parse_predicate_proj != nullptr, "must not be null"); return parse_predicate_proj->isa_IfTrue(); } @@ -253,13 +275,12 @@ class ParsePredicate : public StackObj { assert(is_valid(), "must be valid"); return _success_proj; } + + static bool is_predicate(Node* maybe_success_proj); }; // Utility class for queries on Runtime Predicates. class RuntimePredicate : public StackObj { - static Deoptimization::DeoptReason uncommon_trap_reason(IfProjNode* if_proj); - static bool may_be_runtime_predicate_if(Node* node); - public: static bool is_success_proj(Node* node, Deoptimization::DeoptReason deopt_reason); }; @@ -473,4 +494,17 @@ class ParsePredicateIterator : public StackObj { ParsePredicateNode* next(); }; + +// Special predicate iterator that can be used to walk through predicate entries, regardless of whether the predicate +// belongs to the same loop or not (i.e. leftovers from already folded nodes). The iterator returns the next entry +// to a predicate. +class PredicateEntryIterator : public StackObj { + Node* _current; + + public: + explicit PredicateEntryIterator(Node* start) : _current(start) {}; + + bool has_next() const; + Node* next_entry(); +}; #endif // SHARE_OPTO_PREDICATES_HPP