stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8305635: Replace Parse Predicate IfNode with new ParsePredicateNode and route predicate queries through dedicated classes

Browse Source 
Reviewed-by: thartmann, kvn
This commit is contained in:
Christian Hagedorn 2023-05-25 09:40:19 +00:00
parent f27bc59f85
commit 4f096eb7c9
12 changed files with 427 additions and 327 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/hotspot/share/opto/cfgnode.cpp
View File

@ -1796,8 +1796,8 @@ static Node* split_flow_path(PhaseGVN *phase, PhiNode *phi) {
if( phase->type(n) == Type::TOP ) return nullptr;
if( phi->in(i) == val ) {
hit++;
if (PhaseIdealLoop::find_parse_predicate(r->in(i)) != nullptr) {
return nullptr; // don't split loop entry path
if (Node::may_be_loop_entry(r->in(i))) {
return nullptr; // don't split loop entry path
}
}
}

20
src/hotspot/share/opto/cfgnode.hpp
View File

@ -452,6 +452,26 @@ public:
virtual Node* Ideal(PhaseGVN *phase, bool can_reshape);
};
// Special node that denotes a Parse Predicate added during parsing. A Parse Predicate serves as placeholder to later
// create Runtime Predicates above it. They all share the same uncommon trap. The Parse Predicate will follow the
// Runtime Predicates. Together they form a Regular Predicate Block. There are three kinds of Parse Predicates:
// Loop Parse Predicate, Profiled Loop Parse Predicate (both used by Loop Predication), and Loop Limit Check Parse
// Predicate (used for integer overflow checks when creating a counted loop).
// More information about predicates can be found in loopPredicate.cpp.
class ParsePredicateNode : public IfNode {
Deoptimization::DeoptReason _deopt_reason;
public:
ParsePredicateNode(Node* control, Node* bol, Deoptimization::DeoptReason deopt_reason);
virtual int Opcode() const;
virtual uint size_of() const { return sizeof(*this); }
Deoptimization::DeoptReason deopt_reason() const {
return _deopt_reason;
}
NOT_PRODUCT(void dump_spec(outputStream* st) const;)
};
class IfProjNode : public CProjNode {
public:
IfProjNode(IfNode *ifnode, uint idx) : CProjNode(ifnode,idx) {}

1
src/hotspot/share/opto/classes.hpp
View File

@ -282,6 +282,7 @@ macro(OverflowSubL)
macro(OverflowMulL)
macro(PCTable)
macro(Parm)
macro(ParsePredicate)
macro(PartialSubtypeCheck)
macro(SubTypeCheck)
macro(Phi)

19
src/hotspot/share/opto/graphKit.cpp
View File

@ -3995,20 +3995,21 @@ void GraphKit::add_parse_predicate(Deoptimization::DeoptReason reason, const int
return;
}
Node* cont = _gvn.intcon(1);
Node* opq = _gvn.transform(new Opaque1Node(C, cont));
Node* bol = _gvn.transform(new Conv2BNode(opq));
IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
Node* iffalse = _gvn.transform(new IfFalseNode(iff));
C->add_parse_predicate_opaq(opq);
Node* cont = _gvn.intcon(1);
Node* opaq = _gvn.transform(new Opaque1Node(C, cont));
C->add_parse_predicate_opaq(opaq);
Node* bol = _gvn.transform(new Conv2BNode(opaq));
ParsePredicateNode* parse_predicate = new ParsePredicateNode(control(), bol, reason);
_gvn.set_type(parse_predicate, parse_predicate->Value(&_gvn));
Node* if_false = _gvn.transform(new IfFalseNode(parse_predicate));
{
PreserveJVMState pjvms(this);
set_control(iffalse);
set_control(if_false);
inc_sp(nargs);
uncommon_trap(reason, Deoptimization::Action_maybe_recompile);
}
Node* iftrue = _gvn.transform(new IfTrueNode(iff));
set_control(iftrue);
Node* if_true = _gvn.transform(new IfTrueNode(parse_predicate));
set_control(if_true);
}
// Add Parse Predicates which serve as placeholders to create new Runtime Predicates above them. All

41
src/hotspot/share/opto/ifnode.cpp
View File

@ -242,9 +242,8 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) {
if (phi->in(ii) == con1) {
req_c++;
}
Node* proj = PhaseIdealLoop::find_parse_predicate(r->in(ii));
if (proj != nullptr) {
// Bail out if splitting through a region with a predicate input (could
if (Node::may_be_loop_entry(r->in(ii))) {
// Bail out if splitting through a region with a Parse Predicate input (could
// also be a loop header before loop opts creates a LoopNode for it).
return nullptr;
}
@ -1969,3 +1968,39 @@ Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) {
// Replace dominated IfNode
return dominated_by(prev_dom, igvn);
}
ParsePredicateNode::ParsePredicateNode(Node* control, Node* bol, Deoptimization::DeoptReason deopt_reason)
: IfNode(control, bol, PROB_MAX, COUNT_UNKNOWN),
_deopt_reason(deopt_reason) {
init_class_id(Class_ParsePredicate);
assert(bol->Opcode() == Op_Conv2B && bol->in(1) != nullptr && bol->in(1)->is_Opaque1(), "wrong boolean input");
#ifdef ASSERT
switch (deopt_reason) {
case Deoptimization::Reason_predicate:
case Deoptimization::Reason_profile_predicate:
case Deoptimization::Reason_loop_limit_check:
break;
default:
assert(false, "unsupported deoptimization reason for Parse Predicate");
}
#endif // ASSERT
}
#ifndef PRODUCT
void ParsePredicateNode::dump_spec(outputStream* st) const {
st->print(" #");
switch (_deopt_reason) {
case Deoptimization::DeoptReason::Reason_predicate:
st->print("Loop ");
break;
case Deoptimization::DeoptReason::Reason_profile_predicate:
st->print("Profiled_Loop ");
break;
case Deoptimization::DeoptReason::Reason_loop_limit_check:
st->print("Loop_Limit_Check ");
break;
default:
fatal("unknown kind");
}
}
#endif // NOT PRODUCT

387
src/hotspot/share/opto/loopPredicate.cpp
View File

@ -307,11 +307,18 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, N
// Create new_iff
IdealLoopTree* lp = get_loop(entry);
IfNode* new_iff = nullptr;
if (opcode == Op_If) {
new_iff = new IfNode(entry, iff->in(1), iff->_prob, iff->_fcnt);
} else {
assert(opcode == Op_RangeCheck, "no other if variant here");
new_iff = new RangeCheckNode(entry, iff->in(1), iff->_prob, iff->_fcnt);
switch (opcode) {
case Op_If:
new_iff = new IfNode(entry, iff->in(1), iff->_prob, iff->_fcnt);
break;
case Op_RangeCheck:
new_iff = new RangeCheckNode(entry, iff->in(1), iff->_prob, iff->_fcnt);
break;
case Op_ParsePredicate:
new_iff = new ParsePredicateNode(entry, iff->in(1), reason);
break;
default:
fatal("no other If variant here");
}
register_control(new_iff, lp, entry);
IfProjNode* if_cont;
@ -471,7 +478,7 @@ IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredic
Node* new_entry, Deoptimization::DeoptReason reason,
const bool slow_loop) {
IfProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If,
IfProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_ParsePredicate,
slow_loop);
IfNode* iff = new_predicate_proj->in(0)->as_If();
Node* ctrl = iff->in(0);
@ -586,160 +593,64 @@ IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(Node*
void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) {
LoopNode* head = loop->_head->as_Loop();
bool clone_limit_check = !head->is_CountedLoop();
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
// Search original predicates
ParsePredicateSuccessProj* limit_check_proj = nullptr;
limit_check_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (limit_check_proj != nullptr) {
entry = skip_related_predicates(entry);
}
ParsePredicateSuccessProj* profile_predicate_proj = nullptr;
ParsePredicateSuccessProj* predicate_proj = nullptr;
if (UseProfiledLoopPredicate) {
profile_predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (profile_predicate_proj != nullptr) {
entry = skip_related_predicates(entry);
}
}
if (UseLoopPredicate) {
predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
}
if (predicate_proj != nullptr) { // right pattern that can be used by loop predication
// clone predicate
iffast_pred = clone_parse_predicate_to_unswitched_loop(predicate_proj, iffast_pred, Deoptimization::Reason_predicate, false);
ifslow_pred = clone_parse_predicate_to_unswitched_loop(predicate_proj, ifslow_pred, Deoptimization::Reason_predicate, true);
clone_assertion_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_predicate, predicate_proj,
ParsePredicates parse_predicates(entry);
ParsePredicateSuccessProj* loop_predicate_proj = parse_predicates.loop_predicate_proj();
if (loop_predicate_proj != nullptr) {
// Clone Parse Predicate and Template Assertion Predicates of the Loop Predicate Block.
iffast_pred = clone_parse_predicate_to_unswitched_loop(loop_predicate_proj, iffast_pred,
Deoptimization::Reason_predicate, false);
check_cloned_parse_predicate_for_unswitching(iffast_pred, true);
ifslow_pred = clone_parse_predicate_to_unswitched_loop(loop_predicate_proj, ifslow_pred,
Deoptimization::Reason_predicate, true);
check_cloned_parse_predicate_for_unswitching(ifslow_pred, false);
clone_assertion_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_predicate, loop_predicate_proj,
iffast_pred, ifslow_pred);
check_cloned_parse_predicate_for_unswitching(iffast_pred);
check_cloned_parse_predicate_for_unswitching(ifslow_pred);
}
if (profile_predicate_proj != nullptr) { // right pattern that can be used by loop predication
// clone predicate
iffast_pred = clone_parse_predicate_to_unswitched_loop(profile_predicate_proj, iffast_pred,Deoptimization::Reason_profile_predicate, false);
ifslow_pred = clone_parse_predicate_to_unswitched_loop(profile_predicate_proj, ifslow_pred,Deoptimization::Reason_profile_predicate, true);
ParsePredicateSuccessProj* profiled_loop_predicate_proj = parse_predicates.profiled_loop_predicate_proj();
if (profiled_loop_predicate_proj != nullptr) {
// Clone Parse Predicate and Template Assertion Predicates of the Profiled Loop Predicate Block.
iffast_pred = clone_parse_predicate_to_unswitched_loop(profiled_loop_predicate_proj, iffast_pred,
Deoptimization::Reason_profile_predicate, false);
check_cloned_parse_predicate_for_unswitching(iffast_pred, true);
ifslow_pred = clone_parse_predicate_to_unswitched_loop(profiled_loop_predicate_proj, ifslow_pred,
Deoptimization::Reason_profile_predicate, true);
check_cloned_parse_predicate_for_unswitching(ifslow_pred, false);
clone_assertion_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_profile_predicate,
profile_predicate_proj, iffast_pred, ifslow_pred);
profiled_loop_predicate_proj, iffast_pred, ifslow_pred);
check_cloned_parse_predicate_for_unswitching(iffast_pred);
check_cloned_parse_predicate_for_unswitching(ifslow_pred);
}
if (limit_check_proj != nullptr && clone_limit_check) {
// Clone loop limit check last to insert it before loop.
// Don't clone a limit check which was already finalized
// for this counted loop (only one limit check is needed).
iffast_pred = clone_parse_predicate_to_unswitched_loop(limit_check_proj, iffast_pred,Deoptimization::Reason_loop_limit_check, false);
ifslow_pred = clone_parse_predicate_to_unswitched_loop(limit_check_proj, ifslow_pred,Deoptimization::Reason_loop_limit_check, true);
ParsePredicateSuccessProj* loop_limit_check_predicate_proj = parse_predicates.loop_limit_check_predicate_proj();
if (loop_limit_check_predicate_proj != nullptr && !head->is_CountedLoop()) {
// Don't clone the Loop Limit Check Parse Predicate if we already have a counted loop (a Loop Limit Check Predicate
// is only created when converting a LoopNode to a CountedLoopNode).
iffast_pred = clone_parse_predicate_to_unswitched_loop(loop_limit_check_predicate_proj, iffast_pred,
Deoptimization::Reason_loop_limit_check, false);
check_cloned_parse_predicate_for_unswitching(iffast_pred, true);
check_cloned_parse_predicate_for_unswitching(iffast_pred);
check_cloned_parse_predicate_for_unswitching(ifslow_pred);
ifslow_pred = clone_parse_predicate_to_unswitched_loop(loop_limit_check_predicate_proj, ifslow_pred,
Deoptimization::Reason_loop_limit_check, true);
check_cloned_parse_predicate_for_unswitching(ifslow_pred, false);
}
}
#ifndef PRODUCT
void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry) {
void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry, const bool is_fast_loop) {
assert(new_entry != nullptr, "IfTrue or IfFalse after clone predicate");
if (TraceLoopPredicate) {
tty->print("Loop Predicate cloned: ");
debug_only(new_entry->in(0)->dump(););
tty->print("Parse Predicate cloned to %s loop: ", is_fast_loop ? "fast" : "slow");
new_entry->in(0)->dump();
}
}
#endif
Node* PhaseIdealLoop::skip_related_predicates(Node* entry) {
IfNode* iff = entry->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - entry->as_Proj()->_con);
Node* rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
entry = entry->in(0)->in(0);
while (entry != nullptr && entry->is_Proj() && entry->in(0)->is_If()) {
uncommon_proj = entry->in(0)->as_If()->proj_out(1 - entry->as_Proj()->_con);
if (uncommon_proj->unique_ctrl_out() != rgn)
break;
entry = entry->in(0)->in(0);
}
return entry;
}
Node* PhaseIdealLoop::skip_all_predicates(Node* entry) {
ParsePredicates parse_predicates(entry);
return parse_predicates.get_first_predicate();
}
//--------------------------next_predicate---------------------------------
// Find next related predicate, useful for iterating over all related predicates
IfProjNode* PhaseIdealLoop::next_predicate(IfProjNode* predicate_proj) {
IfNode* iff = predicate_proj->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - predicate_proj->_con);
Node* rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
Node* next = iff->in(0);
if (next != nullptr && next->is_IfProj() && next->in(0)->is_If()) {
uncommon_proj = next->in(0)->as_If()->proj_out(1 - next->as_Proj()->_con);
if (uncommon_proj->unique_ctrl_out() == rgn) { // lead into same region
return next->as_IfProj();
}
}
return nullptr;
}
//--------------------------find_predicate_insertion_point-------------------
// Find a good location to insert a predicate
ParsePredicateSuccessProj* PhaseIdealLoop::find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason) {
if (start_c == nullptr || !start_c->is_IfTrue())
return nullptr;
if (start_c->as_IfTrue()->is_uncommon_trap_if_pattern(reason)) {
return start_c->as_IfTrue();
}
return nullptr;
}
//--------------------------Predicates::Predicates--------------------------
// given loop entry, find all predicates above loop
PhaseIdealLoop::ParsePredicates::ParsePredicates(Node* entry) {
_loop_limit_check_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (_loop_limit_check_predicate != nullptr) {
entry = skip_related_predicates(entry);
}
if (UseProfiledLoopPredicate) {
_profiled_loop_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (_profiled_loop_predicate != nullptr) {
entry = skip_related_predicates(entry);
}
}
if (UseLoopPredicate) {
_loop_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (_loop_predicate != nullptr) {
entry = skip_related_predicates(entry);
}
}
_first_predicate = entry;
}
Node* PhaseIdealLoop::find_parse_predicate(Node* entry) {
Node* predicate = nullptr;
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate != nullptr) { // right pattern that can be used by loop predication
return entry;
}
if (UseLoopPredicate) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (predicate != nullptr) { // right pattern that can be used by loop predication
return entry;
}
}
if (UseProfiledLoopPredicate) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (predicate != nullptr) { // right pattern that can be used by loop predication
return entry;
}
}
return nullptr;
}
//------------------------------Invariance-----------------------------------
// Helper class for loop_predication_impl to compute invariance on the fly and
// clone invariants.
@ -1582,8 +1493,7 @@ IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL
return new_proj;
}
//------------------------------ loop_predication_impl--------------------------
// Insert loop predicates for null checks and range checks
// Insert Hoisted Predicates for null checks and range checks and additional Template Assertion Predicates for range checks.
bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
if (!UseLoopPredicate) return false;
@ -1614,32 +1524,31 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
}
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
ParsePredicateSuccessProj* loop_limit_proj = nullptr;
ParsePredicateSuccessProj* predicate_proj = nullptr;
ParsePredicateSuccessProj* profile_predicate_proj = nullptr;
// Loop limit check predicate should be near the loop.
loop_limit_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (loop_limit_proj != nullptr) {
entry = skip_related_predicates(loop_limit_proj);
ParsePredicates parse_predicates(entry);
bool can_create_loop_predicates = true;
// We cannot add Loop Predicates if:
// - Already added Profiled Loop Predicates (Loop Predicates and Profiled Loop Predicates can be dependent
// through a data node, and thus we should only add new Profiled Loop Predicates which are below Loop Predicates
// in the graph).
// - There are currently no Profiled Loop Predicates, but we have a data node with a control dependency on the Loop
// Parse Predicate (could happen, for example, if we've removed an earlier created Profiled Loop Predicate with
// dominated_by()). We should not create a Loop Predicate for a check that is dependent on this data node because
// the Loop Predicate would end up above the data node with its dependency on the Loop Parse Predicate below. This
// would become unschedulable. However, we can still hoist the check as Profiled Loop Predicate which would end up
// below the Loop Parse Predicate.
if (Predicates::has_profiled_loop_predicates(parse_predicates)
|| (parse_predicates.loop_predicate_proj() != nullptr && parse_predicates.loop_predicate_proj()->outcnt() != 1)) {
can_create_loop_predicates = false;
}
bool has_profile_predicates = false;
profile_predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (profile_predicate_proj != nullptr) {
Node* n = skip_related_predicates(entry);
// Check if predicates were already added to the profile predicate
// block
if (n != entry->in(0)->in(0) || n->outcnt() != 1) {
has_profile_predicates = true;
}
entry = n;
}
predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
ParsePredicateSuccessProj* loop_predicate_proj = parse_predicates.loop_predicate_proj();
ParsePredicateSuccessProj* profiled_loop_predicate_proj = parse_predicates.profiled_loop_predicate_proj();
float loop_trip_cnt = -1;
bool follow_branches = loop_predication_should_follow_branches(loop, profile_predicate_proj, loop_trip_cnt);
bool follow_branches = loop_predication_should_follow_branches(loop, profiled_loop_predicate_proj, loop_trip_cnt);
assert(!follow_branches || loop_trip_cnt >= 0, "negative trip count?");
if (predicate_proj == nullptr && !follow_branches) {
if (loop_predicate_proj == nullptr && !follow_branches) {
#ifndef PRODUCT
if (TraceLoopPredicate) {
tty->print("missing predicate:");
@ -1680,7 +1589,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
bool hoisted = false; // true if at least one proj is promoted
if (!has_profile_predicates) {
if (can_create_loop_predicates) {
while (if_proj_list.size() > 0) {
Node* n = if_proj_list.pop();
@ -1709,13 +1618,15 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
break;
}
if (predicate_proj != nullptr) {
hoisted = loop_predication_impl_helper(loop, if_proj, predicate_proj, cl, zero, invar, Deoptimization::Reason_predicate) | hoisted;
if (loop_predicate_proj != nullptr) {
hoisted = loop_predication_impl_helper(loop, if_proj, loop_predicate_proj, cl, zero, invar,
Deoptimization::Reason_predicate) | hoisted;
}
} // end while
}
if (follow_branches) {
assert(profiled_loop_predicate_proj != nullptr, "sanity check");
PathFrequency pf(loop->_head, this);
// Some projections were skipped by regular predicates because of
@ -1725,7 +1636,8 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
float f = pf.to(if_proj);
if (if_proj->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
f * loop_trip_cnt >= 1) {
hoisted = loop_predication_impl_helper(loop, if_proj->as_IfProj(), profile_predicate_proj, cl, zero, invar, Deoptimization::Reason_profile_predicate) | hoisted;
hoisted = loop_predication_impl_helper(loop, if_proj->as_IfProj(), profiled_loop_predicate_proj, cl, zero, invar,
Deoptimization::Reason_profile_predicate) | hoisted;
}
}
@ -1740,7 +1652,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
for (uint i = 0; i < if_proj_list_freq.size(); i++) {
IfProjNode* if_proj = if_proj_list_freq.at(i)->as_IfProj();
hoisted = loop_predication_impl_helper(loop, if_proj, profile_predicate_proj, cl, zero, invar, Deoptimization::Reason_profile_predicate) | hoisted;
hoisted = loop_predication_impl_helper(loop, if_proj, profiled_loop_predicate_proj, cl, zero, invar, Deoptimization::Reason_profile_predicate) | hoisted;
}
}
@ -1778,3 +1690,140 @@ bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) {
return hoisted;
}
// Skip over all predicates (all Regular Predicate Blocks) starting at the Parse Predicate projection 'node'. Return the
// first node that is not a predicate If node anymore (i.e. entry into the first predicate If on top) or 'node' if 'node'
// is not a Parse Predicate projection.
Node* Predicates::skip_all_predicates(Node* node) {
ParsePredicates parse_predicates(node);
if (parse_predicates.has_any()) {
return skip_all_predicates(parse_predicates);
} else {
return node;
}
}
// Skip over all Runtime Predicates belonging to the given Parse Predicates. Return the first node that is not a predicate
// If node anymore (i.e. entry into the first predicate If on top).
Node* Predicates::skip_all_predicates(ParsePredicates& parse_predicates) {
assert(parse_predicates.has_any(), "must have at least one Parse Predicate");
return skip_predicates_in_block(parse_predicates.get_top_predicate_proj());
}
// Skip over all predicates in a Regular Predicate Block starting at the Parse Predicate projection
// 'parse_predicate_success_proj'. Return the first node not belonging this block anymore (i.e. entry
// into this Regular Predicate Block).
Node* Predicates::skip_predicates_in_block(ParsePredicateSuccessProj* parse_predicate_success_proj) {
IfProjNode* prev;
IfProjNode* next = parse_predicate_success_proj;
do {
prev = next;
next = next_predicate_proj_in_block(next);
} while (next != nullptr);
assert(prev->in(0)->is_If(), "must be predicate If");
return prev->in(0)->in(0);
}
// Find next Runtime Predicate projection in a Regular Predicate Block or return null if there is none.
IfProjNode* Predicates::next_predicate_proj_in_block(IfProjNode* proj) {
IfNode* iff = proj->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - proj->_con);
Node* rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
Node* next = iff->in(0);
if (next != nullptr && next->is_Proj() && next->in(0)->is_If()) {
uncommon_proj = next->in(0)->as_If()->proj_out(1 - next->as_Proj()->_con);
if (uncommon_proj->unique_ctrl_out() == rgn) {
// Same Runtime Predicate Block.
return next->as_IfProj();
}
}
return nullptr;
}
// Is there at least one Profiled Loop Predicate?
bool Predicates::has_profiled_loop_predicates(ParsePredicates& parse_predicates) {
ParsePredicateSuccessProj* profiled_loop_predicate = parse_predicates.profiled_loop_predicate_proj();
if (profiled_loop_predicate == nullptr) {
return false;
}
return Predicates::next_predicate_proj_in_block(profiled_loop_predicate) != nullptr;
}
// Given a node 'starting_proj', check if it is a Parse Predicate success projection.
// If so, find all Parse Predicates above the loop.
ParsePredicates::ParsePredicates(Node* starting_proj) : _top_predicate_proj(nullptr), _starting_proj(nullptr) {
if (starting_proj == nullptr || !starting_proj->is_IfTrue()) {
return; // Not a predicate.
}
_starting_proj = starting_proj->as_IfTrue();
find_parse_predicate_projections();
}
void ParsePredicates::find_parse_predicate_projections() {
Node* maybe_parse_predicate_proj = _starting_proj;
for (int i = 0; i < 3; i++) { // At most 3 Parse Predicates for a loop
if (!is_success_proj(maybe_parse_predicate_proj)) {
break;
}
ParsePredicateSuccessProj* parse_predicate_proj = maybe_parse_predicate_proj->as_IfTrue();
if (!assign_predicate_proj(parse_predicate_proj)) {
// Found a Parse Predicate of another (already removed) loop.
break;
}
_top_predicate_proj = parse_predicate_proj;
maybe_parse_predicate_proj = Predicates::skip_predicates_in_block(parse_predicate_proj);
}
}
// Is 'node' a success (non-UCT) projection of a Parse Predicate?
bool ParsePredicates::is_success_proj(Node* node) {
if (node == nullptr || !node->is_Proj()) {
return false;
}
ParsePredicateNode* parse_predicate = get_parse_predicate_or_null(node);
if (parse_predicate == nullptr) {
return false;
}
return !is_uct_proj(node, parse_predicate->deopt_reason());
}
// Is 'node' a UCT projection of a Parse Predicate of kind 'kind'?
bool ParsePredicates::is_uct_proj(Node* node, Deoptimization::DeoptReason deopt_reason) {
return node->as_Proj()->is_uncommon_trap_proj(deopt_reason);
}
// Check the parent of `parse_predicate_proj` is a ParsePredicateNode. If so return it. Otherwise, return null.
ParsePredicateNode* ParsePredicates::get_parse_predicate_or_null(Node* parse_predicate_proj) {
return parse_predicate_proj->in(0)->isa_ParsePredicate();
}
// Initialize the Parse Predicate projection field that matches the kind of the parent of `parse_predicate_proj`.
bool ParsePredicates::assign_predicate_proj(ParsePredicateSuccessProj* parse_predicate_proj) {
ParsePredicateNode* parse_predicate = get_parse_predicate_or_null(parse_predicate_proj);
assert(parse_predicate != nullptr, "must exist");
Deoptimization::DeoptReason deopt_reason = parse_predicate->deopt_reason();
switch (deopt_reason) {
case Deoptimization::DeoptReason::Reason_predicate:
if (_loop_predicate_proj != nullptr) {
return false;
}
_loop_predicate_proj = parse_predicate_proj;
break;
case Deoptimization::DeoptReason::Reason_profile_predicate:
if (_profiled_loop_predicate_proj != nullptr) {
return false;
}
_profiled_loop_predicate_proj = parse_predicate_proj;
break;
case Deoptimization::DeoptReason::Reason_loop_limit_check:
if (_loop_limit_check_predicate_proj != nullptr) {
return false;
}
_loop_limit_check_predicate_proj = parse_predicate_proj;
break;
default:
fatal("invalid case");
}
return true;
}

37
src/hotspot/share/opto/loopTransform.cpp
View File

@ -489,7 +489,8 @@ uint IdealLoopTree::estimate_peeling(PhaseIdealLoop *phase) {
assert(test->Opcode() == Op_If ||
test->Opcode() == Op_CountedLoopEnd ||
test->Opcode() == Op_LongCountedLoopEnd ||
test->Opcode() == Op_RangeCheck,
test->Opcode() == Op_RangeCheck ||
test->Opcode() == Op_ParsePredicate,
"Check this code when new subtype is added");
// Condition is not a member of this loop?
if (!is_member(phase->get_loop(ctrl)) && is_loop_exit(test)) {
@ -778,11 +779,11 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) {
Node* stride = cl_head->stride();
IdealLoopTree* outer_loop = get_loop(outer_loop_head);
ParsePredicates parse_predicates(new_head->in(LoopNode::EntryControl));
initialize_assertion_predicates_for_peeled_loop(parse_predicates.loop_predicate(),
initialize_assertion_predicates_for_peeled_loop(parse_predicates.loop_predicate_proj(),
outer_loop_head, dd_outer_loop_head,
init, stride, outer_loop,
idx_before_clone, old_new);
initialize_assertion_predicates_for_peeled_loop(parse_predicates.profiled_loop_predicate(),
initialize_assertion_predicates_for_peeled_loop(parse_predicates.profiled_loop_predicate_proj(),
outer_loop_head, dd_outer_loop_head,
init, stride, outer_loop,
idx_before_clone, old_new);
@ -1590,24 +1591,14 @@ void PhaseIdealLoop::copy_assertion_predicates_to_main_loop(CountedLoopNode* pre
const Node_List &old_new) {
if (UseLoopPredicate) {
Node* entry = pre_head->in(LoopNode::EntryControl);
Node* predicate = nullptr;
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate != nullptr) {
entry = skip_related_predicates(entry);
}
Node* profile_predicate = nullptr;
if (UseProfiledLoopPredicate) {
profile_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (profile_predicate != nullptr) {
entry = skip_related_predicates(entry);
}
}
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
copy_assertion_predicates_to_main_loop_helper(predicate, init, stride, outer_loop, outer_main_head, dd_main_head,
idx_before_pre_post, idx_after_post_before_pre, zero_trip_guard_proj_main,
ParsePredicates parse_predicates(entry);
copy_assertion_predicates_to_main_loop_helper(parse_predicates.loop_predicate_proj(), init, stride, outer_loop,
outer_main_head, dd_main_head, idx_before_pre_post,
idx_after_post_before_pre, zero_trip_guard_proj_main,
zero_trip_guard_proj_post, old_new);
copy_assertion_predicates_to_main_loop_helper(profile_predicate, init, stride, outer_loop, outer_main_head, dd_main_head,
idx_before_pre_post, idx_after_post_before_pre, zero_trip_guard_proj_main,
copy_assertion_predicates_to_main_loop_helper(parse_predicates.profiled_loop_predicate_proj(), init, stride,
outer_loop, outer_main_head, dd_main_head, idx_before_pre_post,
idx_after_post_before_pre, zero_trip_guard_proj_main,
zero_trip_guard_proj_post, old_new);
}
}
@ -2140,7 +2131,7 @@ void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(IfProjNode*
Node* control = outer_loop_head->in(LoopNode::EntryControl);
Node* input_proj = control;
predicate_proj = next_predicate(predicate_proj);
predicate_proj = Predicates::next_predicate_proj_in_block(predicate_proj);
while (predicate_proj != nullptr) {
IfNode* iff = predicate_proj->in(0)->as_If();
if (iff->in(1)->Opcode() == Op_Opaque4) {
@ -2166,7 +2157,7 @@ void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(IfProjNode*
}
}
}
predicate_proj = next_predicate(predicate_proj);
predicate_proj = Predicates::next_predicate_proj_in_block(predicate_proj);
}
_igvn.replace_input_of(outer_loop_head, LoopNode::EntryControl, input_proj);
@ -3534,7 +3525,7 @@ bool IdealLoopTree::do_remove_empty_loop(PhaseIdealLoop *phase) {
}
if (needs_guard) {
// Check for an obvious zero trip guard.
Node* in_ctrl = PhaseIdealLoop::skip_all_predicates(cl->skip_predicates());
Node* in_ctrl = Predicates::skip_all_predicates(cl->skip_predicates());
if (in_ctrl->Opcode() == Op_IfTrue || in_ctrl->Opcode() == Op_IfFalse) {
bool maybe_swapped = (in_ctrl->Opcode() == Op_IfFalse);
// The test should look like just the backedge of a CountedLoop

29
src/hotspot/share/opto/loopUnswitch.cpp
View File

@ -117,12 +117,9 @@ IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop) co
// insert a clone of the test that selects which version to
// execute.
void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) {
LoopNode *head = loop->_head->as_Loop();
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
if (find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check) != nullptr
|| (UseProfiledLoopPredicate && find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate) != nullptr)
|| (UseLoopPredicate && find_predicate_insertion_point(entry, Deoptimization::Reason_predicate) != nullptr)) {
if (ParsePredicates::is_success_proj(entry)) {
assert(entry->is_IfProj(), "sanity - must be ifProj since there is at least one predicate");
if (entry->outcnt() > 1) {
// Bailout if there are predicates from which there are additional control dependencies (i.e. from loop
@ -154,29 +151,17 @@ void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) {
#ifdef ASSERT
assert(proj_true->is_IfTrue(), "must be true projection");
entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
Node* parse_predicate = find_parse_predicate(entry);
if (parse_predicate == nullptr) {
ParsePredicates parse_predicates(entry);
if (!parse_predicates.has_any()) {
// No Parse Predicate.
Node* uniqc = proj_true->unique_ctrl_out();
assert((uniqc == head && !head->is_strip_mined()) || (uniqc == head->in(LoopNode::EntryControl)
&& head->is_strip_mined()), "must hold by construction if no predicates");
} else {
// There is at least one Parse Predicate. When calling 'skip_related_predicates' on each found Parse Predicate,
// we should end up at 'proj_true'.
Node* proj_before_first_parse_predicate = skip_related_predicates(entry);
if (UseProfiledLoopPredicate) {
parse_predicate = find_parse_predicate(proj_before_first_parse_predicate);
if (parse_predicate != nullptr) {
proj_before_first_parse_predicate = skip_related_predicates(parse_predicate);
}
}
if (UseLoopPredicate) {
parse_predicate = find_parse_predicate(proj_before_first_parse_predicate);
if (parse_predicate != nullptr) {
proj_before_first_parse_predicate = skip_related_predicates(parse_predicate);
}
}
assert(proj_true == proj_before_first_parse_predicate, "must hold by construction if at least one predicate");
// There is at least one Parse Predicate. When skipping all predicates/Regular Predicate Blocks, we should end up
// at 'proj_true'.
assert(proj_true == Predicates::skip_all_predicates(parse_predicates),
"must hold by construction if at least one Parse Predicate");
}
#endif
// Increment unswitch count

103
src/hotspot/share/opto/loopnode.cpp
View File

@ -561,19 +561,19 @@ Node* PhaseIdealLoop::loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv,
void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop,
SafePointNode* sfpt) {
if (!C->too_many_traps(reason)) {
Node *cont = _igvn.intcon(1);
Node* opq = new Opaque1Node(C, cont);
_igvn.register_new_node_with_optimizer(opq);
Node *bol = new Conv2BNode(opq);
Node* cont = _igvn.intcon(1);
Node* opaq = new Opaque1Node(C, cont);
_igvn.register_new_node_with_optimizer(opaq);
Node* bol = new Conv2BNode(opaq);
_igvn.register_new_node_with_optimizer(bol);
set_subtree_ctrl(bol, false);
IfNode* iff = new IfNode(inner_head->in(LoopNode::EntryControl), bol, PROB_MAX, COUNT_UNKNOWN);
ParsePredicateNode* iff = new ParsePredicateNode(inner_head->in(LoopNode::EntryControl), bol, reason);
register_control(iff, loop, inner_head->in(LoopNode::EntryControl));
Node* iffalse = new IfFalseNode(iff);
register_control(iffalse, _ltree_root, iff);
Node* iftrue = new IfTrueNode(iff);
register_control(iftrue, loop, iff);
C->add_parse_predicate_opaq(opq);
Node* if_false = new IfFalseNode(iff);
register_control(if_false, _ltree_root, iff);
Node* if_true = new IfTrueNode(iff);
register_control(if_true, loop, iff);
C->add_parse_predicate_opaq(opaq);
int trap_request = Deoptimization::make_trap_request(reason, Deoptimization::Action_maybe_recompile);
address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point();
@ -597,7 +597,7 @@ void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Nod
Node *ret = new ParmNode(C->start(), TypeFunc::ReturnAdr);
register_new_node(ret, C->start());
unc->init_req(TypeFunc::Control, iffalse);
unc->init_req(TypeFunc::Control, if_false);
unc->init_req(TypeFunc::I_O, i_o);
unc->init_req(TypeFunc::Memory, mem); // may gc ptrs
unc->init_req(TypeFunc::FramePtr, frame);
@ -609,7 +609,7 @@ void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Nod
for (uint i = TypeFunc::Parms; i < unc->req(); i++) {
set_subtree_ctrl(unc->in(i), false);
}
register_control(unc, _ltree_root, iffalse);
register_control(unc, _ltree_root, if_false);
Node* ctrl = new ProjNode(unc, TypeFunc::Control);
register_control(ctrl, _ltree_root, unc);
@ -617,8 +617,8 @@ void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Nod
register_control(halt, _ltree_root, ctrl);
_igvn.add_input_to(C->root(), halt);
_igvn.replace_input_of(inner_head, LoopNode::EntryControl, iftrue);
set_idom(inner_head, iftrue, dom_depth(inner_head));
_igvn.replace_input_of(inner_head, LoopNode::EntryControl, if_true);
set_idom(inner_head, if_true, dom_depth(inner_head));
}
}
@ -1788,9 +1788,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
assert(!x->as_Loop()->is_loop_nest_inner_loop(), "loop was transformed");
// Generate loop's limit check.
// Loop limit check predicate should be near the loop.
ParsePredicateSuccessProj* loop_limit_check_predicate_parse_proj =
find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check);
if (loop_limit_check_predicate_parse_proj == nullptr) {
if (!ParsePredicates::is_loop_limit_check_predicate_proj(init_control)) {
// The limit check predicate is not generated if this method trapped here before.
#ifdef ASSERT
if (TraceLoopLimitCheck) {
@ -1802,9 +1800,10 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
return false;
}
IfNode* check_iff = loop_limit_check_predicate_parse_proj->in(0)->as_If();
ParsePredicateSuccessProj* loop_limit_check_parse_predicate_proj = init_control->as_IfTrue();
ParsePredicateNode* parse_predicate = loop_limit_check_parse_predicate_proj->in(0)->as_ParsePredicate();
if (!is_dominator(get_ctrl(limit), check_iff->in(0))) {
if (!is_dominator(get_ctrl(limit), parse_predicate->in(0))) {
return false;
}
@ -1819,7 +1818,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
bol = new BoolNode(cmp_limit, BoolTest::ge);
}
insert_loop_limit_check_predicate(loop_limit_check_predicate_parse_proj, cmp_limit, bol);
insert_loop_limit_check_predicate(loop_limit_check_parse_predicate_proj, cmp_limit, bol);
}
// Now we need to canonicalize loop condition.
@ -1832,9 +1831,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
// 'ne' can be replaced with 'gt' only when init > limit.
bt = BoolTest::gt;
} else {
ParsePredicateSuccessProj* loop_limit_check_predicate_parse_proj =
find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check);
if (loop_limit_check_predicate_parse_proj == nullptr) {
if (!ParsePredicates::is_loop_limit_check_predicate_proj(init_control)) {
// The limit check predicate is not generated if this method trapped here before.
#ifdef ASSERT
if (TraceLoopLimitCheck) {
@ -1845,10 +1842,11 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
#endif
return false;
}
IfNode* check_iff = loop_limit_check_predicate_parse_proj->in(0)->as_If();
ParsePredicateSuccessProj* loop_limit_check_parse_predicate_proj = init_control->as_IfTrue();
ParsePredicateNode* parse_predicate = init_control->in(0)->as_ParsePredicate();
if (!is_dominator(get_ctrl(limit), check_iff->in(0)) ||
!is_dominator(get_ctrl(init_trip), check_iff->in(0))) {
if (!is_dominator(get_ctrl(limit), parse_predicate->in(0)) ||
!is_dominator(get_ctrl(init_trip), parse_predicate->in(0))) {
return false;
}
@ -1863,7 +1861,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_
bol = new BoolNode(cmp_limit, BoolTest::gt);
}
insert_loop_limit_check_predicate(loop_limit_check_predicate_parse_proj, cmp_limit, bol);
insert_loop_limit_check_predicate(loop_limit_check_parse_predicate_proj, cmp_limit, bol);
if (stride_con > 0) {
// 'ne' can be replaced with 'lt' only when init < limit.
@ -3954,24 +3952,17 @@ void IdealLoopTree::dump_head() {
tty->sp(2 * _nest);
tty->print("Loop: N%d/N%d ", _head->_idx, _tail->_idx);
if (_irreducible) tty->print(" IRREDUCIBLE");
Node* entry = _head->is_Loop() ? _head->as_Loop()->skip_strip_mined(-1)->in(LoopNode::EntryControl) : _head->in(LoopNode::EntryControl);
Node* predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate != nullptr ) {
Node* entry = _head->is_Loop() ? _head->as_Loop()->skip_strip_mined(-1)->in(LoopNode::EntryControl)
: _head->in(LoopNode::EntryControl);
ParsePredicates parse_predicates(entry);
if (parse_predicates.loop_limit_check_predicate_proj() != nullptr) {
tty->print(" limit_check");
entry = PhaseIdealLoop::skip_related_predicates(entry);
}
if (UseProfiledLoopPredicate) {
predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (predicate != nullptr) {
tty->print(" profile_predicated");
entry = PhaseIdealLoop::skip_related_predicates(entry);
}
if (UseProfiledLoopPredicate && parse_predicates.profiled_loop_predicate_proj() != nullptr) {
tty->print(" profile_predicated");
}
if (UseLoopPredicate) {
predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (predicate != nullptr) {
tty->print(" predicated");
}
if (UseLoopPredicate && parse_predicates.loop_predicate_proj() != nullptr) {
tty->print(" predicated");
}
if (_head->is_CountedLoop()) {
CountedLoopNode *cl = _head->as_CountedLoop();
@ -4078,27 +4069,25 @@ void PhaseIdealLoop::collect_potentially_useful_predicates(IdealLoopTree* loop,
!loop->tail()->is_top()) {
LoopNode* lpn = loop->_head->as_Loop();
Node* entry = lpn->in(LoopNode::EntryControl);
Node* predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check);
if (predicate != nullptr) { // right pattern that can be used by loop predication
assert(entry->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1, "must be");
useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one
entry = skip_related_predicates(entry);
ParsePredicates parse_predicates(entry);
ProjNode* predicate_proj = parse_predicates.loop_limit_check_predicate_proj();
if (predicate_proj != nullptr) { // right pattern that can be used by loop predication
assert(predicate_proj->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1, "must be");
useful_predicates.push(predicate_proj->in(0)->in(1)->in(1)); // good one
}
if (UseProfiledLoopPredicate) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate);
if (predicate != nullptr) { // right pattern that can be used by loop predication
useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one
get_assertion_predicates(entry, useful_predicates, true);
entry = skip_related_predicates(entry);
predicate_proj = parse_predicates.profiled_loop_predicate_proj();
if (predicate_proj != nullptr) { // right pattern that can be used by loop predication
useful_predicates.push(predicate_proj->in(0)->in(1)->in(1)); // good one
get_assertion_predicates(predicate_proj, useful_predicates, true);
}
}
if (UseLoopPredicate) {
predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
if (predicate != nullptr) { // right pattern that can be used by loop predication
useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one
get_assertion_predicates(entry, useful_predicates, true);
predicate_proj = parse_predicates.loop_predicate_proj();
if (predicate_proj != nullptr) { // right pattern that can be used by loop predication
useful_predicates.push(predicate_proj->in(0)->in(1)->in(1)); // good one
get_assertion_predicates(predicate_proj, useful_predicates, true);
}
}
}

102
src/hotspot/share/opto/loopnode.hpp
View File

@ -38,6 +38,7 @@ class IdealLoopTree;
class LoopNode;
class Node;
class OuterStripMinedLoopEndNode;
class ParsePredicates;
class PathFrequency;
class PhaseIdealLoop;
class CountedLoopReserveKit;
@ -1360,46 +1361,6 @@ public:
public:
void register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body = true);
static Node* skip_all_predicates(Node* entry);
static Node* skip_related_predicates(Node* entry);
static IfProjNode* next_predicate(IfProjNode* predicate_proj);
// Find a good location to insert a predicate
static ParsePredicateSuccessProj* find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason);
class ParsePredicates {
private:
ParsePredicateSuccessProj* _loop_predicate = nullptr;
ParsePredicateSuccessProj* _profiled_loop_predicate = nullptr;
ParsePredicateSuccessProj* _loop_limit_check_predicate = nullptr;
Node* _first_predicate = nullptr;
public:
// given loop entry, find all predicates above loop
ParsePredicates(Node* entry);
// Proj of Loop Limit Check Parse Predicate.
ParsePredicateSuccessProj* loop_limit_check_predicate() {
return _loop_limit_check_predicate;
}
// Proj of Profile Loop Parse Predicate.
ParsePredicateSuccessProj* profiled_loop_predicate() {
return _profiled_loop_predicate;
}
// Proj of Loop Parse Predicate.
ParsePredicateSuccessProj* loop_predicate() {
return _loop_predicate;
}
// Proj of first Parse Predicate when walking the graph down from root.
Node* get_first_predicate() {
return _first_predicate;
}
};
// Find a predicate
static Node* find_parse_predicate(Node* entry);
// Construct a range check for a predicate if
BoolNode* rc_predicate(IdealLoopTree *loop, Node* ctrl,
int scale, Node* offset,
@ -1680,7 +1641,7 @@ private:
IfProjNode* clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate,
Deoptimization::DeoptReason reason,
IfProjNode* output_proj);
static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry) PRODUCT_RETURN;
static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN;
bool _created_loop_node;
DEBUG_ONLY(void dump_idoms(Node* early, Node* wrong_lca);)
@ -1952,4 +1913,63 @@ public:
float to(Node* n);
};
// Utility class to work on predicates.
class Predicates {
public:
static Node* skip_all_predicates(Node* node);
static Node* skip_all_predicates(ParsePredicates& parse_predicates);
static Node* skip_predicates_in_block(ParsePredicateSuccessProj* parse_predicate_success_proj);
static IfProjNode* next_predicate_proj_in_block(IfProjNode* proj);
static bool has_profiled_loop_predicates(ParsePredicates& parse_predicates);
};
// Class representing the Parse Predicates that are added during parsing with ParsePredicateNodes.
class ParsePredicates {
private:
ParsePredicateSuccessProj* _loop_predicate_proj = nullptr;
ParsePredicateSuccessProj* _profiled_loop_predicate_proj = nullptr;
ParsePredicateSuccessProj* _loop_limit_check_predicate_proj = nullptr;
// The success projection of the Parse Predicate that comes first when starting from root.
ParsePredicateSuccessProj* _top_predicate_proj;
ParsePredicateSuccessProj* _starting_proj;
void find_parse_predicate_projections();
static bool is_uct_proj(Node* node, Deoptimization::DeoptReason deopt_reason);
static ParsePredicateNode* get_parse_predicate_or_null(Node* proj);
bool assign_predicate_proj(ParsePredicateSuccessProj* parse_predicate_proj);
public:
ParsePredicates(Node* starting_proj);
// Success projection of Loop Parse Predicate.
ParsePredicateSuccessProj* loop_predicate_proj() {
return _loop_predicate_proj;
}
// Success proj of Profiled Loop Parse Predicate.
ParsePredicateSuccessProj* profiled_loop_predicate_proj() {
return _profiled_loop_predicate_proj;
}
// Success proj of Loop Limit Check Parse Predicate.
ParsePredicateSuccessProj* loop_limit_check_predicate_proj() {
return _loop_limit_check_predicate_proj;
}
// Return the success projection of the Parse Predicate that comes first when starting from root.
ParsePredicateSuccessProj* get_top_predicate_proj() {
return _top_predicate_proj;
}
static bool is_success_proj(Node* node);
// Are there any Parse Predicates?
bool has_any() const {
return _top_predicate_proj != nullptr;
}
static bool is_loop_limit_check_predicate_proj(Node* node) {
ParsePredicateNode* parse_predicate = get_parse_predicate_or_null(node);
return parse_predicate != nullptr && parse_predicate->deopt_reason() == Deoptimization::DeoptReason::Reason_loop_limit_check;
}
};
#endif // SHARE_OPTO_LOOPNODE_HPP

3
src/hotspot/share/opto/loopopts.cpp
View File

@ -262,7 +262,8 @@ void PhaseIdealLoop::dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip, b
assert(iff->Opcode() == Op_If ||
iff->Opcode() == Op_CountedLoopEnd ||
iff->Opcode() == Op_LongCountedLoopEnd ||
iff->Opcode() == Op_RangeCheck,
iff->Opcode() == Op_RangeCheck ||
iff->Opcode() == Op_ParsePredicate,
"Check this code when new subtype is added");
int pop = prevdom->Opcode();

8
src/hotspot/share/opto/node.hpp
View File

@ -139,6 +139,7 @@ class Node_List;
class Node_Stack;
class OopMap;
class ParmNode;
class ParsePredicateNode;
class PCTableNode;
class PhaseCCP;
class PhaseGVN;
@ -668,6 +669,7 @@ public:
DEFINE_CLASS_ID(LongCountedLoopEnd, BaseCountedLoopEnd, 1)
DEFINE_CLASS_ID(RangeCheck, If, 1)
DEFINE_CLASS_ID(OuterStripMinedLoopEnd, If, 2)
DEFINE_CLASS_ID(ParsePredicate, If, 3)
DEFINE_CLASS_ID(NeverBranch, MultiBranch, 2)
DEFINE_CLASS_ID(Start, Multi, 2)
DEFINE_CLASS_ID(MemBar, Multi, 3)
@ -942,6 +944,7 @@ public:
DEFINE_CLASS_QUERY(OuterStripMinedLoop)
DEFINE_CLASS_QUERY(OuterStripMinedLoopEnd)
DEFINE_CLASS_QUERY(Parm)
DEFINE_CLASS_QUERY(ParsePredicate)
DEFINE_CLASS_QUERY(PCTable)
DEFINE_CLASS_QUERY(Phi)
DEFINE_CLASS_QUERY(Proj)
@ -1025,6 +1028,11 @@ public:
bool for_post_loop_opts_igvn() const { return (_flags & Flag_for_post_loop_opts_igvn) != 0; }
// Is 'n' possibly a loop entry (i.e. a Parse Predicate projection)?
static bool may_be_loop_entry(Node* n) {
return n != nullptr && n->is_IfProj() && n->in(0)->is_ParsePredicate();
}
//----------------- Optimization
// Get the worst-case Type output for this Node.