diff --git a/src/hotspot/share/opto/cfgnode.cpp b/src/hotspot/share/opto/cfgnode.cpp index c4381b0f93b..0329986e2e6 100644 --- a/src/hotspot/share/opto/cfgnode.cpp +++ b/src/hotspot/share/opto/cfgnode.cpp @@ -1796,8 +1796,8 @@ static Node* split_flow_path(PhaseGVN *phase, PhiNode *phi) { if( phase->type(n) == Type::TOP ) return nullptr; if( phi->in(i) == val ) { hit++; - if (PhaseIdealLoop::find_parse_predicate(r->in(i)) != nullptr) { - return nullptr; // don't split loop entry path + if (Node::may_be_loop_entry(r->in(i))) { + return nullptr; // don't split loop entry path } } } diff --git a/src/hotspot/share/opto/cfgnode.hpp b/src/hotspot/share/opto/cfgnode.hpp index 92ed3cfeac4..c2ab2c1bab8 100644 --- a/src/hotspot/share/opto/cfgnode.hpp +++ b/src/hotspot/share/opto/cfgnode.hpp @@ -452,6 +452,26 @@ public: virtual Node* Ideal(PhaseGVN *phase, bool can_reshape); }; +// Special node that denotes a Parse Predicate added during parsing. A Parse Predicate serves as placeholder to later +// create Runtime Predicates above it. They all share the same uncommon trap. The Parse Predicate will follow the +// Runtime Predicates. Together they form a Regular Predicate Block. There are three kinds of Parse Predicates: +// Loop Parse Predicate, Profiled Loop Parse Predicate (both used by Loop Predication), and Loop Limit Check Parse +// Predicate (used for integer overflow checks when creating a counted loop). +// More information about predicates can be found in loopPredicate.cpp. +class ParsePredicateNode : public IfNode { + Deoptimization::DeoptReason _deopt_reason; + public: + ParsePredicateNode(Node* control, Node* bol, Deoptimization::DeoptReason deopt_reason); + virtual int Opcode() const; + virtual uint size_of() const { return sizeof(*this); } + + Deoptimization::DeoptReason deopt_reason() const { + return _deopt_reason; + } + + NOT_PRODUCT(void dump_spec(outputStream* st) const;) +}; + class IfProjNode : public CProjNode { public: IfProjNode(IfNode *ifnode, uint idx) : CProjNode(ifnode,idx) {} diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index ca8d0614def..892ccc6b8ab 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -282,6 +282,7 @@ macro(OverflowSubL) macro(OverflowMulL) macro(PCTable) macro(Parm) +macro(ParsePredicate) macro(PartialSubtypeCheck) macro(SubTypeCheck) macro(Phi) diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index 58c4191f7ff..4a6f5eee946 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -3995,20 +3995,21 @@ void GraphKit::add_parse_predicate(Deoptimization::DeoptReason reason, const int return; } - Node* cont = _gvn.intcon(1); - Node* opq = _gvn.transform(new Opaque1Node(C, cont)); - Node* bol = _gvn.transform(new Conv2BNode(opq)); - IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN); - Node* iffalse = _gvn.transform(new IfFalseNode(iff)); - C->add_parse_predicate_opaq(opq); + Node* cont = _gvn.intcon(1); + Node* opaq = _gvn.transform(new Opaque1Node(C, cont)); + C->add_parse_predicate_opaq(opaq); + Node* bol = _gvn.transform(new Conv2BNode(opaq)); + ParsePredicateNode* parse_predicate = new ParsePredicateNode(control(), bol, reason); + _gvn.set_type(parse_predicate, parse_predicate->Value(&_gvn)); + Node* if_false = _gvn.transform(new IfFalseNode(parse_predicate)); { PreserveJVMState pjvms(this); - set_control(iffalse); + set_control(if_false); inc_sp(nargs); uncommon_trap(reason, Deoptimization::Action_maybe_recompile); } - Node* iftrue = _gvn.transform(new IfTrueNode(iff)); - set_control(iftrue); + Node* if_true = _gvn.transform(new IfTrueNode(parse_predicate)); + set_control(if_true); } // Add Parse Predicates which serve as placeholders to create new Runtime Predicates above them. All diff --git a/src/hotspot/share/opto/ifnode.cpp b/src/hotspot/share/opto/ifnode.cpp index 2eff1ce9d85..65a58341f3f 100644 --- a/src/hotspot/share/opto/ifnode.cpp +++ b/src/hotspot/share/opto/ifnode.cpp @@ -242,9 +242,8 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) { if (phi->in(ii) == con1) { req_c++; } - Node* proj = PhaseIdealLoop::find_parse_predicate(r->in(ii)); - if (proj != nullptr) { - // Bail out if splitting through a region with a predicate input (could + if (Node::may_be_loop_entry(r->in(ii))) { + // Bail out if splitting through a region with a Parse Predicate input (could // also be a loop header before loop opts creates a LoopNode for it). return nullptr; } @@ -1969,3 +1968,39 @@ Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) { // Replace dominated IfNode return dominated_by(prev_dom, igvn); } + +ParsePredicateNode::ParsePredicateNode(Node* control, Node* bol, Deoptimization::DeoptReason deopt_reason) + : IfNode(control, bol, PROB_MAX, COUNT_UNKNOWN), + _deopt_reason(deopt_reason) { + init_class_id(Class_ParsePredicate); + assert(bol->Opcode() == Op_Conv2B && bol->in(1) != nullptr && bol->in(1)->is_Opaque1(), "wrong boolean input"); +#ifdef ASSERT + switch (deopt_reason) { + case Deoptimization::Reason_predicate: + case Deoptimization::Reason_profile_predicate: + case Deoptimization::Reason_loop_limit_check: + break; + default: + assert(false, "unsupported deoptimization reason for Parse Predicate"); + } +#endif // ASSERT +} + +#ifndef PRODUCT +void ParsePredicateNode::dump_spec(outputStream* st) const { + st->print(" #"); + switch (_deopt_reason) { + case Deoptimization::DeoptReason::Reason_predicate: + st->print("Loop "); + break; + case Deoptimization::DeoptReason::Reason_profile_predicate: + st->print("Profiled_Loop "); + break; + case Deoptimization::DeoptReason::Reason_loop_limit_check: + st->print("Loop_Limit_Check "); + break; + default: + fatal("unknown kind"); + } +} +#endif // NOT PRODUCT diff --git a/src/hotspot/share/opto/loopPredicate.cpp b/src/hotspot/share/opto/loopPredicate.cpp index 067b7c44dee..ab6dbf945cf 100644 --- a/src/hotspot/share/opto/loopPredicate.cpp +++ b/src/hotspot/share/opto/loopPredicate.cpp @@ -307,11 +307,18 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, N // Create new_iff IdealLoopTree* lp = get_loop(entry); IfNode* new_iff = nullptr; - if (opcode == Op_If) { - new_iff = new IfNode(entry, iff->in(1), iff->_prob, iff->_fcnt); - } else { - assert(opcode == Op_RangeCheck, "no other if variant here"); - new_iff = new RangeCheckNode(entry, iff->in(1), iff->_prob, iff->_fcnt); + switch (opcode) { + case Op_If: + new_iff = new IfNode(entry, iff->in(1), iff->_prob, iff->_fcnt); + break; + case Op_RangeCheck: + new_iff = new RangeCheckNode(entry, iff->in(1), iff->_prob, iff->_fcnt); + break; + case Op_ParsePredicate: + new_iff = new ParsePredicateNode(entry, iff->in(1), reason); + break; + default: + fatal("no other If variant here"); } register_control(new_iff, lp, entry); IfProjNode* if_cont; @@ -471,7 +478,7 @@ IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredic Node* new_entry, Deoptimization::DeoptReason reason, const bool slow_loop) { - IfProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_If, + IfProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_ParsePredicate, slow_loop); IfNode* iff = new_predicate_proj->in(0)->as_If(); Node* ctrl = iff->in(0); @@ -586,160 +593,64 @@ IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(Node* void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) { LoopNode* head = loop->_head->as_Loop(); - bool clone_limit_check = !head->is_CountedLoop(); Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); - // Search original predicates - ParsePredicateSuccessProj* limit_check_proj = nullptr; - limit_check_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (limit_check_proj != nullptr) { - entry = skip_related_predicates(entry); - } - ParsePredicateSuccessProj* profile_predicate_proj = nullptr; - ParsePredicateSuccessProj* predicate_proj = nullptr; - if (UseProfiledLoopPredicate) { - profile_predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (profile_predicate_proj != nullptr) { - entry = skip_related_predicates(entry); - } - } - if (UseLoopPredicate) { - predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); - } - if (predicate_proj != nullptr) { // right pattern that can be used by loop predication - // clone predicate - iffast_pred = clone_parse_predicate_to_unswitched_loop(predicate_proj, iffast_pred, Deoptimization::Reason_predicate, false); - ifslow_pred = clone_parse_predicate_to_unswitched_loop(predicate_proj, ifslow_pred, Deoptimization::Reason_predicate, true); - clone_assertion_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_predicate, predicate_proj, + ParsePredicates parse_predicates(entry); + ParsePredicateSuccessProj* loop_predicate_proj = parse_predicates.loop_predicate_proj(); + if (loop_predicate_proj != nullptr) { + // Clone Parse Predicate and Template Assertion Predicates of the Loop Predicate Block. + iffast_pred = clone_parse_predicate_to_unswitched_loop(loop_predicate_proj, iffast_pred, + Deoptimization::Reason_predicate, false); + check_cloned_parse_predicate_for_unswitching(iffast_pred, true); + + ifslow_pred = clone_parse_predicate_to_unswitched_loop(loop_predicate_proj, ifslow_pred, + Deoptimization::Reason_predicate, true); + check_cloned_parse_predicate_for_unswitching(ifslow_pred, false); + + clone_assertion_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_predicate, loop_predicate_proj, iffast_pred, ifslow_pred); - - check_cloned_parse_predicate_for_unswitching(iffast_pred); - check_cloned_parse_predicate_for_unswitching(ifslow_pred); } - if (profile_predicate_proj != nullptr) { // right pattern that can be used by loop predication - // clone predicate - iffast_pred = clone_parse_predicate_to_unswitched_loop(profile_predicate_proj, iffast_pred,Deoptimization::Reason_profile_predicate, false); - ifslow_pred = clone_parse_predicate_to_unswitched_loop(profile_predicate_proj, ifslow_pred,Deoptimization::Reason_profile_predicate, true); + + ParsePredicateSuccessProj* profiled_loop_predicate_proj = parse_predicates.profiled_loop_predicate_proj(); + if (profiled_loop_predicate_proj != nullptr) { + // Clone Parse Predicate and Template Assertion Predicates of the Profiled Loop Predicate Block. + iffast_pred = clone_parse_predicate_to_unswitched_loop(profiled_loop_predicate_proj, iffast_pred, + Deoptimization::Reason_profile_predicate, false); + check_cloned_parse_predicate_for_unswitching(iffast_pred, true); + + ifslow_pred = clone_parse_predicate_to_unswitched_loop(profiled_loop_predicate_proj, ifslow_pred, + Deoptimization::Reason_profile_predicate, true); + check_cloned_parse_predicate_for_unswitching(ifslow_pred, false); + clone_assertion_predicates_to_unswitched_loop(loop, old_new, Deoptimization::Reason_profile_predicate, - profile_predicate_proj, iffast_pred, ifslow_pred); + profiled_loop_predicate_proj, iffast_pred, ifslow_pred); - check_cloned_parse_predicate_for_unswitching(iffast_pred); - check_cloned_parse_predicate_for_unswitching(ifslow_pred); } - if (limit_check_proj != nullptr && clone_limit_check) { - // Clone loop limit check last to insert it before loop. - // Don't clone a limit check which was already finalized - // for this counted loop (only one limit check is needed). - iffast_pred = clone_parse_predicate_to_unswitched_loop(limit_check_proj, iffast_pred,Deoptimization::Reason_loop_limit_check, false); - ifslow_pred = clone_parse_predicate_to_unswitched_loop(limit_check_proj, ifslow_pred,Deoptimization::Reason_loop_limit_check, true); + ParsePredicateSuccessProj* loop_limit_check_predicate_proj = parse_predicates.loop_limit_check_predicate_proj(); + if (loop_limit_check_predicate_proj != nullptr && !head->is_CountedLoop()) { + // Don't clone the Loop Limit Check Parse Predicate if we already have a counted loop (a Loop Limit Check Predicate + // is only created when converting a LoopNode to a CountedLoopNode). + iffast_pred = clone_parse_predicate_to_unswitched_loop(loop_limit_check_predicate_proj, iffast_pred, + Deoptimization::Reason_loop_limit_check, false); + check_cloned_parse_predicate_for_unswitching(iffast_pred, true); - check_cloned_parse_predicate_for_unswitching(iffast_pred); - check_cloned_parse_predicate_for_unswitching(ifslow_pred); + ifslow_pred = clone_parse_predicate_to_unswitched_loop(loop_limit_check_predicate_proj, ifslow_pred, + Deoptimization::Reason_loop_limit_check, true); + check_cloned_parse_predicate_for_unswitching(ifslow_pred, false); } } #ifndef PRODUCT -void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry) { +void PhaseIdealLoop::check_cloned_parse_predicate_for_unswitching(const Node* new_entry, const bool is_fast_loop) { assert(new_entry != nullptr, "IfTrue or IfFalse after clone predicate"); if (TraceLoopPredicate) { - tty->print("Loop Predicate cloned: "); - debug_only(new_entry->in(0)->dump();); + tty->print("Parse Predicate cloned to %s loop: ", is_fast_loop ? "fast" : "slow"); + new_entry->in(0)->dump(); } } #endif -Node* PhaseIdealLoop::skip_related_predicates(Node* entry) { - IfNode* iff = entry->in(0)->as_If(); - ProjNode* uncommon_proj = iff->proj_out(1 - entry->as_Proj()->_con); - Node* rgn = uncommon_proj->unique_ctrl_out(); - assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); - entry = entry->in(0)->in(0); - while (entry != nullptr && entry->is_Proj() && entry->in(0)->is_If()) { - uncommon_proj = entry->in(0)->as_If()->proj_out(1 - entry->as_Proj()->_con); - if (uncommon_proj->unique_ctrl_out() != rgn) - break; - entry = entry->in(0)->in(0); - } - return entry; -} - -Node* PhaseIdealLoop::skip_all_predicates(Node* entry) { - ParsePredicates parse_predicates(entry); - return parse_predicates.get_first_predicate(); -} - -//--------------------------next_predicate--------------------------------- -// Find next related predicate, useful for iterating over all related predicates -IfProjNode* PhaseIdealLoop::next_predicate(IfProjNode* predicate_proj) { - IfNode* iff = predicate_proj->in(0)->as_If(); - ProjNode* uncommon_proj = iff->proj_out(1 - predicate_proj->_con); - Node* rgn = uncommon_proj->unique_ctrl_out(); - assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); - Node* next = iff->in(0); - if (next != nullptr && next->is_IfProj() && next->in(0)->is_If()) { - uncommon_proj = next->in(0)->as_If()->proj_out(1 - next->as_Proj()->_con); - if (uncommon_proj->unique_ctrl_out() == rgn) { // lead into same region - return next->as_IfProj(); - } - } - return nullptr; -} - -//--------------------------find_predicate_insertion_point------------------- -// Find a good location to insert a predicate -ParsePredicateSuccessProj* PhaseIdealLoop::find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason) { - if (start_c == nullptr || !start_c->is_IfTrue()) - return nullptr; - if (start_c->as_IfTrue()->is_uncommon_trap_if_pattern(reason)) { - return start_c->as_IfTrue(); - } - return nullptr; -} - -//--------------------------Predicates::Predicates-------------------------- -// given loop entry, find all predicates above loop -PhaseIdealLoop::ParsePredicates::ParsePredicates(Node* entry) { - _loop_limit_check_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (_loop_limit_check_predicate != nullptr) { - entry = skip_related_predicates(entry); - } - if (UseProfiledLoopPredicate) { - _profiled_loop_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (_profiled_loop_predicate != nullptr) { - entry = skip_related_predicates(entry); - } - } - if (UseLoopPredicate) { - _loop_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); - if (_loop_predicate != nullptr) { - entry = skip_related_predicates(entry); - } - } - _first_predicate = entry; -} - -Node* PhaseIdealLoop::find_parse_predicate(Node* entry) { - Node* predicate = nullptr; - predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (predicate != nullptr) { // right pattern that can be used by loop predication - return entry; - } - if (UseLoopPredicate) { - predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); - if (predicate != nullptr) { // right pattern that can be used by loop predication - return entry; - } - } - if (UseProfiledLoopPredicate) { - predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (predicate != nullptr) { // right pattern that can be used by loop predication - return entry; - } - } - return nullptr; -} - //------------------------------Invariance----------------------------------- // Helper class for loop_predication_impl to compute invariance on the fly and // clone invariants. @@ -1582,8 +1493,7 @@ IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL return new_proj; } -//------------------------------ loop_predication_impl-------------------------- -// Insert loop predicates for null checks and range checks +// Insert Hoisted Predicates for null checks and range checks and additional Template Assertion Predicates for range checks. bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { if (!UseLoopPredicate) return false; @@ -1614,32 +1524,31 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { } Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); - ParsePredicateSuccessProj* loop_limit_proj = nullptr; - ParsePredicateSuccessProj* predicate_proj = nullptr; - ParsePredicateSuccessProj* profile_predicate_proj = nullptr; - // Loop limit check predicate should be near the loop. - loop_limit_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (loop_limit_proj != nullptr) { - entry = skip_related_predicates(loop_limit_proj); + ParsePredicates parse_predicates(entry); + + bool can_create_loop_predicates = true; + // We cannot add Loop Predicates if: + // - Already added Profiled Loop Predicates (Loop Predicates and Profiled Loop Predicates can be dependent + // through a data node, and thus we should only add new Profiled Loop Predicates which are below Loop Predicates + // in the graph). + // - There are currently no Profiled Loop Predicates, but we have a data node with a control dependency on the Loop + // Parse Predicate (could happen, for example, if we've removed an earlier created Profiled Loop Predicate with + // dominated_by()). We should not create a Loop Predicate for a check that is dependent on this data node because + // the Loop Predicate would end up above the data node with its dependency on the Loop Parse Predicate below. This + // would become unschedulable. However, we can still hoist the check as Profiled Loop Predicate which would end up + // below the Loop Parse Predicate. + if (Predicates::has_profiled_loop_predicates(parse_predicates) + || (parse_predicates.loop_predicate_proj() != nullptr && parse_predicates.loop_predicate_proj()->outcnt() != 1)) { + can_create_loop_predicates = false; } - bool has_profile_predicates = false; - profile_predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (profile_predicate_proj != nullptr) { - Node* n = skip_related_predicates(entry); - // Check if predicates were already added to the profile predicate - // block - if (n != entry->in(0)->in(0) || n->outcnt() != 1) { - has_profile_predicates = true; - } - entry = n; - } - predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); + ParsePredicateSuccessProj* loop_predicate_proj = parse_predicates.loop_predicate_proj(); + ParsePredicateSuccessProj* profiled_loop_predicate_proj = parse_predicates.profiled_loop_predicate_proj(); float loop_trip_cnt = -1; - bool follow_branches = loop_predication_should_follow_branches(loop, profile_predicate_proj, loop_trip_cnt); + bool follow_branches = loop_predication_should_follow_branches(loop, profiled_loop_predicate_proj, loop_trip_cnt); assert(!follow_branches || loop_trip_cnt >= 0, "negative trip count?"); - if (predicate_proj == nullptr && !follow_branches) { + if (loop_predicate_proj == nullptr && !follow_branches) { #ifndef PRODUCT if (TraceLoopPredicate) { tty->print("missing predicate:"); @@ -1680,7 +1589,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { bool hoisted = false; // true if at least one proj is promoted - if (!has_profile_predicates) { + if (can_create_loop_predicates) { while (if_proj_list.size() > 0) { Node* n = if_proj_list.pop(); @@ -1709,13 +1618,15 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { break; } - if (predicate_proj != nullptr) { - hoisted = loop_predication_impl_helper(loop, if_proj, predicate_proj, cl, zero, invar, Deoptimization::Reason_predicate) | hoisted; + if (loop_predicate_proj != nullptr) { + hoisted = loop_predication_impl_helper(loop, if_proj, loop_predicate_proj, cl, zero, invar, + Deoptimization::Reason_predicate) | hoisted; } } // end while } if (follow_branches) { + assert(profiled_loop_predicate_proj != nullptr, "sanity check"); PathFrequency pf(loop->_head, this); // Some projections were skipped by regular predicates because of @@ -1725,7 +1636,8 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { float f = pf.to(if_proj); if (if_proj->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) && f * loop_trip_cnt >= 1) { - hoisted = loop_predication_impl_helper(loop, if_proj->as_IfProj(), profile_predicate_proj, cl, zero, invar, Deoptimization::Reason_profile_predicate) | hoisted; + hoisted = loop_predication_impl_helper(loop, if_proj->as_IfProj(), profiled_loop_predicate_proj, cl, zero, invar, + Deoptimization::Reason_profile_predicate) | hoisted; } } @@ -1740,7 +1652,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) { for (uint i = 0; i < if_proj_list_freq.size(); i++) { IfProjNode* if_proj = if_proj_list_freq.at(i)->as_IfProj(); - hoisted = loop_predication_impl_helper(loop, if_proj, profile_predicate_proj, cl, zero, invar, Deoptimization::Reason_profile_predicate) | hoisted; + hoisted = loop_predication_impl_helper(loop, if_proj, profiled_loop_predicate_proj, cl, zero, invar, Deoptimization::Reason_profile_predicate) | hoisted; } } @@ -1778,3 +1690,140 @@ bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) { return hoisted; } + +// Skip over all predicates (all Regular Predicate Blocks) starting at the Parse Predicate projection 'node'. Return the +// first node that is not a predicate If node anymore (i.e. entry into the first predicate If on top) or 'node' if 'node' +// is not a Parse Predicate projection. +Node* Predicates::skip_all_predicates(Node* node) { + ParsePredicates parse_predicates(node); + if (parse_predicates.has_any()) { + return skip_all_predicates(parse_predicates); + } else { + return node; + } +} + +// Skip over all Runtime Predicates belonging to the given Parse Predicates. Return the first node that is not a predicate +// If node anymore (i.e. entry into the first predicate If on top). +Node* Predicates::skip_all_predicates(ParsePredicates& parse_predicates) { + assert(parse_predicates.has_any(), "must have at least one Parse Predicate"); + return skip_predicates_in_block(parse_predicates.get_top_predicate_proj()); +} + +// Skip over all predicates in a Regular Predicate Block starting at the Parse Predicate projection +// 'parse_predicate_success_proj'. Return the first node not belonging this block anymore (i.e. entry +// into this Regular Predicate Block). +Node* Predicates::skip_predicates_in_block(ParsePredicateSuccessProj* parse_predicate_success_proj) { + IfProjNode* prev; + IfProjNode* next = parse_predicate_success_proj; + do { + prev = next; + next = next_predicate_proj_in_block(next); + } while (next != nullptr); + assert(prev->in(0)->is_If(), "must be predicate If"); + return prev->in(0)->in(0); +} + +// Find next Runtime Predicate projection in a Regular Predicate Block or return null if there is none. +IfProjNode* Predicates::next_predicate_proj_in_block(IfProjNode* proj) { + IfNode* iff = proj->in(0)->as_If(); + ProjNode* uncommon_proj = iff->proj_out(1 - proj->_con); + Node* rgn = uncommon_proj->unique_ctrl_out(); + assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct"); + Node* next = iff->in(0); + if (next != nullptr && next->is_Proj() && next->in(0)->is_If()) { + uncommon_proj = next->in(0)->as_If()->proj_out(1 - next->as_Proj()->_con); + if (uncommon_proj->unique_ctrl_out() == rgn) { + // Same Runtime Predicate Block. + return next->as_IfProj(); + } + } + return nullptr; +} + +// Is there at least one Profiled Loop Predicate? +bool Predicates::has_profiled_loop_predicates(ParsePredicates& parse_predicates) { + ParsePredicateSuccessProj* profiled_loop_predicate = parse_predicates.profiled_loop_predicate_proj(); + if (profiled_loop_predicate == nullptr) { + return false; + } + return Predicates::next_predicate_proj_in_block(profiled_loop_predicate) != nullptr; +} + +// Given a node 'starting_proj', check if it is a Parse Predicate success projection. +// If so, find all Parse Predicates above the loop. +ParsePredicates::ParsePredicates(Node* starting_proj) : _top_predicate_proj(nullptr), _starting_proj(nullptr) { + if (starting_proj == nullptr || !starting_proj->is_IfTrue()) { + return; // Not a predicate. + } + _starting_proj = starting_proj->as_IfTrue(); + find_parse_predicate_projections(); +} + +void ParsePredicates::find_parse_predicate_projections() { + Node* maybe_parse_predicate_proj = _starting_proj; + for (int i = 0; i < 3; i++) { // At most 3 Parse Predicates for a loop + if (!is_success_proj(maybe_parse_predicate_proj)) { + break; + } + ParsePredicateSuccessProj* parse_predicate_proj = maybe_parse_predicate_proj->as_IfTrue(); + if (!assign_predicate_proj(parse_predicate_proj)) { + // Found a Parse Predicate of another (already removed) loop. + break; + } + _top_predicate_proj = parse_predicate_proj; + maybe_parse_predicate_proj = Predicates::skip_predicates_in_block(parse_predicate_proj); + } +} + +// Is 'node' a success (non-UCT) projection of a Parse Predicate? +bool ParsePredicates::is_success_proj(Node* node) { + if (node == nullptr || !node->is_Proj()) { + return false; + } + ParsePredicateNode* parse_predicate = get_parse_predicate_or_null(node); + if (parse_predicate == nullptr) { + return false; + } + return !is_uct_proj(node, parse_predicate->deopt_reason()); +} + +// Is 'node' a UCT projection of a Parse Predicate of kind 'kind'? +bool ParsePredicates::is_uct_proj(Node* node, Deoptimization::DeoptReason deopt_reason) { + return node->as_Proj()->is_uncommon_trap_proj(deopt_reason); +} + +// Check the parent of `parse_predicate_proj` is a ParsePredicateNode. If so return it. Otherwise, return null. +ParsePredicateNode* ParsePredicates::get_parse_predicate_or_null(Node* parse_predicate_proj) { + return parse_predicate_proj->in(0)->isa_ParsePredicate(); +} + +// Initialize the Parse Predicate projection field that matches the kind of the parent of `parse_predicate_proj`. +bool ParsePredicates::assign_predicate_proj(ParsePredicateSuccessProj* parse_predicate_proj) { + ParsePredicateNode* parse_predicate = get_parse_predicate_or_null(parse_predicate_proj); + assert(parse_predicate != nullptr, "must exist"); + Deoptimization::DeoptReason deopt_reason = parse_predicate->deopt_reason(); + switch (deopt_reason) { + case Deoptimization::DeoptReason::Reason_predicate: + if (_loop_predicate_proj != nullptr) { + return false; + } + _loop_predicate_proj = parse_predicate_proj; + break; + case Deoptimization::DeoptReason::Reason_profile_predicate: + if (_profiled_loop_predicate_proj != nullptr) { + return false; + } + _profiled_loop_predicate_proj = parse_predicate_proj; + break; + case Deoptimization::DeoptReason::Reason_loop_limit_check: + if (_loop_limit_check_predicate_proj != nullptr) { + return false; + } + _loop_limit_check_predicate_proj = parse_predicate_proj; + break; + default: + fatal("invalid case"); + } + return true; +} diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 2732d2962dc..fb2528a1dc7 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -489,7 +489,8 @@ uint IdealLoopTree::estimate_peeling(PhaseIdealLoop *phase) { assert(test->Opcode() == Op_If || test->Opcode() == Op_CountedLoopEnd || test->Opcode() == Op_LongCountedLoopEnd || - test->Opcode() == Op_RangeCheck, + test->Opcode() == Op_RangeCheck || + test->Opcode() == Op_ParsePredicate, "Check this code when new subtype is added"); // Condition is not a member of this loop? if (!is_member(phase->get_loop(ctrl)) && is_loop_exit(test)) { @@ -778,11 +779,11 @@ void PhaseIdealLoop::do_peeling(IdealLoopTree *loop, Node_List &old_new) { Node* stride = cl_head->stride(); IdealLoopTree* outer_loop = get_loop(outer_loop_head); ParsePredicates parse_predicates(new_head->in(LoopNode::EntryControl)); - initialize_assertion_predicates_for_peeled_loop(parse_predicates.loop_predicate(), + initialize_assertion_predicates_for_peeled_loop(parse_predicates.loop_predicate_proj(), outer_loop_head, dd_outer_loop_head, init, stride, outer_loop, idx_before_clone, old_new); - initialize_assertion_predicates_for_peeled_loop(parse_predicates.profiled_loop_predicate(), + initialize_assertion_predicates_for_peeled_loop(parse_predicates.profiled_loop_predicate_proj(), outer_loop_head, dd_outer_loop_head, init, stride, outer_loop, idx_before_clone, old_new); @@ -1590,24 +1591,14 @@ void PhaseIdealLoop::copy_assertion_predicates_to_main_loop(CountedLoopNode* pre const Node_List &old_new) { if (UseLoopPredicate) { Node* entry = pre_head->in(LoopNode::EntryControl); - Node* predicate = nullptr; - predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (predicate != nullptr) { - entry = skip_related_predicates(entry); - } - Node* profile_predicate = nullptr; - if (UseProfiledLoopPredicate) { - profile_predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (profile_predicate != nullptr) { - entry = skip_related_predicates(entry); - } - } - predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); - copy_assertion_predicates_to_main_loop_helper(predicate, init, stride, outer_loop, outer_main_head, dd_main_head, - idx_before_pre_post, idx_after_post_before_pre, zero_trip_guard_proj_main, + ParsePredicates parse_predicates(entry); + copy_assertion_predicates_to_main_loop_helper(parse_predicates.loop_predicate_proj(), init, stride, outer_loop, + outer_main_head, dd_main_head, idx_before_pre_post, + idx_after_post_before_pre, zero_trip_guard_proj_main, zero_trip_guard_proj_post, old_new); - copy_assertion_predicates_to_main_loop_helper(profile_predicate, init, stride, outer_loop, outer_main_head, dd_main_head, - idx_before_pre_post, idx_after_post_before_pre, zero_trip_guard_proj_main, + copy_assertion_predicates_to_main_loop_helper(parse_predicates.profiled_loop_predicate_proj(), init, stride, + outer_loop, outer_main_head, dd_main_head, idx_before_pre_post, + idx_after_post_before_pre, zero_trip_guard_proj_main, zero_trip_guard_proj_post, old_new); } } @@ -2140,7 +2131,7 @@ void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(IfProjNode* Node* control = outer_loop_head->in(LoopNode::EntryControl); Node* input_proj = control; - predicate_proj = next_predicate(predicate_proj); + predicate_proj = Predicates::next_predicate_proj_in_block(predicate_proj); while (predicate_proj != nullptr) { IfNode* iff = predicate_proj->in(0)->as_If(); if (iff->in(1)->Opcode() == Op_Opaque4) { @@ -2166,7 +2157,7 @@ void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(IfProjNode* } } } - predicate_proj = next_predicate(predicate_proj); + predicate_proj = Predicates::next_predicate_proj_in_block(predicate_proj); } _igvn.replace_input_of(outer_loop_head, LoopNode::EntryControl, input_proj); @@ -3534,7 +3525,7 @@ bool IdealLoopTree::do_remove_empty_loop(PhaseIdealLoop *phase) { } if (needs_guard) { // Check for an obvious zero trip guard. - Node* in_ctrl = PhaseIdealLoop::skip_all_predicates(cl->skip_predicates()); + Node* in_ctrl = Predicates::skip_all_predicates(cl->skip_predicates()); if (in_ctrl->Opcode() == Op_IfTrue || in_ctrl->Opcode() == Op_IfFalse) { bool maybe_swapped = (in_ctrl->Opcode() == Op_IfFalse); // The test should look like just the backedge of a CountedLoop diff --git a/src/hotspot/share/opto/loopUnswitch.cpp b/src/hotspot/share/opto/loopUnswitch.cpp index dfd75b268c0..e2d62965418 100644 --- a/src/hotspot/share/opto/loopUnswitch.cpp +++ b/src/hotspot/share/opto/loopUnswitch.cpp @@ -117,12 +117,9 @@ IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop) co // insert a clone of the test that selects which version to // execute. void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) { - LoopNode *head = loop->_head->as_Loop(); Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); - if (find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check) != nullptr - || (UseProfiledLoopPredicate && find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate) != nullptr) - || (UseLoopPredicate && find_predicate_insertion_point(entry, Deoptimization::Reason_predicate) != nullptr)) { + if (ParsePredicates::is_success_proj(entry)) { assert(entry->is_IfProj(), "sanity - must be ifProj since there is at least one predicate"); if (entry->outcnt() > 1) { // Bailout if there are predicates from which there are additional control dependencies (i.e. from loop @@ -154,29 +151,17 @@ void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) { #ifdef ASSERT assert(proj_true->is_IfTrue(), "must be true projection"); entry = head->skip_strip_mined()->in(LoopNode::EntryControl); - Node* parse_predicate = find_parse_predicate(entry); - if (parse_predicate == nullptr) { + ParsePredicates parse_predicates(entry); + if (!parse_predicates.has_any()) { // No Parse Predicate. Node* uniqc = proj_true->unique_ctrl_out(); assert((uniqc == head && !head->is_strip_mined()) || (uniqc == head->in(LoopNode::EntryControl) && head->is_strip_mined()), "must hold by construction if no predicates"); } else { - // There is at least one Parse Predicate. When calling 'skip_related_predicates' on each found Parse Predicate, - // we should end up at 'proj_true'. - Node* proj_before_first_parse_predicate = skip_related_predicates(entry); - if (UseProfiledLoopPredicate) { - parse_predicate = find_parse_predicate(proj_before_first_parse_predicate); - if (parse_predicate != nullptr) { - proj_before_first_parse_predicate = skip_related_predicates(parse_predicate); - } - } - if (UseLoopPredicate) { - parse_predicate = find_parse_predicate(proj_before_first_parse_predicate); - if (parse_predicate != nullptr) { - proj_before_first_parse_predicate = skip_related_predicates(parse_predicate); - } - } - assert(proj_true == proj_before_first_parse_predicate, "must hold by construction if at least one predicate"); + // There is at least one Parse Predicate. When skipping all predicates/Regular Predicate Blocks, we should end up + // at 'proj_true'. + assert(proj_true == Predicates::skip_all_predicates(parse_predicates), + "must hold by construction if at least one Parse Predicate"); } #endif // Increment unswitch count diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 4940154bef0..960db10296f 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -561,19 +561,19 @@ Node* PhaseIdealLoop::loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv, void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop, SafePointNode* sfpt) { if (!C->too_many_traps(reason)) { - Node *cont = _igvn.intcon(1); - Node* opq = new Opaque1Node(C, cont); - _igvn.register_new_node_with_optimizer(opq); - Node *bol = new Conv2BNode(opq); + Node* cont = _igvn.intcon(1); + Node* opaq = new Opaque1Node(C, cont); + _igvn.register_new_node_with_optimizer(opaq); + Node* bol = new Conv2BNode(opaq); _igvn.register_new_node_with_optimizer(bol); set_subtree_ctrl(bol, false); - IfNode* iff = new IfNode(inner_head->in(LoopNode::EntryControl), bol, PROB_MAX, COUNT_UNKNOWN); + ParsePredicateNode* iff = new ParsePredicateNode(inner_head->in(LoopNode::EntryControl), bol, reason); register_control(iff, loop, inner_head->in(LoopNode::EntryControl)); - Node* iffalse = new IfFalseNode(iff); - register_control(iffalse, _ltree_root, iff); - Node* iftrue = new IfTrueNode(iff); - register_control(iftrue, loop, iff); - C->add_parse_predicate_opaq(opq); + Node* if_false = new IfFalseNode(iff); + register_control(if_false, _ltree_root, iff); + Node* if_true = new IfTrueNode(iff); + register_control(if_true, loop, iff); + C->add_parse_predicate_opaq(opaq); int trap_request = Deoptimization::make_trap_request(reason, Deoptimization::Action_maybe_recompile); address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point(); @@ -597,7 +597,7 @@ void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Nod Node *ret = new ParmNode(C->start(), TypeFunc::ReturnAdr); register_new_node(ret, C->start()); - unc->init_req(TypeFunc::Control, iffalse); + unc->init_req(TypeFunc::Control, if_false); unc->init_req(TypeFunc::I_O, i_o); unc->init_req(TypeFunc::Memory, mem); // may gc ptrs unc->init_req(TypeFunc::FramePtr, frame); @@ -609,7 +609,7 @@ void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Nod for (uint i = TypeFunc::Parms; i < unc->req(); i++) { set_subtree_ctrl(unc->in(i), false); } - register_control(unc, _ltree_root, iffalse); + register_control(unc, _ltree_root, if_false); Node* ctrl = new ProjNode(unc, TypeFunc::Control); register_control(ctrl, _ltree_root, unc); @@ -617,8 +617,8 @@ void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Nod register_control(halt, _ltree_root, ctrl); _igvn.add_input_to(C->root(), halt); - _igvn.replace_input_of(inner_head, LoopNode::EntryControl, iftrue); - set_idom(inner_head, iftrue, dom_depth(inner_head)); + _igvn.replace_input_of(inner_head, LoopNode::EntryControl, if_true); + set_idom(inner_head, if_true, dom_depth(inner_head)); } } @@ -1788,9 +1788,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ assert(!x->as_Loop()->is_loop_nest_inner_loop(), "loop was transformed"); // Generate loop's limit check. // Loop limit check predicate should be near the loop. - ParsePredicateSuccessProj* loop_limit_check_predicate_parse_proj = - find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check); - if (loop_limit_check_predicate_parse_proj == nullptr) { + if (!ParsePredicates::is_loop_limit_check_predicate_proj(init_control)) { // The limit check predicate is not generated if this method trapped here before. #ifdef ASSERT if (TraceLoopLimitCheck) { @@ -1802,9 +1800,10 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ return false; } - IfNode* check_iff = loop_limit_check_predicate_parse_proj->in(0)->as_If(); + ParsePredicateSuccessProj* loop_limit_check_parse_predicate_proj = init_control->as_IfTrue(); + ParsePredicateNode* parse_predicate = loop_limit_check_parse_predicate_proj->in(0)->as_ParsePredicate(); - if (!is_dominator(get_ctrl(limit), check_iff->in(0))) { + if (!is_dominator(get_ctrl(limit), parse_predicate->in(0))) { return false; } @@ -1819,7 +1818,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ bol = new BoolNode(cmp_limit, BoolTest::ge); } - insert_loop_limit_check_predicate(loop_limit_check_predicate_parse_proj, cmp_limit, bol); + insert_loop_limit_check_predicate(loop_limit_check_parse_predicate_proj, cmp_limit, bol); } // Now we need to canonicalize loop condition. @@ -1832,9 +1831,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ // 'ne' can be replaced with 'gt' only when init > limit. bt = BoolTest::gt; } else { - ParsePredicateSuccessProj* loop_limit_check_predicate_parse_proj = - find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check); - if (loop_limit_check_predicate_parse_proj == nullptr) { + if (!ParsePredicates::is_loop_limit_check_predicate_proj(init_control)) { // The limit check predicate is not generated if this method trapped here before. #ifdef ASSERT if (TraceLoopLimitCheck) { @@ -1845,10 +1842,11 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ #endif return false; } - IfNode* check_iff = loop_limit_check_predicate_parse_proj->in(0)->as_If(); + ParsePredicateSuccessProj* loop_limit_check_parse_predicate_proj = init_control->as_IfTrue(); + ParsePredicateNode* parse_predicate = init_control->in(0)->as_ParsePredicate(); - if (!is_dominator(get_ctrl(limit), check_iff->in(0)) || - !is_dominator(get_ctrl(init_trip), check_iff->in(0))) { + if (!is_dominator(get_ctrl(limit), parse_predicate->in(0)) || + !is_dominator(get_ctrl(init_trip), parse_predicate->in(0))) { return false; } @@ -1863,7 +1861,7 @@ bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_ bol = new BoolNode(cmp_limit, BoolTest::gt); } - insert_loop_limit_check_predicate(loop_limit_check_predicate_parse_proj, cmp_limit, bol); + insert_loop_limit_check_predicate(loop_limit_check_parse_predicate_proj, cmp_limit, bol); if (stride_con > 0) { // 'ne' can be replaced with 'lt' only when init < limit. @@ -3954,24 +3952,17 @@ void IdealLoopTree::dump_head() { tty->sp(2 * _nest); tty->print("Loop: N%d/N%d ", _head->_idx, _tail->_idx); if (_irreducible) tty->print(" IRREDUCIBLE"); - Node* entry = _head->is_Loop() ? _head->as_Loop()->skip_strip_mined(-1)->in(LoopNode::EntryControl) : _head->in(LoopNode::EntryControl); - Node* predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (predicate != nullptr ) { + Node* entry = _head->is_Loop() ? _head->as_Loop()->skip_strip_mined(-1)->in(LoopNode::EntryControl) + : _head->in(LoopNode::EntryControl); + ParsePredicates parse_predicates(entry); + if (parse_predicates.loop_limit_check_predicate_proj() != nullptr) { tty->print(" limit_check"); - entry = PhaseIdealLoop::skip_related_predicates(entry); } - if (UseProfiledLoopPredicate) { - predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (predicate != nullptr) { - tty->print(" profile_predicated"); - entry = PhaseIdealLoop::skip_related_predicates(entry); - } + if (UseProfiledLoopPredicate && parse_predicates.profiled_loop_predicate_proj() != nullptr) { + tty->print(" profile_predicated"); } - if (UseLoopPredicate) { - predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); - if (predicate != nullptr) { - tty->print(" predicated"); - } + if (UseLoopPredicate && parse_predicates.loop_predicate_proj() != nullptr) { + tty->print(" predicated"); } if (_head->is_CountedLoop()) { CountedLoopNode *cl = _head->as_CountedLoop(); @@ -4078,27 +4069,25 @@ void PhaseIdealLoop::collect_potentially_useful_predicates(IdealLoopTree* loop, !loop->tail()->is_top()) { LoopNode* lpn = loop->_head->as_Loop(); Node* entry = lpn->in(LoopNode::EntryControl); - - Node* predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); - if (predicate != nullptr) { // right pattern that can be used by loop predication - assert(entry->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1, "must be"); - useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one - entry = skip_related_predicates(entry); + ParsePredicates parse_predicates(entry); + ProjNode* predicate_proj = parse_predicates.loop_limit_check_predicate_proj(); + if (predicate_proj != nullptr) { // right pattern that can be used by loop predication + assert(predicate_proj->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1, "must be"); + useful_predicates.push(predicate_proj->in(0)->in(1)->in(1)); // good one } if (UseProfiledLoopPredicate) { - predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); - if (predicate != nullptr) { // right pattern that can be used by loop predication - useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one - get_assertion_predicates(entry, useful_predicates, true); - entry = skip_related_predicates(entry); + predicate_proj = parse_predicates.profiled_loop_predicate_proj(); + if (predicate_proj != nullptr) { // right pattern that can be used by loop predication + useful_predicates.push(predicate_proj->in(0)->in(1)->in(1)); // good one + get_assertion_predicates(predicate_proj, useful_predicates, true); } } if (UseLoopPredicate) { - predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); - if (predicate != nullptr) { // right pattern that can be used by loop predication - useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one - get_assertion_predicates(entry, useful_predicates, true); + predicate_proj = parse_predicates.loop_predicate_proj(); + if (predicate_proj != nullptr) { // right pattern that can be used by loop predication + useful_predicates.push(predicate_proj->in(0)->in(1)->in(1)); // good one + get_assertion_predicates(predicate_proj, useful_predicates, true); } } } diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 8075b8296d7..e60a8bcd253 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -38,6 +38,7 @@ class IdealLoopTree; class LoopNode; class Node; class OuterStripMinedLoopEndNode; +class ParsePredicates; class PathFrequency; class PhaseIdealLoop; class CountedLoopReserveKit; @@ -1360,46 +1361,6 @@ public: public: void register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body = true); - static Node* skip_all_predicates(Node* entry); - static Node* skip_related_predicates(Node* entry); - static IfProjNode* next_predicate(IfProjNode* predicate_proj); - - // Find a good location to insert a predicate - static ParsePredicateSuccessProj* find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason); - - class ParsePredicates { - private: - ParsePredicateSuccessProj* _loop_predicate = nullptr; - ParsePredicateSuccessProj* _profiled_loop_predicate = nullptr; - ParsePredicateSuccessProj* _loop_limit_check_predicate = nullptr; - Node* _first_predicate = nullptr; - public: - // given loop entry, find all predicates above loop - ParsePredicates(Node* entry); - - // Proj of Loop Limit Check Parse Predicate. - ParsePredicateSuccessProj* loop_limit_check_predicate() { - return _loop_limit_check_predicate; - } - - // Proj of Profile Loop Parse Predicate. - ParsePredicateSuccessProj* profiled_loop_predicate() { - return _profiled_loop_predicate; - } - - // Proj of Loop Parse Predicate. - ParsePredicateSuccessProj* loop_predicate() { - return _loop_predicate; - } - - // Proj of first Parse Predicate when walking the graph down from root. - Node* get_first_predicate() { - return _first_predicate; - } - }; - - // Find a predicate - static Node* find_parse_predicate(Node* entry); // Construct a range check for a predicate if BoolNode* rc_predicate(IdealLoopTree *loop, Node* ctrl, int scale, Node* offset, @@ -1680,7 +1641,7 @@ private: IfProjNode* clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate, Deoptimization::DeoptReason reason, IfProjNode* output_proj); - static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry) PRODUCT_RETURN; + static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN; bool _created_loop_node; DEBUG_ONLY(void dump_idoms(Node* early, Node* wrong_lca);) @@ -1952,4 +1913,63 @@ public: float to(Node* n); }; +// Utility class to work on predicates. +class Predicates { + public: + static Node* skip_all_predicates(Node* node); + static Node* skip_all_predicates(ParsePredicates& parse_predicates); + static Node* skip_predicates_in_block(ParsePredicateSuccessProj* parse_predicate_success_proj); + static IfProjNode* next_predicate_proj_in_block(IfProjNode* proj); + static bool has_profiled_loop_predicates(ParsePredicates& parse_predicates); +}; + +// Class representing the Parse Predicates that are added during parsing with ParsePredicateNodes. +class ParsePredicates { + private: + ParsePredicateSuccessProj* _loop_predicate_proj = nullptr; + ParsePredicateSuccessProj* _profiled_loop_predicate_proj = nullptr; + ParsePredicateSuccessProj* _loop_limit_check_predicate_proj = nullptr; + // The success projection of the Parse Predicate that comes first when starting from root. + ParsePredicateSuccessProj* _top_predicate_proj; + ParsePredicateSuccessProj* _starting_proj; + + void find_parse_predicate_projections(); + static bool is_uct_proj(Node* node, Deoptimization::DeoptReason deopt_reason); + static ParsePredicateNode* get_parse_predicate_or_null(Node* proj); + bool assign_predicate_proj(ParsePredicateSuccessProj* parse_predicate_proj); + public: + ParsePredicates(Node* starting_proj); + + // Success projection of Loop Parse Predicate. + ParsePredicateSuccessProj* loop_predicate_proj() { + return _loop_predicate_proj; + } + + // Success proj of Profiled Loop Parse Predicate. + ParsePredicateSuccessProj* profiled_loop_predicate_proj() { + return _profiled_loop_predicate_proj; + } + + // Success proj of Loop Limit Check Parse Predicate. + ParsePredicateSuccessProj* loop_limit_check_predicate_proj() { + return _loop_limit_check_predicate_proj; + } + + // Return the success projection of the Parse Predicate that comes first when starting from root. + ParsePredicateSuccessProj* get_top_predicate_proj() { + return _top_predicate_proj; + } + + static bool is_success_proj(Node* node); + + // Are there any Parse Predicates? + bool has_any() const { + return _top_predicate_proj != nullptr; + } + + static bool is_loop_limit_check_predicate_proj(Node* node) { + ParsePredicateNode* parse_predicate = get_parse_predicate_or_null(node); + return parse_predicate != nullptr && parse_predicate->deopt_reason() == Deoptimization::DeoptReason::Reason_loop_limit_check; + } +}; #endif // SHARE_OPTO_LOOPNODE_HPP diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index ccfc9eb5f02..599d94a9f08 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -262,7 +262,8 @@ void PhaseIdealLoop::dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip, b assert(iff->Opcode() == Op_If || iff->Opcode() == Op_CountedLoopEnd || iff->Opcode() == Op_LongCountedLoopEnd || - iff->Opcode() == Op_RangeCheck, + iff->Opcode() == Op_RangeCheck || + iff->Opcode() == Op_ParsePredicate, "Check this code when new subtype is added"); int pop = prevdom->Opcode(); diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp index b26c5c60c33..16e7d35cc53 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -139,6 +139,7 @@ class Node_List; class Node_Stack; class OopMap; class ParmNode; +class ParsePredicateNode; class PCTableNode; class PhaseCCP; class PhaseGVN; @@ -668,6 +669,7 @@ public: DEFINE_CLASS_ID(LongCountedLoopEnd, BaseCountedLoopEnd, 1) DEFINE_CLASS_ID(RangeCheck, If, 1) DEFINE_CLASS_ID(OuterStripMinedLoopEnd, If, 2) + DEFINE_CLASS_ID(ParsePredicate, If, 3) DEFINE_CLASS_ID(NeverBranch, MultiBranch, 2) DEFINE_CLASS_ID(Start, Multi, 2) DEFINE_CLASS_ID(MemBar, Multi, 3) @@ -942,6 +944,7 @@ public: DEFINE_CLASS_QUERY(OuterStripMinedLoop) DEFINE_CLASS_QUERY(OuterStripMinedLoopEnd) DEFINE_CLASS_QUERY(Parm) + DEFINE_CLASS_QUERY(ParsePredicate) DEFINE_CLASS_QUERY(PCTable) DEFINE_CLASS_QUERY(Phi) DEFINE_CLASS_QUERY(Proj) @@ -1025,6 +1028,11 @@ public: bool for_post_loop_opts_igvn() const { return (_flags & Flag_for_post_loop_opts_igvn) != 0; } + // Is 'n' possibly a loop entry (i.e. a Parse Predicate projection)? + static bool may_be_loop_entry(Node* n) { + return n != nullptr && n->is_IfProj() && n->in(0)->is_ParsePredicate(); + } + //----------------- Optimization // Get the worst-case Type output for this Node.