8305637: Remove Opaque1 nodes for Parse Predicates and clean up useless predicate elimination

Reviewed-by: roland, kvn
This commit is contained in:
Christian Hagedorn 2023-09-05 12:45:55 +00:00
parent 8647f001bb
commit 4b445753e6
16 changed files with 344 additions and 258 deletions

@ -734,7 +734,7 @@ Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one
for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
Node* u = n->fast_out(j);
if (u->is_CFG()) {
if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
if (!wq.member(u) && !u->as_Proj()->is_uncommon_trap_proj()) {
return NodeSentinel;
}
}
@ -743,7 +743,7 @@ Node* ShenandoahBarrierC2Support::no_branches(Node* c, Node* dom, bool allow_one
}
} else if (c->is_Proj()) {
if (c->is_IfProj()) {
if (c->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) != nullptr) {
if (c->as_Proj()->is_uncommon_trap_if_pattern() != nullptr) {
// continue;
} else {
if (!allow_one_proj) {
@ -1138,7 +1138,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
if (u_t->meet(TypePtr::NULL_PTR) != u_t &&
u->in(0)->Opcode() == Op_IfTrue &&
u->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
u->in(0)->as_Proj()->is_uncommon_trap_if_pattern() &&
u->in(0)->in(0)->is_If() &&
u->in(0)->in(0)->in(1)->Opcode() == Op_Bool &&
u->in(0)->in(0)->in(1)->as_Bool()->_test._test == BoolTest::ne &&

@ -463,8 +463,9 @@ public:
// More information about predicates can be found in loopPredicate.cpp.
class ParsePredicateNode : public IfNode {
Deoptimization::DeoptReason _deopt_reason;
bool _useless; // If the associated loop dies, this parse predicate becomes useless and can be cleaned up by Value().
public:
ParsePredicateNode(Node* control, Node* bol, Deoptimization::DeoptReason deopt_reason);
ParsePredicateNode(Node* control, Deoptimization::DeoptReason deopt_reason, PhaseGVN* gvn);
virtual int Opcode() const;
virtual uint size_of() const { return sizeof(*this); }
@ -472,8 +473,25 @@ class ParsePredicateNode : public IfNode {
return _deopt_reason;
}
bool is_useless() const {
return _useless;
}
void mark_useless() {
_useless = true;
}
void mark_useful() {
_useless = false;
}
Node* uncommon_trap() const;
Node* Ideal(PhaseGVN* phase, bool can_reshape) {
return nullptr; // Don't optimize
}
const Type* Value(PhaseGVN* phase) const;
NOT_PRODUCT(void dump_spec(outputStream* st) const;)
};

@ -360,10 +360,11 @@ void Compile::remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines
assert(found <= 1, "not unique");
}
void Compile::remove_useless_nodes(GrowableArray<Node*>& node_list, Unique_Node_List& useful) {
template<typename N, ENABLE_IF_SDEFN(std::is_base_of<Node, N>::value)>
void Compile::remove_useless_nodes(GrowableArray<N*>& node_list, Unique_Node_List& useful) {
for (int i = node_list.length() - 1; i >= 0; i--) {
Node* n = node_list.at(i);
if (!useful.member(n)) {
N* node = node_list.at(i);
if (!useful.member(node)) {
node_list.delete_at(i); // replaces i-th with last element which is known to be useful (already processed)
}
}
@ -389,6 +390,9 @@ void Compile::remove_useless_node(Node* dead) {
if (dead->Opcode() == Op_Opaque4) {
remove_template_assertion_predicate_opaq(dead);
}
if (dead->is_ParsePredicate()) {
remove_parse_predicate(dead->as_ParsePredicate());
}
if (dead->for_post_loop_opts_igvn()) {
remove_from_post_loop_opts_igvn(dead);
}
@ -436,7 +440,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis
}
remove_useless_nodes(_macro_nodes, useful); // remove useless macro nodes
remove_useless_nodes(_parse_predicate_opaqs, useful); // remove useless Parse Predicate opaque nodes
remove_useless_nodes(_parse_predicates, useful); // remove useless Parse Predicate nodes
remove_useless_nodes(_template_assertion_predicate_opaqs, useful); // remove useless Assertion Predicate opaque nodes
remove_useless_nodes(_expensive_nodes, useful); // remove useless expensive nodes
remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
@ -631,7 +635,7 @@ Compile::Compile( ciEnv* ci_env, ciMethod* target, int osr_bci,
_failure_reason(nullptr),
_intrinsics (comp_arena(), 0, 0, nullptr),
_macro_nodes (comp_arena(), 8, 0, nullptr),
_parse_predicate_opaqs (comp_arena(), 8, 0, nullptr),
_parse_predicates (comp_arena(), 8, 0, nullptr),
_template_assertion_predicate_opaqs (comp_arena(), 8, 0, nullptr),
_expensive_nodes (comp_arena(), 8, 0, nullptr),
_for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
@ -1807,18 +1811,18 @@ bool Compile::can_alias(const TypePtr* adr_type, int alias_idx) {
return adr_idx == alias_idx;
}
// Remove the opaque nodes that protect the Parse Predicates so that all unused
// checks and uncommon_traps will be eliminated from the ideal graph.
void Compile::cleanup_parse_predicates(PhaseIterGVN& igvn) const {
// Mark all ParsePredicateNodes as useless. They will later be removed from the graph in IGVN together with their
// uncommon traps if no Runtime Predicates were created from the Parse Predicates.
void Compile::mark_parse_predicate_nodes_useless(PhaseIterGVN& igvn) {
if (parse_predicate_count() == 0) {
return;
}
for (int i = parse_predicate_count(); i > 0; i--) {
Node* n = parse_predicate_opaque1_node(i - 1);
assert(n->Opcode() == Op_Opaque1, "must be");
igvn.replace_node(n, n->in(1));
for (int i = 0; i < parse_predicate_count(); i++) {
ParsePredicateNode* parse_predicate = _parse_predicates.at(i);
parse_predicate->mark_useless();
igvn._worklist.push(parse_predicate);
}
assert(parse_predicate_count() == 0, "should be clean!");
_parse_predicates.clear();
}
void Compile::record_for_post_loop_opts_igvn(Node* n) {
@ -1851,6 +1855,7 @@ void Compile::process_for_post_loop_opts_igvn(PhaseIterGVN& igvn) {
}
igvn.optimize();
assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed");
assert(C->parse_predicate_count() == 0, "all parse predicates should have been removed now");
// Sometimes IGVN sets major progress (e.g., when processing loop nodes).
if (C->major_progress()) {

@ -69,6 +69,7 @@ class Node_Notes;
class NodeHash;
class NodeCloneInfo;
class OptoReg;
class ParsePredicateNode;
class PhaseCFG;
class PhaseGVN;
class PhaseIterGVN;
@ -357,7 +358,7 @@ class Compile : public Phase {
const char* _failure_reason; // for record_failure/failing pattern
GrowableArray<CallGenerator*> _intrinsics; // List of intrinsics.
GrowableArray<Node*> _macro_nodes; // List of nodes which need to be expanded before matching.
GrowableArray<Node*> _parse_predicate_opaqs; // List of Opaque1 nodes for the Parse Predicates.
GrowableArray<ParsePredicateNode*> _parse_predicates; // List of Parse Predicates.
GrowableArray<Node*> _template_assertion_predicate_opaqs; // List of Opaque4 nodes for Template Assertion Predicates.
GrowableArray<Node*> _expensive_nodes; // List of nodes that are expensive to compute and that we'd better not let the GVN freely common
GrowableArray<Node*> _for_post_loop_igvn; // List of nodes for IGVN after loop opts are over
@ -703,13 +704,13 @@ private:
#endif
int macro_count() const { return _macro_nodes.length(); }
int parse_predicate_count() const { return _parse_predicate_opaqs.length(); }
int parse_predicate_count() const { return _parse_predicates.length(); }
int template_assertion_predicate_count() const { return _template_assertion_predicate_opaqs.length(); }
int expensive_count() const { return _expensive_nodes.length(); }
int coarsened_count() const { return _coarsened_locks.length(); }
Node* macro_node(int idx) const { return _macro_nodes.at(idx); }
Node* parse_predicate_opaque1_node(int idx) const { return _parse_predicate_opaqs.at(idx); }
ParsePredicateNode* parse_predicate(int idx) const { return _parse_predicates.at(idx); }
Node* template_assertion_predicate_opaq_node(int idx) const {
return _template_assertion_predicate_opaqs.at(idx);
@ -728,10 +729,6 @@ private:
// this function may be called twice for a node so we can only remove it
// if it's still existing.
_macro_nodes.remove_if_existing(n);
// remove from _parse_predicate_opaqs list also if it is there
if (parse_predicate_count() > 0) {
_parse_predicate_opaqs.remove_if_existing(n);
}
// Remove from coarsened locks list if present
if (coarsened_count() > 0) {
remove_coarsened_lock(n);
@ -741,16 +738,24 @@ private:
void remove_expensive_node(Node* n) {
_expensive_nodes.remove_if_existing(n);
}
void add_parse_predicate_opaq(Node* n) {
assert(!_parse_predicate_opaqs.contains(n), "duplicate entry in Parse Predicate opaque1 list");
assert(_macro_nodes.contains(n), "should have already been in macro list");
_parse_predicate_opaqs.append(n);
void add_parse_predicate(ParsePredicateNode* n) {
assert(!_parse_predicates.contains(n), "duplicate entry in Parse Predicate list");
_parse_predicates.append(n);
}
void remove_parse_predicate(ParsePredicateNode* n) {
if (parse_predicate_count() > 0) {
_parse_predicates.remove_if_existing(n);
}
}
void add_template_assertion_predicate_opaq(Node* n) {
assert(!_template_assertion_predicate_opaqs.contains(n),
"duplicate entry in template assertion predicate opaque4 list");
_template_assertion_predicate_opaqs.append(n);
}
void remove_template_assertion_predicate_opaq(Node* n) {
if (template_assertion_predicate_count() > 0) {
_template_assertion_predicate_opaqs.remove_if_existing(n);
@ -775,12 +780,7 @@ private:
void sort_macro_nodes();
// Remove the opaque nodes that protect the Parse Predicates so that the unused checks and
// uncommon traps will be eliminated from the graph.
void cleanup_parse_predicates(PhaseIterGVN &igvn) const;
bool is_predicate_opaq(Node* n) const {
return _parse_predicate_opaqs.contains(n);
}
void mark_parse_predicate_nodes_useless(PhaseIterGVN& igvn);
// Are there candidate expensive nodes for optimization?
bool should_optimize_expensive_nodes(PhaseIterGVN &igvn);
@ -1021,7 +1021,8 @@ private:
_vector_reboxing_late_inlines.push(cg);
}
void remove_useless_nodes (GrowableArray<Node*>& node_list, Unique_Node_List &useful);
template<typename N, ENABLE_IF(std::is_base_of<Node, N>::value)>
void remove_useless_nodes(GrowableArray<N*>& node_list, Unique_Node_List& useful);
void remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines, Unique_Node_List &useful);
void remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines, Node* dead);

@ -3996,11 +3996,7 @@ void GraphKit::add_parse_predicate(Deoptimization::DeoptReason reason, const int
return;
}
Node* cont = _gvn.intcon(1);
Node* opaq = _gvn.transform(new Opaque1Node(C, cont));
C->add_parse_predicate_opaq(opaq);
Node* bol = _gvn.transform(new Conv2BNode(opaq));
ParsePredicateNode* parse_predicate = new ParsePredicateNode(control(), bol, reason);
ParsePredicateNode* parse_predicate = new ParsePredicateNode(control(), reason, &_gvn);
_gvn.set_type(parse_predicate, parse_predicate->Value(&_gvn));
Node* if_false = _gvn.transform(new IfFalseNode(parse_predicate));
{

@ -800,7 +800,7 @@ bool IfNode::is_dominator_unc(CallStaticJavaNode* dom_unc, CallStaticJavaNode* u
// Return projection that leads to an uncommon trap if any
ProjNode* IfNode::uncommon_trap_proj(CallStaticJavaNode*& call) const {
for (int i = 0; i < 2; i++) {
call = proj_out(i)->is_uncommon_trap_proj(Deoptimization::Reason_none);
call = proj_out(i)->is_uncommon_trap_proj();
if (call != nullptr) {
return proj_out(i);
}
@ -811,7 +811,7 @@ ProjNode* IfNode::uncommon_trap_proj(CallStaticJavaNode*& call) const {
// Do this If and the dominating If both branch out to an uncommon trap
bool IfNode::has_only_uncommon_traps(ProjNode* proj, ProjNode*& success, ProjNode*& fail, PhaseIterGVN* igvn) {
ProjNode* otherproj = proj->other_if_proj();
CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj(Deoptimization::Reason_none);
CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj();
if (otherproj->outcnt() == 1 && dom_unc != nullptr) {
// We need to re-execute the folded Ifs after deoptimization from the merged traps
@ -1076,8 +1076,8 @@ Node* IfNode::merge_uncommon_traps(ProjNode* proj, ProjNode* success, ProjNode*
ProjNode* otherproj = proj->other_if_proj();
CallStaticJavaNode* unc = success->is_uncommon_trap_proj(Deoptimization::Reason_none);
CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj(Deoptimization::Reason_none);
CallStaticJavaNode* unc = success->is_uncommon_trap_proj();
CallStaticJavaNode* dom_unc = otherproj->is_uncommon_trap_proj();
if (unc != dom_unc) {
Node* r = new RegionNode(3);
@ -1241,13 +1241,13 @@ bool IfNode::is_side_effect_free_test(ProjNode* proj, PhaseIterGVN* igvn) {
if (proj == nullptr) {
return false;
}
CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern();
if (unc != nullptr && proj->outcnt() <= 2) {
if (proj->outcnt() == 1 ||
// Allow simple null check from LoadRange
(is_cmp_with_loadrange(proj) && is_null_check(proj, igvn))) {
CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
CallStaticJavaNode* dom_unc = proj->in(0)->in(0)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern();
CallStaticJavaNode* dom_unc = proj->in(0)->in(0)->as_Proj()->is_uncommon_trap_if_pattern();
assert(dom_unc != nullptr, "is_uncommon_trap_if_pattern returned null");
// reroute_side_effect_free_unc changes the state of this
@ -1278,9 +1278,9 @@ bool IfNode::is_side_effect_free_test(ProjNode* proj, PhaseIterGVN* igvn) {
// where the first CmpI would have prevented it from executing: on a
// trap, we need to restart execution at the state of the first CmpI
void IfNode::reroute_side_effect_free_unc(ProjNode* proj, ProjNode* dom_proj, PhaseIterGVN* igvn) {
CallStaticJavaNode* dom_unc = dom_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
CallStaticJavaNode* dom_unc = dom_proj->is_uncommon_trap_if_pattern();
ProjNode* otherproj = proj->other_if_proj();
CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
CallStaticJavaNode* unc = proj->is_uncommon_trap_if_pattern();
Node* call_proj = dom_unc->unique_ctrl_out();
Node* halt = call_proj->unique_ctrl_out();
@ -1975,11 +1975,13 @@ Node* RangeCheckNode::Ideal(PhaseGVN *phase, bool can_reshape) {
return dominated_by(prev_dom, igvn);
}
ParsePredicateNode::ParsePredicateNode(Node* control, Node* bol, Deoptimization::DeoptReason deopt_reason)
: IfNode(control, bol, PROB_MAX, COUNT_UNKNOWN),
_deopt_reason(deopt_reason) {
ParsePredicateNode::ParsePredicateNode(Node* control, Deoptimization::DeoptReason deopt_reason, PhaseGVN* gvn)
: IfNode(control, gvn->intcon(1), PROB_MAX, COUNT_UNKNOWN),
_deopt_reason(deopt_reason),
_useless(false) {
init_class_id(Class_ParsePredicate);
assert(bol->Opcode() == Op_Conv2B && bol->in(1) != nullptr && bol->in(1)->is_Opaque1(), "wrong boolean input");
gvn->C->add_parse_predicate(this);
gvn->C->record_for_post_loop_opts_igvn(this);
#ifdef ASSERT
switch (deopt_reason) {
case Deoptimization::Reason_predicate:
@ -1999,6 +2001,18 @@ Node* ParsePredicateNode::uncommon_trap() const {
return uct_region_or_call;
}
// Fold this node away once it becomes useless or at latest in post loop opts IGVN.
const Type* ParsePredicateNode::Value(PhaseGVN* phase) const {
if (phase->type(in(0)) == Type::TOP) {
return Type::TOP;
}
if (_useless || phase->C->post_loop_opts_phase()) {
return TypeTuple::IFTRUE;
} else {
return bottom_type();
}
}
#ifndef PRODUCT
void ParsePredicateNode::dump_spec(outputStream* st) const {
st->print(" #");

@ -99,43 +99,43 @@ void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred,
// We will create a region to guard the uct call if there is no one there.
// The continuation projection (if_cont) of the new_iff is returned which
// is an IfTrue projection. This code is also used to clone predicates to cloned loops.
IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, Node* new_entry,
IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason,
const int opcode, const bool rewire_uncommon_proj_phi_inputs) {
assert(cont_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
IfNode* iff = cont_proj->in(0)->as_If();
assert(parse_predicate_proj->is_uncommon_trap_if_pattern(reason), "must be a uct if pattern!");
ParsePredicateNode* parse_predicate = parse_predicate_proj->in(0)->as_ParsePredicate();
ProjNode *uncommon_proj = iff->proj_out(1 - cont_proj->_con);
Node *rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
ProjNode* uncommon_proj = parse_predicate->proj_out(false);
Node* uct_region = uncommon_proj->unique_ctrl_out();
assert(uct_region->is_Region() || uct_region->is_Call(), "must be a region or call uct");
uint proj_index = 1; // region's edge corresponding to uncommon_proj
if (!rgn->is_Region()) { // create a region to guard the call
assert(rgn->is_Call(), "must be call uct");
CallNode* call = rgn->as_Call();
if (!uct_region->is_Region()) { // create a region to guard the call
assert(uct_region->is_Call(), "must be call uct");
CallNode* call = uct_region->as_Call();
IdealLoopTree* loop = get_loop(call);
rgn = new RegionNode(1);
uct_region = new RegionNode(1);
Node* uncommon_proj_orig = uncommon_proj;
uncommon_proj = uncommon_proj->clone()->as_Proj();
register_control(uncommon_proj, loop, iff);
rgn->add_req(uncommon_proj);
register_control(rgn, loop, uncommon_proj);
_igvn.replace_input_of(call, 0, rgn);
register_control(uncommon_proj, loop, parse_predicate);
uct_region->add_req(uncommon_proj);
register_control(uct_region, loop, uncommon_proj);
_igvn.replace_input_of(call, 0, uct_region);
// When called from beautify_loops() idom is not constructed yet.
if (_idom != nullptr) {
set_idom(call, rgn, dom_depth(rgn));
set_idom(call, uct_region, dom_depth(uct_region));
}
// Move nodes pinned on the projection or whose control is set to
// the projection to the region.
lazy_replace(uncommon_proj_orig, rgn);
lazy_replace(uncommon_proj_orig, uct_region);
} else {
// Find region's edge corresponding to uncommon_proj
for (; proj_index < rgn->req(); proj_index++)
if (rgn->in(proj_index) == uncommon_proj) break;
assert(proj_index < rgn->req(), "sanity");
for (; proj_index < uct_region->req(); proj_index++)
if (uct_region->in(proj_index) == uncommon_proj) break;
assert(proj_index < uct_region->req(), "sanity");
}
Node* entry = iff->in(0);
Node* entry = parse_predicate->in(0);
if (new_entry != nullptr) {
// Cloning the predicate to new location.
entry = new_entry;
@ -145,13 +145,13 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, N
IfNode* new_iff = nullptr;
switch (opcode) {
case Op_If:
new_iff = new IfNode(entry, iff->in(1), iff->_prob, iff->_fcnt);
new_iff = new IfNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt);
break;
case Op_RangeCheck:
new_iff = new RangeCheckNode(entry, iff->in(1), iff->_prob, iff->_fcnt);
new_iff = new RangeCheckNode(entry, parse_predicate->in(1), parse_predicate->_prob, parse_predicate->_fcnt);
break;
case Op_ParsePredicate:
new_iff = new ParsePredicateNode(entry, iff->in(1), reason);
new_iff = new ParsePredicateNode(entry, reason, &_igvn);
break;
default:
fatal("no other If variant here");
@ -160,23 +160,19 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, N
IfProjNode* if_cont = new IfTrueNode(new_iff);
IfProjNode* if_uct = new IfFalseNode(new_iff);
if (cont_proj->is_IfFalse()) {
// Swap
IfProjNode* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
}
register_control(if_cont, lp, new_iff);
register_control(if_uct, get_loop(rgn), new_iff);
register_control(if_uct, get_loop(uct_region), new_iff);
_igvn.add_input_to(rgn, if_uct);
_igvn.add_input_to(uct_region, if_uct);
// If rgn has phis add new edges which has the same
// value as on original uncommon_proj pass.
assert(rgn->in(rgn->req() -1) == if_uct, "new edge should be last");
assert(uct_region->in(uct_region->req() - 1) == if_uct, "new edge should be last");
bool has_phi = false;
for (DUIterator_Fast imax, i = rgn->fast_outs(imax); i < imax; i++) {
Node* use = rgn->fast_out(i);
for (DUIterator_Fast imax, i = uct_region->fast_outs(imax); i < imax; i++) {
Node* use = uct_region->fast_out(i);
if (use->is_Phi() && use->outcnt() > 0) {
assert(use->in(0) == rgn, "");
assert(use->in(0) == uct_region, "");
_igvn.rehash_node_delayed(use);
Node* phi_input = use->in(proj_index);
@ -197,21 +193,21 @@ IfProjNode* PhaseIdealLoop::create_new_if_for_predicate(IfProjNode* cont_proj, N
has_phi = true;
}
}
assert(!has_phi || rgn->req() > 3, "no phis when region is created");
assert(!has_phi || uct_region->req() > 3, "no phis when region is created");
if (new_entry == nullptr) {
// Attach if_cont to iff
_igvn.replace_input_of(iff, 0, if_cont);
_igvn.replace_input_of(parse_predicate, 0, if_cont);
if (_idom != nullptr) {
set_idom(iff, if_cont, dom_depth(iff));
set_idom(parse_predicate, if_cont, dom_depth(parse_predicate));
}
}
// When called from beautify_loops() idom is not constructed yet.
if (_idom != nullptr) {
Node* ridom = idom(rgn);
Node* ridom = idom(uct_region);
Node* nrdom = dom_lca_internal(ridom, new_iff);
set_idom(rgn, nrdom, dom_depth(rgn));
set_idom(uct_region, nrdom, dom_depth(uct_region));
}
return if_cont->as_IfProj();
@ -303,24 +299,14 @@ void PhaseIdealLoop::rewire_inputs_of_clones_to_clones(Node* new_ctrl, Node* clo
}
}
IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* predicate_proj,
IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj,
Node* new_entry, Deoptimization::DeoptReason reason,
const bool slow_loop) {
IfProjNode* new_predicate_proj = create_new_if_for_predicate(predicate_proj, new_entry, reason, Op_ParsePredicate,
IfProjNode* new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, new_entry, reason, Op_ParsePredicate,
slow_loop);
IfNode* iff = new_predicate_proj->in(0)->as_If();
Node* ctrl = iff->in(0);
// Match original condition since predicate's projections could be swapped.
assert(predicate_proj->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be");
Node* opq = new Opaque1Node(C, predicate_proj->in(0)->in(1)->in(1)->in(1));
C->add_parse_predicate_opaq(opq);
Node* bol = new Conv2BNode(opq);
register_new_node(opq, ctrl);
register_new_node(bol, ctrl);
_igvn.hash_delete(iff);
iff->set_req(1, bol);
assert(new_predicate_proj->is_IfTrue(), "the success projection of a Parse Predicate is a true projection");
ParsePredicateNode* parse_predicate = new_predicate_proj->in(0)->as_ParsePredicate();
return new_predicate_proj;
}
@ -329,9 +315,11 @@ IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredic
// cloned predicates.
void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
Deoptimization::DeoptReason reason,
IfProjNode* old_predicate_proj, IfProjNode* iffast_pred,
IfProjNode* ifslow_pred) {
assert(iffast_pred->in(0)->is_If() && ifslow_pred->in(0)->is_If(), "sanity check");
IfProjNode* old_predicate_proj,
ParsePredicateSuccessProj* fast_loop_parse_predicate_proj,
ParsePredicateSuccessProj* slow_loop_parse_predicate_proj) {
assert(fast_loop_parse_predicate_proj->in(0)->is_ParsePredicate() &&
slow_loop_parse_predicate_proj->in(0)->is_ParsePredicate(), "sanity check");
// Only need to clone range check predicates as those can be changed and duplicated by inserting pre/main/post loops
// and doing loop unrolling. Push the original predicates on a list to later process them in reverse order to keep the
// original predicate order.
@ -350,9 +338,9 @@ void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree
assert(predicate->is_Proj() && predicate->as_Proj()->is_IfProj(), "predicate must be a projection of an if node");
IfProjNode* predicate_proj = predicate->as_IfProj();
IfProjNode* fast_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, iffast_pred);
IfProjNode* fast_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, fast_loop_parse_predicate_proj);
assert(assertion_predicate_has_loop_opaque_node(fast_proj->in(0)->as_If()), "must find Assertion Predicate for fast loop");
IfProjNode* slow_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, ifslow_pred);
IfProjNode* slow_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, slow_loop_parse_predicate_proj);
assert(assertion_predicate_has_loop_opaque_node(slow_proj->in(0)->as_If()), "must find Assertion Predicate for slow loop");
// Update control dependent data nodes.
@ -378,14 +366,13 @@ void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree
// Put all Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque'
// is set, then the Opaque4 nodes of the Assertion Predicates are put on the list instead of the projections.
void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque) {
IfNode* iff = predicate->in(0)->as_If();
ProjNode* uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con);
ParsePredicateNode* parse_predicate = predicate->in(0)->as_ParsePredicate();
ProjNode* uncommon_proj = parse_predicate->proj_out(1 - predicate->as_Proj()->_con);
Node* rgn = uncommon_proj->unique_ctrl_out();
assert(rgn->is_Region() || rgn->is_Call(), "must be a region or call uct");
assert(iff->in(1)->in(1)->Opcode() == Op_Opaque1, "unexpected predicate shape");
predicate = iff->in(0);
predicate = parse_predicate->in(0);
while (predicate != nullptr && predicate->is_Proj() && predicate->in(0)->is_If()) {
iff = predicate->in(0)->as_If();
IfNode* iff = predicate->in(0)->as_If();
uncommon_proj = iff->proj_out(1 - predicate->as_Proj()->_con);
if (uncommon_proj->unique_ctrl_out() != rgn) {
break;
@ -408,12 +395,12 @@ void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List&
// predicate again).
IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate,
Deoptimization::DeoptReason reason,
IfProjNode* output_proj) {
Node* bol = create_bool_from_template_assertion_predicate(iff, nullptr, nullptr, output_proj);
IfProjNode* if_proj = create_new_if_for_predicate(output_proj, nullptr, reason, iff->Opcode(), false);
ParsePredicateSuccessProj* parse_predicate_proj) {
Node* bol = create_bool_from_template_assertion_predicate(iff, nullptr, nullptr, parse_predicate_proj);
IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, iff->Opcode(), false);
_igvn.replace_input_of(if_proj->in(0), 1, bol);
_igvn.replace_input_of(output_proj->in(0), 0, if_proj);
set_idom(output_proj->in(0), if_proj, dom_depth(if_proj));
_igvn.replace_input_of(parse_predicate_proj->in(0), 0, if_proj);
set_idom(parse_predicate_proj->in(0), if_proj, dom_depth(if_proj));
return if_proj;
}
@ -446,9 +433,13 @@ void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealL
IfProjNode*& iffast_pred,
IfProjNode*& ifslow_pred) {
if (predicate_block->has_parse_predicate()) {
// We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will
// be changed with the complete fix for Assertion Predicates.
clone_parse_predicate_to_unswitched_loops(predicate_block, reason, iffast_pred, ifslow_pred);
assert(iffast_pred->in(0)->is_ParsePredicate() && ifslow_pred->in(0)->is_ParsePredicate(),
"must be success projections of the cloned Parse Predicates");
clone_assertion_predicates_to_unswitched_loop(loop, old_new, reason, predicate_block->parse_predicate_success_proj(),
iffast_pred, ifslow_pred);
iffast_pred->as_IfTrue(), ifslow_pred->as_IfTrue());
}
}
@ -1156,7 +1147,7 @@ void PhaseIdealLoop::loop_predication_follow_branches(Node *n, IdealLoopTree *lo
stack.push(in, 1);
break;
} else if (in->is_IfProj() &&
in->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
in->as_Proj()->is_uncommon_trap_if_pattern() &&
(in->in(0)->Opcode() == Op_If ||
in->in(0)->Opcode() == Op_RangeCheck)) {
if (pf.to(in) * loop_trip_cnt >= 1) {
@ -1305,19 +1296,20 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
// them by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub
// loops are removed properly.
IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
IfProjNode* predicate_proj, IfProjNode* upper_bound_proj,
int scale, Node* offset, Node* init, Node* limit, jint stride,
ParsePredicateSuccessProj* parse_predicate_proj,
IfProjNode* upper_bound_proj, const int scale, Node* offset,
Node* init, Node* limit, const jint stride,
Node* rng, bool& overflow, Deoptimization::DeoptReason reason) {
// First predicate for the initial value on first loop iteration
Node* opaque_init = new OpaqueLoopInitNode(C, init);
register_new_node(opaque_init, upper_bound_proj);
bool negate = (if_proj->_con != predicate_proj->_con);
bool negate = (if_proj->_con != parse_predicate_proj->_con);
BoolNode* bol = rc_predicate(loop, upper_bound_proj, scale, offset, opaque_init, limit, stride, rng,
(stride > 0) != (scale > 0), overflow);
Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over
C->add_template_assertion_predicate_opaq(opaque_bol);
register_new_node(opaque_bol, upper_bound_proj);
IfProjNode* new_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
IfProjNode* new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
assert(opaque_init->outcnt() > 0, "should be used");
@ -1333,14 +1325,14 @@ IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL
register_new_node(max_value, new_proj);
// init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv Phi
max_value = new CastIINode(max_value, loop->_head->as_CountedLoop()->phi()->bottom_type());
register_new_node(max_value, predicate_proj);
register_new_node(max_value, parse_predicate_proj);
bol = rc_predicate(loop, new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0),
overflow);
opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1));
C->add_template_assertion_predicate_opaq(opaque_bol);
register_new_node(opaque_bol, new_proj);
new_proj = create_new_if_for_predicate(predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
assert(max_value->outcnt() > 0, "should be used");
assert(assertion_predicate_has_loop_opaque_node(new_proj->in(0)->as_If()), "unexpected");
@ -1350,13 +1342,7 @@ IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL
// Insert Hoisted Check Predicates for null checks and range checks and additional Template Assertion Predicates for
// range checks.
bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
if (!UseLoopPredicate) return false;
if (!loop->_head->is_Loop()) {
// Could be a simple region when irreducible loops are present.
return false;
}
bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree* loop) {
LoopNode* head = loop->_head->as_Loop();
if (head->unique_ctrl_out()->is_NeverBranch()) {
@ -1436,7 +1422,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
IfProjNode* if_proj = n->as_IfProj();
IfNode* iff = if_proj->in(0)->as_If();
CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern();
if (call == nullptr) {
if (loop->is_loop_exit(iff)) {
// stop processing the remaining projs in the list because the execution of them
@ -1474,7 +1460,7 @@ bool PhaseIdealLoop::loop_predication_impl(IdealLoopTree *loop) {
while (if_proj_list.size() > 0) {
Node* if_proj = if_proj_list.pop();
float f = pf.to(if_proj);
if (if_proj->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none) &&
if (if_proj->as_Proj()->is_uncommon_trap_if_pattern() &&
f * loop_trip_cnt >= 1) {
ParsePredicateSuccessProj* profiled_loop_parse_predicate_proj =
profiled_loop_predicate_block->parse_predicate_success_proj();
@ -1546,7 +1532,7 @@ bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) {
}
// self
if (!_irreducible && !tail()->is_top()) {
if (can_apply_loop_predication()) {
hoisted |= phase->loop_predication_impl(this);
}
@ -1556,3 +1542,7 @@ bool IdealLoopTree::loop_predication( PhaseIdealLoop *phase) {
return hoisted;
}
bool IdealLoopTree::can_apply_loop_predication() {
return _head->is_Loop() && !_irreducible && !tail()->is_top();
}

@ -177,7 +177,7 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) {
// expensive nodes will notice the loop and skip over it to try to
// move the node further up.
if (ctl->is_CountedLoop() && ctl->in(1) != nullptr && ctl->in(1)->in(0) != nullptr && ctl->in(1)->in(0)->is_If()) {
if (!ctl->in(1)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) {
if (!ctl->in(1)->as_Proj()->is_uncommon_trap_if_pattern()) {
break;
}
next = idom(ctl->in(1)->in(0));
@ -191,7 +191,7 @@ Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) {
} else if (parent_ctl->is_CountedLoopEnd() && parent_ctl->as_CountedLoopEnd()->loopnode() != nullptr) {
next = parent_ctl->as_CountedLoopEnd()->loopnode()->init_control();
} else if (parent_ctl->is_If()) {
if (!ctl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) {
if (!ctl->as_Proj()->is_uncommon_trap_if_pattern()) {
break;
}
assert(idom(ctl) == parent_ctl, "strange");
@ -350,11 +350,6 @@ void PhaseIdealLoop::insert_loop_limit_check_predicate(ParsePredicateSuccessProj
Deoptimization::Reason_loop_limit_check,
Op_If);
Node* iff = new_predicate_proj->in(0);
assert(iff->Opcode() == Op_If, "bad graph shape");
Node* conv = iff->in(1);
assert(conv->Opcode() == Op_Conv2B, "bad graph shape");
Node* opaq = conv->in(1);
assert(opaq->Opcode() == Op_Opaque1, "bad graph shape");
cmp_limit = _igvn.register_new_node_with_optimizer(cmp_limit);
bol = _igvn.register_new_node_with_optimizer(bol);
set_subtree_ctrl(bol, false);
@ -564,19 +559,12 @@ Node* PhaseIdealLoop::loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv,
void PhaseIdealLoop::add_parse_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop,
SafePointNode* sfpt) {
if (!C->too_many_traps(reason)) {
Node* cont = _igvn.intcon(1);
Node* opaq = new Opaque1Node(C, cont);
_igvn.register_new_node_with_optimizer(opaq);
Node* bol = new Conv2BNode(opaq);
_igvn.register_new_node_with_optimizer(bol);
set_subtree_ctrl(bol, false);
ParsePredicateNode* iff = new ParsePredicateNode(inner_head->in(LoopNode::EntryControl), bol, reason);
register_control(iff, loop, inner_head->in(LoopNode::EntryControl));
Node* if_false = new IfFalseNode(iff);
register_control(if_false, _ltree_root, iff);
Node* if_true = new IfTrueNode(iff);
register_control(if_true, loop, iff);
C->add_parse_predicate_opaq(opaq);
ParsePredicateNode* parse_predicate = new ParsePredicateNode(inner_head->in(LoopNode::EntryControl), reason, &_igvn);
register_control(parse_predicate, loop, inner_head->in(LoopNode::EntryControl));
Node* if_false = new IfFalseNode(parse_predicate);
register_control(if_false, _ltree_root, parse_predicate);
Node* if_true = new IfTrueNode(parse_predicate);
register_control(if_true, loop, parse_predicate);
int trap_request = Deoptimization::make_trap_request(reason, Deoptimization::Action_maybe_recompile);
address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point();
@ -1089,7 +1077,7 @@ int PhaseIdealLoop::extract_long_range_checks(const IdealLoopTree* loop, jlong s
Node* c = loop->_body.at(i);
if (c->is_IfProj() && c->in(0)->is_RangeCheck()) {
IfProjNode* if_proj = c->as_IfProj();
CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
CallStaticJavaNode* call = if_proj->is_uncommon_trap_if_pattern();
if (call != nullptr) {
Node* range = nullptr;
Node* offset = nullptr;
@ -4052,78 +4040,107 @@ void PhaseIdealLoop::log_loop_tree() {
}
}
//---------------------collect_potentially_useful_predicates-----------------------
// Helper function to collect potentially useful predicates to prevent them from
// being eliminated by PhaseIdealLoop::eliminate_useless_predicates
void PhaseIdealLoop::collect_potentially_useful_predicates(IdealLoopTree* loop, Unique_Node_List &useful_predicates) {
if (loop->_child) { // child
collect_potentially_useful_predicates(loop->_child, useful_predicates);
// Eliminate all Parse and Template Assertion Predicates that are not associated with a loop anymore. The eliminated
// predicates will be removed during the next round of IGVN.
void PhaseIdealLoop::eliminate_useless_predicates() {
if (C->parse_predicate_count() == 0 && C->template_assertion_predicate_count() == 0) {
return; // No predicates left.
}
// self (only loops that we can apply loop predication may use their predicates)
if (loop->_head->is_Loop() &&
!loop->_irreducible &&
!loop->tail()->is_top()) {
LoopNode* lpn = loop->_head->as_Loop();
Node* entry = lpn->in(LoopNode::EntryControl);
const Predicates predicates(entry);
const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block();
if (loop_limit_check_predicate_block->has_parse_predicate()) { // right pattern that can be used by loop predication
IfProjNode* parse_predicate_proj = loop_limit_check_predicate_block->parse_predicate_success_proj();
assert(parse_predicate_proj->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1, "must be");
useful_predicates.push(parse_predicate_proj->in(0)->in(1)->in(1)); // good one
}
if (UseProfiledLoopPredicate) {
const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block();
if (profiled_loop_predicate_block->has_parse_predicate()) { // right pattern that can be used by loop predication
IfProjNode* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj();
useful_predicates.push(parse_predicate_proj->in(0)->in(1)->in(1)); // good one
get_assertion_predicates(parse_predicate_proj, useful_predicates, true);
}
}
eliminate_useless_parse_predicates();
eliminate_useless_template_assertion_predicates();
}
if (UseLoopPredicate) {
const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block();
if (loop_predicate_block->has_parse_predicate()) { // right pattern that can be used by loop predication
IfProjNode* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj();
useful_predicates.push(parse_predicate_proj->in(0)->in(1)->in(1)); // good one
get_assertion_predicates(parse_predicate_proj, useful_predicates, true);
}
}
// Eliminate all Parse Predicates that do not belong to a loop anymore by marking them useless. These will be removed
// during the next round of IGVN.
void PhaseIdealLoop::eliminate_useless_parse_predicates() {
mark_all_parse_predicates_useless();
if (C->has_loops()) {
mark_loop_associated_parse_predicates_useful();
}
add_useless_parse_predicates_to_igvn_worklist();
}
if (loop->_next) { // sibling
collect_potentially_useful_predicates(loop->_next, useful_predicates);
void PhaseIdealLoop::mark_all_parse_predicates_useless() const {
for (int i = 0; i < C->parse_predicate_count(); i++) {
C->parse_predicate(i)->mark_useless();
}
}
//------------------------eliminate_useless_predicates-----------------------------
// Eliminate all inserted predicates if they could not be used by loop predication.
// Note: it will also eliminates loop limits check predicate since it also uses
// Opaque1 node (see Parse::add_predicate()).
void PhaseIdealLoop::eliminate_useless_predicates() {
if (C->parse_predicate_count() == 0 && C->template_assertion_predicate_count() == 0) {
return; // no predicate left
void PhaseIdealLoop::mark_loop_associated_parse_predicates_useful() {
for (LoopTreeIterator iterator(_ltree_root); !iterator.done(); iterator.next()) {
IdealLoopTree* loop = iterator.current();
if (loop->can_apply_loop_predication()) {
mark_useful_parse_predicates_for_loop(loop);
}
}
}
Unique_Node_List useful_predicates; // to store useful predicates
void PhaseIdealLoop::mark_useful_parse_predicates_for_loop(IdealLoopTree* loop) {
Node* entry = loop->_head->in(LoopNode::EntryControl);
const Predicates predicates(entry);
ParsePredicateIterator iterator(predicates);
while (iterator.has_next()) {
iterator.next()->mark_useful();
}
}
void PhaseIdealLoop::add_useless_parse_predicates_to_igvn_worklist() {
for (int i = 0; i < C->parse_predicate_count(); i++) {
ParsePredicateNode* parse_predicate_node = C->parse_predicate(i);
if (parse_predicate_node->is_useless()) {
_igvn._worklist.push(parse_predicate_node);
}
}
}
// Eliminate all Template Assertion Predicates that do not belong to their originally associated loop anymore by
// replacing the Opaque4 node of the If node with true. These nodes will be removed during the next round of IGVN.
void PhaseIdealLoop::eliminate_useless_template_assertion_predicates() {
Unique_Node_List useful_predicates;
if (C->has_loops()) {
collect_potentially_useful_predicates(_ltree_root->_child, useful_predicates);
collect_useful_template_assertion_predicates(useful_predicates);
}
eliminate_useless_template_assertion_predicates(useful_predicates);
}
void PhaseIdealLoop::collect_useful_template_assertion_predicates(Unique_Node_List& useful_predicates) {
for (LoopTreeIterator iterator(_ltree_root); !iterator.done(); iterator.next()) {
IdealLoopTree* loop = iterator.current();
if (loop->can_apply_loop_predication()) {
collect_useful_template_assertion_predicates_for_loop(loop, useful_predicates);
}
}
}
void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(IdealLoopTree* loop,
Unique_Node_List &useful_predicates) {
Node* entry = loop->_head->in(LoopNode::EntryControl);
const Predicates predicates(entry);
if (UseProfiledLoopPredicate) {
const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block();
if (profiled_loop_predicate_block->has_parse_predicate()) {
IfProjNode* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj();
get_assertion_predicates(parse_predicate_proj, useful_predicates, true);
}
}
for (int i = C->parse_predicate_count(); i > 0; i--) {
Node* n = C->parse_predicate_opaque1_node(i - 1);
assert(n->Opcode() == Op_Opaque1, "must be");
if (!useful_predicates.member(n)) { // not in the useful list
_igvn.replace_node(n, n->in(1));
}
if (UseLoopPredicate) {
const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block();
if (loop_predicate_block->has_parse_predicate()) {
IfProjNode* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj();
get_assertion_predicates(parse_predicate_proj, useful_predicates, true);
}
}
}
for (int i = C->template_assertion_predicate_count(); i > 0; i--) {
Node* n = C->template_assertion_predicate_opaq_node(i - 1);
assert(n->Opcode() == Op_Opaque4, "must be");
if (!useful_predicates.member(n)) { // not in the useful list
_igvn.replace_node(n, n->in(2));
void PhaseIdealLoop::eliminate_useless_template_assertion_predicates(Unique_Node_List& useful_predicates) {
for (int i = 0; i < C->template_assertion_predicate_count(); i++) {
Node* opaque4 = C->template_assertion_predicate_opaq_node(i);
assert(opaque4->Opcode() == Op_Opaque4, "must be");
if (!useful_predicates.member(opaque4)) { // not in the useful list
_igvn.replace_node(opaque4, opaque4->in(2));
}
}
}
@ -4561,7 +4578,7 @@ void PhaseIdealLoop::build_and_optimize() {
}
// Perform loop predication before iteration splitting
if (C->has_loops() && !C->major_progress() && (C->parse_predicate_count() > 0)) {
if (UseLoopPredicate && C->has_loops() && !C->major_progress() && (C->parse_predicate_count() > 0)) {
_ltree_root->_child->loop_predication(this);
}
@ -4614,7 +4631,8 @@ void PhaseIdealLoop::build_and_optimize() {
// until no more loop optimizations could be done.
// After that switch predicates off and do more loop optimizations.
if (!C->major_progress() && (C->parse_predicate_count() > 0)) {
C->cleanup_parse_predicates(_igvn);
C->mark_parse_predicate_nodes_useless(_igvn);
assert(C->parse_predicate_count() == 0, "should be zero now");
if (TraceLoopOpts) {
tty->print_cr("PredicatesOff");
}
@ -6096,7 +6114,7 @@ void PhaseIdealLoop::build_loop_late_post_work(Node *n, bool pinned) {
if (!new_ctrl->is_Proj()) {
break;
}
CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern();
if (call == nullptr) {
break;
}

@ -646,6 +646,7 @@ public:
// Perform optimization to use the loop predicates for null checks and range checks.
// Applies to any loop level (not just the innermost one)
bool loop_predication( PhaseIdealLoop *phase);
bool can_apply_loop_predication();
// Perform iteration-splitting on inner loops. Split iterations to
// avoid range checks or one-shot null checks. Returns false if the
@ -1330,8 +1331,9 @@ public:
bool* p_short_scale, int depth);
// Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted
IfProjNode* create_new_if_for_predicate(IfProjNode* cont_proj, Node* new_entry, Deoptimization::DeoptReason reason,
int opcode, bool rewire_uncommon_proj_phi_inputs = false);
IfProjNode* create_new_if_for_predicate(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason, int opcode,
bool rewire_uncommon_proj_phi_inputs = false);
private:
// Helper functions for create_new_if_for_predicate()
@ -1362,15 +1364,27 @@ public:
void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt,
PathFrequency& pf, Node_Stack& stack, VectorSet& seen,
Node_List& if_proj_list);
IfProjNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj, IfProjNode* predicate_proj,
IfProjNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
ParsePredicateSuccessProj* parse_predicate_proj,
IfProjNode* upper_bound_proj, int scale, Node* offset, Node* init, Node* limit,
jint stride, Node* rng, bool& overflow, Deoptimization::DeoptReason reason);
Node* add_range_check_elimination_assertion_predicate(IdealLoopTree* loop, Node* predicate_proj, int scale_con,
Node* offset, Node* limit, jint stride_con, Node* value);
// Helper function to collect predicate for eliminating the useless ones
void collect_potentially_useful_predicates(IdealLoopTree *loop, Unique_Node_List &predicate_opaque1);
void eliminate_useless_predicates();
void eliminate_useless_parse_predicates();
void mark_all_parse_predicates_useless() const;
void mark_loop_associated_parse_predicates_useful();
static void mark_useful_parse_predicates_for_loop(IdealLoopTree* loop);
void add_useless_parse_predicates_to_igvn_worklist();
void eliminate_useless_template_assertion_predicates();
void collect_useful_template_assertion_predicates(Unique_Node_List& useful_predicates);
static void collect_useful_template_assertion_predicates_for_loop(IdealLoopTree* loop, Unique_Node_List& useful_predicates);
void eliminate_useless_template_assertion_predicates(Unique_Node_List& useful_predicates);
void eliminate_useless_zero_trip_guard();
bool has_control_dependencies_from_predicates(LoopNode* head) const;
@ -1621,14 +1635,15 @@ private:
IfProjNode*& ifslow_pred);
void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason,
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred);
IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* predicate_proj, Node* new_entry,
IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason, bool slow_loop);
void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
Deoptimization::DeoptReason reason, IfProjNode* old_predicate_proj,
IfProjNode* iffast_pred, IfProjNode* ifslow_pred);
ParsePredicateSuccessProj* fast_loop_parse_predicate_proj,
ParsePredicateSuccessProj* slow_loop_parse_predicate_proj);
IfProjNode* clone_assertion_predicate_for_unswitched_loops(Node* iff, IfProjNode* predicate,
Deoptimization::DeoptReason reason,
IfProjNode* output_proj);
ParsePredicateSuccessProj* parse_predicate_proj);
static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN;
bool _created_loop_node;

@ -2068,7 +2068,7 @@ void PhaseIdealLoop::clone_loop_handle_data_uses(Node* old, Node_List &old_new,
// make sure the Bool/Cmp input is cloned down to avoid a Phi between
// the AllocateArray node and its ValidLengthTest input that could cause
// split if to break.
if (use->is_If() || use->is_CMove() || C->is_predicate_opaq(use) || use->Opcode() == Op_Opaque4 ||
if (use->is_If() || use->is_CMove() || use->Opcode() == Op_Opaque4 ||
(use->Opcode() == Op_AllocateArray && use->in(AllocateNode::ValidLengthTest) == old)) {
// Since this code is highly unlikely, we lazily build the worklist
// of such Nodes to go split.

@ -537,7 +537,7 @@ Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, Node *sfpt_ctl, BasicType
} else if (mem->is_ArrayCopy()) {
Node* ctl = mem->in(0);
Node* m = mem->in(TypeFunc::Memory);
if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj(Deoptimization::Reason_none)) {
if (sfpt_ctl->is_Proj() && sfpt_ctl->as_Proj()->is_uncommon_trap_proj()) {
// pin the loads in the uncommon trap path
ctl = sfpt_ctl;
m = sfpt_mem;

@ -184,9 +184,9 @@ uint ProjNode::ideal_reg() const {
//-------------------------------is_uncommon_trap_proj----------------------------
// Return uncommon trap call node if proj is for "proj->[region->..]call_uct"
// null otherwise
CallStaticJavaNode* ProjNode::is_uncommon_trap_proj(Deoptimization::DeoptReason reason) {
int path_limit = 10;
Node* out = this;
CallStaticJavaNode* ProjNode::is_uncommon_trap_proj(Deoptimization::DeoptReason reason) const {
const int path_limit = 10;
const Node* out = this;
for (int ct = 0; ct < path_limit; ct++) {
out = out->unique_ctrl_out_or_null();
if (out == nullptr)
@ -213,31 +213,14 @@ CallStaticJavaNode* ProjNode::is_uncommon_trap_proj(Deoptimization::DeoptReason
// |
// V
// other_proj->[region->..]call_uct"
// null otherwise
// "must_reason_predicate" means the uct reason must be Reason_predicate
CallStaticJavaNode* ProjNode::is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason) {
Node *in0 = in(0);
if (!in0->is_If()) return nullptr;
// Variation of a dead If node.
if (in0->outcnt() < 2) return nullptr;
IfNode* iff = in0->as_If();
// we need "If(Conv2B(Opaque1(...)))" pattern for reason_predicate
if (reason != Deoptimization::Reason_none) {
if (iff->in(1)->Opcode() != Op_Conv2B ||
iff->in(1)->in(1)->Opcode() != Op_Opaque1) {
return nullptr;
}
// or null otherwise.
CallStaticJavaNode* ProjNode::is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason) const {
Node* iff = in(0);
if (!iff->is_If() || iff->outcnt() < 2) {
// Not a projection of an If or variation of a dead If node.
return nullptr;
}
ProjNode* other_proj = iff->proj_out(1-_con);
CallStaticJavaNode* call = other_proj->is_uncommon_trap_proj(reason);
if (call != nullptr) {
assert(reason == Deoptimization::Reason_none ||
Compile::current()->is_predicate_opaq(iff->in(1)->in(1)), "should be on the list");
return call;
}
return nullptr;
return other_if_proj()->is_uncommon_trap_proj(reason);
}
ProjNode* ProjNode::other_if_proj() const {

@ -93,13 +93,13 @@ public:
// Return uncommon trap call node if proj is for "proj->[region->..]call_uct"
// null otherwise
CallStaticJavaNode* is_uncommon_trap_proj(Deoptimization::DeoptReason reason);
CallStaticJavaNode* is_uncommon_trap_proj(Deoptimization::DeoptReason reason = Deoptimization::Reason_none) const;
// Return uncommon trap call node for "if(test)-> proj -> ...
// |
// V
// other_proj->[region->..]call_uct"
// null otherwise
CallStaticJavaNode* is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason);
CallStaticJavaNode* is_uncommon_trap_if_pattern(Deoptimization::DeoptReason reason = Deoptimization::Reason_none) const;
// Return other proj node when this is a If proj node
ProjNode* other_if_proj() const;

@ -508,6 +508,10 @@ Node *Node::clone() const {
// If it is applicable, it will happen anyway when the cloned node is registered with IGVN.
n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
}
if (n->is_ParsePredicate()) {
C->add_parse_predicate(n->as_ParsePredicate());
}
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
bs->register_potential_barrier_node(n);
@ -609,6 +613,9 @@ void Node::destruct(PhaseValues* phase) {
if (Opcode() == Op_Opaque4) {
compile->remove_template_assertion_predicate_opaq(this);
}
if (is_ParsePredicate()) {
compile->remove_parse_predicate(as_ParsePredicate());
}
if (for_post_loop_opts_igvn()) {
compile->remove_from_post_loop_opts_igvn(this);
}

@ -69,7 +69,7 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p
}
Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) {
CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern(Deoptimization::Reason_none);
CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern();
if (uct_call == nullptr) {
return Deoptimization::Reason_none;
}
@ -84,6 +84,30 @@ bool RuntimePredicate::is_success_proj(Node* node, Deoptimization::DeoptReason d
}
}
ParsePredicateIterator::ParsePredicateIterator(const Predicates& predicates) : _current_index(0) {
const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block();
if (loop_limit_check_predicate_block->has_parse_predicate()) {
_parse_predicates.push(loop_limit_check_predicate_block->parse_predicate());
}
if (UseProfiledLoopPredicate) {
const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block();
if (profiled_loop_predicate_block->has_parse_predicate()) {
_parse_predicates.push(profiled_loop_predicate_block->parse_predicate());
}
}
if (UseLoopPredicate) {
const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block();
if (loop_predicate_block->has_parse_predicate()) {
_parse_predicates.push(loop_predicate_block->parse_predicate());
}
}
}
ParsePredicateNode* ParsePredicateIterator::next() {
assert(has_next(), "always check has_next() first");
return _parse_predicates.at(_current_index++);
}
// Walk over all Regular Predicates of this block (if any) and return the first node not belonging to the block
// anymore (i.e. entry to the first Regular Predicate in this block if any or `regular_predicate_proj` otherwise).
Node* PredicateBlock::skip_regular_predicates(Node* regular_predicate_proj, Deoptimization::DeoptReason deopt_reason) {

@ -355,4 +355,19 @@ class Predicates : public StackObj {
return _entry != _loop_entry;
}
};
// This class iterates over the Parse Predicates of a loop.
class ParsePredicateIterator : public StackObj {
GrowableArray<ParsePredicateNode*> _parse_predicates;
int _current_index;
public:
ParsePredicateIterator(const Predicates& predicates);
bool has_next() const {
return _current_index < _parse_predicates.length();
}
ParsePredicateNode* next();
};
#endif // SHARE_OPTO_PREDICATES_HPP