8334571: Extract control dependency rewiring out of PhaseIdealLoop::dominated_by() into separate method

Reviewed-by: roland, kvn
This commit is contained in:
Christian Hagedorn 2024-06-24 08:58:02 +00:00
parent 05ff3185ed
commit ca5a438e5a
3 changed files with 56 additions and 39 deletions

@ -1152,7 +1152,6 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl,
ConNode* zero, Invariance& invar, Deoptimization::DeoptReason reason) {
// Following are changed to nonnull when a predicate can be hoisted
IfProjNode* new_predicate_proj = nullptr;
IfNode* iff = if_success_proj->in(0)->as_If();
Node* test = iff->in(1);
if (!test->is_Bool()) { //Conv2B, ...
@ -1163,10 +1162,9 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
if (invar.is_invariant(bol)) {
C->print_method(PHASE_BEFORE_LOOP_PREDICATION_IC, 4, iff);
// Invariant test
new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr,
reason,
iff->Opcode());
Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0);
IfProjNode* hoisted_check_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason,
iff->Opcode());
Node* ctrl = hoisted_check_predicate_proj->in(0)->as_If()->in(0);
BoolNode* hoisted_check_predicate_bool = invar.clone(bol, ctrl)->as_Bool();
// Negate test if necessary (Parse Predicates always have IfTrue as success projection and IfFalse as uncommon trap)
@ -1177,11 +1175,16 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
register_new_node(hoisted_check_predicate_bool, ctrl);
negated = true;
}
IfNode* new_predicate_iff = new_predicate_proj->in(0)->as_If();
IfNode* new_predicate_iff = hoisted_check_predicate_proj->in(0)->as_If();
_igvn.hash_delete(new_predicate_iff);
new_predicate_iff->set_req(1, hoisted_check_predicate_bool);
C->print_method(PHASE_AFTER_LOOP_PREDICATION_IC, 4, new_predicate_proj->in(0));
invar.map_ctrl(if_success_proj, hoisted_check_predicate_proj); // Mark hoisted check as invariant
// Eliminate the old If in the loop body.
dominated_by(hoisted_check_predicate_proj, iff, negated);
C->print_method(PHASE_AFTER_LOOP_PREDICATION_IC, 4, hoisted_check_predicate_proj->in(0));
#ifndef PRODUCT
if (TraceLoopPredicate) {
@ -1193,10 +1196,10 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
}
#endif
} else if (cl != nullptr && loop->is_range_check_if(if_success_proj, this, invar DEBUG_ONLY(COMMA parse_predicate_proj))) {
range_check_predicate = true;
C->print_method(PHASE_BEFORE_LOOP_PREDICATION_RC, 4, iff);
// Range check for counted loops
assert(if_success_proj->is_IfTrue(), "trap must be on false projection for a range check");
IfTrueNode* hoisted_check_proj = if_success_proj->as_IfTrue();
const Node* cmp = bol->in(1)->as_Cmp();
Node* idx = cmp->in(1);
assert(!invar.is_invariant(idx), "index is variant");
@ -1265,10 +1268,18 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
// Fall through into rest of the cleanup code which will move any dependent nodes to the skeleton predicates of the
// upper bound test. We always need to create skeleton predicates in order to properly remove dead loops when later
// splitting the predicated loop into (unreachable) sub-loops (i.e. done by unrolling, peeling, pre/main/post etc.).
new_predicate_proj = add_template_assertion_predicate(iff, loop, if_success_proj, parse_predicate_proj, upper_bound_proj, scale,
offset, init, limit, stride, rng, overflow, reason);
IfTrueNode* template_assertion_predicate_proj =
add_template_assertion_predicate(iff, loop, hoisted_check_proj, parse_predicate_proj, upper_bound_proj, scale,
offset, init, limit, stride, rng, overflow, reason);
C->print_method(PHASE_AFTER_LOOP_PREDICATION_RC, 4, new_predicate_proj->in(0));
// Eliminate the old range check in the loop body.
// When a range check is eliminated, data dependent nodes (Load and range check CastII nodes) are now dependent on 2
// Hoisted Check Predicates (one for the start of the loop, one for the end) but we can only keep track of one control
// dependency: pin the data dependent nodes.
eliminate_hoisted_range_check(hoisted_check_proj, template_assertion_predicate_proj);
invar.map_ctrl(hoisted_check_proj, template_assertion_predicate_proj); // Mark hoisted check as invariant
C->print_method(PHASE_AFTER_LOOP_PREDICATION_RC, 4, template_assertion_predicate_proj->in(0));
#ifndef PRODUCT
if (TraceLoopOpts && !TraceLoopPredicate) {
@ -1281,24 +1292,21 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
// with uncommon trap.
return false;
}
assert(new_predicate_proj != nullptr, "sanity");
// Success - attach condition (new_predicate_bol) to predicate if
invar.map_ctrl(if_success_proj, new_predicate_proj); // so that invariance test can be appropriate
// Eliminate the old If in the loop body
// If a range check is eliminated, data dependent nodes (Load and range check CastII nodes) are now dependent on 2
// Hoisted Check Predicates (one for the start of the loop, one for the end) but we can only keep track of one control
// dependency: pin the data dependent nodes.
dominated_by(new_predicate_proj, iff, if_success_proj->_con != new_predicate_proj->_con, range_check_predicate);
C->set_major_progress();
return true;
}
void PhaseIdealLoop::eliminate_hoisted_range_check(IfTrueNode* hoisted_check_proj,
IfTrueNode* template_assertion_predicate_proj) {
_igvn.replace_input_of(hoisted_check_proj->in(0), 1, _igvn.intcon(1));
rewire_safe_outputs_to_dominator(hoisted_check_proj, template_assertion_predicate_proj, true);
}
// Each newly created Hoisted Check Predicate is accompanied by two Template Assertion Predicates. Later, we initialize
// them by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub
// loops are removed properly.
IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
IfTrueNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
ParsePredicateSuccessProj* parse_predicate_proj,
IfProjNode* upper_bound_proj, const int scale, Node* offset,
Node* init, Node* limit, const jint stride,
@ -1312,7 +1320,7 @@ IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL
Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over
C->add_template_assertion_predicate_opaq(opaque_bol);
register_new_node(opaque_bol, upper_bound_proj);
IfProjNode* new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
IfTrueNode* new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
assert(opaque_init->outcnt() > 0, "should be used");

@ -1377,10 +1377,11 @@ public:
void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt,
PathFrequency& pf, Node_Stack& stack, VectorSet& seen,
Node_List& if_proj_list);
IfProjNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
IfTrueNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
ParsePredicateSuccessProj* parse_predicate_proj,
IfProjNode* upper_bound_proj, int scale, Node* offset, Node* init, Node* limit,
jint stride, Node* rng, bool& overflow, Deoptimization::DeoptReason reason);
void eliminate_hoisted_range_check(IfTrueNode* hoisted_check_proj, IfTrueNode* template_assertion_predicate_proj);
Node* add_range_check_elimination_assertion_predicate(IdealLoopTree* loop, Node* predicate_proj, int scale_con,
Node* offset, Node* limit, int stride_con, Node* value,
bool is_template);
@ -1535,6 +1536,7 @@ public:
// Mark an IfNode as being dominated by a prior test,
// without actually altering the CFG (and hence IDOM info).
void dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip = false, bool pin_array_access_nodes = false);
void rewire_safe_outputs_to_dominator(Node* source, Node* dominator, bool pin_array_access_nodes);
// Split Node 'n' through merge point
RegionNode* split_thru_region(Node* n, RegionNode* region);

@ -340,24 +340,31 @@ void PhaseIdealLoop::dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip, b
// I can assume this path reaches an infinite loop. In this case it's not
// important to optimize the data Nodes - either the whole compilation will
// be tossed or this path (and all data Nodes) will go dead.
if (iff->outcnt() != 2) return;
if (iff->outcnt() != 2) {
return;
}
// Make control-dependent data Nodes on the live path (path that will remain
// once the dominated IF is removed) become control-dependent on the
// dominating projection.
Node* dp = iff->proj_out_or_null(pop == Op_IfTrue);
if (dp == nullptr)
if (dp == nullptr) {
return;
}
IdealLoopTree* old_loop = get_loop(dp);
rewire_safe_outputs_to_dominator(dp, prevdom, pin_array_access_nodes);
}
for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) {
Node* cd = dp->fast_out(i); // Control-dependent node
void PhaseIdealLoop::rewire_safe_outputs_to_dominator(Node* source, Node* dominator, const bool pin_array_access_nodes) {
IdealLoopTree* old_loop = get_loop(source);
for (DUIterator_Fast imax, i = source->fast_outs(imax); i < imax; i++) {
Node* out = source->fast_out(i); // Control-dependent node
// Do not rewire Div and Mod nodes which could have a zero divisor to avoid skipping their zero check.
if (cd->depends_only_on_test() && _igvn.no_dependent_zero_check(cd)) {
assert(cd->in(0) == dp, "");
_igvn.replace_input_of(cd, 0, prevdom);
if (out->depends_only_on_test() && _igvn.no_dependent_zero_check(out)) {
assert(out->in(0) == source, "must be control dependent on source");
_igvn.replace_input_of(out, 0, dominator);
if (pin_array_access_nodes) {
// Because of Loop Predication, Loads and range check Cast nodes that are control dependent on this range
// check (that is about to be removed) now depend on multiple dominating Hoisted Check Predicates. After the
@ -365,21 +372,21 @@ void PhaseIdealLoop::dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip, b
// in the graph. To ensure that these Loads/Casts do not float above any of the dominating checks (even when the
// lowest dominating check is later replaced by yet another dominating check), we need to pin them at the lowest
// dominating check.
Node* clone = cd->pin_array_access_node();
Node* clone = out->pin_array_access_node();
if (clone != nullptr) {
clone = _igvn.register_new_node_with_optimizer(clone, cd);
_igvn.replace_node(cd, clone);
cd = clone;
clone = _igvn.register_new_node_with_optimizer(clone, out);
_igvn.replace_node(out, clone);
out = clone;
}
}
set_early_ctrl(cd, false);
IdealLoopTree* new_loop = get_loop(get_ctrl(cd));
set_early_ctrl(out, false);
IdealLoopTree* new_loop = get_loop(get_ctrl(out));
if (old_loop != new_loop) {
if (!old_loop->_child) {
old_loop->_body.yank(cd);
old_loop->_body.yank(out);
}
if (!new_loop->_child) {
new_loop->_body.push(cd);
new_loop->_body.push(out);
}
}
--i;