8334571: Extract control dependency rewiring out of PhaseIdealLoop::dominated_by() into separate method
Reviewed-by: roland, kvn
This commit is contained in:
parent
05ff3185ed
commit
ca5a438e5a
src/hotspot/share/opto
@ -1152,7 +1152,6 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
|
||||
ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl,
|
||||
ConNode* zero, Invariance& invar, Deoptimization::DeoptReason reason) {
|
||||
// Following are changed to nonnull when a predicate can be hoisted
|
||||
IfProjNode* new_predicate_proj = nullptr;
|
||||
IfNode* iff = if_success_proj->in(0)->as_If();
|
||||
Node* test = iff->in(1);
|
||||
if (!test->is_Bool()) { //Conv2B, ...
|
||||
@ -1163,10 +1162,9 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
|
||||
if (invar.is_invariant(bol)) {
|
||||
C->print_method(PHASE_BEFORE_LOOP_PREDICATION_IC, 4, iff);
|
||||
// Invariant test
|
||||
new_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr,
|
||||
reason,
|
||||
iff->Opcode());
|
||||
Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0);
|
||||
IfProjNode* hoisted_check_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason,
|
||||
iff->Opcode());
|
||||
Node* ctrl = hoisted_check_predicate_proj->in(0)->as_If()->in(0);
|
||||
BoolNode* hoisted_check_predicate_bool = invar.clone(bol, ctrl)->as_Bool();
|
||||
|
||||
// Negate test if necessary (Parse Predicates always have IfTrue as success projection and IfFalse as uncommon trap)
|
||||
@ -1177,11 +1175,16 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
|
||||
register_new_node(hoisted_check_predicate_bool, ctrl);
|
||||
negated = true;
|
||||
}
|
||||
IfNode* new_predicate_iff = new_predicate_proj->in(0)->as_If();
|
||||
IfNode* new_predicate_iff = hoisted_check_predicate_proj->in(0)->as_If();
|
||||
_igvn.hash_delete(new_predicate_iff);
|
||||
new_predicate_iff->set_req(1, hoisted_check_predicate_bool);
|
||||
|
||||
C->print_method(PHASE_AFTER_LOOP_PREDICATION_IC, 4, new_predicate_proj->in(0));
|
||||
invar.map_ctrl(if_success_proj, hoisted_check_predicate_proj); // Mark hoisted check as invariant
|
||||
|
||||
// Eliminate the old If in the loop body.
|
||||
dominated_by(hoisted_check_predicate_proj, iff, negated);
|
||||
|
||||
C->print_method(PHASE_AFTER_LOOP_PREDICATION_IC, 4, hoisted_check_predicate_proj->in(0));
|
||||
|
||||
#ifndef PRODUCT
|
||||
if (TraceLoopPredicate) {
|
||||
@ -1193,10 +1196,10 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
|
||||
}
|
||||
#endif
|
||||
} else if (cl != nullptr && loop->is_range_check_if(if_success_proj, this, invar DEBUG_ONLY(COMMA parse_predicate_proj))) {
|
||||
range_check_predicate = true;
|
||||
C->print_method(PHASE_BEFORE_LOOP_PREDICATION_RC, 4, iff);
|
||||
// Range check for counted loops
|
||||
assert(if_success_proj->is_IfTrue(), "trap must be on false projection for a range check");
|
||||
IfTrueNode* hoisted_check_proj = if_success_proj->as_IfTrue();
|
||||
const Node* cmp = bol->in(1)->as_Cmp();
|
||||
Node* idx = cmp->in(1);
|
||||
assert(!invar.is_invariant(idx), "index is variant");
|
||||
@ -1265,10 +1268,18 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
|
||||
// Fall through into rest of the cleanup code which will move any dependent nodes to the skeleton predicates of the
|
||||
// upper bound test. We always need to create skeleton predicates in order to properly remove dead loops when later
|
||||
// splitting the predicated loop into (unreachable) sub-loops (i.e. done by unrolling, peeling, pre/main/post etc.).
|
||||
new_predicate_proj = add_template_assertion_predicate(iff, loop, if_success_proj, parse_predicate_proj, upper_bound_proj, scale,
|
||||
offset, init, limit, stride, rng, overflow, reason);
|
||||
IfTrueNode* template_assertion_predicate_proj =
|
||||
add_template_assertion_predicate(iff, loop, hoisted_check_proj, parse_predicate_proj, upper_bound_proj, scale,
|
||||
offset, init, limit, stride, rng, overflow, reason);
|
||||
|
||||
C->print_method(PHASE_AFTER_LOOP_PREDICATION_RC, 4, new_predicate_proj->in(0));
|
||||
// Eliminate the old range check in the loop body.
|
||||
// When a range check is eliminated, data dependent nodes (Load and range check CastII nodes) are now dependent on 2
|
||||
// Hoisted Check Predicates (one for the start of the loop, one for the end) but we can only keep track of one control
|
||||
// dependency: pin the data dependent nodes.
|
||||
eliminate_hoisted_range_check(hoisted_check_proj, template_assertion_predicate_proj);
|
||||
invar.map_ctrl(hoisted_check_proj, template_assertion_predicate_proj); // Mark hoisted check as invariant
|
||||
|
||||
C->print_method(PHASE_AFTER_LOOP_PREDICATION_RC, 4, template_assertion_predicate_proj->in(0));
|
||||
|
||||
#ifndef PRODUCT
|
||||
if (TraceLoopOpts && !TraceLoopPredicate) {
|
||||
@ -1281,24 +1292,21 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
|
||||
// with uncommon trap.
|
||||
return false;
|
||||
}
|
||||
assert(new_predicate_proj != nullptr, "sanity");
|
||||
// Success - attach condition (new_predicate_bol) to predicate if
|
||||
invar.map_ctrl(if_success_proj, new_predicate_proj); // so that invariance test can be appropriate
|
||||
|
||||
// Eliminate the old If in the loop body
|
||||
// If a range check is eliminated, data dependent nodes (Load and range check CastII nodes) are now dependent on 2
|
||||
// Hoisted Check Predicates (one for the start of the loop, one for the end) but we can only keep track of one control
|
||||
// dependency: pin the data dependent nodes.
|
||||
dominated_by(new_predicate_proj, iff, if_success_proj->_con != new_predicate_proj->_con, range_check_predicate);
|
||||
|
||||
C->set_major_progress();
|
||||
return true;
|
||||
}
|
||||
|
||||
void PhaseIdealLoop::eliminate_hoisted_range_check(IfTrueNode* hoisted_check_proj,
|
||||
IfTrueNode* template_assertion_predicate_proj) {
|
||||
_igvn.replace_input_of(hoisted_check_proj->in(0), 1, _igvn.intcon(1));
|
||||
rewire_safe_outputs_to_dominator(hoisted_check_proj, template_assertion_predicate_proj, true);
|
||||
}
|
||||
|
||||
// Each newly created Hoisted Check Predicate is accompanied by two Template Assertion Predicates. Later, we initialize
|
||||
// them by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub
|
||||
// loops are removed properly.
|
||||
IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
|
||||
IfTrueNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
|
||||
ParsePredicateSuccessProj* parse_predicate_proj,
|
||||
IfProjNode* upper_bound_proj, const int scale, Node* offset,
|
||||
Node* init, Node* limit, const jint stride,
|
||||
@ -1312,7 +1320,7 @@ IfProjNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealL
|
||||
Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over
|
||||
C->add_template_assertion_predicate_opaq(opaque_bol);
|
||||
register_new_node(opaque_bol, upper_bound_proj);
|
||||
IfProjNode* new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
|
||||
IfTrueNode* new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode());
|
||||
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
|
||||
assert(opaque_init->outcnt() > 0, "should be used");
|
||||
|
||||
|
@ -1377,10 +1377,11 @@ public:
|
||||
void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt,
|
||||
PathFrequency& pf, Node_Stack& stack, VectorSet& seen,
|
||||
Node_List& if_proj_list);
|
||||
IfProjNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
|
||||
IfTrueNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj,
|
||||
ParsePredicateSuccessProj* parse_predicate_proj,
|
||||
IfProjNode* upper_bound_proj, int scale, Node* offset, Node* init, Node* limit,
|
||||
jint stride, Node* rng, bool& overflow, Deoptimization::DeoptReason reason);
|
||||
void eliminate_hoisted_range_check(IfTrueNode* hoisted_check_proj, IfTrueNode* template_assertion_predicate_proj);
|
||||
Node* add_range_check_elimination_assertion_predicate(IdealLoopTree* loop, Node* predicate_proj, int scale_con,
|
||||
Node* offset, Node* limit, int stride_con, Node* value,
|
||||
bool is_template);
|
||||
@ -1535,6 +1536,7 @@ public:
|
||||
// Mark an IfNode as being dominated by a prior test,
|
||||
// without actually altering the CFG (and hence IDOM info).
|
||||
void dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip = false, bool pin_array_access_nodes = false);
|
||||
void rewire_safe_outputs_to_dominator(Node* source, Node* dominator, bool pin_array_access_nodes);
|
||||
|
||||
// Split Node 'n' through merge point
|
||||
RegionNode* split_thru_region(Node* n, RegionNode* region);
|
||||
|
@ -340,24 +340,31 @@ void PhaseIdealLoop::dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip, b
|
||||
// I can assume this path reaches an infinite loop. In this case it's not
|
||||
// important to optimize the data Nodes - either the whole compilation will
|
||||
// be tossed or this path (and all data Nodes) will go dead.
|
||||
if (iff->outcnt() != 2) return;
|
||||
if (iff->outcnt() != 2) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Make control-dependent data Nodes on the live path (path that will remain
|
||||
// once the dominated IF is removed) become control-dependent on the
|
||||
// dominating projection.
|
||||
Node* dp = iff->proj_out_or_null(pop == Op_IfTrue);
|
||||
|
||||
if (dp == nullptr)
|
||||
if (dp == nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
IdealLoopTree* old_loop = get_loop(dp);
|
||||
rewire_safe_outputs_to_dominator(dp, prevdom, pin_array_access_nodes);
|
||||
}
|
||||
|
||||
for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) {
|
||||
Node* cd = dp->fast_out(i); // Control-dependent node
|
||||
void PhaseIdealLoop::rewire_safe_outputs_to_dominator(Node* source, Node* dominator, const bool pin_array_access_nodes) {
|
||||
IdealLoopTree* old_loop = get_loop(source);
|
||||
|
||||
for (DUIterator_Fast imax, i = source->fast_outs(imax); i < imax; i++) {
|
||||
Node* out = source->fast_out(i); // Control-dependent node
|
||||
// Do not rewire Div and Mod nodes which could have a zero divisor to avoid skipping their zero check.
|
||||
if (cd->depends_only_on_test() && _igvn.no_dependent_zero_check(cd)) {
|
||||
assert(cd->in(0) == dp, "");
|
||||
_igvn.replace_input_of(cd, 0, prevdom);
|
||||
if (out->depends_only_on_test() && _igvn.no_dependent_zero_check(out)) {
|
||||
assert(out->in(0) == source, "must be control dependent on source");
|
||||
_igvn.replace_input_of(out, 0, dominator);
|
||||
if (pin_array_access_nodes) {
|
||||
// Because of Loop Predication, Loads and range check Cast nodes that are control dependent on this range
|
||||
// check (that is about to be removed) now depend on multiple dominating Hoisted Check Predicates. After the
|
||||
@ -365,21 +372,21 @@ void PhaseIdealLoop::dominated_by(IfProjNode* prevdom, IfNode* iff, bool flip, b
|
||||
// in the graph. To ensure that these Loads/Casts do not float above any of the dominating checks (even when the
|
||||
// lowest dominating check is later replaced by yet another dominating check), we need to pin them at the lowest
|
||||
// dominating check.
|
||||
Node* clone = cd->pin_array_access_node();
|
||||
Node* clone = out->pin_array_access_node();
|
||||
if (clone != nullptr) {
|
||||
clone = _igvn.register_new_node_with_optimizer(clone, cd);
|
||||
_igvn.replace_node(cd, clone);
|
||||
cd = clone;
|
||||
clone = _igvn.register_new_node_with_optimizer(clone, out);
|
||||
_igvn.replace_node(out, clone);
|
||||
out = clone;
|
||||
}
|
||||
}
|
||||
set_early_ctrl(cd, false);
|
||||
IdealLoopTree* new_loop = get_loop(get_ctrl(cd));
|
||||
set_early_ctrl(out, false);
|
||||
IdealLoopTree* new_loop = get_loop(get_ctrl(out));
|
||||
if (old_loop != new_loop) {
|
||||
if (!old_loop->_child) {
|
||||
old_loop->_body.yank(cd);
|
||||
old_loop->_body.yank(out);
|
||||
}
|
||||
if (!new_loop->_child) {
|
||||
new_loop->_body.push(cd);
|
||||
new_loop->_body.push(out);
|
||||
}
|
||||
}
|
||||
--i;
|
||||
|
Loading…
x
Reference in New Issue
Block a user