stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8341328: Refactor initial Assertion Predicate creation into separate classes

Browse Source 
Reviewed-by: thartmann, kvn
This commit is contained in:
Christian Hagedorn 2024-10-17 05:08:59 +00:00
parent 58d39c317e
commit 22a1feea74
6 changed files with 411 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/hotspot/share/opto/ifnode.cpp
View File

@ -1849,6 +1849,9 @@ void IfNode::dump_spec(outputStream* st) const {
case AssertionPredicateType::LastValue: case AssertionPredicateType::LastValue:
st->print("#Last Value Assertion Predicate "); st->print("#Last Value Assertion Predicate ");
break; break;
case AssertionPredicateType::FinalIv:
st->print("#Final IV Assertion Predicate ");
break;
case AssertionPredicateType::None: case AssertionPredicateType::None:
// No Assertion Predicate // No Assertion Predicate
break; break;

77
src/hotspot/share/opto/loopPredicate.cpp
View File

@ -1153,7 +1153,8 @@ void PhaseIdealLoop::loop_predication_follow_branches(Node *n, IdealLoopTree *lo
bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNode* if_success_proj, bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNode* if_success_proj,
ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl, ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl,
ConNode* zero, Invariance& invar, Deoptimization::DeoptReason reason) { ConNode* zero, Invariance& invar,
Deoptimization::DeoptReason deopt_reason) {
// Following are changed to nonnull when a predicate can be hoisted // Following are changed to nonnull when a predicate can be hoisted
IfNode* iff = if_success_proj->in(0)->as_If(); IfNode* iff = if_success_proj->in(0)->as_If();
Node* test = iff->in(1); Node* test = iff->in(1);
@ -1165,7 +1166,7 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
if (invar.is_invariant(bol)) { if (invar.is_invariant(bol)) {
C->print_method(PHASE_BEFORE_LOOP_PREDICATION_IC, 4, iff); C->print_method(PHASE_BEFORE_LOOP_PREDICATION_IC, 4, iff);
// Invariant test // Invariant test
IfProjNode* hoisted_check_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, IfProjNode* hoisted_check_predicate_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, deopt_reason,
iff->Opcode()); iff->Opcode());
Node* ctrl = hoisted_check_predicate_proj->in(0)->as_If()->in(0); Node* ctrl = hoisted_check_predicate_proj->in(0)->as_If()->in(0);
BoolNode* hoisted_check_predicate_bool = invar.clone(bol, ctrl)->as_Bool(); BoolNode* hoisted_check_predicate_bool = invar.clone(bol, ctrl)->as_Bool();
@ -1206,9 +1207,9 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
const Node* cmp = bol->in(1)->as_Cmp(); const Node* cmp = bol->in(1)->as_Cmp();
Node* idx = cmp->in(1); Node* idx = cmp->in(1);
assert(!invar.is_invariant(idx), "index is variant"); assert(!invar.is_invariant(idx), "index is variant");
Node* rng = cmp->in(2); Node* range = cmp->in(2);
assert(rng->Opcode() == Op_LoadRange || iff->is_RangeCheck() || _igvn.type(rng)->is_int()->_lo >= 0, "must be"); assert(range->Opcode() == Op_LoadRange || iff->is_RangeCheck() || _igvn.type(range)->is_int()->_lo >= 0, "must be");
assert(invar.is_invariant(rng), "range must be invariant"); assert(invar.is_invariant(range), "range must be invariant");
int scale = 1; int scale = 1;
Node* offset = zero; Node* offset = zero;
bool ok = is_scaled_iv_plus_offset(idx, cl->phi(), &scale, &offset); bool ok = is_scaled_iv_plus_offset(idx, cl->phi(), &scale, &offset);
@ -1237,7 +1238,7 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
// late schedule will place invariant things in the loop. // late schedule will place invariant things in the loop.
ParsePredicateNode* parse_predicate = parse_predicate_proj->in(0)->as_ParsePredicate(); ParsePredicateNode* parse_predicate = parse_predicate_proj->in(0)->as_ParsePredicate();
Node* ctrl = parse_predicate->in(0); Node* ctrl = parse_predicate->in(0);
rng = invar.clone(rng, ctrl); range = invar.clone(range, ctrl);
if (offset && offset != zero) { if (offset && offset != zero) {
assert(invar.is_invariant(offset), "offset must be loop invariant"); assert(invar.is_invariant(offset), "offset must be loop invariant");
offset = invar.clone(offset, ctrl); offset = invar.clone(offset, ctrl);
@ -1245,10 +1246,10 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
// If predicate expressions may overflow in the integer range, longs are used. // If predicate expressions may overflow in the integer range, longs are used.
bool overflow = false; bool overflow = false;
// Test the lower bound // Test the lower bound
BoolNode* lower_bound_bol = rc_predicate(ctrl, scale, offset, init, limit, stride, rng, false, overflow); BoolNode* lower_bound_bol = rc_predicate(ctrl, scale, offset, init, limit, stride, range, false, overflow);
const int if_opcode = iff->Opcode(); const int if_opcode = iff->Opcode();
IfProjNode* lower_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : if_opcode); IfProjNode* lower_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, deopt_reason, overflow ? Op_If : if_opcode);
IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If(); IfNode* lower_bound_iff = lower_bound_proj->in(0)->as_If();
_igvn.hash_delete(lower_bound_iff); _igvn.hash_delete(lower_bound_iff);
lower_bound_iff->set_req(1, lower_bound_bol); lower_bound_iff->set_req(1, lower_bound_bol);
@ -1257,9 +1258,9 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
} }
// Test the upper bound // Test the upper bound
BoolNode* upper_bound_bol = rc_predicate(lower_bound_proj, scale, offset, init, limit, stride, rng, true, overflow); BoolNode* upper_bound_bol = rc_predicate(lower_bound_proj, scale, offset, init, limit, stride, range, true, overflow);
IfProjNode* upper_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : if_opcode); IfProjNode* upper_bound_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, deopt_reason, overflow ? Op_If : if_opcode);
assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate"); assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate");
IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If(); IfNode* upper_bound_iff = upper_bound_proj->in(0)->as_If();
_igvn.hash_delete(upper_bound_iff); _igvn.hash_delete(upper_bound_iff);
@ -1272,8 +1273,8 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
// upper bound test. We always need to create skeleton predicates in order to properly remove dead loops when later // upper bound test. We always need to create skeleton predicates in order to properly remove dead loops when later
// splitting the predicated loop into (unreachable) sub-loops (i.e. done by unrolling, peeling, pre/main/post etc.). // splitting the predicated loop into (unreachable) sub-loops (i.e. done by unrolling, peeling, pre/main/post etc.).
IfTrueNode* template_assertion_predicate_proj = IfTrueNode* template_assertion_predicate_proj =
add_template_assertion_predicate(iff, loop, hoisted_check_proj, parse_predicate_proj, upper_bound_proj, scale, create_template_assertion_predicate(if_opcode, cl, parse_predicate_proj, upper_bound_proj, scale, offset, range,
offset, init, limit, stride, rng, overflow, reason); deopt_reason);
// Eliminate the old range check in the loop body. // Eliminate the old range check in the loop body.
// When a range check is eliminated, data dependent nodes (Load and range check CastII nodes) are now dependent on 2 // When a range check is eliminated, data dependent nodes (Load and range check CastII nodes) are now dependent on 2
@ -1309,53 +1310,15 @@ void PhaseIdealLoop::eliminate_hoisted_range_check(IfTrueNode* hoisted_check_pro
// Each newly created Hoisted Check Predicate is accompanied by two Template Assertion Predicates. Later, we initialize // Each newly created Hoisted Check Predicate is accompanied by two Template Assertion Predicates. Later, we initialize
// them by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub // them by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub
// loops are removed properly. // loops are removed properly.
IfTrueNode* PhaseIdealLoop::add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj, IfTrueNode* PhaseIdealLoop::create_template_assertion_predicate(const int if_opcode, CountedLoopNode* loop_head,
ParsePredicateSuccessProj* parse_predicate_proj, ParsePredicateSuccessProj* parse_predicate_proj,
IfProjNode* upper_bound_proj, const int scale, Node* offset, IfProjNode* new_control, const int scale, Node* offset,
Node* init, Node* limit, const jint stride, Node* range, Deoptimization::DeoptReason deopt_reason) {
Node* rng, bool& overflow, Deoptimization::DeoptReason reason) {
// First predicate for the initial value on first loop iteration
Node* opaque_init = new OpaqueLoopInitNode(C, init);
register_new_node(opaque_init, upper_bound_proj);
bool negate = (if_proj->_con != parse_predicate_proj->_con);
BoolNode* bol = rc_predicate(upper_bound_proj, scale, offset, opaque_init, limit, stride, rng,
(stride > 0) != (scale > 0), overflow);
Node* opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1)); // This will go away once loop opts are over
C->add_template_assertion_predicate_opaq(opaque_bol);
register_new_node(opaque_bol, upper_bound_proj);
IfTrueNode* new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode(),
false NOT_PRODUCT(COMMA AssertionPredicateType::InitValue));
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
assert(opaque_init->outcnt() > 0, "should be used");
// Second predicate for init + (current stride - initial stride) TemplateAssertionPredicateCreator template_assertion_predicate_creator(loop_head, scale, offset, range, this);
// This is identical to the previous predicate initially but as return template_assertion_predicate_creator.create_with_uncommon_trap(new_control, parse_predicate_proj, deopt_reason,
// unrolling proceeds current stride is updated. if_opcode);
Node* init_stride = loop->_head->as_CountedLoop()->stride();
Node* opaque_stride = new OpaqueLoopStrideNode(C, init_stride);
register_new_node(opaque_stride, new_proj);
Node* max_value = new SubINode(opaque_stride, init_stride);
register_new_node(max_value, new_proj);
max_value = new AddINode(opaque_init, max_value);
register_new_node(max_value, new_proj);
// init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv Phi
const Type* type_iv = loop->_head->as_CountedLoop()->phi()->bottom_type();
assert(!type_iv->is_int()->is_con(), "constant indicates one loop iteration for which we bailed out earlier");
max_value = new CastIINode(new_proj, max_value, type_iv);
register_new_node(max_value, new_proj);
bol = rc_predicate(new_proj, scale, offset, max_value, limit, stride, rng, (stride > 0) != (scale > 0),
overflow);
opaque_bol = new Opaque4Node(C, bol, _igvn.intcon(1));
C->add_template_assertion_predicate_opaq(opaque_bol);
register_new_node(opaque_bol, new_proj);
new_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, overflow ? Op_If : iff->Opcode(),
false NOT_PRODUCT(COMMA AssertionPredicateType::LastValue));
_igvn.replace_input_of(new_proj->in(0), 1, opaque_bol);
assert(max_value->outcnt() > 0, "should be used");
assert(assertion_predicate_has_loop_opaque_node(new_proj->in(0)->as_If()), "unexpected");
return new_proj;
} }
// Insert Hoisted Check Predicates for null checks and range checks and additional Template Assertion Predicates for // Insert Hoisted Check Predicates for null checks and range checks and additional Template Assertion Predicates for

122
src/hotspot/share/opto/loopTransform.cpp
View File

@ -1461,12 +1461,13 @@ void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride)
// Create an Initialized Assertion Predicate from the template_assertion_predicate // Create an Initialized Assertion Predicate from the template_assertion_predicate
IfTrueNode* PhaseIdealLoop::create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init, IfTrueNode* PhaseIdealLoop::create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init,
Node* new_stride, Node* control) { Node* new_stride, Node* new_control) {
assert(assertion_predicate_has_loop_opaque_node(template_assertion_predicate), assert(assertion_predicate_has_loop_opaque_node(template_assertion_predicate),
"must find OpaqueLoop* nodes for Template Assertion Predicate"); "must find OpaqueLoop* nodes for Template Assertion Predicate");
InitializedAssertionPredicateCreator initialized_assertion_predicate(template_assertion_predicate, new_init, InitializedAssertionPredicateCreator initialized_assertion_predicate(this);
new_stride, this); IfTrueNode* success_proj = initialized_assertion_predicate.create_from_template(template_assertion_predicate,
IfTrueNode* success_proj = initialized_assertion_predicate.create(control); new_control, new_init, new_stride);
assert(!assertion_predicate_has_loop_opaque_node(success_proj->in(0)->as_If()), assert(!assertion_predicate_has_loop_opaque_node(success_proj->in(0)->as_If()),
"Initialized Assertion Predicates do not have OpaqueLoop* nodes in the bool expression anymore"); "Initialized Assertion Predicates do not have OpaqueLoop* nodes in the bool expression anymore");
return success_proj; return success_proj;
@ -1477,30 +1478,18 @@ IfTrueNode* PhaseIdealLoop::create_initialized_assertion_predicate(IfNode* templ
// We keep the Opaque4 node since it's still a template. Since the templates are eventually removed after loop opts, // We keep the Opaque4 node since it's still a template. Since the templates are eventually removed after loop opts,
// these are never executed. We therefore insert a Halt node instead of an uncommon trap. // these are never executed. We therefore insert a Halt node instead of an uncommon trap.
Node* PhaseIdealLoop::clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, Node* PhaseIdealLoop::clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj,
Node* control, IdealLoopTree* outer_loop, Node* input_proj) { Node* control, IdealLoopTree* outer_loop, Node* new_control) {
assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes for Template Assertion Predicate"); assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes for Template Assertion Predicate");
TemplateAssertionExpression template_assertion_expression(iff->in(1)->as_Opaque4()); TemplateAssertionExpression template_assertion_expression(iff->in(1)->as_Opaque4());
assert(new_init->is_OpaqueLoopInit(), "only for creating new Template Assertion Predicates"); assert(new_init->is_OpaqueLoopInit(), "only for creating new Template Assertion Predicates");
Opaque4Node* new_opaque_node = template_assertion_expression.clone_and_replace_init(new_init, control, this); Opaque4Node* new_opaque_node = template_assertion_expression.clone_and_replace_init(new_init, control, this);
Node* proj = predicate->clone(); AssertionPredicateIfCreator assertion_predicate_if_creator(this);
Node* other_proj = uncommon_proj->clone(); IfTrueNode* success_proj =
Node* new_iff = iff->clone(); assertion_predicate_if_creator.create_for_template(new_control, iff->Opcode(), new_opaque_node
new_iff->set_req(1, new_opaque_node); NOT_PRODUCT(COMMA iff->assertion_predicate_type()));
proj->set_req(0, new_iff); assert(assertion_predicate_has_loop_opaque_node(success_proj->in(0)->as_If()),
other_proj->set_req(0, new_iff);
Node* frame = new ParmNode(C->start(), TypeFunc::FramePtr);
register_new_node(frame, C->start());
Node* halt = new HaltNode(other_proj, frame, "Template Assertion Predicates are always removed before code generation");
_igvn.add_input_to(C->root(), halt);
new_iff->set_req(0, input_proj);
register_control(new_iff, outer_loop == _ltree_root ? _ltree_root : outer_loop->_parent, input_proj);
register_control(proj, outer_loop == _ltree_root ? _ltree_root : outer_loop->_parent, new_iff);
register_control(other_proj, _ltree_root, new_iff);
register_control(halt, _ltree_root, other_proj);
assert(assertion_predicate_has_loop_opaque_node(proj->in(0)->as_If()),
"Template Assertion Predicates must have OpaqueLoop* nodes in the bool expression"); "Template Assertion Predicates must have OpaqueLoop* nodes in the bool expression");
return proj; return success_proj;
} }
void PhaseIdealLoop::copy_assertion_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, void PhaseIdealLoop::copy_assertion_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride,
@ -2732,40 +2721,6 @@ bool PhaseIdealLoop::is_scaled_iv_plus_extra_offset(Node* exp1, Node* offset3, N
return false; return false;
} }
// Same as PhaseIdealLoop::duplicate_predicates() but for range checks
// eliminated by iteration splitting.
Node* PhaseIdealLoop::add_range_check_elimination_assertion_predicate(
IdealLoopTree* loop, Node* ctrl, const int scale_con, Node* offset, Node* limit, jint stride_con, Node* value,
const bool is_template NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) {
bool overflow = false;
BoolNode* bol = rc_predicate(ctrl, scale_con, offset, value, nullptr, stride_con,
limit, (stride_con > 0) != (scale_con > 0), overflow);
Node* opaque_assertion_predicate;
if (is_template) {
opaque_assertion_predicate = new Opaque4Node(C, bol, _igvn.intcon(1));
} else {
opaque_assertion_predicate = new OpaqueInitializedAssertionPredicateNode(bol, C);
}
register_new_node(opaque_assertion_predicate, ctrl);
IfNode* new_iff = nullptr;
if (overflow) {
new_iff = new IfNode(ctrl, opaque_assertion_predicate, PROB_MAX, COUNT_UNKNOWN);
} else {
new_iff = new RangeCheckNode(ctrl, opaque_assertion_predicate, PROB_MAX, COUNT_UNKNOWN);
}
register_control(new_iff, loop->_parent, ctrl);
Node* iffalse = new IfFalseNode(new_iff);
register_control(iffalse, _ltree_root, new_iff);
ProjNode* iftrue = new IfTrueNode(new_iff);
register_control(iftrue, loop->_parent, new_iff);
Node *frame = new ParmNode(C->start(), TypeFunc::FramePtr);
register_new_node(frame, C->start());
Node* halt = new HaltNode(iffalse, frame, "range check predicate failed which is impossible");
register_control(halt, _ltree_root, iffalse);
_igvn.add_input_to(C->root(), halt);
return iftrue;
}
//------------------------------do_range_check--------------------------------- //------------------------------do_range_check---------------------------------
// Eliminate range-checks and other trip-counter vs loop-invariant tests. // Eliminate range-checks and other trip-counter vs loop-invariant tests.
void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) { void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) {
@ -2974,45 +2929,39 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) {
Node* opaque_init = new OpaqueLoopInitNode(C, init); Node* opaque_init = new OpaqueLoopInitNode(C, init);
register_new_node(opaque_init, loop_entry); register_new_node(opaque_init, loop_entry);
InitializedAssertionPredicateCreator initialized_assertion_predicate_creator(this);
if (abs_stride_is_one) { if (abs_stride_is_one) {
// If the main loop becomes empty and the array access for this range check is sunk out of the loop, the index // If the main loop becomes empty and the array access for this range check is sunk out of the loop, the index
// for the array access will be set to the index value of the final iteration which could be out of loop. // for the array access will be set to the index value of the final iteration which could be out of loop.
// Add an Assertion Predicate for that corner case. The final iv is computed from LoopLimit which is the // Add an Initialized Assertion Predicate for that corner case. The final iv is computed from LoopLimit which
// LoopNode::limit() only if abs(stride) == 1 otherwise the computation depends on LoopNode::init_trip() as // is the LoopNode::limit() only if abs(stride) == 1 otherwise the computation depends on LoopNode::init_trip()
// well. When LoopLimit only depends on LoopNode::limit(), there are cases where the zero trip guard for the // as well. When LoopLimit only depends on LoopNode::limit(), there are cases where the zero trip guard for
// main loop doesn't constant fold after range check elimination but, the array access for the final // the main loop doesn't constant fold after range check elimination but, the array access for the final
// iteration of the main loop is out of bound and the index for that access is out of range for the range // iteration of the main loop is out of bound and the index for that access is out of range for the range
// check CastII. // check CastII.
loop_entry = add_range_check_elimination_assertion_predicate(loop, loop_entry, scale_con, int_offset, // Note that we do not need to emit a Template Assertion Predicate to update this predicate. When further
int_limit, stride_con, final_iv_placeholder, false); // splitting this loop, the final IV will still be the same. When unrolling the loop, we will remove a
// previously added Initialized Assertion Predicate here. But then abs(stride) is greater than 1, and we
// cannot remove an empty loop with a constant limit when init is not a constant as well. We will use
// a LoopLimitCheck node that can only be folded if the zero grip guard is also foldable.
loop_entry = initialized_assertion_predicate_creator.create(final_iv_placeholder, loop_entry, stride_con,
scale_con, int_offset, int_limit NOT_PRODUCT(
COMMA AssertionPredicateType::FinalIv));
assert(!assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); assert(!assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected");
} }
// Initialized Assertion Predicate for the value of the initial main-loop.
loop_entry = add_range_check_elimination_assertion_predicate(loop, loop_entry, scale_con, int_offset,
int_limit, stride_con, init, false);
assert(!assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected");
// Add two Template Assertion Predicates to create new Initialized Assertion Predicates from when either // Add two Template Assertion Predicates to create new Initialized Assertion Predicates from when either
// unrolling or splitting this main-loop further. // unrolling or splitting this main-loop further.
loop_entry = add_range_check_elimination_assertion_predicate( TemplateAssertionPredicateCreator template_assertion_predicate_creator(cl, scale_con , int_offset, int_limit,
loop, loop_entry, scale_con, int_offset, int_limit, stride_con, opaque_init, true this);
NOT_PRODUCT(COMMA AssertionPredicateType::InitValue)); loop_entry = template_assertion_predicate_creator.create_with_halt(loop_entry);
assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected");
Node* opaque_stride = new OpaqueLoopStrideNode(C, cl->stride()); // Initialized Assertion Predicate for the value of the initial main-loop.
register_new_node(opaque_stride, loop_entry); loop_entry = initialized_assertion_predicate_creator.create(init, loop_entry, stride_con, scale_con,
Node* max_value = new SubINode(opaque_stride, cl->stride()); int_offset, int_limit NOT_PRODUCT(COMMA
register_new_node(max_value, loop_entry); AssertionPredicateType::InitValue));
max_value = new AddINode(opaque_init, max_value); assert(!assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected");
register_new_node(max_value, loop_entry);
// init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv Phi
max_value = new CastIINode(loop_entry, max_value, loop->_head->as_CountedLoop()->phi()->bottom_type());
register_new_node(max_value, loop_entry);
loop_entry = add_range_check_elimination_assertion_predicate(
loop, loop_entry, scale_con, int_offset, int_limit, stride_con, max_value, true
NOT_PRODUCT(COMMA AssertionPredicateType::LastValue));
assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected");
} else { } else {
if (PrintOpto) { if (PrintOpto) {
@ -3072,9 +3021,6 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) {
--imax; --imax;
} }
} }
C->print_method(PHASE_AFTER_RANGE_CHECK_ELIMINATION, 4, cl);
} // End of is IF } // End of is IF
} }
if (loop_entry != cl->skip_strip_mined()->in(LoopNode::EntryControl)) { if (loop_entry != cl->skip_strip_mined()->in(LoopNode::EntryControl)) {
@ -3138,6 +3084,8 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) {
set_ctrl(opqzm, new_limit_ctrl); set_ctrl(opqzm, new_limit_ctrl);
set_ctrl(iffm->in(1)->in(1), new_limit_ctrl); set_ctrl(iffm->in(1)->in(1), new_limit_ctrl);
set_ctrl(iffm->in(1), new_limit_ctrl); set_ctrl(iffm->in(1), new_limit_ctrl);
C->print_method(PHASE_AFTER_RANGE_CHECK_ELIMINATION, 4, cl);
} }
// Adjust control for node and its inputs (and inputs of its inputs) to be above the pre end // Adjust control for node and its inputs (and inputs of its inputs) to be above the pre end

15
src/hotspot/share/opto/loopnode.hpp
View File

@ -951,7 +951,7 @@ private:
uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main, uint idx_after_post_before_pre, Node* zero_trip_guard_proj_main,
Node* zero_trip_guard_proj_post, const Node_List& old_new); Node* zero_trip_guard_proj_post, const Node_List& old_new);
Node* clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, Node* control, Node* clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj, Node* control,
IdealLoopTree* outer_loop, Node* input_proj); IdealLoopTree* outer_loop, Node* new_control);
IfTrueNode* create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init, IfTrueNode* create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init,
Node* new_stride, Node* control); Node* new_stride, Node* control);
static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride); static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);
@ -1384,20 +1384,17 @@ public:
private: private:
bool loop_predication_impl_helper(IdealLoopTree* loop, IfProjNode* if_success_proj, bool loop_predication_impl_helper(IdealLoopTree* loop, IfProjNode* if_success_proj,
ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl, ConNode* zero, ParsePredicateSuccessProj* parse_predicate_proj, CountedLoopNode* cl, ConNode* zero,
Invariance& invar, Deoptimization::DeoptReason reason); Invariance& invar, Deoptimization::DeoptReason deopt_reason);
bool can_create_loop_predicates(const PredicateBlock* profiled_loop_predicate_block) const; bool can_create_loop_predicates(const PredicateBlock* profiled_loop_predicate_block) const;
bool loop_predication_should_follow_branches(IdealLoopTree* loop, float& loop_trip_cnt); bool loop_predication_should_follow_branches(IdealLoopTree* loop, float& loop_trip_cnt);
void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt, void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt,
PathFrequency& pf, Node_Stack& stack, VectorSet& seen, PathFrequency& pf, Node_Stack& stack, VectorSet& seen,
Node_List& if_proj_list); Node_List& if_proj_list);
IfTrueNode* add_template_assertion_predicate(IfNode* iff, IdealLoopTree* loop, IfProjNode* if_proj, IfTrueNode* create_template_assertion_predicate(int if_opcode, CountedLoopNode* loop_head,
ParsePredicateSuccessProj* parse_predicate_proj, ParsePredicateSuccessProj* parse_predicate_proj,
IfProjNode* upper_bound_proj, int scale, Node* offset, Node* init, Node* limit, IfProjNode* new_control, int scale, Node* offset,
jint stride, Node* rng, bool& overflow, Deoptimization::DeoptReason reason); Node* range, Deoptimization::DeoptReason deopt_reason);
void eliminate_hoisted_range_check(IfTrueNode* hoisted_check_proj, IfTrueNode* template_assertion_predicate_proj); void eliminate_hoisted_range_check(IfTrueNode* hoisted_check_proj, IfTrueNode* template_assertion_predicate_proj);
Node* add_range_check_elimination_assertion_predicate(
IdealLoopTree* loop, Node* predicate_proj, int scale_con, Node* offset, Node* limit, int stride_con, Node* value,
bool is_template NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type = AssertionPredicateType::None));
// Helper function to collect predicate for eliminating the useless ones // Helper function to collect predicate for eliminating the useless ones
void eliminate_useless_predicates(); void eliminate_useless_predicates();

328
src/hotspot/share/opto/predicates.cpp
View File

@ -23,7 +23,9 @@
*/ */
#include "precompiled.hpp" #include "precompiled.hpp"
#include "opto/addnode.hpp"
#include "opto/callnode.hpp" #include "opto/callnode.hpp"
#include "opto/castnode.hpp"
#include "opto/loopnode.hpp" #include "opto/loopnode.hpp"
#include "opto/node.hpp" #include "opto/node.hpp"
#include "opto/predicates.hpp" #include "opto/predicates.hpp"
@ -247,11 +249,11 @@ Opaque4Node* TemplateAssertionExpression::clone_and_replace_init(Node* new_init,
// Same as clone() but instead of cloning the OpaqueLoopInit and OpaqueLoopStride node, we replace them with the provided // Same as clone() but instead of cloning the OpaqueLoopInit and OpaqueLoopStride node, we replace them with the provided
// 'new_init' and 'new_stride' nodes, respectively. // 'new_init' and 'new_stride' nodes, respectively.
Opaque4Node* TemplateAssertionExpression::clone_and_replace_init_and_stride(Node* new_init, Node* new_stride, Opaque4Node* TemplateAssertionExpression::clone_and_replace_init_and_stride(Node* new_control, Node* new_init,
Node* new_ctrl, Node* new_stride,
PhaseIdealLoop* phase) { PhaseIdealLoop* phase) {
ReplaceInitAndStrideStrategy replace_init_and_stride_strategy(new_init, new_stride); ReplaceInitAndStrideStrategy replace_init_and_stride_strategy(new_init, new_stride);
return clone(replace_init_and_stride_strategy, new_ctrl, phase); return clone(replace_init_and_stride_strategy, new_control, phase);
} }
// Class to collect data nodes from a source to target nodes by following the inputs of the source node recursively. // Class to collect data nodes from a source to target nodes by following the inputs of the source node recursively.
@ -370,86 +372,296 @@ bool TemplateAssertionExpressionNode::is_template_assertion_predicate(Node* node
return node->is_If() && node->in(1)->is_Opaque4(); return node->is_If() && node->in(1)->is_Opaque4();
} }
InitializedAssertionPredicateCreator::InitializedAssertionPredicateCreator(IfNode* template_assertion_predicate, Node* new_init, // This class creates the Assertion Predicate expression to be used for a Template or Initialized Assertion Predicate.
Node* new_stride, PhaseIdealLoop* phase) class AssertionPredicateExpressionCreator : public StackObj {
: _template_assertion_predicate(template_assertion_predicate), PhaseIdealLoop* const _phase;
_new_init(new_init), const jint _stride;
_new_stride(new_stride), const int _scale;
_phase(phase) {} Node* const _offset;
Node* const _range;
const bool _upper;
// Create an Initialized Assertion Predicate at the provided control from the _template_assertion_predicate. public:
AssertionPredicateExpressionCreator(const int stride, const int scale, Node* offset, Node* range,
PhaseIdealLoop* phase)
: _phase(phase),
_stride(stride),
_scale(scale),
_offset(offset),
_range(range),
_upper((_stride > 0) != (_scale > 0)) {} // Make sure rc_predicate() chooses the "scale*init + offset" case.
// Create the expression for a Template Assertion Predicate with an Opaque4 node.
Opaque4Node* create_for_template(Node* new_control, Node* operand, bool& does_overflow) const {
BoolNode* bool_for_expression = _phase->rc_predicate(new_control, _scale, _offset, operand, nullptr,
_stride, _range, _upper, does_overflow);
return create_opaque4_node(new_control, bool_for_expression);
}
private:
Opaque4Node* create_opaque4_node(Node* new_control, BoolNode* bool_for_expression) const {
Compile* C = _phase->C;
Opaque4Node* new_expression = new Opaque4Node(C, bool_for_expression, _phase->igvn().intcon(1));
C->add_template_assertion_predicate_opaq(new_expression);
_phase->register_new_node(new_expression, new_control);
return new_expression;
}
public:
// Create the expression for an Initialized Assertion Predicate with an OpaqueInitializedAssertionPredicate node.
OpaqueInitializedAssertionPredicateNode* create_for_initialized(Node* new_control, Node* operand,
bool& does_overflow) const {
BoolNode* bool_for_expression = _phase->rc_predicate(new_control, _scale, _offset, operand, nullptr,
_stride, _range, _upper, does_overflow);
return create_opaque_initialized_assertion_predicate_node(new_control, bool_for_expression);
}
private:
OpaqueInitializedAssertionPredicateNode* create_opaque_initialized_assertion_predicate_node(
Node* new_control, BoolNode* bool_for_expression) const {
OpaqueInitializedAssertionPredicateNode* new_expression =
new OpaqueInitializedAssertionPredicateNode(bool_for_expression, _phase->C);
_phase->register_new_node(new_expression, new_control);
return new_expression;
}
};
// Creates an If with a success and a fail path with the given assertion_expression. The only difference to
// create_for_initialized() is that we use a template specific Halt message on the fail path.
IfTrueNode* AssertionPredicateIfCreator::create_for_template(Node* new_control, const int if_opcode,
Node* assertion_expression NOT_PRODUCT(COMMA
const AssertionPredicateType assertion_predicate_type)) {
const char* halt_message = "Template Assertion Predicates are always removed before code generation";
return create(new_control, if_opcode, assertion_expression, halt_message NOT_PRODUCT(COMMA assertion_predicate_type));
}
// Creates an If with a success and a fail path with the given assertion_expression. The only difference to
// create_for_template() is that we use a initialized specific Halt message on the fail path.
IfTrueNode* AssertionPredicateIfCreator::create_for_initialized(Node* new_control, const int if_opcode,
Node* assertion_expression NOT_PRODUCT(COMMA
const AssertionPredicateType assertion_predicate_type)) {
const char* halt_message = "Initialized Assertion Predicate cannot fail";
return create(new_control, if_opcode, assertion_expression, halt_message NOT_PRODUCT(COMMA assertion_predicate_type));
}
// Creates the If node for an Assertion Predicate with a success path and a fail path having a Halt node:
//
// new_control assertion_expression
// \ /
// If
// / \
// success fail path
// proj with Halt
//
IfTrueNode* AssertionPredicateIfCreator::create(Node* new_control, const int if_opcode, Node* assertion_expression,
const char* halt_message NOT_PRODUCT(COMMA
const AssertionPredicateType assertion_predicate_type)) {
assert(assertion_expression->is_Opaque4() || assertion_expression->is_OpaqueInitializedAssertionPredicate(),
"not a valid assertion expression");
IdealLoopTree* loop = _phase->get_loop(new_control);
IfNode* if_node = create_if_node(new_control, if_opcode, assertion_expression, loop
NOT_PRODUCT(COMMA assertion_predicate_type));
create_fail_path(if_node, loop, halt_message);
return create_success_path(if_node, loop);
}
IfNode* AssertionPredicateIfCreator::create_if_node(Node* new_control, const int if_opcode, Node* assertion_expression,
IdealLoopTree* loop NOT_PRODUCT(COMMA
const AssertionPredicateType assertion_predicate_type)) {
IfNode* if_node;
if (if_opcode == Op_If) {
if_node = new IfNode(new_control, assertion_expression, PROB_MAX, COUNT_UNKNOWN
NOT_PRODUCT(COMMA assertion_predicate_type));
} else {
assert(if_opcode == Op_RangeCheck, "must be range check");
if_node = new RangeCheckNode(new_control, assertion_expression, PROB_MAX, COUNT_UNKNOWN
NOT_PRODUCT(COMMA assertion_predicate_type));
}
_phase->register_control(if_node, loop, new_control);
return if_node;
}
IfTrueNode* AssertionPredicateIfCreator::create_success_path(IfNode* if_node, IdealLoopTree* loop) {
IfTrueNode* success_proj = new IfTrueNode(if_node);
_phase->register_control(success_proj, loop, if_node);
return success_proj;
}
void AssertionPredicateIfCreator::create_fail_path(IfNode* if_node, IdealLoopTree* loop, const char* halt_message) {
IfFalseNode* fail_proj = new IfFalseNode(if_node);
_phase->register_control(fail_proj, loop, if_node);
create_halt_node(fail_proj, loop, halt_message);
}
void AssertionPredicateIfCreator::create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop,
const char* halt_message) {
StartNode* start_node = _phase->C->start();
Node* frame = new ParmNode(start_node, TypeFunc::FramePtr);
_phase->register_new_node(frame, start_node);
Node* halt = new HaltNode(fail_proj, frame, halt_message);
_phase->igvn().add_input_to(_phase->C->root(), halt);
_phase->register_control(halt, loop, fail_proj);
}
// Creates an init and last value Template Assertion Predicate connected together from a Parse Predicate with an UCT on
// the failing path. Returns the success projection of the last value Template Assertion Predicate.
IfTrueNode* TemplateAssertionPredicateCreator::create_with_uncommon_trap(
Node* new_control, ParsePredicateSuccessProj* parse_predicate_success_proj,
const Deoptimization::DeoptReason deopt_reason, const int if_opcode) {
OpaqueLoopInitNode* opaque_init = create_opaque_init(new_control);
bool does_overflow;
Opaque4Node* template_assertion_predicate_expression = create_for_init_value(new_control, opaque_init,
does_overflow);
IfTrueNode* template_predicate_success_proj =
create_if_node_with_uncommon_trap(template_assertion_predicate_expression, parse_predicate_success_proj,
deopt_reason, if_opcode, does_overflow
NOT_PRODUCT(COMMA AssertionPredicateType::InitValue));
template_assertion_predicate_expression = create_for_last_value(template_predicate_success_proj, opaque_init,
does_overflow);
return create_if_node_with_uncommon_trap(template_assertion_predicate_expression, parse_predicate_success_proj,
deopt_reason, if_opcode, does_overflow
NOT_PRODUCT(COMMA AssertionPredicateType::LastValue));
}
OpaqueLoopInitNode* TemplateAssertionPredicateCreator::create_opaque_init(Node* new_control) {
OpaqueLoopInitNode* opaque_init = new OpaqueLoopInitNode(_phase->C, _loop_head->init_trip());
_phase->register_new_node(opaque_init, new_control);
return opaque_init;
}
Opaque4Node* TemplateAssertionPredicateCreator::create_for_init_value(Node* new_control, OpaqueLoopInitNode* opaque_init,
bool& does_overflow) const {
AssertionPredicateExpressionCreator expression_creator(_loop_head->stride_con(), _scale, _offset, _range, _phase);
return expression_creator.create_for_template(new_control, opaque_init, does_overflow);
}
IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_uncommon_trap(
Opaque4Node* template_assertion_predicate_expression, ParsePredicateSuccessProj* parse_predicate_success_proj,
const Deoptimization::DeoptReason deopt_reason, const int if_opcode, const bool does_overflow
NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) {
IfTrueNode* success_proj = _phase->create_new_if_for_predicate(parse_predicate_success_proj, nullptr, deopt_reason,
does_overflow ? Op_If : if_opcode, false
NOT_PRODUCT(COMMA assertion_predicate_type));
_phase->igvn().replace_input_of(success_proj->in(0), 1, template_assertion_predicate_expression);
return success_proj;
}
Opaque4Node* TemplateAssertionPredicateCreator::create_for_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init,
bool& does_overflow) const {
Node* last_value = create_last_value(new_control, opaque_init);
AssertionPredicateExpressionCreator expression_creator(_loop_head->stride_con(), _scale, _offset, _range, _phase);
return expression_creator.create_for_template(new_control, last_value, does_overflow);
}
Node* TemplateAssertionPredicateCreator::create_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init) const {
Node* init_stride = _loop_head->stride();
Node* opaque_stride = new OpaqueLoopStrideNode(_phase->C, init_stride);
_phase->register_new_node(opaque_stride, new_control);
Node* last_value = new SubINode(opaque_stride, init_stride);
_phase->register_new_node(last_value, new_control);
last_value = new AddINode(opaque_init, last_value);
_phase->register_new_node(last_value, new_control);
// init + (current stride - initial stride) is within the loop so narrow its type by leveraging the type of the iv phi
last_value = new CastIINode(new_control, last_value, _loop_head->phi()->bottom_type());
_phase->register_new_node(last_value, new_control);
return last_value;
}
IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_halt(
Node* new_control, Opaque4Node* template_assertion_predicate_expression, bool does_overflow
NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) {
AssertionPredicateIfCreator assertion_predicate_if_creator(_phase);
return assertion_predicate_if_creator.create_for_template(new_control, does_overflow ? Op_If : Op_RangeCheck,
template_assertion_predicate_expression
NOT_PRODUCT(COMMA assertion_predicate_type));
}
// Creates an init and last value Template Assertion Predicate connected together with a Halt node on the failing path.
// Returns the success projection of the last value Template Assertion Predicate latter.
IfTrueNode* TemplateAssertionPredicateCreator::create_with_halt(Node* new_control) {
OpaqueLoopInitNode* opaque_init = create_opaque_init(new_control);
bool does_overflow;
Opaque4Node* template_assertion_predicate_expression = create_for_init_value(new_control, opaque_init,
does_overflow);
IfTrueNode* template_predicate_success_proj =
create_if_node_with_halt(new_control, template_assertion_predicate_expression, does_overflow
NOT_PRODUCT(COMMA AssertionPredicateType::InitValue));
template_assertion_predicate_expression = create_for_last_value(template_predicate_success_proj, opaque_init,
does_overflow);
return create_if_node_with_halt(template_predicate_success_proj, template_assertion_predicate_expression,
does_overflow NOT_PRODUCT(COMMA AssertionPredicateType::LastValue));
}
InitializedAssertionPredicateCreator::InitializedAssertionPredicateCreator(PhaseIdealLoop* phase)
: _phase(phase) {}
// Create an Initialized Assertion Predicate from the provided template_assertion_predicate at 'new_control'.
// We clone the Template Assertion Expression and replace: // We clone the Template Assertion Expression and replace:
// - Opaque4 with OpaqueInitializedAssertionPredicate // - Opaque4 with OpaqueInitializedAssertionPredicate
// - OpaqueLoop*Nodes with _new_init and _new_stride, respectively. // - OpaqueLoop*Nodes with new_init and _ew_stride, respectively.
// //
// / init stride // / init stride
// | | | // | | |
// | OpaqueLoopInitNode OpaqueLoopStrideNode / _new_init _new_stride // | OpaqueLoopInitNode OpaqueLoopStrideNode / new_init new_stride
// Template | \ / | \ / // Template | \ / | \ /
// Assertion | ... Assertion | ... // Assertion | ... Assertion | ...
// Expression | | Expression | | // Expression | | Expression | |
// | Bool | new Bool // | Bool | new Bool
// | | | | // | | | |
// \ Opaque4 ======> control \ OpaqueInitializedAssertionPredicate // \ Opaque4 ======> new_control \ OpaqueInitializedAssertionPredicate
// | \ / // | \ /
// If new If // If new If
// / \ / \ // / \ / \
// success fail path new success new Halt // success fail path new success new Halt
// proj (Halt or UCT) proj // proj (Halt or UCT) proj
// //
IfTrueNode* InitializedAssertionPredicateCreator::create(Node* control) { IfTrueNode* InitializedAssertionPredicateCreator::create_from_template(IfNode* template_assertion_predicate,
IdealLoopTree* loop = _phase->get_loop(control); Node* new_control, Node* new_init,
OpaqueInitializedAssertionPredicateNode* assertion_expression = create_assertion_expression(control); Node* new_stride) {
IfNode* if_node = create_if_node(control, assertion_expression, loop); OpaqueInitializedAssertionPredicateNode* assertion_expression =
create_fail_path(if_node, loop); create_assertion_expression_from_template(template_assertion_predicate, new_control, new_init, new_stride);
return create_success_path(if_node, loop); return create_control_nodes(new_control, template_assertion_predicate->Opcode(), assertion_expression
NOT_PRODUCT(COMMA template_assertion_predicate->assertion_predicate_type()));
} }
// Create a new Assertion Expression to be used as bool input for the Initialized Assertion Predicate IfNode. // Create a new Initialized Assertion Predicate directly without a template.
OpaqueInitializedAssertionPredicateNode* InitializedAssertionPredicateCreator::create_assertion_expression(Node* control) { IfTrueNode* InitializedAssertionPredicateCreator::create(Node* operand, Node* new_control, const jint stride,
Opaque4Node* template_opaque = _template_assertion_predicate->in(1)->as_Opaque4(); const int scale, Node* offset, Node* range NOT_PRODUCT(COMMA
AssertionPredicateType assertion_predicate_type)) {
AssertionPredicateExpressionCreator expression_creator(stride, scale, offset, range, _phase);
bool does_overflow;
OpaqueInitializedAssertionPredicateNode* assertion_expression =
expression_creator.create_for_initialized(new_control, operand, does_overflow);
return create_control_nodes(new_control, does_overflow ? Op_If : Op_RangeCheck, assertion_expression
NOT_PRODUCT(COMMA assertion_predicate_type));
}
// Creates the CFG nodes for the Initialized Assertion Predicate.
IfTrueNode* InitializedAssertionPredicateCreator::create_control_nodes(
Node* new_control, const int if_opcode, OpaqueInitializedAssertionPredicateNode* assertion_expression
NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type)) {
AssertionPredicateIfCreator assertion_predicate_if_creator(_phase);
return assertion_predicate_if_creator.create_for_initialized(new_control, if_opcode, assertion_expression
NOT_PRODUCT(COMMA assertion_predicate_type));
}
// Create a new Assertion Expression based from the given template to be used as bool input for the Initialized
// Assertion Predicate IfNode.
OpaqueInitializedAssertionPredicateNode*
InitializedAssertionPredicateCreator::create_assertion_expression_from_template(IfNode* template_assertion_predicate,
Node* new_control, Node* new_init,
Node* new_stride) {
Opaque4Node* template_opaque = template_assertion_predicate->in(1)->as_Opaque4();
TemplateAssertionExpression template_assertion_expression(template_opaque); TemplateAssertionExpression template_assertion_expression(template_opaque);
Opaque4Node* tmp_opaque = template_assertion_expression.clone_and_replace_init_and_stride(_new_init, _new_stride, Opaque4Node* tmp_opaque = template_assertion_expression.clone_and_replace_init_and_stride(new_control, new_init,
control, _phase); new_stride,
_phase);
OpaqueInitializedAssertionPredicateNode* assertion_expression = OpaqueInitializedAssertionPredicateNode* assertion_expression =
new OpaqueInitializedAssertionPredicateNode(tmp_opaque->in(1)->as_Bool(), _phase->C); new OpaqueInitializedAssertionPredicateNode(tmp_opaque->in(1)->as_Bool(), _phase->C);
_phase->register_new_node(assertion_expression, control); _phase->register_new_node(assertion_expression, new_control);
return assertion_expression; return assertion_expression;
} }
IfNode* InitializedAssertionPredicateCreator::create_if_node(Node* control,
OpaqueInitializedAssertionPredicateNode* assertion_expression,
IdealLoopTree* loop) {
const int if_opcode = _template_assertion_predicate->Opcode();
NOT_PRODUCT(const AssertionPredicateType assertion_predicate_type = _template_assertion_predicate->assertion_predicate_type();)
IfNode* if_node = if_opcode == Op_If ?
new IfNode(control, assertion_expression, PROB_MAX, COUNT_UNKNOWN NOT_PRODUCT(COMMA assertion_predicate_type)) :
new RangeCheckNode(control, assertion_expression, PROB_MAX, COUNT_UNKNOWN NOT_PRODUCT(COMMA assertion_predicate_type));
_phase->register_control(if_node, loop, control);
return if_node;
}
IfTrueNode* InitializedAssertionPredicateCreator::create_success_path(IfNode* if_node, IdealLoopTree* loop) {
IfTrueNode* success_proj = new IfTrueNode(if_node);
_phase->register_control(success_proj, loop, if_node);
return success_proj;
}
void InitializedAssertionPredicateCreator::create_fail_path(IfNode* if_node, IdealLoopTree* loop) {
IfFalseNode* fail_proj = new IfFalseNode(if_node);
_phase->register_control(fail_proj, loop, if_node);
create_halt_node(fail_proj, loop);
}
void InitializedAssertionPredicateCreator::create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop) {
StartNode* start_node = _phase->C->start();
Node* frame = new ParmNode(start_node, TypeFunc::FramePtr);
_phase->register_new_node(frame, start_node);
Node* halt = new HaltNode(fail_proj, frame, "Initialized Assertion Predicate cannot fail");
_phase->igvn().add_input_to(_phase->C->root(), halt);
_phase->register_control(halt, loop, fail_proj);
}
#ifndef PRODUCT #ifndef PRODUCT
void PredicateBlock::dump() const { void PredicateBlock::dump() const {
dump(""); dump("");

93
src/hotspot/share/opto/predicates.hpp
View File

@ -207,7 +207,9 @@ class TemplateAssertionPredicate;
enum class AssertionPredicateType { enum class AssertionPredicateType {
None, // Not an Assertion Predicate None, // Not an Assertion Predicate
InitValue, InitValue,
LastValue LastValue,
// Used for the Initialized Assertion Predicate emitted during Range Check Elimination for the final IV value.
FinalIv
}; };
#endif // NOT PRODUCT #endif // NOT PRODUCT
@ -442,8 +444,9 @@ class TemplateAssertionExpression : public StackObj {
public: public:
Opaque4Node* clone(Node* new_ctrl, PhaseIdealLoop* phase); Opaque4Node* clone(Node* new_ctrl, PhaseIdealLoop* phase);
Opaque4Node* clone_and_replace_init(Node* new_init, Node* new_ctrl,PhaseIdealLoop* phase); Opaque4Node* clone_and_replace_init(Node* new_init, Node* new_ctrl, PhaseIdealLoop* phase);
Opaque4Node* clone_and_replace_init_and_stride(Node* new_init, Node* new_stride, Node* new_ctrl, PhaseIdealLoop* phase); Opaque4Node* clone_and_replace_init_and_stride(Node* new_control, Node* new_init, Node* new_stride,
PhaseIdealLoop* phase);
}; };
// Class to represent a node being part of a Template Assertion Expression. Note that this is not an IR node. // Class to represent a node being part of a Template Assertion Expression. Note that this is not an IR node.
@ -520,26 +523,84 @@ class TemplateAssertionExpressionNode : public StackObj {
} }
}; };
// This class creates a new Initialized Assertion Predicate. // This class is used to create the actual If node with a success path and a fail path with a Halt node.
class InitializedAssertionPredicateCreator : public StackObj { class AssertionPredicateIfCreator : public StackObj {
IfNode* const _template_assertion_predicate;
Node* const _new_init;
Node* const _new_stride;
PhaseIdealLoop* const _phase; PhaseIdealLoop* const _phase;
public: public:
InitializedAssertionPredicateCreator(IfNode* template_assertion_predicate, Node* new_init, Node* new_stride, explicit AssertionPredicateIfCreator(PhaseIdealLoop* const phase) : _phase(phase) {}
PhaseIdealLoop* phase); NONCOPYABLE(AssertionPredicateIfCreator);
IfTrueNode* create_for_initialized(Node* new_control, int if_opcode, Node* assertion_expression
NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type));
IfTrueNode* create_for_template(Node* new_control, int if_opcode, Node* assertion_expression
NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type));
private:
IfTrueNode* create(Node* new_control, int if_opcode, Node* assertion_expression, const char* halt_message
NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type));
IfNode* create_if_node(Node* new_control, int if_opcode, Node* assertion_expression, IdealLoopTree* loop
NOT_PRODUCT(COMMA const AssertionPredicateType assertion_predicate_type));
IfTrueNode* create_success_path(IfNode* if_node, IdealLoopTree* loop);
void create_fail_path(IfNode* if_node, IdealLoopTree* loop, const char* halt_message);
void create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop, const char* halt_message);
};
// This class is used to create a Template Assertion Predicate either with an UCT or a Halt Node from scratch.
class TemplateAssertionPredicateCreator : public StackObj {
CountedLoopNode* const _loop_head;
const int _scale;
Node* const _offset;
Node* const _range;
PhaseIdealLoop* const _phase;
OpaqueLoopInitNode* create_opaque_init(Node* new_control);
Opaque4Node* create_for_init_value(Node* new_control, OpaqueLoopInitNode* opaque_init, bool& does_overflow) const;
Opaque4Node* create_for_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init, bool& does_overflow) const;
Node* create_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init) const;
IfTrueNode* create_if_node_with_uncommon_trap(Opaque4Node* template_assertion_predicate_expression,
ParsePredicateSuccessProj* parse_predicate_success_proj,
Deoptimization::DeoptReason deopt_reason, int if_opcode,
bool does_overflow
NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type));
IfTrueNode* create_if_node_with_halt(Node* new_control, Opaque4Node* template_assertion_predicate_expression,
bool does_overflow
NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type));
public:
TemplateAssertionPredicateCreator(CountedLoopNode* loop_head, int scale, Node* offset, Node* range,
PhaseIdealLoop* phase)
: _loop_head(loop_head),
_scale(scale),
_offset(offset),
_range(range),
_phase(phase) {}
NONCOPYABLE(TemplateAssertionPredicateCreator);
IfTrueNode* create_with_uncommon_trap(Node* new_control, ParsePredicateSuccessProj* parse_predicate_success_proj,
Deoptimization::DeoptReason deopt_reason, int if_opcode);
IfTrueNode* create_with_halt(Node* new_control);
};
// This class creates a new Initialized Assertion Predicate either from a template or from scratch.
class InitializedAssertionPredicateCreator : public StackObj {
PhaseIdealLoop* const _phase;
public:
explicit InitializedAssertionPredicateCreator(PhaseIdealLoop* phase);
NONCOPYABLE(InitializedAssertionPredicateCreator); NONCOPYABLE(InitializedAssertionPredicateCreator);
IfTrueNode* create(Node* control); IfTrueNode* create_from_template(IfNode* template_assertion_predicate, Node* new_control, Node* new_init,
Node* new_stride);
IfTrueNode* create(Node* operand, Node* new_control, jint stride, int scale, Node* offset, Node* range
NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type));
private: private:
OpaqueInitializedAssertionPredicateNode* create_assertion_expression(Node* control); OpaqueInitializedAssertionPredicateNode* create_assertion_expression_from_template(IfNode* template_assertion_predicate,
IfNode* create_if_node(Node* control, OpaqueInitializedAssertionPredicateNode* assertion_expression, IdealLoopTree* loop); Node* new_control, Node* new_init,
void create_fail_path(IfNode* if_node, IdealLoopTree* loop); Node* new_stride);
void create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop); IfTrueNode* create_control_nodes(Node* new_control, int if_opcode,
IfTrueNode* create_success_path(IfNode* if_node, IdealLoopTree* loop); OpaqueInitializedAssertionPredicateNode* assertion_expression
NOT_PRODUCT(COMMA AssertionPredicateType assertion_predicate_type));
}; };
// This class iterates through all predicates of a Regular Predicate Block and applies the given visitor to each. // This class iterates through all predicates of a Regular Predicate Block and applies the given visitor to each.