stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8342047: Create Template Assertion Predicates with Halt nodes only instead of uncommon traps

Browse Source 
Reviewed-by: epeter, thartmann
This commit is contained in:
Christian Hagedorn 2024-11-14 07:13:19 +00:00
parent 23a8c71d3b
commit c977ef7b45
6 changed files with 136 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
src/hotspot/share/opto/loopPredicate.cpp
View File

@ -283,62 +283,57 @@ IfProjNode* PhaseIdealLoop::clone_parse_predicate_to_unswitched_loop(ParsePredic
return new_predicate_proj; return new_predicate_proj;
} }
// Clones Assertion Predicates to both unswitched loops starting at 'old_predicate_proj' by following its control inputs. // Clones Template Assertion Predicates to both unswitched loops starting at 'old_predicate_proj' by following its
// It also rewires the control edges of data nodes with dependencies in the loop from the old predicates to the new // control inputs. It also rewires the control edges of data nodes with dependencies in the loop from the old predicates
// cloned predicates. // to the new cloned predicates.
void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
Deoptimization::DeoptReason reason,
ParsePredicateSuccessProj* old_parse_predicate_proj, ParsePredicateSuccessProj* old_parse_predicate_proj,
ParsePredicateSuccessProj* fast_loop_parse_predicate_proj, ParsePredicateNode* true_path_loop_parse_predicate,
ParsePredicateSuccessProj* slow_loop_parse_predicate_proj) { ParsePredicateNode* false_path_loop_parse_predicate) {
assert(fast_loop_parse_predicate_proj->in(0)->is_ParsePredicate() && // Push the original Template Assertion Predicates on a list to later process them in reverse order to keep the
slow_loop_parse_predicate_proj->in(0)->is_ParsePredicate(), "sanity check");
// Only need to clone range check predicates as those can be changed and duplicated by inserting pre/main/post loops
// and doing loop unrolling. Push the original predicates on a list to later process them in reverse order to keep the
// original predicate order. // original predicate order.
Unique_Node_List list; Unique_Node_List list;
get_assertion_predicates(old_parse_predicate_proj, list); get_template_assertion_predicates(old_parse_predicate_proj, list);
Node_List to_process; Node_List to_process;
// Process in reverse order such that 'create_new_if_for_predicate' can be used in
// 'clone_assertion_predicate_for_unswitched_loops' and the original order is maintained.
for (int i = list.size() - 1; i >= 0; i--) { for (int i = list.size() - 1; i >= 0; i--) {
Node* predicate = list.at(i); IfTrueNode* template_assertion_predicate_success_proj = list.at(i)->as_IfTrue();
assert(predicate->in(0)->is_If(), "must be If node"); assert(template_assertion_predicate_success_proj->in(0)->is_If(), "must be If node");
IfNode* iff = predicate->in(0)->as_If();
assert(predicate->is_Proj() && predicate->as_Proj()->is_IfProj(), "predicate must be a projection of an if node");
IfProjNode* predicate_proj = predicate->as_IfProj();
IfProjNode* fast_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, fast_loop_parse_predicate_proj); IfTrueNode* true_path_loop_proj =
assert(assertion_predicate_has_loop_opaque_node(fast_proj->in(0)->as_If()), "must find Assertion Predicate for fast loop"); clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj,
IfProjNode* slow_proj = clone_assertion_predicate_for_unswitched_loops(iff, predicate_proj, reason, slow_loop_parse_predicate_proj); true_path_loop_parse_predicate);
assert(assertion_predicate_has_loop_opaque_node(slow_proj->in(0)->as_If()), "must find Assertion Predicate for slow loop"); IfTrueNode* false_path_loop_proj =
clone_assertion_predicate_for_unswitched_loops(template_assertion_predicate_success_proj,
false_path_loop_parse_predicate);
// Update control dependent data nodes. // Update control dependent data nodes.
for (DUIterator j = predicate->outs(); predicate->has_out(j); j++) { for (DUIterator j = template_assertion_predicate_success_proj->outs();
Node* fast_node = predicate->out(j); template_assertion_predicate_success_proj->has_out(j);
if (loop->is_member(get_loop(ctrl_or_self(fast_node)))) { j++) {
assert(fast_node->in(0) == predicate, "only control edge"); Node* true_path_loop_node = template_assertion_predicate_success_proj->out(j);
Node* slow_node = old_new[fast_node->_idx]; if (loop->is_member(get_loop(ctrl_or_self(true_path_loop_node)))) {
assert(slow_node->in(0) == predicate, "only control edge"); assert(true_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge");
_igvn.replace_input_of(fast_node, 0, fast_proj); Node* false_path_loop_node = old_new[true_path_loop_node->_idx];
to_process.push(slow_node); assert(false_path_loop_node->in(0) == template_assertion_predicate_success_proj, "only control edge");
_igvn.replace_input_of(true_path_loop_node, 0, true_path_loop_proj);
to_process.push(false_path_loop_node);
--j; --j;
} }
} }
// Have to delay updates to the slow loop so uses of predicate are not modified while we iterate on them. // Have to delay updates to the false path loop so uses of predicate are not modified while we iterate on them.
while (to_process.size() > 0) { while (to_process.size() > 0) {
Node* slow_node = to_process.pop(); Node* slow_node = to_process.pop();
_igvn.replace_input_of(slow_node, 0, slow_proj); _igvn.replace_input_of(slow_node, 0, false_path_loop_proj);
} }
} }
} }
// Put all Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque' // Put all Template Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque'
// is set, then the OpaqueTemplateAssertionPredicateNode nodes of the Assertion Predicates are put on the list instead // is set, then the OpaqueTemplateAssertionPredicateNode nodes of the Assertion Predicates are put on the list instead
// of the projections. // of the projections.
void PhaseIdealLoop::get_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, void PhaseIdealLoop::get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list,
const bool get_opaque) { const bool get_opaque) {
Deoptimization::DeoptReason deopt_reason = parse_predicate_proj->in(0)->as_ParsePredicate()->deopt_reason(); Deoptimization::DeoptReason deopt_reason = parse_predicate_proj->in(0)->as_ParsePredicate()->deopt_reason();
PredicateBlockIterator predicate_iterator(parse_predicate_proj, deopt_reason); PredicateBlockIterator predicate_iterator(parse_predicate_proj, deopt_reason);
TemplateAssertionPredicateCollector template_assertion_predicate_collector(list, get_opaque); TemplateAssertionPredicateCollector template_assertion_predicate_collector(list, get_opaque);
@ -348,39 +343,38 @@ void PhaseIdealLoop::get_assertion_predicates(ParsePredicateSuccessProj* parse_p
// Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon // Clone an Assertion Predicate for an unswitched loop. OpaqueLoopInit and OpaqueLoopStride nodes are cloned and uncommon
// traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned // traps are kept for the predicate (a Halt node is used later when creating pre/main/post loops and copying this cloned
// predicate again). // predicate again).
IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfNode* template_assertion_predicate, IfTrueNode*
IfProjNode* predicate, PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj,
Deoptimization::DeoptReason reason, ParsePredicateNode* unswitched_loop_parse_predicate) {
ParsePredicateSuccessProj* parse_predicate_proj) { TemplateAssertionPredicate template_assertion_predicate(template_assertion_predicate_success_proj);
TemplateAssertionExpression template_assertion_expression(template_assertion_predicate->in(1)->as_OpaqueTemplateAssertionPredicate()); IfTrueNode* template_success_proj = template_assertion_predicate.clone(unswitched_loop_parse_predicate->in(0), this);
OpaqueTemplateAssertionPredicateNode* cloned_opaque_node = template_assertion_expression.clone(parse_predicate_proj->in(0)->in(0), this); assert(assertion_predicate_has_loop_opaque_node(template_success_proj->in(0)->as_If()),
IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason, "must find Assertion Predicate for fast loop");
template_assertion_predicate->Opcode(), false); _igvn.replace_input_of(unswitched_loop_parse_predicate, 0, template_success_proj);
_igvn.replace_input_of(if_proj->in(0), 1, cloned_opaque_node); set_idom(unswitched_loop_parse_predicate, template_success_proj, dom_depth(template_success_proj));
_igvn.replace_input_of(parse_predicate_proj->in(0), 0, if_proj); return template_success_proj;
set_idom(parse_predicate_proj->in(0), if_proj, dom_depth(if_proj));
return if_proj;
} }
// Clone the old Parse Predicates and Assertion Predicates before the unswitch If to the unswitched loops after the // Clone the old Parse Predicates and Assertion Predicates before the unswitch If to the unswitched loops after the
// unswitch If. // unswitch If.
void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred) { IfProjNode*& true_path_loop_entry,
IfProjNode*& false_path_loop_entry) {
LoopNode* head = loop->_head->as_Loop(); LoopNode* head = loop->_head->as_Loop();
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl); Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
const Predicates predicates(entry); const Predicates predicates(entry);
clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.loop_predicate_block(), clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.loop_predicate_block(),
Deoptimization::Reason_predicate, iffast_pred, ifslow_pred); Deoptimization::Reason_predicate, true_path_loop_entry, false_path_loop_entry);
clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.profiled_loop_predicate_block(), clone_loop_predication_predicates_to_unswitched_loop(loop, old_new, predicates.profiled_loop_predicate_block(),
Deoptimization::Reason_profile_predicate, iffast_pred, ifslow_pred); Deoptimization::Reason_profile_predicate, true_path_loop_entry, false_path_loop_entry);
const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block(); const PredicateBlock* loop_limit_check_predicate_block = predicates.loop_limit_check_predicate_block();
if (loop_limit_check_predicate_block->has_parse_predicate() && !head->is_CountedLoop()) { if (loop_limit_check_predicate_block->has_parse_predicate() && !head->is_CountedLoop()) {
// Don't clone the Loop Limit Check Parse Predicate if we already have a counted loop (a Loop Limit Check Predicate // Don't clone the Loop Limit Check Parse Predicate if we already have a counted loop (a Loop Limit Check Predicate
// is only created when converting a LoopNode to a CountedLoopNode). // is only created when converting a LoopNode to a CountedLoopNode).
clone_parse_predicate_to_unswitched_loops(loop_limit_check_predicate_block, Deoptimization::Reason_loop_limit_check, clone_parse_predicate_to_unswitched_loops(loop_limit_check_predicate_block, Deoptimization::Reason_loop_limit_check,
iffast_pred, ifslow_pred); true_path_loop_entry, false_path_loop_entry);
} }
} }
@ -388,16 +382,17 @@ void PhaseIdealLoop::clone_parse_and_assertion_predicates_to_unswitched_loop(Ide
void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, void PhaseIdealLoop::clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
const PredicateBlock* predicate_block, const PredicateBlock* predicate_block,
Deoptimization::DeoptReason reason, Deoptimization::DeoptReason reason,
IfProjNode*& iffast_pred, IfProjNode*& true_path_loop_entry,
IfProjNode*& ifslow_pred) { IfProjNode*& false_path_loop_entry) {
if (predicate_block->has_parse_predicate()) { if (predicate_block->has_parse_predicate()) {
// We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will // We currently only clone Assertion Predicates if there are Parse Predicates. This is not entirely correct and will
// be changed with the complete fix for Assertion Predicates. // be changed with the complete fix for Assertion Predicates.
clone_parse_predicate_to_unswitched_loops(predicate_block, reason, iffast_pred, ifslow_pred); clone_parse_predicate_to_unswitched_loops(predicate_block, reason, true_path_loop_entry, false_path_loop_entry);
assert(iffast_pred->in(0)->is_ParsePredicate() && ifslow_pred->in(0)->is_ParsePredicate(), assert(true_path_loop_entry->in(0)->is_ParsePredicate() && false_path_loop_entry->in(0)->is_ParsePredicate(),
"must be success projections of the cloned Parse Predicates"); "must be success projections of the cloned Parse Predicates");
clone_assertion_predicates_to_unswitched_loop(loop, old_new, reason, predicate_block->parse_predicate_success_proj(), clone_assertion_predicates_to_unswitched_loop(loop, old_new, predicate_block->parse_predicate_success_proj(),
iffast_pred->as_IfTrue(), ifslow_pred->as_IfTrue()); true_path_loop_entry->in(0)->as_ParsePredicate(),
false_path_loop_entry->in(0)->as_ParsePredicate());
} }
} }
@ -1250,9 +1245,9 @@ bool PhaseIdealLoop::loop_predication_impl_helper(IdealLoopTree* loop, IfProjNod
// Fall through into rest of the cleanup code which will move any dependent nodes to the skeleton predicates of the // Fall through into rest of the cleanup code which will move any dependent nodes to the skeleton predicates of the
// upper bound test. We always need to create skeleton predicates in order to properly remove dead loops when later // upper bound test. We always need to create skeleton predicates in order to properly remove dead loops when later
// splitting the predicated loop into (unreachable) sub-loops (i.e. done by unrolling, peeling, pre/main/post etc.). // splitting the predicated loop into (unreachable) sub-loops (i.e. done by unrolling, peeling, pre/main/post etc.).
IfTrueNode* template_assertion_predicate_proj = IfTrueNode* template_assertion_predicate_proj = create_template_assertion_predicate(cl, parse_predicate,
create_template_assertion_predicate(if_opcode, cl, parse_predicate_proj, upper_bound_proj, scale, offset, range, upper_bound_proj, scale, offset,
deopt_reason); range);
// Eliminate the old range check in the loop body. // Eliminate the old range check in the loop body.
// When a range check is eliminated, data dependent nodes (Load and range check CastII nodes) are now dependent on 2 // When a range check is eliminated, data dependent nodes (Load and range check CastII nodes) are now dependent on 2
@ -1288,15 +1283,16 @@ void PhaseIdealLoop::eliminate_hoisted_range_check(IfTrueNode* hoisted_check_pro
// Each newly created Hoisted Check Predicate is accompanied by two Template Assertion Predicates. Later, we initialize // Each newly created Hoisted Check Predicate is accompanied by two Template Assertion Predicates. Later, we initialize
// them by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub // them by making a copy of them when splitting a loop into sub loops. The Assertion Predicates ensure that dead sub
// loops are removed properly. // loops are removed properly.
IfTrueNode* PhaseIdealLoop::create_template_assertion_predicate(const int if_opcode, CountedLoopNode* loop_head, IfTrueNode* PhaseIdealLoop::create_template_assertion_predicate(CountedLoopNode* loop_head,
ParsePredicateSuccessProj* parse_predicate_proj, ParsePredicateNode* parse_predicate,
IfProjNode* new_control, const int scale, Node* offset, IfProjNode* new_control, const int scale, Node* offset,
Node* range, Deoptimization::DeoptReason deopt_reason) { Node* range) {
TemplateAssertionPredicateCreator template_assertion_predicate_creator(loop_head, scale, offset, range, this); TemplateAssertionPredicateCreator template_assertion_predicate_creator(loop_head, scale, offset, range, this);
return template_assertion_predicate_creator.create_with_uncommon_trap(new_control, parse_predicate_proj, deopt_reason, IfTrueNode* template_success_proj = template_assertion_predicate_creator.create(new_control);
if_opcode); _igvn.replace_input_of(parse_predicate, 0, template_success_proj);
set_idom(parse_predicate, template_success_proj, dom_depth(template_success_proj));
return template_success_proj;
} }
// Insert Hoisted Check Predicates for null checks and range checks and additional Template Assertion Predicates for // Insert Hoisted Check Predicates for null checks and range checks and additional Template Assertion Predicates for

2
src/hotspot/share/opto/loopTransform.cpp
View File

@ -2768,7 +2768,7 @@ void PhaseIdealLoop::do_range_check(IdealLoopTree *loop, Node_List &old_new) {
// unrolling or splitting this main-loop further. // unrolling or splitting this main-loop further.
TemplateAssertionPredicateCreator template_assertion_predicate_creator(cl, scale_con , int_offset, int_limit, TemplateAssertionPredicateCreator template_assertion_predicate_creator(cl, scale_con , int_offset, int_limit,
this); this);
loop_entry = template_assertion_predicate_creator.create_with_halt(loop_entry); loop_entry = template_assertion_predicate_creator.create(loop_entry);
assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected"); assert(assertion_predicate_has_loop_opaque_node(loop_entry->in(0)->as_If()), "unexpected");
// Initialized Assertion Predicate for the value of the initial main-loop. // Initialized Assertion Predicate for the value of the initial main-loop.

6
src/hotspot/share/opto/loopnode.cpp
View File

@ -2834,7 +2834,7 @@ Node* CountedLoopNode::skip_assertion_predicates_with_halt() {
ctrl = skip_strip_mined()->in(LoopNode::EntryControl); ctrl = skip_strip_mined()->in(LoopNode::EntryControl);
} }
if (is_main_loop() || is_post_loop()) { if (is_main_loop() || is_post_loop()) {
AssertionPredicatesWithHalt assertion_predicates(ctrl); AssertionPredicates assertion_predicates(ctrl);
return assertion_predicates.entry(); return assertion_predicates.entry();
} }
return ctrl; return ctrl;
@ -4470,7 +4470,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal
const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block(); const PredicateBlock* profiled_loop_predicate_block = predicates.profiled_loop_predicate_block();
if (profiled_loop_predicate_block->has_parse_predicate()) { if (profiled_loop_predicate_block->has_parse_predicate()) {
ParsePredicateSuccessProj* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj(); ParsePredicateSuccessProj* parse_predicate_proj = profiled_loop_predicate_block->parse_predicate_success_proj();
get_assertion_predicates(parse_predicate_proj, useful_predicates, true); get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true);
} }
} }
@ -4478,7 +4478,7 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal
const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block(); const PredicateBlock* loop_predicate_block = predicates.loop_predicate_block();
if (loop_predicate_block->has_parse_predicate()) { if (loop_predicate_block->has_parse_predicate()) {
ParsePredicateSuccessProj* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj(); ParsePredicateSuccessProj* parse_predicate_proj = loop_predicate_block->parse_predicate_success_proj();
get_assertion_predicates(parse_predicate_proj, useful_predicates, true); get_template_assertion_predicates(parse_predicate_proj, useful_predicates, true);
} }
} }
} }

27
src/hotspot/share/opto/loopnode.hpp
View File

@ -947,7 +947,7 @@ private:
DEBUG_ONLY(static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);) DEBUG_ONLY(static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);)
private: private:
DEBUG_ONLY(static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);) DEBUG_ONLY(static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);)
static void get_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, bool get_opaque = false); static void get_template_assertion_predicates(ParsePredicateSuccessProj* parse_predicate_proj, Unique_Node_List& list, bool get_opaque = false);
void update_main_loop_assertion_predicates(CountedLoopNode* main_loop_head); void update_main_loop_assertion_predicates(CountedLoopNode* main_loop_head);
void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head, void initialize_assertion_predicates_for_peeled_loop(CountedLoopNode* peeled_loop_head,
CountedLoopNode* remaining_loop_head, CountedLoopNode* remaining_loop_head,
@ -1389,10 +1389,8 @@ public:
void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt, void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt,
PathFrequency& pf, Node_Stack& stack, VectorSet& seen, PathFrequency& pf, Node_Stack& stack, VectorSet& seen,
Node_List& if_proj_list); Node_List& if_proj_list);
IfTrueNode* create_template_assertion_predicate(int if_opcode, CountedLoopNode* loop_head, IfTrueNode* create_template_assertion_predicate(CountedLoopNode* loop_head, ParsePredicateNode* parse_predicate,
ParsePredicateSuccessProj* parse_predicate_proj, IfProjNode* new_control, int scale, Node* offset, Node* range);
IfProjNode* new_control, int scale, Node* offset,
Node* range, Deoptimization::DeoptReason deopt_reason);
void eliminate_hoisted_range_check(IfTrueNode* hoisted_check_proj, IfTrueNode* template_assertion_predicate_proj); void eliminate_hoisted_range_check(IfTrueNode* hoisted_check_proj, IfTrueNode* template_assertion_predicate_proj);
// Helper function to collect predicate for eliminating the useless ones // Helper function to collect predicate for eliminating the useless ones
@ -1661,23 +1659,24 @@ private:
public: public:
// Clone Parse Predicates to slow and fast loop when unswitching a loop // Clone Parse Predicates to slow and fast loop when unswitching a loop
void clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, void clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred); IfProjNode*& true_path_loop_entry,
IfProjNode*& false_path_loop_entry);
private: private:
void clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, void clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
const PredicateBlock* predicate_block, const PredicateBlock* predicate_block,
Deoptimization::DeoptReason reason, IfProjNode*& iffast_pred, Deoptimization::DeoptReason reason,
IfProjNode*& ifslow_pred); IfProjNode*& true_path_loop_entry,
IfProjNode*& false_path_loop_entry);
void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason, void clone_parse_predicate_to_unswitched_loops(const PredicateBlock* predicate_block, Deoptimization::DeoptReason reason,
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred); IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred);
IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry, IfProjNode* clone_parse_predicate_to_unswitched_loop(ParsePredicateSuccessProj* parse_predicate_proj, Node* new_entry,
Deoptimization::DeoptReason reason, bool slow_loop); Deoptimization::DeoptReason reason, bool slow_loop);
void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, void clone_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
Deoptimization::DeoptReason reason, ParsePredicateSuccessProj* old_parse_predicate_proj, ParsePredicateSuccessProj* old_parse_predicate_proj,
ParsePredicateSuccessProj* fast_loop_parse_predicate_proj, ParsePredicateNode* true_path_loop_parse_predicate,
ParsePredicateSuccessProj* slow_loop_parse_predicate_proj); ParsePredicateNode* false_path_loop_parse_predicate);
IfProjNode* clone_assertion_predicate_for_unswitched_loops(IfNode* template_assertion_predicate, IfProjNode* predicate, IfTrueNode* clone_assertion_predicate_for_unswitched_loops(IfTrueNode* template_assertion_predicate_success_proj,
Deoptimization::DeoptReason reason, ParsePredicateNode* unswitched_loop_parse_predicate);
ParsePredicateSuccessProj* parse_predicate_proj);
static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN; static void check_cloned_parse_predicate_for_unswitching(const Node* new_entry, bool is_fast_loop) PRODUCT_RETURN;
bool _created_loop_node; bool _created_loop_node;

100
src/hotspot/share/opto/predicates.cpp
View File

@ -33,10 +33,10 @@
// Walk over all Initialized Assertion Predicates and return the entry into the first Initialized Assertion Predicate // Walk over all Initialized Assertion Predicates and return the entry into the first Initialized Assertion Predicate
// (i.e. not belonging to an Initialized Assertion Predicate anymore) // (i.e. not belonging to an Initialized Assertion Predicate anymore)
Node* AssertionPredicatesWithHalt::find_entry(Node* start_proj) { Node* AssertionPredicates::find_entry(Node* start_proj) {
assert(start_proj != nullptr, "should not be null"); assert(start_proj != nullptr, "should not be null");
Node* entry = start_proj; Node* entry = start_proj;
while (AssertionPredicateWithHalt::is_predicate(entry)) { while (AssertionPredicate::is_predicate(entry)) {
entry = entry->in(0)->in(0); entry = entry->in(0)->in(0);
} }
return entry; return entry;
@ -48,7 +48,7 @@ bool may_be_assertion_predicate_if(const Node* node) {
return node->is_IfTrue() && RegularPredicate::may_be_predicate_if(node->as_IfProj()); return node->is_IfTrue() && RegularPredicate::may_be_predicate_if(node->as_IfProj());
} }
bool AssertionPredicateWithHalt::is_predicate(const Node* maybe_success_proj) { bool AssertionPredicate::is_predicate(const Node* maybe_success_proj) {
if (!may_be_assertion_predicate_if(maybe_success_proj)) { if (!may_be_assertion_predicate_if(maybe_success_proj)) {
return false; return false;
} }
@ -57,14 +57,14 @@ bool AssertionPredicateWithHalt::is_predicate(const Node* maybe_success_proj) {
// Check if the If node of `predicate_proj` has an OpaqueTemplateAssertionPredicate (Template Assertion Predicate) or // Check if the If node of `predicate_proj` has an OpaqueTemplateAssertionPredicate (Template Assertion Predicate) or
// an OpaqueInitializedAssertionPredicate (Initialized Assertion Predicate) node as input. // an OpaqueInitializedAssertionPredicate (Initialized Assertion Predicate) node as input.
bool AssertionPredicateWithHalt::has_assertion_predicate_opaque(const Node* predicate_proj) { bool AssertionPredicate::has_assertion_predicate_opaque(const Node* predicate_proj) {
IfNode* iff = predicate_proj->in(0)->as_If(); IfNode* iff = predicate_proj->in(0)->as_If();
Node* bol = iff->in(1); Node* bol = iff->in(1);
return bol->is_OpaqueTemplateAssertionPredicate() || bol->is_OpaqueInitializedAssertionPredicate(); return bol->is_OpaqueTemplateAssertionPredicate() || bol->is_OpaqueInitializedAssertionPredicate();
} }
// Check if the other projection (UCT projection) of `success_proj` has a Halt node as output. // Check if the other projection (UCT projection) of `success_proj` has a Halt node as output.
bool AssertionPredicateWithHalt::has_halt(const Node* success_proj) { bool AssertionPredicate::has_halt(const Node* success_proj) {
ProjNode* other_proj = success_proj->as_IfProj()->other_if_proj(); ProjNode* other_proj = success_proj->as_IfProj()->other_if_proj();
return other_proj->outcnt() == 1 && other_proj->unique_out()->Opcode() == Op_Halt; return other_proj->outcnt() == 1 && other_proj->unique_out()->Opcode() == Op_Halt;
} }
@ -82,7 +82,7 @@ ParsePredicateNode* ParsePredicate::init_parse_predicate(Node* parse_predicate_p
return nullptr; return nullptr;
} }
Deoptimization::DeoptReason RegularPredicateWithUCT::uncommon_trap_reason(IfProjNode* if_proj) { Deoptimization::DeoptReason RuntimePredicate::uncommon_trap_reason(IfProjNode* if_proj) {
CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern(); CallStaticJavaNode* uct_call = if_proj->is_uncommon_trap_if_pattern();
if (uct_call == nullptr) { if (uct_call == nullptr) {
return Deoptimization::Reason_none; return Deoptimization::Reason_none;
@ -90,7 +90,7 @@ Deoptimization::DeoptReason RegularPredicateWithUCT::uncommon_trap_reason(IfProj
return Deoptimization::trap_request_reason(uct_call->uncommon_trap_request()); return Deoptimization::trap_request_reason(uct_call->uncommon_trap_request());
} }
bool RegularPredicateWithUCT::is_predicate(Node* maybe_success_proj) { bool RuntimePredicate::is_predicate(Node* maybe_success_proj) {
if (RegularPredicate::may_be_predicate_if(maybe_success_proj)) { if (RegularPredicate::may_be_predicate_if(maybe_success_proj)) {
return has_valid_uncommon_trap(maybe_success_proj); return has_valid_uncommon_trap(maybe_success_proj);
} else { } else {
@ -98,7 +98,7 @@ bool RegularPredicateWithUCT::is_predicate(Node* maybe_success_proj) {
} }
} }
bool RegularPredicateWithUCT::has_valid_uncommon_trap(const Node* success_proj) { bool RuntimePredicate::has_valid_uncommon_trap(const Node* success_proj) {
assert(RegularPredicate::may_be_predicate_if(success_proj), "must have been checked before"); assert(RegularPredicate::may_be_predicate_if(success_proj), "must have been checked before");
const Deoptimization::DeoptReason deopt_reason = uncommon_trap_reason(success_proj->as_IfProj()); const Deoptimization::DeoptReason deopt_reason = uncommon_trap_reason(success_proj->as_IfProj());
return (deopt_reason == Deoptimization::Reason_loop_limit_check || return (deopt_reason == Deoptimization::Reason_loop_limit_check ||
@ -106,7 +106,7 @@ bool RegularPredicateWithUCT::has_valid_uncommon_trap(const Node* success_proj)
deopt_reason == Deoptimization::Reason_profile_predicate); deopt_reason == Deoptimization::Reason_profile_predicate);
} }
bool RegularPredicateWithUCT::is_predicate(const Node* node, Deoptimization::DeoptReason deopt_reason) { bool RuntimePredicate::is_predicate(const Node* node, const Deoptimization::DeoptReason deopt_reason) {
if (RegularPredicate::may_be_predicate_if(node)) { if (RegularPredicate::may_be_predicate_if(node)) {
return deopt_reason == uncommon_trap_reason(node->as_IfProj()); return deopt_reason == uncommon_trap_reason(node->as_IfProj());
} else { } else {
@ -128,16 +128,6 @@ bool RegularPredicate::may_be_predicate_if(const Node* node) {
return false; return false;
} }
// Runtime Predicates always have an UCT since they could normally fail at runtime. In this case we execute the trap
// on the failing path.
bool RuntimePredicate::is_predicate(Node* node) {
return RegularPredicateWithUCT::is_predicate(node);
}
bool RuntimePredicate::is_predicate(Node* node, Deoptimization::DeoptReason deopt_reason) {
return RegularPredicateWithUCT::is_predicate(node, deopt_reason);
}
// Rewire any non-CFG nodes dependent on this Template Assertion Predicate (i.e. with a control input to this // Rewire any non-CFG nodes dependent on this Template Assertion Predicate (i.e. with a control input to this
// Template Assertion Predicate) to the 'target_predicate' based on the 'data_in_loop_body' check. // Template Assertion Predicate) to the 'target_predicate' based on the 'data_in_loop_body' check.
void TemplateAssertionPredicate::rewire_loop_data_dependencies(IfTrueNode* target_predicate, void TemplateAssertionPredicate::rewire_loop_data_dependencies(IfTrueNode* target_predicate,
@ -161,6 +151,21 @@ bool TemplateAssertionPredicate::is_predicate(Node* node) {
return if_node->in(1)->is_OpaqueTemplateAssertionPredicate(); return if_node->in(1)->is_OpaqueTemplateAssertionPredicate();
} }
// Clone this Template Assertion Predicate and replace the OpaqueLoopInitNode with the provided 'new_opaque_init' node.
IfTrueNode* TemplateAssertionPredicate::clone(Node* new_control, PhaseIdealLoop* phase) const {
assert(PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(_if_node),
"must find OpaqueLoop* nodes for Template Assertion Predicate");
TemplateAssertionExpression template_assertion_expression(opaque_node());
OpaqueTemplateAssertionPredicateNode* new_opaque_node = template_assertion_expression.clone(new_control, phase);
AssertionPredicateIfCreator assertion_predicate_if_creator(phase);
IfTrueNode* success_proj = assertion_predicate_if_creator.create_for_template(new_control, _if_node->Opcode(),
new_opaque_node,
_if_node->assertion_predicate_type());
assert(PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(success_proj->in(0)->as_If()),
"Template Assertion Predicates must have OpaqueLoop* nodes in the bool expression");
return success_proj;
}
// Clone this Template Assertion Predicate and replace the OpaqueLoopInitNode with the provided 'new_opaque_init' node. // Clone this Template Assertion Predicate and replace the OpaqueLoopInitNode with the provided 'new_opaque_init' node.
IfTrueNode* TemplateAssertionPredicate::clone_and_replace_init(Node* new_control, OpaqueLoopInitNode* new_opaque_init, IfTrueNode* TemplateAssertionPredicate::clone_and_replace_init(Node* new_control, OpaqueLoopInitNode* new_opaque_init,
PhaseIdealLoop* phase) const { PhaseIdealLoop* phase) const {
@ -168,7 +173,7 @@ IfTrueNode* TemplateAssertionPredicate::clone_and_replace_init(Node* new_control
"must find OpaqueLoop* nodes for Template Assertion Predicate"); "must find OpaqueLoop* nodes for Template Assertion Predicate");
TemplateAssertionExpression template_assertion_expression(opaque_node()); TemplateAssertionExpression template_assertion_expression(opaque_node());
OpaqueTemplateAssertionPredicateNode* new_opaque_node = OpaqueTemplateAssertionPredicateNode* new_opaque_node =
template_assertion_expression.clone_and_replace_init(new_opaque_init, new_control, phase); template_assertion_expression.clone_and_replace_init(new_control, new_opaque_init, phase);
AssertionPredicateIfCreator assertion_predicate_if_creator(phase); AssertionPredicateIfCreator assertion_predicate_if_creator(phase);
IfTrueNode* success_proj = assertion_predicate_if_creator.create_for_template(new_control, _if_node->Opcode(), IfTrueNode* success_proj = assertion_predicate_if_creator.create_for_template(new_control, _if_node->Opcode(),
new_opaque_node, new_opaque_node,
@ -295,16 +300,16 @@ class ReplaceInitAndStrideStrategy : public TransformStrategyForOpaqueLoopNodes
// OpaqueTemplateAssertionPredicate to and including the OpaqueLoop* nodes). The cloned nodes are rewired to reflect the // OpaqueTemplateAssertionPredicate to and including the OpaqueLoop* nodes). The cloned nodes are rewired to reflect the
// same graph structure as found for this Template Assertion Expression. The cloned nodes get 'new_ctrl' as ctrl. There // same graph structure as found for this Template Assertion Expression. The cloned nodes get 'new_ctrl' as ctrl. There
// is no other update done for the cloned nodes. Return the newly cloned OpaqueTemplateAssertionPredicate. // is no other update done for the cloned nodes. Return the newly cloned OpaqueTemplateAssertionPredicate.
OpaqueTemplateAssertionPredicateNode* TemplateAssertionExpression::clone(Node* new_ctrl, PhaseIdealLoop* phase) { OpaqueTemplateAssertionPredicateNode* TemplateAssertionExpression::clone(Node* new_control, PhaseIdealLoop* phase) {
CloneStrategy clone_init_and_stride_strategy(phase, new_ctrl); CloneStrategy clone_init_and_stride_strategy(phase, new_control);
return clone(clone_init_and_stride_strategy, new_ctrl, phase); return clone(clone_init_and_stride_strategy, new_control, phase);
} }
// Same as clone() but instead of cloning the OpaqueLoopInitNode, we replace it with the provided 'new_init' node. // Same as clone() but instead of cloning the OpaqueLoopInitNode, we replace it with the provided 'new_init' node.
OpaqueTemplateAssertionPredicateNode* OpaqueTemplateAssertionPredicateNode*
TemplateAssertionExpression::clone_and_replace_init(Node* new_init, Node* new_ctrl, PhaseIdealLoop* phase) { TemplateAssertionExpression::clone_and_replace_init(Node* new_control, Node* new_init, PhaseIdealLoop* phase) {
ReplaceInitAndCloneStrideStrategy replace_init_and_clone_stride_strategy(new_init, new_ctrl, phase); ReplaceInitAndCloneStrideStrategy replace_init_and_clone_stride_strategy(new_init, new_control, phase);
return clone(replace_init_and_clone_stride_strategy, new_ctrl, phase); return clone(replace_init_and_clone_stride_strategy, new_control, phase);
} }
// Same as clone() but instead of cloning the OpaqueLoopInit and OpaqueLoopStride node, we replace them with the provided // Same as clone() but instead of cloning the OpaqueLoopInit and OpaqueLoopStride node, we replace them with the provided
@ -617,26 +622,6 @@ void AssertionPredicateIfCreator::create_halt_node(IfFalseNode* fail_proj, Ideal
_phase->register_control(halt, loop, fail_proj); _phase->register_control(halt, loop, fail_proj);
} }
// Creates an init and last value Template Assertion Predicate connected together from a Parse Predicate with an UCT on
// the failing path. Returns the success projection of the last value Template Assertion Predicate.
IfTrueNode* TemplateAssertionPredicateCreator::create_with_uncommon_trap(
Node* new_control, ParsePredicateSuccessProj* parse_predicate_success_proj,
const Deoptimization::DeoptReason deopt_reason, const int if_opcode) {
OpaqueLoopInitNode* opaque_init = create_opaque_init(new_control);
bool does_overflow;
OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression =
create_for_init_value(new_control, opaque_init, does_overflow);
IfTrueNode* template_predicate_success_proj =
create_if_node_with_uncommon_trap(template_assertion_predicate_expression, parse_predicate_success_proj,
deopt_reason, if_opcode, does_overflow,
AssertionPredicateType::InitValue);
template_assertion_predicate_expression = create_for_last_value(template_predicate_success_proj, opaque_init,
does_overflow);
return create_if_node_with_uncommon_trap(template_assertion_predicate_expression, parse_predicate_success_proj,
deopt_reason, if_opcode, does_overflow,
AssertionPredicateType::LastValue);
}
OpaqueLoopInitNode* TemplateAssertionPredicateCreator::create_opaque_init(Node* new_control) { OpaqueLoopInitNode* TemplateAssertionPredicateCreator::create_opaque_init(Node* new_control) {
OpaqueLoopInitNode* opaque_init = new OpaqueLoopInitNode(_phase->C, _loop_head->init_trip()); OpaqueLoopInitNode* opaque_init = new OpaqueLoopInitNode(_phase->C, _loop_head->init_trip());
_phase->register_new_node(opaque_init, new_control); _phase->register_new_node(opaque_init, new_control);
@ -650,17 +635,6 @@ TemplateAssertionPredicateCreator::create_for_init_value(Node* new_control, Opaq
return expression_creator.create_for_template(new_control, opaque_init, does_overflow); return expression_creator.create_for_template(new_control, opaque_init, does_overflow);
} }
IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_uncommon_trap(
OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression,
ParsePredicateSuccessProj* parse_predicate_success_proj, const Deoptimization::DeoptReason deopt_reason,
const int if_opcode, const bool does_overflow, const AssertionPredicateType assertion_predicate_type) {
IfTrueNode* success_proj = _phase->create_new_if_for_predicate(parse_predicate_success_proj, nullptr, deopt_reason,
does_overflow ? Op_If : if_opcode, false,
assertion_predicate_type);
_phase->igvn().replace_input_of(success_proj->in(0), 1, template_assertion_predicate_expression);
return success_proj;
}
OpaqueTemplateAssertionPredicateNode* OpaqueTemplateAssertionPredicateNode*
TemplateAssertionPredicateCreator::create_for_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init, TemplateAssertionPredicateCreator::create_for_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init,
bool& does_overflow) const { bool& does_overflow) const {
@ -683,7 +657,7 @@ Node* TemplateAssertionPredicateCreator::create_last_value(Node* new_control, Op
return last_value; return last_value;
} }
IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_halt( IfTrueNode* TemplateAssertionPredicateCreator::create_if_node(
Node* new_control, OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression, Node* new_control, OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression,
const bool does_overflow, const AssertionPredicateType assertion_predicate_type) { const bool does_overflow, const AssertionPredicateType assertion_predicate_type) {
AssertionPredicateIfCreator assertion_predicate_if_creator(_phase); AssertionPredicateIfCreator assertion_predicate_if_creator(_phase);
@ -694,18 +668,18 @@ IfTrueNode* TemplateAssertionPredicateCreator::create_if_node_with_halt(
// Creates an init and last value Template Assertion Predicate connected together with a Halt node on the failing path. // Creates an init and last value Template Assertion Predicate connected together with a Halt node on the failing path.
// Returns the success projection of the last value Template Assertion Predicate latter. // Returns the success projection of the last value Template Assertion Predicate latter.
IfTrueNode* TemplateAssertionPredicateCreator::create_with_halt(Node* new_control) { IfTrueNode* TemplateAssertionPredicateCreator::create(Node* new_control) {
OpaqueLoopInitNode* opaque_init = create_opaque_init(new_control); OpaqueLoopInitNode* opaque_init = create_opaque_init(new_control);
bool does_overflow; bool does_overflow;
OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression = OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression =
create_for_init_value(new_control, opaque_init, does_overflow); create_for_init_value(new_control, opaque_init, does_overflow);
IfTrueNode* template_predicate_success_proj = IfTrueNode* template_predicate_success_proj =
create_if_node_with_halt(new_control, template_assertion_predicate_expression, does_overflow, create_if_node(new_control, template_assertion_predicate_expression, does_overflow,
AssertionPredicateType::InitValue); AssertionPredicateType::InitValue);
template_assertion_predicate_expression = create_for_last_value(template_predicate_success_proj, opaque_init, template_assertion_predicate_expression = create_for_last_value(template_predicate_success_proj, opaque_init,
does_overflow); does_overflow);
return create_if_node_with_halt(template_predicate_success_proj, template_assertion_predicate_expression, return create_if_node(template_predicate_success_proj, template_assertion_predicate_expression,
does_overflow, AssertionPredicateType::LastValue); does_overflow, AssertionPredicateType::LastValue);
} }
InitializedAssertionPredicateCreator::InitializedAssertionPredicateCreator(PhaseIdealLoop* phase) InitializedAssertionPredicateCreator::InitializedAssertionPredicateCreator(PhaseIdealLoop* phase)

56
src/hotspot/share/opto/predicates.hpp
View File

@ -252,15 +252,15 @@ class NodeInLoopBody : public StackObj {
virtual bool check(Node* node) const = 0; virtual bool check(Node* node) const = 0;
}; };
// Class to represent Assertion Predicates with a HaltNode instead of an UCT (i.e. either an Initialized Assertion // Class to represent Assertion Predicates (i.e. either Initialized and/or Template Assertion Predicates).
// Predicate or a Template Assertion Predicate created after the initial one at Loop Predication). class AssertionPredicates : public StackObj {
class AssertionPredicatesWithHalt : public StackObj { Node* const _entry;
Node* _entry;
static Node* find_entry(Node* start_proj); static Node* find_entry(Node* start_proj);
public: public:
AssertionPredicatesWithHalt(Node* assertion_predicate_proj) : _entry(find_entry(assertion_predicate_proj)) {} explicit AssertionPredicates(Node* assertion_predicate_proj) : _entry(find_entry(assertion_predicate_proj)) {}
NONCOPYABLE(AssertionPredicates);
// Returns the control input node into the first assertion predicate If. If there are no assertion predicates, it // Returns the control input node into the first assertion predicate If. If there are no assertion predicates, it
// returns the same node initially passed to the constructor. // returns the same node initially passed to the constructor.
@ -269,15 +269,14 @@ class AssertionPredicatesWithHalt : public StackObj {
} }
}; };
// Class to represent a single Assertion Predicate with a HaltNode. This could either be: // Class to represent a single Assertion Predicate. This could either be:
// - A Template Assertion Predicate. // - A Template Assertion Predicate.
// - An Initialized Assertion Predicate. // - An Initialized Assertion Predicate.
// Note that all other Regular Predicates have an UCT node. class AssertionPredicate : public StackObj {
class AssertionPredicateWithHalt : public StackObj {
static bool has_assertion_predicate_opaque(const Node* predicate_proj); static bool has_assertion_predicate_opaque(const Node* predicate_proj);
static bool has_halt(const Node* success_proj);
public: public:
static bool is_predicate(const Node* maybe_success_proj); static bool is_predicate(const Node* maybe_success_proj);
static bool has_halt(const Node* success_proj);
}; };
// Utility class representing a Regular Predicate which is either a Runtime Predicate or an Assertion Predicate. // Utility class representing a Regular Predicate which is either a Runtime Predicate or an Assertion Predicate.
@ -286,19 +285,6 @@ class RegularPredicate : public StackObj {
static bool may_be_predicate_if(const Node* node); static bool may_be_predicate_if(const Node* node);
}; };
// Class to represent a single Regular Predicate with an UCT. This could either be:
// - A Runtime Predicate
// - A Template Assertion Predicate
// Note that all other Regular Predicates have a Halt node.
class RegularPredicateWithUCT : public StackObj {
static Deoptimization::DeoptReason uncommon_trap_reason(IfProjNode* if_proj);
public:
static bool is_predicate(Node* maybe_success_proj);
static bool is_predicate(const Node* node, Deoptimization::DeoptReason deopt_reason);
static bool has_valid_uncommon_trap(const Node* success_proj);
};
// Class to represent a Parse Predicate. // Class to represent a Parse Predicate.
class ParsePredicate : public Predicate { class ParsePredicate : public Predicate {
ParsePredicateSuccessProj* _success_proj; ParsePredicateSuccessProj* _success_proj;
@ -356,6 +342,8 @@ class RuntimePredicate : public Predicate {
private: private:
static bool is_predicate(Node* maybe_success_proj); static bool is_predicate(Node* maybe_success_proj);
static bool has_valid_uncommon_trap(const Node* success_proj);
static Deoptimization::DeoptReason uncommon_trap_reason(IfProjNode* if_proj);
public: public:
Node* entry() const override { Node* entry() const override {
@ -370,7 +358,7 @@ class RuntimePredicate : public Predicate {
return _success_proj; return _success_proj;
} }
static bool is_predicate(Node* node, Deoptimization::DeoptReason deopt_reason); static bool is_predicate(const Node* node, Deoptimization::DeoptReason deopt_reason);
}; };
// Class to represent a Template Assertion Predicate. // Class to represent a Template Assertion Predicate.
@ -405,6 +393,7 @@ class TemplateAssertionPredicate : public Predicate {
return _if_node->assertion_predicate_type() == AssertionPredicateType::LastValue; return _if_node->assertion_predicate_type() == AssertionPredicateType::LastValue;
} }
IfTrueNode* clone(Node* new_control, PhaseIdealLoop* phase) const;
IfTrueNode* clone_and_replace_init(Node* new_control, OpaqueLoopInitNode* new_opaque_init, PhaseIdealLoop* phase) const; IfTrueNode* clone_and_replace_init(Node* new_control, OpaqueLoopInitNode* new_opaque_init, PhaseIdealLoop* phase) const;
void replace_opaque_stride_input(Node* new_stride, PhaseIterGVN& igvn) const; void replace_opaque_stride_input(Node* new_stride, PhaseIterGVN& igvn) const;
IfTrueNode* initialize(PhaseIdealLoop* phase, Node* new_control) const; IfTrueNode* initialize(PhaseIdealLoop* phase, Node* new_control) const;
@ -466,8 +455,9 @@ class TemplateAssertionExpression : public StackObj {
Node* new_ctrl, PhaseIdealLoop* phase); Node* new_ctrl, PhaseIdealLoop* phase);
public: public:
OpaqueTemplateAssertionPredicateNode* clone(Node* new_ctrl, PhaseIdealLoop* phase); OpaqueTemplateAssertionPredicateNode* clone(Node* new_control, PhaseIdealLoop* phase);
OpaqueTemplateAssertionPredicateNode* clone_and_replace_init(Node* new_init, Node* new_ctrl, PhaseIdealLoop* phase); OpaqueTemplateAssertionPredicateNode* clone_and_replace_init(Node* new_control, Node* new_init,
PhaseIdealLoop* phase);
OpaqueTemplateAssertionPredicateNode* clone_and_replace_init_and_stride(Node* new_control, Node* new_init, OpaqueTemplateAssertionPredicateNode* clone_and_replace_init_and_stride(Node* new_control, Node* new_init,
Node* new_stride, PhaseIdealLoop* phase); Node* new_stride, PhaseIdealLoop* phase);
void replace_opaque_stride_input(Node* new_stride, PhaseIterGVN& igvn); void replace_opaque_stride_input(Node* new_stride, PhaseIterGVN& igvn);
@ -570,7 +560,7 @@ class AssertionPredicateIfCreator : public StackObj {
void create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop, const char* halt_message); void create_halt_node(IfFalseNode* fail_proj, IdealLoopTree* loop, const char* halt_message);
}; };
// This class is used to create a Template Assertion Predicate either with an UCT or a Halt Node from scratch. // This class is used to create a Template Assertion Predicate either with a Halt Node from scratch.
class TemplateAssertionPredicateCreator : public StackObj { class TemplateAssertionPredicateCreator : public StackObj {
CountedLoopNode* const _loop_head; CountedLoopNode* const _loop_head;
const int _scale; const int _scale;
@ -584,13 +574,9 @@ class TemplateAssertionPredicateCreator : public StackObj {
OpaqueTemplateAssertionPredicateNode* create_for_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init, OpaqueTemplateAssertionPredicateNode* create_for_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init,
bool& does_overflow) const; bool& does_overflow) const;
Node* create_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init) const; Node* create_last_value(Node* new_control, OpaqueLoopInitNode* opaque_init) const;
IfTrueNode* create_if_node_with_uncommon_trap(OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression, IfTrueNode* create_if_node(Node* new_control,
ParsePredicateSuccessProj* parse_predicate_success_proj, OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression,
Deoptimization::DeoptReason deopt_reason, int if_opcode, bool does_overflow, AssertionPredicateType assertion_predicate_type);
bool does_overflow, AssertionPredicateType assertion_predicate_type);
IfTrueNode* create_if_node_with_halt(Node* new_control,
OpaqueTemplateAssertionPredicateNode* template_assertion_predicate_expression,
bool does_overflow, AssertionPredicateType assertion_predicate_type);
public: public:
TemplateAssertionPredicateCreator(CountedLoopNode* loop_head, int scale, Node* offset, Node* range, TemplateAssertionPredicateCreator(CountedLoopNode* loop_head, int scale, Node* offset, Node* range,
@ -602,9 +588,7 @@ class TemplateAssertionPredicateCreator : public StackObj {
_phase(phase) {} _phase(phase) {}
NONCOPYABLE(TemplateAssertionPredicateCreator); NONCOPYABLE(TemplateAssertionPredicateCreator);
IfTrueNode* create_with_uncommon_trap(Node* new_control, ParsePredicateSuccessProj* parse_predicate_success_proj, IfTrueNode* create(Node* new_control);
Deoptimization::DeoptReason deopt_reason, int if_opcode);
IfTrueNode* create_with_halt(Node* new_control);
}; };
// This class creates a new Initialized Assertion Predicate either from a template or from scratch. // This class creates a new Initialized Assertion Predicate either from a template or from scratch.