stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8324775: C2 SuperWord: refactor visited sets

Browse Source 
Reviewed-by: kvn, chagedorn
This commit is contained in:
Emanuel Peter 2024-01-29 08:46:34 +00:00
parent 422020c4d6
commit 6ad78ca8a5
2 changed files with 74 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
src/hotspot/share/opto/superword.cpp
View File

@ -57,10 +57,7 @@ SuperWord::SuperWord(PhaseIdealLoop* phase) :
_clone_map(phase->C->clone_map()), // map of nodes created in cloning
_align_to_ref(nullptr), // memory reference to align vectors to
_dg(_arena), // dependence graph
_visited(arena()), // visited node set
_post_visited(arena()), // post visited node set
_nlist(arena(), 8, 0, nullptr), // scratch list of nodes
_stk(arena(), 8, 0, nullptr), // scratch stack of nodes
_lpt(nullptr), // loop tree node
_lp(nullptr), // CountedLoopNode
_loop_reductions(arena()), // reduction nodes in the current loop
@ -1059,54 +1056,71 @@ bool SuperWord::isomorphic(Node* s1, Node* s2) {
//------------------------------independent---------------------------
// Is there no data path from s1 to s2 or s2 to s1?
bool SuperWord::independent(Node* s1, Node* s2) {
// assert(s1->Opcode() == s2->Opcode(), "check isomorphic first");
int d1 = depth(s1);
int d2 = depth(s2);
if (d1 == d2) return s1 != s2;
if (d1 == d2) {
// Same depth:
// 1) same node -> dependent
// 2) different nodes -> same level implies there is no path
return s1 != s2;
}
// Traversal starting at the deeper node to find the shallower one.
Node* deep = d1 > d2 ? s1 : s2;
Node* shallow = d1 > d2 ? s2 : s1;
int min_d = MIN2(d1, d2); // prune traversal at min_d
visited_clear();
return independent_path(shallow, deep);
ResourceMark rm;
Unique_Node_List worklist;
worklist.push(deep);
for (uint i = 0; i < worklist.size(); i++) {
Node* n = worklist.at(i);
for (DepPreds preds(n, _dg); !preds.done(); preds.next()) {
Node* pred = preds.current();
if (in_bb(pred) && depth(pred) >= min_d) {
if (pred == shallow) {
return false; // found it -> dependent
}
worklist.push(pred);
}
}
}
return true; // not found -> independent
}
//------------------------------find_dependence---------------------
// Is any s1 in p dependent on any s2 in p? Yes: return such a s2. No: return nullptr.
// Are all nodes in nodes list mutually independent?
// We could query independent(s1, s2) for all pairs, but that results
// in O(p.size * p.size) graph traversals. We can do it all in one BFS!
// Start the BFS traversal at all nodes from the pack. Traverse DepPreds
// recursively, for nodes that have at least depth min_d, which is the
// smallest depth of all nodes from the pack. Once we have traversed all
// those nodes, and have not found another node from the pack, we know
// that all nodes in the pack are independent.
Node* SuperWord::find_dependence(Node_List* p) {
if (is_marked_reduction(p->at(0))) {
return nullptr; // ignore reductions
}
// in O(size * size) graph traversals. We can do it all in one BFS!
// Start the BFS traversal at all nodes from the nodes list. Traverse
// Preds recursively, for nodes that have at least depth min_d, which
// is the smallest depth of all nodes from the nodes list. Once we have
// traversed all those nodes, and have not found another node from the
// nodes list, we know that all nodes in the nodes list are independent.
bool SuperWord::mutually_independent(Node_List* nodes) const {
ResourceMark rm;
Unique_Node_List worklist; // traversal queue
int min_d = depth(p->at(0));
visited_clear();
for (uint k = 0; k < p->size(); k++) {
Node* n = p->at(k);
Unique_Node_List worklist;
VectorSet nodes_set;
int min_d = depth(nodes->at(0));
for (uint k = 0; k < nodes->size(); k++) {
Node* n = nodes->at(k);
min_d = MIN2(min_d, depth(n));
worklist.push(n); // start traversal at all nodes in p
visited_set(n); // mark node
worklist.push(n); // start traversal at all nodes in nodes list
nodes_set.set(bb_idx(n));
}
for (uint i = 0; i < worklist.size(); i++) {
Node* n = worklist.at(i);
for (DepPreds preds(n, _dg); !preds.done(); preds.next()) {
Node* pred = preds.current();
if (in_bb(pred) && depth(pred) >= min_d) {
if (visited_test(pred)) { // marked as in p?
return pred;
if (nodes_set.test(bb_idx(pred))) {
return false; // found one -> dependent
}
worklist.push(pred);
}
}
}
return nullptr;
return true; // not found -> independent
}
//--------------------------have_similar_inputs-----------------------
@ -1154,27 +1168,6 @@ bool SuperWord::reduction(Node* s1, Node* s2) {
return retValue;
}
//------------------------------independent_path------------------------------
// Helper for independent
bool SuperWord::independent_path(Node* shallow, Node* deep, uint dp) {
if (dp >= 1000) return false; // stop deep recursion
visited_set(deep);
int shal_depth = depth(shallow);
assert(shal_depth <= depth(deep), "must be");
for (DepPreds preds(deep, _dg); !preds.done(); preds.next()) {
Node* pred = preds.current();
if (in_bb(pred) && !visited_test(pred)) {
if (shallow == pred) {
return false;
}
if (shal_depth < depth(pred) && !independent_path(shallow, pred, dp+1)) {
return false;
}
}
}
return true;
}
//------------------------------set_alignment---------------------------
void SuperWord::set_alignment(Node* s1, Node* s2, int align) {
set_alignment(s1, align);
@ -1567,13 +1560,13 @@ void SuperWord::combine_packs() {
for (int i = 0; i < _packset.length(); i++) {
Node_List* p = _packset.at(i);
if (p != nullptr) {
Node* dependence = find_dependence(p);
if (dependence != nullptr) {
// reductions are trivially connected
if (!is_marked_reduction(p->at(0)) &&
!mutually_independent(p)) {
#ifndef PRODUCT
if (TraceSuperWord) {
tty->cr();
tty->print_cr("WARNING: Found dependency at distance greater than 1.");
dependence->dump();
tty->print_cr("In pack[%d]", i);
print_pack(p);
}
@ -1956,24 +1949,16 @@ void SuperWord::verify_packs() {
// Verify independence at pack level.
for (int i = 0; i < _packset.length(); i++) {
Node_List* p = _packset.at(i);
Node* dependence = find_dependence(p);
if (dependence != nullptr) {
tty->print_cr("Other nodes in pack have dependence on:");
dependence->dump();
tty->print_cr("The following nodes are not independent:");
for (uint k = 0; k < p->size(); k++) {
Node* n = p->at(k);
if (!independent(n, dependence)) {
n->dump();
}
}
tty->print_cr("They are all from pack[%d]", i);
if (!is_marked_reduction(p->at(0)) &&
!mutually_independent(p)) {
tty->print_cr("FAILURE: nodes not mutually independent in pack[%d]", i);
print_pack(p);
assert(false, "pack nodes not mutually independent");
}
assert(dependence == nullptr, "all nodes in pack must be mutually independent");
}
// Verify all nodes in packset have my_pack set correctly.
ResourceMark rm;
Unique_Node_List processed;
for (int i = 0; i < _packset.length(); i++) {
Node_List* p = _packset.at(i);
@ -2944,7 +2929,6 @@ bool SuperWord::is_vector_use(Node* use, int u_idx) {
bool SuperWord::construct_bb() {
Node* entry = bb();
assert(_stk.length() == 0, "stk is empty");
assert(_block.length() == 0, "block is empty");
assert(_data_entry.length() == 0, "data_entry is empty");
assert(_mem_slice_head.length() == 0, "mem_slice_head is empty");
@ -3001,31 +2985,33 @@ bool SuperWord::construct_bb() {
// Create an RPO list of nodes in block
visited_clear();
post_visited_clear();
ResourceMark rm;
GrowableArray<Node*> stack;
VectorSet visited;
VectorSet post_visited;
// Push all non-control nodes with no inputs from within block, then control entry
for (int j = 0; j < _data_entry.length(); j++) {
Node* n = _data_entry.at(j);
visited_set(n);
_stk.push(n);
visited.set(bb_idx(n));
stack.push(n);
}
visited_set(entry);
_stk.push(entry);
visited.set(bb_idx(entry));
stack.push(entry);
// Do a depth first walk over out edges
int rpo_idx = bb_ct - 1;
int size;
int reduction_uses = 0;
while ((size = _stk.length()) > 0) {
Node* n = _stk.top(); // Leave node on stack
if (!visited_test_set(n)) {
while ((size = stack.length()) > 0) {
Node* n = stack.top(); // Leave node on stack
if (!visited.test_set(bb_idx(n))) {
// forward arc in graph
} else if (!post_visited_test(n)) {
} else if (!post_visited.test(bb_idx(n))) {
// cross or back arc
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
Node *use = n->fast_out(i);
if (in_bb(use) && !visited_test(use) &&
if (in_bb(use) && !visited.test(bb_idx(use)) &&
// Don't go around backedge
(!use->is_Phi() || n == entry)) {
if (is_marked_reduction(use)) {
@ -3036,20 +3022,20 @@ bool SuperWord::construct_bb() {
reduction_uses++;
}
}
_stk.push(use);
stack.push(use);
}
}
if (_stk.length() == size) {
if (stack.length() == size) {
// There were no additional uses, post visit node now
_stk.pop(); // Remove node from stack
stack.pop(); // Remove node from stack
assert(rpo_idx >= 0, "");
_block.at_put_grow(rpo_idx, n);
rpo_idx--;
post_visited_set(n);
assert(rpo_idx >= 0 || _stk.is_empty(), "");
post_visited.set(bb_idx(n));
assert(rpo_idx >= 0 || stack.is_empty(), "");
}
} else {
_stk.pop(); // Remove post-visited node from stack
stack.pop(); // Remove post-visited node from stack
}
}//while

20
src/hotspot/share/opto/superword.hpp
View File

@ -224,10 +224,7 @@ class SuperWord : public ResourceObj {
DepGraph _dg; // Dependence graph
// Scratch pads
VectorSet _visited; // Visited set
VectorSet _post_visited; // Post-visited set
GrowableArray<Node*> _nlist; // List of nodes
GrowableArray<Node*> _stk; // Stack of nodes
public:
SuperWord(PhaseIdealLoop* phase);
@ -306,15 +303,6 @@ class SuperWord : public ResourceObj {
private:
void set_bb_idx(Node* n, int i) { _bb_idx.at_put_grow(n->_idx, i); }
// visited set accessors
void visited_clear() { _visited.clear(); }
void visited_set(Node* n) { return _visited.set(bb_idx(n)); }
int visited_test(Node* n) { return _visited.test(bb_idx(n)); }
int visited_test_set(Node* n) { return _visited.test_set(bb_idx(n)); }
void post_visited_clear() { _post_visited.clear(); }
void post_visited_set(Node* n) { return _post_visited.set(bb_idx(n)); }
int post_visited_test(Node* n) { return _post_visited.test(bb_idx(n)); }
// Ensure node_info contains element "i"
void grow_node_info(int i) { if (i >= _node_info.length()) _node_info.at_put_grow(i, SWNodeInfo::initial); }
@ -326,7 +314,7 @@ class SuperWord : public ResourceObj {
void set_alignment(Node* n, int a) { int i = bb_idx(n); grow_node_info(i); _node_info.adr_at(i)->_alignment = a; }
// Max expression (DAG) depth from beginning of the block for each node
int depth(Node* n) { return _node_info.adr_at(bb_idx(n))->_depth; }
int depth(Node* n) const { return _node_info.adr_at(bb_idx(n))->_depth; }
void set_depth(Node* n, int d) { int i = bb_idx(n); grow_node_info(i); _node_info.adr_at(i)->_depth = d; }
// vector element type
@ -429,15 +417,13 @@ private:
bool isomorphic(Node* s1, Node* s2);
// Is there no data path from s1 to s2 or s2 to s1?
bool independent(Node* s1, Node* s2);
// Is any s1 in p dependent on any s2 in p? Yes: return such a s2. No: return nullptr.
Node* find_dependence(Node_List* p);
// Are all nodes in nodes list mutually independent?
bool mutually_independent(Node_List* nodes) const;
// For a node pair (s1, s2) which is isomorphic and independent,
// do s1 and s2 have similar input edges?
bool have_similar_inputs(Node* s1, Node* s2);
// Is there a data path between s1 and s2 and both are reductions?
bool reduction(Node* s1, Node* s2);
// Helper for independent
bool independent_path(Node* shallow, Node* deep, uint dp=0);
void set_alignment(Node* s1, Node* s2, int align);
int data_size(Node* s);
// Extend packset by following use->def and def->use links from pack members.