stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8238690: C2: Handle vector shifts by constant and non-constant scalar uniformly

Browse Source 
Reviewed-by: thartmann, sviswanathan
This commit is contained in:
Vladimir Ivanov 2020-02-11 14:55:43 +03:00
parent 0d84fe9872
commit 74e68b4092
8 changed files with 71 additions and 69 deletions

42
src/hotspot/cpu/aarch64/aarch64.ad

@ -17312,7 +17312,7 @@ instruct vsrl16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 4 ||
n->as_Vector()->length() == 8);
match(Set dst (LShiftVB src shift));
match(Set dst (LShiftVB src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (8B)" %}
ins_encode %{
@ -17331,7 +17331,7 @@ instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 16);
match(Set dst (LShiftVB src shift));
match(Set dst (LShiftVB src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (16B)" %}
ins_encode %{
@ -17351,7 +17351,7 @@ instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 4 ||
n->as_Vector()->length() == 8);
match(Set dst (RShiftVB src shift));
match(Set dst (RShiftVB src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (8B)" %}
ins_encode %{
@ -17365,7 +17365,7 @@ instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 16);
match(Set dst (RShiftVB src shift));
match(Set dst (RShiftVB src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (16B)" %}
ins_encode %{
@ -17380,7 +17380,7 @@ instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 4 ||
n->as_Vector()->length() == 8);
match(Set dst (URShiftVB src shift));
match(Set dst (URShiftVB src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (8B)" %}
ins_encode %{
@ -17399,7 +17399,7 @@ instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
instruct vsrl16B_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 16);
match(Set dst (URShiftVB src shift));
match(Set dst (URShiftVB src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (16B)" %}
ins_encode %{
@ -17516,7 +17516,7 @@ instruct vsrl8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2 ||
n->as_Vector()->length() == 4);
match(Set dst (LShiftVS src shift));
match(Set dst (LShiftVS src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (4H)" %}
ins_encode %{
@ -17535,7 +17535,7 @@ instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (LShiftVS src shift));
match(Set dst (LShiftVS src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (8H)" %}
ins_encode %{
@ -17555,7 +17555,7 @@ instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2 ||
n->as_Vector()->length() == 4);
match(Set dst (RShiftVS src shift));
match(Set dst (RShiftVS src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (4H)" %}
ins_encode %{
@ -17569,7 +17569,7 @@ instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (RShiftVS src shift));
match(Set dst (RShiftVS src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (8H)" %}
ins_encode %{
@ -17584,7 +17584,7 @@ instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2 ||
n->as_Vector()->length() == 4);
match(Set dst (URShiftVS src shift));
match(Set dst (URShiftVS src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (4H)" %}
ins_encode %{
@ -17603,7 +17603,7 @@ instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
instruct vsrl8S_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (URShiftVS src shift));
match(Set dst (URShiftVS src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (8H)" %}
ins_encode %{
@ -17716,7 +17716,7 @@ instruct vsrl4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (LShiftVI src shift));
match(Set dst (LShiftVI src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (2S)" %}
ins_encode %{
@ -17729,7 +17729,7 @@ instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (LShiftVI src shift));
match(Set dst (LShiftVI src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (4S)" %}
ins_encode %{
@ -17742,7 +17742,7 @@ instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (RShiftVI src shift));
match(Set dst (RShiftVI src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (2S)" %}
ins_encode %{
@ -17755,7 +17755,7 @@ instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (RShiftVI src shift));
match(Set dst (RShiftVI src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (4S)" %}
ins_encode %{
@ -17768,7 +17768,7 @@ instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (URShiftVI src shift));
match(Set dst (URShiftVI src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (2S)" %}
ins_encode %{
@ -17781,7 +17781,7 @@ instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
instruct vsrl4I_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (URShiftVI src shift));
match(Set dst (URShiftVI src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (4S)" %}
ins_encode %{
@ -17841,7 +17841,7 @@ instruct vsrl2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (LShiftVL src shift));
match(Set dst (LShiftVL src (LShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "shl $dst, $src, $shift\t# vector (2D)" %}
ins_encode %{
@ -17854,7 +17854,7 @@ instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (RShiftVL src shift));
match(Set dst (RShiftVL src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "sshr $dst, $src, $shift\t# vector (2D)" %}
ins_encode %{
@ -17867,7 +17867,7 @@ instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (URShiftVL src shift));
match(Set dst (URShiftVL src (RShiftCntV shift)));
ins_cost(INSN_COST);
format %{ "ushr $dst, $src, $shift\t# vector (2D)" %}
ins_encode %{

22
src/hotspot/cpu/arm/arm.ad

@ -10618,7 +10618,7 @@ instruct vsl16B_reg(vecX dst, vecX src, vecX shift) %{
instruct vsl8B_immI(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (LShiftVB src shift));
match(Set dst (LShiftVB src (LShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@ -10634,7 +10634,7 @@ instruct vsl8B_immI(vecD dst, vecD src, immI shift) %{
instruct vsl16B_immI(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 16);
match(Set dst (LShiftVB src shift));
match(Set dst (LShiftVB src (LShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@ -10673,7 +10673,7 @@ instruct vsl8S_reg(vecX dst, vecX src, vecX shift) %{
instruct vsl4S_immI(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (LShiftVS src shift));
match(Set dst (LShiftVS src (LShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@ -10728,7 +10728,7 @@ instruct vsl4I_reg(vecX dst, vecX src, vecX shift) %{
instruct vsl2I_immI(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
match(Set dst (LShiftVI src shift));
match(Set dst (LShiftVI src (LShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@ -10744,7 +10744,7 @@ instruct vsl2I_immI(vecD dst, vecD src, immI shift) %{
instruct vsl4I_immI(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
match(Set dst (LShiftVI src shift));
match(Set dst (LShiftVI src (LShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@ -10772,7 +10772,7 @@ instruct vsl2L_reg(vecX dst, vecX src, vecX shift) %{
instruct vsl2L_immI(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (LShiftVL src shift));
match(Set dst (LShiftVL src (LShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@ -10795,7 +10795,7 @@ instruct vsl2L_immI(vecX dst, vecX src, immI shift) %{
// Chars vector logical right shift
instruct vsrl4S_immI(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (URShiftVS src shift));
match(Set dst (URShiftVS src (RShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@ -10811,7 +10811,7 @@ instruct vsrl4S_immI(vecD dst, vecD src, immI shift) %{
instruct vsrl8S_immI(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (URShiftVS src shift));
match(Set dst (URShiftVS src (RShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@ -10828,7 +10828,7 @@ instruct vsrl8S_immI(vecX dst, vecX src, immI shift) %{
// Integers vector logical right shift
instruct vsrl2I_immI(vecD dst, vecD src, immI shift) %{
predicate(n->as_Vector()->length() == 2 && VM_Version::has_simd());
match(Set dst (URShiftVI src shift));
match(Set dst (URShiftVI src (RShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@ -10844,7 +10844,7 @@ instruct vsrl2I_immI(vecD dst, vecD src, immI shift) %{
instruct vsrl4I_immI(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 4 && VM_Version::has_simd());
match(Set dst (URShiftVI src shift));
match(Set dst (URShiftVI src (RShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{
@ -10861,7 +10861,7 @@ instruct vsrl4I_immI(vecX dst, vecX src, immI shift) %{
// Longs vector logical right shift
instruct vsrl2L_immI(vecX dst, vecX src, immI shift) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (URShiftVL src shift));
match(Set dst (URShiftVL src (RShiftCntV shift)));
size(4);
ins_cost(DEFAULT_COST); // FIXME
format %{

12
src/hotspot/cpu/x86/x86.ad

@ -5254,18 +5254,6 @@ instruct vshiftcnt(vec dst, rRegI cnt) %{
ins_pipe( pipe_slow );
%}
instruct vshiftcntimm(vec dst, immI8 cnt, rRegI tmp) %{
match(Set dst cnt);
effect(TEMP tmp);
format %{ "movl $tmp,$cnt\t"
"movdl $dst,$tmp\t! load shift count" %}
ins_encode %{
__ movl($tmp$$Register, $cnt$$constant);
__ movdl($dst$$XMMRegister, $tmp$$Register);
%}
ins_pipe( pipe_slow );
%}
// Byte vector shift
instruct vshiftB(vec dst, vec src, vec shift, vec tmp, rRegI scratch) %{
predicate(n->as_Vector()->length() <= 8);

37
src/hotspot/share/opto/matcher.cpp

@ -2010,6 +2010,15 @@ bool Matcher::is_bmi_pattern(Node *n, Node *m) {
}
#endif // X86
bool Matcher::is_vshift_con_pattern(Node *n, Node *m) {
if (n != NULL && m != NULL) {
return VectorNode::is_vector_shift(n) &&
VectorNode::is_vector_shift_count(m) && m->in(1)->is_Con();
}
return false;
}
bool Matcher::clone_base_plus_offset_address(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
Node *off = m->in(AddPNode::Offset);
if (off->is_Con()) {
@ -2090,6 +2099,10 @@ void Matcher::find_shared( Node *n ) {
continue;
}
#endif
if (is_vshift_con_pattern(n, m)) {
mstack.push(m, Visit);
continue;
}
// Clone addressing expressions as they are "free" in memory access instructions
if (mem_op && i == mem_addr_idx && mop == Op_AddP &&
@ -2525,22 +2538,16 @@ void Matcher::do_postselect_cleanup() {
//----------------------------------------------------------------------
// Convert (leg)Vec to (leg)Vec[SDXYZ].
MachOper* Matcher::specialize_vector_operand_helper(MachNode* m, uint opnd_idx, const Type* t) {
MachOper* Matcher::specialize_vector_operand_helper(MachNode* m, uint opnd_idx, const TypeVect* vt) {
MachOper* original_opnd = m->_opnds[opnd_idx];
uint ideal_reg = t->ideal_reg();
uint ideal_reg = vt->ideal_reg();
// Handle special cases.
if (t->isa_vect()) {
// LShiftCntV/RShiftCntV report wide vector type, but Matcher::vector_shift_count_ideal_reg() as ideal register (see vectornode.hpp).
// Look for shift count use sites as well (at vector shift nodes).
int opc = m->ideal_Opcode();
if ((VectorNode::is_shift_count(opc) && opnd_idx == 0) || // DEF operand of LShiftCntV/RShiftCntV
(VectorNode::is_vector_shift(opc) && opnd_idx == 2)) { // shift operand of a vector shift node
ideal_reg = Matcher::vector_shift_count_ideal_reg(t->is_vect()->length_in_bytes());
}
} else {
// Chain instructions which convert scalar to vector (e.g., vshiftcntimm on x86) don't have vector type.
int size_in_bytes = 4 * type2size[t->basic_type()];
ideal_reg = Matcher::vector_ideal_reg(size_in_bytes);
// LShiftCntV/RShiftCntV report wide vector type, but Matcher::vector_shift_count_ideal_reg() as ideal register (see vectornode.hpp).
// Look for shift count use sites as well (at vector shift nodes).
int opc = m->ideal_Opcode();
if ((VectorNode::is_vector_shift_count(opc) && opnd_idx == 0) || // DEF operand of LShiftCntV/RShiftCntV
(VectorNode::is_vector_shift(opc) && opnd_idx == 2)) { // shift operand of a vector shift node
ideal_reg = Matcher::vector_shift_count_ideal_reg(vt->length_in_bytes());
}
return Matcher::specialize_generic_vector_operand(original_opnd, ideal_reg, false);
}
@ -2575,7 +2582,7 @@ MachOper* Matcher::specialize_vector_operand(MachNode* m, uint opnd_idx) {
}
}
}
return specialize_vector_operand_helper(m, opnd_idx, def->bottom_type());
return specialize_vector_operand_helper(m, opnd_idx, def->bottom_type()->is_vect());
}
void Matcher::specialize_mach_node(MachNode* m) {

4
src/hotspot/share/opto/matcher.hpp

@ -125,6 +125,8 @@ private:
bool is_bmi_pattern(Node *n, Node *m);
#endif
bool is_vshift_con_pattern(Node *n, Node *m);
// Debug and profile information for nodes in old space:
GrowableArray<Node_Notes*>* _old_node_note_array;
@ -517,7 +519,7 @@ public:
void specialize_mach_node(MachNode* m);
void specialize_temp_node(MachTempNode* tmp, MachNode* use, uint idx);
MachOper* specialize_vector_operand(MachNode* m, uint opnd_idx);
MachOper* specialize_vector_operand_helper(MachNode* m, uint opnd_idx, const Type* t);
MachOper* specialize_vector_operand_helper(MachNode* m, uint opnd_idx, const TypeVect* vt);
static MachOper* specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp);

10
src/hotspot/share/opto/superword.cpp

@ -2706,13 +2706,11 @@ Node* SuperWord::vector_opd(Node_List* p, int opd_idx) {
NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("Should be int type only");})
return NULL;
}
// Move non constant shift count into vector register.
cnt = VectorNode::shift_count(p0, cnt, vlen, velt_basic_type(p0));
}
if (cnt != opd) {
_igvn.register_new_node_with_optimizer(cnt);
_phase->set_ctrl(cnt, _phase->get_ctrl(opd));
}
// Move shift count into vector register.
cnt = VectorNode::shift_count(p0, cnt, vlen, velt_basic_type(p0));
_igvn.register_new_node_with_optimizer(cnt);
_phase->set_ctrl(cnt, _phase->get_ctrl(opd));
return cnt;
}
assert(!opd->is_StoreVector(), "such vector is not expected here");

4
src/hotspot/share/opto/vectornode.cpp

@ -467,7 +467,7 @@ VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t) {
}
VectorNode* VectorNode::shift_count(Node* shift, Node* cnt, uint vlen, BasicType bt) {
assert(VectorNode::is_shift(shift) && !cnt->is_Con(), "only variable shift count");
assert(VectorNode::is_shift(shift), "sanity");
// Match shift count type with shift vector type.
const TypeVect* vt = TypeVect::make(bt, vlen);
switch (shift->Opcode()) {
@ -506,7 +506,7 @@ bool VectorNode::is_vector_shift(int opc) {
}
}
bool VectorNode::is_shift_count(int opc) {
bool VectorNode::is_vector_shift_count(int opc) {
assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode");
switch (opc) {
case Op_RShiftCntV:

9
src/hotspot/share/opto/vectornode.hpp

@ -76,7 +76,14 @@ class VectorNode : public TypeNode {
static void vector_operands(Node* n, uint* start, uint* end);
static bool is_vector_shift(int opc);
static bool is_shift_count(int opc);
static bool is_vector_shift_count(int opc);
static bool is_vector_shift(Node* n) {
return is_vector_shift(n->Opcode());
}
static bool is_vector_shift_count(Node* n) {
return is_vector_shift_count(n->Opcode());
}
};
//===========================Vector=ALU=Operations=============================