stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8309893: Integrate ReplicateB/S/I/L/F/D nodes to Replicate node

Browse Source 
Reviewed-by: jbhateja, epeter, fgao
This commit is contained in:
Eric Liu 2023-11-13 02:19:52 +00:00
parent caf71810f8
commit 50f41d6370
20 changed files with 554 additions and 785 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/hotspot/cpu/aarch64/aarch64.ad
View File

@ -2587,7 +2587,7 @@ bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
return false;
}
if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
return false;
}

274
src/hotspot/cpu/aarch64/aarch64_vector.ad
View File

@ -85,6 +85,7 @@ source_hpp %{
bool assert_not_var_shift(const Node* n);
Assembler::SIMD_Arrangement get_arrangement(const Node* n);
Assembler::SIMD_RegVariant get_reg_variant(const Node* n);
%}
source %{
@ -327,6 +328,11 @@ source %{
return Assembler::esize2arrangement((uint)type2aelembytes(bt),
/* isQ */ length_in_bytes == 16);
}
Assembler::SIMD_RegVariant get_reg_variant(const Node* n) {
BasicType bt = Matcher::vector_element_basic_type(n);
return Assembler::elemType_to_regVariant(bt);
}
%}
@ -666,7 +672,7 @@ instruct vaddD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
instruct vaddImmB(vReg dst_src, immBAddSubV con) %{
predicate(UseSVE > 0);
match(Set dst_src (AddVB dst_src (ReplicateB con)));
match(Set dst_src (AddVB dst_src (Replicate con)));
format %{ "vaddImmB $dst_src, $dst_src, $con" %}
ins_encode %{
int val = (int)$con$$constant;
@ -681,7 +687,7 @@ instruct vaddImmB(vReg dst_src, immBAddSubV con) %{
instruct vaddImmS(vReg dst_src, immIAddSubV con) %{
predicate(UseSVE > 0);
match(Set dst_src (AddVS dst_src (ReplicateS con)));
match(Set dst_src (AddVS dst_src (Replicate con)));
format %{ "vaddImmS $dst_src, $dst_src, $con" %}
ins_encode %{
int val = (int)$con$$constant;
@ -696,7 +702,7 @@ instruct vaddImmS(vReg dst_src, immIAddSubV con) %{
instruct vaddImmI(vReg dst_src, immIAddSubV con) %{
predicate(UseSVE > 0);
match(Set dst_src (AddVI dst_src (ReplicateI con)));
match(Set dst_src (AddVI dst_src (Replicate con)));
format %{ "vaddImmI $dst_src, $dst_src, $con" %}
ins_encode %{
int val = (int)$con$$constant;
@ -711,7 +717,7 @@ instruct vaddImmI(vReg dst_src, immIAddSubV con) %{
instruct vaddImmL(vReg dst_src, immLAddSubV con) %{
predicate(UseSVE > 0);
match(Set dst_src (AddVL dst_src (ReplicateL con)));
match(Set dst_src (AddVL dst_src (Replicate con)));
format %{ "vaddImmL $dst_src, $dst_src, $con" %}
ins_encode %{
int val = (int)$con$$constant;
@ -1190,8 +1196,8 @@ instruct vand_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
// vector and reg imm (unpredicated)
instruct vandImmB(vReg dst_src, immBLog con) %{
predicate(UseSVE > 0);
match(Set dst_src (AndV dst_src (ReplicateB con)));
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst_src (AndV dst_src (Replicate con)));
format %{ "vandImmB $dst_src, $dst_src, $con" %}
ins_encode %{
__ sve_and($dst_src$$FloatRegister, __ B, (uint64_t)($con$$constant));
@ -1200,8 +1206,8 @@ instruct vandImmB(vReg dst_src, immBLog con) %{
%}
instruct vandImmS(vReg dst_src, immSLog con) %{
predicate(UseSVE > 0);
match(Set dst_src (AndV dst_src (ReplicateS con)));
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst_src (AndV dst_src (Replicate con)));
format %{ "vandImmS $dst_src, $dst_src, $con" %}
ins_encode %{
__ sve_and($dst_src$$FloatRegister, __ H, (uint64_t)($con$$constant));
@ -1210,8 +1216,8 @@ instruct vandImmS(vReg dst_src, immSLog con) %{
%}
instruct vandImmI(vReg dst_src, immILog con) %{
predicate(UseSVE > 0);
match(Set dst_src (AndV dst_src (ReplicateI con)));
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst_src (AndV dst_src (Replicate con)));
format %{ "vandImmI $dst_src, $dst_src, $con" %}
ins_encode %{
__ sve_and($dst_src$$FloatRegister, __ S, (uint64_t)($con$$constant));
@ -1220,8 +1226,8 @@ instruct vandImmI(vReg dst_src, immILog con) %{
%}
instruct vandImmL(vReg dst_src, immLLog con) %{
predicate(UseSVE > 0);
match(Set dst_src (AndV dst_src (ReplicateL con)));
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst_src (AndV dst_src (Replicate con)));
format %{ "vandImmL $dst_src, $dst_src, $con" %}
ins_encode %{
__ sve_and($dst_src$$FloatRegister, __ D, (uint64_t)($con$$constant));
@ -1266,8 +1272,8 @@ instruct vor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
// vector or reg imm (unpredicated)
instruct vorImmB(vReg dst_src, immBLog con) %{
predicate(UseSVE > 0);
match(Set dst_src (OrV dst_src (ReplicateB con)));
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst_src (OrV dst_src (Replicate con)));
format %{ "vorImmB $dst_src, $dst_src, $con" %}
ins_encode %{
__ sve_orr($dst_src$$FloatRegister, __ B, (uint64_t)($con$$constant));
@ -1276,8 +1282,8 @@ instruct vorImmB(vReg dst_src, immBLog con) %{
%}
instruct vorImmS(vReg dst_src, immSLog con) %{
predicate(UseSVE > 0);
match(Set dst_src (OrV dst_src (ReplicateS con)));
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst_src (OrV dst_src (Replicate con)));
format %{ "vorImmS $dst_src, $dst_src, $con" %}
ins_encode %{
__ sve_orr($dst_src$$FloatRegister, __ H, (uint64_t)($con$$constant));
@ -1286,8 +1292,8 @@ instruct vorImmS(vReg dst_src, immSLog con) %{
%}
instruct vorImmI(vReg dst_src, immILog con) %{
predicate(UseSVE > 0);
match(Set dst_src (OrV dst_src (ReplicateI con)));
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst_src (OrV dst_src (Replicate con)));
format %{ "vorImmI $dst_src, $dst_src, $con" %}
ins_encode %{
__ sve_orr($dst_src$$FloatRegister, __ S, (uint64_t)($con$$constant));
@ -1296,8 +1302,8 @@ instruct vorImmI(vReg dst_src, immILog con) %{
%}
instruct vorImmL(vReg dst_src, immLLog con) %{
predicate(UseSVE > 0);
match(Set dst_src (OrV dst_src (ReplicateL con)));
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst_src (OrV dst_src (Replicate con)));
format %{ "vorImmL $dst_src, $dst_src, $con" %}
ins_encode %{
__ sve_orr($dst_src$$FloatRegister, __ D, (uint64_t)($con$$constant));
@ -1342,8 +1348,8 @@ instruct vxor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
// vector xor reg imm (unpredicated)
instruct vxorImmB(vReg dst_src, immBLog con) %{
predicate(UseSVE > 0);
match(Set dst_src (XorV dst_src (ReplicateB con)));
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst_src (XorV dst_src (Replicate con)));
format %{ "vxorImmB $dst_src, $dst_src, $con" %}
ins_encode %{
__ sve_eor($dst_src$$FloatRegister, __ B, (uint64_t)($con$$constant));
@ -1352,8 +1358,8 @@ instruct vxorImmB(vReg dst_src, immBLog con) %{
%}
instruct vxorImmS(vReg dst_src, immSLog con) %{
predicate(UseSVE > 0);
match(Set dst_src (XorV dst_src (ReplicateS con)));
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst_src (XorV dst_src (Replicate con)));
format %{ "vxorImmS $dst_src, $dst_src, $con" %}
ins_encode %{
__ sve_eor($dst_src$$FloatRegister, __ H, (uint64_t)($con$$constant));
@ -1362,8 +1368,8 @@ instruct vxorImmS(vReg dst_src, immSLog con) %{
%}
instruct vxorImmI(vReg dst_src, immILog con) %{
predicate(UseSVE > 0);
match(Set dst_src (XorV dst_src (ReplicateI con)));
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst_src (XorV dst_src (Replicate con)));
format %{ "vxorImmI $dst_src, $dst_src, $con" %}
ins_encode %{
__ sve_eor($dst_src$$FloatRegister, __ S, (uint64_t)($con$$constant));
@ -1372,8 +1378,8 @@ instruct vxorImmI(vReg dst_src, immILog con) %{
%}
instruct vxorImmL(vReg dst_src, immLLog con) %{
predicate(UseSVE > 0);
match(Set dst_src (XorV dst_src (ReplicateL con)));
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst_src (XorV dst_src (Replicate con)));
format %{ "vxorImmL $dst_src, $dst_src, $con" %}
ins_encode %{
__ sve_eor($dst_src$$FloatRegister, __ D, (uint64_t)($con$$constant));
@ -1410,9 +1416,7 @@ instruct veor3_sve(vReg dst_src1, vReg src2, vReg src3) %{
// vector not
instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
match(Set dst (XorV src (ReplicateB m1)));
match(Set dst (XorV src (ReplicateS m1)));
match(Set dst (XorV src (ReplicateI m1)));
match(Set dst (XorV src (Replicate m1)));
format %{ "vnotI $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
@ -1428,7 +1432,7 @@ instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
%}
instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
match(Set dst (XorV src (ReplicateL m1)));
match(Set dst (XorV src (Replicate m1)));
format %{ "vnotL $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
@ -1447,13 +1451,10 @@ instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
instruct vnotI_masked(vReg dst_src, immI_M1 m1, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src (XorV (Binary dst_src (ReplicateB m1)) pg));
match(Set dst_src (XorV (Binary dst_src (ReplicateS m1)) pg));
match(Set dst_src (XorV (Binary dst_src (ReplicateI m1)) pg));
match(Set dst_src (XorV (Binary dst_src (Replicate m1)) pg));
format %{ "vnotI_masked $dst_src, $pg, $dst_src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_not($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
__ sve_not($dst_src$$FloatRegister, get_reg_variant(this),
$pg$$PRegister, $dst_src$$FloatRegister);
%}
ins_pipe(pipe_slow);
@ -1461,11 +1462,10 @@ instruct vnotI_masked(vReg dst_src, immI_M1 m1, pRegGov pg) %{
instruct vnotL_masked(vReg dst_src, immL_M1 m1, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src (XorV (Binary dst_src (ReplicateL m1)) pg));
match(Set dst_src (XorV (Binary dst_src (Replicate m1)) pg));
format %{ "vnotL_masked $dst_src, $pg, $dst_src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_not($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
__ sve_not($dst_src$$FloatRegister, get_reg_variant(this),
$pg$$PRegister, $dst_src$$FloatRegister);
%}
ins_pipe(pipe_slow);
@ -1476,9 +1476,7 @@ instruct vnotL_masked(vReg dst_src, immL_M1 m1, pRegGov pg) %{
// vector and_not
instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));
match(Set dst (AndV src1 (XorV src2 (Replicate m1))));
format %{ "vand_notI $dst, $src1, $src2" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
@ -1494,7 +1492,7 @@ instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
%}
instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));
match(Set dst (AndV src1 (XorV src2 (Replicate m1))));
format %{ "vand_notL $dst, $src1, $src2" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
@ -1513,13 +1511,10 @@ instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
instruct vand_notI_masked(vReg dst_src1, vReg src2, immI_M1 m1, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateB m1))) pg));
match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateS m1))) pg));
match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateI m1))) pg));
match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (Replicate m1))) pg));
format %{ "vand_notI_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_bic($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
__ sve_bic($dst_src1$$FloatRegister, get_reg_variant(this),
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
@ -1527,11 +1522,10 @@ instruct vand_notI_masked(vReg dst_src1, vReg src2, immI_M1 m1, pRegGov pg) %{
instruct vand_notL_masked(vReg dst_src1, vReg src2, immL_M1 m1, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateL m1))) pg));
match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (Replicate m1))) pg));
format %{ "vand_notL_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_bic($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
__ sve_bic($dst_src1$$FloatRegister, get_reg_variant(this),
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
@ -4190,38 +4184,8 @@ instruct vcvtFtoHF_sve(vReg dst, vReg src, vReg tmp) %{
// replicate from reg
instruct replicateB(vReg dst, iRegIorL2I src) %{
match(Set dst (ReplicateB src));
format %{ "replicateB $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$Register);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ B, $src$$Register);
}
%}
ins_pipe(pipe_slow);
%}
instruct replicateS(vReg dst, iRegIorL2I src) %{
match(Set dst (ReplicateS src));
format %{ "replicateS $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$Register);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ H, $src$$Register);
}
%}
ins_pipe(pipe_slow);
%}
instruct replicateI(vReg dst, iRegIorL2I src) %{
match(Set dst (ReplicateI src));
match(Set dst (Replicate src));
format %{ "replicateI $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
@ -4229,14 +4193,14 @@ instruct replicateI(vReg dst, iRegIorL2I src) %{
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$Register);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ S, $src$$Register);
__ sve_dup($dst$$FloatRegister, get_reg_variant(this), $src$$Register);
}
%}
ins_pipe(pipe_slow);
%}
instruct replicateL(vReg dst, iRegL src) %{
match(Set dst (ReplicateL src));
match(Set dst (Replicate src));
format %{ "replicateL $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
@ -4244,14 +4208,14 @@ instruct replicateL(vReg dst, iRegL src) %{
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$Register);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ D, $src$$Register);
__ sve_dup($dst$$FloatRegister, get_reg_variant(this), $src$$Register);
}
%}
ins_pipe(pipe_slow);
%}
instruct replicateF(vReg dst, vRegF src) %{
match(Set dst (ReplicateF src));
match(Set dst (Replicate src));
format %{ "replicateF $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
@ -4266,7 +4230,7 @@ instruct replicateF(vReg dst, vRegF src) %{
%}
instruct replicateD(vReg dst, vRegD src) %{
match(Set dst (ReplicateD src));
match(Set dst (Replicate src));
format %{ "replicateD $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
@ -4283,10 +4247,9 @@ instruct replicateD(vReg dst, vRegD src) %{
// replicate from imm
instruct replicateI_imm_le128b(vReg dst, immI con) %{
predicate(Matcher::vector_length_in_bytes(n) <= 16);
match(Set dst (ReplicateB con));
match(Set dst (ReplicateS con));
match(Set dst (ReplicateI con));
predicate(Matcher::vector_length_in_bytes(n) <= 16 &&
Matcher::is_non_long_integral_vector(n));
match(Set dst (Replicate con));
format %{ "replicateI_imm_le128b $dst, $con\t# vector <= 128 bits" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
@ -4304,8 +4267,9 @@ instruct replicateI_imm_le128b(vReg dst, immI con) %{
%}
instruct replicateB_imm8_gt128b(vReg dst, immI8 con) %{
predicate(Matcher::vector_length_in_bytes(n) > 16);
match(Set dst (ReplicateB con));
predicate(Matcher::vector_length_in_bytes(n) > 16 &&
Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (Replicate con));
format %{ "replicateB_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
@ -4314,31 +4278,22 @@ instruct replicateB_imm8_gt128b(vReg dst, immI8 con) %{
ins_pipe(pipe_slow);
%}
instruct replicateS_imm8_gt128b(vReg dst, immI8_shift8 con) %{
predicate(Matcher::vector_length_in_bytes(n) > 16);
match(Set dst (ReplicateS con));
format %{ "replicateS_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ H, (int)($con$$constant));
%}
ins_pipe(pipe_slow);
%}
instruct replicateI_imm8_gt128b(vReg dst, immI8_shift8 con) %{
predicate(Matcher::vector_length_in_bytes(n) > 16);
match(Set dst (ReplicateI con));
predicate(Matcher::vector_length_in_bytes(n) > 16 &&
(Matcher::vector_element_basic_type(n) == T_SHORT ||
Matcher::vector_element_basic_type(n) == T_INT));
match(Set dst (Replicate con));
format %{ "replicateI_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ S, (int)($con$$constant));
__ sve_dup($dst$$FloatRegister, get_reg_variant(this), (int)($con$$constant));
%}
ins_pipe(pipe_slow);
%}
instruct replicateL_imm_128b(vReg dst, immL con) %{
predicate(Matcher::vector_length_in_bytes(n) == 16);
match(Set dst (ReplicateL con));
match(Set dst (Replicate con));
format %{ "replicateL_imm_128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
__ mov($dst$$FloatRegister, __ T2D, (uint64_t)($con$$constant));
@ -4348,7 +4303,7 @@ instruct replicateL_imm_128b(vReg dst, immL con) %{
instruct replicateL_imm8_gt128b(vReg dst, immL8_shift8 con) %{
predicate(Matcher::vector_length_in_bytes(n) > 16);
match(Set dst (ReplicateL con));
match(Set dst (Replicate con));
format %{ "replicateL_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
@ -5169,15 +5124,13 @@ instruct vmaskcmp_neon(vReg dst, vReg src1, vReg src2, immI cond) %{
instruct vmaskcmp_zeroI_neon(vReg dst, vReg src, immI0 zero, immI_cmp_cond cond) %{
predicate(UseSVE == 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateB zero)) cond));
match(Set dst (VectorMaskCmp (Binary src (ReplicateS zero)) cond));
match(Set dst (VectorMaskCmp (Binary src (ReplicateI zero)) cond));
match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
format %{ "vmaskcmp_zeroI_neon $dst, $src, #0, $cond" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ neon_compare_zero($dst$$FloatRegister, bt, $src$$FloatRegister,
__ neon_compare_zero($dst$$FloatRegister, Matcher::vector_element_basic_type(this),
$src$$FloatRegister,
condition, /* isQ */ length_in_bytes == 16);
%}
ins_pipe(pipe_slow);
@ -5185,12 +5138,13 @@ instruct vmaskcmp_zeroI_neon(vReg dst, vReg src, immI0 zero, immI_cmp_cond cond)
instruct vmaskcmp_zeroL_neon(vReg dst, vReg src, immL0 zero, immI_cmp_cond cond) %{
predicate(UseSVE == 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateL zero)) cond));
match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
format %{ "vmaskcmp_zeroL_neon $dst, $src, #0, $cond" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ neon_compare_zero($dst$$FloatRegister, T_LONG, $src$$FloatRegister,
__ neon_compare_zero($dst$$FloatRegister, T_LONG,
$src$$FloatRegister,
condition, /* isQ */ length_in_bytes == 16);
%}
ins_pipe(pipe_slow);
@ -5198,12 +5152,13 @@ instruct vmaskcmp_zeroL_neon(vReg dst, vReg src, immL0 zero, immI_cmp_cond cond)
instruct vmaskcmp_zeroF_neon(vReg dst, vReg src, immF0 zero, immI_cmp_cond cond) %{
predicate(UseSVE == 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateF zero)) cond));
match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
format %{ "vmaskcmp_zeroF_neon $dst, $src, #0, $cond" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ neon_compare_zero($dst$$FloatRegister, T_FLOAT, $src$$FloatRegister,
__ neon_compare_zero($dst$$FloatRegister, T_FLOAT,
$src$$FloatRegister,
condition, /* isQ */ length_in_bytes == 16);
%}
ins_pipe(pipe_slow);
@ -5211,12 +5166,13 @@ instruct vmaskcmp_zeroF_neon(vReg dst, vReg src, immF0 zero, immI_cmp_cond cond)
instruct vmaskcmp_zeroD_neon(vReg dst, vReg src, immD0 zero, immI_cmp_cond cond) %{
predicate(UseSVE == 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateD zero)) cond));
match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
format %{ "vmaskcmp_zeroD_neon $dst, $src, #0, $cond" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ neon_compare_zero($dst$$FloatRegister, T_DOUBLE, $src$$FloatRegister,
__ neon_compare_zero($dst$$FloatRegister, T_DOUBLE,
$src$$FloatRegister,
condition, /* isQ */ length_in_bytes == 16);
%}
ins_pipe(pipe_slow);
@ -5238,114 +5194,62 @@ instruct vmaskcmp_sve(pReg dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %
ins_pipe(pipe_slow);
%}
instruct vmaskcmp_immB_sve(pReg dst, vReg src, immI5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateB imm)) cond));
effect(KILL cr);
format %{ "vmaskcmp_immB_sve $dst, $src, $imm, $cond\t# KILL cr" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_cmp(condition, $dst$$PRegister, __ B, ptrue, $src$$FloatRegister, (int)$imm$$constant);
%}
ins_pipe(pipe_slow);
%}
instruct vmaskcmpU_immB_sve(pReg dst, vReg src, immIU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateB imm)) cond));
effect(KILL cr);
format %{ "vmaskcmpU_immB_sve $dst, $src, $imm, $cond\t# KILL cr" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_cmp(condition, $dst$$PRegister, __ B, ptrue, $src$$FloatRegister, (int)$imm$$constant);
%}
ins_pipe(pipe_slow);
%}
instruct vmaskcmp_immS_sve(pReg dst, vReg src, immI5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateS imm)) cond));
effect(KILL cr);
format %{ "vmaskcmp_immS_sve $dst, $src, $imm, $cond\t# KILL cr" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_cmp(condition, $dst$$PRegister, __ H, ptrue, $src$$FloatRegister, (int)$imm$$constant);
%}
ins_pipe(pipe_slow);
%}
instruct vmaskcmpU_immS_sve(pReg dst, vReg src, immIU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateS imm)) cond));
effect(KILL cr);
format %{ "vmaskcmpU_immS_sve $dst, $src, $imm, $cond\t# KILL cr" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_cmp(condition, $dst$$PRegister, __ H, ptrue, $src$$FloatRegister, (int)$imm$$constant);
%}
ins_pipe(pipe_slow);
%}
instruct vmaskcmp_immI_sve(pReg dst, vReg src, immI5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateI imm)) cond));
match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
effect(KILL cr);
format %{ "vmaskcmp_immI_sve $dst, $src, $imm, $cond\t# KILL cr" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_cmp(condition, $dst$$PRegister, __ S, ptrue, $src$$FloatRegister, (int)$imm$$constant);
__ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
ptrue, $src$$FloatRegister, (int)$imm$$constant);
%}
ins_pipe(pipe_slow);
%}
instruct vmaskcmpU_immI_sve(pReg dst, vReg src, immIU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateI imm)) cond));
match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
effect(KILL cr);
format %{ "vmaskcmpU_immI_sve $dst, $src, $imm, $cond\t# KILL cr" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_cmp(condition, $dst$$PRegister, __ S, ptrue, $src$$FloatRegister, (int)$imm$$constant);
__ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
ptrue, $src$$FloatRegister, (int)$imm$$constant);
%}
ins_pipe(pipe_slow);
%}
instruct vmaskcmp_immL_sve(pReg dst, vReg src, immL5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateL imm)) cond));
match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
effect(KILL cr);
format %{ "vmaskcmp_immL_sve $dst, $src, $imm, $cond\t# KILL cr" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_cmp(condition, $dst$$PRegister, __ D, ptrue, $src$$FloatRegister, (int)$imm$$constant);
__ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
ptrue, $src$$FloatRegister, (int)$imm$$constant);
%}
ins_pipe(pipe_slow);
%}
instruct vmaskcmpU_immL_sve(pReg dst, vReg src, immLU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateL imm)) cond));
match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
effect(KILL cr);
format %{ "vmaskcmpU_immL_sve $dst, $src, $imm, $cond\t# KILL cr" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_cmp(condition, $dst$$PRegister, __ D, ptrue, $src$$FloatRegister, (int)$imm$$constant);
__ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
ptrue, $src$$FloatRegister, (int)$imm$$constant);
%}
ins_pipe(pipe_slow);
%}

279
src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
View File

@ -75,6 +75,7 @@ source_hpp %{
bool assert_not_var_shift(const Node* n);
Assembler::SIMD_Arrangement get_arrangement(const Node* n);
Assembler::SIMD_RegVariant get_reg_variant(const Node* n);
%}
source %{
@ -317,6 +318,11 @@ source %{
return Assembler::esize2arrangement((uint)type2aelembytes(bt),
/* isQ */ length_in_bytes == 16);
}
Assembler::SIMD_RegVariant get_reg_variant(const Node* n) {
BasicType bt = Matcher::vector_element_basic_type(n);
return Assembler::elemType_to_regVariant(bt);
}
%}
@ -477,7 +483,7 @@ dnl VADD_IMM(type, imm_type, size)
define(`VADD_IMM', `
instruct vaddImm$1(vReg dst_src, $2 con) %{
predicate(UseSVE > 0);
match(Set dst_src (AddV$1 dst_src (Replicate$1 con)));
match(Set dst_src (AddV$1 dst_src (Replicate con)));
format %{ "vaddImm$1 $dst_src, $dst_src, $con" %}
ins_encode %{
int val = (int)$con$$constant;
@ -618,12 +624,27 @@ BINARY_OP_NEON_SVE_PAIRWISE(vdivD, DivVD, fdiv, sve_fdiv, D)
BINARY_OP_PREDICATE(vdivF, DivVF, sve_fdiv, S)
BINARY_OP_PREDICATE(vdivD, DivVD, sve_fdiv, D)
dnl
dnl BITWISE_OP($1, $2, $3, $4 )
dnl BITWISE_OP(rule_name, op_name, insn_neon, insn_sve)
define(`BITWISE_OP', `
instruct $1(vReg dst, vReg src1, vReg src2) %{
dnl BITWISE_OP_IMM($1, $2, $3, $4, $5, $6 )
dnl BITWISE_OP_IMM(rule_name, type, op_name, insn, size, basic_type)
define(`BITWISE_OP_IMM', `
instruct $1(vReg dst_src, imm$2Log con) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == $6);
match(Set dst_src ($3 dst_src (Replicate con)));
format %{ "$1 $dst_src, $dst_src, $con" %}
ins_encode %{
__ $4($dst_src$$FloatRegister, __ $5, (uint64_t)($con$$constant));
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl BITWISE_OPS($1, $2, $3, $4 )
dnl BITWISE_OPS(rule_name, op_name, insn_neon, insn_sve)
define(`BITWISE_OPS',
`// vector $1
instruct v$1(vReg dst, vReg src1, vReg src2) %{
match(Set dst ($2 src1 src2));
format %{ "$1 $dst, $src1, $src2" %}
format %{ "v$1 $dst, $src1, $src2" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
@ -635,78 +656,39 @@ instruct $1(vReg dst, vReg src1, vReg src2) %{
}
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl BITWISE_OP_PREDICATE($1, $2, $3 )
dnl BITWISE_OP_PREDICATE(rule_name, op_name, insn)
define(`BITWISE_OP_PREDICATE', `
instruct $1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
%}
// vector $1 - predicated
instruct v$1_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
predicate(UseSVE > 0);
match(Set dst_src1 ($2 (Binary dst_src1 src2) pg));
format %{ "$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
format %{ "v$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ $3($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
__ $4($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl BITWISE_OP_IMM($1, $2, $3, $4, $5 )
dnl BITWISE_OP_IMM(rule_name, type, op_name, insn, size)
define(`BITWISE_OP_IMM', `
instruct $1(vReg dst_src, imm$2Log con) %{
predicate(UseSVE > 0);
match(Set dst_src ($3 dst_src (Replicate$2 con)));
format %{ "$1 $dst_src, $dst_src, $con" %}
ins_encode %{
__ $4($dst_src$$FloatRegister, __ $5, (uint64_t)($con$$constant));
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
%}
// vector $1 reg imm (unpredicated)
BITWISE_OP_IMM(v$1ImmB, B, $2, $4, B, T_BYTE)
BITWISE_OP_IMM(v$1ImmS, S, $2, $4, H, T_SHORT)
BITWISE_OP_IMM(v$1ImmI, I, $2, $4, S, T_INT)
BITWISE_OP_IMM(v$1ImmL, L, $2, $4, D, T_LONG)')dnl
// ------------------------------ Vector and -----------------------------------
// vector and
BITWISE_OP(vand, AndV, andr, sve_and)
// vector and - predicated
BITWISE_OP_PREDICATE(vand, AndV, sve_and)
// vector and reg imm (unpredicated)
BITWISE_OP_IMM(vandImmB, B, AndV, sve_and, B)
BITWISE_OP_IMM(vandImmS, S, AndV, sve_and, H)
BITWISE_OP_IMM(vandImmI, I, AndV, sve_and, S)
BITWISE_OP_IMM(vandImmL, L, AndV, sve_and, D)
BITWISE_OPS(and, AndV, andr, sve_and)
// ------------------------------ Vector or ------------------------------------
// vector or
BITWISE_OP(vor, OrV, orr, sve_orr)
// vector or - predicated
BITWISE_OP_PREDICATE(vor, OrV, sve_orr)
// vector or reg imm (unpredicated)
BITWISE_OP_IMM(vorImmB, B, OrV, sve_orr, B)
BITWISE_OP_IMM(vorImmS, S, OrV, sve_orr, H)
BITWISE_OP_IMM(vorImmI, I, OrV, sve_orr, S)
BITWISE_OP_IMM(vorImmL, L, OrV, sve_orr, D)
BITWISE_OPS(or, OrV, orr, sve_orr)
// ------------------------------ Vector xor -----------------------------------
// vector xor
BITWISE_OP(vxor, XorV, eor, sve_eor)
// vector xor - predicated
BITWISE_OP_PREDICATE(vxor, XorV, sve_eor)
// vector xor reg imm (unpredicated)
BITWISE_OP_IMM(vxorImmB, B, XorV, sve_eor, B)
BITWISE_OP_IMM(vxorImmS, S, XorV, sve_eor, H)
BITWISE_OP_IMM(vxorImmI, I, XorV, sve_eor, S)
BITWISE_OP_IMM(vxorImmL, L, XorV, sve_eor, D)
BITWISE_OPS(xor, XorV, eor, sve_eor)
// vector eor3 (unpredicated)
@ -734,18 +716,12 @@ instruct veor3_sve(vReg dst_src1, vReg src2, vReg src3) %{
// ------------------------------ Vector not -----------------------------------
dnl
define(`MATCH_RULE', `ifelse($1, I,
`match(Set dst (XorV src (ReplicateB m1)));
match(Set dst (XorV src (ReplicateS m1)));
match(Set dst (XorV src (ReplicateI m1)));',
`match(Set dst (XorV src (ReplicateL m1)));')')dnl
dnl
dnl VECTOR_NOT($1 )
dnl VECTOR_NOT(type)
define(`VECTOR_NOT', `
instruct vnot$1`'(vReg dst, vReg src, imm$1_M1 m1) %{
MATCH_RULE($1)
match(Set dst (XorV src (Replicate m1)));
format %{ "vnot$1 $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
@ -763,48 +739,34 @@ dnl
// vector not
VECTOR_NOT(I)
VECTOR_NOT(L)
undefine(MATCH_RULE)
dnl
define(`MATCH_RULE', `ifelse($1, I,
`match(Set dst_src (XorV (Binary dst_src (ReplicateB m1)) pg));
match(Set dst_src (XorV (Binary dst_src (ReplicateS m1)) pg));
match(Set dst_src (XorV (Binary dst_src (ReplicateI m1)) pg));',
`match(Set dst_src (XorV (Binary dst_src (ReplicateL m1)) pg));')')dnl
dnl
dnl VECTOR_NOT_PREDICATE($1 )
dnl VECTOR_NOT_PREDICATE(type)
define(`VECTOR_NOT_PREDICATE', `
instruct vnot$1_masked`'(vReg dst_src, imm$1_M1 m1, pRegGov pg) %{
predicate(UseSVE > 0);
MATCH_RULE($1)
match(Set dst_src (XorV (Binary dst_src (Replicate m1)) pg));
format %{ "vnot$1_masked $dst_src, $pg, $dst_src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_not($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
__ sve_not($dst_src$$FloatRegister, get_reg_variant(this),
$pg$$PRegister, $dst_src$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// vector not - predicated
VECTOR_NOT_PREDICATE(I)
VECTOR_NOT_PREDICATE(L)
undefine(MATCH_RULE)
dnl
// ------------------------------ Vector and_not -------------------------------
dnl
define(`MATCH_RULE', `ifelse($1, I,
`match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));',
`match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));')')dnl
// ------------------------------ Vector and_not -------------------------------
dnl
dnl VECTOR_AND_NOT($1 )
dnl VECTOR_AND_NOT(type)
define(`VECTOR_AND_NOT', `
instruct vand_not$1`'(vReg dst, vReg src1, vReg src2, imm$1_M1 m1) %{
MATCH_RULE($1)
match(Set dst (AndV src1 (XorV src2 (Replicate m1))));
format %{ "vand_not$1 $dst, $src1, $src2" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
@ -819,36 +781,30 @@ instruct vand_not$1`'(vReg dst, vReg src1, vReg src2, imm$1_M1 m1) %{
ins_pipe(pipe_slow);
%}')dnl
dnl
// vector and_not
VECTOR_AND_NOT(I)
VECTOR_AND_NOT(L)
undefine(MATCH_RULE)
dnl
define(`MATCH_RULE', `ifelse($1, I,
`match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateB m1))) pg));
match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateS m1))) pg));
match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateI m1))) pg));',
`match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateL m1))) pg));')')dnl
dnl
dnl VECTOR_AND_NOT_PREDICATE($1 )
dnl VECTOR_AND_NOT_PREDICATE(type)
define(`VECTOR_AND_NOT_PREDICATE', `
instruct vand_not$1_masked`'(vReg dst_src1, vReg src2, imm$1_M1 m1, pRegGov pg) %{
predicate(UseSVE > 0);
MATCH_RULE($1)
match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (Replicate m1))) pg));
format %{ "vand_not$1_masked $dst_src1, $pg, $dst_src1, $src2" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_bic($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
__ sve_bic($dst_src1$$FloatRegister, get_reg_variant(this),
$pg$$PRegister, $src2$$FloatRegister);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// vector and_not - predicated
VECTOR_AND_NOT_PREDICATE(I)
VECTOR_AND_NOT_PREDICATE(L)
undefine(MATCH_RULE)
dnl
dnl UNARY_OP($1, $2, $3, $4, $5 )
dnl UNARY_OP(rule_name, op_name, insn_neon, insn_sve, size)
@ -2764,7 +2720,7 @@ dnl REPLICATE_INT($1, $2, $3 )
dnl REPLICATE_INT(type, arg_type, size)
define(`REPLICATE_INT', `
instruct replicate$1(vReg dst, $2 src) %{
match(Set dst (Replicate$1 src));
match(Set dst (Replicate src));
format %{ "replicate$1 $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
@ -2772,7 +2728,7 @@ instruct replicate$1(vReg dst, $2 src) %{
__ dup($dst$$FloatRegister, get_arrangement(this), $src$$Register);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ $3, $src$$Register);
__ sve_dup($dst$$FloatRegister, get_reg_variant(this), $src$$Register);
}
%}
ins_pipe(pipe_slow);
@ -2782,7 +2738,7 @@ dnl REPLICATE_FP($1, $2 )
dnl REPLICATE_FP(type, size)
define(`REPLICATE_FP', `
instruct replicate$1(vReg dst, vReg$1 src) %{
match(Set dst (Replicate$1 src));
match(Set dst (Replicate src));
format %{ "replicate$1 $dst, $src" %}
ins_encode %{
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
@ -2796,35 +2752,18 @@ instruct replicate$1(vReg dst, vReg$1 src) %{
ins_pipe(pipe_slow);
%}')dnl
dnl
dnl REPLICATE_IMM_SVE($1, $2, $3 )
dnl REPLICATE_IMM_SVE(type, arg_type, size)
define(`REPLICATE_IMM_SVE', `
instruct replicate$1_imm8_gt128b(vReg dst, $2 con) %{
predicate(Matcher::vector_length_in_bytes(n) > 16);
match(Set dst (Replicate$1 con));
format %{ "replicate$1_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ $3, (int)($con$$constant));
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
// replicate from reg
REPLICATE_INT(B, iRegIorL2I, B)
REPLICATE_INT(S, iRegIorL2I, H)
REPLICATE_INT(I, iRegIorL2I, S)
REPLICATE_INT(L, iRegL, D)
REPLICATE_FP(F, S)
REPLICATE_FP(D, D)
REPLICATE_INT(I, iRegIorL2I)
REPLICATE_INT(L, iRegL)
REPLICATE_FP(F, S, T_FLOAT )
REPLICATE_FP(D, D, T_DOUBLE)
// replicate from imm
instruct replicateI_imm_le128b(vReg dst, immI con) %{
predicate(Matcher::vector_length_in_bytes(n) <= 16);
match(Set dst (ReplicateB con));
match(Set dst (ReplicateS con));
match(Set dst (ReplicateI con));
predicate(Matcher::vector_length_in_bytes(n) <= 16 &&
Matcher::is_non_long_integral_vector(n));
match(Set dst (Replicate con));
format %{ "replicateI_imm_le128b $dst, $con\t# vector <= 128 bits" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
@ -2840,22 +2779,52 @@ instruct replicateI_imm_le128b(vReg dst, immI con) %{
%}
ins_pipe(pipe_slow);
%}
dnl
REPLICATE_IMM_SVE(B, immI8, B)
REPLICATE_IMM_SVE(S, immI8_shift8, H)
REPLICATE_IMM_SVE(I, immI8_shift8, S)
instruct replicateB_imm8_gt128b(vReg dst, immI8 con) %{
predicate(Matcher::vector_length_in_bytes(n) > 16 &&
Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (Replicate con));
format %{ "replicateB_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ B, (int)($con$$constant));
%}
ins_pipe(pipe_slow);
%}
instruct replicateI_imm8_gt128b(vReg dst, immI8_shift8 con) %{
predicate(Matcher::vector_length_in_bytes(n) > 16 &&
(Matcher::vector_element_basic_type(n) == T_SHORT ||
Matcher::vector_element_basic_type(n) == T_INT));
match(Set dst (Replicate con));
format %{ "replicateI_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, get_reg_variant(this), (int)($con$$constant));
%}
ins_pipe(pipe_slow);
%}
instruct replicateL_imm_128b(vReg dst, immL con) %{
predicate(Matcher::vector_length_in_bytes(n) == 16);
match(Set dst (ReplicateL con));
match(Set dst (Replicate con));
format %{ "replicateL_imm_128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
__ mov($dst$$FloatRegister, __ T2D, (uint64_t)($con$$constant));
%}
ins_pipe(pipe_slow);
%}
dnl
REPLICATE_IMM_SVE(L, immL8_shift8, D)
instruct replicateL_imm8_gt128b(vReg dst, immL8_shift8 con) %{
predicate(Matcher::vector_length_in_bytes(n) > 16);
match(Set dst (Replicate con));
format %{ "replicateL_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
ins_encode %{
assert(UseSVE > 0, "must be sve");
__ sve_dup($dst$$FloatRegister, __ D, (int)($con$$constant));
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector insert --------------------------------
@ -3561,39 +3530,24 @@ instruct vmaskcmp_neon(vReg dst, vReg src1, vReg src2, immI cond) %{
%}
ins_pipe(pipe_slow);
%}
instruct vmaskcmp_zeroI_neon(vReg dst, vReg src, immI0 zero, immI_cmp_cond cond) %{
predicate(UseSVE == 0);
match(Set dst (VectorMaskCmp (Binary src (ReplicateB zero)) cond));
match(Set dst (VectorMaskCmp (Binary src (ReplicateS zero)) cond));
match(Set dst (VectorMaskCmp (Binary src (ReplicateI zero)) cond));
format %{ "vmaskcmp_zeroI_neon $dst, $src, #0, $cond" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ neon_compare_zero($dst$$FloatRegister, bt, $src$$FloatRegister,
condition, /* isQ */ length_in_bytes == 16);
%}
ins_pipe(pipe_slow);
%}
dnl
dnl VMASKCMP_ZERO_NEON($1, $2 )
dnl VMASKCMP_ZERO_NEON(type, basic_type)
define(`VMASKCMP_ZERO_NEON', `
instruct vmaskcmp_zero$1_neon(vReg dst, vReg src, imm`$1'0 zero, immI_cmp_cond cond) %{
predicate(UseSVE == 0);
match(Set dst (VectorMaskCmp (Binary src (Replicate$1 zero)) cond));
match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
format %{ "vmaskcmp_zero$1_neon $dst, $src, #0, $cond" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
__ neon_compare_zero($dst$$FloatRegister, $2, $src$$FloatRegister,
__ neon_compare_zero($dst$$FloatRegister, ifelse($1, I, Matcher::vector_element_basic_type(this), $2),
$src$$FloatRegister,
condition, /* isQ */ length_in_bytes == 16);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl
VMASKCMP_ZERO_NEON(I)
VMASKCMP_ZERO_NEON(L, T_LONG)
VMASKCMP_ZERO_NEON(F, T_FLOAT)
VMASKCMP_ZERO_NEON(D, T_DOUBLE)
@ -3617,27 +3571,24 @@ dnl
dnl VMASKCMP_SVE_IMM($1 , $2 , $3 , $4 )
dnl VMASKCMP_SVE_IMM(element_size, element_type, type_imm, type_condition)
define(`VMASKCMP_SVE_IMM', `
instruct vmask$4_imm$2_sve(pReg dst, vReg src, $3 imm, immI_$4_cond cond, rFlagsReg cr) %{
instruct vmask$3_imm$1_sve(pReg dst, vReg src, $2 imm, immI_$3_cond cond, rFlagsReg cr) %{
predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src (Replicate$2 imm)) cond));
match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
effect(KILL cr);
format %{ "vmask$4_imm$2_sve $dst, $src, $imm, $cond\t# KILL cr" %}
format %{ "vmask$3_imm$1_sve $dst, $src, $imm, $cond\t# KILL cr" %}
ins_encode %{
Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
assert(length_in_bytes == MaxVectorSize, "invalid vector length");
__ sve_cmp(condition, $dst$$PRegister, __ $1, ptrue, $src$$FloatRegister, (int)$imm$$constant);
__ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
ptrue, $src$$FloatRegister, (int)$imm$$constant);
%}
ins_pipe(pipe_slow);
%}')dnl
VMASKCMP_SVE_IMM(B, B, immI5, cmp)
VMASKCMP_SVE_IMM(B, B, immIU7, cmpU)
VMASKCMP_SVE_IMM(H, S, immI5, cmp)
VMASKCMP_SVE_IMM(H, S, immIU7, cmpU)
VMASKCMP_SVE_IMM(S, I, immI5, cmp)
VMASKCMP_SVE_IMM(S, I, immIU7, cmpU)
VMASKCMP_SVE_IMM(D, L, immL5, cmp)
VMASKCMP_SVE_IMM(D, L, immLU7, cmpU)
VMASKCMP_SVE_IMM(I, immI5, cmp)
VMASKCMP_SVE_IMM(I, immIU7, cmpU)
VMASKCMP_SVE_IMM(L, immL5, cmp)
VMASKCMP_SVE_IMM(L, immLU7, cmpU)
instruct vmaskcmp_masked(pReg dst, vReg src1, vReg src2, immI cond,
pRegGov pg, rFlagsReg cr) %{

135
src/hotspot/cpu/arm/arm.ad
View File

@ -9462,8 +9462,9 @@ instruct storeV16(memoryvld mem, vecX src) %{
// Replicate scalar to packed byte values in Double register
instruct Repl8B_reg(vecD dst, iRegI src, iRegI tmp) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length() == 8 &&
Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (Replicate src));
ins_cost(DEFAULT_COST*4);
effect(TEMP tmp);
size(16);
@ -9484,8 +9485,9 @@ instruct Repl8B_reg(vecD dst, iRegI src, iRegI tmp) %{
// Replicate scalar to packed byte values in Double register
instruct Repl8B_reg_simd(vecD dst, iRegI src) %{
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (Replicate src));
size(4);
format %{ "VDUP.8 $dst,$src\t" %}
@ -9499,8 +9501,9 @@ instruct Repl8B_reg_simd(vecD dst, iRegI src) %{
// Replicate scalar to packed byte values in Double register pair
instruct Repl16B_reg(vecX dst, iRegI src) %{
predicate(n->as_Vector()->length_in_bytes() == 16);
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length_in_bytes() == 16 &&
Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (Replicate src));
size(4);
format %{ "VDUP.8 $dst.Q,$src\t" %}
@ -9514,8 +9517,9 @@ instruct Repl16B_reg(vecX dst, iRegI src) %{
// Replicate scalar constant to packed byte values in Double register
instruct Repl8B_immI(vecD dst, immI src, iRegI tmp) %{
predicate(n->as_Vector()->length() == 8);
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length() == 8 &&
Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (Replicate src));
ins_cost(DEFAULT_COST*2);
effect(TEMP tmp);
size(12);
@ -9529,8 +9533,9 @@ instruct Repl8B_immI(vecD dst, immI src, iRegI tmp) %{
// Replicate scalar constant to packed byte values in Double register
// TODO: support negative constants with MVNI?
instruct Repl8B_immU8(vecD dst, immU8 src) %{
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (Replicate src));
size(4);
format %{ "VMOV.U8 $dst,$src" %}
@ -9544,8 +9549,9 @@ instruct Repl8B_immU8(vecD dst, immU8 src) %{
// Replicate scalar constant to packed byte values in Double register pair
instruct Repl16B_immU8(vecX dst, immU8 src) %{
predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (Replicate src));
size(4);
format %{ "VMOV.U8 $dst.Q,$src" %}
@ -9559,8 +9565,9 @@ instruct Repl16B_immU8(vecX dst, immU8 src) %{
// Replicate scalar to packed short/char values into Double register
instruct Repl4S_reg(vecD dst, iRegI src, iRegI tmp) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst (Replicate src));
ins_cost(DEFAULT_COST*3);
effect(TEMP tmp);
size(12);
@ -9579,8 +9586,9 @@ instruct Repl4S_reg(vecD dst, iRegI src, iRegI tmp) %{
// Replicate scalar to packed byte values in Double register
instruct Repl4S_reg_simd(vecD dst, iRegI src) %{
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst (Replicate src));
size(4);
format %{ "VDUP.16 $dst,$src\t" %}
@ -9594,8 +9602,9 @@ instruct Repl4S_reg_simd(vecD dst, iRegI src) %{
// Replicate scalar to packed byte values in Double register pair
instruct Repl8S_reg(vecX dst, iRegI src) %{
predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst (Replicate src));
size(4);
format %{ "VDUP.16 $dst.Q,$src\t" %}
@ -9610,8 +9619,9 @@ instruct Repl8S_reg(vecX dst, iRegI src) %{
// Replicate scalar constant to packed short/char values in Double register
instruct Repl4S_immI(vecD dst, immI src, iRegP tmp) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst (Replicate src));
effect(TEMP tmp);
size(12);
ins_cost(DEFAULT_COST*4); // FIXME
@ -9624,8 +9634,9 @@ instruct Repl4S_immI(vecD dst, immI src, iRegP tmp) %{
// Replicate scalar constant to packed byte values in Double register
instruct Repl4S_immU8(vecD dst, immU8 src) %{
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst (Replicate src));
size(4);
format %{ "VMOV.U16 $dst,$src" %}
@ -9639,8 +9650,9 @@ instruct Repl4S_immU8(vecD dst, immU8 src) %{
// Replicate scalar constant to packed byte values in Double register pair
instruct Repl8S_immU8(vecX dst, immU8 src) %{
predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst (Replicate src));
size(4);
format %{ "VMOV.U16 $dst.Q,$src" %}
@ -9654,8 +9666,9 @@ instruct Repl8S_immU8(vecX dst, immU8 src) %{
// Replicate scalar to packed int values in Double register
instruct Repl2I_reg(vecD dst, iRegI src) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst (Replicate src));
size(4);
format %{ "FMDRR $dst,$src,$src\t" %}
@ -9667,8 +9680,9 @@ instruct Repl2I_reg(vecD dst, iRegI src) %{
// Replicate scalar to packed int values in Double register pair
instruct Repl4I_reg(vecX dst, iRegI src) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst (Replicate src));
ins_cost(DEFAULT_COST*2);
size(8);
@ -9685,8 +9699,9 @@ instruct Repl4I_reg(vecX dst, iRegI src) %{
// Replicate scalar to packed int values in Double register
instruct Repl2I_reg_simd(vecD dst, iRegI src) %{
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst (Replicate src));
size(4);
format %{ "VDUP.32 $dst.D,$src\t" %}
@ -9700,8 +9715,9 @@ instruct Repl2I_reg_simd(vecD dst, iRegI src) %{
// Replicate scalar to packed int values in Double register pair
instruct Repl4I_reg_simd(vecX dst, iRegI src) %{
predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst (Replicate src));
size(4);
format %{ "VDUP.32 $dst.Q,$src\t" %}
@ -9716,8 +9732,9 @@ instruct Repl4I_reg_simd(vecX dst, iRegI src) %{
// Replicate scalar zero constant to packed int values in Double register
instruct Repl2I_immI(vecD dst, immI src, iRegI tmp) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst (Replicate src));
effect(TEMP tmp);
size(12);
ins_cost(DEFAULT_COST*4); // FIXME
@ -9730,8 +9747,9 @@ instruct Repl2I_immI(vecD dst, immI src, iRegI tmp) %{
// Replicate scalar constant to packed byte values in Double register
instruct Repl2I_immU8(vecD dst, immU8 src) %{
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst (Replicate src));
size(4);
format %{ "VMOV.I32 $dst.D,$src" %}
@ -9745,8 +9763,9 @@ instruct Repl2I_immU8(vecD dst, immU8 src) %{
// Replicate scalar constant to packed byte values in Double register pair
instruct Repl4I_immU8(vecX dst, immU8 src) %{
predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst (Replicate src));
size(4);
format %{ "VMOV.I32 $dst.Q,$src" %}
@ -9760,8 +9779,9 @@ instruct Repl4I_immU8(vecX dst, immU8 src) %{
// Replicate scalar to packed byte values in Double register pair
instruct Repl2L_reg(vecX dst, iRegL src) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (ReplicateL src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (Replicate src));
size(8);
ins_cost(DEFAULT_COST*2); // FIXME
@ -9778,8 +9798,9 @@ instruct Repl2L_reg(vecX dst, iRegL src) %{
// Replicate scalar to packed float values in Double register
instruct Repl2F_regI(vecD dst, iRegI src) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (ReplicateF src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (Replicate src));
size(4);
format %{ "FMDRR $dst.D,$src,$src\t" %}
@ -9791,8 +9812,9 @@ instruct Repl2F_regI(vecD dst, iRegI src) %{
// Replicate scalar to packed float values in Double register
instruct Repl2F_reg_vfp(vecD dst, regF src) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (ReplicateF src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (Replicate src));
size(4*2);
ins_cost(DEFAULT_COST*2); // FIXME
@ -9805,8 +9827,9 @@ instruct Repl2F_reg_vfp(vecD dst, regF src) %{
// Replicate scalar to packed float values in Double register
instruct Repl2F_reg_simd(vecD dst, regF src) %{
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd());
match(Set dst (ReplicateF src));
predicate(n->as_Vector()->length_in_bytes() == 8 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (Replicate src));
size(4);
ins_cost(DEFAULT_COST); // FIXME
@ -9820,8 +9843,9 @@ instruct Repl2F_reg_simd(vecD dst, regF src) %{
// Replicate scalar to packed float values in Double register pair
instruct Repl4F_reg(vecX dst, regF src, iRegI tmp) %{
predicate(n->as_Vector()->length() == 4);
match(Set dst (ReplicateF src));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (Replicate src));
effect(TEMP tmp);
size(4*3);
ins_cost(DEFAULT_COST*3); // FIXME
@ -9840,8 +9864,9 @@ instruct Repl4F_reg(vecX dst, regF src, iRegI tmp) %{
// Replicate scalar to packed float values in Double register pair
instruct Repl4F_reg_simd(vecX dst, regF src) %{
predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd());
match(Set dst (ReplicateF src));
predicate(n->as_Vector()->length_in_bytes() == 16 && VM_Version::has_simd() &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (Replicate src));
size(4);
ins_cost(DEFAULT_COST); // FIXME
@ -9855,8 +9880,9 @@ instruct Repl4F_reg_simd(vecX dst, regF src) %{
// Replicate scalar zero constant to packed float values in Double register
instruct Repl2F_immI(vecD dst, immF src, iRegI tmp) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (ReplicateF src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (Replicate src));
effect(TEMP tmp);
size(12);
ins_cost(DEFAULT_COST*4); // FIXME
@ -9869,8 +9895,9 @@ instruct Repl2F_immI(vecD dst, immF src, iRegI tmp) %{
// Replicate scalar to packed double float values in Double register pair
instruct Repl2D_reg(vecX dst, regD src) %{
predicate(n->as_Vector()->length() == 2);
match(Set dst (ReplicateD src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_DOUBLE);
match(Set dst (Replicate src));
size(4*2);
ins_cost(DEFAULT_COST*2); // FIXME

143
src/hotspot/cpu/ppc/ppc.ad
View File

@ -13455,8 +13455,9 @@ instruct repl56(iRegLdst dst) %{
%}
instruct repl8B_reg_Ex(iRegLdst dst, iRegIsrc src) %{
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length() == 8);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 8 &&
Matcher::vector_element_basic_type(n) == T_BYTE);
expand %{
moveReg(dst, src);
repl56(dst);
@ -13466,8 +13467,9 @@ instruct repl8B_reg_Ex(iRegLdst dst, iRegIsrc src) %{
%}
instruct repl8B_immI0(iRegLdst dst, immI_0 zero) %{
match(Set dst (ReplicateB zero));
predicate(n->as_Vector()->length() == 8);
match(Set dst (Replicate zero));
predicate(n->as_Vector()->length() == 8 &&
Matcher::vector_element_basic_type(n) == T_BYTE);
format %{ "LI $dst, #0 \t// replicate8B" %}
size(4);
ins_encode %{
@ -13477,8 +13479,9 @@ instruct repl8B_immI0(iRegLdst dst, immI_0 zero) %{
%}
instruct repl8B_immIminus1(iRegLdst dst, immI_minus1 src) %{
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length() == 8);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 8 &&
Matcher::vector_element_basic_type(n) == T_BYTE);
format %{ "LI $dst, #-1 \t// replicate8B" %}
size(4);
ins_encode %{
@ -13488,8 +13491,9 @@ instruct repl8B_immIminus1(iRegLdst dst, immI_minus1 src) %{
%}
instruct repl16B_reg_Ex(vecX dst, iRegIsrc src) %{
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length() == 16);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 16 &&
Matcher::vector_element_basic_type(n) == T_BYTE);
expand %{
iRegLdst tmpL;
@ -13504,8 +13508,9 @@ instruct repl16B_reg_Ex(vecX dst, iRegIsrc src) %{
%}
instruct repl16B_immI0(vecX dst, immI_0 zero) %{
match(Set dst (ReplicateB zero));
predicate(n->as_Vector()->length() == 16);
match(Set dst (Replicate zero));
predicate(n->as_Vector()->length() == 16 &&
Matcher::vector_element_basic_type(n) == T_BYTE);
format %{ "XXLXOR $dst, $zero \t// replicate16B" %}
size(4);
@ -13516,8 +13521,9 @@ instruct repl16B_immI0(vecX dst, immI_0 zero) %{
%}
instruct repl16B_immIminus1(vecX dst, immI_minus1 src) %{
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length() == 16);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 16 &&
Matcher::vector_element_basic_type(n) == T_BYTE);
format %{ "XXLEQV $dst, $src \t// replicate16B" %}
size(4);
@ -13528,8 +13534,9 @@ instruct repl16B_immIminus1(vecX dst, immI_minus1 src) %{
%}
instruct repl4S_reg_Ex(iRegLdst dst, iRegIsrc src) %{
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length() == 4);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_SHORT);
expand %{
moveReg(dst, src);
repl48(dst);
@ -13538,8 +13545,9 @@ instruct repl4S_reg_Ex(iRegLdst dst, iRegIsrc src) %{
%}
instruct repl4S_immI0(iRegLdst dst, immI_0 zero) %{
match(Set dst (ReplicateS zero));
predicate(n->as_Vector()->length() == 4);
match(Set dst (Replicate zero));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_SHORT);
format %{ "LI $dst, #0 \t// replicate4S" %}
size(4);
ins_encode %{
@ -13549,8 +13557,9 @@ instruct repl4S_immI0(iRegLdst dst, immI_0 zero) %{
%}
instruct repl4S_immIminus1(iRegLdst dst, immI_minus1 src) %{
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length() == 4);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_SHORT);
format %{ "LI $dst, -1 \t// replicate4S" %}
size(4);
ins_encode %{
@ -13560,8 +13569,9 @@ instruct repl4S_immIminus1(iRegLdst dst, immI_minus1 src) %{
%}
instruct repl8S_reg_Ex(vecX dst, iRegIsrc src) %{
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length() == 8);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 8 &&
Matcher::vector_element_basic_type(n) == T_SHORT);
expand %{
iRegLdst tmpL;
@ -13576,8 +13586,9 @@ instruct repl8S_reg_Ex(vecX dst, iRegIsrc src) %{
%}
instruct repl8S_immI0(vecX dst, immI_0 zero) %{
match(Set dst (ReplicateS zero));
predicate(n->as_Vector()->length() == 8);
match(Set dst (Replicate zero));
predicate(n->as_Vector()->length() == 8 &&
Matcher::vector_element_basic_type(n) == T_SHORT);
format %{ "XXLXOR $dst, $zero \t// replicate8S" %}
size(4);
@ -13588,8 +13599,9 @@ instruct repl8S_immI0(vecX dst, immI_0 zero) %{
%}
instruct repl8S_immIminus1(vecX dst, immI_minus1 src) %{
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length() == 8);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 8 &&
Matcher::vector_element_basic_type(n) == T_SHORT);
format %{ "XXLEQV $dst, $src \t// replicate8S" %}
size(4);
@ -13600,8 +13612,9 @@ instruct repl8S_immIminus1(vecX dst, immI_minus1 src) %{
%}
instruct repl2I_reg_Ex(iRegLdst dst, iRegIsrc src) %{
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_INT);
ins_cost(2 * DEFAULT_COST);
expand %{
moveReg(dst, src);
@ -13610,8 +13623,9 @@ instruct repl2I_reg_Ex(iRegLdst dst, iRegIsrc src) %{
%}
instruct repl2I_immI0(iRegLdst dst, immI_0 zero) %{
match(Set dst (ReplicateI zero));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate zero));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_INT);
format %{ "LI $dst, #0 \t// replicate2I" %}
size(4);
ins_encode %{
@ -13621,8 +13635,9 @@ instruct repl2I_immI0(iRegLdst dst, immI_0 zero) %{
%}
instruct repl2I_immIminus1(iRegLdst dst, immI_minus1 src) %{
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_INT);
format %{ "LI $dst, -1 \t// replicate2I" %}
size(4);
ins_encode %{
@ -13632,8 +13647,9 @@ instruct repl2I_immIminus1(iRegLdst dst, immI_minus1 src) %{
%}
instruct repl4I_reg_Ex(vecX dst, iRegIsrc src) %{
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length() == 4);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_INT);
ins_cost(2 * DEFAULT_COST);
expand %{
@ -13648,8 +13664,9 @@ instruct repl4I_reg_Ex(vecX dst, iRegIsrc src) %{
%}
instruct repl4I_immI0(vecX dst, immI_0 zero) %{
match(Set dst (ReplicateI zero));
predicate(n->as_Vector()->length() == 4);
match(Set dst (Replicate zero));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_INT);
format %{ "XXLXOR $dst, $zero \t// replicate4I" %}
size(4);
@ -13660,8 +13677,9 @@ instruct repl4I_immI0(vecX dst, immI_0 zero) %{
%}
instruct repl4I_immIminus1(vecX dst, immI_minus1 src) %{
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length() == 4);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_INT);
format %{ "XXLEQV $dst, $dst, $dst \t// replicate4I" %}
size(4);
@ -13673,8 +13691,9 @@ instruct repl4I_immIminus1(vecX dst, immI_minus1 src) %{
// Move float to int register via stack, replicate.
instruct repl2F_reg_Ex(iRegLdst dst, regF src) %{
match(Set dst (ReplicateF src));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
ins_cost(2 * MEMORY_REF_COST + DEFAULT_COST);
expand %{
stackSlotL tmpS;
@ -13688,8 +13707,9 @@ instruct repl2F_reg_Ex(iRegLdst dst, regF src) %{
// Replicate scalar constant to packed float values in Double register
instruct repl2F_immF_Ex(iRegLdst dst, immF src) %{
match(Set dst (ReplicateF src));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
ins_cost(5 * DEFAULT_COST);
format %{ "LD $dst, offset, $constanttablebase\t// load replicated float $src $src from table, postalloc expanded" %}
@ -13698,8 +13718,9 @@ instruct repl2F_immF_Ex(iRegLdst dst, immF src) %{
// Replicate scalar zero constant to packed float values in Double register
instruct repl2F_immF0(iRegLdst dst, immF_0 zero) %{
match(Set dst (ReplicateF zero));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate zero));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
format %{ "LI $dst, #0 \t// replicate2F" %}
ins_encode %{
@ -14215,8 +14236,9 @@ instruct overflowMulL_reg_reg(flagsRegCR0 cr0, iRegLsrc op1, iRegLsrc op2) %{
%}
instruct repl4F_reg_Ex(vecX dst, regF src) %{
match(Set dst (ReplicateF src));
predicate(n->as_Vector()->length() == 4);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
ins_cost(DEFAULT_COST);
expand %{
vecX tmpV;
@ -14228,8 +14250,9 @@ instruct repl4F_reg_Ex(vecX dst, regF src) %{
%}
instruct repl4F_immF_Ex(vecX dst, immF src, iRegLdst tmp) %{
match(Set dst (ReplicateF src));
predicate(n->as_Vector()->length() == 4);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
effect(TEMP tmp);
ins_cost(10 * DEFAULT_COST);
@ -14237,8 +14260,9 @@ instruct repl4F_immF_Ex(vecX dst, immF src, iRegLdst tmp) %{
%}
instruct repl4F_immF0(vecX dst, immF_0 zero) %{
match(Set dst (ReplicateF zero));
predicate(n->as_Vector()->length() == 4);
match(Set dst (Replicate zero));
predicate(n->as_Vector()->length() == 4 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
format %{ "XXLXOR $dst, $zero \t// replicate4F" %}
ins_encode %{
@ -14248,8 +14272,9 @@ instruct repl4F_immF0(vecX dst, immF_0 zero) %{
%}
instruct repl2D_reg_Ex(vecX dst, regD src) %{
match(Set dst (ReplicateD src));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_DOUBLE);
format %{ "XXPERMDI $dst, $src, $src, 0 \t// Splat doubleword" %}
size(4);
@ -14260,8 +14285,9 @@ instruct repl2D_reg_Ex(vecX dst, regD src) %{
%}
instruct repl2D_immD0(vecX dst, immD_0 zero) %{
match(Set dst (ReplicateD zero));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate zero));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_DOUBLE);
format %{ "XXLXOR $dst, $zero \t// replicate2D" %}
size(4);
@ -14306,7 +14332,8 @@ instruct xxpermdi(vecX dst, vecX src1, vecX src2, immI8 zero) %{
%}
instruct repl2L_reg_Ex(vecX dst, iRegLsrc src) %{
match(Set dst (ReplicateL src));
predicate(Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 2);
expand %{
vecX tmpV;
@ -14317,8 +14344,9 @@ instruct repl2L_reg_Ex(vecX dst, iRegLsrc src) %{
%}
instruct repl2L_immI0(vecX dst, immI_0 zero) %{
match(Set dst (ReplicateL zero));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate zero));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_LONG);
format %{ "XXLXOR $dst, $zero \t// replicate2L" %}
size(4);
@ -14329,8 +14357,9 @@ instruct repl2L_immI0(vecX dst, immI_0 zero) %{
%}
instruct repl2L_immIminus1(vecX dst, immI_minus1 src) %{
match(Set dst (ReplicateL src));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_LONG);
format %{ "XXLEQV $dst, $src \t// replicate2L" %}
size(4);

22
src/hotspot/cpu/riscv/riscv_v.ad
View File

@ -1684,9 +1684,8 @@ instruct vreduce_minD_masked(fRegD dst, fRegD src1, vReg src2, vRegMask_V0 v0, v
// vector replicate
instruct replicate(vReg dst, iRegIorL2I src) %{
match(Set dst (ReplicateB src));
match(Set dst (ReplicateS src));
match(Set dst (ReplicateI src));
predicate(Matcher::is_non_long_integral_vector(n));
match(Set dst (Replicate src));
ins_cost(VEC_COST);
format %{ "replicate $dst, $src" %}
ins_encode %{
@ -1698,7 +1697,8 @@ instruct replicate(vReg dst, iRegIorL2I src) %{
%}
instruct replicateL(vReg dst, iRegL src) %{
match(Set dst (ReplicateL src));
predicate(Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (Replicate src));
ins_cost(VEC_COST);
format %{ "replicateL $dst, $src" %}
ins_encode %{
@ -1709,9 +1709,8 @@ instruct replicateL(vReg dst, iRegL src) %{
%}
instruct replicate_imm5(vReg dst, immI5 con) %{
match(Set dst (ReplicateB con));
match(Set dst (ReplicateS con));
match(Set dst (ReplicateI con));
predicate(Matcher::is_non_long_integral_vector(n));
match(Set dst (Replicate con));
ins_cost(VEC_COST);
format %{ "replicate_imm5 $dst, $con" %}
ins_encode %{
@ -1723,7 +1722,8 @@ instruct replicate_imm5(vReg dst, immI5 con) %{
%}
instruct replicateL_imm5(vReg dst, immL5 con) %{
match(Set dst (ReplicateL con));
predicate(Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (Replicate con));
ins_cost(VEC_COST);
format %{ "replicateL_imm5 $dst, $con" %}
ins_encode %{
@ -1734,7 +1734,8 @@ instruct replicateL_imm5(vReg dst, immL5 con) %{
%}
instruct replicateF(vReg dst, fRegF src) %{
match(Set dst (ReplicateF src));
predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (Replicate src));
ins_cost(VEC_COST);
format %{ "replicateF $dst, $src" %}
ins_encode %{
@ -1745,7 +1746,8 @@ instruct replicateF(vReg dst, fRegF src) %{
%}
instruct replicateD(vReg dst, fRegD src) %{
match(Set dst (ReplicateD src));
predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
match(Set dst (Replicate src));
ins_cost(VEC_COST);
format %{ "replicateD $dst, $src" %}
ins_encode %{

66
src/hotspot/cpu/s390/s390.ad
View File

@ -10446,9 +10446,9 @@ instruct expand_Repl2I_reg(iRegL dst, iRegL src) %{
// Exploit rotate_then_insert, if available
// Replicate scalar byte to packed byte values (8 Bytes).
instruct Repl8B_reg_risbg(iRegL dst, iRegI src, flagsReg cr) %{
match(Set dst (ReplicateB src));
match(Set dst (Replicate src));
effect(KILL cr);
predicate((n->as_Vector()->length() == 8));
predicate(n->as_Vector()->length() == 8 && Matcher::vector_element_basic_type(n) == T_BYTE);
format %{ "REPLIC8B $dst,$src\t # pack8B" %}
ins_encode %{
if ($dst$$Register != $src$$Register) {
@ -10463,8 +10463,8 @@ instruct Repl8B_reg_risbg(iRegL dst, iRegI src, flagsReg cr) %{
// Replicate scalar byte to packed byte values (8 Bytes).
instruct Repl8B_imm(iRegL dst, immB_n0m1 src) %{
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length() == 8);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 8 && Matcher::vector_element_basic_type(n) == T_BYTE);
ins_should_rematerialize(true);
format %{ "REPLIC8B $dst,$src\t # pack8B imm" %}
ins_encode %{
@ -10481,8 +10481,8 @@ instruct Repl8B_imm(iRegL dst, immB_n0m1 src) %{
// Replicate scalar byte to packed byte values (8 Bytes).
instruct Repl8B_imm0(iRegL dst, immI_0 src) %{
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length() == 8);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 8 && Matcher::vector_element_basic_type(n) == T_BYTE);
ins_should_rematerialize(true);
format %{ "REPLIC8B $dst,$src\t # pack8B imm0" %}
ins_encode %{ __ z_laz($dst$$Register, 0, Z_R0); %}
@ -10491,8 +10491,8 @@ instruct Repl8B_imm0(iRegL dst, immI_0 src) %{
// Replicate scalar byte to packed byte values (8 Bytes).
instruct Repl8B_immm1(iRegL dst, immB_minus1 src) %{
match(Set dst (ReplicateB src));
predicate(n->as_Vector()->length() == 8);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 8 && Matcher::vector_element_basic_type(n) == T_BYTE);
ins_should_rematerialize(true);
format %{ "REPLIC8B $dst,$src\t # pack8B immm1" %}
ins_encode %{ __ z_lghi($dst$$Register, -1); %}
@ -10502,9 +10502,9 @@ instruct Repl8B_immm1(iRegL dst, immB_minus1 src) %{
// Exploit rotate_then_insert, if available
// Replicate scalar short to packed short values (8 Bytes).
instruct Repl4S_reg_risbg(iRegL dst, iRegI src, flagsReg cr) %{
match(Set dst (ReplicateS src));
match(Set dst (Replicate src));
effect(KILL cr);
predicate((n->as_Vector()->length() == 4));
predicate((n->as_Vector()->length() == 4) && Matcher::vector_element_basic_type(n) == T_SHORT);
format %{ "REPLIC4S $dst,$src\t # pack4S" %}
ins_encode %{
if ($dst$$Register != $src$$Register) {
@ -10518,8 +10518,8 @@ instruct Repl4S_reg_risbg(iRegL dst, iRegI src, flagsReg cr) %{
// Replicate scalar short to packed short values (8 Bytes).
instruct Repl4S_imm(iRegL dst, immS_n0m1 src) %{
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length() == 4);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 4 && Matcher::vector_element_basic_type(n) == T_SHORT);
ins_should_rematerialize(true);
format %{ "REPLIC4S $dst,$src\t # pack4S imm" %}
ins_encode %{
@ -10536,8 +10536,8 @@ instruct Repl4S_imm(iRegL dst, immS_n0m1 src) %{
// Replicate scalar short to packed short values (8 Bytes).
instruct Repl4S_imm0(iRegL dst, immI_0 src) %{
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length() == 4);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 4 && Matcher::vector_element_basic_type(n) == T_SHORT);
ins_should_rematerialize(true);
format %{ "REPLIC4S $dst,$src\t # pack4S imm0" %}
ins_encode %{ __ z_laz($dst$$Register, 0, Z_R0); %}
@ -10546,8 +10546,8 @@ instruct Repl4S_imm0(iRegL dst, immI_0 src) %{
// Replicate scalar short to packed short values (8 Bytes).
instruct Repl4S_immm1(iRegL dst, immS_minus1 src) %{
match(Set dst (ReplicateS src));
predicate(n->as_Vector()->length() == 4);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 4 && Matcher::vector_element_basic_type(n) == T_SHORT);
ins_should_rematerialize(true);
format %{ "REPLIC4S $dst,$src\t # pack4S immm1" %}
ins_encode %{ __ z_lghi($dst$$Register, -1); %}
@ -10557,9 +10557,9 @@ instruct Repl4S_immm1(iRegL dst, immS_minus1 src) %{
// Exploit rotate_then_insert, if available.
// Replicate scalar int to packed int values (8 Bytes).
instruct Repl2I_reg_risbg(iRegL dst, iRegI src, flagsReg cr) %{
match(Set dst (ReplicateI src));
match(Set dst (Replicate src));
effect(KILL cr);
predicate((n->as_Vector()->length() == 2));
predicate((n->as_Vector()->length() == 2) && Matcher::vector_element_basic_type(n) == T_INT);
format %{ "REPLIC2I $dst,$src\t # pack2I" %}
ins_encode %{
if ($dst$$Register != $src$$Register) {
@ -10572,8 +10572,8 @@ instruct Repl2I_reg_risbg(iRegL dst, iRegI src, flagsReg cr) %{
// Replicate scalar int to packed int values (8 Bytes).
instruct Repl2I_imm(iRegL dst, immI_n0m1 src) %{
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 2 && Matcher::vector_element_basic_type(n) == T_INT);
ins_should_rematerialize(true);
format %{ "REPLIC2I $dst,$src\t # pack2I imm" %}
ins_encode %{
@ -10588,8 +10588,8 @@ instruct Repl2I_imm(iRegL dst, immI_n0m1 src) %{
// Replicate scalar int to packed int values (8 Bytes).
instruct Repl2I_imm0(iRegL dst, immI_0 src) %{
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 2 && Matcher::vector_element_basic_type(n) == T_INT);
ins_should_rematerialize(true);
format %{ "REPLIC2I $dst,$src\t # pack2I imm0" %}
ins_encode %{ __ z_laz($dst$$Register, 0, Z_R0); %}
@ -10598,8 +10598,8 @@ instruct Repl2I_imm0(iRegL dst, immI_0 src) %{
// Replicate scalar int to packed int values (8 Bytes).
instruct Repl2I_immm1(iRegL dst, immI_minus1 src) %{
match(Set dst (ReplicateI src));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 2 && Matcher::vector_element_basic_type(n) == T_INT);
ins_should_rematerialize(true);
format %{ "REPLIC2I $dst,$src\t # pack2I immm1" %}
ins_encode %{ __ z_lghi($dst$$Register, -1); %}
@ -10609,9 +10609,10 @@ instruct Repl2I_immm1(iRegL dst, immI_minus1 src) %{
//
instruct Repl2F_reg_indirect(iRegL dst, regF src, flagsReg cr) %{
match(Set dst (ReplicateF src));
match(Set dst (Replicate src));
effect(KILL cr);
predicate(!VM_Version::has_FPSupportEnhancements() && n->as_Vector()->length() == 2);
predicate(!VM_Version::has_FPSupportEnhancements() && n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
format %{ "REPLIC2F $dst,$src\t # pack2F indirect" %}
expand %{
stackSlotF tmp;
@ -10624,9 +10625,10 @@ instruct Repl2F_reg_indirect(iRegL dst, regF src, flagsReg cr) %{
// Replicate scalar float to packed float values in GREG (8 Bytes).
instruct Repl2F_reg_direct(iRegL dst, regF src, flagsReg cr) %{
match(Set dst (ReplicateF src));
match(Set dst (Replicate src));
effect(KILL cr);
predicate(VM_Version::has_FPSupportEnhancements() && n->as_Vector()->length() == 2);
predicate(VM_Version::has_FPSupportEnhancements() && n->as_Vector()->length() == 2 &&
Matcher::vector_element_basic_type(n) == T_FLOAT);
format %{ "REPLIC2F $dst,$src\t # pack2F direct" %}
ins_encode %{
assert(VM_Version::has_FPSupportEnhancements(), "encoder should never be called on old H/W");
@ -10641,8 +10643,8 @@ instruct Repl2F_reg_direct(iRegL dst, regF src, flagsReg cr) %{
// Replicate scalar float immediate to packed float values in GREG (8 Bytes).
instruct Repl2F_imm(iRegL dst, immF src) %{
match(Set dst (ReplicateF src));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 2 && Matcher::vector_element_basic_type(n) == T_FLOAT);
ins_should_rematerialize(true);
format %{ "REPLIC2F $dst,$src\t # pack2F imm" %}
ins_encode %{
@ -10660,8 +10662,8 @@ instruct Repl2F_imm(iRegL dst, immF src) %{
// Replicate scalar float immediate zeroes to packed float values in GREG (8 Bytes).
// Do this only for 'real' zeroes, especially don't loose sign of negative zeroes.
instruct Repl2F_imm0(iRegL dst, immFp0 src) %{
match(Set dst (ReplicateF src));
predicate(n->as_Vector()->length() == 2);
match(Set dst (Replicate src));
predicate(n->as_Vector()->length() == 2 && Matcher::vector_element_basic_type(n) == T_FLOAT);
ins_should_rematerialize(true);
format %{ "REPLIC2F $dst,$src\t # pack2F imm0" %}
ins_encode %{ __ z_laz($dst$$Register, 0, Z_R0); %}

171
src/hotspot/cpu/x86/x86.ad
View File

@ -4158,43 +4158,37 @@ instruct scatter_masked(memory mem, vec src, vec idx, kReg mask, kReg ktmp, rReg
// Replicate byte scalar to be vector
instruct vReplB_reg(vec dst, rRegI src) %{
predicate(UseAVX >= 2);
match(Set dst (ReplicateB src));
predicate(Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (Replicate src));
format %{ "replicateB $dst,$src" %}
ins_encode %{
uint vlen = Matcher::vector_length(this);
int vlen_enc = vector_length_encoding(this);
if (vlen == 64 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
assert(VM_Version::supports_avx512bw(), "required"); // 512-bit byte vectors assume AVX512BW
__ evpbroadcastb($dst$$XMMRegister, $src$$Register, vlen_enc);
if (UseAVX >= 2) {
int vlen_enc = vector_length_encoding(this);
if (vlen == 64 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
assert(VM_Version::supports_avx512bw(), "required"); // 512-bit byte vectors assume AVX512BW
__ evpbroadcastb($dst$$XMMRegister, $src$$Register, vlen_enc);
} else {
__ movdl($dst$$XMMRegister, $src$$Register);
__ vpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
}
} else {
assert(UseAVX < 2, "");
__ movdl($dst$$XMMRegister, $src$$Register);
__ vpbroadcastb($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
}
%}
ins_pipe( pipe_slow );
%}
instruct ReplB_reg(vec dst, rRegI src) %{
predicate(UseAVX < 2);
match(Set dst (ReplicateB src));
format %{ "replicateB $dst,$src" %}
ins_encode %{
uint vlen = Matcher::vector_length(this);
__ movdl($dst$$XMMRegister, $src$$Register);
__ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
__ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
if (vlen >= 16) {
assert(vlen == 16, "");
__ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
__ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
__ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
if (vlen >= 16) {
assert(vlen == 16, "");
__ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
}
}
%}
ins_pipe( pipe_slow );
%}
instruct ReplB_mem(vec dst, memory mem) %{
predicate(UseAVX >= 2);
match(Set dst (ReplicateB (LoadB mem)));
predicate(UseAVX >= 2 && Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (Replicate (LoadB mem)));
format %{ "replicateB $dst,$mem" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
@ -4206,43 +4200,36 @@ instruct ReplB_mem(vec dst, memory mem) %{
// ====================ReplicateS=======================================
instruct vReplS_reg(vec dst, rRegI src) %{
predicate(UseAVX >= 2);
match(Set dst (ReplicateS src));
predicate(Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst (Replicate src));
format %{ "replicateS $dst,$src" %}
ins_encode %{
uint vlen = Matcher::vector_length(this);
int vlen_enc = vector_length_encoding(this);
if (vlen == 32 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
assert(VM_Version::supports_avx512bw(), "required"); // 512-bit short vectors assume AVX512BW
__ evpbroadcastw($dst$$XMMRegister, $src$$Register, vlen_enc);
if (UseAVX >= 2) {
if (vlen == 32 || VM_Version::supports_avx512vlbw()) { // AVX512VL for <512bit operands
assert(VM_Version::supports_avx512bw(), "required"); // 512-bit short vectors assume AVX512BW
__ evpbroadcastw($dst$$XMMRegister, $src$$Register, vlen_enc);
} else {
__ movdl($dst$$XMMRegister, $src$$Register);
__ vpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
}
} else {
assert(UseAVX < 2, "");
__ movdl($dst$$XMMRegister, $src$$Register);
__ vpbroadcastw($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
}
%}
ins_pipe( pipe_slow );
%}
instruct ReplS_reg(vec dst, rRegI src) %{
predicate(UseAVX < 2);
match(Set dst (ReplicateS src));
format %{ "replicateS $dst,$src" %}
ins_encode %{
uint vlen = Matcher::vector_length(this);
int vlen_enc = vector_length_encoding(this);
__ movdl($dst$$XMMRegister, $src$$Register);
__ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
if (vlen >= 8) {
assert(vlen == 8, "");
__ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
__ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
if (vlen >= 8) {
assert(vlen == 8, "");
__ punpcklqdq($dst$$XMMRegister, $dst$$XMMRegister);
}
}
%}
ins_pipe( pipe_slow );
%}
instruct ReplS_mem(vec dst, memory mem) %{
predicate(UseAVX >= 2);
match(Set dst (ReplicateS (LoadS mem)));
predicate(UseAVX >= 2 && Matcher::vector_element_basic_type(n) == T_SHORT);
match(Set dst (Replicate (LoadS mem)));
format %{ "replicateS $dst,$mem" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
@ -4254,7 +4241,8 @@ instruct ReplS_mem(vec dst, memory mem) %{
// ====================ReplicateI=======================================
instruct ReplI_reg(vec dst, rRegI src) %{
match(Set dst (ReplicateI src));
predicate(Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst (Replicate src));
format %{ "replicateI $dst,$src" %}
ins_encode %{
uint vlen = Matcher::vector_length(this);
@ -4273,7 +4261,8 @@ instruct ReplI_reg(vec dst, rRegI src) %{
%}
instruct ReplI_mem(vec dst, memory mem) %{
match(Set dst (ReplicateI (LoadI mem)));
predicate(Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst (Replicate (LoadI mem)));
format %{ "replicateI $dst,$mem" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
@ -4290,9 +4279,8 @@ instruct ReplI_mem(vec dst, memory mem) %{
%}
instruct ReplI_imm(vec dst, immI con) %{
match(Set dst (ReplicateB con));
match(Set dst (ReplicateS con));
match(Set dst (ReplicateI con));
predicate(Matcher::is_non_long_integral_vector(n));
match(Set dst (Replicate con));
format %{ "replicateI $dst,$con" %}
ins_encode %{
InternalAddress addr = $constantaddress(Matcher::vector_element_basic_type(this),
@ -4308,9 +4296,8 @@ instruct ReplI_imm(vec dst, immI con) %{
// Replicate scalar zero to be vector
instruct ReplI_zero(vec dst, immI_0 zero) %{
match(Set dst (ReplicateB zero));
match(Set dst (ReplicateS zero));
match(Set dst (ReplicateI zero));
predicate(Matcher::is_non_long_integral_vector(n));
match(Set dst (Replicate zero));
format %{ "replicateI $dst,$zero" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
@ -4324,10 +4311,8 @@ instruct ReplI_zero(vec dst, immI_0 zero) %{
%}
instruct ReplI_M1(vec dst, immI_M1 con) %{
predicate(UseSSE >= 2);
match(Set dst (ReplicateB con));
match(Set dst (ReplicateS con));
match(Set dst (ReplicateI con));
predicate(UseSSE >= 2 && Matcher::is_non_long_integral_vector(n));
match(Set dst (Replicate con));
format %{ "vallones $dst" %}
ins_encode %{
int vector_len = vector_length_encoding(this);
@ -4341,7 +4326,8 @@ instruct ReplI_M1(vec dst, immI_M1 con) %{
#ifdef _LP64
// Replicate long (8 byte) scalar to be vector
instruct ReplL_reg(vec dst, rRegL src) %{
match(Set dst (ReplicateL src));
predicate(Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (Replicate src));
format %{ "replicateL $dst,$src" %}
ins_encode %{
int vlen = Matcher::vector_length(this);
@ -4361,8 +4347,8 @@ instruct ReplL_reg(vec dst, rRegL src) %{
#else // _LP64
// Replicate long (8 byte) scalar to be vector
instruct ReplL_reg(vec dst, eRegL src, vec tmp) %{
predicate(Matcher::vector_length(n) <= 4);
match(Set dst (ReplicateL src));
predicate(Matcher::vector_length(n) <= 4 && Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (Replicate src));
effect(TEMP dst, USE src, TEMP tmp);
format %{ "replicateL $dst,$src" %}
ins_encode %{
@ -4390,8 +4376,8 @@ instruct ReplL_reg(vec dst, eRegL src, vec tmp) %{
%}
instruct ReplL_reg_leg(legVec dst, eRegL src, legVec tmp) %{
predicate(Matcher::vector_length(n) == 8);
match(Set dst (ReplicateL src));
predicate(Matcher::vector_length(n) == 8 && Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (Replicate src));
effect(TEMP dst, USE src, TEMP tmp);
format %{ "replicateL $dst,$src" %}
ins_encode %{
@ -4415,7 +4401,8 @@ instruct ReplL_reg_leg(legVec dst, eRegL src, legVec tmp) %{
#endif // _LP64
instruct ReplL_mem(vec dst, memory mem) %{
match(Set dst (ReplicateL (LoadL mem)));
predicate(Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (Replicate (LoadL mem)));
format %{ "replicateL $dst,$mem" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
@ -4433,7 +4420,8 @@ instruct ReplL_mem(vec dst, memory mem) %{
// Replicate long (8 byte) scalar immediate to be vector by loading from const table.
instruct ReplL_imm(vec dst, immL con) %{
match(Set dst (ReplicateL con));
predicate(Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (Replicate con));
format %{ "replicateL $dst,$con" %}
ins_encode %{
InternalAddress addr = $constantaddress(T_LONG, vreplicate_imm(T_LONG, $con$$constant, 1));
@ -4444,7 +4432,8 @@ instruct ReplL_imm(vec dst, immL con) %{
%}
instruct ReplL_zero(vec dst, immL0 zero) %{
match(Set dst (ReplicateL zero));
predicate(Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (Replicate zero));
format %{ "replicateL $dst,$zero" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
@ -4458,8 +4447,8 @@ instruct ReplL_zero(vec dst, immL0 zero) %{
%}
instruct ReplL_M1(vec dst, immL_M1 con) %{
predicate(UseSSE >= 2);
match(Set dst (ReplicateL con));
predicate(UseSSE >= 2 && Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (Replicate con));
format %{ "vallones $dst" %}
ins_encode %{
int vector_len = vector_length_encoding(this);
@ -4471,8 +4460,8 @@ instruct ReplL_M1(vec dst, immL_M1 con) %{
// ====================ReplicateF=======================================
instruct vReplF_reg(vec dst, vlRegF src) %{
predicate(UseAVX > 0);
match(Set dst (ReplicateF src));
predicate(UseAVX > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (Replicate src));
format %{ "replicateF $dst,$src" %}
ins_encode %{
uint vlen = Matcher::vector_length(this);
@ -4491,8 +4480,8 @@ instruct vReplF_reg(vec dst, vlRegF src) %{
%}
instruct ReplF_reg(vec dst, vlRegF src) %{
predicate(UseAVX == 0);
match(Set dst (ReplicateF src));
predicate(UseAVX == 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (Replicate src));
format %{ "replicateF $dst,$src" %}
ins_encode %{
__ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
@ -4501,8 +4490,8 @@ instruct ReplF_reg(vec dst, vlRegF src) %{
%}
instruct ReplF_mem(vec dst, memory mem) %{
predicate(UseAVX > 0);
match(Set dst (ReplicateF (LoadF mem)));
predicate(UseAVX > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (Replicate (LoadF mem)));
format %{ "replicateF $dst,$mem" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
@ -4513,7 +4502,8 @@ instruct ReplF_mem(vec dst, memory mem) %{
// Replicate float scalar immediate to be vector by loading from const table.
instruct ReplF_imm(vec dst, immF con) %{
match(Set dst (ReplicateF con));
predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (Replicate con));
format %{ "replicateF $dst,$con" %}
ins_encode %{
InternalAddress addr = $constantaddress(T_FLOAT, vreplicate_imm(T_FLOAT, $con$$constant,
@ -4525,7 +4515,8 @@ instruct ReplF_imm(vec dst, immF con) %{
%}
instruct ReplF_zero(vec dst, immF0 zero) %{
match(Set dst (ReplicateF zero));
predicate(Matcher::vector_element_basic_type(n) == T_FLOAT);
match(Set dst (Replicate zero));
format %{ "replicateF $dst,$zero" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
@ -4542,8 +4533,8 @@ instruct ReplF_zero(vec dst, immF0 zero) %{
// Replicate double (8 bytes) scalar to be vector
instruct vReplD_reg(vec dst, vlRegD src) %{
predicate(UseSSE >= 3);
match(Set dst (ReplicateD src));
predicate(UseSSE >= 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
match(Set dst (Replicate src));
format %{ "replicateD $dst,$src" %}
ins_encode %{
uint vlen = Matcher::vector_length(this);
@ -4562,8 +4553,8 @@ instruct vReplD_reg(vec dst, vlRegD src) %{
%}
instruct ReplD_reg(vec dst, vlRegD src) %{
predicate(UseSSE < 3);
match(Set dst (ReplicateD src));
predicate(UseSSE < 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
match(Set dst (Replicate src));
format %{ "replicateD $dst,$src" %}
ins_encode %{
__ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x44);
@ -4572,8 +4563,8 @@ instruct ReplD_reg(vec dst, vlRegD src) %{
%}
instruct ReplD_mem(vec dst, memory mem) %{
predicate(UseSSE >= 3);
match(Set dst (ReplicateD (LoadD mem)));
predicate(UseSSE >= 3 && Matcher::vector_element_basic_type(n) == T_DOUBLE);
match(Set dst (Replicate (LoadD mem)));
format %{ "replicateD $dst,$mem" %}
ins_encode %{
if (Matcher::vector_length(this) >= 4) {
@ -4588,7 +4579,8 @@ instruct ReplD_mem(vec dst, memory mem) %{
// Replicate double (8 byte) scalar immediate to be vector by loading from const table.
instruct ReplD_imm(vec dst, immD con) %{
match(Set dst (ReplicateD con));
predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
match(Set dst (Replicate con));
format %{ "replicateD $dst,$con" %}
ins_encode %{
InternalAddress addr = $constantaddress(T_DOUBLE, vreplicate_imm(T_DOUBLE, $con$$constant, 1));
@ -4599,7 +4591,8 @@ instruct ReplD_imm(vec dst, immD con) %{
%}
instruct ReplD_zero(vec dst, immD0 zero) %{
match(Set dst (ReplicateD zero));
predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
match(Set dst (Replicate zero));
format %{ "replicateD $dst,$zero" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);

2
src/hotspot/share/adlc/formssel.cpp
View File

@ -4251,7 +4251,7 @@ bool MatchRule::is_vector() const {
"LShiftVB","LShiftVS","LShiftVI","LShiftVL",
"RShiftVB","RShiftVS","RShiftVI","RShiftVL",
"URShiftVB","URShiftVS","URShiftVI","URShiftVL",
"ReplicateB","ReplicateS","ReplicateI","ReplicateL","ReplicateF","ReplicateD","ReverseV","ReverseBytesV",
"Replicate","ReverseV","ReverseBytesV",
"RoundDoubleModeV","RotateLeftV" , "RotateRightV", "LoadVector","StoreVector",
"LoadVectorGather", "StoreVectorScatter", "LoadVectorGatherMasked", "StoreVectorScatterMasked",
"VectorTest", "VectorLoadMask", "VectorStoreMask", "VectorBlend", "VectorInsert",

7
src/hotspot/share/opto/classes.hpp
View File

@ -466,12 +466,7 @@ macro(PackF)
macro(PackD)
macro(Pack2L)
macro(Pack2D)
macro(ReplicateB)
macro(ReplicateS)
macro(ReplicateI)
macro(ReplicateL)
macro(ReplicateF)
macro(ReplicateD)
macro(Replicate)
macro(RoundVF)
macro(RoundVD)
macro(Extract)

6
src/hotspot/share/opto/matcher.cpp
View File

@ -2795,6 +2795,12 @@ BasicType Matcher::vector_element_basic_type(const MachNode* use, const MachOper
return def->bottom_type()->is_vect()->element_basic_type();
}
bool Matcher::is_non_long_integral_vector(const Node* n) {
BasicType bt = vector_element_basic_type(n);
assert(bt != T_CHAR, "char is not allowed in vector");
return is_subword_type(bt) || bt == T_INT;
}
#ifdef ASSERT
bool Matcher::verify_after_postselect_cleanup() {
assert(!C->failing(), "sanity");

3
src/hotspot/share/opto/matcher.hpp
View File

@ -377,6 +377,9 @@ public:
static BasicType vector_element_basic_type(const Node* n);
static BasicType vector_element_basic_type(const MachNode* use, const MachOper* opnd);
// Vector element basic type is non double word integral type.
static bool is_non_long_integral_vector(const Node* n);
// Check if given booltest condition is unsigned or not
static inline bool is_unsigned_booltest_pred(int bt) {
return ((bt & BoolTest::unsigned_compare) == BoolTest::unsigned_compare);

2
src/hotspot/share/opto/superword.cpp
View File

@ -2686,7 +2686,7 @@ bool SuperWord::output() {
C->record_failure(C2Compiler::retry_no_superword());
return false; // bailout
}
if (VectorNode::is_invariant_vector(in1) && (node_isa_reduction == false) && (n->is_Add() || n->is_Mul())) {
if (in1->Opcode() == Op_Replicate && (node_isa_reduction == false) && (n->is_Add() || n->is_Mul())) {
// Move invariant vector input into second position to avoid register spilling.
Node* tmp = in1;
in1 = in2;

11
src/hotspot/share/opto/vectorIntrinsics.cpp
View File

@ -74,8 +74,7 @@ bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicTyp
// has_scalar_args flag is true only for non-constant scalar shift count,
// since in this case shift needs to be broadcasted.
if (!Matcher::match_rule_supported_vector(opc, num_elem, elem_bt) ||
(has_scalar_args &&
!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
(has_scalar_args && !arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
is_supported = false;
}
@ -625,7 +624,7 @@ bool LibraryCallKit::inline_vector_shuffle_iota() {
if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed) ||
!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed) ||
!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
!arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed)) {
return false;
}
@ -869,7 +868,7 @@ bool LibraryCallKit::inline_vector_frombits_coerced() {
bool is_mask = is_vector_mask(vbox_klass);
int bcast_mode = mode->get_con();
VectorMaskUseType checkFlags = (VectorMaskUseType)(is_mask ? VecMaskUseAll : VecMaskNotUsed);
int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : VectorNode::replicate_opcode(elem_bt);
int opc = bcast_mode == VectorSupport::MODE_BITS_COERCED_LONG_TO_MASK ? Op_VectorLongToMask : Op_Replicate;
if (!arch_supports_vector(opc, num_elem, elem_bt, checkFlags, true /*has_scalar_args*/)) {
if (C->print_intrinsics()) {
@ -2100,7 +2099,7 @@ bool LibraryCallKit::inline_vector_rearrange() {
if(!is_masked_op ||
(!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed))) {
!arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: arity=2 op=shuffle/rearrange vlen=%d etype=%s ismask=no",
num_elem, type2name(elem_bt));
@ -3107,7 +3106,7 @@ bool LibraryCallKit::inline_index_partially_in_upper_range() {
bool supports_mask_gen = arch_supports_vector(Op_VectorMaskGen, num_elem, elem_bt, VecMaskUseStore);
if (!supports_mask_gen) {
if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed) ||
!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed) ||
!arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed) ||
!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: vlen=%d etype=%s", num_elem, type2name(elem_bt));

78
src/hotspot/share/opto/vectornode.cpp
View File

@ -387,28 +387,6 @@ int VectorNode::scalar_opcode(int sopc, BasicType bt) {
}
}
int VectorNode::replicate_opcode(BasicType bt) {
switch(bt) {
case T_BOOLEAN:
case T_BYTE:
return Op_ReplicateB;
case T_SHORT:
case T_CHAR:
return Op_ReplicateS;
case T_INT:
return Op_ReplicateI;
case T_LONG:
return Op_ReplicateL;
case T_FLOAT:
return Op_ReplicateF;
case T_DOUBLE:
return Op_ReplicateD;
default:
assert(false, "wrong type: %s", type2name(bt));
return 0;
}
}
// Limits on vector size (number of elements) for auto-vectorization.
bool VectorNode::vector_size_supported_superword(const BasicType bt, int size) {
return Matcher::superword_max_vector_size(bt) >= size &&
@ -506,7 +484,7 @@ bool VectorNode::is_vector_integral_negate_supported(int opc, uint vlen, BasicTy
// Negate is implemented with "(SubVI/L (ReplicateI/L 0) src)", if NegVI/L is not supported.
int sub_opc = (bt == T_LONG) ? Op_SubL : Op_SubI;
if (Matcher::match_rule_supported_vector(VectorNode::opcode(sub_opc, bt), vlen, bt) &&
Matcher::match_rule_supported_vector(VectorNode::replicate_opcode(bt), vlen, bt)) {
Matcher::match_rule_supported_vector(Op_Replicate, vlen, bt)) {
return true;
}
} else {
@ -519,7 +497,7 @@ bool VectorNode::is_vector_integral_negate_supported(int opc, uint vlen, BasicTy
int add_opc = (bt == T_LONG) ? Op_AddL : Op_AddI;
if (Matcher::match_rule_supported_vector_masked(Op_XorV, vlen, bt) &&
Matcher::match_rule_supported_vector_masked(VectorNode::opcode(add_opc, bt), vlen, bt) &&
Matcher::match_rule_supported_vector(VectorNode::replicate_opcode(bt), vlen, bt)) {
Matcher::match_rule_supported_vector(Op_Replicate, vlen, bt)) {
return true;
}
}
@ -626,22 +604,6 @@ bool VectorNode::is_vshift_cnt(Node* n) {
return is_vshift_cnt_opcode(n->Opcode());
}
// Check if input is loop invariant vector.
bool VectorNode::is_invariant_vector(Node* n) {
// Only Replicate vector nodes are loop invariant for now.
switch (n->Opcode()) {
case Op_ReplicateB:
case Op_ReplicateS:
case Op_ReplicateI:
case Op_ReplicateL:
case Op_ReplicateF:
case Op_ReplicateD:
return true;
default:
return false;
}
}
// [Start, end) half-open range defining which operands are vectors
void VectorNode::vector_operands(Node* n, uint* start, uint* end) {
switch (n->Opcode()) {
@ -863,25 +825,7 @@ VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t, boo
const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen)
: TypeVect::make(bt, vlen);
switch (bt) {
case T_BOOLEAN:
case T_BYTE:
return new ReplicateBNode(s, vt);
case T_CHAR:
case T_SHORT:
return new ReplicateSNode(s, vt);
case T_INT:
return new ReplicateINode(s, vt);
case T_LONG:
return new ReplicateLNode(s, vt);
case T_FLOAT:
return new ReplicateFNode(s, vt);
case T_DOUBLE:
return new ReplicateDNode(s, vt);
default:
fatal("Type '%s' is not supported for vectors", type2name(bt));
return nullptr;
}
return new ReplicateNode(s, vt);
}
VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) {
@ -966,10 +910,9 @@ static bool is_con(Node* n, long con) {
// Return true if every bit in this vector is 1.
bool VectorNode::is_all_ones_vector(Node* n) {
switch (n->Opcode()) {
case Op_ReplicateB:
case Op_ReplicateS:
case Op_ReplicateI:
case Op_ReplicateL:
case Op_Replicate:
return is_integral_type(n->bottom_type()->is_vect()->element_basic_type()) &&
is_con(n->in(1), -1);
case Op_MaskAll:
return is_con(n->in(1), -1);
default:
@ -980,10 +923,9 @@ bool VectorNode::is_all_ones_vector(Node* n) {
// Return true if every bit in this vector is 0.
bool VectorNode::is_all_zeros_vector(Node* n) {
switch (n->Opcode()) {
case Op_ReplicateB:
case Op_ReplicateS:
case Op_ReplicateI:
case Op_ReplicateL:
case Op_Replicate:
return is_integral_type(n->bottom_type()->is_vect()->element_basic_type()) &&
is_con(n->in(1), 0);
case Op_MaskAll:
return is_con(n->in(1), 0);
default:
@ -1631,7 +1573,7 @@ Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_
int shift = cnt_type->get_con() & shift_mask;
shiftRCnt = phase->intcon(shift);
shiftLCnt = phase->intcon(shift_mask + 1 - shift);
} else if (VectorNode::is_invariant_vector(cnt)) {
} else if (cnt->Opcode() == Op_Replicate) {
// Scalar variable shift, handle replicates generated by auto vectorizer.
cnt = cnt->in(1);
if (bt == T_LONG) {

51
src/hotspot/share/opto/vectornode.hpp
View File

@ -94,7 +94,6 @@ class VectorNode : public TypeNode {
static int opcode(int sopc, BasicType bt); // scalar_opc -> vector_opc
static int scalar_opcode(int vopc, BasicType bt); // vector_opc -> scalar_opc
static int replicate_opcode(BasicType bt);
// Limits on vector size (number of elements) for auto-vectorization.
static bool vector_size_supported_superword(const BasicType bt, int size);
@ -109,7 +108,6 @@ class VectorNode : public TypeNode {
static bool is_vector_rotate_supported(int opc, uint vlen, BasicType bt);
static bool is_vector_integral_negate_supported(int opc, uint vlen, BasicType bt, bool use_predicate);
static bool is_populate_index_supported(BasicType bt);
static bool is_invariant_vector(Node* n);
// Return true if every bit in this vector is 1.
static bool is_all_ones_vector(Node* n);
// Return true if every bit in this vector is 0.
@ -1135,51 +1133,12 @@ class XorVMaskNode : public XorVNode {
//=========================Promote_Scalar_to_Vector============================
//------------------------------ReplicateBNode---------------------------------
// Replicate byte scalar to be vector
class ReplicateBNode : public VectorNode {
class ReplicateNode : public VectorNode {
public:
ReplicateBNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
virtual int Opcode() const;
};
//------------------------------ReplicateSNode---------------------------------
// Replicate short scalar to be vector
class ReplicateSNode : public VectorNode {
public:
ReplicateSNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
virtual int Opcode() const;
};
//------------------------------ReplicateINode---------------------------------
// Replicate int scalar to be vector
class ReplicateINode : public VectorNode {
public:
ReplicateINode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
virtual int Opcode() const;
};
//------------------------------ReplicateLNode---------------------------------
// Replicate long scalar to be vector
class ReplicateLNode : public VectorNode {
public:
ReplicateLNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
virtual int Opcode() const;
};
//------------------------------ReplicateFNode---------------------------------
// Replicate float scalar to be vector
class ReplicateFNode : public VectorNode {
public:
ReplicateFNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
virtual int Opcode() const;
};
//------------------------------ReplicateDNode---------------------------------
// Replicate double scalar to be vector
class ReplicateDNode : public VectorNode {
public:
ReplicateDNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {}
ReplicateNode(Node* in1, const TypeVect* vt) : VectorNode(in1, vt) {
assert(vt->element_basic_type() != T_BOOLEAN, "not support");
assert(vt->element_basic_type() != T_CHAR, "not support");
}
virtual int Opcode() const;
};

7
src/hotspot/share/runtime/vmStructs.cpp
View File

@ -1749,12 +1749,7 @@
declare_c2_type(MinVNode, VectorNode) \
declare_c2_type(LoadVectorNode, LoadNode) \
declare_c2_type(StoreVectorNode, StoreNode) \
declare_c2_type(ReplicateBNode, VectorNode) \
declare_c2_type(ReplicateSNode, VectorNode) \
declare_c2_type(ReplicateINode, VectorNode) \
declare_c2_type(ReplicateLNode, VectorNode) \
declare_c2_type(ReplicateFNode, VectorNode) \
declare_c2_type(ReplicateDNode, VectorNode) \
declare_c2_type(ReplicateNode, VectorNode) \
declare_c2_type(PopulateIndexNode, VectorNode) \
declare_c2_type(PackNode, VectorNode) \
declare_c2_type(PackBNode, PackNode) \

62
test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java
View File

@ -1137,52 +1137,34 @@ public class IRNode {
trapNodes(RANGE_CHECK_TRAP,"range_check");
}
public static final String REPLICATE_B = PREFIX + "REPLICATE_B" + POSTFIX;
public static final String REPLICATE_B = VECTOR_PREFIX + "REPLICATE_B" + POSTFIX;
static {
String regex = START + "ReplicateB" + MID + END;
IR_NODE_MAPPINGS.put(REPLICATE_B, new SinglePhaseRangeEntry(CompilePhase.PRINT_IDEAL, regex,
CompilePhase.AFTER_CLOOPS,
CompilePhase.BEFORE_MATCHING));
vectorNode(REPLICATE_B, "Replicate", TYPE_BYTE);
}
public static final String REPLICATE_S = PREFIX + "REPLICATE_S" + POSTFIX;
public static final String REPLICATE_S = VECTOR_PREFIX + "REPLICATE_S" + POSTFIX;
static {
String regex = START + "ReplicateS" + MID + END;
IR_NODE_MAPPINGS.put(REPLICATE_S, new SinglePhaseRangeEntry(CompilePhase.PRINT_IDEAL, regex,
CompilePhase.AFTER_CLOOPS,
CompilePhase.BEFORE_MATCHING));
vectorNode(REPLICATE_S, "Replicate", TYPE_SHORT);
}
public static final String REPLICATE_I = PREFIX + "REPLICATE_I" + POSTFIX;
public static final String REPLICATE_I = VECTOR_PREFIX + "REPLICATE_I" + POSTFIX;
static {
String regex = START + "ReplicateI" + MID + END;
IR_NODE_MAPPINGS.put(REPLICATE_I, new SinglePhaseRangeEntry(CompilePhase.PRINT_IDEAL, regex,
CompilePhase.AFTER_CLOOPS,
CompilePhase.BEFORE_MATCHING));
vectorNode(REPLICATE_I, "Replicate", TYPE_INT);
}
public static final String REPLICATE_L = PREFIX + "REPLICATE_L" + POSTFIX;
public static final String REPLICATE_L = VECTOR_PREFIX + "REPLICATE_L" + POSTFIX;
static {
String regex = START + "ReplicateL" + MID + END;
IR_NODE_MAPPINGS.put(REPLICATE_L, new SinglePhaseRangeEntry(CompilePhase.PRINT_IDEAL, regex,
CompilePhase.AFTER_CLOOPS,
CompilePhase.BEFORE_MATCHING));
vectorNode(REPLICATE_L, "Replicate", TYPE_LONG);
}
public static final String REPLICATE_F = PREFIX + "REPLICATE_F" + POSTFIX;
public static final String REPLICATE_F = VECTOR_PREFIX + "REPLICATE_F" + POSTFIX;
static {
String regex = START + "ReplicateF" + MID + END;
IR_NODE_MAPPINGS.put(REPLICATE_F, new SinglePhaseRangeEntry(CompilePhase.PRINT_IDEAL, regex,
CompilePhase.AFTER_CLOOPS,
CompilePhase.BEFORE_MATCHING));
vectorNode(REPLICATE_F, "Replicate", TYPE_FLOAT);
}
public static final String REPLICATE_D = PREFIX + "REPLICATE_D" + POSTFIX;
public static final String REPLICATE_D = VECTOR_PREFIX + "REPLICATE_D" + POSTFIX;
static {
String regex = START + "ReplicateD" + MID + END;
IR_NODE_MAPPINGS.put(REPLICATE_D, new SinglePhaseRangeEntry(CompilePhase.PRINT_IDEAL, regex,
CompilePhase.AFTER_CLOOPS,
CompilePhase.BEFORE_MATCHING));
vectorNode(REPLICATE_D, "Replicate", TYPE_DOUBLE);
}
public static final String REVERSE_BYTES_VB = VECTOR_PREFIX + "REVERSE_BYTES_VB" + POSTFIX;
@ -1952,26 +1934,6 @@ public class IRNode {
machOnlyNameRegex(VMASK_CMP_ZERO_D_NEON, "vmaskcmp_zeroD_neon");
}
public static final String VMASK_CMP_IMM_B_SVE = PREFIX + "VMASK_CMP_IMM_B_SVE" + POSTFIX;
static {
machOnlyNameRegex(VMASK_CMP_IMM_B_SVE, "vmaskcmp_immB_sve");
}
public static final String VMASK_CMPU_IMM_B_SVE = PREFIX + "VMASK_CMPU_IMM_B_SVE" + POSTFIX;
static {
machOnlyNameRegex(VMASK_CMPU_IMM_B_SVE, "vmaskcmpU_immB_sve");
}
public static final String VMASK_CMP_IMM_S_SVE = PREFIX + "VMASK_CMP_IMM_S_SVE" + POSTFIX;
static {
machOnlyNameRegex(VMASK_CMP_IMM_S_SVE, "vmaskcmp_immS_sve");
}
public static final String VMASK_CMPU_IMM_S_SVE = PREFIX + "VMASK_CMPU_IMM_S_SVE" + POSTFIX;
static {
machOnlyNameRegex(VMASK_CMPU_IMM_S_SVE, "vmaskcmpU_immS_sve");
}
public static final String VMASK_CMP_IMM_I_SVE = PREFIX + "VMASK_CMP_IMM_I_SVE" + POSTFIX;
static {
machOnlyNameRegex(VMASK_CMP_IMM_I_SVE, "vmaskcmp_immI_sve");

16
test/hotspot/jtreg/compiler/vectorapi/VectorCompareWithImmTest.java
View File

@ -127,7 +127,7 @@ public class VectorCompareWithImmTest {
}
@Test
@IR(counts = { IRNode.VMASK_CMP_IMM_B_SVE, ">= 1" })
@IR(counts = { IRNode.VMASK_CMP_IMM_I_SVE, ">= 1" })
public static void testByteGTInRange() {
ByteVector av = ByteVector.fromArray(B_SPECIES, ba, 0);
av.compare(VectorOperators.GT, 12).intoArray(br, 0);
@ -140,7 +140,7 @@ public class VectorCompareWithImmTest {
}
@Test
@IR(counts = { IRNode.VMASK_CMPU_IMM_B_SVE, ">= 1" })
@IR(counts = { IRNode.VMASK_CMPU_IMM_I_SVE, ">= 1" })
public static void testByteUnsignedGTInRange() {
ByteVector av = ByteVector.fromArray(B_SPECIES, ba, 0);
av.compare(VectorOperators.UNSIGNED_GT, 64).intoArray(br, 0);
@ -153,21 +153,21 @@ public class VectorCompareWithImmTest {
}
@Test
@IR(failOn = { IRNode.VMASK_CMP_IMM_B_SVE })
@IR(failOn = { IRNode.VMASK_CMP_IMM_I_SVE })
public static void testByteGTOutOfRange() {
ByteVector av = ByteVector.fromArray(B_SPECIES, ba, 0);
av.compare(VectorOperators.GT, -91).intoArray(br, 0);
}
@Test
@IR(failOn = { IRNode.VMASK_CMPU_IMM_B_SVE })
@IR(failOn = { IRNode.VMASK_CMPU_IMM_I_SVE })
public static void testByteUnsignedGTOutOfRange() {
ByteVector av = ByteVector.fromArray(B_SPECIES, ba, 0);
av.compare(VectorOperators.UNSIGNED_GT, -91).intoArray(br, 0);
}
@Test
@IR(counts = { IRNode.VMASK_CMP_IMM_S_SVE, ">= 1" })
@IR(counts = { IRNode.VMASK_CMP_IMM_I_SVE, ">= 1" })
public static void testShortGEInRange() {
ShortVector av = ShortVector.fromArray(S_SPECIES, sa, 0);
av.compare(VectorOperators.GE, 5).intoArray(sr, 0);
@ -180,7 +180,7 @@ public class VectorCompareWithImmTest {
}
@Test
@IR(counts = { IRNode.VMASK_CMPU_IMM_S_SVE, ">= 1" })
@IR(counts = { IRNode.VMASK_CMPU_IMM_I_SVE, ">= 1" })
public static void testShortUnsignedGEInRange() {
ShortVector av = ShortVector.fromArray(S_SPECIES, sa, 0);
av.compare(VectorOperators.UNSIGNED_GE, 56).intoArray(sr, 0);
@ -193,14 +193,14 @@ public class VectorCompareWithImmTest {
}
@Test
@IR(failOn = { IRNode.VMASK_CMP_IMM_S_SVE })
@IR(failOn = { IRNode.VMASK_CMP_IMM_I_SVE })
public static void testShortGEOutOfRange() {
ShortVector av = ShortVector.fromArray(S_SPECIES, sa, 0);
av.compare(VectorOperators.GE, -85).intoArray(sr, 0);
}
@Test
@IR(failOn = { IRNode.VMASK_CMPU_IMM_S_SVE })
@IR(failOn = { IRNode.VMASK_CMPU_IMM_I_SVE })
public static void testShortUnsignedGEOutOfRange() {
ShortVector av = ShortVector.fromArray(S_SPECIES, sa, 0);
av.compare(VectorOperators.UNSIGNED_GE, -85).intoArray(sr, 0);

2
test/hotspot/jtreg/compiler/vectorization/TestBufferVectorization.java
View File

@ -244,7 +244,7 @@ public class TestBufferVectorization {
return; // bufferHeap uses Long type for memory accesses which are not vectorized in 32-bit VM
}
out.shouldContain("ReplicateI");
out.shouldContain("Replicate");
out.shouldContain("LoadVector");
out.shouldContain("AddVI");
out.shouldContain("StoreVector");