stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8294261: AArch64: Use pReg instead of pRegGov when possible

Browse Source 
Reviewed-by: ngasson, xgong
This commit is contained in:
Ningsheng Jian 2022-10-11 01:06:28 +00:00
parent 891156a744
commit 4b17d28a6d
6 changed files with 86 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/hotspot/cpu/aarch64/aarch64.ad
View File

@ -460,6 +460,7 @@ alloc_class chunk1(
); );
alloc_class chunk2 ( alloc_class chunk2 (
// Governing predicates for load/store and arithmetic
P0, P0,
P1, P1,
P2, P2,
@ -467,8 +468,8 @@ alloc_class chunk2 (
P4, P4,
P5, P5,
P6, P6,
P7,
// Extra predicates
P8, P8,
P9, P9,
P10, P10,
@ -477,6 +478,9 @@ alloc_class chunk2 (
P13, P13,
P14, P14,
P15, P15,
// Preserved for all-true predicate
P7,
); );
alloc_class chunk3(RFLAGS); alloc_class chunk3(RFLAGS);
@ -5538,6 +5542,7 @@ operand pRegGov()
%{ %{
constraint(ALLOC_IN_RC(gov_pr)); constraint(ALLOC_IN_RC(gov_pr));
match(RegVectMask); match(RegVectMask);
match(pReg);
op_cost(0); op_cost(0);
format %{ %} format %{ %}
interface(REG_INTER); interface(REG_INTER);

78
src/hotspot/cpu/aarch64/aarch64_vector.ad
View File

@ -4659,7 +4659,7 @@ instruct vloadmask_neon(vReg dst, vReg src) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vloadmaskB_sve(pRegGov dst, vReg src, rFlagsReg cr) %{ instruct vloadmaskB_sve(pReg dst, vReg src, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE); predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (VectorLoadMask src)); match(Set dst (VectorLoadMask src));
effect(KILL cr); effect(KILL cr);
@ -4671,7 +4671,7 @@ instruct vloadmaskB_sve(pRegGov dst, vReg src, rFlagsReg cr) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vloadmask_extend_sve(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{ instruct vloadmask_extend_sve(pReg dst, vReg src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE); predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE);
match(Set dst (VectorLoadMask src)); match(Set dst (VectorLoadMask src));
effect(TEMP tmp, KILL cr); effect(TEMP tmp, KILL cr);
@ -4685,7 +4685,7 @@ instruct vloadmask_extend_sve(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vloadmaskB_masked(pRegGov dst, vReg src, pRegGov pg, rFlagsReg cr) %{ instruct vloadmaskB_masked(pReg dst, vReg src, pRegGov pg, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE); predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (VectorLoadMask src pg)); match(Set dst (VectorLoadMask src pg));
effect(KILL cr); effect(KILL cr);
@ -4697,7 +4697,7 @@ instruct vloadmaskB_masked(pRegGov dst, vReg src, pRegGov pg, rFlagsReg cr) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vloadmask_extend_masked(pRegGov dst, vReg src, pRegGov pg, vReg tmp, rFlagsReg cr) %{ instruct vloadmask_extend_masked(pReg dst, vReg src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE); predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE);
match(Set dst (VectorLoadMask src pg)); match(Set dst (VectorLoadMask src pg));
effect(TEMP tmp, KILL cr); effect(TEMP tmp, KILL cr);
@ -4751,7 +4751,7 @@ instruct vstoremask_narrow_neon(vReg dst, vReg src, immI_gt_1 size) %{
// vector store mask - sve // vector store mask - sve
instruct vstoremaskB_sve(vReg dst, pRegGov src, immI_1 size) %{ instruct vstoremaskB_sve(vReg dst, pReg src, immI_1 size) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (VectorStoreMask src size)); match(Set dst (VectorStoreMask src size));
format %{ "vstoremaskB_sve $dst, $src" %} format %{ "vstoremaskB_sve $dst, $src" %}
@ -4761,7 +4761,7 @@ instruct vstoremaskB_sve(vReg dst, pRegGov src, immI_1 size) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vstoremask_narrow_sve(vReg dst, pRegGov src, immI_gt_1 size, vReg tmp) %{ instruct vstoremask_narrow_sve(vReg dst, pReg src, immI_gt_1 size, vReg tmp) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (VectorStoreMask src size)); match(Set dst (VectorStoreMask src size));
effect(TEMP_DEF dst, TEMP tmp); effect(TEMP_DEF dst, TEMP tmp);
@ -4778,7 +4778,7 @@ instruct vstoremask_narrow_sve(vReg dst, pRegGov src, immI_gt_1 size, vReg tmp)
// Combined rules for vector mask load when the vector element type is not T_BYTE // Combined rules for vector mask load when the vector element type is not T_BYTE
// VectorLoadMask+LoadVector, and the VectorLoadMask is unpredicated. // VectorLoadMask+LoadVector, and the VectorLoadMask is unpredicated.
instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{ instruct vloadmask_loadV(pReg dst, indirect mem, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1); type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
match(Set dst (VectorLoadMask (LoadVector mem))); match(Set dst (VectorLoadMask (LoadVector mem)));
@ -4800,7 +4800,7 @@ instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
%} %}
// VectorLoadMask+LoadVector, and the VectorLoadMask is predicated. // VectorLoadMask+LoadVector, and the VectorLoadMask is predicated.
instruct vloadmask_loadV_masked(pRegGov dst, indirect mem, pRegGov pg, instruct vloadmask_loadV_masked(pReg dst, indirect mem, pRegGov pg,
vReg tmp, rFlagsReg cr) %{ vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1); type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
@ -4821,7 +4821,7 @@ instruct vloadmask_loadV_masked(pRegGov dst, indirect mem, pRegGov pg,
%} %}
// VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is unpredicated. // VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is unpredicated.
instruct vloadmask_loadVMasked(pRegGov dst, vmemA mem, pRegGov pg, vReg tmp, rFlagsReg cr) %{ instruct vloadmask_loadVMasked(pReg dst, vmemA mem, pRegGov pg, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1); type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
match(Set dst (VectorLoadMask (LoadVectorMasked mem pg))); match(Set dst (VectorLoadMask (LoadVectorMasked mem pg)));
@ -4848,7 +4848,7 @@ instruct vloadmask_loadVMasked(pRegGov dst, vmemA mem, pRegGov pg, vReg tmp, rFl
%} %}
// VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is predicated. // VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is predicated.
instruct vloadmask_loadVMasked_masked(pRegGov dst, vmemA mem, pRegGov pg1, pRegGov pg2, instruct vloadmask_loadVMasked_masked(pReg dst, vmemA mem, pRegGov pg1, pRegGov pg2,
vReg tmp, rFlagsReg cr) %{ vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1); type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
@ -4878,7 +4878,7 @@ instruct vloadmask_loadVMasked_masked(pRegGov dst, vmemA mem, pRegGov pg1, pRegG
// Combined rules for vector mask store when the vector element type is not T_BYTE // Combined rules for vector mask store when the vector element type is not T_BYTE
// StoreVector+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize. // StoreVector+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize.
instruct storeV_vstoremask(indirect mem, pRegGov src, immI_gt_1 esize, vReg tmp) %{ instruct storeV_vstoremask(indirect mem, pReg src, immI_gt_1 esize, vReg tmp) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize); Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
match(Set mem (StoreVector mem (VectorStoreMask src esize))); match(Set mem (StoreVector mem (VectorStoreMask src esize)));
@ -4899,7 +4899,7 @@ instruct storeV_vstoremask(indirect mem, pRegGov src, immI_gt_1 esize, vReg tmp)
%} %}
// StoreVector+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize. // StoreVector+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize.
instruct storeV_vstoremask_masked(indirect mem, pRegGov src, immI_gt_1 esize, instruct storeV_vstoremask_masked(indirect mem, pReg src, immI_gt_1 esize,
vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{ vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize); Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
@ -4921,7 +4921,7 @@ instruct storeV_vstoremask_masked(indirect mem, pRegGov src, immI_gt_1 esize,
%} %}
// StoreVectorMasked+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize. // StoreVectorMasked+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize.
instruct storeVMasked_vstoremask(vmemA mem, pRegGov src, pRegGov pg, immI_gt_1 esize, vReg tmp) %{ instruct storeVMasked_vstoremask(vmemA mem, pReg src, pRegGov pg, immI_gt_1 esize, vReg tmp) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize); Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
match(Set mem (StoreVectorMasked mem (Binary (VectorStoreMask src esize) pg))); match(Set mem (StoreVectorMasked mem (Binary (VectorStoreMask src esize) pg)));
@ -4947,7 +4947,7 @@ instruct storeVMasked_vstoremask(vmemA mem, pRegGov src, pRegGov pg, immI_gt_1 e
%} %}
// StoreVectorMasked+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize. // StoreVectorMasked+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize.
instruct storeVMasked_vstoremask_masked(vmemA mem, pRegGov src, pRegGov pg, immI_gt_1 esize, instruct storeVMasked_vstoremask_masked(vmemA mem, pReg src, pRegGov pg, immI_gt_1 esize,
vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{ vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize); Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
@ -4977,7 +4977,7 @@ instruct storeVMasked_vstoremask_masked(vmemA mem, pRegGov src, pRegGov pg, immI
// vector mask logical ops: and/or/xor/and_not // vector mask logical ops: and/or/xor/and_not
instruct vmask_and(pRegGov pd, pRegGov pn, pRegGov pm) %{ instruct vmask_and(pReg pd, pReg pn, pReg pm) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd (AndVMask pn pm)); match(Set pd (AndVMask pn pm));
format %{ "vmask_and $pd, $pn, $pm" %} format %{ "vmask_and $pd, $pn, $pm" %}
@ -4987,7 +4987,7 @@ instruct vmask_and(pRegGov pd, pRegGov pn, pRegGov pm) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmask_or(pRegGov pd, pRegGov pn, pRegGov pm) %{ instruct vmask_or(pReg pd, pReg pn, pReg pm) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd (OrVMask pn pm)); match(Set pd (OrVMask pn pm));
format %{ "vmask_or $pd, $pn, $pm" %} format %{ "vmask_or $pd, $pn, $pm" %}
@ -4997,7 +4997,7 @@ instruct vmask_or(pRegGov pd, pRegGov pn, pRegGov pm) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmask_xor(pRegGov pd, pRegGov pn, pRegGov pm) %{ instruct vmask_xor(pReg pd, pReg pn, pReg pm) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd (XorVMask pn pm)); match(Set pd (XorVMask pn pm));
format %{ "vmask_xor $pd, $pn, $pm" %} format %{ "vmask_xor $pd, $pn, $pm" %}
@ -5007,7 +5007,7 @@ instruct vmask_xor(pRegGov pd, pRegGov pn, pRegGov pm) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmask_and_notI(pRegGov pd, pRegGov pn, pRegGov pm, immI_M1 m1) %{ instruct vmask_and_notI(pReg pd, pReg pn, pReg pm, immI_M1 m1) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1)))); match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
format %{ "vmask_and_notI $pd, $pn, $pm" %} format %{ "vmask_and_notI $pd, $pn, $pm" %}
@ -5017,7 +5017,7 @@ instruct vmask_and_notI(pRegGov pd, pRegGov pn, pRegGov pm, immI_M1 m1) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmask_and_notL(pRegGov pd, pRegGov pn, pRegGov pm, immL_M1 m1) %{ instruct vmask_and_notL(pReg pd, pReg pn, pReg pm, immL_M1 m1) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1)))); match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
format %{ "vmask_and_notL $pd, $pn, $pm" %} format %{ "vmask_and_notL $pd, $pn, $pm" %}
@ -5045,7 +5045,7 @@ instruct vmaskcmp_neon(vReg dst, vReg src1, vReg src2, immI cond) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmaskcmp_sve(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{ instruct vmaskcmp_sve(pReg dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src1 src2) cond)); match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
effect(KILL cr); effect(KILL cr);
@ -5060,7 +5060,7 @@ instruct vmaskcmp_sve(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, instruct vmaskcmp_masked(pReg dst, vReg src1, vReg src2, immI cond,
pRegGov pg, rFlagsReg cr) %{ pRegGov pg, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg))); match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
@ -5087,7 +5087,7 @@ instruct vmaskcast_same_esize_neon(vReg dst_src) %{
ins_pipe(pipe_class_empty); ins_pipe(pipe_class_empty);
%} %}
instruct vmaskcast_same_esize_sve(pRegGov dst_src) %{ instruct vmaskcast_same_esize_sve(pReg dst_src) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst_src (VectorMaskCast dst_src)); match(Set dst_src (VectorMaskCast dst_src));
@ -5097,7 +5097,7 @@ instruct vmaskcast_same_esize_sve(pRegGov dst_src) %{
ins_pipe(pipe_class_empty); ins_pipe(pipe_class_empty);
%} %}
instruct vmaskcast_extend(pRegGov dst, pReg src) %{ instruct vmaskcast_extend(pReg dst, pReg src) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1))); Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst (VectorMaskCast src)); match(Set dst (VectorMaskCast src));
@ -5114,7 +5114,7 @@ instruct vmaskcast_extend(pRegGov dst, pReg src) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmaskcast_narrow(pRegGov dst, pReg src) %{ instruct vmaskcast_narrow(pReg dst, pReg src) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1))); Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst (VectorMaskCast src)); match(Set dst (VectorMaskCast src));
@ -5133,7 +5133,7 @@ instruct vmaskcast_narrow(pRegGov dst, pReg src) %{
// vector mask reinterpret // vector mask reinterpret
instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{ instruct vmask_reinterpret_same_esize(pReg dst_src) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length(n) == Matcher::vector_length(n->in(1)) && Matcher::vector_length(n) == Matcher::vector_length(n->in(1)) &&
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
@ -5144,7 +5144,7 @@ instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
ins_pipe(pipe_class_empty); ins_pipe(pipe_class_empty);
%} %}
instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{ instruct vmask_reinterpret_diff_esize(pReg dst, pReg src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
@ -5290,7 +5290,7 @@ instruct vmask_firsttrue_sve(iRegINoSp dst, pReg src, pReg ptmp) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmask_firsttrue_masked(iRegINoSp dst, pReg src, pRegGov pg, pReg ptmp) %{ instruct vmask_firsttrue_masked(iRegINoSp dst, pReg src, pReg pg, pReg ptmp) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (VectorMaskFirstTrue src pg)); match(Set dst (VectorMaskFirstTrue src pg));
effect(TEMP ptmp); effect(TEMP ptmp);
@ -5402,7 +5402,7 @@ instruct vmask_tolong_sve(iRegLNoSp dst, pReg src, vReg tmp1, vReg tmp2) %{
// fromlong // fromlong
instruct vmask_fromlong(pRegGov dst, iRegL src, vReg tmp1, vReg tmp2) %{ instruct vmask_fromlong(pReg dst, iRegL src, vReg tmp1, vReg tmp2) %{
match(Set dst (VectorLongToMask src)); match(Set dst (VectorLongToMask src));
effect(TEMP tmp1, TEMP tmp2); effect(TEMP tmp1, TEMP tmp2);
format %{ "vmask_fromlong $dst, $src\t# vector (sve2). KILL $tmp1, $tmp2" %} format %{ "vmask_fromlong $dst, $src\t# vector (sve2). KILL $tmp1, $tmp2" %}
@ -5419,7 +5419,7 @@ instruct vmask_fromlong(pRegGov dst, iRegL src, vReg tmp1, vReg tmp2) %{
// maskAll // maskAll
instruct vmaskAll_immI(pRegGov dst, immI src, rFlagsReg cr) %{ instruct vmaskAll_immI(pReg dst, immI src, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (MaskAll src)); match(Set dst (MaskAll src));
effect(KILL cr); effect(KILL cr);
@ -5437,7 +5437,7 @@ instruct vmaskAll_immI(pRegGov dst, immI src, rFlagsReg cr) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{ instruct vmaskAllI(pReg dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (MaskAll src)); match(Set dst (MaskAll src));
effect(TEMP tmp, KILL cr); effect(TEMP tmp, KILL cr);
@ -5453,7 +5453,7 @@ instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmaskAllI_masked(pRegGov dst, iRegIorL2I src, pRegGov pg, vReg tmp, rFlagsReg cr) %{ instruct vmaskAllI_masked(pReg dst, iRegIorL2I src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (MaskAll src pg)); match(Set dst (MaskAll src pg));
effect(TEMP tmp, KILL cr); effect(TEMP tmp, KILL cr);
@ -5468,7 +5468,7 @@ instruct vmaskAllI_masked(pRegGov dst, iRegIorL2I src, pRegGov pg, vReg tmp, rFl
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmaskAll_immL(pRegGov dst, immL src, rFlagsReg cr) %{ instruct vmaskAll_immL(pReg dst, immL src, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (MaskAll src)); match(Set dst (MaskAll src));
effect(KILL cr); effect(KILL cr);
@ -5486,7 +5486,7 @@ instruct vmaskAll_immL(pRegGov dst, immL src, rFlagsReg cr) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{ instruct vmaskAllL(pReg dst, iRegL src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (MaskAll src)); match(Set dst (MaskAll src));
effect(TEMP tmp, KILL cr); effect(TEMP tmp, KILL cr);
@ -5502,7 +5502,7 @@ instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmaskAllL_masked(pRegGov dst, iRegL src, pRegGov pg, vReg tmp, rFlagsReg cr) %{ instruct vmaskAllL_masked(pReg dst, iRegL src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (MaskAll src pg)); match(Set dst (MaskAll src pg));
effect(TEMP tmp, KILL cr); effect(TEMP tmp, KILL cr);
@ -5519,7 +5519,7 @@ instruct vmaskAllL_masked(pRegGov dst, iRegL src, pRegGov pg, vReg tmp, rFlagsRe
// vetcor mask generation // vetcor mask generation
instruct vmask_gen_I(pRegGov pd, iRegIorL2I src, rFlagsReg cr) %{ instruct vmask_gen_I(pReg pd, iRegIorL2I src, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd (VectorMaskGen (ConvI2L src))); match(Set pd (VectorMaskGen (ConvI2L src)));
effect(KILL cr); effect(KILL cr);
@ -5531,7 +5531,7 @@ instruct vmask_gen_I(pRegGov pd, iRegIorL2I src, rFlagsReg cr) %{
ins_pipe(pipe_class_default); ins_pipe(pipe_class_default);
%} %}
instruct vmask_gen_L(pRegGov pd, iRegL src, rFlagsReg cr) %{ instruct vmask_gen_L(pReg pd, iRegL src, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd (VectorMaskGen src)); match(Set pd (VectorMaskGen src));
effect(KILL cr); effect(KILL cr);
@ -5543,7 +5543,7 @@ instruct vmask_gen_L(pRegGov pd, iRegL src, rFlagsReg cr) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmask_gen_imm(pRegGov pd, immL con, rFlagsReg cr) %{ instruct vmask_gen_imm(pReg pd, immL con, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd (VectorMaskGen con)); match(Set pd (VectorMaskGen con));
effect(KILL cr); effect(KILL cr);
@ -5837,7 +5837,7 @@ instruct vtest_anytrue_neon(iRegINoSp dst, vReg src1, vReg src2, vReg tmp, rFlag
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vtest_anytrue_sve(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr) %{ instruct vtest_anytrue_sve(iRegINoSp dst, pReg src1, pReg src2, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne); static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
match(Set dst (VectorTest src1 src2)); match(Set dst (VectorTest src1 src2));
@ -5871,7 +5871,7 @@ instruct vtest_alltrue_neon(iRegINoSp dst, vReg src1, vReg src2, vReg tmp, rFlag
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vtest_alltrue_sve(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr) %{ instruct vtest_alltrue_sve(iRegINoSp dst, pReg src1, pReg src2, pReg ptmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow); static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
match(Set dst (VectorTest src1 src2)); match(Set dst (VectorTest src1 src2));

66
src/hotspot/cpu/aarch64/aarch64_vector_ad.m4
View File

@ -3093,7 +3093,7 @@ instruct vloadmask_neon(vReg dst, vReg src) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vloadmaskB_sve(pRegGov dst, vReg src, rFlagsReg cr) %{ instruct vloadmaskB_sve(pReg dst, vReg src, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE); predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (VectorLoadMask src)); match(Set dst (VectorLoadMask src));
effect(KILL cr); effect(KILL cr);
@ -3105,7 +3105,7 @@ instruct vloadmaskB_sve(pRegGov dst, vReg src, rFlagsReg cr) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vloadmask_extend_sve(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{ instruct vloadmask_extend_sve(pReg dst, vReg src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE); predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE);
match(Set dst (VectorLoadMask src)); match(Set dst (VectorLoadMask src));
effect(TEMP tmp, KILL cr); effect(TEMP tmp, KILL cr);
@ -3119,7 +3119,7 @@ instruct vloadmask_extend_sve(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vloadmaskB_masked(pRegGov dst, vReg src, pRegGov pg, rFlagsReg cr) %{ instruct vloadmaskB_masked(pReg dst, vReg src, pRegGov pg, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE); predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (VectorLoadMask src pg)); match(Set dst (VectorLoadMask src pg));
effect(KILL cr); effect(KILL cr);
@ -3131,7 +3131,7 @@ instruct vloadmaskB_masked(pRegGov dst, vReg src, pRegGov pg, rFlagsReg cr) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vloadmask_extend_masked(pRegGov dst, vReg src, pRegGov pg, vReg tmp, rFlagsReg cr) %{ instruct vloadmask_extend_masked(pReg dst, vReg src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE); predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE);
match(Set dst (VectorLoadMask src pg)); match(Set dst (VectorLoadMask src pg));
effect(TEMP tmp, KILL cr); effect(TEMP tmp, KILL cr);
@ -3185,7 +3185,7 @@ instruct vstoremask_narrow_neon(vReg dst, vReg src, immI_gt_1 size) %{
// vector store mask - sve // vector store mask - sve
instruct vstoremaskB_sve(vReg dst, pRegGov src, immI_1 size) %{ instruct vstoremaskB_sve(vReg dst, pReg src, immI_1 size) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (VectorStoreMask src size)); match(Set dst (VectorStoreMask src size));
format %{ "vstoremaskB_sve $dst, $src" %} format %{ "vstoremaskB_sve $dst, $src" %}
@ -3195,7 +3195,7 @@ instruct vstoremaskB_sve(vReg dst, pRegGov src, immI_1 size) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vstoremask_narrow_sve(vReg dst, pRegGov src, immI_gt_1 size, vReg tmp) %{ instruct vstoremask_narrow_sve(vReg dst, pReg src, immI_gt_1 size, vReg tmp) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (VectorStoreMask src size)); match(Set dst (VectorStoreMask src size));
effect(TEMP_DEF dst, TEMP tmp); effect(TEMP_DEF dst, TEMP tmp);
@ -3212,7 +3212,7 @@ instruct vstoremask_narrow_sve(vReg dst, pRegGov src, immI_gt_1 size, vReg tmp)
// Combined rules for vector mask load when the vector element type is not T_BYTE // Combined rules for vector mask load when the vector element type is not T_BYTE
// VectorLoadMask+LoadVector, and the VectorLoadMask is unpredicated. // VectorLoadMask+LoadVector, and the VectorLoadMask is unpredicated.
instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{ instruct vloadmask_loadV(pReg dst, indirect mem, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1); type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
match(Set dst (VectorLoadMask (LoadVector mem))); match(Set dst (VectorLoadMask (LoadVector mem)));
@ -3234,7 +3234,7 @@ instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
%} %}
// VectorLoadMask+LoadVector, and the VectorLoadMask is predicated. // VectorLoadMask+LoadVector, and the VectorLoadMask is predicated.
instruct vloadmask_loadV_masked(pRegGov dst, indirect mem, pRegGov pg, instruct vloadmask_loadV_masked(pReg dst, indirect mem, pRegGov pg,
vReg tmp, rFlagsReg cr) %{ vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1); type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
@ -3255,7 +3255,7 @@ instruct vloadmask_loadV_masked(pRegGov dst, indirect mem, pRegGov pg,
%} %}
// VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is unpredicated. // VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is unpredicated.
instruct vloadmask_loadVMasked(pRegGov dst, vmemA mem, pRegGov pg, vReg tmp, rFlagsReg cr) %{ instruct vloadmask_loadVMasked(pReg dst, vmemA mem, pRegGov pg, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1); type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
match(Set dst (VectorLoadMask (LoadVectorMasked mem pg))); match(Set dst (VectorLoadMask (LoadVectorMasked mem pg)));
@ -3282,7 +3282,7 @@ instruct vloadmask_loadVMasked(pRegGov dst, vmemA mem, pRegGov pg, vReg tmp, rFl
%} %}
// VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is predicated. // VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is predicated.
instruct vloadmask_loadVMasked_masked(pRegGov dst, vmemA mem, pRegGov pg1, pRegGov pg2, instruct vloadmask_loadVMasked_masked(pReg dst, vmemA mem, pRegGov pg1, pRegGov pg2,
vReg tmp, rFlagsReg cr) %{ vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
type2aelembytes(Matcher::vector_element_basic_type(n)) > 1); type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
@ -3312,7 +3312,7 @@ instruct vloadmask_loadVMasked_masked(pRegGov dst, vmemA mem, pRegGov pg1, pRegG
// Combined rules for vector mask store when the vector element type is not T_BYTE // Combined rules for vector mask store when the vector element type is not T_BYTE
// StoreVector+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize. // StoreVector+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize.
instruct storeV_vstoremask(indirect mem, pRegGov src, immI_gt_1 esize, vReg tmp) %{ instruct storeV_vstoremask(indirect mem, pReg src, immI_gt_1 esize, vReg tmp) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize); Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
match(Set mem (StoreVector mem (VectorStoreMask src esize))); match(Set mem (StoreVector mem (VectorStoreMask src esize)));
@ -3333,7 +3333,7 @@ instruct storeV_vstoremask(indirect mem, pRegGov src, immI_gt_1 esize, vReg tmp)
%} %}
// StoreVector+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize. // StoreVector+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize.
instruct storeV_vstoremask_masked(indirect mem, pRegGov src, immI_gt_1 esize, instruct storeV_vstoremask_masked(indirect mem, pReg src, immI_gt_1 esize,
vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{ vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize); Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
@ -3355,7 +3355,7 @@ instruct storeV_vstoremask_masked(indirect mem, pRegGov src, immI_gt_1 esize,
%} %}
// StoreVectorMasked+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize. // StoreVectorMasked+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize.
instruct storeVMasked_vstoremask(vmemA mem, pRegGov src, pRegGov pg, immI_gt_1 esize, vReg tmp) %{ instruct storeVMasked_vstoremask(vmemA mem, pReg src, pRegGov pg, immI_gt_1 esize, vReg tmp) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize); Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
match(Set mem (StoreVectorMasked mem (Binary (VectorStoreMask src esize) pg))); match(Set mem (StoreVectorMasked mem (Binary (VectorStoreMask src esize) pg)));
@ -3381,7 +3381,7 @@ instruct storeVMasked_vstoremask(vmemA mem, pRegGov src, pRegGov pg, immI_gt_1 e
%} %}
// StoreVectorMasked+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize. // StoreVectorMasked+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize.
instruct storeVMasked_vstoremask_masked(vmemA mem, pRegGov src, pRegGov pg, immI_gt_1 esize, instruct storeVMasked_vstoremask_masked(vmemA mem, pReg src, pRegGov pg, immI_gt_1 esize,
vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{ vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize); Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
@ -3413,7 +3413,7 @@ dnl
dnl VMASK_BITWISE_OP($1, $2, $3 ) dnl VMASK_BITWISE_OP($1, $2, $3 )
dnl VMASK_BITWISE_OP(type, op_name, insn) dnl VMASK_BITWISE_OP(type, op_name, insn)
define(`VMASK_BITWISE_OP', ` define(`VMASK_BITWISE_OP', `
instruct vmask_$1(pRegGov pd, pRegGov pn, pRegGov pm) %{ instruct vmask_$1(pReg pd, pReg pn, pReg pm) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd ($2 pn pm)); match(Set pd ($2 pn pm));
format %{ "vmask_$1 $pd, $pn, $pm" %} format %{ "vmask_$1 $pd, $pn, $pm" %}
@ -3426,7 +3426,7 @@ dnl
dnl VMASK_AND_NOT($1 ) dnl VMASK_AND_NOT($1 )
dnl VMASK_AND_NOT(type) dnl VMASK_AND_NOT(type)
define(`VMASK_AND_NOT', ` define(`VMASK_AND_NOT', `
instruct vmask_and_not$1(pRegGov pd, pRegGov pn, pRegGov pm, imm$1_M1 m1) %{ instruct vmask_and_not$1(pReg pd, pReg pn, pReg pm, imm$1_M1 m1) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1)))); match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
format %{ "vmask_and_not$1 $pd, $pn, $pm" %} format %{ "vmask_and_not$1 $pd, $pn, $pm" %}
@ -3461,7 +3461,7 @@ instruct vmaskcmp_neon(vReg dst, vReg src1, vReg src2, immI cond) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmaskcmp_sve(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{ instruct vmaskcmp_sve(pReg dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src1 src2) cond)); match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
effect(KILL cr); effect(KILL cr);
@ -3476,7 +3476,7 @@ instruct vmaskcmp_sve(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, instruct vmaskcmp_masked(pReg dst, vReg src1, vReg src2, immI cond,
pRegGov pg, rFlagsReg cr) %{ pRegGov pg, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg))); match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
@ -3503,7 +3503,7 @@ instruct vmaskcast_same_esize_neon(vReg dst_src) %{
ins_pipe(pipe_class_empty); ins_pipe(pipe_class_empty);
%} %}
instruct vmaskcast_same_esize_sve(pRegGov dst_src) %{ instruct vmaskcast_same_esize_sve(pReg dst_src) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst_src (VectorMaskCast dst_src)); match(Set dst_src (VectorMaskCast dst_src));
@ -3513,7 +3513,7 @@ instruct vmaskcast_same_esize_sve(pRegGov dst_src) %{
ins_pipe(pipe_class_empty); ins_pipe(pipe_class_empty);
%} %}
instruct vmaskcast_extend(pRegGov dst, pReg src) %{ instruct vmaskcast_extend(pReg dst, pReg src) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1))); Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst (VectorMaskCast src)); match(Set dst (VectorMaskCast src));
@ -3530,7 +3530,7 @@ instruct vmaskcast_extend(pRegGov dst, pReg src) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmaskcast_narrow(pRegGov dst, pReg src) %{ instruct vmaskcast_narrow(pReg dst, pReg src) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1))); Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)));
match(Set dst (VectorMaskCast src)); match(Set dst (VectorMaskCast src));
@ -3549,7 +3549,7 @@ instruct vmaskcast_narrow(pRegGov dst, pReg src) %{
// vector mask reinterpret // vector mask reinterpret
instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{ instruct vmask_reinterpret_same_esize(pReg dst_src) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length(n) == Matcher::vector_length(n->in(1)) && Matcher::vector_length(n) == Matcher::vector_length(n->in(1)) &&
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
@ -3560,7 +3560,7 @@ instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
ins_pipe(pipe_class_empty); ins_pipe(pipe_class_empty);
%} %}
instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{ instruct vmask_reinterpret_diff_esize(pReg dst, pReg src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) && Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
@ -3706,7 +3706,7 @@ instruct vmask_firsttrue_sve(iRegINoSp dst, pReg src, pReg ptmp) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmask_firsttrue_masked(iRegINoSp dst, pReg src, pRegGov pg, pReg ptmp) %{ instruct vmask_firsttrue_masked(iRegINoSp dst, pReg src, pReg pg, pReg ptmp) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (VectorMaskFirstTrue src pg)); match(Set dst (VectorMaskFirstTrue src pg));
effect(TEMP ptmp); effect(TEMP ptmp);
@ -3818,7 +3818,7 @@ instruct vmask_tolong_sve(iRegLNoSp dst, pReg src, vReg tmp1, vReg tmp2) %{
// fromlong // fromlong
instruct vmask_fromlong(pRegGov dst, iRegL src, vReg tmp1, vReg tmp2) %{ instruct vmask_fromlong(pReg dst, iRegL src, vReg tmp1, vReg tmp2) %{
match(Set dst (VectorLongToMask src)); match(Set dst (VectorLongToMask src));
effect(TEMP tmp1, TEMP tmp2); effect(TEMP tmp1, TEMP tmp2);
format %{ "vmask_fromlong $dst, $src\t# vector (sve2). KILL $tmp1, $tmp2" %} format %{ "vmask_fromlong $dst, $src\t# vector (sve2). KILL $tmp1, $tmp2" %}
@ -3838,7 +3838,7 @@ dnl
dnl VMASKALL_IMM($1, $2 ) dnl VMASKALL_IMM($1, $2 )
dnl VMASKALL_IMM(type, var_type) dnl VMASKALL_IMM(type, var_type)
define(`VMASKALL_IMM', ` define(`VMASKALL_IMM', `
instruct vmaskAll_imm$1(pRegGov dst, imm$1 src, rFlagsReg cr) %{ instruct vmaskAll_imm$1(pReg dst, imm$1 src, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (MaskAll src)); match(Set dst (MaskAll src));
effect(KILL cr); effect(KILL cr);
@ -3859,7 +3859,7 @@ dnl
dnl VMASKALL($1, $2 ) dnl VMASKALL($1, $2 )
dnl VMASKALL(type, arg_type) dnl VMASKALL(type, arg_type)
define(`VMASKALL', ` define(`VMASKALL', `
instruct vmaskAll$1(pRegGov dst, $2 src, vReg tmp, rFlagsReg cr) %{ instruct vmaskAll$1(pReg dst, $2 src, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (MaskAll src)); match(Set dst (MaskAll src));
effect(TEMP tmp, KILL cr); effect(TEMP tmp, KILL cr);
@ -3878,7 +3878,7 @@ dnl
dnl VMASKALL_PREDICATE($1, $2 ) dnl VMASKALL_PREDICATE($1, $2 )
dnl VMASKALL_PREDICATE(type, arg_type) dnl VMASKALL_PREDICATE(type, arg_type)
define(`VMASKALL_PREDICATE', ` define(`VMASKALL_PREDICATE', `
instruct vmaskAll$1_masked(pRegGov dst, $2 src, pRegGov pg, vReg tmp, rFlagsReg cr) %{ instruct vmaskAll$1_masked(pReg dst, $2 src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set dst (MaskAll src pg)); match(Set dst (MaskAll src pg));
effect(TEMP tmp, KILL cr); effect(TEMP tmp, KILL cr);
@ -3902,7 +3902,7 @@ VMASKALL_PREDICATE(L, iRegL)
// vetcor mask generation // vetcor mask generation
instruct vmask_gen_I(pRegGov pd, iRegIorL2I src, rFlagsReg cr) %{ instruct vmask_gen_I(pReg pd, iRegIorL2I src, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd (VectorMaskGen (ConvI2L src))); match(Set pd (VectorMaskGen (ConvI2L src)));
effect(KILL cr); effect(KILL cr);
@ -3914,7 +3914,7 @@ instruct vmask_gen_I(pRegGov pd, iRegIorL2I src, rFlagsReg cr) %{
ins_pipe(pipe_class_default); ins_pipe(pipe_class_default);
%} %}
instruct vmask_gen_L(pRegGov pd, iRegL src, rFlagsReg cr) %{ instruct vmask_gen_L(pReg pd, iRegL src, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd (VectorMaskGen src)); match(Set pd (VectorMaskGen src));
effect(KILL cr); effect(KILL cr);
@ -3926,7 +3926,7 @@ instruct vmask_gen_L(pRegGov pd, iRegL src, rFlagsReg cr) %{
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vmask_gen_imm(pRegGov pd, immL con, rFlagsReg cr) %{ instruct vmask_gen_imm(pReg pd, immL con, rFlagsReg cr) %{
predicate(UseSVE > 0); predicate(UseSVE > 0);
match(Set pd (VectorMaskGen con)); match(Set pd (VectorMaskGen con));
effect(KILL cr); effect(KILL cr);
@ -4209,7 +4209,7 @@ instruct vtest_anytrue_neon(iRegINoSp dst, vReg src1, vReg src2, vReg tmp, rFlag
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vtest_anytrue_sve(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr) %{ instruct vtest_anytrue_sve(iRegINoSp dst, pReg src1, pReg src2, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne); static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
match(Set dst (VectorTest src1 src2)); match(Set dst (VectorTest src1 src2));
@ -4243,7 +4243,7 @@ instruct vtest_alltrue_neon(iRegINoSp dst, vReg src1, vReg src2, vReg tmp, rFlag
ins_pipe(pipe_slow); ins_pipe(pipe_slow);
%} %}
instruct vtest_alltrue_sve(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr) %{ instruct vtest_alltrue_sve(iRegINoSp dst, pReg src1, pReg src2, pReg ptmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && predicate(UseSVE > 0 &&
static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow); static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
match(Set dst (VectorTest src1 src2)); match(Set dst (VectorTest src1 src2));

12
src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
View File

@ -2394,9 +2394,9 @@ int MacroAssembler::push_p(unsigned int bitset, Register stack) {
return 0; return 0;
} }
unsigned char regs[PRegister::number_of_saved_registers]; unsigned char regs[PRegister::number_of_registers];
int count = 0; int count = 0;
for (int reg = 0; reg < PRegister::number_of_saved_registers; reg++) { for (int reg = 0; reg < PRegister::number_of_registers; reg++) {
if (1 & bitset) if (1 & bitset)
regs[count++] = reg; regs[count++] = reg;
bitset >>= 1; bitset >>= 1;
@ -2431,9 +2431,9 @@ int MacroAssembler::pop_p(unsigned int bitset, Register stack) {
return 0; return 0;
} }
unsigned char regs[PRegister::number_of_saved_registers]; unsigned char regs[PRegister::number_of_registers];
int count = 0; int count = 0;
for (int reg = 0; reg < PRegister::number_of_saved_registers; reg++) { for (int reg = 0; reg < PRegister::number_of_registers; reg++) {
if (1 & bitset) if (1 & bitset)
regs[count++] = reg; regs[count++] = reg;
bitset >>= 1; bitset >>= 1;
@ -2937,7 +2937,7 @@ void MacroAssembler::push_CPU_state(bool save_vectors, bool use_sve,
} }
if (save_vectors && use_sve && total_predicate_in_bytes > 0) { if (save_vectors && use_sve && total_predicate_in_bytes > 0) {
sub(sp, sp, total_predicate_in_bytes); sub(sp, sp, total_predicate_in_bytes);
for (int i = 0; i < PRegister::number_of_saved_registers; i++) { for (int i = 0; i < PRegister::number_of_registers; i++) {
sve_str(as_PRegister(i), Address(sp, i)); sve_str(as_PRegister(i), Address(sp, i));
} }
} }
@ -2946,7 +2946,7 @@ void MacroAssembler::push_CPU_state(bool save_vectors, bool use_sve,
void MacroAssembler::pop_CPU_state(bool restore_vectors, bool use_sve, void MacroAssembler::pop_CPU_state(bool restore_vectors, bool use_sve,
int sve_vector_size_in_bytes, int total_predicate_in_bytes) { int sve_vector_size_in_bytes, int total_predicate_in_bytes) {
if (restore_vectors && use_sve && total_predicate_in_bytes > 0) { if (restore_vectors && use_sve && total_predicate_in_bytes > 0) {
for (int i = PRegister::number_of_saved_registers - 1; i >= 0; i--) { for (int i = PRegister::number_of_registers - 1; i >= 0; i--) {
sve_ldr(as_PRegister(i), Address(sp, i)); sve_ldr(as_PRegister(i), Address(sp, i));
} }
add(sp, sp, total_predicate_in_bytes); add(sp, sp, total_predicate_in_bytes);

5
src/hotspot/cpu/aarch64/register_aarch64.hpp
View File

@ -296,11 +296,6 @@ public:
enum { enum {
number_of_registers = 16, number_of_registers = 16,
number_of_governing_registers = 8, number_of_governing_registers = 8,
// p0-p7 are governing predicates for load/store and arithmetic, but p7 is
// preserved as an all-true predicate in OpenJDK. And since we don't support
// non-governing predicate registers allocation for non-temp register, the
// predicate registers to be saved are p0-p6.
number_of_saved_registers = number_of_governing_registers - 1,
max_slots_per_register = 1 max_slots_per_register = 1
}; };

15
src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
View File

@ -161,11 +161,8 @@ int RegisterSaver::v0_offset_in_bytes() {
int RegisterSaver::total_sve_predicate_in_bytes() { int RegisterSaver::total_sve_predicate_in_bytes() {
#ifdef COMPILER2 #ifdef COMPILER2
if (_save_vectors && Matcher::supports_scalable_vector()) { if (_save_vectors && Matcher::supports_scalable_vector()) {
// The number of total predicate bytes is unlikely to be a multiple return (Matcher::scalable_vector_reg_size(T_BYTE) >> LogBitsPerByte) *
// of 16 bytes so we manually align it up. PRegister::number_of_registers;
return align_up(Matcher::scalable_predicate_reg_slots() *
VMRegImpl::stack_slot_size *
PRegister::number_of_saved_registers, 16);
} }
#endif #endif
return 0; return 0;
@ -251,14 +248,6 @@ OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_
oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), r->as_VMReg()); oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), r->as_VMReg());
} }
if (_save_vectors && use_sve) {
for (int i = 0; i < PRegister::number_of_saved_registers; i++) {
PRegister r = as_PRegister(i);
int sp_offset = sve_predicate_size_in_slots * i;
oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), r->as_VMReg());
}
}
return oop_map; return oop_map;
} }