stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8282926: AArch64: Optimize out WHILELO with PTRUE

Browse Source 
Reviewed-by: njian, ngasson
This commit is contained in:
Eric Liu 2022-03-30 14:57:50 +00:00 committed by Nick Gasson
parent 720e751f35
commit e8e9b8dc89
3 changed files with 103 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
src/hotspot/cpu/aarch64/aarch64_sve.ad
View File

@ -305,12 +305,12 @@ instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
match(Set dst (LoadVector mem));
effect(TEMP pgtmp, KILL cr);
ins_cost(6 * SVE_COST);
format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
format %{ "sve_ptrue $pgtmp, vector_length\n\t"
"sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
FloatRegister dst_reg = as_FloatRegister($dst$$reg);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
@ -325,12 +325,12 @@ instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
match(Set mem (StoreVector mem src));
effect(TEMP pgtmp, KILL cr);
ins_cost(5 * SVE_COST);
format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
format %{ "sve_ptrue $pgtmp, vector_length\n\t"
"sve_str $src, $pgtmp, $mem\t# store vector partial" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
__ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this, $src));
FloatRegister src_reg = as_FloatRegister($src$$reg);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
@ -365,8 +365,8 @@ instruct loadV_masked_partial(vReg dst, vmemA mem, pRegGov pg, pRegGov pgtmp, rF
format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated partial (sve)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
__ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
@ -400,8 +400,8 @@ instruct storeV_masked_partial(vReg src, vmemA mem, pRegGov pg, pRegGov pgtmp, r
format %{ "sve_str $mem, $pg, $src\t# store vector predicated partial (sve)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
__ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this, $src));
__ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
@ -438,7 +438,7 @@ instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
effect(TEMP tmp, KILL cr);
ins_cost(3 * SVE_COST);
format %{ "sve_dup $tmp, $src\n\t"
"sve_ptrue_lanecnt $dst\n\t"
"sve_ptrue $dst, vector_length\n\t"
"sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
@ -481,7 +481,7 @@ instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{
effect(TEMP tmp, KILL cr);
ins_cost(3 * SVE_COST);
format %{ "sve_dup $tmp, $src\n\t"
"sve_ptrue_lanecnt $dst\n\t"
"sve_ptrue $dst, vector_length\n\t"
"sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (D)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
@ -591,7 +591,7 @@ instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
length_in_bytes_src : length_in_bytes_dst;
assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
"invalid vector length");
__ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
__ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
__ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
__ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
@ -2201,7 +2201,7 @@ instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov p
// expected vector element type. Convert the vector to predicate.
BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
@ -2248,7 +2248,7 @@ instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
__ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
@ -2326,8 +2326,7 @@ instruct reduce_addI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vt
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -2344,8 +2343,7 @@ instruct reduce_addL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
ins_cost(2 * SVE_COST);
format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -2361,8 +2359,7 @@ instruct reduce_addF_partial(vRegF src1_dst, vReg src2, pRegGov ptmp, rFlagsReg
effect(TEMP ptmp, KILL cr);
format %{ "sve_reduce_addF $src1_dst, $src1_dst, $src2\t# addF reduction partial (sve) (S)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
__ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
%}
@ -2377,8 +2374,7 @@ instruct reduce_addD_partial(vRegD src1_dst, vReg src2, pRegGov ptmp, rFlagsReg
effect(TEMP ptmp, KILL cr);
format %{ "sve_reduce_addD $src1_dst, $src1_dst, $src2\t# addD reduction partial (sve) (D)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
%}
@ -2455,8 +2451,7 @@ instruct reduce_addI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, v
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
@ -2475,8 +2470,7 @@ instruct reduce_addL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_addL $dst, $src1, $pg, $src2\t# addL reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
@ -2494,8 +2488,7 @@ instruct reduce_addF_masked_partial(vRegF src1_dst, vReg src2, pRegGov pg, pRegG
ins_cost(SVE_COST);
format %{ "sve_reduce_addF $src1_dst, $pg, $src2\t# addF reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
@ -2512,8 +2505,7 @@ instruct reduce_addD_masked_partial(vRegD src1_dst, vReg src2, pRegGov pg, pRegG
ins_cost(SVE_COST);
format %{ "sve_reduce_addD $src1_dst, $pg, $src2\t# addD reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
@ -2569,8 +2561,7 @@ instruct reduce_andI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vt
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -2588,8 +2579,7 @@ instruct reduce_andL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
ins_cost(2 * SVE_COST);
format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -2644,8 +2634,7 @@ instruct reduce_andI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, v
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
@ -2665,8 +2654,7 @@ instruct reduce_andL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_andL $dst, $src1, $pg, $src2\t# andL reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
@ -2723,8 +2711,7 @@ instruct reduce_orI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtm
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -2742,8 +2729,7 @@ instruct reduce_orL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
ins_cost(2 * SVE_COST);
format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -2798,8 +2784,7 @@ instruct reduce_orI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vR
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
@ -2819,8 +2804,7 @@ instruct reduce_orL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD v
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_orL $dst, $src1, $pg, $src2\t# orL reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
@ -2877,8 +2861,7 @@ instruct reduce_eorI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vt
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -2896,8 +2879,7 @@ instruct reduce_eorL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
ins_cost(2 * SVE_COST);
format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -2952,8 +2934,7 @@ instruct reduce_eorI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, v
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
@ -2973,8 +2954,7 @@ instruct reduce_eorL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_eorL $dst, $src1, $pg, $src2\t# eorL reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
@ -3033,8 +3013,7 @@ instruct reduce_maxI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vt
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -3052,8 +3031,7 @@ instruct reduce_maxL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
ins_cost(2 * SVE_COST);
format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -3086,8 +3064,7 @@ instruct reduce_maxF_partial(vRegF dst, vRegF src1, vReg src2,
effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
__ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
__ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
%}
@ -3119,8 +3096,7 @@ instruct reduce_maxD_partial(vRegD dst, vRegD src1, vReg src2,
effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
__ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
%}
@ -3178,8 +3154,7 @@ instruct reduce_maxI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, v
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
@ -3199,8 +3174,7 @@ instruct reduce_maxL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_maxL $dst, $src1, $pg, $src2\t# maxL reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
@ -3250,8 +3224,7 @@ instruct reduce_maxF_masked_partial(vRegF dst, vRegF src1, vReg src2, pRegGov pg
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_fmaxv(as_FloatRegister($dst$$reg), __ S,
@ -3271,8 +3244,7 @@ instruct reduce_maxD_masked_partial(vRegD dst, vRegD src1, vReg src2, pRegGov pg
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_fmaxv(as_FloatRegister($dst$$reg), __ D,
@ -3331,8 +3303,7 @@ instruct reduce_minI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vt
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -3350,8 +3321,7 @@ instruct reduce_minL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
ins_cost(2 * SVE_COST);
format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -3384,8 +3354,7 @@ instruct reduce_minF_partial(vRegF dst, vRegF src1, vReg src2,
effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
__ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
__ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
%}
@ -3417,8 +3386,7 @@ instruct reduce_minD_partial(vRegD dst, vRegD src1, vReg src2,
effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
__ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
%}
@ -3476,8 +3444,7 @@ instruct reduce_minI_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, v
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
@ -3497,8 +3464,7 @@ instruct reduce_minL_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_minL $dst, $src1, $pg, $src2\t# minL reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
@ -3548,8 +3514,7 @@ instruct reduce_minF_masked_partial(vRegF dst, vRegF src1, vReg src2, pRegGov pg
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_fminv(as_FloatRegister($dst$$reg), __ S,
@ -3569,8 +3534,7 @@ instruct reduce_minD_masked_partial(vRegD dst, vRegD src1, vReg src2, pRegGov pg
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_fminv(as_FloatRegister($dst$$reg), __ D,
@ -5128,8 +5092,7 @@ instruct vtest_alltrue_partial(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGo
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src1);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
Matcher::vector_length(this, $src1));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src1));
__ sve_eors(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($src1$$reg), as_PRegister($src2$$reg));
__ csetw(as_Register($dst$$reg), Assembler::EQ);
@ -5149,8 +5112,7 @@ instruct vtest_anytrue_partial(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGo
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src1);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
Matcher::vector_length(this, $src1));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src1));
__ sve_ands(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($src1$$reg), as_PRegister($src2$$reg));
__ csetw(as_Register($dst$$reg), Assembler::NE);
@ -5385,7 +5347,7 @@ instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsR
ins_cost(2 * SVE_COST + INSN_COST);
format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
__ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
%}
@ -5402,8 +5364,7 @@ instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsR
ins_cost(3 * SVE_COST + INSN_COST);
format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
@ -5456,8 +5417,7 @@ instruct gatherI_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pR
ins_cost(3 * SVE_COST);
format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (S)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
@ -5476,7 +5436,7 @@ instruct gatherL_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pR
ins_cost(4 * SVE_COST);
format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (D)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
@ -5531,8 +5491,7 @@ instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlags
ins_cost(2 * SVE_COST + INSN_COST);
format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src));
__ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
%}
@ -5549,8 +5508,7 @@ instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlags
ins_cost(3 * SVE_COST + INSN_COST);
format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src));
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
@ -5603,8 +5561,7 @@ instruct scatterI_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, p
ins_cost(3 * SVE_COST);
format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (S)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
@ -5623,8 +5580,7 @@ instruct scatterL_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, p
ins_cost(4 * SVE_COST);
format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (D)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
@ -5742,7 +5698,7 @@ instruct vmask_truecount_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg c
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
__ sve_cntp($dst$$Register, size, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
%}
ins_pipe(pipe_slow);
@ -5758,7 +5714,7 @@ instruct vmask_firsttrue_partial(iRegINoSp dst, pReg src, pReg ptmp1, pReg ptmp2
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp1$$reg), size,
__ sve_ptrue_lanecnt(as_PRegister($ptmp1$$reg), size,
Matcher::vector_length(this, $src));
__ sve_brkb(as_PRegister($ptmp2$$reg), as_PRegister($ptmp1$$reg), as_PRegister($src$$reg), false);
__ sve_cntp($dst$$Register, size, as_PRegister($ptmp1$$reg), as_PRegister($ptmp2$$reg));
@ -5776,7 +5732,7 @@ instruct vmask_lasttrue_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
__ sve_and(as_PRegister($ptmp$$reg), ptrue, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
__ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg));
%}

95
src/hotspot/cpu/aarch64/aarch64_sve_ad.m4
View File

@ -242,12 +242,12 @@ instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
match(Set dst (LoadVector mem));
effect(TEMP pgtmp, KILL cr);
ins_cost(6 * SVE_COST);
format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
format %{ "sve_ptrue $pgtmp, vector_length\n\t"
"sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
FloatRegister dst_reg = as_FloatRegister($dst$$reg);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
@ -262,12 +262,12 @@ instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
match(Set mem (StoreVector mem src));
effect(TEMP pgtmp, KILL cr);
ins_cost(5 * SVE_COST);
format %{ "sve_whilelo_zr_imm $pgtmp, vector_length\n\t"
format %{ "sve_ptrue $pgtmp, vector_length\n\t"
"sve_str $src, $pgtmp, $mem\t# store vector partial" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
__ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this, $src));
FloatRegister src_reg = as_FloatRegister($src$$reg);
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
@ -302,8 +302,8 @@ instruct loadV_masked_partial(vReg dst, vmemA mem, pRegGov pg, pRegGov pgtmp, rF
format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated partial (sve)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this));
__ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
@ -337,8 +337,8 @@ instruct storeV_masked_partial(vReg src, vmemA mem, pRegGov pg, pRegGov pgtmp, r
format %{ "sve_str $mem, $pg, $src\t# store vector predicated partial (sve)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
__ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
Matcher::vector_length(this, $src));
__ sve_and(as_PRegister($pgtmp$$reg), as_PRegister($pgtmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
@ -380,7 +380,7 @@ instruct vmaskAll$1(pRegGov dst, ifelse($1, `I', iRegIorL2I, iRegL) src, vReg tm
effect(TEMP tmp, KILL cr);
ins_cost(3 * SVE_COST);
format %{ "sve_dup $tmp, $src\n\t"
"sve_ptrue_lanecnt $dst\n\t"
"sve_ptrue $dst, vector_length\n\t"
"sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) ($2)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
@ -473,7 +473,7 @@ instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
length_in_bytes_src : length_in_bytes_dst;
assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
"invalid vector length");
__ sve_whilelo_zr_imm(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
__ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
__ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
__ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
@ -1262,7 +1262,7 @@ instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov p
// expected vector element type. Convert the vector to predicate.
BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
@ -1309,7 +1309,7 @@ instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
__ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
@ -1381,8 +1381,7 @@ instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtm
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -1406,8 +1405,7 @@ instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
ins_cost(2 * SVE_COST);
format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -1443,8 +1441,7 @@ instruct reduce_$1_partial($3 src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %
effect(TEMP ptmp, KILL cr);
format %{ "sve_reduce_$1 $src1_dst, $src1_dst, $src2\t# $1 reduction partial (sve) ($4)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ $4, Matcher::vector_length(this, $src2));
__ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
%}
@ -1515,8 +1512,7 @@ instruct reduce_$1I_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vR
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
@ -1542,8 +1538,7 @@ instruct reduce_$1L_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD v
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
@ -1580,8 +1575,7 @@ instruct reduce_$1_masked_partial($3 src1_dst, vReg src2, pRegGov pg, pRegGov pt
ins_cost(SVE_COST);
format %{ "sve_reduce_$1 $src1_dst, $pg, $src2\t# $1 reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ $4, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_fadda(as_FloatRegister($src1_dst$$reg), __ $4,
@ -1704,8 +1698,7 @@ instruct reduce_$1I_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtm
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -1726,8 +1719,7 @@ instruct reduce_$1L_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
ins_cost(2 * SVE_COST);
format %{ "sve_reduce_$1L $dst, $src1, $src2\t# $1L reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
$src1$$Register, as_FloatRegister($src2$$reg),
as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
@ -1793,8 +1785,7 @@ instruct reduce_$1I_masked_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vR
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src2);
Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), variant,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
@ -1817,8 +1808,7 @@ instruct reduce_$1L_masked_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD v
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_$1L $dst, $src1, $pg, $src2\t# $1L reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
@ -1859,8 +1849,7 @@ instruct reduce_$1$2_partial($5 dst, $5 src1, vReg src2,
effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
format %{ "sve_reduce_$1$2 $dst, $src1, $src2\t# $1$2 reduction partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ $4, Matcher::vector_length(this, $src2));
__ sve_f$1v(as_FloatRegister($dst$$reg), __ $4, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
__ f`$1'translit($4, `SD', `sd')(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
%}
@ -1898,8 +1887,7 @@ instruct reduce_$1$2_masked_partial($5 dst, $5 src1, vReg src2, pRegGov pg,
ins_cost(3 * SVE_COST);
format %{ "sve_reduce_$1$2 $dst, $src1, $pg, $src2\t# $1$2 reduction predicated partial (sve)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ $4,
Matcher::vector_length(this, $src2));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ $4, Matcher::vector_length(this, $src2));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_f$1v(as_FloatRegister($dst$$reg), __ $4,
@ -2603,8 +2591,7 @@ instruct vtest_$1_partial`'(iRegINoSp dst, pRegGov src1, pRegGov src2, pRegGov p
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src1);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size,
Matcher::vector_length(this, $src1));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src1));
__ $3(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($src1$$reg), as_PRegister($src2$$reg));
__ csetw(as_Register($dst$$reg), Assembler::$4);
@ -2826,7 +2813,7 @@ instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsR
ins_cost(2 * SVE_COST + INSN_COST);
format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
__ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
%}
@ -2843,8 +2830,7 @@ instruct gatherL_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsR
ins_cost(3 * SVE_COST + INSN_COST);
format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
@ -2897,8 +2883,7 @@ instruct gatherI_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pR
ins_cost(3 * SVE_COST);
format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (S)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
@ -2917,7 +2902,7 @@ instruct gatherL_masked_partial(vReg dst, indirect mem, vReg idx, pRegGov pg, pR
ins_cost(4 * SVE_COST);
format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated partial (D)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
@ -2972,8 +2957,7 @@ instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlags
ins_cost(2 * SVE_COST + INSN_COST);
format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src));
__ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
%}
@ -2990,8 +2974,7 @@ instruct scatterL_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlags
ins_cost(3 * SVE_COST + INSN_COST);
format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src));
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
__ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
as_Register($mem$$base), as_FloatRegister($idx$$reg));
@ -3044,8 +3027,7 @@ instruct scatterI_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, p
ins_cost(3 * SVE_COST);
format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (S)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ S,
Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
@ -3064,8 +3046,7 @@ instruct scatterL_masked_partial(indirect mem, vReg src, vReg idx, pRegGov pg, p
ins_cost(4 * SVE_COST);
format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated partial (D)" %}
ins_encode %{
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), __ D,
Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src));
__ sve_and(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg),
as_PRegister($pg$$reg), as_PRegister($pg$$reg));
__ sve_uunpklo(as_FloatRegister($idx$$reg), __ D, as_FloatRegister($idx$$reg));
@ -3169,7 +3150,7 @@ instruct vmask_truecount_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg c
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
__ sve_cntp($dst$$Register, size, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
%}
ins_pipe(pipe_slow);
@ -3185,7 +3166,7 @@ instruct vmask_firsttrue_partial(iRegINoSp dst, pReg src, pReg ptmp1, pReg ptmp2
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp1$$reg), size,
__ sve_ptrue_lanecnt(as_PRegister($ptmp1$$reg), size,
Matcher::vector_length(this, $src));
__ sve_brkb(as_PRegister($ptmp2$$reg), as_PRegister($ptmp1$$reg), as_PRegister($src$$reg), false);
__ sve_cntp($dst$$Register, size, as_PRegister($ptmp1$$reg), as_PRegister($ptmp2$$reg));
@ -3203,7 +3184,7 @@ instruct vmask_lasttrue_partial(iRegINoSp dst, pReg src, pReg ptmp, rFlagsReg cr
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_whilelo_zr_imm(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
__ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
__ sve_and(as_PRegister($ptmp$$reg), ptrue, as_PRegister($ptmp$$reg), as_PRegister($src$$reg));
__ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg));
%}

10
src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -91,14 +91,6 @@
// in the range of [0, lane_cnt), or to false otherwise.
void sve_ptrue_lanecnt(PRegister dst, SIMD_RegVariant size, int lane_cnt);
// Generate predicate through whilelo, by comparing ZR with an unsigned
// immediate. rscratch1 will be clobbered.
inline void sve_whilelo_zr_imm(PRegister pd, SIMD_RegVariant size, uint imm) {
assert(UseSVE > 0, "not supported");
mov(rscratch1, imm);
sve_whilelo(pd, size, zr, rscratch1);
}
// Extract a scalar element from an sve vector at position 'idx'.
// rscratch1 will be clobbered.
// T could be FloatRegister or Register.