8283435: AArch64: [vectorapi] Optimize SVE lane/withLane operations for 64/128-bit vector sizes

Reviewed-by: njian, ngasson
This commit is contained in:
Eric Liu 2022-04-27 01:22:27 +00:00 committed by Pengfei Li
parent 16ebe40a1b
commit d3ea4b7bb4
9 changed files with 751 additions and 467 deletions

@ -1750,104 +1750,42 @@ instruct reduce_eor2L(iRegLNoSp dst, iRegL isrc, vecX vsrc, iRegLNoSp tmp)
// ------------------------------ Vector insert ---------------------------------
instruct insert8B(vecD dst, vecD src, iRegIorL2I val, immI idx)
instruct insertID(vecD dst, vecD src, iRegIorL2I val, immI idx)
%{
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
predicate((n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
n->bottom_type()->is_vect()->element_basic_type() == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(INSN_COST);
ins_cost(2 * INSN_COST);
format %{ "orr $dst, T8B, $src, $src\n\t"
"mov $dst, B, $idx, $val\t# insert into vector(8B)" %}
"mov $dst, B/H/S, $idx, $val\t# insert into vector (B/H/S)" %}
ins_encode %{
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T8B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ mov(as_FloatRegister($dst$$reg), __ B, $idx$$constant, $val$$Register);
__ mov(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(Matcher::vector_element_basic_type(this)),
$idx$$constant, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct insert16B(vecX dst, vecX src, iRegIorL2I val, immI idx)
instruct insertIX(vecX dst, vecX src, iRegIorL2I val, immI idx)
%{
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
predicate((n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
n->bottom_type()->is_vect()->element_basic_type() == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(INSN_COST);
ins_cost(2 * INSN_COST);
format %{ "orr $dst, T16B, $src, $src\n\t"
"mov $dst, B, $idx, $val\t# insert into vector(16B)" %}
"mov $dst, B/H/S, $idx, $val\t# insert into vector (B/H/S)" %}
ins_encode %{
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ mov(as_FloatRegister($dst$$reg), __ B, $idx$$constant, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct insert4S(vecD dst, vecD src, iRegIorL2I val, immI idx)
%{
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(INSN_COST);
format %{ "orr $dst, T8B, $src, $src\n\t"
"mov $dst, H, $idx, $val\t# insert into vector(4S)" %}
ins_encode %{
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T8B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ mov(as_FloatRegister($dst$$reg), __ H, $idx$$constant, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct insert8S(vecX dst, vecX src, iRegIorL2I val, immI idx)
%{
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(INSN_COST);
format %{ "orr $dst, T16B, $src, $src\n\t"
"mov $dst, H, $idx, $val\t# insert into vector(8S)" %}
ins_encode %{
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ mov(as_FloatRegister($dst$$reg), __ H, $idx$$constant, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct insert2I(vecD dst, vecD src, iRegIorL2I val, immI idx)
%{
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_INT);
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(INSN_COST);
format %{ "orr $dst, T8B, $src, $src\n\t"
"mov $dst, S, $idx, $val\t# insert into vector(2I)" %}
ins_encode %{
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T8B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ mov(as_FloatRegister($dst$$reg), __ S, $idx$$constant, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct insert4I(vecX dst, vecX src, iRegIorL2I val, immI idx)
%{
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_INT);
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(INSN_COST);
format %{ "orr $dst, T16B, $src, $src\n\t"
"mov $dst, S, $idx, $val\t# insert into vector(4I)" %}
ins_encode %{
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ mov(as_FloatRegister($dst$$reg), __ S, $idx$$constant, $val$$Register);
__ mov(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(Matcher::vector_element_basic_type(this)),
$idx$$constant, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
@ -1856,15 +1794,16 @@ instruct insert2L(vecX dst, vecX src, iRegL val, immI idx)
%{
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(INSN_COST);
ins_cost(2 * INSN_COST);
format %{ "orr $dst, T16B, $src, $src\n\t"
"mov $dst, D, $idx, $val\t# insert into vector(2L)" %}
"mov $dst, D, $idx, $val\t# insert into vector (D)" %}
ins_encode %{
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ mov(as_FloatRegister($dst$$reg), __ D, $idx$$constant, $val$$Register);
__ mov(as_FloatRegister($dst$$reg), __ D,
$idx$$constant, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
@ -1873,7 +1812,7 @@ instruct insert2F(vecD dst, vecD src, vRegF val, immI idx)
%{
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(INSN_COST);
ins_cost(2 * INSN_COST);
effect(TEMP_DEF dst);
format %{ "orr $dst, T8B, $src, $src\n\t"
"ins $dst, S, $val, $idx, 0\t# insert into vector(2F)" %}
@ -1890,7 +1829,7 @@ instruct insert4F(vecX dst, vecX src, vRegF val, immI idx)
%{
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(INSN_COST);
ins_cost(2 * INSN_COST);
effect(TEMP_DEF dst);
format %{ "orr $dst, T16B, $src, $src\n\t"
"ins $dst, S, $val, $idx, 0\t# insert into vector(4F)" %}
@ -1907,7 +1846,7 @@ instruct insert2D(vecX dst, vecX src, vRegD val, immI idx)
%{
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(INSN_COST);
ins_cost(2 * INSN_COST);
effect(TEMP_DEF dst);
format %{ "orr $dst, T16B, $src, $src\n\t"
"ins $dst, D, $val, $idx, 0\t# insert into vector(2D)" %}
@ -2013,8 +1952,15 @@ instruct extract2F(vRegF dst, vecD src, immI idx)
ins_cost(INSN_COST);
format %{ "ins $dst, S, $src, 0, $idx\t# extract from vector(2F)" %}
ins_encode %{
__ ins(as_FloatRegister($dst$$reg), __ S,
as_FloatRegister($src$$reg), 0, $idx$$constant);
if ((0 == $idx$$constant) &&
(as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg))) {
/* empty */
} else if ($idx$$constant == 0) {
__ fmovs(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
} else {
__ ins(as_FloatRegister($dst$$reg), __ S,
as_FloatRegister($src$$reg), 0, $idx$$constant);
}
%}
ins_pipe(pipe_class_default);
%}
@ -2026,8 +1972,15 @@ instruct extract4F(vRegF dst, vecX src, immI idx)
ins_cost(INSN_COST);
format %{ "ins $dst, S, $src, 0, $idx\t# extract from vector(4F)" %}
ins_encode %{
__ ins(as_FloatRegister($dst$$reg), __ S,
as_FloatRegister($src$$reg), 0, $idx$$constant);
if ((0 == $idx$$constant) &&
(as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg))) {
/* empty */
} else if ($idx$$constant == 0) {
__ fmovs(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
} else {
__ ins(as_FloatRegister($dst$$reg), __ S,
as_FloatRegister($src$$reg), 0, $idx$$constant);
}
%}
ins_pipe(pipe_class_default);
%}
@ -2039,8 +1992,15 @@ instruct extract2D(vRegD dst, vecX src, immI idx)
ins_cost(INSN_COST);
format %{ "ins $dst, D, $src, 0, $idx\t# extract from vector(2D)" %}
ins_encode %{
__ ins(as_FloatRegister($dst$$reg), __ D,
as_FloatRegister($src$$reg), 0, $idx$$constant);
if ((0 == $idx$$constant) &&
(as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg))) {
/* empty */
} else if ($idx$$constant == 0) {
__ fmovd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
} else {
__ ins(as_FloatRegister($dst$$reg), __ D,
as_FloatRegister($src$$reg), 0, $idx$$constant);
}
%}
ins_pipe(pipe_class_default);
%}

@ -887,53 +887,55 @@ REDUCE_LOGIC_OP_2L(eor, Xor, eor )
dnl
// ------------------------------ Vector insert ---------------------------------
dnl VECTOR_INSERT_I($1, $2, $3, $4, $5)
dnl VECTOR_INSERT_I(rule_name, vector_length_in_bytes, reg_variant, vreg, ireg)
define(`VECTOR_INSERT_I', `
instruct insert$1$2`'(vec$3 dst, vec$3 src, iReg$4`'ORL2I($4) val, immI idx)
instruct $1($4 dst, $4 src, $5 val, immI idx)
%{
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($2));
predicate(ifelse($3, D, n->bottom_type()->is_vect()->element_basic_type() == T_LONG,
(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
n->bottom_type()->is_vect()->element_basic_type() == T_INT)));
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(INSN_COST);
format %{ "orr $dst, T$5, $src, $src\n\t"
"mov $dst, iTYPE2SIMD($2), $idx, $val\t# insert into vector($1$2)" %}
ins_cost(2 * INSN_COST);
format %{ "orr $dst, T$2B, $src, $src\n\t"
"mov $dst, $3, $idx, $val\t`#' insert into vector ($3)" %}
ins_encode %{
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T$5,
__ orr(as_FloatRegister($dst$$reg), __ T$2B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ mov(as_FloatRegister($dst$$reg), __ iTYPE2SIMD($2), $idx$$constant, $val$$Register);
__ mov(as_FloatRegister($dst$$reg), __ ifelse($3, D, D, elemType_to_regVariant(Matcher::vector_element_basic_type(this))),
$idx$$constant, $val$$Register);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl $1 $2 $3 $4 $5
VECTOR_INSERT_I(8, B, D, I, 8B)
VECTOR_INSERT_I(16, B, X, I, 16B)
VECTOR_INSERT_I(4, S, D, I, 8B)
VECTOR_INSERT_I(8, S, X, I, 16B)
VECTOR_INSERT_I(2, I, D, I, 8B)
VECTOR_INSERT_I(4, I, X, I, 16B)
VECTOR_INSERT_I(2, L, X, L, 16B)
dnl $1 $2 $3 $4 $5
VECTOR_INSERT_I(insertID, 8, B/H/S, vecD, iRegIorL2I)
VECTOR_INSERT_I(insertIX, 16, B/H/S, vecX, iRegIorL2I)
VECTOR_INSERT_I(insert2L, 16, D, vecX, iRegL)
dnl
define(`VECTOR_INSERT_F', `
instruct insert$1`'(vec$2 dst, vec$2 src, vReg$3 val, immI idx)
instruct insert$3`'(vec$2 dst, vec$2 src, vReg$1 val, immI idx)
%{
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($3));
predicate(n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($1));
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(INSN_COST);
ins_cost(2 * INSN_COST);
effect(TEMP_DEF dst);
format %{ "orr $dst, T$4, $src, $src\n\t"
"ins $dst, $5, $val, $idx, 0\t# insert into vector($1)" %}
format %{ "orr $dst, ifelse($2, D, T8B, T16B), $src, $src\n\t"
"ins $dst, ifelse($1, F, S, D), $val, $idx, 0\t# insert into vector($3)" %}
ins_encode %{
__ orr(as_FloatRegister($dst$$reg), __ T$4,
__ orr(as_FloatRegister($dst$$reg), __ ifelse($2, D, T8B, T16B),
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ ins(as_FloatRegister($dst$$reg), __ $5,
__ ins(as_FloatRegister($dst$$reg), __ ifelse($1, F, S, D),
as_FloatRegister($val$$reg), $idx$$constant, 0);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl $1 $2 $3 $4 $5
VECTOR_INSERT_F(2F, D, F, 8B, S)
VECTOR_INSERT_F(4F, X, F, 16B, S)
VECTOR_INSERT_F(2D, X, D, 16B, D)
dnl $1 $2 $3
VECTOR_INSERT_F(F, D, 2F)
VECTOR_INSERT_F(F, X, 4F)
VECTOR_INSERT_F(D, X, 2D)
dnl
// ------------------------------ Vector extract ---------------------------------
@ -966,8 +968,15 @@ instruct extract$1$2`'(vReg$2 dst, vec$3 src, immI idx)
ins_cost(INSN_COST);
format %{ "ins $dst, $4, $src, 0, $idx\t# extract from vector($1$2)" %}
ins_encode %{
__ ins(as_FloatRegister($dst$$reg), __ $4,
as_FloatRegister($src$$reg), 0, $idx$$constant);
if ((0 == $idx$$constant) &&
(as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg))) {
/* empty */
} else if ($idx$$constant == 0) {
__ ifelse($2, F, fmovs, fmovd)(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
} else {
__ ins(as_FloatRegister($dst$$reg), __ $4,
as_FloatRegister($src$$reg), 0, $idx$$constant);
}
%}
ins_pipe(pipe_class_default);
%}')dnl

@ -4707,91 +4707,148 @@ instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
// ------------------------------ Vector extract ---------------------------------
instruct extractB(iRegINoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
instruct extractB(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
%{
predicate(UseSVE > 0);
predicate(UseSVE > 0 && n->in(2)->get_int() >= 16);
match(Set dst (ExtractB src idx));
effect(TEMP pgtmp, KILL cr);
effect(TEMP vtmp);
ins_cost(2 * SVE_COST);
format %{ "sve_extract $dst, B, $pgtmp, $src, $idx\n\t"
"sbfmw $dst, $dst, 0U, 7U\t# extract from vector(B)" %}
format %{ "sve_extract_integral $dst, B, $src, $idx\t# extract from vector(B)" %}
ins_encode %{
__ sve_extract(as_Register($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
as_FloatRegister($src$$reg), (int)($idx$$constant));
__ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 7U);
__ sve_extract_integral(as_Register($dst$$reg), __ B, as_FloatRegister($src$$reg),
(int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
%}
ins_pipe(pipe_slow);
%}
instruct extractS(iRegINoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
instruct extractS(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
%{
predicate(UseSVE > 0);
predicate(UseSVE > 0 && n->in(2)->get_int() >= 8);
match(Set dst (ExtractS src idx));
effect(TEMP pgtmp, KILL cr);
effect(TEMP vtmp);
ins_cost(2 * SVE_COST);
format %{ "sve_extract $dst, H, $pgtmp, $src, $idx\n\t"
"sbfmw $dst, $dst, 0U, 15U\t# extract from vector(S)" %}
format %{ "sve_extract_integral $dst, H, $src, $idx\t# extract from vector(S)" %}
ins_encode %{
__ sve_extract(as_Register($dst$$reg), __ H, as_PRegister($pgtmp$$reg),
as_FloatRegister($src$$reg), (int)($idx$$constant));
__ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
__ sve_extract_integral(as_Register($dst$$reg), __ H, as_FloatRegister($src$$reg),
(int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
%}
ins_pipe(pipe_slow);
%}
instruct extractI(iRegINoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
instruct extractI(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
%{
predicate(UseSVE > 0);
predicate(UseSVE > 0 && n->in(2)->get_int() >= 4);
match(Set dst (ExtractI src idx));
effect(TEMP pgtmp, KILL cr);
effect(TEMP vtmp);
ins_cost(2 * SVE_COST);
format %{ "sve_extract $dst, S, $pgtmp, $src, $idx\t# extract from vector(I)" %}
format %{ "sve_extract_integral $dst, S, $src, $idx\t# extract from vector(I)" %}
ins_encode %{
__ sve_extract(as_Register($dst$$reg), __ S, as_PRegister($pgtmp$$reg),
as_FloatRegister($src$$reg), (int)($idx$$constant));
__ sve_extract_integral(as_Register($dst$$reg), __ S, as_FloatRegister($src$$reg),
(int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
%}
ins_pipe(pipe_slow);
%}
instruct extractL(iRegLNoSp dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
instruct extractL(iRegLNoSp dst, vReg src, immI idx, vReg vtmp)
%{
predicate(UseSVE > 0);
predicate(UseSVE > 0 && n->in(2)->get_int() >= 2);
match(Set dst (ExtractL src idx));
effect(TEMP pgtmp, KILL cr);
effect(TEMP vtmp);
ins_cost(2 * SVE_COST);
format %{ "sve_extract $dst, D, $pgtmp, $src, $idx\t# extract from vector(L)" %}
format %{ "sve_extract_integral $dst, D, $src, $idx\t# extract from vector(L)" %}
ins_encode %{
__ sve_extract(as_Register($dst$$reg), __ D, as_PRegister($pgtmp$$reg),
as_FloatRegister($src$$reg), (int)($idx$$constant));
__ sve_extract_integral(as_Register($dst$$reg), __ D, as_FloatRegister($src$$reg),
(int)($idx$$constant), /* is_signed */ false, as_FloatRegister($vtmp$$reg));
%}
ins_pipe(pipe_slow);
%}
instruct extractF(vRegF dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
instruct extractB_LT16(iRegINoSp dst, vReg src, immI idx)
%{
predicate(UseSVE > 0 && n->in(2)->get_int() < 16);
match(Set dst (ExtractB src idx));
ins_cost(INSN_COST);
format %{ "smov $dst, B, $src, $idx\t# extract from vector(B)" %}
ins_encode %{
__ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ B, $idx$$constant);
%}
ins_pipe(pipe_class_default);
%}
instruct extractS_LT8(iRegINoSp dst, vReg src, immI idx)
%{
predicate(UseSVE > 0 && n->in(2)->get_int() < 8);
match(Set dst (ExtractS src idx));
ins_cost(INSN_COST);
format %{ "smov $dst, H, $src, $idx\t# extract from vector(S)" %}
ins_encode %{
__ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ H, $idx$$constant);
%}
ins_pipe(pipe_class_default);
%}
instruct extractI_LT4(iRegINoSp dst, vReg src, immI idx)
%{
predicate(UseSVE > 0 && n->in(2)->get_int() < 4);
match(Set dst (ExtractI src idx));
ins_cost(INSN_COST);
format %{ "smov $dst, S, $src, $idx\t# extract from vector(I)" %}
ins_encode %{
__ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ S, $idx$$constant);
%}
ins_pipe(pipe_class_default);
%}
instruct extractL_LT2(iRegLNoSp dst, vReg src, immI idx)
%{
predicate(UseSVE > 0 && n->in(2)->get_int() < 2);
match(Set dst (ExtractL src idx));
ins_cost(INSN_COST);
format %{ "umov $dst, D, $src, $idx\t# extract from vector(L)" %}
ins_encode %{
__ umov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ D, $idx$$constant);
%}
ins_pipe(pipe_class_default);
%}
instruct extractF(vRegF dst, vReg src, immI idx)
%{
predicate(UseSVE > 0);
match(Set dst (ExtractF src idx));
effect(TEMP pgtmp, KILL cr);
ins_cost(2 * SVE_COST);
format %{ "sve_extract $dst, S, $pgtmp, $src, $idx\t# extract from vector(F)" %}
format %{ "sve_extract_f $dst, S, $src, $idx\t# extract from vector(F)" %}
ins_encode %{
__ sve_extract(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg),
as_FloatRegister($src$$reg), (int)($idx$$constant));
if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
/* empty */
} else if ($idx$$constant == 0) {
__ fmovs(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
} else if ($idx$$constant < 4) {
__ ins(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg), 0, (int)($idx$$constant));
} else {
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 2);
}
%}
ins_pipe(pipe_slow);
%}
instruct extractD(vRegD dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
instruct extractD(vRegD dst, vReg src, immI idx)
%{
predicate(UseSVE > 0);
match(Set dst (ExtractD src idx));
effect(TEMP pgtmp, KILL cr);
ins_cost(2 * SVE_COST);
format %{ "sve_extract $dst, D, $pgtmp, $src, $idx\t# extract from vector(D)" %}
format %{ "sve_extract_d $dst, D, $src, $idx\t# extract from vector(D)" %}
ins_encode %{
__ sve_extract(as_FloatRegister($dst$$reg), __ D, as_PRegister($pgtmp$$reg),
as_FloatRegister($src$$reg), (int)($idx$$constant));
if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
/* empty */
} else if ($idx$$constant == 0) {
__ fmovd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
} else if ($idx$$constant == 1) {
__ ins(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), 0, 1);
} else {
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 3);
}
%}
ins_pipe(pipe_slow);
%}
@ -4833,34 +4890,152 @@ instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
%}
// ------------------------------ Vector insert ---------------------------------
instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, pRegGov pgtmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
instruct insertI_le128bits(vReg dst, vReg src, iRegIorL2I val, immI idx) %{
predicate(UseSVE > 0 &&
(Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16) &&
(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
n->bottom_type()->is_vect()->element_basic_type() == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
ins_cost(2 * INSN_COST);
format %{ "orr $dst, T8/16B, $src, $src\n\t"
"mov $dst, B/H/S, $idx, $val\t# insertI into vector(64/128bits)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ mov(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(Matcher::vector_element_basic_type(this)),
$idx$$constant, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct insertI_small_index(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
Matcher::vector_length_in_bytes(n) > 16 &&
(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
n->bottom_type()->is_vect()->element_basic_type() == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP vtmp, TEMP pgtmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $dst, -16, 1\t# (B/H/S)\n\t"
"sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
format %{ "sve_index $vtmp, -16, 1\t# (B/H/S)\n\t"
"sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_index(as_FloatRegister($dst$$reg), size, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
__ block_comment("insert into vector (B/H/S) {");
__ sve_index(as_FloatRegister($vtmp$$reg), size, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
__ block_comment("} insert into vector (B/H/S)");
%}
ins_pipe(pipe_slow);
%}
instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
Matcher::vector_length_in_bytes(n) > 16 &&
(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
n->bottom_type()->is_vect()->element_basic_type() == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr);
ins_cost(5 * SVE_COST);
format %{ "sve_index $vtmp1, 0, 1\t# (B/H/S)\n\t"
"sve_dup $vtmp2, $idx\t# (B/H/S)\n\t"
"sve_cmpeq $pgtmp, $vtmp1, $vtmp2\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ block_comment("insert into vector (B/H/S) {");
__ sve_index(as_FloatRegister($vtmp1$$reg), size, 0, 1);
__ sve_dup(as_FloatRegister($vtmp2$$reg), size, (int)($idx$$constant));
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
__ block_comment("} insert into vector (B/H/S)");
%}
ins_pipe(pipe_slow);
%}
instruct insertL_128bits(vReg dst, vReg src, iRegL val, immI idx) %{
predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(2 * INSN_COST);
format %{ "orr $dst, T16B, $src, $src\n\t"
"mov $dst, D, $idx, $val\t# insertL into vector(128bits)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ mov(as_FloatRegister($dst$$reg), __ D, $idx$$constant, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct insertL(vReg dst, vReg src, iRegL val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) > 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP vtmp, TEMP pgtmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $vtmp, D, -16, 1\n\t"
"sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %}
ins_encode %{
__ block_comment("insert into vector (L) {");
__ sve_index(as_FloatRegister($vtmp$$reg), __ D, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), __ D,
as_PRegister($pgtmp$$reg), $val$$Register);
__ block_comment("} insert into vector (L)");
%}
ins_pipe(pipe_slow);
%}
instruct insertF_le128bits(vReg dst, vReg src, vRegF val, immI idx) %{
predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
(Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16));
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(2 * INSN_COST);
effect(TEMP_DEF dst);
format %{ "orr $dst, T8/16B, $src, $src\n\t"
"ins $dst, S, $val, $idx, 0\t# insertF into vector(64/128bits)" %}
ins_encode %{
__ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ ins(as_FloatRegister($dst$$reg), __ S,
as_FloatRegister($val$$reg), $idx$$constant, 0);
%}
ins_pipe(pipe_slow);
%}
instruct insertF_small_index(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
Matcher::vector_length_in_bytes(n) > 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
@ -4870,87 +5045,20 @@ instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, r
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
ins_encode %{
__ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
__ block_comment("insert into vector (F) {");
__ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
__ block_comment("} insert into vector (F)");
%}
ins_pipe(pipe_slow);
%}
instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
n->bottom_type()->is_vect()->element_basic_type() == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
ins_cost(5 * SVE_COST);
format %{ "sve_index $tmp1, 0, 1\t# (B/H/S)\n\t"
"sve_dup $dst, $idx\t# (B/H/S)\n\t"
"sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_index(as_FloatRegister($tmp1$$reg), size, 0, 1);
__ sve_dup(as_FloatRegister($dst$$reg), size, (int)($idx$$constant));
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
%}
ins_pipe(pipe_slow);
%}
instruct insertL(vReg dst, vReg src, iRegL val, immI idx, pRegGov pgtmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 &&
n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $dst, D, -16, 1\n\t"
"sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %}
ins_encode %{
__ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ D, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
%}
ins_pipe(pipe_slow);
%}
instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 &&
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $dst, D, -16, 1\n\t"
"sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %}
ins_encode %{
__ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ D, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
%}
ins_pipe(pipe_slow);
%}
instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
Matcher::vector_length_in_bytes(n) > 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
@ -4961,15 +5069,58 @@ instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgt
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
ins_encode %{
__ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
__ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
__ sve_orr(as_FloatRegister($dst$$reg),
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ S,
as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
__ block_comment("insert into vector (F) {");
__ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
__ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ S,
as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
__ block_comment("} insert into vector (F)");
%}
ins_pipe(pipe_slow);
%}
instruct insertD_128bits(vReg dst, vReg src, vRegD val, immI idx) %{
predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(2 * INSN_COST);
effect(TEMP_DEF dst);
format %{ "orr $dst, T16B, $src, $src\n\t"
"ins $dst, D, $val, $idx, 0\t# insertD into vector(128bits)" %}
ins_encode %{
__ orr(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ ins(as_FloatRegister($dst$$reg), __ D,
as_FloatRegister($val$$reg), $idx$$constant, 0);
%}
ins_pipe(pipe_slow);
%}
instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) > 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $dst, D, -16, 1\n\t"
"sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %}
ins_encode %{
__ block_comment("insert into vector (D) {");
__ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ D,
as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
__ block_comment("} insert into vector (D)");
%}
ins_pipe(pipe_slow);
%}
@ -5320,19 +5471,18 @@ instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
ins_pipe(pipe_slow);
%}
instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{
instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2) %{
predicate(UseSVE > 0 &&
n->in(1)->bottom_type()->is_vect()->length() <= 64);
match(Set dst (VectorMaskToLong src));
effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr);
effect(TEMP vtmp1, TEMP vtmp2);
ins_cost(13 * SVE_COST);
format %{ "vmask_tolong $dst, $src\t# vector mask tolong (sve)" %}
ins_encode %{
__ sve_vmask_tolong(as_Register($dst$$reg), as_PRegister($src$$reg),
Matcher::vector_element_basic_type(this, $src),
Matcher::vector_length(this, $src),
as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg),
as_PRegister($pgtmp$$reg));
as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
%}
ins_pipe(pipe_slow);
%}

@ -2353,46 +2353,86 @@ instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
dnl
dnl
// ------------------------------ Vector extract ---------------------------------
define(`VECTOR_EXTRACT_SXT', `
instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
%{
predicate(UseSVE > 0);
match(Set dst (Extract$1 src idx));
effect(TEMP pgtmp, KILL cr);
ins_cost(2 * SVE_COST);
format %{ "sve_extract $dst, $3, $pgtmp, $src, $idx\n\t"
"sbfmw $dst, $dst, 0U, $5\t# extract from vector($1)" %}
ins_encode %{
__ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pgtmp$$reg),
as_FloatRegister($src$$reg), (int)($idx$$constant));
__ sbfmw(as_$4($dst$$reg), as_$4($dst$$reg), 0U, $5);
%}
ins_pipe(pipe_slow);
%}')dnl
dnl $1 $2 $3 $4 $5
VECTOR_EXTRACT_SXT(B, iRegINoSp, B, Register, 7U)
VECTOR_EXTRACT_SXT(S, iRegINoSp, H, Register, 15U)
dnl
define(`VECTOR_EXTRACT', `
instruct extract$1`'($2 dst, vReg src, immI idx, pRegGov pgtmp, rFlagsReg cr)
define(`VECTOR_EXTRACT_I', `
instruct extract$1`'($3 dst, vReg src, immI idx, vReg vtmp)
%{
predicate(UseSVE > 0);
predicate(UseSVE > 0 && n->in(2)->get_int() >= $2);
match(Set dst (Extract$1 src idx));
effect(TEMP pgtmp, KILL cr);
effect(TEMP vtmp);
ins_cost(2 * SVE_COST);
format %{ "sve_extract $dst, $3, $pgtmp, $src, $idx\t# extract from vector($1)" %}
format %{ "sve_extract_integral $dst, $4, $src, $idx\t# extract from vector($1)" %}
ins_encode %{
__ sve_extract(as_$4($dst$$reg), __ $3, as_PRegister($pgtmp$$reg),
as_FloatRegister($src$$reg), (int)($idx$$constant));
__ sve_extract_integral(as_Register($dst$$reg), __ $4, as_FloatRegister($src$$reg),
(int)($idx$$constant), /* is_signed */ ifelse($1, L, false, true), as_FloatRegister($vtmp$$reg));
%}
ins_pipe(pipe_slow);
%}')dnl
dnl $1 $2 $3 $4
VECTOR_EXTRACT(I, iRegINoSp, S, Register)
VECTOR_EXTRACT(L, iRegLNoSp, D, Register)
VECTOR_EXTRACT(F, vRegF, S, FloatRegister)
VECTOR_EXTRACT(D, vRegD, D, FloatRegister)
dnl $1 $2 $3 $4
VECTOR_EXTRACT_I(B, 16, iRegINoSp, B)
VECTOR_EXTRACT_I(S, 8, iRegINoSp, H)
VECTOR_EXTRACT_I(I, 4, iRegINoSp, S)
VECTOR_EXTRACT_I(L, 2, iRegLNoSp, D)
dnl
define(`VECTOR_EXTRACT_I_LT', `
instruct extract$1_LT$2`'($3 dst, vReg src, immI idx)
%{
predicate(UseSVE > 0 && n->in(2)->get_int() < $2);
match(Set dst (Extract$1 src idx));
ins_cost(INSN_COST);
format %{ "ifelse($4, D, umov, smov) $dst, $4, $src, $idx\t# extract from vector($1)" %}
ins_encode %{
__ ifelse($4, D, umov, smov)(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ $4, $idx$$constant);
%}
ins_pipe(pipe_class_default);
%}')dnl
dnl $1 $2 $3 $4
VECTOR_EXTRACT_I_LT(B, 16, iRegINoSp, B)
VECTOR_EXTRACT_I_LT(S, 8, iRegINoSp, H)
VECTOR_EXTRACT_I_LT(I, 4, iRegINoSp, S)
VECTOR_EXTRACT_I_LT(L, 2, iRegLNoSp, D)
instruct extractF(vRegF dst, vReg src, immI idx)
%{
predicate(UseSVE > 0);
match(Set dst (ExtractF src idx));
ins_cost(2 * SVE_COST);
format %{ "sve_extract_f $dst, S, $src, $idx\t# extract from vector(F)" %}
ins_encode %{
if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
/* empty */
} else if ($idx$$constant == 0) {
__ fmovs(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
} else if ($idx$$constant < 4) {
__ ins(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg), 0, (int)($idx$$constant));
} else {
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 2);
}
%}
ins_pipe(pipe_slow);
%}
instruct extractD(vRegD dst, vReg src, immI idx)
%{
predicate(UseSVE > 0);
match(Set dst (ExtractD src idx));
ins_cost(2 * SVE_COST);
format %{ "sve_extract_d $dst, D, $src, $idx\t# extract from vector(D)" %}
ins_encode %{
if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
/* empty */
} else if ($idx$$constant == 0) {
__ fmovd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
} else if ($idx$$constant == 1) {
__ ins(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), 0, 1);
} else {
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 3);
}
%}
ins_pipe(pipe_slow);
%}
// ------------------------------- VectorTest ----------------------------------
@ -2432,34 +2472,152 @@ instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
dnl
// ------------------------------ Vector insert ---------------------------------
instruct insertI_small(vReg dst, vReg src, iRegIorL2I val, immI idx, pRegGov pgtmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
instruct insertI_le128bits(vReg dst, vReg src, iRegIorL2I val, immI idx) %{
predicate(UseSVE > 0 &&
(Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16) &&
(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
n->bottom_type()->is_vect()->element_basic_type() == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
ins_cost(2 * INSN_COST);
format %{ "orr $dst, T8/16B, $src, $src\n\t"
"mov $dst, B/H/S, $idx, $val\t# insertI into vector(64/128bits)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ mov(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(Matcher::vector_element_basic_type(this)),
$idx$$constant, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct insertI_small_index(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
Matcher::vector_length_in_bytes(n) > 16 &&
(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
n->bottom_type()->is_vect()->element_basic_type() == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP vtmp, TEMP pgtmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $dst, -16, 1\t# (B/H/S)\n\t"
"sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
format %{ "sve_index $vtmp, -16, 1\t# (B/H/S)\n\t"
"sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_index(as_FloatRegister($dst$$reg), size, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
__ block_comment("insert into vector (B/H/S) {");
__ sve_index(as_FloatRegister($vtmp$$reg), size, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
__ block_comment("} insert into vector (B/H/S)");
%}
ins_pipe(pipe_slow);
%}
instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() <= 32 &&
instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
Matcher::vector_length_in_bytes(n) > 16 &&
(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
n->bottom_type()->is_vect()->element_basic_type() == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr);
ins_cost(5 * SVE_COST);
format %{ "sve_index $vtmp1, 0, 1\t# (B/H/S)\n\t"
"sve_dup $vtmp2, $idx\t# (B/H/S)\n\t"
"sve_cmpeq $pgtmp, $vtmp1, $vtmp2\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ block_comment("insert into vector (B/H/S) {");
__ sve_index(as_FloatRegister($vtmp1$$reg), size, 0, 1);
__ sve_dup(as_FloatRegister($vtmp2$$reg), size, (int)($idx$$constant));
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
__ block_comment("} insert into vector (B/H/S)");
%}
ins_pipe(pipe_slow);
%}
instruct insertL_128bits(vReg dst, vReg src, iRegL val, immI idx) %{
predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(2 * INSN_COST);
format %{ "orr $dst, T16B, $src, $src\n\t"
"mov $dst, D, $idx, $val\t# insertL into vector(128bits)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ orr(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ mov(as_FloatRegister($dst$$reg), __ D, $idx$$constant, $val$$Register);
%}
ins_pipe(pipe_slow);
%}
instruct insertL(vReg dst, vReg src, iRegL val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) > 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP vtmp, TEMP pgtmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $vtmp, D, -16, 1\n\t"
"sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %}
ins_encode %{
__ block_comment("insert into vector (L) {");
__ sve_index(as_FloatRegister($vtmp$$reg), __ D, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
}
__ sve_cpy(as_FloatRegister($dst$$reg), __ D,
as_PRegister($pgtmp$$reg), $val$$Register);
__ block_comment("} insert into vector (L)");
%}
ins_pipe(pipe_slow);
%}
instruct insertF_le128bits(vReg dst, vReg src, vRegF val, immI idx) %{
predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
(Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16));
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(2 * INSN_COST);
effect(TEMP_DEF dst);
format %{ "orr $dst, T8/16B, $src, $src\n\t"
"ins $dst, S, $val, $idx, 0\t# insertF into vector(64/128bits)" %}
ins_encode %{
__ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ ins(as_FloatRegister($dst$$reg), __ S,
as_FloatRegister($val$$reg), $idx$$constant, 0);
%}
ins_pipe(pipe_slow);
%}
instruct insertF_small_index(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
Matcher::vector_length_in_bytes(n) > 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
@ -2469,71 +2627,20 @@ instruct insertF_small(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, r
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
ins_encode %{
__ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
__ block_comment("insert into vector (F) {");
__ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
__ block_comment("} insert into vector (F)");
%}
ins_pipe(pipe_slow);
%}
instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
(n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
n->bottom_type()->is_vect()->element_basic_type() == T_INT));
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
ins_cost(5 * SVE_COST);
format %{ "sve_index $tmp1, 0, 1\t# (B/H/S)\n\t"
"sve_dup $dst, $idx\t# (B/H/S)\n\t"
"sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this, $src);
Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
__ sve_index(as_FloatRegister($tmp1$$reg), size, 0, 1);
__ sve_dup(as_FloatRegister($dst$$reg), size, (int)($idx$$constant));
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), as_Register($val$$reg));
%}
ins_pipe(pipe_slow);
%}
dnl
dnl
define(`VECTOR_INSERT_D', `
instruct insert$1`'(vReg dst, vReg src, $2 val, immI idx, pRegGov pgtmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 &&
n->bottom_type()->is_vect()->element_basic_type() == T_`'TYPE2DATATYPE($1));
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $dst, $3, -16, 1\n\t"
"sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector ($1)" %}
ins_encode %{
__ sve_index(as_FloatRegister($dst$$reg), __ $3, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ $3, ptrue,
as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ $3, as_PRegister($pgtmp$$reg), as_$4($val$$reg));
%}
ins_pipe(pipe_slow);
%}')dnl
dnl $1 $2 $3 $4
VECTOR_INSERT_D(L, iRegL, D, Register)
VECTOR_INSERT_D(D, vRegD, D, FloatRegister)
instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr)
%{
predicate(UseSVE > 0 && n->as_Vector()->length() > 32 &&
instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
Matcher::vector_length_in_bytes(n) > 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
@ -2544,15 +2651,58 @@ instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgt
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
ins_encode %{
__ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
__ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
__ sve_orr(as_FloatRegister($dst$$reg),
as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ S,
as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
__ block_comment("insert into vector (F) {");
__ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
__ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ S,
as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
__ block_comment("} insert into vector (F)");
%}
ins_pipe(pipe_slow);
%}
instruct insertD_128bits(vReg dst, vReg src, vRegD val, immI idx) %{
predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
match(Set dst (VectorInsert (Binary src val) idx));
ins_cost(2 * INSN_COST);
effect(TEMP_DEF dst);
format %{ "orr $dst, T16B, $src, $src\n\t"
"ins $dst, D, $val, $idx, 0\t# insertD into vector(128bits)" %}
ins_encode %{
__ orr(as_FloatRegister($dst$$reg), __ T16B,
as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
__ ins(as_FloatRegister($dst$$reg), __ D,
as_FloatRegister($val$$reg), $idx$$constant, 0);
%}
ins_pipe(pipe_slow);
%}
instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
predicate(UseSVE > 0 &&
Matcher::vector_length_in_bytes(n) > 16 &&
n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
match(Set dst (VectorInsert (Binary src val) idx));
effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
ins_cost(4 * SVE_COST);
format %{ "sve_index $dst, D, -16, 1\n\t"
"sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
"sve_orr $dst, $src, $src\n\t"
"sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %}
ins_encode %{
__ block_comment("insert into vector (D) {");
__ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
__ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
__ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
as_FloatRegister($src$$reg));
__ sve_cpy(as_FloatRegister($dst$$reg), __ D,
as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
__ block_comment("} insert into vector (D)");
%}
ins_pipe(pipe_slow);
%}
@ -2889,19 +3039,18 @@ instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
ins_pipe(pipe_slow);
%}
instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{
instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2) %{
predicate(UseSVE > 0 &&
n->in(1)->bottom_type()->is_vect()->length() <= 64);
match(Set dst (VectorMaskToLong src));
effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr);
effect(TEMP vtmp1, TEMP vtmp2);
ins_cost(13 * SVE_COST);
format %{ "vmask_tolong $dst, $src\t# vector mask tolong (sve)" %}
ins_encode %{
__ sve_vmask_tolong(as_Register($dst$$reg), as_PRegister($src$$reg),
Matcher::vector_element_basic_type(this, $src),
Matcher::vector_length(this, $src),
as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg),
as_PRegister($pgtmp$$reg));
as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
%}
ins_pipe(pipe_slow);
%}

@ -3506,6 +3506,13 @@ public:
f(0b11, 15, 14), prf(Pg, 10), rf(Zn, 5), rf(Zd, 0);
}
// SVE Permute Vector - Extract
void sve_ext(FloatRegister Zdn, FloatRegister Zm, int imm8) {
starti;
f(0b00000101001, 31, 21), f(imm8 >> 3, 20, 16), f(0b000, 15, 13);
f(imm8 & 0b111, 12, 10), rf(Zm, 5), rf(Zdn, 0);
}
// SVE Integer/Floating-Point Compare - Vectors
#define INSN(NAME, op1, op2, fp) \
void NAME(Condition cond, PRegister Pd, SIMD_RegVariant T, PRegister Pg, \

@ -960,10 +960,10 @@ void C2_MacroAssembler::bytemask_compress(Register dst) {
// in dst, at most the first 64 lane elements.
// Clobbers: rscratch1
void C2_MacroAssembler::sve_vmask_tolong(Register dst, PRegister src, BasicType bt, int lane_cnt,
FloatRegister vtmp1, FloatRegister vtmp2, PRegister pgtmp) {
assert(pgtmp->is_governing(), "This register has to be a governing predicate register.");
FloatRegister vtmp1, FloatRegister vtmp2) {
assert(lane_cnt <= 64 && is_power_of_2(lane_cnt), "Unsupported lane count");
assert_different_registers(dst, rscratch1);
assert_different_registers(vtmp1, vtmp2);
Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
@ -981,7 +981,7 @@ void C2_MacroAssembler::sve_vmask_tolong(Register dst, PRegister src, BasicType
// Repeat on higher bytes and join the results.
// Compress 8 bytes in each iteration.
for (int idx = 1; idx < (lane_cnt / 8); idx++) {
idx == 1 ? fmovhid(rscratch1, vtmp1) : sve_extract(rscratch1, D, pgtmp, vtmp1, idx);
sve_extract_integral(rscratch1, D, vtmp1, idx, /* is_signed */ false, vtmp2);
bytemask_compress(rscratch1);
orr(dst, dst, rscratch1, Assembler::LSL, idx << 3);
}
@ -1268,6 +1268,21 @@ void C2_MacroAssembler::sve_ptrue_lanecnt(PRegister dst, SIMD_RegVariant size, i
}
}
// Extract a scalar element from an sve vector at position 'idx'.
// The input elements in src are expected to be of integral type.
void C2_MacroAssembler::sve_extract_integral(Register dst, SIMD_RegVariant size, FloatRegister src, int idx,
bool is_signed, FloatRegister vtmp) {
assert(UseSVE > 0 && size != Q, "unsupported");
assert(!(is_signed && size == D), "signed extract (D) not supported.");
if (regVariant_to_elemBits(size) * idx < 128) { // generate lower cost NEON instruction
is_signed ? smov(dst, src, size, idx) : umov(dst, src, size, idx);
} else {
sve_orr(vtmp, src, src);
sve_ext(vtmp, vtmp, idx << size);
is_signed ? smov(dst, vtmp, size, 0) : umov(dst, vtmp, size, 0);
}
}
// java.lang.Math::round intrinsics
void C2_MacroAssembler::vector_round_neon(FloatRegister dst, FloatRegister src, FloatRegister tmp1,
@ -1337,4 +1352,3 @@ void C2_MacroAssembler::vector_round_sve(FloatRegister dst, FloatRegister src, F
sve_fcvtzs(dst, T, ptrue, dst, T);
// result in dst
}

@ -61,7 +61,7 @@
// Pack the lowest-numbered bit of each mask element in src into a long value
// in dst, at most the first 64 lane elements.
void sve_vmask_tolong(Register dst, PRegister src, BasicType bt, int lane_cnt,
FloatRegister vtmp1, FloatRegister vtmp2, PRegister pgtmp);
FloatRegister vtmp1, FloatRegister vtmp2);
// SIMD&FP comparison
void neon_compare(FloatRegister dst, BasicType bt, FloatRegister src1,
@ -92,16 +92,9 @@
void sve_ptrue_lanecnt(PRegister dst, SIMD_RegVariant size, int lane_cnt);
// Extract a scalar element from an sve vector at position 'idx'.
// rscratch1 will be clobbered.
// T could be FloatRegister or Register.
template<class T>
inline void sve_extract(T dst, SIMD_RegVariant size, PRegister pg, FloatRegister src, int idx) {
assert(UseSVE > 0, "not supported");
assert(pg->is_governing(), "This register has to be a governing predicate register");
mov(rscratch1, idx);
sve_whilele(pg, size, zr, rscratch1);
sve_lastb(dst, size, pg, src);
}
// The input elements in src are expected to be of integral type.
void sve_extract_integral(Register dst, SIMD_RegVariant size, FloatRegister src, int idx,
bool is_signed, FloatRegister vtmp);
// java.lang.Math::round intrinsics
void vector_round_neon(FloatRegister dst, FloatRegister src, FloatRegister tmp1,

@ -1811,6 +1811,7 @@ generate(SpecialCases, [["ccmn", "__ ccmn(zr, zr, 3u, Assembler::LE);",
["uzp2", "__ sve_uzp2(p0, __ D, p0, p1);", "uzp2\tp0.d, p0.d, p1.d"],
["punpklo", "__ sve_punpklo(p1, p0);", "punpklo\tp1.h, p0.b"],
["punpkhi", "__ sve_punpkhi(p1, p0);", "punpkhi\tp1.h, p0.b"],
["ext", "__ sve_ext(z17, z16, 63);", "ext\tz17.b, z17.b, z16.b, #63"],
])
print "\n// FloatImmediateOp"

@ -955,6 +955,7 @@
__ sve_uzp2(p0, __ D, p0, p1); // uzp2 p0.d, p0.d, p1.d
__ sve_punpklo(p1, p0); // punpklo p1.h, p0.b
__ sve_punpkhi(p1, p0); // punpkhi p1.h, p0.b
__ sve_ext(z17, z16, 63); // ext z17.b, z17.b, z16.b, #63
// FloatImmediateOp
__ fmovd(v0, 2.0); // fmov d0, #2.0
@ -1211,30 +1212,30 @@
0x9101a1a0, 0xb10a5cc8, 0xd10810aa, 0xf10fd061,
0x120cb166, 0x321764bc, 0x52174681, 0x720c0227,
0x9241018e, 0xb25a2969, 0xd278b411, 0xf26aad01,
0x14000000, 0x17ffffd7, 0x140003e3, 0x94000000,
0x97ffffd4, 0x940003e0, 0x3400000a, 0x34fffa2a,
0x34007baa, 0x35000008, 0x35fff9c8, 0x35007b48,
0xb400000b, 0xb4fff96b, 0xb4007aeb, 0xb500001d,
0xb5fff91d, 0xb5007a9d, 0x10000013, 0x10fff8b3,
0x10007a33, 0x90000013, 0x36300016, 0x3637f836,
0x363079b6, 0x3758000c, 0x375ff7cc, 0x3758794c,
0x14000000, 0x17ffffd7, 0x140003e4, 0x94000000,
0x97ffffd4, 0x940003e1, 0x3400000a, 0x34fffa2a,
0x34007bca, 0x35000008, 0x35fff9c8, 0x35007b68,
0xb400000b, 0xb4fff96b, 0xb4007b0b, 0xb500001d,
0xb5fff91d, 0xb5007abd, 0x10000013, 0x10fff8b3,
0x10007a53, 0x90000013, 0x36300016, 0x3637f836,
0x363079d6, 0x3758000c, 0x375ff7cc, 0x3758796c,
0x128313a0, 0x528a32c7, 0x7289173b, 0x92ab3acc,
0xd2a0bf94, 0xf2c285e8, 0x9358722f, 0x330e652f,
0x53067f3b, 0x93577c53, 0xb34a1aac, 0xd35a4016,
0x13946c63, 0x93c3dbc8, 0x54000000, 0x54fff5a0,
0x54007720, 0x54000001, 0x54fff541, 0x540076c1,
0x54000002, 0x54fff4e2, 0x54007662, 0x54000002,
0x54fff482, 0x54007602, 0x54000003, 0x54fff423,
0x540075a3, 0x54000003, 0x54fff3c3, 0x54007543,
0x54000004, 0x54fff364, 0x540074e4, 0x54000005,
0x54fff305, 0x54007485, 0x54000006, 0x54fff2a6,
0x54007426, 0x54000007, 0x54fff247, 0x540073c7,
0x54000008, 0x54fff1e8, 0x54007368, 0x54000009,
0x54fff189, 0x54007309, 0x5400000a, 0x54fff12a,
0x540072aa, 0x5400000b, 0x54fff0cb, 0x5400724b,
0x5400000c, 0x54fff06c, 0x540071ec, 0x5400000d,
0x54fff00d, 0x5400718d, 0x5400000e, 0x54ffefae,
0x5400712e, 0x5400000f, 0x54ffef4f, 0x540070cf,
0x54007740, 0x54000001, 0x54fff541, 0x540076e1,
0x54000002, 0x54fff4e2, 0x54007682, 0x54000002,
0x54fff482, 0x54007622, 0x54000003, 0x54fff423,
0x540075c3, 0x54000003, 0x54fff3c3, 0x54007563,
0x54000004, 0x54fff364, 0x54007504, 0x54000005,
0x54fff305, 0x540074a5, 0x54000006, 0x54fff2a6,
0x54007446, 0x54000007, 0x54fff247, 0x540073e7,
0x54000008, 0x54fff1e8, 0x54007388, 0x54000009,
0x54fff189, 0x54007329, 0x5400000a, 0x54fff12a,
0x540072ca, 0x5400000b, 0x54fff0cb, 0x5400726b,
0x5400000c, 0x54fff06c, 0x5400720c, 0x5400000d,
0x54fff00d, 0x540071ad, 0x5400000e, 0x54ffefae,
0x5400714e, 0x5400000f, 0x54ffef4f, 0x540070ef,
0xd40658e1, 0xd4014d22, 0xd4046543, 0xd4273f60,
0xd44cad80, 0xd503201f, 0xd503203f, 0xd503205f,
0xd503209f, 0xd50320bf, 0xd503219f, 0xd50323bf,
@ -1410,56 +1411,56 @@
0x25d8e104, 0x25d8e184, 0x2518e407, 0x05214800,
0x05614800, 0x05a14800, 0x05e14800, 0x05214c00,
0x05614c00, 0x05a14c00, 0x05e14c00, 0x05304001,
0x05314001, 0x1e601000, 0x1e603000, 0x1e621000,
0x1e623000, 0x1e641000, 0x1e643000, 0x1e661000,
0x1e663000, 0x1e681000, 0x1e683000, 0x1e6a1000,
0x1e6a3000, 0x1e6c1000, 0x1e6c3000, 0x1e6e1000,
0x1e6e3000, 0x1e701000, 0x1e703000, 0x1e721000,
0x1e723000, 0x1e741000, 0x1e743000, 0x1e761000,
0x1e763000, 0x1e781000, 0x1e783000, 0x1e7a1000,
0x1e7a3000, 0x1e7c1000, 0x1e7c3000, 0x1e7e1000,
0x1e7e3000, 0xf82c815f, 0xf8300047, 0xf823126d,
0xf8312070, 0xf82133cb, 0xf82551e8, 0xf83d401e,
0xf8347287, 0xf83762bc, 0xf8bb80b9, 0xf8a10217,
0xf8bf1185, 0xf8a921fc, 0xf8bd33f6, 0xf8b350bf,
0xf8ae43f0, 0xf8b0729b, 0xf8b0616c, 0xf8e983c6,
0xf8f1039b, 0xf8fe1147, 0xf8f4208a, 0xf8f83231,
0xf8f653a3, 0xf8ef4276, 0xf8f37056, 0xf8ef6186,
0xf87081ab, 0xf87703c1, 0xf8731225, 0xf86222d0,
0xf86d32aa, 0xf87d519b, 0xf87b4023, 0xf87f7278,
0xf8716389, 0xb83b80ef, 0xb83503f7, 0xb83913e2,
0xb83b2150, 0xb8373073, 0xb8305320, 0xb83a4057,
0xb830708c, 0xb83c63be, 0xb8b080db, 0xb8a901fd,
0xb8a710e4, 0xb8af22e9, 0xb8a83382, 0xb8b550bf,
0xb8bb4220, 0xb8af7344, 0xb8a862dc, 0xb8fb833b,
0xb8f70080, 0xb8e61010, 0xb8e4202f, 0xb8ea30a7,
0xb8ea50fc, 0xb8f442b7, 0xb8e6710b, 0xb8f160df,
0xb8718182, 0xb87e007d, 0xb87b13b6, 0xb86e238d,
0xb87130b8, 0xb862514e, 0xb870436b, 0xb877708c,
0xb8766091, 0xce304661, 0xce0c09cc, 0xce748c70,
0xce863cb7, 0xce7b8191, 0xce668610, 0xcec08382,
0xce668883, 0x25a0cdd1, 0x25a1c86c, 0x058000b8,
0x054242ca, 0x0500051e, 0x2520cf00, 0x25e1c951,
0x058039ea, 0x05400e1b, 0x05009891, 0x2520c09c,
0x25a1d448, 0x05801e36, 0x05400516, 0x050039fe,
0x2520ce0b, 0x25a1d0c8, 0x058074d9, 0x05404531,
0x05031e84, 0x2560cf1a, 0x2561dda2, 0x058026a3,
0x05404c35, 0x05007851, 0x25a0d293, 0x25a1de96,
0x05808874, 0x05423bb1, 0x050030e4, 0x04680102,
0x04be0638, 0x658103c4, 0x65800993, 0x65910707,
0x04d6a53b, 0x04c00e17, 0x04da1696, 0x049089bc,
0x045b1787, 0x049aad6b, 0x04991901, 0x0493922d,
0x04518064, 0x04900dc7, 0x0417afa4, 0x04deaaa0,
0x04980123, 0x04080b1c, 0x04ca06f3, 0x04c1154d,
0x04dcb3cc, 0x65c083ae, 0x65cd94f5, 0x65c68342,
0x65c79229, 0x65c28440, 0x04dda56e, 0x6582b3ae,
0x6580a2c3, 0x6581bb63, 0x65cdb4f3, 0x65818cb5,
0x65f186b9, 0x65b30120, 0x65f12dc7, 0x65f1af0b,
0x65eec9f1, 0x65a7fed6, 0x65aa5f65, 0x65b47aae,
0x04c55723, 0x0441723d, 0x042d33ae, 0x04be3051,
0x047d32b6, 0x04e03048, 0x05a06ad7, 0x05776f59,
0x041a3435, 0x0498356a, 0x04d93917, 0x04883671,
0x04ca35a4, 0x65c73fd6, 0x658631d1, 0x65983e8c,
0x04012da1,
0x05314001, 0x05271e11, 0x1e601000, 0x1e603000,
0x1e621000, 0x1e623000, 0x1e641000, 0x1e643000,
0x1e661000, 0x1e663000, 0x1e681000, 0x1e683000,
0x1e6a1000, 0x1e6a3000, 0x1e6c1000, 0x1e6c3000,
0x1e6e1000, 0x1e6e3000, 0x1e701000, 0x1e703000,
0x1e721000, 0x1e723000, 0x1e741000, 0x1e743000,
0x1e761000, 0x1e763000, 0x1e781000, 0x1e783000,
0x1e7a1000, 0x1e7a3000, 0x1e7c1000, 0x1e7c3000,
0x1e7e1000, 0x1e7e3000, 0xf82c815f, 0xf8300047,
0xf823126d, 0xf8312070, 0xf82133cb, 0xf82551e8,
0xf83d401e, 0xf8347287, 0xf83762bc, 0xf8bb80b9,
0xf8a10217, 0xf8bf1185, 0xf8a921fc, 0xf8bd33f6,
0xf8b350bf, 0xf8ae43f0, 0xf8b0729b, 0xf8b0616c,
0xf8e983c6, 0xf8f1039b, 0xf8fe1147, 0xf8f4208a,
0xf8f83231, 0xf8f653a3, 0xf8ef4276, 0xf8f37056,
0xf8ef6186, 0xf87081ab, 0xf87703c1, 0xf8731225,
0xf86222d0, 0xf86d32aa, 0xf87d519b, 0xf87b4023,
0xf87f7278, 0xf8716389, 0xb83b80ef, 0xb83503f7,
0xb83913e2, 0xb83b2150, 0xb8373073, 0xb8305320,
0xb83a4057, 0xb830708c, 0xb83c63be, 0xb8b080db,
0xb8a901fd, 0xb8a710e4, 0xb8af22e9, 0xb8a83382,
0xb8b550bf, 0xb8bb4220, 0xb8af7344, 0xb8a862dc,
0xb8fb833b, 0xb8f70080, 0xb8e61010, 0xb8e4202f,
0xb8ea30a7, 0xb8ea50fc, 0xb8f442b7, 0xb8e6710b,
0xb8f160df, 0xb8718182, 0xb87e007d, 0xb87b13b6,
0xb86e238d, 0xb87130b8, 0xb862514e, 0xb870436b,
0xb877708c, 0xb8766091, 0xce304661, 0xce0c09cc,
0xce748c70, 0xce863cb7, 0xce7b8191, 0xce668610,
0xcec08382, 0xce668883, 0x25a0cdd1, 0x25a1c86c,
0x058000b8, 0x054242ca, 0x0500051e, 0x2520cf00,
0x25e1c951, 0x058039ea, 0x05400e1b, 0x05009891,
0x2520c09c, 0x25a1d448, 0x05801e36, 0x05400516,
0x050039fe, 0x2520ce0b, 0x25a1d0c8, 0x058074d9,
0x05404531, 0x05031e84, 0x2560cf1a, 0x2561dda2,
0x058026a3, 0x05404c35, 0x05007851, 0x25a0d293,
0x25a1de96, 0x05808874, 0x05423bb1, 0x050030e4,
0x04680102, 0x04be0638, 0x658103c4, 0x65800993,
0x65910707, 0x04d6a53b, 0x04c00e17, 0x04da1696,
0x049089bc, 0x045b1787, 0x049aad6b, 0x04991901,
0x0493922d, 0x04518064, 0x04900dc7, 0x0417afa4,
0x04deaaa0, 0x04980123, 0x04080b1c, 0x04ca06f3,
0x04c1154d, 0x04dcb3cc, 0x65c083ae, 0x65cd94f5,
0x65c68342, 0x65c79229, 0x65c28440, 0x04dda56e,
0x6582b3ae, 0x6580a2c3, 0x6581bb63, 0x65cdb4f3,
0x65818cb5, 0x65f186b9, 0x65b30120, 0x65f12dc7,
0x65f1af0b, 0x65eec9f1, 0x65a7fed6, 0x65aa5f65,
0x65b47aae, 0x04c55723, 0x0441723d, 0x042d33ae,
0x04be3051, 0x047d32b6, 0x04e03048, 0x05a06ad7,
0x05776f59, 0x041a3435, 0x0498356a, 0x04d93917,
0x04883671, 0x04ca35a4, 0x65c73fd6, 0x658631d1,
0x65983e8c, 0x04012da1,
};
// END Generated code -- do not edit