8310459: [BACKOUT] 8304450: [vectorapi] Refactor VectorShuffle implementation

Reviewed-by: thartmann, sviswanathan
This commit is contained in:
Jatin Bhateja 2023-06-26 18:35:03 +00:00
parent 815ac6eeb3
commit ff9a754109
64 changed files with 2086 additions and 2736 deletions

@ -315,10 +315,6 @@ source %{
}
}
const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) {
return false;
}
// Assert that the given node is not a variable shift.
bool assert_not_var_shift(const Node* n) {
assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
@ -6162,6 +6158,41 @@ instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector shuffle -------------------------------
instruct loadshuffle(vReg dst, vReg src) %{
match(Set dst (VectorLoadShuffle src));
format %{ "loadshuffle $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (bt == T_BYTE) {
if ($dst$$FloatRegister != $src$$FloatRegister) {
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister, $src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
}
}
} else {
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
// 4S/8S, 4I, 4F
__ uxtl($dst$$FloatRegister, __ T8H, $src$$FloatRegister, __ T8B);
if (type2aelembytes(bt) == 4) {
__ uxtl($dst$$FloatRegister, __ T4S, $dst$$FloatRegister, __ T4H);
}
} else {
assert(UseSVE > 0, "must be sve");
__ sve_vector_extend($dst$$FloatRegister, __ elemType_to_regVariant(bt),
$src$$FloatRegister, __ B);
}
}
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector rearrange -----------------------------
// Here is an example that rearranges a NEON vector with 4 ints:
@ -6184,7 +6215,6 @@ instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{
// need to lookup 2/4 bytes as a group. For VectorRearrange long, we use bsl
// to implement rearrange.
// Maybe move the shuffle preparation to VectorLoadShuffle
instruct rearrange_HS_neon(vReg dst, vReg src, vReg shuffle, vReg tmp1, vReg tmp2) %{
predicate(UseSVE == 0 &&
(Matcher::vector_element_basic_type(n) == T_SHORT ||

@ -305,10 +305,6 @@ source %{
}
}
const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) {
return false;
}
// Assert that the given node is not a variable shift.
bool assert_not_var_shift(const Node* n) {
assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
@ -4428,6 +4424,41 @@ instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector shuffle -------------------------------
instruct loadshuffle(vReg dst, vReg src) %{
match(Set dst (VectorLoadShuffle src));
format %{ "loadshuffle $dst, $src" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint length_in_bytes = Matcher::vector_length_in_bytes(this);
if (bt == T_BYTE) {
if ($dst$$FloatRegister != $src$$FloatRegister) {
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
__ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
$src$$FloatRegister, $src$$FloatRegister);
} else {
assert(UseSVE > 0, "must be sve");
__ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
}
}
} else {
if (VM_Version::use_neon_for_vector(length_in_bytes)) {
// 4S/8S, 4I, 4F
__ uxtl($dst$$FloatRegister, __ T8H, $src$$FloatRegister, __ T8B);
if (type2aelembytes(bt) == 4) {
__ uxtl($dst$$FloatRegister, __ T4S, $dst$$FloatRegister, __ T4H);
}
} else {
assert(UseSVE > 0, "must be sve");
__ sve_vector_extend($dst$$FloatRegister, __ elemType_to_regVariant(bt),
$src$$FloatRegister, __ B);
}
}
%}
ins_pipe(pipe_slow);
%}
// ------------------------------ Vector rearrange -----------------------------
// Here is an example that rearranges a NEON vector with 4 ints:
@ -4450,7 +4481,6 @@ instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{
// need to lookup 2/4 bytes as a group. For VectorRearrange long, we use bsl
// to implement rearrange.
// Maybe move the shuffle preparation to VectorLoadShuffle
instruct rearrange_HS_neon(vReg dst, vReg src, vReg shuffle, vReg tmp1, vReg tmp2) %{
predicate(UseSVE == 0 &&
(Matcher::vector_element_basic_type(n) == T_SHORT ||

@ -1025,10 +1025,6 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect*
return false;
}
const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) {
return false;
}
const RegMask* Matcher::predicate_reg_mask(void) {
return NULL;
}

@ -2189,10 +2189,6 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect*
return false;
}
const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) {
return false;
}
const RegMask* Matcher::predicate_reg_mask(void) {
return NULL;
}

@ -90,9 +90,6 @@ source %{
return false;
}
const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) {
return false;
}
%}
definitions %{
@ -3985,4 +3982,4 @@ instruct vtest_alltrue_branch(cmpOpEqNe cop, vRegMask op1, vRegMask op2, label l
__ enc_cmpEqNe_imm0_branch($cop$$cmpcode, t0, *($lbl$$label), /* is_far */ true);
%}
ins_pipe(pipe_slow);
%}
%}

@ -1529,10 +1529,6 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect*
return false;
}
const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) {
return false;
}
const RegMask* Matcher::predicate_reg_mask(void) {
return NULL;
}

@ -2151,19 +2151,6 @@ const bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect*
return false;
}
// Return true if Vector::rearrange needs preparation of the shuffle argument
const bool Matcher::vector_needs_load_shuffle(BasicType elem_bt, int vlen) {
switch (elem_bt) {
case T_BYTE: return false;
case T_SHORT: return !VM_Version::supports_avx512bw();
case T_INT: return !VM_Version::supports_avx();
case T_LONG: return vlen < 8 && !VM_Version::supports_avx512vl();
default:
ShouldNotReachHere();
return false;
}
}
MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
assert(Matcher::is_generic_vector(generic_opnd), "not generic");
bool legacy = (generic_opnd->opcode() == LEGVEC);
@ -8340,6 +8327,17 @@ instruct VectorPopulateLIndex(vec dst, rRegL src1, immI_1 src2, vec vtmp) %{
//-------------------------------- Rearrange ----------------------------------
// LoadShuffle/Rearrange for Byte
instruct loadShuffleB(vec dst) %{
predicate(Matcher::vector_element_basic_type(n) == T_BYTE);
match(Set dst (VectorLoadShuffle dst));
format %{ "vector_load_shuffle $dst, $dst" %}
ins_encode %{
// empty
%}
ins_pipe( pipe_slow );
%}
instruct rearrangeB(vec dst, vec shuffle) %{
predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
Matcher::vector_length(n) < 32);
@ -8406,7 +8404,7 @@ instruct rearrangeB_evex_vbmi(vec dst, vec src, vec shuffle) %{
instruct loadShuffleS(vec dst, vec src, vec vtmp) %{
predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
!VM_Version::supports_avx512bw());
Matcher::vector_length(n) <= 16 && !VM_Version::supports_avx512bw()); // NB! aligned with rearrangeS
match(Set dst (VectorLoadShuffle src));
effect(TEMP dst, TEMP vtmp);
format %{ "vector_load_shuffle $dst, $src\t! using $vtmp as TEMP" %}
@ -8417,7 +8415,7 @@ instruct loadShuffleS(vec dst, vec src, vec vtmp) %{
if (UseAVX == 0) {
assert(vlen_in_bytes <= 16, "required");
// Multiply each shuffle by two to get byte index
__ movdqu($vtmp$$XMMRegister, $src$$XMMRegister);
__ pmovzxbw($vtmp$$XMMRegister, $src$$XMMRegister);
__ psllw($vtmp$$XMMRegister, 1);
// Duplicate to create 2 copies of byte index
@ -8432,7 +8430,8 @@ instruct loadShuffleS(vec dst, vec src, vec vtmp) %{
assert(UseAVX > 1 || vlen_in_bytes <= 16, "required");
int vlen_enc = vector_length_encoding(this);
// Multiply each shuffle by two to get byte index
__ vpsllw($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc);
__ vpmovzxbw($vtmp$$XMMRegister, $src$$XMMRegister, vlen_enc);
__ vpsllw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 1, vlen_enc);
// Duplicate to create 2 copies of byte index
__ vpsllw($dst$$XMMRegister, $vtmp$$XMMRegister, 8, vlen_enc);
@ -8479,6 +8478,21 @@ instruct rearrangeS_avx(legVec dst, legVec src, vec shuffle, legVec vtmp1, legVe
ins_pipe( pipe_slow );
%}
instruct loadShuffleS_evex(vec dst, vec src) %{
predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
VM_Version::supports_avx512bw());
match(Set dst (VectorLoadShuffle src));
format %{ "vector_load_shuffle $dst, $src" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
if (!VM_Version::supports_avx512vl()) {
vlen_enc = Assembler::AVX_512bit;
}
__ vpmovzxbw($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
instruct rearrangeS_evex(vec dst, vec src, vec shuffle) %{
predicate(Matcher::vector_element_basic_type(n) == T_SHORT &&
VM_Version::supports_avx512bw());
@ -8509,7 +8523,7 @@ instruct loadShuffleI(vec dst, vec src, vec vtmp) %{
// only byte shuffle instruction available on these platforms
// Duplicate and multiply each shuffle by 4
__ movdqu($vtmp$$XMMRegister, $src$$XMMRegister);
__ pmovzxbd($vtmp$$XMMRegister, $src$$XMMRegister);
__ pshuflw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0);
__ pshufhw($vtmp$$XMMRegister, $vtmp$$XMMRegister, 0xA0);
__ psllw($vtmp$$XMMRegister, 2);
@ -8538,6 +8552,18 @@ instruct rearrangeI(vec dst, vec shuffle) %{
ins_pipe( pipe_slow );
%}
instruct loadShuffleI_avx(vec dst, vec src) %{
predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
UseAVX > 0);
match(Set dst (VectorLoadShuffle src));
format %{ "vector_load_shuffle $dst, $src" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
__ vpmovzxbd($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
instruct rearrangeI_avx(vec dst, vec src, vec shuffle) %{
predicate((Matcher::vector_element_basic_type(n) == T_INT || Matcher::vector_element_basic_type(n) == T_FLOAT) &&
UseAVX > 0);
@ -8567,7 +8593,8 @@ instruct loadShuffleL(vec dst, vec src, vec vtmp) %{
// only double word shuffle instruction available on these platforms
// Multiply each shuffle by two to get double word index
__ vpsllq($vtmp$$XMMRegister, $src$$XMMRegister, 1, vlen_enc);
__ vpmovzxbq($vtmp$$XMMRegister, $src$$XMMRegister, vlen_enc);
__ vpsllq($vtmp$$XMMRegister, $vtmp$$XMMRegister, 1, vlen_enc);
// Duplicate each double word shuffle
__ vpsllq($dst$$XMMRegister, $vtmp$$XMMRegister, 32, vlen_enc);
@ -8593,6 +8620,20 @@ instruct rearrangeL(vec dst, vec src, vec shuffle) %{
ins_pipe( pipe_slow );
%}
instruct loadShuffleL_evex(vec dst, vec src) %{
predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
(Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl()));
match(Set dst (VectorLoadShuffle src));
format %{ "vector_load_shuffle $dst, $src" %}
ins_encode %{
assert(UseAVX > 2, "required");
int vlen_enc = vector_length_encoding(this);
__ vpmovzxbq($dst$$XMMRegister, $src$$XMMRegister, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
instruct rearrangeL_evex(vec dst, vec src, vec shuffle) %{
predicate(is_double_word_type(Matcher::vector_element_basic_type(n)) && // T_LONG, T_DOUBLE
(Matcher::vector_length(n) == 8 || VM_Version::supports_avx512vl()));

@ -965,6 +965,24 @@ class methodHandle;
"Ljdk/internal/vm/vector/VectorSupport$VectorPayload;") \
do_name(vector_frombits_coerced_name, "fromBitsCoerced") \
\
do_intrinsic(_VectorShuffleIota, jdk_internal_vm_vector_VectorSupport, vector_shuffle_step_iota_name, vector_shuffle_step_iota_sig, F_S) \
do_signature(vector_shuffle_step_iota_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorSpecies;" \
"IIII" \
"Ljdk/internal/vm/vector/VectorSupport$ShuffleIotaOperation;)" \
"Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;") \
do_name(vector_shuffle_step_iota_name, "shuffleIota") \
\
do_intrinsic(_VectorShuffleToVector, jdk_internal_vm_vector_VectorSupport, vector_shuffle_to_vector_name, vector_shuffle_to_vector_sig, F_S) \
do_signature(vector_shuffle_to_vector_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
"Ljava/lang/Class;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;" \
"ILjdk/internal/vm/vector/VectorSupport$ShuffleToVectorOperation;)" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;") \
do_name(vector_shuffle_to_vector_name, "shuffleToVector") \
\
do_intrinsic(_VectorLoadOp, jdk_internal_vm_vector_VectorSupport, vector_load_op_name, vector_load_op_sig, F_S) \
do_signature(vector_load_op_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \

@ -747,6 +747,8 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method) {
case vmIntrinsics::_VectorBinaryOp:
case vmIntrinsics::_VectorTernaryOp:
case vmIntrinsics::_VectorFromBitsCoerced:
case vmIntrinsics::_VectorShuffleIota:
case vmIntrinsics::_VectorShuffleToVector:
case vmIntrinsics::_VectorLoadOp:
case vmIntrinsics::_VectorLoadMaskedOp:
case vmIntrinsics::_VectorStoreOp:

@ -904,7 +904,7 @@ class GraphKit : public Phase {
// Vector API support (implemented in vectorIntrinsics.cpp)
Node* box_vector(Node* in, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool deoptimize_on_exception = false);
Node* unbox_vector(Node* in, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem);
Node* unbox_vector(Node* in, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector = false);
Node* vector_shift_count(Node* cnt, int shift_op, BasicType bt, int num_elem);
};

@ -698,8 +698,12 @@ bool LibraryCallKit::try_to_inline(int predicate) {
return inline_vector_nary_operation(3);
case vmIntrinsics::_VectorFromBitsCoerced:
return inline_vector_frombits_coerced();
case vmIntrinsics::_VectorShuffleIota:
return inline_vector_shuffle_iota();
case vmIntrinsics::_VectorMaskOp:
return inline_vector_mask_operation();
case vmIntrinsics::_VectorShuffleToVector:
return inline_vector_shuffle_to_vector();
case vmIntrinsics::_VectorLoadOp:
return inline_vector_mem_operation(/*is_store=*/false);
case vmIntrinsics::_VectorLoadMaskedOp:

@ -345,6 +345,8 @@ class LibraryCallKit : public GraphKit {
// Vector API support
bool inline_vector_nary_operation(int n);
bool inline_vector_frombits_coerced();
bool inline_vector_shuffle_to_vector();
bool inline_vector_shuffle_iota();
bool inline_vector_mask_operation();
bool inline_vector_mem_operation(bool is_store);
bool inline_vector_mem_masked_operation(bool is_store);

@ -340,8 +340,6 @@ public:
static const bool vector_needs_partial_operations(Node* node, const TypeVect* vt);
static const bool vector_needs_load_shuffle(BasicType elem_bt, int vlen);
static const RegMask* predicate_reg_mask(void);
static const TypeVectMask* predicate_reg_type(const Type* elemTy, int length);

@ -36,6 +36,11 @@ static bool is_vector_mask(ciKlass* klass) {
return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
}
static bool is_vector_shuffle(ciKlass* klass) {
return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
}
void PhaseVector::optimize_vector_boxes() {
Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]);
@ -459,6 +464,8 @@ void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) {
if (is_vector_mask(from_kls)) {
bt = T_BOOLEAN;
} else if (is_vector_shuffle(from_kls)) {
bt = T_BYTE;
}
ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbols::payload_name(),
@ -503,6 +510,9 @@ void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) {
if (is_vector_mask(from_kls)) {
vec_val_load = gvn.transform(new VectorLoadMaskNode(vec_val_load, TypeVect::makemask(masktype, num_elem)));
} else if (is_vector_shuffle(from_kls) && !vec_unbox->is_shuffle_to_vector()) {
assert(vec_unbox->bottom_type()->is_vect()->element_basic_type() == masktype, "expect shuffle type consistency");
vec_val_load = gvn.transform(new VectorLoadShuffleNode(vec_val_load, TypeVect::make(masktype, num_elem)));
}
gvn.hash_delete(vec_unbox);

@ -63,6 +63,10 @@ static bool is_vector_mask(ciKlass* klass) {
return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
}
static bool is_vector_shuffle(ciKlass* klass) {
return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
}
bool LibraryCallKit::arch_supports_vector_rotate(int opc, int num_elem, BasicType elem_bt,
VectorMaskUseType mask_use_type, bool has_scalar_args) {
bool is_supported = true;
@ -156,7 +160,7 @@ Node* GraphKit::box_vector(Node* vector, const TypeInstPtr* vbox_type, BasicType
return gvn().transform(vbox);
}
Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem) {
Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType elem_bt, int num_elem, bool shuffle_to_vector) {
assert(EnableVectorSupport, "");
const TypeInstPtr* vbox_type_v = gvn().type(v)->is_instptr();
if (vbox_type->instance_klass() != vbox_type_v->instance_klass()) {
@ -167,7 +171,7 @@ Node* GraphKit::unbox_vector(Node* v, const TypeInstPtr* vbox_type, BasicType el
}
assert(check_vbox(vbox_type), "");
const TypeVect* vt = TypeVect::make(elem_bt, num_elem, is_vector_mask(vbox_type->instance_klass()));
Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory()));
Node* unbox = gvn().transform(new VectorUnboxNode(C, vt, v, merged_memory(), shuffle_to_vector));
return unbox;
}
@ -578,6 +582,103 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) {
return true;
}
// <Sh extends VectorShuffle<E>, E>
// Sh ShuffleIota(Class<?> E, Class<?> shuffleClass, Vector.Species<E> s, int length,
// int start, int step, int wrap, ShuffleIotaOperation<Sh, E> defaultImpl)
bool LibraryCallKit::inline_vector_shuffle_iota() {
const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
const TypeInt* start_val = gvn().type(argument(4))->isa_int();
const TypeInt* step_val = gvn().type(argument(5))->isa_int();
const TypeInt* wrap = gvn().type(argument(6))->isa_int();
Node* start = argument(4);
Node* step = argument(5);
if (shuffle_klass == nullptr || vlen == nullptr || start_val == nullptr || step_val == nullptr || wrap == nullptr) {
return false; // dead code
}
if (!vlen->is_con() || !is_power_of_2(vlen->get_con()) ||
shuffle_klass->const_oop() == nullptr || !wrap->is_con()) {
return false; // not enough info for intrinsification
}
if (!is_klass_initialized(shuffle_klass)) {
if (C->print_intrinsics()) {
tty->print_cr(" ** klass argument not initialized");
}
return false;
}
int do_wrap = wrap->get_con();
int num_elem = vlen->get_con();
BasicType elem_bt = T_BYTE;
if (!arch_supports_vector(VectorNode::replicate_opcode(elem_bt), num_elem, elem_bt, VecMaskNotUsed)) {
return false;
}
if (!arch_supports_vector(Op_AddVB, num_elem, elem_bt, VecMaskNotUsed)) {
return false;
}
if (!arch_supports_vector(Op_AndV, num_elem, elem_bt, VecMaskNotUsed)) {
return false;
}
if (!arch_supports_vector(Op_VectorLoadConst, num_elem, elem_bt, VecMaskNotUsed)) {
return false;
}
if (!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad)) {
return false;
}
if (!arch_supports_vector(Op_VectorMaskCmp, num_elem, elem_bt, VecMaskUseStore)) {
return false;
}
const Type * type_bt = Type::get_const_basic_type(elem_bt);
const TypeVect * vt = TypeVect::make(type_bt, num_elem);
Node* res = gvn().transform(new VectorLoadConstNode(gvn().makecon(TypeInt::ZERO), vt));
if(!step_val->is_con() || !is_power_of_2(step_val->get_con())) {
Node* bcast_step = gvn().transform(VectorNode::scalar2vector(step, num_elem, type_bt));
res = gvn().transform(VectorNode::make(Op_MulI, res, bcast_step, num_elem, elem_bt));
} else if (step_val->get_con() > 1) {
Node* cnt = gvn().makecon(TypeInt::make(log2i_exact(step_val->get_con())));
Node* shift_cnt = vector_shift_count(cnt, Op_LShiftI, elem_bt, num_elem);
res = gvn().transform(VectorNode::make(Op_LShiftVB, res, shift_cnt, vt));
}
if (!start_val->is_con() || start_val->get_con() != 0) {
Node* bcast_start = gvn().transform(VectorNode::scalar2vector(start, num_elem, type_bt));
res = gvn().transform(VectorNode::make(Op_AddI, res, bcast_start, num_elem, elem_bt));
}
Node * mod_val = gvn().makecon(TypeInt::make(num_elem-1));
Node * bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, type_bt));
if(do_wrap) {
// Wrap the indices greater than lane count.
res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
} else {
ConINode* pred_node = (ConINode*)gvn().makecon(TypeInt::make(BoolTest::ge));
Node * lane_cnt = gvn().makecon(TypeInt::make(num_elem));
Node * bcast_lane_cnt = gvn().transform(VectorNode::scalar2vector(lane_cnt, num_elem, type_bt));
const TypeVect* vmask_type = TypeVect::makemask(elem_bt, num_elem);
Node* mask = gvn().transform(new VectorMaskCmpNode(BoolTest::ge, bcast_lane_cnt, res, pred_node, vmask_type));
// Make the indices greater than lane count as -ve values. This matches the java side implementation.
res = gvn().transform(VectorNode::make(Op_AndI, res, bcast_mod, num_elem, elem_bt));
Node * biased_val = gvn().transform(VectorNode::make(Op_SubI, res, bcast_lane_cnt, num_elem, elem_bt));
res = gvn().transform(new VectorBlendNode(biased_val, res, mask));
}
ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
// Wrap it up in VectorBox to keep object type information.
res = box_vector(res, shuffle_box_type, elem_bt, num_elem);
set_result(res);
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
return true;
}
// <E, M>
// long maskReductionCoerced(int oper, Class<? extends M> maskClass, Class<?> elemClass,
// int length, M m, VectorMaskOp<M> defaultImpl)
@ -615,7 +716,7 @@ bool LibraryCallKit::inline_vector_mask_operation() {
const Type* elem_ty = Type::get_const_basic_type(elem_bt);
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem);
Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true);
if (mask_vec == nullptr) {
if (C->print_intrinsics()) {
tty->print_cr(" ** unbox failed mask=%s",
@ -638,6 +739,71 @@ bool LibraryCallKit::inline_vector_mask_operation() {
return true;
}
// public static
// <V,
// Sh extends VectorShuffle<E>,
// E>
// V shuffleToVector(Class<? extends Vector<E>> vclass, Class<E> elementType,
// Class<? extends Sh> shuffleClass, Sh s, int length,
// ShuffleToVectorOperation<V, Sh, E> defaultImpl)
bool LibraryCallKit::inline_vector_shuffle_to_vector() {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
const TypeInstPtr* elem_klass = gvn().type(argument(1))->isa_instptr();
const TypeInstPtr* shuffle_klass = gvn().type(argument(2))->isa_instptr();
Node* shuffle = argument(3);
const TypeInt* vlen = gvn().type(argument(4))->isa_int();
if (vector_klass == nullptr || elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr) {
return false; // dead code
}
if (!vlen->is_con() || vector_klass->const_oop() == nullptr || shuffle_klass->const_oop() == nullptr) {
return false; // not enough info for intrinsification
}
if (!is_klass_initialized(shuffle_klass) || !is_klass_initialized(vector_klass) ) {
if (C->print_intrinsics()) {
tty->print_cr(" ** klass argument not initialized");
}
return false;
}
int num_elem = vlen->get_con();
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
BasicType elem_bt = elem_type->basic_type();
if (num_elem < 4) {
return false;
}
int cast_vopc = VectorCastNode::opcode(-1, T_BYTE); // from shuffle of type T_BYTE
// Make sure that cast is implemented to particular type/size combination.
if (!arch_supports_vector(cast_vopc, num_elem, elem_bt, VecMaskNotUsed)) {
if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s",
cast_vopc, num_elem, type2name(elem_bt));
}
return false;
}
ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
// Unbox shuffle with true flag to indicate its load shuffle to vector
// shuffle is a byte array
Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, T_BYTE, num_elem, true);
// cast byte to target element type
shuffle_vec = gvn().transform(VectorCastNode::make(cast_vopc, shuffle_vec, elem_bt, num_elem));
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
const TypeInstPtr* vec_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
// Box vector
Node* res = box_vector(shuffle_vec, vec_box_type, elem_bt, num_elem);
set_result(res);
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
return true;
}
// public static
// <M,
// S extends VectorSpecies<E>,
@ -1892,22 +2058,14 @@ bool LibraryCallKit::inline_vector_rearrange() {
}
return false; // should be primitive type
}
BasicType elem_bt = elem_type->basic_type();
BasicType shuffle_bt = elem_bt;
if (shuffle_bt == T_FLOAT) {
shuffle_bt = T_INT;
} else if (shuffle_bt == T_DOUBLE) {
shuffle_bt = T_LONG;
}
int num_elem = vlen->get_con();
bool need_load_shuffle = Matcher::vector_needs_load_shuffle(shuffle_bt, num_elem);
if (need_load_shuffle && !arch_supports_vector(Op_VectorLoadShuffle, num_elem, shuffle_bt, VecMaskNotUsed)) {
if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)) {
if (C->print_intrinsics()) {
tty->print_cr(" ** not supported: arity=0 op=load/shuffle vlen=%d etype=%s ismask=no",
num_elem, type2name(shuffle_bt));
num_elem, type2name(elem_bt));
}
return false; // not supported
}
@ -1944,8 +2102,6 @@ bool LibraryCallKit::inline_vector_rearrange() {
Node* v1 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
Node* shuffle = unbox_vector(argument(6), shbox_type, shuffle_bt, num_elem);
const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
const TypeVect* st = TypeVect::make(shuffle_bt, num_elem);
if (v1 == nullptr || shuffle == nullptr) {
return false; // operand unboxing failed
@ -1965,16 +2121,13 @@ bool LibraryCallKit::inline_vector_rearrange() {
}
}
if (need_load_shuffle) {
shuffle = gvn().transform(new VectorLoadShuffleNode(shuffle, st));
}
Node* rearrange = new VectorRearrangeNode(v1, shuffle);
if (is_masked_op) {
if (use_predicate) {
rearrange->add_req(mask);
rearrange->add_flag(Node::Flag_is_predicated_vector);
} else {
const TypeVect* vt = v1->bottom_type()->is_vect();
rearrange = gvn().transform(rearrange);
Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
@ -2268,7 +2421,9 @@ bool LibraryCallKit::inline_vector_convert() {
ciKlass* vbox_klass_from = vector_klass_from->const_oop()->as_instance()->java_lang_Class_klass();
ciKlass* vbox_klass_to = vector_klass_to->const_oop()->as_instance()->java_lang_Class_klass();
if (is_vector_shuffle(vbox_klass_from)) {
return false; // vector shuffles aren't supported
}
bool is_mask = is_vector_mask(vbox_klass_from);
ciType* elem_type_from = elem_klass_from->const_oop()->as_instance()->java_mirror_type();

@ -1753,13 +1753,19 @@ Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) {
if (in_vt->length() == out_vt->length()) {
Node* value = vbox->in(VectorBoxNode::Value);
bool is_vector_mask = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
bool is_vector_mask = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
if (is_vector_mask) {
// VectorUnbox (VectorBox vmask) ==> VectorMaskCast vmask
const TypeVect* vmask_type = TypeVect::makemask(out_vt->element_basic_type(), out_vt->length());
return new VectorMaskCastNode(value, vmask_type);
} else if (is_vector_shuffle) {
if (!is_shuffle_to_vector()) {
// VectorUnbox (VectorBox vshuffle) ==> VectorLoadShuffle vshuffle
return new VectorLoadShuffleNode(value, out_vt);
}
} else {
// Vector type mismatch is only supported for masks, but sometimes it happens in pathological cases.
// Vector type mismatch is only supported for masks and shuffles, but sometimes it happens in pathological cases.
}
} else {
// Vector length mismatch.

@ -1482,8 +1482,11 @@ class VectorRearrangeNode : public VectorNode {
class VectorLoadShuffleNode : public VectorNode {
public:
VectorLoadShuffleNode(Node* in, const TypeVect* vt)
: VectorNode(in, vt) {}
: VectorNode(in, vt) {
assert(in->bottom_type()->is_vect()->element_basic_type() == T_BYTE, "must be BYTE");
}
int GetOutShuffleSize() const { return type2aelembytes(vect_type()->element_basic_type()); }
virtual int Opcode() const;
};
@ -1714,11 +1717,14 @@ class VectorBoxAllocateNode : public CallStaticJavaNode {
};
class VectorUnboxNode : public VectorNode {
private:
bool _shuffle_to_vector;
protected:
uint size_of() const { return sizeof(*this); }
public:
VectorUnboxNode(Compile* C, const TypeVect* vec_type, Node* obj, Node* mem)
VectorUnboxNode(Compile* C, const TypeVect* vec_type, Node* obj, Node* mem, bool shuffle_to_vector)
: VectorNode(mem, obj, vec_type) {
_shuffle_to_vector = shuffle_to_vector;
init_class_id(Class_VectorUnbox);
init_flags(Flag_is_macro);
C->add_macro_node(this);
@ -1729,6 +1735,7 @@ class VectorUnboxNode : public VectorNode {
Node* mem() const { return in(1); }
virtual Node* Identity(PhaseGVN* phase);
Node* Ideal(PhaseGVN* phase, bool can_reshape);
bool is_shuffle_to_vector() { return _shuffle_to_vector; }
};
class RotateRightVNode : public VectorNode {

@ -73,6 +73,10 @@ bool VectorSupport::is_vector_mask(Klass* klass) {
return klass->is_subclass_of(vmClasses::vector_VectorMask_klass());
}
bool VectorSupport::is_vector_shuffle(Klass* klass) {
return klass->is_subclass_of(vmClasses::vector_VectorShuffle_klass());
}
BasicType VectorSupport::klass2bt(InstanceKlass* ik) {
assert(ik->is_subclass_of(vmClasses::vector_VectorPayload_klass()), "%s not a VectorPayload", ik->name()->as_C_string());
fieldDescriptor fd; // find_field initializes fd if found
@ -83,7 +87,9 @@ BasicType VectorSupport::klass2bt(InstanceKlass* ik) {
assert(fd.is_static(), "");
assert(fd.offset() > 0, "");
if (is_vector_mask(ik)) {
if (is_vector_shuffle(ik)) {
return T_BYTE;
} else if (is_vector_mask(ik)) {
return T_BOOLEAN;
} else { // vector and mask
oop value = ik->java_mirror()->obj_field(fd.offset());

@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -147,5 +147,6 @@ class VectorSupport : AllStatic {
static bool is_vector(Klass* klass);
static bool is_vector_mask(Klass* klass);
static bool is_vector_shuffle(Klass* klass);
};
#endif // SHARE_PRIMS_VECTORSUPPORT_HPP

@ -227,6 +227,42 @@ public class VectorSupport {
return defaultImpl.apply(offset, limit);
}
/* ============================================================================ */
public interface ShuffleIotaOperation<S extends VectorSpecies<?>,
SH extends VectorShuffle<?>> {
SH apply(int length, int start, int step, S s);
}
@IntrinsicCandidate
public static
<E,
S extends VectorSpecies<E>,
SH extends VectorShuffle<E>>
SH shuffleIota(Class<E> eClass, Class<? extends SH> shClass, S s,
int length,
int start, int step, int wrap,
ShuffleIotaOperation<S, SH> defaultImpl) {
assert isNonCapturingLambda(defaultImpl) : defaultImpl;
return defaultImpl.apply(length, start, step, s);
}
public interface ShuffleToVectorOperation<V extends Vector<?>,
SH extends VectorShuffle<?>> {
V apply(SH sh);
}
@IntrinsicCandidate
public static
<V extends Vector<E>,
SH extends VectorShuffle<E>,
E>
V shuffleToVector(Class<? extends Vector<E>> vClass, Class<E> eClass, Class<? extends SH> shClass, SH sh,
int length,
ShuffleToVectorOperation<V, SH> defaultImpl) {
assert isNonCapturingLambda(defaultImpl) : defaultImpl;
return defaultImpl.apply(sh);
}
/* ============================================================================ */
public interface IndexOperation<V extends Vector<?>,
S extends VectorSpecies<?>> {

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -26,16 +26,53 @@ package jdk.incubator.vector;
import java.util.function.IntUnaryOperator;
import jdk.internal.vm.annotation.ForceInline;
import jdk.internal.vm.vector.VectorSupport;
abstract class AbstractShuffle<E> extends VectorShuffle<E> {
static final IntUnaryOperator IDENTITY = i -> i;
// Internal representation allows for a maximum index of E.MAX_VALUE - 1
// Internal representation allows for a maximum index of 256
// Values are clipped to [-VLENGTH..VLENGTH-1].
AbstractShuffle(Object indices) {
super(indices);
AbstractShuffle(int length, byte[] reorder) {
super(reorder);
assert(length == reorder.length);
assert(indexesInRange(reorder));
}
AbstractShuffle(int length, int[] reorder) {
this(length, reorder, 0);
}
AbstractShuffle(int length, int[] reorder, int offset) {
super(prepare(length, reorder, offset));
}
AbstractShuffle(int length, IntUnaryOperator f) {
super(prepare(length, f));
}
private static byte[] prepare(int length, int[] reorder, int offset) {
byte[] a = new byte[length];
for (int i = 0; i < length; i++) {
int si = reorder[offset + i];
si = partiallyWrapIndex(si, length);
a[i] = (byte) si;
}
return a;
}
private static byte[] prepare(int length, IntUnaryOperator f) {
byte[] a = new byte[length];
for (int i = 0; i < a.length; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, length);
a[i] = (byte) si;
}
return a;
}
byte[] reorder() {
return (byte[])getPayload();
}
/*package-private*/
@ -47,90 +84,89 @@ abstract class AbstractShuffle<E> extends VectorShuffle<E> {
return vspecies();
}
/*package-private*/
abstract AbstractVector<?> toBitsVector();
final AbstractVector<?> toBitsVectorTemplate() {
AbstractSpecies<?> dsp = vspecies().asIntegral();
Class<?> etype = dsp.elementType();
Class<?> rvtype = dsp.dummyVector().getClass();
return VectorSupport.convert(VectorSupport.VECTOR_OP_REINTERPRET,
getClass(), etype, length(),
rvtype, etype, length(),
this, dsp,
(v, s) -> v.toBitsVector0());
}
abstract AbstractVector<?> toBitsVector0();
@Override
@ForceInline
public final Vector<E> toVector() {
return toBitsVector().castShape(vspecies(), 0);
}
@Override
@ForceInline
public final int[] toArray() {
int[] res = new int[length()];
intoArray(res, 0);
return res;
}
@Override
@ForceInline
public final <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
if (length() != s.length()) {
throw new IllegalArgumentException("VectorShuffle length and species length differ");
public void intoArray(int[] a, int offset) {
byte[] reorder = reorder();
int vlen = reorder.length;
for (int i = 0; i < vlen; i++) {
int sourceIndex = reorder[i];
assert(sourceIndex >= -vlen && sourceIndex < vlen);
a[offset + i] = sourceIndex;
}
return toBitsVector().toShuffle((AbstractSpecies<F>) s);
}
@Override
@ForceInline
public int[] toArray() {
byte[] reorder = reorder();
int[] a = new int[reorder.length];
intoArray(a, 0);
return a;
}
/*package-private*/
@ForceInline
final
AbstractVector<E>
toVectorTemplate() {
// Note that the values produced by laneSource
// are already clipped. At this point we convert
// them from internal ints (or bytes) into the ETYPE.
// FIXME: Use a conversion intrinsic for this operation.
// https://bugs.openjdk.org/browse/JDK-8225740
return (AbstractVector<E>) vspecies().fromIntValues(toArray());
}
@ForceInline
public final VectorShuffle<E> checkIndexes() {
if (VectorIntrinsics.VECTOR_ACCESS_OOB_CHECK == 0) {
return this;
}
Vector<?> shufvec = this.toBitsVector();
VectorMask<?> vecmask = shufvec.compare(VectorOperators.LT, 0);
Vector<E> shufvec = this.toVector();
VectorMask<E> vecmask = shufvec.compare(VectorOperators.LT, vspecies().zero());
if (vecmask.anyTrue()) {
int[] indices = toArray();
throw checkIndexFailed(indices[vecmask.firstTrue()], length());
byte[] reorder = reorder();
throw checkIndexFailed(reorder[vecmask.firstTrue()], length());
}
return this;
}
@Override
@ForceInline
public final VectorShuffle<E> wrapIndexes() {
Vector<E> shufvec = this.toVector();
VectorMask<E> vecmask = shufvec.compare(VectorOperators.LT, vspecies().zero());
if (vecmask.anyTrue()) {
// FIXME: vectorize this
byte[] reorder = reorder();
return wrapAndRebuild(reorder);
}
return this;
}
@ForceInline
public final VectorShuffle<E> wrapAndRebuild(byte[] oldReorder) {
int length = oldReorder.length;
byte[] reorder = new byte[length];
for (int i = 0; i < length; i++) {
int si = oldReorder[i];
// FIXME: This does not work unless it's a power of 2.
if ((length & (length - 1)) == 0) {
si += si & length; // power-of-two optimization
} else if (si < 0) {
// non-POT code requires a conditional add
si += length;
}
assert(si >= 0 && si < length);
reorder[i] = (byte) si;
}
return vspecies().dummyVector().shuffleFromBytes(reorder);
}
@ForceInline
public final VectorMask<E> laneIsValid() {
Vector<?> shufvec = this.toBitsVector();
return shufvec.compare(VectorOperators.GE, 0)
.cast(vspecies());
}
@ForceInline
@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public final VectorShuffle<E> rearrange(VectorShuffle<E> shuffle) {
Vector v = toBitsVector();
return (VectorShuffle<E>) v.rearrange(shuffle.cast(vspecies().asIntegral()))
.toShuffle()
.cast(vspecies());
}
@ForceInline
@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public final VectorShuffle<E> wrapIndexes() {
Vector v = toBitsVector();
if ((length() & (length() - 1)) == 0) {
v = v.lanewise(VectorOperators.AND, length() - 1);
} else {
v = v.blend(v.lanewise(VectorOperators.ADD, length()),
v.compare(VectorOperators.LT, 0));
}
return (VectorShuffle<E>) v.toShuffle().cast(vspecies());
Vector<E> shufvec = this.toVector();
return shufvec.compare(VectorOperators.GE, vspecies().zero());
}
@Override
@ -184,4 +220,21 @@ abstract class AbstractShuffle<E> extends VectorShuffle<E> {
String msg = "required an index in [0.."+max+"] but found "+index;
return new IndexOutOfBoundsException(msg);
}
static boolean indexesInRange(byte[] reorder) {
int length = reorder.length;
for (byte si : reorder) {
if (si >= length || si < -length) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(reorder));
throw new AssertionError(msg);
}
return false;
}
}
return true;
}
}

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -48,8 +48,6 @@ abstract class AbstractSpecies<E> extends jdk.internal.vm.vector.VectorSupport.V
@Stable
final Class<? extends AbstractMask<E>> maskType;
@Stable
final Class<? extends AbstractShuffle<E>> shuffleType;
@Stable
final Function<Object, ? extends AbstractVector<E>> vectorFactory;
@Stable
@ -63,13 +61,11 @@ abstract class AbstractSpecies<E> extends jdk.internal.vm.vector.VectorSupport.V
LaneType laneType,
Class<? extends AbstractVector<E>> vectorType,
Class<? extends AbstractMask<E>> maskType,
Class<? extends AbstractShuffle<E>> shuffleType,
Function<Object, ? extends AbstractVector<E>> vectorFactory) {
this.vectorShape = vectorShape;
this.laneType = laneType;
this.vectorType = vectorType;
this.maskType = maskType;
this.shuffleType = shuffleType;
this.vectorFactory = vectorFactory;
// derived values:
@ -166,11 +162,6 @@ abstract class AbstractSpecies<E> extends jdk.internal.vm.vector.VectorSupport.V
return maskType;
}
@ForceInline
final Class<? extends AbstractShuffle<E>> shuffleType() {
return shuffleType;
}
@Override
@ForceInline
public final int elementSize() {

@ -1,5 +1,5 @@
/*
* Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -188,57 +188,12 @@ abstract class AbstractVector<E> extends Vector<E> {
abstract AbstractMask<E> maskFromArray(boolean[] bits);
abstract <F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp);
abstract AbstractShuffle<E> iotaShuffle();
/*package-private*/
@ForceInline
final <F> VectorShuffle<F> toShuffleTemplate(AbstractSpecies<F> dsp) {
Class<?> etype = vspecies().elementType();
Class<?> dvtype = dsp.shuffleType();
Class<?> dtype = dsp.asIntegral().elementType();
int dlength = dsp.dummyVector().length();
return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
getClass(), etype, length(),
dvtype, dtype, dlength,
this, dsp,
AbstractVector::toShuffle0);
}
abstract AbstractShuffle<E> iotaShuffle(int start, int step, boolean wrap);
abstract <F> VectorShuffle<F> toShuffle0(AbstractSpecies<F> dsp);
@ForceInline
public final
VectorShuffle<E> toShuffle() {
return toShuffle(vspecies());
}
abstract VectorShuffle<E> iotaShuffle();
@ForceInline
@SuppressWarnings({"rawtypes", "unchecked"})
final VectorShuffle<E> iotaShuffle(int start, int step, boolean wrap) {
if (start == 0 && step == 1) {
return iotaShuffle();
}
if ((length() & (length() - 1)) != 0) {
return wrap ? shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i * step + start, length())))
: shuffleFromOp(i -> i * step + start);
}
AbstractSpecies<?> species = vspecies().asIntegral();
Vector iota = species.iota();
iota = iota.lanewise(VectorOperators.MUL, step)
.lanewise(VectorOperators.ADD, start);
Vector wrapped = iota.lanewise(VectorOperators.AND, length() - 1);
if (!wrap) {
Vector wrappedEx = wrapped.lanewise(VectorOperators.SUB, length());
VectorMask<?> mask = wrapped.compare(VectorOperators.EQ, iota);
wrapped = wrappedEx.blend(wrapped, mask);
}
return ((AbstractVector) wrapped).toShuffle(vspecies());
}
/*do not alias this byte array*/
abstract AbstractShuffle<E> shuffleFromBytes(byte[] reorder);
abstract AbstractShuffle<E> shuffleFromArray(int[] indexes, int i);

@ -141,9 +141,24 @@ final class Byte128Vector extends ByteVector {
@ForceInline
Byte128Shuffle iotaShuffle() { return Byte128Shuffle.IOTA; }
@ForceInline
Byte128Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Byte128Shuffle)VectorSupport.shuffleIota(ETYPE, Byte128Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Byte128Shuffle)VectorSupport.shuffleIota(ETYPE, Byte128Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Byte128Shuffle shuffleFromArray(int[] indices, int i) { return new Byte128Shuffle(indices, i); }
Byte128Shuffle shuffleFromBytes(byte[] reorder) { return new Byte128Shuffle(reorder); }
@Override
@ForceInline
Byte128Shuffle shuffleFromArray(int[] indexes, int i) { return new Byte128Shuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class Byte128Vector extends ByteVector {
return (long) super.reduceLanesTemplate(op, Byte128Mask.class, (Byte128Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Byte> toShuffle() {
return super.toShuffleTemplate(Byte128Shuffle.class); // specialize
}
// Specialized unary testing
@ -791,26 +804,23 @@ final class Byte128Vector extends ByteVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Byte> ETYPE = byte.class; // used by the JVM
Byte128Shuffle(byte[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Byte128Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Byte128Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Byte128Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Byte128Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Byte128Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
byte[] indices() {
return (byte[])getPayload();
public Byte128Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public ByteSpecies vspecies() {
return VSPECIES;
}
@ -825,76 +835,33 @@ final class Byte128Vector extends ByteVector {
@Override
@ForceInline
Byte128Vector toBitsVector() {
return (Byte128Vector) super.toBitsVectorTemplate();
public Byte128Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Byte128Shuffle.class, this, VLENGTH,
(s) -> ((Byte128Vector)(((AbstractShuffle<Byte>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
ByteVector toBitsVector0() {
return Byte128Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
VectorSpecies<Integer> species = IntVector.SPECIES_128;
Vector<Byte> v = toBitsVector();
v.convertShape(VectorOperators.B2I, species, 0)
.reinterpretAsInts()
.intoArray(a, offset);
v.convertShape(VectorOperators.B2I, species, 1)
.reinterpretAsInts()
.intoArray(a, offset + species.length());
v.convertShape(VectorOperators.B2I, species, 2)
.reinterpretAsInts()
.intoArray(a, offset + species.length() * 2);
v.convertShape(VectorOperators.B2I, species, 3)
.reinterpretAsInts()
.intoArray(a, offset + species.length() * 3);
}
private static byte[] prepare(int[] indices, int offset) {
byte[] a = new byte[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (byte)si;
public Byte128Shuffle rearrange(VectorShuffle<Byte> shuffle) {
Byte128Shuffle s = (Byte128Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static byte[] prepare(IntUnaryOperator f) {
byte[] a = new byte[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (byte)si;
}
return a;
}
private static boolean indicesInRange(byte[] indices) {
int length = indices.length;
for (byte si : indices) {
if (si >= (byte)length || si < (byte)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Byte128Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Byte256Vector extends ByteVector {
@ForceInline
Byte256Shuffle iotaShuffle() { return Byte256Shuffle.IOTA; }
@ForceInline
Byte256Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Byte256Shuffle)VectorSupport.shuffleIota(ETYPE, Byte256Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Byte256Shuffle)VectorSupport.shuffleIota(ETYPE, Byte256Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Byte256Shuffle shuffleFromArray(int[] indices, int i) { return new Byte256Shuffle(indices, i); }
Byte256Shuffle shuffleFromBytes(byte[] reorder) { return new Byte256Shuffle(reorder); }
@Override
@ForceInline
Byte256Shuffle shuffleFromArray(int[] indexes, int i) { return new Byte256Shuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class Byte256Vector extends ByteVector {
return (long) super.reduceLanesTemplate(op, Byte256Mask.class, (Byte256Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Byte> toShuffle() {
return super.toShuffleTemplate(Byte256Shuffle.class); // specialize
}
// Specialized unary testing
@ -823,26 +836,23 @@ final class Byte256Vector extends ByteVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Byte> ETYPE = byte.class; // used by the JVM
Byte256Shuffle(byte[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Byte256Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Byte256Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Byte256Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Byte256Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Byte256Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
byte[] indices() {
return (byte[])getPayload();
public Byte256Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public ByteSpecies vspecies() {
return VSPECIES;
}
@ -857,76 +867,33 @@ final class Byte256Vector extends ByteVector {
@Override
@ForceInline
Byte256Vector toBitsVector() {
return (Byte256Vector) super.toBitsVectorTemplate();
public Byte256Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Byte256Shuffle.class, this, VLENGTH,
(s) -> ((Byte256Vector)(((AbstractShuffle<Byte>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
ByteVector toBitsVector0() {
return Byte256Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
VectorSpecies<Integer> species = IntVector.SPECIES_256;
Vector<Byte> v = toBitsVector();
v.convertShape(VectorOperators.B2I, species, 0)
.reinterpretAsInts()
.intoArray(a, offset);
v.convertShape(VectorOperators.B2I, species, 1)
.reinterpretAsInts()
.intoArray(a, offset + species.length());
v.convertShape(VectorOperators.B2I, species, 2)
.reinterpretAsInts()
.intoArray(a, offset + species.length() * 2);
v.convertShape(VectorOperators.B2I, species, 3)
.reinterpretAsInts()
.intoArray(a, offset + species.length() * 3);
}
private static byte[] prepare(int[] indices, int offset) {
byte[] a = new byte[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (byte)si;
public Byte256Shuffle rearrange(VectorShuffle<Byte> shuffle) {
Byte256Shuffle s = (Byte256Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static byte[] prepare(IntUnaryOperator f) {
byte[] a = new byte[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (byte)si;
}
return a;
}
private static boolean indicesInRange(byte[] indices) {
int length = indices.length;
for (byte si : indices) {
if (si >= (byte)length || si < (byte)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Byte256Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Byte512Vector extends ByteVector {
@ForceInline
Byte512Shuffle iotaShuffle() { return Byte512Shuffle.IOTA; }
@ForceInline
Byte512Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Byte512Shuffle)VectorSupport.shuffleIota(ETYPE, Byte512Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Byte512Shuffle)VectorSupport.shuffleIota(ETYPE, Byte512Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Byte512Shuffle shuffleFromArray(int[] indices, int i) { return new Byte512Shuffle(indices, i); }
Byte512Shuffle shuffleFromBytes(byte[] reorder) { return new Byte512Shuffle(reorder); }
@Override
@ForceInline
Byte512Shuffle shuffleFromArray(int[] indexes, int i) { return new Byte512Shuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class Byte512Vector extends ByteVector {
return (long) super.reduceLanesTemplate(op, Byte512Mask.class, (Byte512Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Byte> toShuffle() {
return super.toShuffleTemplate(Byte512Shuffle.class); // specialize
}
// Specialized unary testing
@ -887,26 +900,23 @@ final class Byte512Vector extends ByteVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Byte> ETYPE = byte.class; // used by the JVM
Byte512Shuffle(byte[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Byte512Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Byte512Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Byte512Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Byte512Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Byte512Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
byte[] indices() {
return (byte[])getPayload();
public Byte512Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public ByteSpecies vspecies() {
return VSPECIES;
}
@ -921,76 +931,33 @@ final class Byte512Vector extends ByteVector {
@Override
@ForceInline
Byte512Vector toBitsVector() {
return (Byte512Vector) super.toBitsVectorTemplate();
public Byte512Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Byte512Shuffle.class, this, VLENGTH,
(s) -> ((Byte512Vector)(((AbstractShuffle<Byte>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
ByteVector toBitsVector0() {
return Byte512Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
VectorSpecies<Integer> species = IntVector.SPECIES_512;
Vector<Byte> v = toBitsVector();
v.convertShape(VectorOperators.B2I, species, 0)
.reinterpretAsInts()
.intoArray(a, offset);
v.convertShape(VectorOperators.B2I, species, 1)
.reinterpretAsInts()
.intoArray(a, offset + species.length());
v.convertShape(VectorOperators.B2I, species, 2)
.reinterpretAsInts()
.intoArray(a, offset + species.length() * 2);
v.convertShape(VectorOperators.B2I, species, 3)
.reinterpretAsInts()
.intoArray(a, offset + species.length() * 3);
}
private static byte[] prepare(int[] indices, int offset) {
byte[] a = new byte[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (byte)si;
public Byte512Shuffle rearrange(VectorShuffle<Byte> shuffle) {
Byte512Shuffle s = (Byte512Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static byte[] prepare(IntUnaryOperator f) {
byte[] a = new byte[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (byte)si;
}
return a;
}
private static boolean indicesInRange(byte[] indices) {
int length = indices.length;
for (byte si : indices) {
if (si >= (byte)length || si < (byte)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Byte512Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Byte64Vector extends ByteVector {
@ForceInline
Byte64Shuffle iotaShuffle() { return Byte64Shuffle.IOTA; }
@ForceInline
Byte64Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Byte64Shuffle)VectorSupport.shuffleIota(ETYPE, Byte64Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Byte64Shuffle)VectorSupport.shuffleIota(ETYPE, Byte64Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Byte64Shuffle shuffleFromArray(int[] indices, int i) { return new Byte64Shuffle(indices, i); }
Byte64Shuffle shuffleFromBytes(byte[] reorder) { return new Byte64Shuffle(reorder); }
@Override
@ForceInline
Byte64Shuffle shuffleFromArray(int[] indexes, int i) { return new Byte64Shuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class Byte64Vector extends ByteVector {
return (long) super.reduceLanesTemplate(op, Byte64Mask.class, (Byte64Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Byte> toShuffle() {
return super.toShuffleTemplate(Byte64Shuffle.class); // specialize
}
// Specialized unary testing
@ -775,26 +788,23 @@ final class Byte64Vector extends ByteVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Byte> ETYPE = byte.class; // used by the JVM
Byte64Shuffle(byte[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Byte64Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Byte64Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Byte64Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Byte64Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Byte64Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
byte[] indices() {
return (byte[])getPayload();
public Byte64Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public ByteSpecies vspecies() {
return VSPECIES;
}
@ -809,76 +819,33 @@ final class Byte64Vector extends ByteVector {
@Override
@ForceInline
Byte64Vector toBitsVector() {
return (Byte64Vector) super.toBitsVectorTemplate();
public Byte64Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Byte64Shuffle.class, this, VLENGTH,
(s) -> ((Byte64Vector)(((AbstractShuffle<Byte>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
ByteVector toBitsVector0() {
return Byte64Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
VectorSpecies<Integer> species = IntVector.SPECIES_64;
Vector<Byte> v = toBitsVector();
v.convertShape(VectorOperators.B2I, species, 0)
.reinterpretAsInts()
.intoArray(a, offset);
v.convertShape(VectorOperators.B2I, species, 1)
.reinterpretAsInts()
.intoArray(a, offset + species.length());
v.convertShape(VectorOperators.B2I, species, 2)
.reinterpretAsInts()
.intoArray(a, offset + species.length() * 2);
v.convertShape(VectorOperators.B2I, species, 3)
.reinterpretAsInts()
.intoArray(a, offset + species.length() * 3);
}
private static byte[] prepare(int[] indices, int offset) {
byte[] a = new byte[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (byte)si;
public Byte64Shuffle rearrange(VectorShuffle<Byte> shuffle) {
Byte64Shuffle s = (Byte64Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static byte[] prepare(IntUnaryOperator f) {
byte[] a = new byte[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (byte)si;
}
return a;
}
private static boolean indicesInRange(byte[] indices) {
int length = indices.length;
for (byte si : indices) {
if (si >= (byte)length || si < (byte)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Byte64Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class ByteMaxVector extends ByteVector {
@ForceInline
ByteMaxShuffle iotaShuffle() { return ByteMaxShuffle.IOTA; }
@ForceInline
ByteMaxShuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (ByteMaxShuffle)VectorSupport.shuffleIota(ETYPE, ByteMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (ByteMaxShuffle)VectorSupport.shuffleIota(ETYPE, ByteMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
ByteMaxShuffle shuffleFromArray(int[] indices, int i) { return new ByteMaxShuffle(indices, i); }
ByteMaxShuffle shuffleFromBytes(byte[] reorder) { return new ByteMaxShuffle(reorder); }
@Override
@ForceInline
ByteMaxShuffle shuffleFromArray(int[] indexes, int i) { return new ByteMaxShuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class ByteMaxVector extends ByteVector {
return (long) super.reduceLanesTemplate(op, ByteMaxMask.class, (ByteMaxMask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Byte> toShuffle() {
return super.toShuffleTemplate(ByteMaxShuffle.class); // specialize
}
// Specialized unary testing
@ -761,26 +774,23 @@ final class ByteMaxVector extends ByteVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Byte> ETYPE = byte.class; // used by the JVM
ByteMaxShuffle(byte[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
ByteMaxShuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
ByteMaxShuffle(int[] indices, int i) {
this(prepare(indices, i));
public ByteMaxShuffle(int[] reorder) {
super(VLENGTH, reorder);
}
ByteMaxShuffle(IntUnaryOperator fn) {
this(prepare(fn));
public ByteMaxShuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
byte[] indices() {
return (byte[])getPayload();
public ByteMaxShuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public ByteSpecies vspecies() {
return VSPECIES;
}
@ -795,76 +805,33 @@ final class ByteMaxVector extends ByteVector {
@Override
@ForceInline
ByteMaxVector toBitsVector() {
return (ByteMaxVector) super.toBitsVectorTemplate();
public ByteMaxVector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, ByteMaxShuffle.class, this, VLENGTH,
(s) -> ((ByteMaxVector)(((AbstractShuffle<Byte>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
ByteVector toBitsVector0() {
return ByteMaxVector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
VectorSpecies<Integer> species = IntVector.SPECIES_MAX;
Vector<Byte> v = toBitsVector();
v.convertShape(VectorOperators.B2I, species, 0)
.reinterpretAsInts()
.intoArray(a, offset);
v.convertShape(VectorOperators.B2I, species, 1)
.reinterpretAsInts()
.intoArray(a, offset + species.length());
v.convertShape(VectorOperators.B2I, species, 2)
.reinterpretAsInts()
.intoArray(a, offset + species.length() * 2);
v.convertShape(VectorOperators.B2I, species, 3)
.reinterpretAsInts()
.intoArray(a, offset + species.length() * 3);
}
private static byte[] prepare(int[] indices, int offset) {
byte[] a = new byte[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (byte)si;
public ByteMaxShuffle rearrange(VectorShuffle<Byte> shuffle) {
ByteMaxShuffle s = (ByteMaxShuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static byte[] prepare(IntUnaryOperator f) {
byte[] a = new byte[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (byte)si;
}
return a;
}
private static boolean indicesInRange(byte[] indices) {
int length = indices.length;
for (byte si : indices) {
if (si >= (byte)length || si < (byte)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new ByteMaxShuffle(r);
}
}

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -1071,7 +1071,7 @@ public abstract class ByteVector extends AbstractVector<Byte> {
// and broadcast, but it would be more surprising not to continue
// the obvious pattern started by unary and binary.
/**
/**
* {@inheritDoc} <!--workaround-->
* @see #lanewise(VectorOperators.Ternary,byte,byte,VectorMask)
* @see #lanewise(VectorOperators.Ternary,Vector,byte,VectorMask)
@ -2480,8 +2480,8 @@ public abstract class ByteVector extends AbstractVector<Byte> {
}
@ForceInline
final <F>
VectorShuffle<F> toShuffle0(AbstractSpecies<F> dsp) {
private final
VectorShuffle<Byte> toShuffle0(ByteSpecies dsp) {
byte[] a = toArray();
int[] sa = new int[a.length];
for (int i = 0; i < a.length; i++) {
@ -2490,6 +2490,18 @@ public abstract class ByteVector extends AbstractVector<Byte> {
return VectorShuffle.fromArray(dsp, sa, 0);
}
/*package-private*/
@ForceInline
final
VectorShuffle<Byte> toShuffleTemplate(Class<?> shuffleType) {
ByteSpecies vsp = vspecies();
return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
getClass(), byte.class, length(),
shuffleType, byte.class, length(),
this, vsp,
ByteVector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
@ -4084,10 +4096,9 @@ public abstract class ByteVector extends AbstractVector<Byte> {
private ByteSpecies(VectorShape shape,
Class<? extends ByteVector> vectorType,
Class<? extends AbstractMask<Byte>> maskType,
Class<? extends AbstractShuffle<Byte>> shuffleType,
Function<Object, ByteVector> vectorFactory) {
super(shape, LaneType.of(byte.class),
vectorType, maskType, shuffleType,
vectorType, maskType,
vectorFactory);
assert(this.elementSize() == Byte.SIZE);
}
@ -4363,7 +4374,6 @@ public abstract class ByteVector extends AbstractVector<Byte> {
= new ByteSpecies(VectorShape.S_64_BIT,
Byte64Vector.class,
Byte64Vector.Byte64Mask.class,
Byte64Vector.Byte64Shuffle.class,
Byte64Vector::new);
/** Species representing {@link ByteVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */
@ -4371,7 +4381,6 @@ public abstract class ByteVector extends AbstractVector<Byte> {
= new ByteSpecies(VectorShape.S_128_BIT,
Byte128Vector.class,
Byte128Vector.Byte128Mask.class,
Byte128Vector.Byte128Shuffle.class,
Byte128Vector::new);
/** Species representing {@link ByteVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */
@ -4379,7 +4388,6 @@ public abstract class ByteVector extends AbstractVector<Byte> {
= new ByteSpecies(VectorShape.S_256_BIT,
Byte256Vector.class,
Byte256Vector.Byte256Mask.class,
Byte256Vector.Byte256Shuffle.class,
Byte256Vector::new);
/** Species representing {@link ByteVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */
@ -4387,7 +4395,6 @@ public abstract class ByteVector extends AbstractVector<Byte> {
= new ByteSpecies(VectorShape.S_512_BIT,
Byte512Vector.class,
Byte512Vector.Byte512Mask.class,
Byte512Vector.Byte512Shuffle.class,
Byte512Vector::new);
/** Species representing {@link ByteVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */
@ -4395,7 +4402,6 @@ public abstract class ByteVector extends AbstractVector<Byte> {
= new ByteSpecies(VectorShape.S_Max_BIT,
ByteMaxVector.class,
ByteMaxVector.ByteMaxMask.class,
ByteMaxVector.ByteMaxShuffle.class,
ByteMaxVector::new);
/**

@ -141,9 +141,24 @@ final class Double128Vector extends DoubleVector {
@ForceInline
Double128Shuffle iotaShuffle() { return Double128Shuffle.IOTA; }
@ForceInline
Double128Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Double128Shuffle)VectorSupport.shuffleIota(ETYPE, Double128Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Double128Shuffle)VectorSupport.shuffleIota(ETYPE, Double128Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Double128Shuffle shuffleFromArray(int[] indices, int i) { return new Double128Shuffle(indices, i); }
Double128Shuffle shuffleFromBytes(byte[] reorder) { return new Double128Shuffle(reorder); }
@Override
@ForceInline
Double128Shuffle shuffleFromArray(int[] indexes, int i) { return new Double128Shuffle(indexes, i); }
@Override
@ForceInline
@ -329,11 +344,9 @@ final class Double128Vector extends DoubleVector {
return (long) super.reduceLanesTemplate(op, Double128Mask.class, (Double128Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Double> toShuffle() {
return super.toShuffleTemplate(Double128Shuffle.class); // specialize
}
// Specialized unary testing
@ -750,28 +763,25 @@ final class Double128Vector extends DoubleVector {
static final class Double128Shuffle extends AbstractShuffle<Double> {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Long> ETYPE = long.class; // used by the JVM
static final Class<Double> ETYPE = double.class; // used by the JVM
Double128Shuffle(long[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Double128Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Double128Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Double128Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Double128Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Double128Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
long[] indices() {
return (long[])getPayload();
public Double128Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public DoubleSpecies vspecies() {
return VSPECIES;
}
@ -779,94 +789,40 @@ final class Double128Vector extends DoubleVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Long.MAX_VALUE);
assert(Long.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Double128Shuffle IOTA = new Double128Shuffle(IDENTITY);
@Override
@ForceInline
Long128Vector toBitsVector() {
return (Long128Vector) super.toBitsVectorTemplate();
public Double128Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Double128Shuffle.class, this, VLENGTH,
(s) -> ((Double128Vector)(((AbstractShuffle<Double>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
LongVector toBitsVector0() {
return Long128Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
switch (length()) {
case 1 -> a[offset] = laneSource(0);
case 2 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_64, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 4 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_128, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 8 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_256, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 16 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_512, 0)
.reinterpretAsInts()
.intoArray(a, offset);
default -> {
VectorIntrinsics.checkFromIndexSize(offset, length(), a.length);
for (int i = 0; i < length(); i++) {
a[offset + i] = laneSource(i);
}
}
public Double128Shuffle rearrange(VectorShuffle<Double> shuffle) {
Double128Shuffle s = (Double128Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
}
private static long[] prepare(int[] indices, int offset) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static long[] prepare(IntUnaryOperator f) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static boolean indicesInRange(long[] indices) {
int length = indices.length;
for (long si : indices) {
if (si >= (long)length || si < (long)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Double128Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Double256Vector extends DoubleVector {
@ForceInline
Double256Shuffle iotaShuffle() { return Double256Shuffle.IOTA; }
@ForceInline
Double256Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Double256Shuffle)VectorSupport.shuffleIota(ETYPE, Double256Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Double256Shuffle)VectorSupport.shuffleIota(ETYPE, Double256Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Double256Shuffle shuffleFromArray(int[] indices, int i) { return new Double256Shuffle(indices, i); }
Double256Shuffle shuffleFromBytes(byte[] reorder) { return new Double256Shuffle(reorder); }
@Override
@ForceInline
Double256Shuffle shuffleFromArray(int[] indexes, int i) { return new Double256Shuffle(indexes, i); }
@Override
@ForceInline
@ -329,11 +344,9 @@ final class Double256Vector extends DoubleVector {
return (long) super.reduceLanesTemplate(op, Double256Mask.class, (Double256Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Double> toShuffle() {
return super.toShuffleTemplate(Double256Shuffle.class); // specialize
}
// Specialized unary testing
@ -754,28 +767,25 @@ final class Double256Vector extends DoubleVector {
static final class Double256Shuffle extends AbstractShuffle<Double> {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Long> ETYPE = long.class; // used by the JVM
static final Class<Double> ETYPE = double.class; // used by the JVM
Double256Shuffle(long[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Double256Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Double256Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Double256Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Double256Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Double256Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
long[] indices() {
return (long[])getPayload();
public Double256Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public DoubleSpecies vspecies() {
return VSPECIES;
}
@ -783,94 +793,40 @@ final class Double256Vector extends DoubleVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Long.MAX_VALUE);
assert(Long.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Double256Shuffle IOTA = new Double256Shuffle(IDENTITY);
@Override
@ForceInline
Long256Vector toBitsVector() {
return (Long256Vector) super.toBitsVectorTemplate();
public Double256Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Double256Shuffle.class, this, VLENGTH,
(s) -> ((Double256Vector)(((AbstractShuffle<Double>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
LongVector toBitsVector0() {
return Long256Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
switch (length()) {
case 1 -> a[offset] = laneSource(0);
case 2 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_64, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 4 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_128, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 8 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_256, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 16 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_512, 0)
.reinterpretAsInts()
.intoArray(a, offset);
default -> {
VectorIntrinsics.checkFromIndexSize(offset, length(), a.length);
for (int i = 0; i < length(); i++) {
a[offset + i] = laneSource(i);
}
}
public Double256Shuffle rearrange(VectorShuffle<Double> shuffle) {
Double256Shuffle s = (Double256Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
}
private static long[] prepare(int[] indices, int offset) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static long[] prepare(IntUnaryOperator f) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static boolean indicesInRange(long[] indices) {
int length = indices.length;
for (long si : indices) {
if (si >= (long)length || si < (long)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Double256Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Double512Vector extends DoubleVector {
@ForceInline
Double512Shuffle iotaShuffle() { return Double512Shuffle.IOTA; }
@ForceInline
Double512Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Double512Shuffle)VectorSupport.shuffleIota(ETYPE, Double512Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Double512Shuffle)VectorSupport.shuffleIota(ETYPE, Double512Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Double512Shuffle shuffleFromArray(int[] indices, int i) { return new Double512Shuffle(indices, i); }
Double512Shuffle shuffleFromBytes(byte[] reorder) { return new Double512Shuffle(reorder); }
@Override
@ForceInline
Double512Shuffle shuffleFromArray(int[] indexes, int i) { return new Double512Shuffle(indexes, i); }
@Override
@ForceInline
@ -329,11 +344,9 @@ final class Double512Vector extends DoubleVector {
return (long) super.reduceLanesTemplate(op, Double512Mask.class, (Double512Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Double> toShuffle() {
return super.toShuffleTemplate(Double512Shuffle.class); // specialize
}
// Specialized unary testing
@ -762,28 +775,25 @@ final class Double512Vector extends DoubleVector {
static final class Double512Shuffle extends AbstractShuffle<Double> {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Long> ETYPE = long.class; // used by the JVM
static final Class<Double> ETYPE = double.class; // used by the JVM
Double512Shuffle(long[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Double512Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Double512Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Double512Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Double512Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Double512Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
long[] indices() {
return (long[])getPayload();
public Double512Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public DoubleSpecies vspecies() {
return VSPECIES;
}
@ -791,94 +801,40 @@ final class Double512Vector extends DoubleVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Long.MAX_VALUE);
assert(Long.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Double512Shuffle IOTA = new Double512Shuffle(IDENTITY);
@Override
@ForceInline
Long512Vector toBitsVector() {
return (Long512Vector) super.toBitsVectorTemplate();
public Double512Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Double512Shuffle.class, this, VLENGTH,
(s) -> ((Double512Vector)(((AbstractShuffle<Double>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
LongVector toBitsVector0() {
return Long512Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
switch (length()) {
case 1 -> a[offset] = laneSource(0);
case 2 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_64, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 4 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_128, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 8 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_256, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 16 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_512, 0)
.reinterpretAsInts()
.intoArray(a, offset);
default -> {
VectorIntrinsics.checkFromIndexSize(offset, length(), a.length);
for (int i = 0; i < length(); i++) {
a[offset + i] = laneSource(i);
}
}
public Double512Shuffle rearrange(VectorShuffle<Double> shuffle) {
Double512Shuffle s = (Double512Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
}
private static long[] prepare(int[] indices, int offset) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static long[] prepare(IntUnaryOperator f) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static boolean indicesInRange(long[] indices) {
int length = indices.length;
for (long si : indices) {
if (si >= (long)length || si < (long)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Double512Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Double64Vector extends DoubleVector {
@ForceInline
Double64Shuffle iotaShuffle() { return Double64Shuffle.IOTA; }
@ForceInline
Double64Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Double64Shuffle)VectorSupport.shuffleIota(ETYPE, Double64Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Double64Shuffle)VectorSupport.shuffleIota(ETYPE, Double64Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Double64Shuffle shuffleFromArray(int[] indices, int i) { return new Double64Shuffle(indices, i); }
Double64Shuffle shuffleFromBytes(byte[] reorder) { return new Double64Shuffle(reorder); }
@Override
@ForceInline
Double64Shuffle shuffleFromArray(int[] indexes, int i) { return new Double64Shuffle(indexes, i); }
@Override
@ForceInline
@ -329,11 +344,9 @@ final class Double64Vector extends DoubleVector {
return (long) super.reduceLanesTemplate(op, Double64Mask.class, (Double64Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Double> toShuffle() {
return super.toShuffleTemplate(Double64Shuffle.class); // specialize
}
// Specialized unary testing
@ -748,28 +761,25 @@ final class Double64Vector extends DoubleVector {
static final class Double64Shuffle extends AbstractShuffle<Double> {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Long> ETYPE = long.class; // used by the JVM
static final Class<Double> ETYPE = double.class; // used by the JVM
Double64Shuffle(long[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Double64Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Double64Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Double64Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Double64Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Double64Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
long[] indices() {
return (long[])getPayload();
public Double64Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public DoubleSpecies vspecies() {
return VSPECIES;
}
@ -777,94 +787,40 @@ final class Double64Vector extends DoubleVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Long.MAX_VALUE);
assert(Long.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Double64Shuffle IOTA = new Double64Shuffle(IDENTITY);
@Override
@ForceInline
Long64Vector toBitsVector() {
return (Long64Vector) super.toBitsVectorTemplate();
public Double64Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Double64Shuffle.class, this, VLENGTH,
(s) -> ((Double64Vector)(((AbstractShuffle<Double>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
LongVector toBitsVector0() {
return Long64Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
switch (length()) {
case 1 -> a[offset] = laneSource(0);
case 2 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_64, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 4 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_128, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 8 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_256, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 16 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_512, 0)
.reinterpretAsInts()
.intoArray(a, offset);
default -> {
VectorIntrinsics.checkFromIndexSize(offset, length(), a.length);
for (int i = 0; i < length(); i++) {
a[offset + i] = laneSource(i);
}
}
public Double64Shuffle rearrange(VectorShuffle<Double> shuffle) {
Double64Shuffle s = (Double64Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
}
private static long[] prepare(int[] indices, int offset) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static long[] prepare(IntUnaryOperator f) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static boolean indicesInRange(long[] indices) {
int length = indices.length;
for (long si : indices) {
if (si >= (long)length || si < (long)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Double64Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class DoubleMaxVector extends DoubleVector {
@ForceInline
DoubleMaxShuffle iotaShuffle() { return DoubleMaxShuffle.IOTA; }
@ForceInline
DoubleMaxShuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (DoubleMaxShuffle)VectorSupport.shuffleIota(ETYPE, DoubleMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (DoubleMaxShuffle)VectorSupport.shuffleIota(ETYPE, DoubleMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
DoubleMaxShuffle shuffleFromArray(int[] indices, int i) { return new DoubleMaxShuffle(indices, i); }
DoubleMaxShuffle shuffleFromBytes(byte[] reorder) { return new DoubleMaxShuffle(reorder); }
@Override
@ForceInline
DoubleMaxShuffle shuffleFromArray(int[] indexes, int i) { return new DoubleMaxShuffle(indexes, i); }
@Override
@ForceInline
@ -329,11 +344,9 @@ final class DoubleMaxVector extends DoubleVector {
return (long) super.reduceLanesTemplate(op, DoubleMaxMask.class, (DoubleMaxMask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Double> toShuffle() {
return super.toShuffleTemplate(DoubleMaxShuffle.class); // specialize
}
// Specialized unary testing
@ -747,28 +760,25 @@ final class DoubleMaxVector extends DoubleVector {
static final class DoubleMaxShuffle extends AbstractShuffle<Double> {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Long> ETYPE = long.class; // used by the JVM
static final Class<Double> ETYPE = double.class; // used by the JVM
DoubleMaxShuffle(long[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
DoubleMaxShuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
DoubleMaxShuffle(int[] indices, int i) {
this(prepare(indices, i));
public DoubleMaxShuffle(int[] reorder) {
super(VLENGTH, reorder);
}
DoubleMaxShuffle(IntUnaryOperator fn) {
this(prepare(fn));
public DoubleMaxShuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
long[] indices() {
return (long[])getPayload();
public DoubleMaxShuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public DoubleSpecies vspecies() {
return VSPECIES;
}
@ -776,94 +786,40 @@ final class DoubleMaxVector extends DoubleVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Long.MAX_VALUE);
assert(Long.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final DoubleMaxShuffle IOTA = new DoubleMaxShuffle(IDENTITY);
@Override
@ForceInline
LongMaxVector toBitsVector() {
return (LongMaxVector) super.toBitsVectorTemplate();
public DoubleMaxVector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, DoubleMaxShuffle.class, this, VLENGTH,
(s) -> ((DoubleMaxVector)(((AbstractShuffle<Double>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
LongVector toBitsVector0() {
return LongMaxVector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
switch (length()) {
case 1 -> a[offset] = laneSource(0);
case 2 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_64, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 4 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_128, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 8 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_256, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 16 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_512, 0)
.reinterpretAsInts()
.intoArray(a, offset);
default -> {
VectorIntrinsics.checkFromIndexSize(offset, length(), a.length);
for (int i = 0; i < length(); i++) {
a[offset + i] = laneSource(i);
}
}
public DoubleMaxShuffle rearrange(VectorShuffle<Double> shuffle) {
DoubleMaxShuffle s = (DoubleMaxShuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
}
private static long[] prepare(int[] indices, int offset) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static long[] prepare(IntUnaryOperator f) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static boolean indicesInRange(long[] indices) {
int length = indices.length;
for (long si : indices) {
if (si >= (long)length || si < (long)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new DoubleMaxShuffle(r);
}
}

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -953,7 +953,7 @@ public abstract class DoubleVector extends AbstractVector<Double> {
// and broadcast, but it would be more surprising not to continue
// the obvious pattern started by unary and binary.
/**
/**
* {@inheritDoc} <!--workaround-->
* @see #lanewise(VectorOperators.Ternary,double,double,VectorMask)
* @see #lanewise(VectorOperators.Ternary,Vector,double,VectorMask)
@ -2322,8 +2322,8 @@ public abstract class DoubleVector extends AbstractVector<Double> {
}
@ForceInline
final <F>
VectorShuffle<F> toShuffle0(AbstractSpecies<F> dsp) {
private final
VectorShuffle<Double> toShuffle0(DoubleSpecies dsp) {
double[] a = toArray();
int[] sa = new int[a.length];
for (int i = 0; i < a.length; i++) {
@ -2332,6 +2332,18 @@ public abstract class DoubleVector extends AbstractVector<Double> {
return VectorShuffle.fromArray(dsp, sa, 0);
}
/*package-private*/
@ForceInline
final
VectorShuffle<Double> toShuffleTemplate(Class<?> shuffleType) {
DoubleSpecies vsp = vspecies();
return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
getClass(), double.class, length(),
shuffleType, byte.class, length(),
this, vsp,
DoubleVector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
@ -3687,10 +3699,9 @@ public abstract class DoubleVector extends AbstractVector<Double> {
private DoubleSpecies(VectorShape shape,
Class<? extends DoubleVector> vectorType,
Class<? extends AbstractMask<Double>> maskType,
Class<? extends AbstractShuffle<Double>> shuffleType,
Function<Object, DoubleVector> vectorFactory) {
super(shape, LaneType.of(double.class),
vectorType, maskType, shuffleType,
vectorType, maskType,
vectorFactory);
assert(this.elementSize() == Double.SIZE);
}
@ -3966,7 +3977,6 @@ public abstract class DoubleVector extends AbstractVector<Double> {
= new DoubleSpecies(VectorShape.S_64_BIT,
Double64Vector.class,
Double64Vector.Double64Mask.class,
Double64Vector.Double64Shuffle.class,
Double64Vector::new);
/** Species representing {@link DoubleVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */
@ -3974,7 +3984,6 @@ public abstract class DoubleVector extends AbstractVector<Double> {
= new DoubleSpecies(VectorShape.S_128_BIT,
Double128Vector.class,
Double128Vector.Double128Mask.class,
Double128Vector.Double128Shuffle.class,
Double128Vector::new);
/** Species representing {@link DoubleVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */
@ -3982,7 +3991,6 @@ public abstract class DoubleVector extends AbstractVector<Double> {
= new DoubleSpecies(VectorShape.S_256_BIT,
Double256Vector.class,
Double256Vector.Double256Mask.class,
Double256Vector.Double256Shuffle.class,
Double256Vector::new);
/** Species representing {@link DoubleVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */
@ -3990,7 +3998,6 @@ public abstract class DoubleVector extends AbstractVector<Double> {
= new DoubleSpecies(VectorShape.S_512_BIT,
Double512Vector.class,
Double512Vector.Double512Mask.class,
Double512Vector.Double512Shuffle.class,
Double512Vector::new);
/** Species representing {@link DoubleVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */
@ -3998,7 +4005,6 @@ public abstract class DoubleVector extends AbstractVector<Double> {
= new DoubleSpecies(VectorShape.S_Max_BIT,
DoubleMaxVector.class,
DoubleMaxVector.DoubleMaxMask.class,
DoubleMaxVector.DoubleMaxShuffle.class,
DoubleMaxVector::new);
/**

@ -141,9 +141,24 @@ final class Float128Vector extends FloatVector {
@ForceInline
Float128Shuffle iotaShuffle() { return Float128Shuffle.IOTA; }
@ForceInline
Float128Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Float128Shuffle)VectorSupport.shuffleIota(ETYPE, Float128Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Float128Shuffle)VectorSupport.shuffleIota(ETYPE, Float128Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Float128Shuffle shuffleFromArray(int[] indices, int i) { return new Float128Shuffle(indices, i); }
Float128Shuffle shuffleFromBytes(byte[] reorder) { return new Float128Shuffle(reorder); }
@Override
@ForceInline
Float128Shuffle shuffleFromArray(int[] indexes, int i) { return new Float128Shuffle(indexes, i); }
@Override
@ForceInline
@ -329,11 +344,9 @@ final class Float128Vector extends FloatVector {
return (long) super.reduceLanesTemplate(op, Float128Mask.class, (Float128Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Float> toShuffle() {
return super.toShuffleTemplate(Float128Shuffle.class); // specialize
}
// Specialized unary testing
@ -754,28 +767,25 @@ final class Float128Vector extends FloatVector {
static final class Float128Shuffle extends AbstractShuffle<Float> {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Integer> ETYPE = int.class; // used by the JVM
static final Class<Float> ETYPE = float.class; // used by the JVM
Float128Shuffle(int[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Float128Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Float128Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Float128Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Float128Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Float128Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
int[] indices() {
return (int[])getPayload();
public Float128Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public FloatSpecies vspecies() {
return VSPECIES;
}
@ -783,70 +793,40 @@ final class Float128Vector extends FloatVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Integer.MAX_VALUE);
assert(Integer.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Float128Shuffle IOTA = new Float128Shuffle(IDENTITY);
@Override
@ForceInline
Int128Vector toBitsVector() {
return (Int128Vector) super.toBitsVectorTemplate();
public Float128Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Float128Shuffle.class, this, VLENGTH,
(s) -> ((Float128Vector)(((AbstractShuffle<Float>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
IntVector toBitsVector0() {
return Int128Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
toBitsVector().intoArray(a, offset);
}
private static int[] prepare(int[] indices, int offset) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
public Float128Shuffle rearrange(VectorShuffle<Float> shuffle) {
Float128Shuffle s = (Float128Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static int[] prepare(IntUnaryOperator f) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
}
return a;
}
private static boolean indicesInRange(int[] indices) {
int length = indices.length;
for (int si : indices) {
if (si >= (int)length || si < (int)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Float128Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Float256Vector extends FloatVector {
@ForceInline
Float256Shuffle iotaShuffle() { return Float256Shuffle.IOTA; }
@ForceInline
Float256Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Float256Shuffle)VectorSupport.shuffleIota(ETYPE, Float256Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Float256Shuffle)VectorSupport.shuffleIota(ETYPE, Float256Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Float256Shuffle shuffleFromArray(int[] indices, int i) { return new Float256Shuffle(indices, i); }
Float256Shuffle shuffleFromBytes(byte[] reorder) { return new Float256Shuffle(reorder); }
@Override
@ForceInline
Float256Shuffle shuffleFromArray(int[] indexes, int i) { return new Float256Shuffle(indexes, i); }
@Override
@ForceInline
@ -329,11 +344,9 @@ final class Float256Vector extends FloatVector {
return (long) super.reduceLanesTemplate(op, Float256Mask.class, (Float256Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Float> toShuffle() {
return super.toShuffleTemplate(Float256Shuffle.class); // specialize
}
// Specialized unary testing
@ -762,28 +775,25 @@ final class Float256Vector extends FloatVector {
static final class Float256Shuffle extends AbstractShuffle<Float> {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Integer> ETYPE = int.class; // used by the JVM
static final Class<Float> ETYPE = float.class; // used by the JVM
Float256Shuffle(int[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Float256Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Float256Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Float256Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Float256Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Float256Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
int[] indices() {
return (int[])getPayload();
public Float256Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public FloatSpecies vspecies() {
return VSPECIES;
}
@ -791,70 +801,40 @@ final class Float256Vector extends FloatVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Integer.MAX_VALUE);
assert(Integer.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Float256Shuffle IOTA = new Float256Shuffle(IDENTITY);
@Override
@ForceInline
Int256Vector toBitsVector() {
return (Int256Vector) super.toBitsVectorTemplate();
public Float256Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Float256Shuffle.class, this, VLENGTH,
(s) -> ((Float256Vector)(((AbstractShuffle<Float>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
IntVector toBitsVector0() {
return Int256Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
toBitsVector().intoArray(a, offset);
}
private static int[] prepare(int[] indices, int offset) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
public Float256Shuffle rearrange(VectorShuffle<Float> shuffle) {
Float256Shuffle s = (Float256Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static int[] prepare(IntUnaryOperator f) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
}
return a;
}
private static boolean indicesInRange(int[] indices) {
int length = indices.length;
for (int si : indices) {
if (si >= (int)length || si < (int)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Float256Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Float512Vector extends FloatVector {
@ForceInline
Float512Shuffle iotaShuffle() { return Float512Shuffle.IOTA; }
@ForceInline
Float512Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Float512Shuffle)VectorSupport.shuffleIota(ETYPE, Float512Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Float512Shuffle)VectorSupport.shuffleIota(ETYPE, Float512Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Float512Shuffle shuffleFromArray(int[] indices, int i) { return new Float512Shuffle(indices, i); }
Float512Shuffle shuffleFromBytes(byte[] reorder) { return new Float512Shuffle(reorder); }
@Override
@ForceInline
Float512Shuffle shuffleFromArray(int[] indexes, int i) { return new Float512Shuffle(indexes, i); }
@Override
@ForceInline
@ -329,11 +344,9 @@ final class Float512Vector extends FloatVector {
return (long) super.reduceLanesTemplate(op, Float512Mask.class, (Float512Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Float> toShuffle() {
return super.toShuffleTemplate(Float512Shuffle.class); // specialize
}
// Specialized unary testing
@ -778,28 +791,25 @@ final class Float512Vector extends FloatVector {
static final class Float512Shuffle extends AbstractShuffle<Float> {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Integer> ETYPE = int.class; // used by the JVM
static final Class<Float> ETYPE = float.class; // used by the JVM
Float512Shuffle(int[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Float512Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Float512Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Float512Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Float512Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Float512Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
int[] indices() {
return (int[])getPayload();
public Float512Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public FloatSpecies vspecies() {
return VSPECIES;
}
@ -807,70 +817,40 @@ final class Float512Vector extends FloatVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Integer.MAX_VALUE);
assert(Integer.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Float512Shuffle IOTA = new Float512Shuffle(IDENTITY);
@Override
@ForceInline
Int512Vector toBitsVector() {
return (Int512Vector) super.toBitsVectorTemplate();
public Float512Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Float512Shuffle.class, this, VLENGTH,
(s) -> ((Float512Vector)(((AbstractShuffle<Float>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
IntVector toBitsVector0() {
return Int512Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
toBitsVector().intoArray(a, offset);
}
private static int[] prepare(int[] indices, int offset) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
public Float512Shuffle rearrange(VectorShuffle<Float> shuffle) {
Float512Shuffle s = (Float512Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static int[] prepare(IntUnaryOperator f) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
}
return a;
}
private static boolean indicesInRange(int[] indices) {
int length = indices.length;
for (int si : indices) {
if (si >= (int)length || si < (int)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Float512Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Float64Vector extends FloatVector {
@ForceInline
Float64Shuffle iotaShuffle() { return Float64Shuffle.IOTA; }
@ForceInline
Float64Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Float64Shuffle)VectorSupport.shuffleIota(ETYPE, Float64Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Float64Shuffle)VectorSupport.shuffleIota(ETYPE, Float64Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Float64Shuffle shuffleFromArray(int[] indices, int i) { return new Float64Shuffle(indices, i); }
Float64Shuffle shuffleFromBytes(byte[] reorder) { return new Float64Shuffle(reorder); }
@Override
@ForceInline
Float64Shuffle shuffleFromArray(int[] indexes, int i) { return new Float64Shuffle(indexes, i); }
@Override
@ForceInline
@ -329,11 +344,9 @@ final class Float64Vector extends FloatVector {
return (long) super.reduceLanesTemplate(op, Float64Mask.class, (Float64Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Float> toShuffle() {
return super.toShuffleTemplate(Float64Shuffle.class); // specialize
}
// Specialized unary testing
@ -750,28 +763,25 @@ final class Float64Vector extends FloatVector {
static final class Float64Shuffle extends AbstractShuffle<Float> {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Integer> ETYPE = int.class; // used by the JVM
static final Class<Float> ETYPE = float.class; // used by the JVM
Float64Shuffle(int[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Float64Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Float64Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Float64Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Float64Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Float64Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
int[] indices() {
return (int[])getPayload();
public Float64Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public FloatSpecies vspecies() {
return VSPECIES;
}
@ -779,70 +789,40 @@ final class Float64Vector extends FloatVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Integer.MAX_VALUE);
assert(Integer.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Float64Shuffle IOTA = new Float64Shuffle(IDENTITY);
@Override
@ForceInline
Int64Vector toBitsVector() {
return (Int64Vector) super.toBitsVectorTemplate();
public Float64Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Float64Shuffle.class, this, VLENGTH,
(s) -> ((Float64Vector)(((AbstractShuffle<Float>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
IntVector toBitsVector0() {
return Int64Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
toBitsVector().intoArray(a, offset);
}
private static int[] prepare(int[] indices, int offset) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
public Float64Shuffle rearrange(VectorShuffle<Float> shuffle) {
Float64Shuffle s = (Float64Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static int[] prepare(IntUnaryOperator f) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
}
return a;
}
private static boolean indicesInRange(int[] indices) {
int length = indices.length;
for (int si : indices) {
if (si >= (int)length || si < (int)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Float64Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class FloatMaxVector extends FloatVector {
@ForceInline
FloatMaxShuffle iotaShuffle() { return FloatMaxShuffle.IOTA; }
@ForceInline
FloatMaxShuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (FloatMaxShuffle)VectorSupport.shuffleIota(ETYPE, FloatMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (FloatMaxShuffle)VectorSupport.shuffleIota(ETYPE, FloatMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
FloatMaxShuffle shuffleFromArray(int[] indices, int i) { return new FloatMaxShuffle(indices, i); }
FloatMaxShuffle shuffleFromBytes(byte[] reorder) { return new FloatMaxShuffle(reorder); }
@Override
@ForceInline
FloatMaxShuffle shuffleFromArray(int[] indexes, int i) { return new FloatMaxShuffle(indexes, i); }
@Override
@ForceInline
@ -329,11 +344,9 @@ final class FloatMaxVector extends FloatVector {
return (long) super.reduceLanesTemplate(op, FloatMaxMask.class, (FloatMaxMask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Float> toShuffle() {
return super.toShuffleTemplate(FloatMaxShuffle.class); // specialize
}
// Specialized unary testing
@ -747,28 +760,25 @@ final class FloatMaxVector extends FloatVector {
static final class FloatMaxShuffle extends AbstractShuffle<Float> {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Integer> ETYPE = int.class; // used by the JVM
static final Class<Float> ETYPE = float.class; // used by the JVM
FloatMaxShuffle(int[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
FloatMaxShuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
FloatMaxShuffle(int[] indices, int i) {
this(prepare(indices, i));
public FloatMaxShuffle(int[] reorder) {
super(VLENGTH, reorder);
}
FloatMaxShuffle(IntUnaryOperator fn) {
this(prepare(fn));
public FloatMaxShuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
int[] indices() {
return (int[])getPayload();
public FloatMaxShuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public FloatSpecies vspecies() {
return VSPECIES;
}
@ -776,70 +786,40 @@ final class FloatMaxVector extends FloatVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Integer.MAX_VALUE);
assert(Integer.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final FloatMaxShuffle IOTA = new FloatMaxShuffle(IDENTITY);
@Override
@ForceInline
IntMaxVector toBitsVector() {
return (IntMaxVector) super.toBitsVectorTemplate();
public FloatMaxVector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, FloatMaxShuffle.class, this, VLENGTH,
(s) -> ((FloatMaxVector)(((AbstractShuffle<Float>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
IntVector toBitsVector0() {
return IntMaxVector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
toBitsVector().intoArray(a, offset);
}
private static int[] prepare(int[] indices, int offset) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
public FloatMaxShuffle rearrange(VectorShuffle<Float> shuffle) {
FloatMaxShuffle s = (FloatMaxShuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static int[] prepare(IntUnaryOperator f) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
}
return a;
}
private static boolean indicesInRange(int[] indices) {
int length = indices.length;
for (int si : indices) {
if (si >= (int)length || si < (int)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new FloatMaxShuffle(r);
}
}

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -953,7 +953,7 @@ public abstract class FloatVector extends AbstractVector<Float> {
// and broadcast, but it would be more surprising not to continue
// the obvious pattern started by unary and binary.
/**
/**
* {@inheritDoc} <!--workaround-->
* @see #lanewise(VectorOperators.Ternary,float,float,VectorMask)
* @see #lanewise(VectorOperators.Ternary,Vector,float,VectorMask)
@ -2334,8 +2334,8 @@ public abstract class FloatVector extends AbstractVector<Float> {
}
@ForceInline
final <F>
VectorShuffle<F> toShuffle0(AbstractSpecies<F> dsp) {
private final
VectorShuffle<Float> toShuffle0(FloatSpecies dsp) {
float[] a = toArray();
int[] sa = new int[a.length];
for (int i = 0; i < a.length; i++) {
@ -2344,6 +2344,18 @@ public abstract class FloatVector extends AbstractVector<Float> {
return VectorShuffle.fromArray(dsp, sa, 0);
}
/*package-private*/
@ForceInline
final
VectorShuffle<Float> toShuffleTemplate(Class<?> shuffleType) {
FloatSpecies vsp = vspecies();
return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
getClass(), float.class, length(),
shuffleType, byte.class, length(),
this, vsp,
FloatVector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
@ -3637,10 +3649,9 @@ public abstract class FloatVector extends AbstractVector<Float> {
private FloatSpecies(VectorShape shape,
Class<? extends FloatVector> vectorType,
Class<? extends AbstractMask<Float>> maskType,
Class<? extends AbstractShuffle<Float>> shuffleType,
Function<Object, FloatVector> vectorFactory) {
super(shape, LaneType.of(float.class),
vectorType, maskType, shuffleType,
vectorType, maskType,
vectorFactory);
assert(this.elementSize() == Float.SIZE);
}
@ -3916,7 +3927,6 @@ public abstract class FloatVector extends AbstractVector<Float> {
= new FloatSpecies(VectorShape.S_64_BIT,
Float64Vector.class,
Float64Vector.Float64Mask.class,
Float64Vector.Float64Shuffle.class,
Float64Vector::new);
/** Species representing {@link FloatVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */
@ -3924,7 +3934,6 @@ public abstract class FloatVector extends AbstractVector<Float> {
= new FloatSpecies(VectorShape.S_128_BIT,
Float128Vector.class,
Float128Vector.Float128Mask.class,
Float128Vector.Float128Shuffle.class,
Float128Vector::new);
/** Species representing {@link FloatVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */
@ -3932,7 +3941,6 @@ public abstract class FloatVector extends AbstractVector<Float> {
= new FloatSpecies(VectorShape.S_256_BIT,
Float256Vector.class,
Float256Vector.Float256Mask.class,
Float256Vector.Float256Shuffle.class,
Float256Vector::new);
/** Species representing {@link FloatVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */
@ -3940,7 +3948,6 @@ public abstract class FloatVector extends AbstractVector<Float> {
= new FloatSpecies(VectorShape.S_512_BIT,
Float512Vector.class,
Float512Vector.Float512Mask.class,
Float512Vector.Float512Shuffle.class,
Float512Vector::new);
/** Species representing {@link FloatVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */
@ -3948,7 +3955,6 @@ public abstract class FloatVector extends AbstractVector<Float> {
= new FloatSpecies(VectorShape.S_Max_BIT,
FloatMaxVector.class,
FloatMaxVector.FloatMaxMask.class,
FloatMaxVector.FloatMaxShuffle.class,
FloatMaxVector::new);
/**

@ -141,9 +141,24 @@ final class Int128Vector extends IntVector {
@ForceInline
Int128Shuffle iotaShuffle() { return Int128Shuffle.IOTA; }
@ForceInline
Int128Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Int128Shuffle)VectorSupport.shuffleIota(ETYPE, Int128Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Int128Shuffle)VectorSupport.shuffleIota(ETYPE, Int128Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Int128Shuffle shuffleFromArray(int[] indices, int i) { return new Int128Shuffle(indices, i); }
Int128Shuffle shuffleFromBytes(byte[] reorder) { return new Int128Shuffle(reorder); }
@Override
@ForceInline
Int128Shuffle shuffleFromArray(int[] indexes, int i) { return new Int128Shuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class Int128Vector extends IntVector {
return (long) super.reduceLanesTemplate(op, Int128Mask.class, (Int128Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Integer> toShuffle() {
return super.toShuffleTemplate(Int128Shuffle.class); // specialize
}
// Specialized unary testing
@ -767,26 +780,23 @@ final class Int128Vector extends IntVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Integer> ETYPE = int.class; // used by the JVM
Int128Shuffle(int[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Int128Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Int128Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Int128Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Int128Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Int128Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
int[] indices() {
return (int[])getPayload();
public Int128Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public IntSpecies vspecies() {
return VSPECIES;
}
@ -794,70 +804,40 @@ final class Int128Vector extends IntVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Integer.MAX_VALUE);
assert(Integer.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Int128Shuffle IOTA = new Int128Shuffle(IDENTITY);
@Override
@ForceInline
Int128Vector toBitsVector() {
return (Int128Vector) super.toBitsVectorTemplate();
public Int128Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Int128Shuffle.class, this, VLENGTH,
(s) -> ((Int128Vector)(((AbstractShuffle<Integer>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
IntVector toBitsVector0() {
return Int128Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
toBitsVector().intoArray(a, offset);
}
private static int[] prepare(int[] indices, int offset) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
public Int128Shuffle rearrange(VectorShuffle<Integer> shuffle) {
Int128Shuffle s = (Int128Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static int[] prepare(IntUnaryOperator f) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
}
return a;
}
private static boolean indicesInRange(int[] indices) {
int length = indices.length;
for (int si : indices) {
if (si >= (int)length || si < (int)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Int128Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Int256Vector extends IntVector {
@ForceInline
Int256Shuffle iotaShuffle() { return Int256Shuffle.IOTA; }
@ForceInline
Int256Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Int256Shuffle)VectorSupport.shuffleIota(ETYPE, Int256Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Int256Shuffle)VectorSupport.shuffleIota(ETYPE, Int256Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Int256Shuffle shuffleFromArray(int[] indices, int i) { return new Int256Shuffle(indices, i); }
Int256Shuffle shuffleFromBytes(byte[] reorder) { return new Int256Shuffle(reorder); }
@Override
@ForceInline
Int256Shuffle shuffleFromArray(int[] indexes, int i) { return new Int256Shuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class Int256Vector extends IntVector {
return (long) super.reduceLanesTemplate(op, Int256Mask.class, (Int256Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Integer> toShuffle() {
return super.toShuffleTemplate(Int256Shuffle.class); // specialize
}
// Specialized unary testing
@ -775,26 +788,23 @@ final class Int256Vector extends IntVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Integer> ETYPE = int.class; // used by the JVM
Int256Shuffle(int[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Int256Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Int256Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Int256Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Int256Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Int256Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
int[] indices() {
return (int[])getPayload();
public Int256Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public IntSpecies vspecies() {
return VSPECIES;
}
@ -802,70 +812,40 @@ final class Int256Vector extends IntVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Integer.MAX_VALUE);
assert(Integer.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Int256Shuffle IOTA = new Int256Shuffle(IDENTITY);
@Override
@ForceInline
Int256Vector toBitsVector() {
return (Int256Vector) super.toBitsVectorTemplate();
public Int256Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Int256Shuffle.class, this, VLENGTH,
(s) -> ((Int256Vector)(((AbstractShuffle<Integer>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
IntVector toBitsVector0() {
return Int256Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
toBitsVector().intoArray(a, offset);
}
private static int[] prepare(int[] indices, int offset) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
public Int256Shuffle rearrange(VectorShuffle<Integer> shuffle) {
Int256Shuffle s = (Int256Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static int[] prepare(IntUnaryOperator f) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
}
return a;
}
private static boolean indicesInRange(int[] indices) {
int length = indices.length;
for (int si : indices) {
if (si >= (int)length || si < (int)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Int256Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Int512Vector extends IntVector {
@ForceInline
Int512Shuffle iotaShuffle() { return Int512Shuffle.IOTA; }
@ForceInline
Int512Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Int512Shuffle)VectorSupport.shuffleIota(ETYPE, Int512Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Int512Shuffle)VectorSupport.shuffleIota(ETYPE, Int512Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Int512Shuffle shuffleFromArray(int[] indices, int i) { return new Int512Shuffle(indices, i); }
Int512Shuffle shuffleFromBytes(byte[] reorder) { return new Int512Shuffle(reorder); }
@Override
@ForceInline
Int512Shuffle shuffleFromArray(int[] indexes, int i) { return new Int512Shuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class Int512Vector extends IntVector {
return (long) super.reduceLanesTemplate(op, Int512Mask.class, (Int512Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Integer> toShuffle() {
return super.toShuffleTemplate(Int512Shuffle.class); // specialize
}
// Specialized unary testing
@ -791,26 +804,23 @@ final class Int512Vector extends IntVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Integer> ETYPE = int.class; // used by the JVM
Int512Shuffle(int[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Int512Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Int512Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Int512Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Int512Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Int512Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
int[] indices() {
return (int[])getPayload();
public Int512Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public IntSpecies vspecies() {
return VSPECIES;
}
@ -818,70 +828,40 @@ final class Int512Vector extends IntVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Integer.MAX_VALUE);
assert(Integer.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Int512Shuffle IOTA = new Int512Shuffle(IDENTITY);
@Override
@ForceInline
Int512Vector toBitsVector() {
return (Int512Vector) super.toBitsVectorTemplate();
public Int512Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Int512Shuffle.class, this, VLENGTH,
(s) -> ((Int512Vector)(((AbstractShuffle<Integer>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
IntVector toBitsVector0() {
return Int512Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
toBitsVector().intoArray(a, offset);
}
private static int[] prepare(int[] indices, int offset) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
public Int512Shuffle rearrange(VectorShuffle<Integer> shuffle) {
Int512Shuffle s = (Int512Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static int[] prepare(IntUnaryOperator f) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
}
return a;
}
private static boolean indicesInRange(int[] indices) {
int length = indices.length;
for (int si : indices) {
if (si >= (int)length || si < (int)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Int512Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Int64Vector extends IntVector {
@ForceInline
Int64Shuffle iotaShuffle() { return Int64Shuffle.IOTA; }
@ForceInline
Int64Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Int64Shuffle)VectorSupport.shuffleIota(ETYPE, Int64Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Int64Shuffle)VectorSupport.shuffleIota(ETYPE, Int64Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Int64Shuffle shuffleFromArray(int[] indices, int i) { return new Int64Shuffle(indices, i); }
Int64Shuffle shuffleFromBytes(byte[] reorder) { return new Int64Shuffle(reorder); }
@Override
@ForceInline
Int64Shuffle shuffleFromArray(int[] indexes, int i) { return new Int64Shuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class Int64Vector extends IntVector {
return (long) super.reduceLanesTemplate(op, Int64Mask.class, (Int64Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Integer> toShuffle() {
return super.toShuffleTemplate(Int64Shuffle.class); // specialize
}
// Specialized unary testing
@ -763,26 +776,23 @@ final class Int64Vector extends IntVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Integer> ETYPE = int.class; // used by the JVM
Int64Shuffle(int[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Int64Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Int64Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Int64Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Int64Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Int64Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
int[] indices() {
return (int[])getPayload();
public Int64Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public IntSpecies vspecies() {
return VSPECIES;
}
@ -790,70 +800,40 @@ final class Int64Vector extends IntVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Integer.MAX_VALUE);
assert(Integer.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Int64Shuffle IOTA = new Int64Shuffle(IDENTITY);
@Override
@ForceInline
Int64Vector toBitsVector() {
return (Int64Vector) super.toBitsVectorTemplate();
public Int64Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Int64Shuffle.class, this, VLENGTH,
(s) -> ((Int64Vector)(((AbstractShuffle<Integer>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
IntVector toBitsVector0() {
return Int64Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
toBitsVector().intoArray(a, offset);
}
private static int[] prepare(int[] indices, int offset) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
public Int64Shuffle rearrange(VectorShuffle<Integer> shuffle) {
Int64Shuffle s = (Int64Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static int[] prepare(IntUnaryOperator f) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
}
return a;
}
private static boolean indicesInRange(int[] indices) {
int length = indices.length;
for (int si : indices) {
if (si >= (int)length || si < (int)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Int64Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class IntMaxVector extends IntVector {
@ForceInline
IntMaxShuffle iotaShuffle() { return IntMaxShuffle.IOTA; }
@ForceInline
IntMaxShuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (IntMaxShuffle)VectorSupport.shuffleIota(ETYPE, IntMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (IntMaxShuffle)VectorSupport.shuffleIota(ETYPE, IntMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
IntMaxShuffle shuffleFromArray(int[] indices, int i) { return new IntMaxShuffle(indices, i); }
IntMaxShuffle shuffleFromBytes(byte[] reorder) { return new IntMaxShuffle(reorder); }
@Override
@ForceInline
IntMaxShuffle shuffleFromArray(int[] indexes, int i) { return new IntMaxShuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class IntMaxVector extends IntVector {
return (long) super.reduceLanesTemplate(op, IntMaxMask.class, (IntMaxMask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Integer> toShuffle() {
return super.toShuffleTemplate(IntMaxShuffle.class); // specialize
}
// Specialized unary testing
@ -772,26 +785,23 @@ final class IntMaxVector extends IntVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Integer> ETYPE = int.class; // used by the JVM
IntMaxShuffle(int[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
IntMaxShuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
IntMaxShuffle(int[] indices, int i) {
this(prepare(indices, i));
public IntMaxShuffle(int[] reorder) {
super(VLENGTH, reorder);
}
IntMaxShuffle(IntUnaryOperator fn) {
this(prepare(fn));
public IntMaxShuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
int[] indices() {
return (int[])getPayload();
public IntMaxShuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public IntSpecies vspecies() {
return VSPECIES;
}
@ -799,70 +809,40 @@ final class IntMaxVector extends IntVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Integer.MAX_VALUE);
assert(Integer.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final IntMaxShuffle IOTA = new IntMaxShuffle(IDENTITY);
@Override
@ForceInline
IntMaxVector toBitsVector() {
return (IntMaxVector) super.toBitsVectorTemplate();
public IntMaxVector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, IntMaxShuffle.class, this, VLENGTH,
(s) -> ((IntMaxVector)(((AbstractShuffle<Integer>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
IntVector toBitsVector0() {
return IntMaxVector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
toBitsVector().intoArray(a, offset);
}
private static int[] prepare(int[] indices, int offset) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
public IntMaxShuffle rearrange(VectorShuffle<Integer> shuffle) {
IntMaxShuffle s = (IntMaxShuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static int[] prepare(IntUnaryOperator f) {
int[] a = new int[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (int)si;
}
return a;
}
private static boolean indicesInRange(int[] indices) {
int length = indices.length;
for (int si : indices) {
if (si >= (int)length || si < (int)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new IntMaxShuffle(r);
}
}

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -1074,7 +1074,7 @@ public abstract class IntVector extends AbstractVector<Integer> {
// and broadcast, but it would be more surprising not to continue
// the obvious pattern started by unary and binary.
/**
/**
* {@inheritDoc} <!--workaround-->
* @see #lanewise(VectorOperators.Ternary,int,int,VectorMask)
* @see #lanewise(VectorOperators.Ternary,Vector,int,VectorMask)
@ -2465,8 +2465,8 @@ public abstract class IntVector extends AbstractVector<Integer> {
}
@ForceInline
final <F>
VectorShuffle<F> toShuffle0(AbstractSpecies<F> dsp) {
private final
VectorShuffle<Integer> toShuffle0(IntSpecies dsp) {
int[] a = toArray();
int[] sa = new int[a.length];
for (int i = 0; i < a.length; i++) {
@ -2475,6 +2475,18 @@ public abstract class IntVector extends AbstractVector<Integer> {
return VectorShuffle.fromArray(dsp, sa, 0);
}
/*package-private*/
@ForceInline
final
VectorShuffle<Integer> toShuffleTemplate(Class<?> shuffleType) {
IntSpecies vsp = vspecies();
return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
getClass(), int.class, length(),
shuffleType, byte.class, length(),
this, vsp,
IntVector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
@ -3793,10 +3805,9 @@ public abstract class IntVector extends AbstractVector<Integer> {
private IntSpecies(VectorShape shape,
Class<? extends IntVector> vectorType,
Class<? extends AbstractMask<Integer>> maskType,
Class<? extends AbstractShuffle<Integer>> shuffleType,
Function<Object, IntVector> vectorFactory) {
super(shape, LaneType.of(int.class),
vectorType, maskType, shuffleType,
vectorType, maskType,
vectorFactory);
assert(this.elementSize() == Integer.SIZE);
}
@ -4072,7 +4083,6 @@ public abstract class IntVector extends AbstractVector<Integer> {
= new IntSpecies(VectorShape.S_64_BIT,
Int64Vector.class,
Int64Vector.Int64Mask.class,
Int64Vector.Int64Shuffle.class,
Int64Vector::new);
/** Species representing {@link IntVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */
@ -4080,7 +4090,6 @@ public abstract class IntVector extends AbstractVector<Integer> {
= new IntSpecies(VectorShape.S_128_BIT,
Int128Vector.class,
Int128Vector.Int128Mask.class,
Int128Vector.Int128Shuffle.class,
Int128Vector::new);
/** Species representing {@link IntVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */
@ -4088,7 +4097,6 @@ public abstract class IntVector extends AbstractVector<Integer> {
= new IntSpecies(VectorShape.S_256_BIT,
Int256Vector.class,
Int256Vector.Int256Mask.class,
Int256Vector.Int256Shuffle.class,
Int256Vector::new);
/** Species representing {@link IntVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */
@ -4096,7 +4104,6 @@ public abstract class IntVector extends AbstractVector<Integer> {
= new IntSpecies(VectorShape.S_512_BIT,
Int512Vector.class,
Int512Vector.Int512Mask.class,
Int512Vector.Int512Shuffle.class,
Int512Vector::new);
/** Species representing {@link IntVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */
@ -4104,7 +4111,6 @@ public abstract class IntVector extends AbstractVector<Integer> {
= new IntSpecies(VectorShape.S_Max_BIT,
IntMaxVector.class,
IntMaxVector.IntMaxMask.class,
IntMaxVector.IntMaxShuffle.class,
IntMaxVector::new);
/**

@ -136,9 +136,24 @@ final class Long128Vector extends LongVector {
@ForceInline
Long128Shuffle iotaShuffle() { return Long128Shuffle.IOTA; }
@ForceInline
Long128Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Long128Shuffle)VectorSupport.shuffleIota(ETYPE, Long128Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Long128Shuffle)VectorSupport.shuffleIota(ETYPE, Long128Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Long128Shuffle shuffleFromArray(int[] indices, int i) { return new Long128Shuffle(indices, i); }
Long128Shuffle shuffleFromBytes(byte[] reorder) { return new Long128Shuffle(reorder); }
@Override
@ForceInline
Long128Shuffle shuffleFromArray(int[] indexes, int i) { return new Long128Shuffle(indexes, i); }
@Override
@ForceInline
@ -337,11 +352,9 @@ final class Long128Vector extends LongVector {
return (long) super.reduceLanesTemplate(op, Long128Mask.class, (Long128Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Long> toShuffle() {
return super.toShuffleTemplate(Long128Shuffle.class); // specialize
}
// Specialized unary testing
@ -753,26 +766,23 @@ final class Long128Vector extends LongVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Long> ETYPE = long.class; // used by the JVM
Long128Shuffle(long[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Long128Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Long128Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Long128Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Long128Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Long128Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
long[] indices() {
return (long[])getPayload();
public Long128Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public LongSpecies vspecies() {
return VSPECIES;
}
@ -780,94 +790,40 @@ final class Long128Vector extends LongVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Long.MAX_VALUE);
assert(Long.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Long128Shuffle IOTA = new Long128Shuffle(IDENTITY);
@Override
@ForceInline
Long128Vector toBitsVector() {
return (Long128Vector) super.toBitsVectorTemplate();
public Long128Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Long128Shuffle.class, this, VLENGTH,
(s) -> ((Long128Vector)(((AbstractShuffle<Long>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
LongVector toBitsVector0() {
return Long128Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
switch (length()) {
case 1 -> a[offset] = laneSource(0);
case 2 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_64, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 4 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_128, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 8 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_256, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 16 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_512, 0)
.reinterpretAsInts()
.intoArray(a, offset);
default -> {
VectorIntrinsics.checkFromIndexSize(offset, length(), a.length);
for (int i = 0; i < length(); i++) {
a[offset + i] = laneSource(i);
}
}
public Long128Shuffle rearrange(VectorShuffle<Long> shuffle) {
Long128Shuffle s = (Long128Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
}
private static long[] prepare(int[] indices, int offset) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static long[] prepare(IntUnaryOperator f) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static boolean indicesInRange(long[] indices) {
int length = indices.length;
for (long si : indices) {
if (si >= (long)length || si < (long)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Long128Shuffle(r);
}
}

@ -136,9 +136,24 @@ final class Long256Vector extends LongVector {
@ForceInline
Long256Shuffle iotaShuffle() { return Long256Shuffle.IOTA; }
@ForceInline
Long256Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Long256Shuffle)VectorSupport.shuffleIota(ETYPE, Long256Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Long256Shuffle)VectorSupport.shuffleIota(ETYPE, Long256Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Long256Shuffle shuffleFromArray(int[] indices, int i) { return new Long256Shuffle(indices, i); }
Long256Shuffle shuffleFromBytes(byte[] reorder) { return new Long256Shuffle(reorder); }
@Override
@ForceInline
Long256Shuffle shuffleFromArray(int[] indexes, int i) { return new Long256Shuffle(indexes, i); }
@Override
@ForceInline
@ -337,11 +352,9 @@ final class Long256Vector extends LongVector {
return (long) super.reduceLanesTemplate(op, Long256Mask.class, (Long256Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Long> toShuffle() {
return super.toShuffleTemplate(Long256Shuffle.class); // specialize
}
// Specialized unary testing
@ -757,26 +770,23 @@ final class Long256Vector extends LongVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Long> ETYPE = long.class; // used by the JVM
Long256Shuffle(long[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Long256Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Long256Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Long256Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Long256Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Long256Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
long[] indices() {
return (long[])getPayload();
public Long256Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public LongSpecies vspecies() {
return VSPECIES;
}
@ -784,94 +794,40 @@ final class Long256Vector extends LongVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Long.MAX_VALUE);
assert(Long.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Long256Shuffle IOTA = new Long256Shuffle(IDENTITY);
@Override
@ForceInline
Long256Vector toBitsVector() {
return (Long256Vector) super.toBitsVectorTemplate();
public Long256Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Long256Shuffle.class, this, VLENGTH,
(s) -> ((Long256Vector)(((AbstractShuffle<Long>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
LongVector toBitsVector0() {
return Long256Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
switch (length()) {
case 1 -> a[offset] = laneSource(0);
case 2 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_64, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 4 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_128, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 8 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_256, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 16 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_512, 0)
.reinterpretAsInts()
.intoArray(a, offset);
default -> {
VectorIntrinsics.checkFromIndexSize(offset, length(), a.length);
for (int i = 0; i < length(); i++) {
a[offset + i] = laneSource(i);
}
}
public Long256Shuffle rearrange(VectorShuffle<Long> shuffle) {
Long256Shuffle s = (Long256Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
}
private static long[] prepare(int[] indices, int offset) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static long[] prepare(IntUnaryOperator f) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static boolean indicesInRange(long[] indices) {
int length = indices.length;
for (long si : indices) {
if (si >= (long)length || si < (long)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Long256Shuffle(r);
}
}

@ -136,9 +136,24 @@ final class Long512Vector extends LongVector {
@ForceInline
Long512Shuffle iotaShuffle() { return Long512Shuffle.IOTA; }
@ForceInline
Long512Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Long512Shuffle)VectorSupport.shuffleIota(ETYPE, Long512Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Long512Shuffle)VectorSupport.shuffleIota(ETYPE, Long512Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Long512Shuffle shuffleFromArray(int[] indices, int i) { return new Long512Shuffle(indices, i); }
Long512Shuffle shuffleFromBytes(byte[] reorder) { return new Long512Shuffle(reorder); }
@Override
@ForceInline
Long512Shuffle shuffleFromArray(int[] indexes, int i) { return new Long512Shuffle(indexes, i); }
@Override
@ForceInline
@ -337,11 +352,9 @@ final class Long512Vector extends LongVector {
return (long) super.reduceLanesTemplate(op, Long512Mask.class, (Long512Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Long> toShuffle() {
return super.toShuffleTemplate(Long512Shuffle.class); // specialize
}
// Specialized unary testing
@ -765,26 +778,23 @@ final class Long512Vector extends LongVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Long> ETYPE = long.class; // used by the JVM
Long512Shuffle(long[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Long512Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Long512Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Long512Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Long512Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Long512Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
long[] indices() {
return (long[])getPayload();
public Long512Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public LongSpecies vspecies() {
return VSPECIES;
}
@ -792,94 +802,40 @@ final class Long512Vector extends LongVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Long.MAX_VALUE);
assert(Long.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Long512Shuffle IOTA = new Long512Shuffle(IDENTITY);
@Override
@ForceInline
Long512Vector toBitsVector() {
return (Long512Vector) super.toBitsVectorTemplate();
public Long512Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Long512Shuffle.class, this, VLENGTH,
(s) -> ((Long512Vector)(((AbstractShuffle<Long>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
LongVector toBitsVector0() {
return Long512Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
switch (length()) {
case 1 -> a[offset] = laneSource(0);
case 2 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_64, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 4 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_128, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 8 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_256, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 16 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_512, 0)
.reinterpretAsInts()
.intoArray(a, offset);
default -> {
VectorIntrinsics.checkFromIndexSize(offset, length(), a.length);
for (int i = 0; i < length(); i++) {
a[offset + i] = laneSource(i);
}
}
public Long512Shuffle rearrange(VectorShuffle<Long> shuffle) {
Long512Shuffle s = (Long512Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
}
private static long[] prepare(int[] indices, int offset) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static long[] prepare(IntUnaryOperator f) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static boolean indicesInRange(long[] indices) {
int length = indices.length;
for (long si : indices) {
if (si >= (long)length || si < (long)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Long512Shuffle(r);
}
}

@ -136,9 +136,24 @@ final class Long64Vector extends LongVector {
@ForceInline
Long64Shuffle iotaShuffle() { return Long64Shuffle.IOTA; }
@ForceInline
Long64Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Long64Shuffle)VectorSupport.shuffleIota(ETYPE, Long64Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Long64Shuffle)VectorSupport.shuffleIota(ETYPE, Long64Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Long64Shuffle shuffleFromArray(int[] indices, int i) { return new Long64Shuffle(indices, i); }
Long64Shuffle shuffleFromBytes(byte[] reorder) { return new Long64Shuffle(reorder); }
@Override
@ForceInline
Long64Shuffle shuffleFromArray(int[] indexes, int i) { return new Long64Shuffle(indexes, i); }
@Override
@ForceInline
@ -337,11 +352,9 @@ final class Long64Vector extends LongVector {
return (long) super.reduceLanesTemplate(op, Long64Mask.class, (Long64Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Long> toShuffle() {
return super.toShuffleTemplate(Long64Shuffle.class); // specialize
}
// Specialized unary testing
@ -751,26 +764,23 @@ final class Long64Vector extends LongVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Long> ETYPE = long.class; // used by the JVM
Long64Shuffle(long[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Long64Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Long64Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Long64Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Long64Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Long64Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
long[] indices() {
return (long[])getPayload();
public Long64Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public LongSpecies vspecies() {
return VSPECIES;
}
@ -778,94 +788,40 @@ final class Long64Vector extends LongVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Long.MAX_VALUE);
assert(Long.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Long64Shuffle IOTA = new Long64Shuffle(IDENTITY);
@Override
@ForceInline
Long64Vector toBitsVector() {
return (Long64Vector) super.toBitsVectorTemplate();
public Long64Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Long64Shuffle.class, this, VLENGTH,
(s) -> ((Long64Vector)(((AbstractShuffle<Long>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
LongVector toBitsVector0() {
return Long64Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
switch (length()) {
case 1 -> a[offset] = laneSource(0);
case 2 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_64, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 4 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_128, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 8 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_256, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 16 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_512, 0)
.reinterpretAsInts()
.intoArray(a, offset);
default -> {
VectorIntrinsics.checkFromIndexSize(offset, length(), a.length);
for (int i = 0; i < length(); i++) {
a[offset + i] = laneSource(i);
}
}
public Long64Shuffle rearrange(VectorShuffle<Long> shuffle) {
Long64Shuffle s = (Long64Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
}
private static long[] prepare(int[] indices, int offset) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static long[] prepare(IntUnaryOperator f) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static boolean indicesInRange(long[] indices) {
int length = indices.length;
for (long si : indices) {
if (si >= (long)length || si < (long)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Long64Shuffle(r);
}
}

@ -136,9 +136,24 @@ final class LongMaxVector extends LongVector {
@ForceInline
LongMaxShuffle iotaShuffle() { return LongMaxShuffle.IOTA; }
@ForceInline
LongMaxShuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (LongMaxShuffle)VectorSupport.shuffleIota(ETYPE, LongMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (LongMaxShuffle)VectorSupport.shuffleIota(ETYPE, LongMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
LongMaxShuffle shuffleFromArray(int[] indices, int i) { return new LongMaxShuffle(indices, i); }
LongMaxShuffle shuffleFromBytes(byte[] reorder) { return new LongMaxShuffle(reorder); }
@Override
@ForceInline
LongMaxShuffle shuffleFromArray(int[] indexes, int i) { return new LongMaxShuffle(indexes, i); }
@Override
@ForceInline
@ -337,11 +352,9 @@ final class LongMaxVector extends LongVector {
return (long) super.reduceLanesTemplate(op, LongMaxMask.class, (LongMaxMask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Long> toShuffle() {
return super.toShuffleTemplate(LongMaxShuffle.class); // specialize
}
// Specialized unary testing
@ -751,26 +764,23 @@ final class LongMaxVector extends LongVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Long> ETYPE = long.class; // used by the JVM
LongMaxShuffle(long[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
LongMaxShuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
LongMaxShuffle(int[] indices, int i) {
this(prepare(indices, i));
public LongMaxShuffle(int[] reorder) {
super(VLENGTH, reorder);
}
LongMaxShuffle(IntUnaryOperator fn) {
this(prepare(fn));
public LongMaxShuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
long[] indices() {
return (long[])getPayload();
public LongMaxShuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public LongSpecies vspecies() {
return VSPECIES;
}
@ -778,94 +788,40 @@ final class LongMaxVector extends LongVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Long.MAX_VALUE);
assert(Long.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final LongMaxShuffle IOTA = new LongMaxShuffle(IDENTITY);
@Override
@ForceInline
LongMaxVector toBitsVector() {
return (LongMaxVector) super.toBitsVectorTemplate();
public LongMaxVector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, LongMaxShuffle.class, this, VLENGTH,
(s) -> ((LongMaxVector)(((AbstractShuffle<Long>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
LongVector toBitsVector0() {
return LongMaxVector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
switch (length()) {
case 1 -> a[offset] = laneSource(0);
case 2 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_64, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 4 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_128, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 8 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_256, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 16 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_512, 0)
.reinterpretAsInts()
.intoArray(a, offset);
default -> {
VectorIntrinsics.checkFromIndexSize(offset, length(), a.length);
for (int i = 0; i < length(); i++) {
a[offset + i] = laneSource(i);
}
}
public LongMaxShuffle rearrange(VectorShuffle<Long> shuffle) {
LongMaxShuffle s = (LongMaxShuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
}
private static long[] prepare(int[] indices, int offset) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static long[] prepare(IntUnaryOperator f) {
long[] a = new long[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (long)si;
}
return a;
}
private static boolean indicesInRange(long[] indices) {
int length = indices.length;
for (long si : indices) {
if (si >= (long)length || si < (long)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new LongMaxShuffle(r);
}
}

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -987,7 +987,7 @@ public abstract class LongVector extends AbstractVector<Long> {
// and broadcast, but it would be more surprising not to continue
// the obvious pattern started by unary and binary.
/**
/**
* {@inheritDoc} <!--workaround-->
* @see #lanewise(VectorOperators.Ternary,long,long,VectorMask)
* @see #lanewise(VectorOperators.Ternary,Vector,long,VectorMask)
@ -2331,8 +2331,8 @@ public abstract class LongVector extends AbstractVector<Long> {
}
@ForceInline
final <F>
VectorShuffle<F> toShuffle0(AbstractSpecies<F> dsp) {
private final
VectorShuffle<Long> toShuffle0(LongSpecies dsp) {
long[] a = toArray();
int[] sa = new int[a.length];
for (int i = 0; i < a.length; i++) {
@ -2341,6 +2341,18 @@ public abstract class LongVector extends AbstractVector<Long> {
return VectorShuffle.fromArray(dsp, sa, 0);
}
/*package-private*/
@ForceInline
final
VectorShuffle<Long> toShuffleTemplate(Class<?> shuffleType) {
LongSpecies vsp = vspecies();
return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
getClass(), long.class, length(),
shuffleType, byte.class, length(),
this, vsp,
LongVector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
@ -3728,10 +3740,9 @@ public abstract class LongVector extends AbstractVector<Long> {
private LongSpecies(VectorShape shape,
Class<? extends LongVector> vectorType,
Class<? extends AbstractMask<Long>> maskType,
Class<? extends AbstractShuffle<Long>> shuffleType,
Function<Object, LongVector> vectorFactory) {
super(shape, LaneType.of(long.class),
vectorType, maskType, shuffleType,
vectorType, maskType,
vectorFactory);
assert(this.elementSize() == Long.SIZE);
}
@ -3998,7 +4009,6 @@ public abstract class LongVector extends AbstractVector<Long> {
= new LongSpecies(VectorShape.S_64_BIT,
Long64Vector.class,
Long64Vector.Long64Mask.class,
Long64Vector.Long64Shuffle.class,
Long64Vector::new);
/** Species representing {@link LongVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */
@ -4006,7 +4016,6 @@ public abstract class LongVector extends AbstractVector<Long> {
= new LongSpecies(VectorShape.S_128_BIT,
Long128Vector.class,
Long128Vector.Long128Mask.class,
Long128Vector.Long128Shuffle.class,
Long128Vector::new);
/** Species representing {@link LongVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */
@ -4014,7 +4023,6 @@ public abstract class LongVector extends AbstractVector<Long> {
= new LongSpecies(VectorShape.S_256_BIT,
Long256Vector.class,
Long256Vector.Long256Mask.class,
Long256Vector.Long256Shuffle.class,
Long256Vector::new);
/** Species representing {@link LongVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */
@ -4022,7 +4030,6 @@ public abstract class LongVector extends AbstractVector<Long> {
= new LongSpecies(VectorShape.S_512_BIT,
Long512Vector.class,
Long512Vector.Long512Mask.class,
Long512Vector.Long512Shuffle.class,
Long512Vector::new);
/** Species representing {@link LongVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */
@ -4030,7 +4037,6 @@ public abstract class LongVector extends AbstractVector<Long> {
= new LongSpecies(VectorShape.S_Max_BIT,
LongMaxVector.class,
LongMaxVector.LongMaxMask.class,
LongMaxVector.LongMaxShuffle.class,
LongMaxVector::new);
/**

@ -141,9 +141,24 @@ final class Short128Vector extends ShortVector {
@ForceInline
Short128Shuffle iotaShuffle() { return Short128Shuffle.IOTA; }
@ForceInline
Short128Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Short128Shuffle)VectorSupport.shuffleIota(ETYPE, Short128Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Short128Shuffle)VectorSupport.shuffleIota(ETYPE, Short128Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Short128Shuffle shuffleFromArray(int[] indices, int i) { return new Short128Shuffle(indices, i); }
Short128Shuffle shuffleFromBytes(byte[] reorder) { return new Short128Shuffle(reorder); }
@Override
@ForceInline
Short128Shuffle shuffleFromArray(int[] indexes, int i) { return new Short128Shuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class Short128Vector extends ShortVector {
return (long) super.reduceLanesTemplate(op, Short128Mask.class, (Short128Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Short> toShuffle() {
return super.toShuffleTemplate(Short128Shuffle.class); // specialize
}
// Specialized unary testing
@ -775,26 +788,23 @@ final class Short128Vector extends ShortVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Short> ETYPE = short.class; // used by the JVM
Short128Shuffle(short[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Short128Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Short128Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Short128Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Short128Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Short128Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
short[] indices() {
return (short[])getPayload();
public Short128Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public ShortSpecies vspecies() {
return VSPECIES;
}
@ -802,77 +812,40 @@ final class Short128Vector extends ShortVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Short.MAX_VALUE);
assert(Short.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Short128Shuffle IOTA = new Short128Shuffle(IDENTITY);
@Override
@ForceInline
Short128Vector toBitsVector() {
return (Short128Vector) super.toBitsVectorTemplate();
public Short128Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Short128Shuffle.class, this, VLENGTH,
(s) -> ((Short128Vector)(((AbstractShuffle<Short>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
ShortVector toBitsVector0() {
return Short128Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
VectorSpecies<Integer> species = IntVector.SPECIES_128;
Vector<Short> v = toBitsVector();
v.convertShape(VectorOperators.S2I, species, 0)
.reinterpretAsInts()
.intoArray(a, offset);
v.convertShape(VectorOperators.S2I, species, 1)
.reinterpretAsInts()
.intoArray(a, offset + species.length());
}
private static short[] prepare(int[] indices, int offset) {
short[] a = new short[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (short)si;
public Short128Shuffle rearrange(VectorShuffle<Short> shuffle) {
Short128Shuffle s = (Short128Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static short[] prepare(IntUnaryOperator f) {
short[] a = new short[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (short)si;
}
return a;
}
private static boolean indicesInRange(short[] indices) {
int length = indices.length;
for (short si : indices) {
if (si >= (short)length || si < (short)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Short128Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Short256Vector extends ShortVector {
@ForceInline
Short256Shuffle iotaShuffle() { return Short256Shuffle.IOTA; }
@ForceInline
Short256Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Short256Shuffle)VectorSupport.shuffleIota(ETYPE, Short256Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Short256Shuffle)VectorSupport.shuffleIota(ETYPE, Short256Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Short256Shuffle shuffleFromArray(int[] indices, int i) { return new Short256Shuffle(indices, i); }
Short256Shuffle shuffleFromBytes(byte[] reorder) { return new Short256Shuffle(reorder); }
@Override
@ForceInline
Short256Shuffle shuffleFromArray(int[] indexes, int i) { return new Short256Shuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class Short256Vector extends ShortVector {
return (long) super.reduceLanesTemplate(op, Short256Mask.class, (Short256Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Short> toShuffle() {
return super.toShuffleTemplate(Short256Shuffle.class); // specialize
}
// Specialized unary testing
@ -791,26 +804,23 @@ final class Short256Vector extends ShortVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Short> ETYPE = short.class; // used by the JVM
Short256Shuffle(short[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Short256Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Short256Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Short256Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Short256Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Short256Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
short[] indices() {
return (short[])getPayload();
public Short256Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public ShortSpecies vspecies() {
return VSPECIES;
}
@ -818,77 +828,40 @@ final class Short256Vector extends ShortVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Short.MAX_VALUE);
assert(Short.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Short256Shuffle IOTA = new Short256Shuffle(IDENTITY);
@Override
@ForceInline
Short256Vector toBitsVector() {
return (Short256Vector) super.toBitsVectorTemplate();
public Short256Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Short256Shuffle.class, this, VLENGTH,
(s) -> ((Short256Vector)(((AbstractShuffle<Short>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
ShortVector toBitsVector0() {
return Short256Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
VectorSpecies<Integer> species = IntVector.SPECIES_256;
Vector<Short> v = toBitsVector();
v.convertShape(VectorOperators.S2I, species, 0)
.reinterpretAsInts()
.intoArray(a, offset);
v.convertShape(VectorOperators.S2I, species, 1)
.reinterpretAsInts()
.intoArray(a, offset + species.length());
}
private static short[] prepare(int[] indices, int offset) {
short[] a = new short[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (short)si;
public Short256Shuffle rearrange(VectorShuffle<Short> shuffle) {
Short256Shuffle s = (Short256Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static short[] prepare(IntUnaryOperator f) {
short[] a = new short[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (short)si;
}
return a;
}
private static boolean indicesInRange(short[] indices) {
int length = indices.length;
for (short si : indices) {
if (si >= (short)length || si < (short)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Short256Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Short512Vector extends ShortVector {
@ForceInline
Short512Shuffle iotaShuffle() { return Short512Shuffle.IOTA; }
@ForceInline
Short512Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Short512Shuffle)VectorSupport.shuffleIota(ETYPE, Short512Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Short512Shuffle)VectorSupport.shuffleIota(ETYPE, Short512Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Short512Shuffle shuffleFromArray(int[] indices, int i) { return new Short512Shuffle(indices, i); }
Short512Shuffle shuffleFromBytes(byte[] reorder) { return new Short512Shuffle(reorder); }
@Override
@ForceInline
Short512Shuffle shuffleFromArray(int[] indexes, int i) { return new Short512Shuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class Short512Vector extends ShortVector {
return (long) super.reduceLanesTemplate(op, Short512Mask.class, (Short512Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Short> toShuffle() {
return super.toShuffleTemplate(Short512Shuffle.class); // specialize
}
// Specialized unary testing
@ -823,26 +836,23 @@ final class Short512Vector extends ShortVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Short> ETYPE = short.class; // used by the JVM
Short512Shuffle(short[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Short512Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Short512Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Short512Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Short512Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Short512Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
short[] indices() {
return (short[])getPayload();
public Short512Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public ShortSpecies vspecies() {
return VSPECIES;
}
@ -850,77 +860,40 @@ final class Short512Vector extends ShortVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Short.MAX_VALUE);
assert(Short.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Short512Shuffle IOTA = new Short512Shuffle(IDENTITY);
@Override
@ForceInline
Short512Vector toBitsVector() {
return (Short512Vector) super.toBitsVectorTemplate();
public Short512Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Short512Shuffle.class, this, VLENGTH,
(s) -> ((Short512Vector)(((AbstractShuffle<Short>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
ShortVector toBitsVector0() {
return Short512Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
VectorSpecies<Integer> species = IntVector.SPECIES_512;
Vector<Short> v = toBitsVector();
v.convertShape(VectorOperators.S2I, species, 0)
.reinterpretAsInts()
.intoArray(a, offset);
v.convertShape(VectorOperators.S2I, species, 1)
.reinterpretAsInts()
.intoArray(a, offset + species.length());
}
private static short[] prepare(int[] indices, int offset) {
short[] a = new short[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (short)si;
public Short512Shuffle rearrange(VectorShuffle<Short> shuffle) {
Short512Shuffle s = (Short512Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static short[] prepare(IntUnaryOperator f) {
short[] a = new short[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (short)si;
}
return a;
}
private static boolean indicesInRange(short[] indices) {
int length = indices.length;
for (short si : indices) {
if (si >= (short)length || si < (short)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Short512Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class Short64Vector extends ShortVector {
@ForceInline
Short64Shuffle iotaShuffle() { return Short64Shuffle.IOTA; }
@ForceInline
Short64Shuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (Short64Shuffle)VectorSupport.shuffleIota(ETYPE, Short64Shuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (Short64Shuffle)VectorSupport.shuffleIota(ETYPE, Short64Shuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
Short64Shuffle shuffleFromArray(int[] indices, int i) { return new Short64Shuffle(indices, i); }
Short64Shuffle shuffleFromBytes(byte[] reorder) { return new Short64Shuffle(reorder); }
@Override
@ForceInline
Short64Shuffle shuffleFromArray(int[] indexes, int i) { return new Short64Shuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class Short64Vector extends ShortVector {
return (long) super.reduceLanesTemplate(op, Short64Mask.class, (Short64Mask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Short> toShuffle() {
return super.toShuffleTemplate(Short64Shuffle.class); // specialize
}
// Specialized unary testing
@ -767,26 +780,23 @@ final class Short64Vector extends ShortVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Short> ETYPE = short.class; // used by the JVM
Short64Shuffle(short[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
Short64Shuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
Short64Shuffle(int[] indices, int i) {
this(prepare(indices, i));
public Short64Shuffle(int[] reorder) {
super(VLENGTH, reorder);
}
Short64Shuffle(IntUnaryOperator fn) {
this(prepare(fn));
public Short64Shuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
short[] indices() {
return (short[])getPayload();
public Short64Shuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public ShortSpecies vspecies() {
return VSPECIES;
}
@ -794,77 +804,40 @@ final class Short64Vector extends ShortVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Short.MAX_VALUE);
assert(Short.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final Short64Shuffle IOTA = new Short64Shuffle(IDENTITY);
@Override
@ForceInline
Short64Vector toBitsVector() {
return (Short64Vector) super.toBitsVectorTemplate();
public Short64Vector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, Short64Shuffle.class, this, VLENGTH,
(s) -> ((Short64Vector)(((AbstractShuffle<Short>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
ShortVector toBitsVector0() {
return Short64Vector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
VectorSpecies<Integer> species = IntVector.SPECIES_64;
Vector<Short> v = toBitsVector();
v.convertShape(VectorOperators.S2I, species, 0)
.reinterpretAsInts()
.intoArray(a, offset);
v.convertShape(VectorOperators.S2I, species, 1)
.reinterpretAsInts()
.intoArray(a, offset + species.length());
}
private static short[] prepare(int[] indices, int offset) {
short[] a = new short[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (short)si;
public Short64Shuffle rearrange(VectorShuffle<Short> shuffle) {
Short64Shuffle s = (Short64Shuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static short[] prepare(IntUnaryOperator f) {
short[] a = new short[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (short)si;
}
return a;
}
private static boolean indicesInRange(short[] indices) {
int length = indices.length;
for (short si : indices) {
if (si >= (short)length || si < (short)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new Short64Shuffle(r);
}
}

@ -141,9 +141,24 @@ final class ShortMaxVector extends ShortVector {
@ForceInline
ShortMaxShuffle iotaShuffle() { return ShortMaxShuffle.IOTA; }
@ForceInline
ShortMaxShuffle iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return (ShortMaxShuffle)VectorSupport.shuffleIota(ETYPE, ShortMaxShuffle.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return (ShortMaxShuffle)VectorSupport.shuffleIota(ETYPE, ShortMaxShuffle.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
ShortMaxShuffle shuffleFromArray(int[] indices, int i) { return new ShortMaxShuffle(indices, i); }
ShortMaxShuffle shuffleFromBytes(byte[] reorder) { return new ShortMaxShuffle(reorder); }
@Override
@ForceInline
ShortMaxShuffle shuffleFromArray(int[] indexes, int i) { return new ShortMaxShuffle(indexes, i); }
@Override
@ForceInline
@ -342,11 +357,9 @@ final class ShortMaxVector extends ShortVector {
return (long) super.reduceLanesTemplate(op, ShortMaxMask.class, (ShortMaxMask) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<Short> toShuffle() {
return super.toShuffleTemplate(ShortMaxShuffle.class); // specialize
}
// Specialized unary testing
@ -761,26 +774,23 @@ final class ShortMaxVector extends ShortVector {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<Short> ETYPE = short.class; // used by the JVM
ShortMaxShuffle(short[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
ShortMaxShuffle(byte[] reorder) {
super(VLENGTH, reorder);
}
ShortMaxShuffle(int[] indices, int i) {
this(prepare(indices, i));
public ShortMaxShuffle(int[] reorder) {
super(VLENGTH, reorder);
}
ShortMaxShuffle(IntUnaryOperator fn) {
this(prepare(fn));
public ShortMaxShuffle(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
short[] indices() {
return (short[])getPayload();
public ShortMaxShuffle(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public ShortSpecies vspecies() {
return VSPECIES;
}
@ -788,77 +798,40 @@ final class ShortMaxVector extends ShortVector {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < Short.MAX_VALUE);
assert(Short.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final ShortMaxShuffle IOTA = new ShortMaxShuffle(IDENTITY);
@Override
@ForceInline
ShortMaxVector toBitsVector() {
return (ShortMaxVector) super.toBitsVectorTemplate();
public ShortMaxVector toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, ShortMaxShuffle.class, this, VLENGTH,
(s) -> ((ShortMaxVector)(((AbstractShuffle<Short>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
ShortVector toBitsVector0() {
return ShortMaxVector.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
VectorSpecies<Integer> species = IntVector.SPECIES_MAX;
Vector<Short> v = toBitsVector();
v.convertShape(VectorOperators.S2I, species, 0)
.reinterpretAsInts()
.intoArray(a, offset);
v.convertShape(VectorOperators.S2I, species, 1)
.reinterpretAsInts()
.intoArray(a, offset + species.length());
}
private static short[] prepare(int[] indices, int offset) {
short[] a = new short[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (short)si;
public ShortMaxShuffle rearrange(VectorShuffle<Short> shuffle) {
ShortMaxShuffle s = (ShortMaxShuffle) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
return a;
}
private static short[] prepare(IntUnaryOperator f) {
short[] a = new short[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = (short)si;
}
return a;
}
private static boolean indicesInRange(short[] indices) {
int length = indices.length;
for (short si : indices) {
if (si >= (short)length || si < (short)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new ShortMaxShuffle(r);
}
}

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -1071,7 +1071,7 @@ public abstract class ShortVector extends AbstractVector<Short> {
// and broadcast, but it would be more surprising not to continue
// the obvious pattern started by unary and binary.
/**
/**
* {@inheritDoc} <!--workaround-->
* @see #lanewise(VectorOperators.Ternary,short,short,VectorMask)
* @see #lanewise(VectorOperators.Ternary,Vector,short,VectorMask)
@ -2481,8 +2481,8 @@ public abstract class ShortVector extends AbstractVector<Short> {
}
@ForceInline
final <F>
VectorShuffle<F> toShuffle0(AbstractSpecies<F> dsp) {
private final
VectorShuffle<Short> toShuffle0(ShortSpecies dsp) {
short[] a = toArray();
int[] sa = new int[a.length];
for (int i = 0; i < a.length; i++) {
@ -2491,6 +2491,18 @@ public abstract class ShortVector extends AbstractVector<Short> {
return VectorShuffle.fromArray(dsp, sa, 0);
}
/*package-private*/
@ForceInline
final
VectorShuffle<Short> toShuffleTemplate(Class<?> shuffleType) {
ShortSpecies vsp = vspecies();
return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
getClass(), short.class, length(),
shuffleType, byte.class, length(),
this, vsp,
ShortVector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
@ -4078,10 +4090,9 @@ public abstract class ShortVector extends AbstractVector<Short> {
private ShortSpecies(VectorShape shape,
Class<? extends ShortVector> vectorType,
Class<? extends AbstractMask<Short>> maskType,
Class<? extends AbstractShuffle<Short>> shuffleType,
Function<Object, ShortVector> vectorFactory) {
super(shape, LaneType.of(short.class),
vectorType, maskType, shuffleType,
vectorType, maskType,
vectorFactory);
assert(this.elementSize() == Short.SIZE);
}
@ -4357,7 +4368,6 @@ public abstract class ShortVector extends AbstractVector<Short> {
= new ShortSpecies(VectorShape.S_64_BIT,
Short64Vector.class,
Short64Vector.Short64Mask.class,
Short64Vector.Short64Shuffle.class,
Short64Vector::new);
/** Species representing {@link ShortVector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */
@ -4365,7 +4375,6 @@ public abstract class ShortVector extends AbstractVector<Short> {
= new ShortSpecies(VectorShape.S_128_BIT,
Short128Vector.class,
Short128Vector.Short128Mask.class,
Short128Vector.Short128Shuffle.class,
Short128Vector::new);
/** Species representing {@link ShortVector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */
@ -4373,7 +4382,6 @@ public abstract class ShortVector extends AbstractVector<Short> {
= new ShortSpecies(VectorShape.S_256_BIT,
Short256Vector.class,
Short256Vector.Short256Mask.class,
Short256Vector.Short256Shuffle.class,
Short256Vector::new);
/** Species representing {@link ShortVector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */
@ -4381,7 +4389,6 @@ public abstract class ShortVector extends AbstractVector<Short> {
= new ShortSpecies(VectorShape.S_512_BIT,
Short512Vector.class,
Short512Vector.Short512Mask.class,
Short512Vector.Short512Shuffle.class,
Short512Vector::new);
/** Species representing {@link ShortVector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */
@ -4389,7 +4396,6 @@ public abstract class ShortVector extends AbstractVector<Short> {
= new ShortSpecies(VectorShape.S_Max_BIT,
ShortMaxVector.class,
ShortMaxVector.ShortMaxMask.class,
ShortMaxVector.ShortMaxShuffle.class,
ShortMaxVector::new);
/**

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -124,7 +124,6 @@ public enum VectorShape {
* @throws IllegalArgumentException if no such vector shape exists
* @see #vectorBitSize()
*/
@ForceInline
public static VectorShape forBitSize(int bitSize) {
switch (bitSize) {
case 64:

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -133,8 +133,8 @@ import java.util.function.IntUnaryOperator;
*/
@SuppressWarnings("exports")
public abstract class VectorShuffle<E> extends jdk.internal.vm.vector.VectorSupport.VectorShuffle<E> {
VectorShuffle(Object indices) {
super(indices);
VectorShuffle(byte[] reorder) {
super(reorder);
}
/**
@ -556,7 +556,7 @@ public abstract class VectorShuffle<E> extends jdk.internal.vm.vector.VectorSupp
* @param i the lane index
* @return the {@code int} lane element at lane index {@code i}
*/
public abstract int laneSource(int i);
public int laneSource(int i) { return toArray()[i]; }
/**
* Rearranges the lane elements of this shuffle selecting lane indexes

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -24,8 +24,6 @@
*/
package jdk.incubator.vector;
import jdk.internal.vm.annotation.ForceInline;
import java.lang.foreign.MemorySegment;
import java.nio.ByteOrder;
@ -344,7 +342,6 @@ public interface VectorSpecies<E> {
* @see #withLanes(Class)
* @see #withShape(VectorShape)
*/
@ForceInline
static <E> VectorSpecies<E> of(Class<E> elementType, VectorShape shape) {
LaneType laneType = LaneType.of(elementType);
return AbstractSpecies.findSpecies(elementType, laneType, shape);
@ -370,7 +367,6 @@ public interface VectorSpecies<E> {
* or if the given type is not a valid {@code ETYPE}
* @see VectorSpecies#ofPreferred(Class)
*/
@ForceInline
static <E> VectorSpecies<E> ofLargestShape(Class<E> etype) {
return VectorSpecies.of(etype, VectorShape.largestShapeFor(etype));
}
@ -414,7 +410,6 @@ public interface VectorSpecies<E> {
* @see VectorShape#preferredShape()
* @see VectorSpecies#ofLargestShape(Class)
*/
@ForceInline
public static <E> VectorSpecies<E> ofPreferred(Class<E> etype) {
return of(etype, VectorShape.preferredShape());
}
@ -437,7 +432,6 @@ public interface VectorSpecies<E> {
* if the given {@code elementType} argument is not
* a valid vector {@code ETYPE}
*/
@ForceInline
static int elementSize(Class<?> elementType) {
return LaneType.of(elementType).elementSize;
}

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -1204,7 +1204,7 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
// and broadcast, but it would be more surprising not to continue
// the obvious pattern started by unary and binary.
/**
/**
* {@inheritDoc} <!--workaround-->
* @see #lanewise(VectorOperators.Ternary,$type$,$type$,VectorMask)
* @see #lanewise(VectorOperators.Ternary,Vector,$type$,VectorMask)
@ -2857,8 +2857,8 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
}
@ForceInline
final <F>
VectorShuffle<F> toShuffle0(AbstractSpecies<F> dsp) {
private final
VectorShuffle<$Boxtype$> toShuffle0($Type$Species dsp) {
$type$[] a = toArray();
int[] sa = new int[a.length];
for (int i = 0; i < a.length; i++) {
@ -2867,6 +2867,18 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
return VectorShuffle.fromArray(dsp, sa, 0);
}
/*package-private*/
@ForceInline
final
VectorShuffle<$Boxtype$> toShuffleTemplate(Class<?> shuffleType) {
$Type$Species vsp = vspecies();
return VectorSupport.convert(VectorSupport.VECTOR_OP_CAST,
getClass(), $type$.class, length(),
shuffleType, byte.class, length(),
this, vsp,
$Type$Vector::toShuffle0);
}
/**
* {@inheritDoc} <!--workaround-->
* @since 19
@ -5336,10 +5348,9 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
private $Type$Species(VectorShape shape,
Class<? extends $abstractvectortype$> vectorType,
Class<? extends AbstractMask<$Boxtype$>> maskType,
Class<? extends AbstractShuffle<$Boxtype$>> shuffleType,
Function<Object, $abstractvectortype$> vectorFactory) {
super(shape, LaneType.of($type$.class),
vectorType, maskType, shuffleType,
vectorType, maskType,
vectorFactory);
assert(this.elementSize() == $Boxtype$.SIZE);
}
@ -5622,7 +5633,6 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
= new $Type$Species(VectorShape.S_64_BIT,
$Type$64Vector.class,
$Type$64Vector.$Type$64Mask.class,
$Type$64Vector.$Type$64Shuffle.class,
$Type$64Vector::new);
/** Species representing {@link $Type$Vector}s of {@link VectorShape#S_128_BIT VectorShape.S_128_BIT}. */
@ -5630,7 +5640,6 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
= new $Type$Species(VectorShape.S_128_BIT,
$Type$128Vector.class,
$Type$128Vector.$Type$128Mask.class,
$Type$128Vector.$Type$128Shuffle.class,
$Type$128Vector::new);
/** Species representing {@link $Type$Vector}s of {@link VectorShape#S_256_BIT VectorShape.S_256_BIT}. */
@ -5638,7 +5647,6 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
= new $Type$Species(VectorShape.S_256_BIT,
$Type$256Vector.class,
$Type$256Vector.$Type$256Mask.class,
$Type$256Vector.$Type$256Shuffle.class,
$Type$256Vector::new);
/** Species representing {@link $Type$Vector}s of {@link VectorShape#S_512_BIT VectorShape.S_512_BIT}. */
@ -5646,7 +5654,6 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
= new $Type$Species(VectorShape.S_512_BIT,
$Type$512Vector.class,
$Type$512Vector.$Type$512Mask.class,
$Type$512Vector.$Type$512Shuffle.class,
$Type$512Vector::new);
/** Species representing {@link $Type$Vector}s of {@link VectorShape#S_Max_BIT VectorShape.S_Max_BIT}. */
@ -5654,7 +5661,6 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
= new $Type$Species(VectorShape.S_Max_BIT,
$Type$MaxVector.class,
$Type$MaxVector.$Type$MaxMask.class,
$Type$MaxVector.$Type$MaxShuffle.class,
$Type$MaxVector::new);
/**

@ -143,9 +143,24 @@ final class $vectortype$ extends $abstractvectortype$ {
@ForceInline
$shuffletype$ iotaShuffle() { return $shuffletype$.IOTA; }
@ForceInline
$shuffletype$ iotaShuffle(int start, int step, boolean wrap) {
if (wrap) {
return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 1,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (VectorIntrinsics.wrapToRange(i*lstep + lstart, l))));
} else {
return ($shuffletype$)VectorSupport.shuffleIota(ETYPE, $shuffletype$.class, VSPECIES, VLENGTH, start, step, 0,
(l, lstart, lstep, s) -> s.shuffleFromOp(i -> (i*lstep + lstart)));
}
}
@Override
@ForceInline
$shuffletype$ shuffleFromArray(int[] indices, int i) { return new $shuffletype$(indices, i); }
$shuffletype$ shuffleFromBytes(byte[] reorder) { return new $shuffletype$(reorder); }
@Override
@ForceInline
$shuffletype$ shuffleFromArray(int[] indexes, int i) { return new $shuffletype$(indexes, i); }
@Override
@ForceInline
@ -346,11 +361,9 @@ final class $vectortype$ extends $abstractvectortype$ {
return (long) super.reduceLanesTemplate(op, $masktype$.class, ($masktype$) m); // specialized
}
@Override
@ForceInline
public final
<F> VectorShuffle<F> toShuffle(AbstractSpecies<F> dsp) {
return super.toShuffleTemplate(dsp);
public VectorShuffle<$Boxtype$> toShuffle() {
return super.toShuffleTemplate($shuffletype$.class); // specialize
}
// Specialized unary testing
@ -1047,28 +1060,25 @@ final class $vectortype$ extends $abstractvectortype$ {
static final class $shuffletype$ extends AbstractShuffle<$Boxtype$> {
static final int VLENGTH = VSPECIES.laneCount(); // used by the JVM
static final Class<$Boxbitstype$> ETYPE = $bitstype$.class; // used by the JVM
static final Class<$Boxtype$> ETYPE = $type$.class; // used by the JVM
$shuffletype$($bitstype$[] indices) {
super(indices);
assert(VLENGTH == indices.length);
assert(indicesInRange(indices));
$shuffletype$(byte[] reorder) {
super(VLENGTH, reorder);
}
$shuffletype$(int[] indices, int i) {
this(prepare(indices, i));
public $shuffletype$(int[] reorder) {
super(VLENGTH, reorder);
}
$shuffletype$(IntUnaryOperator fn) {
this(prepare(fn));
public $shuffletype$(int[] reorder, int i) {
super(VLENGTH, reorder, i);
}
$bitstype$[] indices() {
return ($bitstype$[])getPayload();
public $shuffletype$(IntUnaryOperator fn) {
super(VLENGTH, fn);
}
@Override
@ForceInline
public $Type$Species vspecies() {
return VSPECIES;
}
@ -1076,125 +1086,40 @@ final class $vectortype$ extends $abstractvectortype$ {
static {
// There must be enough bits in the shuffle lanes to encode
// VLENGTH valid indexes and VLENGTH exceptional ones.
assert(VLENGTH < $Boxbitstype$.MAX_VALUE);
assert($Boxbitstype$.MIN_VALUE <= -VLENGTH);
assert(VLENGTH < Byte.MAX_VALUE);
assert(Byte.MIN_VALUE <= -VLENGTH);
}
static final $shuffletype$ IOTA = new $shuffletype$(IDENTITY);
@Override
@ForceInline
$bitsvectortype$ toBitsVector() {
return ($bitsvectortype$) super.toBitsVectorTemplate();
public $vectortype$ toVector() {
return VectorSupport.shuffleToVector(VCLASS, ETYPE, $shuffletype$.class, this, VLENGTH,
(s) -> (($vectortype$)(((AbstractShuffle<$Boxtype$>)(s)).toVectorTemplate())));
}
@Override
@ForceInline
$Bitstype$Vector toBitsVector0() {
return $bitsvectortype$.VSPECIES.dummyVector().vectorFactory(indices());
public <F> VectorShuffle<F> cast(VectorSpecies<F> s) {
AbstractSpecies<F> species = (AbstractSpecies<F>) s;
if (length() != species.laneCount())
throw new IllegalArgumentException("VectorShuffle length and species length differ");
int[] shuffleArray = toArray();
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public int laneSource(int i) {
return (int)toBitsVector().lane(i);
}
@Override
@ForceInline
public void intoArray(int[] a, int offset) {
#if[byte]
VectorSpecies<Integer> species = IntVector.SPECIES_$BITS$;
Vector<Byte> v = toBitsVector();
v.convertShape(VectorOperators.B2I, species, 0)
.reinterpretAsInts()
.intoArray(a, offset);
v.convertShape(VectorOperators.B2I, species, 1)
.reinterpretAsInts()
.intoArray(a, offset + species.length());
v.convertShape(VectorOperators.B2I, species, 2)
.reinterpretAsInts()
.intoArray(a, offset + species.length() * 2);
v.convertShape(VectorOperators.B2I, species, 3)
.reinterpretAsInts()
.intoArray(a, offset + species.length() * 3);
#end[byte]
#if[short]
VectorSpecies<Integer> species = IntVector.SPECIES_$BITS$;
Vector<Short> v = toBitsVector();
v.convertShape(VectorOperators.S2I, species, 0)
.reinterpretAsInts()
.intoArray(a, offset);
v.convertShape(VectorOperators.S2I, species, 1)
.reinterpretAsInts()
.intoArray(a, offset + species.length());
#end[short]
#if[intOrFloat]
toBitsVector().intoArray(a, offset);
#end[intOrFloat]
#if[longOrDouble]
switch (length()) {
case 1 -> a[offset] = laneSource(0);
case 2 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_64, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 4 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_128, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 8 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_256, 0)
.reinterpretAsInts()
.intoArray(a, offset);
case 16 -> toBitsVector()
.convertShape(VectorOperators.L2I, IntVector.SPECIES_512, 0)
.reinterpretAsInts()
.intoArray(a, offset);
default -> {
VectorIntrinsics.checkFromIndexSize(offset, length(), a.length);
for (int i = 0; i < length(); i++) {
a[offset + i] = laneSource(i);
}
}
public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) {
$shuffletype$ s = ($shuffletype$) shuffle;
byte[] reorder1 = reorder();
byte[] reorder2 = s.reorder();
byte[] r = new byte[reorder1.length];
for (int i = 0; i < reorder1.length; i++) {
int ssi = reorder2[i];
r[i] = reorder1[ssi]; // throws on exceptional index
}
#end[longOrDouble]
}
private static $bitstype$[] prepare(int[] indices, int offset) {
$bitstype$[] a = new $bitstype$[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = indices[offset + i];
si = partiallyWrapIndex(si, VLENGTH);
a[i] = ($bitstype$)si;
}
return a;
}
private static $bitstype$[] prepare(IntUnaryOperator f) {
$bitstype$[] a = new $bitstype$[VLENGTH];
for (int i = 0; i < VLENGTH; i++) {
int si = f.applyAsInt(i);
si = partiallyWrapIndex(si, VLENGTH);
a[i] = ($bitstype$)si;
}
return a;
}
private static boolean indicesInRange($bitstype$[] indices) {
int length = indices.length;
for ($bitstype$ si : indices) {
if (si >= ($bitstype$)length || si < ($bitstype$)(-length)) {
boolean assertsEnabled = false;
assert(assertsEnabled = true);
if (assertsEnabled) {
String msg = ("index "+si+"out of range ["+length+"] in "+
java.util.Arrays.toString(indices));
throw new AssertionError(msg);
}
return false;
}
}
return true;
return new $shuffletype$(r);
}
}