8340079: Modify rearrange/selectFrom Vector API methods to perform wrapIndexes instead of checkIndexes

Reviewed-by: jbhateja, psandoz
This commit is contained in:
Sandhya Viswanathan 2024-10-01 22:48:31 +00:00
parent d2e77089aa
commit 83dcb02d77
47 changed files with 663 additions and 148 deletions

@ -4357,7 +4357,7 @@ bool MatchRule::is_vector() const {
"RoundDoubleModeV","RotateLeftV" , "RotateRightV", "LoadVector","StoreVector",
"LoadVectorGather", "StoreVectorScatter", "LoadVectorGatherMasked", "StoreVectorScatterMasked",
"VectorTest", "VectorLoadMask", "VectorStoreMask", "VectorBlend", "VectorInsert",
"VectorRearrange","VectorLoadShuffle", "VectorLoadConst",
"VectorRearrange", "VectorLoadShuffle", "VectorLoadConst",
"VectorCastB2X", "VectorCastS2X", "VectorCastI2X",
"VectorCastL2X", "VectorCastF2X", "VectorCastD2X", "VectorCastF2HF", "VectorCastHF2F",
"VectorUCastB2X", "VectorUCastS2X", "VectorUCastI2X",

@ -1008,6 +1008,15 @@ class methodHandle;
"Ljdk/internal/vm/vector/VectorSupport$Vector;") \
do_name(vector_shuffle_to_vector_name, "shuffleToVector") \
\
do_intrinsic(_VectorWrapShuffleIndexes, jdk_internal_vm_vector_VectorSupport, vector_wrap_shuffle_indexes_name, \
vector_wrap_shuffle_indexes_sig, F_S) \
do_signature(vector_wrap_shuffle_indexes_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;" \
"ILjdk/internal/vm/vector/VectorSupport$WrapShuffleIndexesOperation;)" \
"Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;") \
do_name(vector_wrap_shuffle_indexes_name, "wrapShuffleIndexes") \
\
do_intrinsic(_VectorLoadOp, jdk_internal_vm_vector_VectorSupport, vector_load_op_name, vector_load_op_sig, F_S) \
do_signature(vector_load_op_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
@ -1129,6 +1138,18 @@ class methodHandle;
"Ljdk/internal/vm/vector/VectorSupport$Vector;") \
do_name(vector_rearrange_name, "rearrangeOp") \
\
do_intrinsic(_VectorSelectFrom, jdk_internal_vm_vector_VectorSupport, vector_select_from_name, vector_select_from_sig, F_S) \
do_signature(vector_select_from_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \
"Ljava/lang/Class;" \
"I" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
"Ljdk/internal/vm/vector/VectorSupport$VectorSelectFromOp;)" \
"Ljdk/internal/vm/vector/VectorSupport$Vector;") \
do_name(vector_select_from_name, "selectFromOp") \
\
do_intrinsic(_VectorExtract, jdk_internal_vm_vector_VectorSupport, vector_extract_name, vector_extract_sig, F_S) \
do_signature(vector_extract_sig, "(Ljava/lang/Class;" \
"Ljava/lang/Class;" \

@ -811,6 +811,7 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
case vmIntrinsics::_VectorFromBitsCoerced:
case vmIntrinsics::_VectorShuffleIota:
case vmIntrinsics::_VectorShuffleToVector:
case vmIntrinsics::_VectorWrapShuffleIndexes:
case vmIntrinsics::_VectorLoadOp:
case vmIntrinsics::_VectorLoadMaskedOp:
case vmIntrinsics::_VectorStoreOp:
@ -821,6 +822,7 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
case vmIntrinsics::_VectorTest:
case vmIntrinsics::_VectorBlend:
case vmIntrinsics::_VectorRearrange:
case vmIntrinsics::_VectorSelectFrom:
case vmIntrinsics::_VectorCompare:
case vmIntrinsics::_VectorBroadcastInt:
case vmIntrinsics::_VectorConvert:

@ -717,6 +717,8 @@ bool LibraryCallKit::try_to_inline(int predicate) {
return inline_vector_mask_operation();
case vmIntrinsics::_VectorShuffleToVector:
return inline_vector_shuffle_to_vector();
case vmIntrinsics::_VectorWrapShuffleIndexes:
return inline_vector_wrap_shuffle_indexes();
case vmIntrinsics::_VectorLoadOp:
return inline_vector_mem_operation(/*is_store=*/false);
case vmIntrinsics::_VectorLoadMaskedOp:
@ -737,6 +739,8 @@ bool LibraryCallKit::try_to_inline(int predicate) {
return inline_vector_blend();
case vmIntrinsics::_VectorRearrange:
return inline_vector_rearrange();
case vmIntrinsics::_VectorSelectFrom:
return inline_vector_select_from();
case vmIntrinsics::_VectorCompare:
return inline_vector_compare();
case vmIntrinsics::_VectorBroadcastInt:

@ -353,6 +353,7 @@ class LibraryCallKit : public GraphKit {
bool inline_vector_nary_operation(int n);
bool inline_vector_frombits_coerced();
bool inline_vector_shuffle_to_vector();
bool inline_vector_wrap_shuffle_indexes();
bool inline_vector_shuffle_iota();
Node* partially_wrap_indexes(Node* index_vec, int num_elem, BasicType type_bt);
bool inline_vector_mask_operation();
@ -363,6 +364,7 @@ class LibraryCallKit : public GraphKit {
bool inline_vector_test();
bool inline_vector_blend();
bool inline_vector_rearrange();
bool inline_vector_select_from();
bool inline_vector_compare();
bool inline_vector_broadcast_int();
bool inline_vector_convert();

@ -757,6 +757,64 @@ bool LibraryCallKit::inline_vector_shuffle_to_vector() {
return true;
}
// public static
// <E,
// SH extends VectorShuffle<E>>
// SH wrapShuffleIndexes(Class<E> eClass, Class<? extends SH> shClass, SH sh, int length,
// ShuffleWrapIndexesOperation<SH> defaultImpl)
bool LibraryCallKit::inline_vector_wrap_shuffle_indexes() {
const TypeInstPtr* elem_klass = gvn().type(argument(0))->isa_instptr();
const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
Node* shuffle = argument(2);
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
if (elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr ||
!vlen->is_con() || shuffle_klass->const_oop() == nullptr) {
// not enough info for intrinsification
return false;
}
if (!is_klass_initialized(shuffle_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
}
int num_elem = vlen->get_con();
if ((num_elem < 4) || !is_power_of_2(num_elem)) {
log_if_needed(" ** vlen < 4 or not power of two=%d", num_elem);
return false;
}
// Shuffles use byte array based backing storage
BasicType shuffle_bt = T_BYTE;
if (!arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
!arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
log_if_needed(" ** not supported: op=wrapShuffleIndexes vlen=%d etype=%s",
num_elem, type2name(shuffle_bt));
return false;
}
ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
// Unbox shuffle with true flag to indicate its load shuffle to vector
// shuffle is a byte array
Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, shuffle_bt, num_elem, true);
const TypeVect* vt = TypeVect::make(shuffle_bt, num_elem);
const Type* shuffle_type_bt = Type::get_const_basic_type(shuffle_bt);
Node* mod_mask = gvn().makecon(TypeInt::make(num_elem-1));
Node* bcast_mod_mask = gvn().transform(VectorNode::scalar2vector(mod_mask, num_elem, shuffle_type_bt));
// Wrap the indices greater than lane count.
Node* res = gvn().transform(VectorNode::make(Op_AndV, shuffle_vec, bcast_mod_mask, vt));
// Wrap it up in VectorBox to keep object type information.
res = box_vector(res, shuffle_box_type, shuffle_bt, num_elem);
set_result(res);
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(shuffle_bt))));
return true;
}
// public static
// <M,
// S extends VectorSpecies<E>,
@ -2044,6 +2102,150 @@ static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr,
return addr;
}
// public static
// <V extends Vector<E>,
// M extends VectorMask<E>,
// E>
// V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
// int length, V v1, V v2, M m,
// VectorSelectFromOp<V, M> defaultImpl)
bool LibraryCallKit::inline_vector_select_from() {
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
vector_klass->const_oop() == nullptr ||
elem_klass->const_oop() == nullptr ||
!vlen->is_con()) {
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
NodeClassNames[argument(0)->Opcode()],
NodeClassNames[argument(2)->Opcode()],
NodeClassNames[argument(3)->Opcode()]);
return false; // not enough info for intrinsification
}
if (!is_klass_initialized(vector_klass)) {
log_if_needed(" ** klass argument not initialized");
return false;
}
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
if (!elem_type->is_primitive_type()) {
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
return false; // should be primitive type
}
BasicType elem_bt = elem_type->basic_type();
int num_elem = vlen->get_con();
if (!is_power_of_2(num_elem)) {
log_if_needed(" ** vlen not power of two=%d", num_elem);
return false;
}
int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)||
!arch_supports_vector(Op_AndV, num_elem, T_BYTE, VecMaskNotUsed) ||
!arch_supports_vector(Op_Replicate, num_elem, T_BYTE, VecMaskNotUsed) ||
!arch_supports_vector(cast_vopc, num_elem, T_BYTE, VecMaskNotUsed)) {
log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
num_elem, type2name(elem_bt));
return false; // not supported
}
bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
bool use_predicate = is_masked_op;
if (is_masked_op &&
(mask_klass == nullptr ||
mask_klass->const_oop() == nullptr ||
!is_klass_initialized(mask_klass))) {
log_if_needed(" ** mask_klass argument not initialized");
return false; // not supported
}
VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
use_predicate = false;
if(!is_masked_op ||
(!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
!arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
num_elem, type2name(elem_bt), is_masked_op);
return false; // not supported
}
}
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
// v1 is the index vector
Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
// v2 is the vector being rearranged
Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
if (v1 == nullptr) {
log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
return false; // operand unboxing failed
}
if (v2 == nullptr) {
log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
return false; // operand unboxing failed
}
Node* mask = nullptr;
if (is_masked_op) {
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
if (mask == nullptr) {
log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
return false;
}
}
// cast index vector from elem_bt vector to byte vector
const Type * byte_bt = Type::get_const_basic_type(T_BYTE);
const TypeVect * byte_vt = TypeVect::make(byte_bt, num_elem);
Node* byte_shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, T_BYTE, num_elem));
// wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
// this is a simple AND operation as we come here only for power of two vector length
Node* mod_val = gvn().makecon(TypeInt::make(num_elem-1));
Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, byte_bt));
byte_shuffle = gvn().transform(VectorNode::make(Op_AndV, byte_shuffle, bcast_mod, byte_vt));
// load the shuffle to use in rearrange
const TypeVect * shuffle_vt = TypeVect::make(elem_bt, num_elem);
Node* load_shuffle = gvn().transform(new VectorLoadShuffleNode(byte_shuffle, shuffle_vt));
// and finally rearrange
Node* rearrange = new VectorRearrangeNode(v2, load_shuffle);
if (is_masked_op) {
if (use_predicate) {
// masked rearrange is supported so use that directly
rearrange->add_req(mask);
rearrange->add_flag(Node::Flag_is_predicated_vector);
} else {
// masked rearrange is not supported so emulate usig blend
const TypeVect* vt = v1->bottom_type()->is_vect();
rearrange = gvn().transform(rearrange);
// create a zero vector with each lane element set as zero
Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
// For each lane for which mask is set, blend in the rearranged lane into zero vector
rearrange = new VectorBlendNode(zerovec, rearrange, mask);
}
}
rearrange = gvn().transform(rearrange);
// box the result
Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
set_result(box);
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
return true;
}
Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
assert(UseVectorStubs, "sanity");
assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");

@ -263,6 +263,20 @@ public class VectorSupport {
return defaultImpl.apply(sh);
}
public interface WrapShuffleIndexesOperation<SH extends VectorShuffle<?>> {
SH apply(SH sh);
}
@IntrinsicCandidate
public static
<E,
SH extends VectorShuffle<E>>
SH wrapShuffleIndexes(Class<E> eClass, Class<? extends SH> shClass, SH sh, int length,
WrapShuffleIndexesOperation<SH> defaultImpl) {
assert isNonCapturingLambda(defaultImpl) : defaultImpl;
return defaultImpl.apply(sh);
}
/* ============================================================================ */
public interface IndexOperation<V extends Vector<?>,
S extends VectorSpecies<?>> {
@ -605,6 +619,23 @@ public class VectorSupport {
return defaultImpl.apply(v, sh, m);
}
public interface VectorSelectFromOp<V extends Vector<?>,
M extends VectorMask<?>> {
V apply(V v1, V v2, M m);
}
@IntrinsicCandidate
public static
<V extends Vector<E>,
M extends VectorMask<E>,
E>
V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
int length, V v1, V v2, M m,
VectorSelectFromOp<V, M> defaultImpl) {
assert isNonCapturingLambda(defaultImpl) : defaultImpl;
return defaultImpl.apply(v1, v2, m);
}
/* ============================================================================ */
public interface VectorBlendOp<V extends Vector<?>,

@ -133,7 +133,7 @@ abstract class AbstractShuffle<E> extends VectorShuffle<E> {
}
@ForceInline
public final VectorShuffle<E> wrapIndexes() {
public final VectorShuffle<E> wrapIndexesTemplate() {
Vector<E> shufvec = this.toVector();
VectorMask<E> vecmask = shufvec.compare(VectorOperators.LT, vspecies().zero());
if (vecmask.anyTrue()) {

@ -503,7 +503,7 @@ final class Byte128Vector extends ByteVector {
VectorMask<Byte> m) {
return (Byte128Vector)
super.selectFromTemplate((Byte128Vector) v,
(Byte128Mask) m); // specialize
Byte128Mask.class, (Byte128Mask) m); // specialize
}
@ -860,6 +860,13 @@ final class Byte128Vector extends ByteVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Byte128Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Byte128Shuffle.class, this, VLENGTH,
(s) -> ((Byte128Shuffle)(((AbstractShuffle<Byte>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Byte128Shuffle rearrange(VectorShuffle<Byte> shuffle) {

@ -503,7 +503,7 @@ final class Byte256Vector extends ByteVector {
VectorMask<Byte> m) {
return (Byte256Vector)
super.selectFromTemplate((Byte256Vector) v,
(Byte256Mask) m); // specialize
Byte256Mask.class, (Byte256Mask) m); // specialize
}
@ -892,6 +892,13 @@ final class Byte256Vector extends ByteVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Byte256Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Byte256Shuffle.class, this, VLENGTH,
(s) -> ((Byte256Shuffle)(((AbstractShuffle<Byte>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Byte256Shuffle rearrange(VectorShuffle<Byte> shuffle) {

@ -503,7 +503,7 @@ final class Byte512Vector extends ByteVector {
VectorMask<Byte> m) {
return (Byte512Vector)
super.selectFromTemplate((Byte512Vector) v,
(Byte512Mask) m); // specialize
Byte512Mask.class, (Byte512Mask) m); // specialize
}
@ -956,6 +956,13 @@ final class Byte512Vector extends ByteVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Byte512Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Byte512Shuffle.class, this, VLENGTH,
(s) -> ((Byte512Shuffle)(((AbstractShuffle<Byte>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Byte512Shuffle rearrange(VectorShuffle<Byte> shuffle) {

@ -503,7 +503,7 @@ final class Byte64Vector extends ByteVector {
VectorMask<Byte> m) {
return (Byte64Vector)
super.selectFromTemplate((Byte64Vector) v,
(Byte64Mask) m); // specialize
Byte64Mask.class, (Byte64Mask) m); // specialize
}
@ -844,6 +844,13 @@ final class Byte64Vector extends ByteVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Byte64Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Byte64Shuffle.class, this, VLENGTH,
(s) -> ((Byte64Shuffle)(((AbstractShuffle<Byte>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Byte64Shuffle rearrange(VectorShuffle<Byte> shuffle) {

@ -503,7 +503,7 @@ final class ByteMaxVector extends ByteVector {
VectorMask<Byte> m) {
return (ByteMaxVector)
super.selectFromTemplate((ByteMaxVector) v,
(ByteMaxMask) m); // specialize
ByteMaxMask.class, (ByteMaxMask) m); // specialize
}
@ -830,6 +830,13 @@ final class ByteMaxVector extends ByteVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public ByteMaxShuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, ByteMaxShuffle.class, this, VLENGTH,
(s) -> ((ByteMaxShuffle)(((AbstractShuffle<Byte>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public ByteMaxShuffle rearrange(VectorShuffle<Byte> shuffle) {

@ -2393,17 +2393,18 @@ public abstract class ByteVector extends AbstractVector<Byte> {
*/
@Override
public abstract
ByteVector rearrange(VectorShuffle<Byte> m);
ByteVector rearrange(VectorShuffle<Byte> shuffle);
/*package-private*/
@ForceInline
final
<S extends VectorShuffle<Byte>>
ByteVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
shuffle.checkIndexes();
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, null, byte.class, length(),
this, shuffle, null,
this, ws, null,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return v1.lane(ei);
@ -2428,17 +2429,14 @@ public abstract class ByteVector extends AbstractVector<Byte> {
M m) {
m.check(masktype, this);
VectorMask<Byte> valid = shuffle.laneIsValid();
if (m.andNot(valid).anyTrue()) {
shuffle.checkIndexes();
throw new AssertionError();
}
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, masktype, byte.class, length(),
this, shuffle, m,
this, ws, m,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
}));
}
@ -2551,7 +2549,10 @@ public abstract class ByteVector extends AbstractVector<Byte> {
/*package-private*/
@ForceInline
final ByteVector selectFromTemplate(ByteVector v) {
return v.rearrange(this.toShuffle());
return (ByteVector)VectorSupport.selectFromOp(getClass(), null, byte.class,
length(), this, v, null,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle()));
}
/**
@ -2563,9 +2564,15 @@ public abstract class ByteVector extends AbstractVector<Byte> {
/*package-private*/
@ForceInline
final ByteVector selectFromTemplate(ByteVector v,
AbstractMask<Byte> m) {
return v.rearrange(this.toShuffle(), m);
final
<M extends VectorMask<Byte>>
ByteVector selectFromTemplate(ByteVector v,
Class<M> masktype, M m) {
m.check(masktype, this);
return (ByteVector)VectorSupport.selectFromOp(getClass(), masktype, byte.class,
length(), this, v, m,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle(), _m));
}
/// Ternary operations

@ -490,7 +490,7 @@ final class Double128Vector extends DoubleVector {
VectorMask<Double> m) {
return (Double128Vector)
super.selectFromTemplate((Double128Vector) v,
(Double128Mask) m); // specialize
Double128Mask.class, (Double128Mask) m); // specialize
}
@ -821,6 +821,13 @@ final class Double128Vector extends DoubleVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Double128Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Double128Shuffle.class, this, VLENGTH,
(s) -> ((Double128Shuffle)(((AbstractShuffle<Double>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Double128Shuffle rearrange(VectorShuffle<Double> shuffle) {

@ -490,7 +490,7 @@ final class Double256Vector extends DoubleVector {
VectorMask<Double> m) {
return (Double256Vector)
super.selectFromTemplate((Double256Vector) v,
(Double256Mask) m); // specialize
Double256Mask.class, (Double256Mask) m); // specialize
}
@ -825,6 +825,13 @@ final class Double256Vector extends DoubleVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Double256Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Double256Shuffle.class, this, VLENGTH,
(s) -> ((Double256Shuffle)(((AbstractShuffle<Double>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Double256Shuffle rearrange(VectorShuffle<Double> shuffle) {

@ -490,7 +490,7 @@ final class Double512Vector extends DoubleVector {
VectorMask<Double> m) {
return (Double512Vector)
super.selectFromTemplate((Double512Vector) v,
(Double512Mask) m); // specialize
Double512Mask.class, (Double512Mask) m); // specialize
}
@ -833,6 +833,13 @@ final class Double512Vector extends DoubleVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Double512Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Double512Shuffle.class, this, VLENGTH,
(s) -> ((Double512Shuffle)(((AbstractShuffle<Double>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Double512Shuffle rearrange(VectorShuffle<Double> shuffle) {

@ -490,7 +490,7 @@ final class Double64Vector extends DoubleVector {
VectorMask<Double> m) {
return (Double64Vector)
super.selectFromTemplate((Double64Vector) v,
(Double64Mask) m); // specialize
Double64Mask.class, (Double64Mask) m); // specialize
}
@ -819,6 +819,13 @@ final class Double64Vector extends DoubleVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Double64Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Double64Shuffle.class, this, VLENGTH,
(s) -> ((Double64Shuffle)(((AbstractShuffle<Double>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Double64Shuffle rearrange(VectorShuffle<Double> shuffle) {

@ -490,7 +490,7 @@ final class DoubleMaxVector extends DoubleVector {
VectorMask<Double> m) {
return (DoubleMaxVector)
super.selectFromTemplate((DoubleMaxVector) v,
(DoubleMaxMask) m); // specialize
DoubleMaxMask.class, (DoubleMaxMask) m); // specialize
}
@ -818,6 +818,13 @@ final class DoubleMaxVector extends DoubleVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public DoubleMaxShuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, DoubleMaxShuffle.class, this, VLENGTH,
(s) -> ((DoubleMaxShuffle)(((AbstractShuffle<Double>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public DoubleMaxShuffle rearrange(VectorShuffle<Double> shuffle) {

@ -2235,17 +2235,18 @@ public abstract class DoubleVector extends AbstractVector<Double> {
*/
@Override
public abstract
DoubleVector rearrange(VectorShuffle<Double> m);
DoubleVector rearrange(VectorShuffle<Double> shuffle);
/*package-private*/
@ForceInline
final
<S extends VectorShuffle<Double>>
DoubleVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
shuffle.checkIndexes();
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, null, double.class, length(),
this, shuffle, null,
this, ws, null,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return v1.lane(ei);
@ -2270,17 +2271,14 @@ public abstract class DoubleVector extends AbstractVector<Double> {
M m) {
m.check(masktype, this);
VectorMask<Double> valid = shuffle.laneIsValid();
if (m.andNot(valid).anyTrue()) {
shuffle.checkIndexes();
throw new AssertionError();
}
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, masktype, double.class, length(),
this, shuffle, m,
this, ws, m,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
}));
}
@ -2393,7 +2391,10 @@ public abstract class DoubleVector extends AbstractVector<Double> {
/*package-private*/
@ForceInline
final DoubleVector selectFromTemplate(DoubleVector v) {
return v.rearrange(this.toShuffle());
return (DoubleVector)VectorSupport.selectFromOp(getClass(), null, double.class,
length(), this, v, null,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle()));
}
/**
@ -2405,9 +2406,15 @@ public abstract class DoubleVector extends AbstractVector<Double> {
/*package-private*/
@ForceInline
final DoubleVector selectFromTemplate(DoubleVector v,
AbstractMask<Double> m) {
return v.rearrange(this.toShuffle(), m);
final
<M extends VectorMask<Double>>
DoubleVector selectFromTemplate(DoubleVector v,
Class<M> masktype, M m) {
m.check(masktype, this);
return (DoubleVector)VectorSupport.selectFromOp(getClass(), masktype, double.class,
length(), this, v, m,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle(), _m));
}
/// Ternary operations

@ -490,7 +490,7 @@ final class Float128Vector extends FloatVector {
VectorMask<Float> m) {
return (Float128Vector)
super.selectFromTemplate((Float128Vector) v,
(Float128Mask) m); // specialize
Float128Mask.class, (Float128Mask) m); // specialize
}
@ -825,6 +825,13 @@ final class Float128Vector extends FloatVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Float128Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Float128Shuffle.class, this, VLENGTH,
(s) -> ((Float128Shuffle)(((AbstractShuffle<Float>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Float128Shuffle rearrange(VectorShuffle<Float> shuffle) {

@ -490,7 +490,7 @@ final class Float256Vector extends FloatVector {
VectorMask<Float> m) {
return (Float256Vector)
super.selectFromTemplate((Float256Vector) v,
(Float256Mask) m); // specialize
Float256Mask.class, (Float256Mask) m); // specialize
}
@ -833,6 +833,13 @@ final class Float256Vector extends FloatVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Float256Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Float256Shuffle.class, this, VLENGTH,
(s) -> ((Float256Shuffle)(((AbstractShuffle<Float>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Float256Shuffle rearrange(VectorShuffle<Float> shuffle) {

@ -490,7 +490,7 @@ final class Float512Vector extends FloatVector {
VectorMask<Float> m) {
return (Float512Vector)
super.selectFromTemplate((Float512Vector) v,
(Float512Mask) m); // specialize
Float512Mask.class, (Float512Mask) m); // specialize
}
@ -849,6 +849,13 @@ final class Float512Vector extends FloatVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Float512Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Float512Shuffle.class, this, VLENGTH,
(s) -> ((Float512Shuffle)(((AbstractShuffle<Float>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Float512Shuffle rearrange(VectorShuffle<Float> shuffle) {

@ -490,7 +490,7 @@ final class Float64Vector extends FloatVector {
VectorMask<Float> m) {
return (Float64Vector)
super.selectFromTemplate((Float64Vector) v,
(Float64Mask) m); // specialize
Float64Mask.class, (Float64Mask) m); // specialize
}
@ -821,6 +821,13 @@ final class Float64Vector extends FloatVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Float64Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Float64Shuffle.class, this, VLENGTH,
(s) -> ((Float64Shuffle)(((AbstractShuffle<Float>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Float64Shuffle rearrange(VectorShuffle<Float> shuffle) {

@ -490,7 +490,7 @@ final class FloatMaxVector extends FloatVector {
VectorMask<Float> m) {
return (FloatMaxVector)
super.selectFromTemplate((FloatMaxVector) v,
(FloatMaxMask) m); // specialize
FloatMaxMask.class, (FloatMaxMask) m); // specialize
}
@ -818,6 +818,13 @@ final class FloatMaxVector extends FloatVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public FloatMaxShuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, FloatMaxShuffle.class, this, VLENGTH,
(s) -> ((FloatMaxShuffle)(((AbstractShuffle<Float>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public FloatMaxShuffle rearrange(VectorShuffle<Float> shuffle) {

@ -2247,17 +2247,18 @@ public abstract class FloatVector extends AbstractVector<Float> {
*/
@Override
public abstract
FloatVector rearrange(VectorShuffle<Float> m);
FloatVector rearrange(VectorShuffle<Float> shuffle);
/*package-private*/
@ForceInline
final
<S extends VectorShuffle<Float>>
FloatVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
shuffle.checkIndexes();
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, null, float.class, length(),
this, shuffle, null,
this, ws, null,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return v1.lane(ei);
@ -2282,17 +2283,14 @@ public abstract class FloatVector extends AbstractVector<Float> {
M m) {
m.check(masktype, this);
VectorMask<Float> valid = shuffle.laneIsValid();
if (m.andNot(valid).anyTrue()) {
shuffle.checkIndexes();
throw new AssertionError();
}
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, masktype, float.class, length(),
this, shuffle, m,
this, ws, m,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
}));
}
@ -2405,7 +2403,10 @@ public abstract class FloatVector extends AbstractVector<Float> {
/*package-private*/
@ForceInline
final FloatVector selectFromTemplate(FloatVector v) {
return v.rearrange(this.toShuffle());
return (FloatVector)VectorSupport.selectFromOp(getClass(), null, float.class,
length(), this, v, null,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle()));
}
/**
@ -2417,9 +2418,15 @@ public abstract class FloatVector extends AbstractVector<Float> {
/*package-private*/
@ForceInline
final FloatVector selectFromTemplate(FloatVector v,
AbstractMask<Float> m) {
return v.rearrange(this.toShuffle(), m);
final
<M extends VectorMask<Float>>
FloatVector selectFromTemplate(FloatVector v,
Class<M> masktype, M m) {
m.check(masktype, this);
return (FloatVector)VectorSupport.selectFromOp(getClass(), masktype, float.class,
length(), this, v, m,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle(), _m));
}
/// Ternary operations

@ -503,7 +503,7 @@ final class Int128Vector extends IntVector {
VectorMask<Integer> m) {
return (Int128Vector)
super.selectFromTemplate((Int128Vector) v,
(Int128Mask) m); // specialize
Int128Mask.class, (Int128Mask) m); // specialize
}
@ -836,6 +836,13 @@ final class Int128Vector extends IntVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Int128Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Int128Shuffle.class, this, VLENGTH,
(s) -> ((Int128Shuffle)(((AbstractShuffle<Integer>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Int128Shuffle rearrange(VectorShuffle<Integer> shuffle) {

@ -503,7 +503,7 @@ final class Int256Vector extends IntVector {
VectorMask<Integer> m) {
return (Int256Vector)
super.selectFromTemplate((Int256Vector) v,
(Int256Mask) m); // specialize
Int256Mask.class, (Int256Mask) m); // specialize
}
@ -844,6 +844,13 @@ final class Int256Vector extends IntVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Int256Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Int256Shuffle.class, this, VLENGTH,
(s) -> ((Int256Shuffle)(((AbstractShuffle<Integer>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Int256Shuffle rearrange(VectorShuffle<Integer> shuffle) {

@ -503,7 +503,7 @@ final class Int512Vector extends IntVector {
VectorMask<Integer> m) {
return (Int512Vector)
super.selectFromTemplate((Int512Vector) v,
(Int512Mask) m); // specialize
Int512Mask.class, (Int512Mask) m); // specialize
}
@ -860,6 +860,13 @@ final class Int512Vector extends IntVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Int512Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Int512Shuffle.class, this, VLENGTH,
(s) -> ((Int512Shuffle)(((AbstractShuffle<Integer>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Int512Shuffle rearrange(VectorShuffle<Integer> shuffle) {

@ -503,7 +503,7 @@ final class Int64Vector extends IntVector {
VectorMask<Integer> m) {
return (Int64Vector)
super.selectFromTemplate((Int64Vector) v,
(Int64Mask) m); // specialize
Int64Mask.class, (Int64Mask) m); // specialize
}
@ -832,6 +832,13 @@ final class Int64Vector extends IntVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Int64Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Int64Shuffle.class, this, VLENGTH,
(s) -> ((Int64Shuffle)(((AbstractShuffle<Integer>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Int64Shuffle rearrange(VectorShuffle<Integer> shuffle) {

@ -503,7 +503,7 @@ final class IntMaxVector extends IntVector {
VectorMask<Integer> m) {
return (IntMaxVector)
super.selectFromTemplate((IntMaxVector) v,
(IntMaxMask) m); // specialize
IntMaxMask.class, (IntMaxMask) m); // specialize
}
@ -841,6 +841,13 @@ final class IntMaxVector extends IntVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public IntMaxShuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, IntMaxShuffle.class, this, VLENGTH,
(s) -> ((IntMaxShuffle)(((AbstractShuffle<Integer>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public IntMaxShuffle rearrange(VectorShuffle<Integer> shuffle) {

@ -2378,17 +2378,18 @@ public abstract class IntVector extends AbstractVector<Integer> {
*/
@Override
public abstract
IntVector rearrange(VectorShuffle<Integer> m);
IntVector rearrange(VectorShuffle<Integer> shuffle);
/*package-private*/
@ForceInline
final
<S extends VectorShuffle<Integer>>
IntVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
shuffle.checkIndexes();
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, null, int.class, length(),
this, shuffle, null,
this, ws, null,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return v1.lane(ei);
@ -2413,17 +2414,14 @@ public abstract class IntVector extends AbstractVector<Integer> {
M m) {
m.check(masktype, this);
VectorMask<Integer> valid = shuffle.laneIsValid();
if (m.andNot(valid).anyTrue()) {
shuffle.checkIndexes();
throw new AssertionError();
}
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, masktype, int.class, length(),
this, shuffle, m,
this, ws, m,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
}));
}
@ -2536,7 +2534,10 @@ public abstract class IntVector extends AbstractVector<Integer> {
/*package-private*/
@ForceInline
final IntVector selectFromTemplate(IntVector v) {
return v.rearrange(this.toShuffle());
return (IntVector)VectorSupport.selectFromOp(getClass(), null, int.class,
length(), this, v, null,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle()));
}
/**
@ -2548,9 +2549,15 @@ public abstract class IntVector extends AbstractVector<Integer> {
/*package-private*/
@ForceInline
final IntVector selectFromTemplate(IntVector v,
AbstractMask<Integer> m) {
return v.rearrange(this.toShuffle(), m);
final
<M extends VectorMask<Integer>>
IntVector selectFromTemplate(IntVector v,
Class<M> masktype, M m) {
m.check(masktype, this);
return (IntVector)VectorSupport.selectFromOp(getClass(), masktype, int.class,
length(), this, v, m,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle(), _m));
}
/// Ternary operations

@ -493,7 +493,7 @@ final class Long128Vector extends LongVector {
VectorMask<Long> m) {
return (Long128Vector)
super.selectFromTemplate((Long128Vector) v,
(Long128Mask) m); // specialize
Long128Mask.class, (Long128Mask) m); // specialize
}
@ -822,6 +822,13 @@ final class Long128Vector extends LongVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Long128Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Long128Shuffle.class, this, VLENGTH,
(s) -> ((Long128Shuffle)(((AbstractShuffle<Long>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Long128Shuffle rearrange(VectorShuffle<Long> shuffle) {

@ -493,7 +493,7 @@ final class Long256Vector extends LongVector {
VectorMask<Long> m) {
return (Long256Vector)
super.selectFromTemplate((Long256Vector) v,
(Long256Mask) m); // specialize
Long256Mask.class, (Long256Mask) m); // specialize
}
@ -826,6 +826,13 @@ final class Long256Vector extends LongVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Long256Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Long256Shuffle.class, this, VLENGTH,
(s) -> ((Long256Shuffle)(((AbstractShuffle<Long>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Long256Shuffle rearrange(VectorShuffle<Long> shuffle) {

@ -493,7 +493,7 @@ final class Long512Vector extends LongVector {
VectorMask<Long> m) {
return (Long512Vector)
super.selectFromTemplate((Long512Vector) v,
(Long512Mask) m); // specialize
Long512Mask.class, (Long512Mask) m); // specialize
}
@ -834,6 +834,13 @@ final class Long512Vector extends LongVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Long512Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Long512Shuffle.class, this, VLENGTH,
(s) -> ((Long512Shuffle)(((AbstractShuffle<Long>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Long512Shuffle rearrange(VectorShuffle<Long> shuffle) {

@ -493,7 +493,7 @@ final class Long64Vector extends LongVector {
VectorMask<Long> m) {
return (Long64Vector)
super.selectFromTemplate((Long64Vector) v,
(Long64Mask) m); // specialize
Long64Mask.class, (Long64Mask) m); // specialize
}
@ -820,6 +820,13 @@ final class Long64Vector extends LongVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Long64Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Long64Shuffle.class, this, VLENGTH,
(s) -> ((Long64Shuffle)(((AbstractShuffle<Long>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Long64Shuffle rearrange(VectorShuffle<Long> shuffle) {

@ -493,7 +493,7 @@ final class LongMaxVector extends LongVector {
VectorMask<Long> m) {
return (LongMaxVector)
super.selectFromTemplate((LongMaxVector) v,
(LongMaxMask) m); // specialize
LongMaxMask.class, (LongMaxMask) m); // specialize
}
@ -820,6 +820,13 @@ final class LongMaxVector extends LongVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public LongMaxShuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, LongMaxShuffle.class, this, VLENGTH,
(s) -> ((LongMaxShuffle)(((AbstractShuffle<Long>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public LongMaxShuffle rearrange(VectorShuffle<Long> shuffle) {

@ -2244,17 +2244,18 @@ public abstract class LongVector extends AbstractVector<Long> {
*/
@Override
public abstract
LongVector rearrange(VectorShuffle<Long> m);
LongVector rearrange(VectorShuffle<Long> shuffle);
/*package-private*/
@ForceInline
final
<S extends VectorShuffle<Long>>
LongVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
shuffle.checkIndexes();
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, null, long.class, length(),
this, shuffle, null,
this, ws, null,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return v1.lane(ei);
@ -2279,17 +2280,14 @@ public abstract class LongVector extends AbstractVector<Long> {
M m) {
m.check(masktype, this);
VectorMask<Long> valid = shuffle.laneIsValid();
if (m.andNot(valid).anyTrue()) {
shuffle.checkIndexes();
throw new AssertionError();
}
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, masktype, long.class, length(),
this, shuffle, m,
this, ws, m,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
}));
}
@ -2402,7 +2400,10 @@ public abstract class LongVector extends AbstractVector<Long> {
/*package-private*/
@ForceInline
final LongVector selectFromTemplate(LongVector v) {
return v.rearrange(this.toShuffle());
return (LongVector)VectorSupport.selectFromOp(getClass(), null, long.class,
length(), this, v, null,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle()));
}
/**
@ -2414,9 +2415,15 @@ public abstract class LongVector extends AbstractVector<Long> {
/*package-private*/
@ForceInline
final LongVector selectFromTemplate(LongVector v,
AbstractMask<Long> m) {
return v.rearrange(this.toShuffle(), m);
final
<M extends VectorMask<Long>>
LongVector selectFromTemplate(LongVector v,
Class<M> masktype, M m) {
m.check(masktype, this);
return (LongVector)VectorSupport.selectFromOp(getClass(), masktype, long.class,
length(), this, v, m,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle(), _m));
}
/// Ternary operations

@ -503,7 +503,7 @@ final class Short128Vector extends ShortVector {
VectorMask<Short> m) {
return (Short128Vector)
super.selectFromTemplate((Short128Vector) v,
(Short128Mask) m); // specialize
Short128Mask.class, (Short128Mask) m); // specialize
}
@ -844,6 +844,13 @@ final class Short128Vector extends ShortVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Short128Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Short128Shuffle.class, this, VLENGTH,
(s) -> ((Short128Shuffle)(((AbstractShuffle<Short>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Short128Shuffle rearrange(VectorShuffle<Short> shuffle) {

@ -503,7 +503,7 @@ final class Short256Vector extends ShortVector {
VectorMask<Short> m) {
return (Short256Vector)
super.selectFromTemplate((Short256Vector) v,
(Short256Mask) m); // specialize
Short256Mask.class, (Short256Mask) m); // specialize
}
@ -860,6 +860,13 @@ final class Short256Vector extends ShortVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Short256Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Short256Shuffle.class, this, VLENGTH,
(s) -> ((Short256Shuffle)(((AbstractShuffle<Short>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Short256Shuffle rearrange(VectorShuffle<Short> shuffle) {

@ -503,7 +503,7 @@ final class Short512Vector extends ShortVector {
VectorMask<Short> m) {
return (Short512Vector)
super.selectFromTemplate((Short512Vector) v,
(Short512Mask) m); // specialize
Short512Mask.class, (Short512Mask) m); // specialize
}
@ -892,6 +892,13 @@ final class Short512Vector extends ShortVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Short512Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Short512Shuffle.class, this, VLENGTH,
(s) -> ((Short512Shuffle)(((AbstractShuffle<Short>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Short512Shuffle rearrange(VectorShuffle<Short> shuffle) {

@ -503,7 +503,7 @@ final class Short64Vector extends ShortVector {
VectorMask<Short> m) {
return (Short64Vector)
super.selectFromTemplate((Short64Vector) v,
(Short64Mask) m); // specialize
Short64Mask.class, (Short64Mask) m); // specialize
}
@ -836,6 +836,13 @@ final class Short64Vector extends ShortVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public Short64Shuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, Short64Shuffle.class, this, VLENGTH,
(s) -> ((Short64Shuffle)(((AbstractShuffle<Short>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public Short64Shuffle rearrange(VectorShuffle<Short> shuffle) {

@ -503,7 +503,7 @@ final class ShortMaxVector extends ShortVector {
VectorMask<Short> m) {
return (ShortMaxVector)
super.selectFromTemplate((ShortMaxVector) v,
(ShortMaxMask) m); // specialize
ShortMaxMask.class, (ShortMaxMask) m); // specialize
}
@ -830,6 +830,13 @@ final class ShortMaxVector extends ShortVector {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public ShortMaxShuffle wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, ShortMaxShuffle.class, this, VLENGTH,
(s) -> ((ShortMaxShuffle)(((AbstractShuffle<Short>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public ShortMaxShuffle rearrange(VectorShuffle<Short> shuffle) {

@ -2394,17 +2394,18 @@ public abstract class ShortVector extends AbstractVector<Short> {
*/
@Override
public abstract
ShortVector rearrange(VectorShuffle<Short> m);
ShortVector rearrange(VectorShuffle<Short> shuffle);
/*package-private*/
@ForceInline
final
<S extends VectorShuffle<Short>>
ShortVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
shuffle.checkIndexes();
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, null, short.class, length(),
this, shuffle, null,
this, ws, null,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return v1.lane(ei);
@ -2429,17 +2430,14 @@ public abstract class ShortVector extends AbstractVector<Short> {
M m) {
m.check(masktype, this);
VectorMask<Short> valid = shuffle.laneIsValid();
if (m.andNot(valid).anyTrue()) {
shuffle.checkIndexes();
throw new AssertionError();
}
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, masktype, short.class, length(),
this, shuffle, m,
this, ws, m,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
}));
}
@ -2552,7 +2550,10 @@ public abstract class ShortVector extends AbstractVector<Short> {
/*package-private*/
@ForceInline
final ShortVector selectFromTemplate(ShortVector v) {
return v.rearrange(this.toShuffle());
return (ShortVector)VectorSupport.selectFromOp(getClass(), null, short.class,
length(), this, v, null,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle()));
}
/**
@ -2564,9 +2565,15 @@ public abstract class ShortVector extends AbstractVector<Short> {
/*package-private*/
@ForceInline
final ShortVector selectFromTemplate(ShortVector v,
AbstractMask<Short> m) {
return v.rearrange(this.toShuffle(), m);
final
<M extends VectorMask<Short>>
ShortVector selectFromTemplate(ShortVector v,
Class<M> masktype, M m) {
m.check(masktype, this);
return (ShortVector)VectorSupport.selectFromOp(getClass(), masktype, short.class,
length(), this, v, m,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle(), _m));
}
/// Ternary operations

@ -2614,17 +2614,14 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
* elements of this vector.
*
* For each lane {@code N} of the shuffle, and for each lane
* source index {@code I=s.laneSource(N)} in the shuffle,
* source index {@code I=s.wrapIndex(s.laneSource(N))} in the shuffle,
* the output lane {@code N} obtains the value from
* the input vector at lane {@code I}.
*
* @param s the shuffle controlling lane index selection
* @return the rearrangement of the lane elements of this vector
* @throws IndexOutOfBoundsException if there are any exceptional
* source indexes in the shuffle
* @see #rearrange(VectorShuffle,VectorMask)
* @see #rearrange(VectorShuffle,Vector)
* @see VectorShuffle#laneIsValid()
*/
public abstract Vector<E> rearrange(VectorShuffle<E> s);
@ -2636,27 +2633,22 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
* elements of this vector.
*
* For each lane {@code N} of the shuffle, and for each lane
* source index {@code I=s.laneSource(N)} in the shuffle,
* source index {@code I=s.wrapIndex(s.laneSource(N))} in the shuffle,
* the output lane {@code N} obtains the value from
* the input vector at lane {@code I} if the mask is set.
* Otherwise the output lane {@code N} is set to zero.
*
* <p> This method returns the value of this pseudocode:
* <pre>{@code
* Vector<E> r = this.rearrange(s.wrapIndexes());
* VectorMask<E> valid = s.laneIsValid();
* if (m.andNot(valid).anyTrue()) throw ...;
* Vector<E> r = this.rearrange(s);
* return broadcast(0).blend(r, m);
* }</pre>
*
* @param s the shuffle controlling lane index selection
* @param m the mask controlling application of the shuffle
* @return the rearrangement of the lane elements of this vector
* @throws IndexOutOfBoundsException if there are any exceptional
* source indexes in the shuffle where the mask is set
* @see #rearrange(VectorShuffle)
* @see #rearrange(VectorShuffle,Vector)
* @see VectorShuffle#laneIsValid()
*/
public abstract Vector<E> rearrange(VectorShuffle<E> s, VectorMask<E> m);
@ -2747,7 +2739,7 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
* this vector.
*
* For each lane {@code N} of this vector, and for each lane
* value {@code I=this.lane(N)} in this vector,
* value {@code I=wrapIndex(this.lane(N))} in this vector,
* the output lane {@code N} obtains the value from
* the argument vector at lane {@code I}.
*
@ -2760,8 +2752,6 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
*
* @param v the vector supplying the result values
* @return the rearrangement of the lane elements of {@code v}
* @throws IndexOutOfBoundsException if any invalid
* source indexes are found in {@code this}
* @see #rearrange(VectorShuffle)
*/
public abstract Vector<E> selectFrom(Vector<E> v);
@ -2787,9 +2777,6 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
* @param v the vector supplying the result values
* @param m the mask controlling selection from {@code v}
* @return the rearrangement of the lane elements of {@code v}
* @throws IndexOutOfBoundsException if any invalid
* source indexes are found in {@code this},
* in a lane which is set in the mask
* @see #selectFrom(Vector)
* @see #rearrange(VectorShuffle,VectorMask)
*/

@ -2770,17 +2770,18 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
*/
@Override
public abstract
$abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> m);
$abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> shuffle);
/*package-private*/
@ForceInline
final
<S extends VectorShuffle<$Boxtype$>>
$abstractvectortype$ rearrangeTemplate(Class<S> shuffletype, S shuffle) {
shuffle.checkIndexes();
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, null, $type$.class, length(),
this, shuffle, null,
this, ws, null,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return v1.lane(ei);
@ -2805,17 +2806,14 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
M m) {
m.check(masktype, this);
VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
if (m.andNot(valid).anyTrue()) {
shuffle.checkIndexes();
throw new AssertionError();
}
@SuppressWarnings("unchecked")
S ws = (S) shuffle.wrapIndexes();
return VectorSupport.rearrangeOp(
getClass(), shuffletype, masktype, $type$.class, length(),
this, shuffle, m,
this, ws, m,
(v1, s_, m_) -> v1.uOp((i, a) -> {
int ei = s_.laneSource(i);
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
}));
}
@ -2928,7 +2926,10 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
/*package-private*/
@ForceInline
final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v) {
return v.rearrange(this.toShuffle());
return ($Type$Vector)VectorSupport.selectFromOp(getClass(), null, $type$.class,
length(), this, v, null,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle()));
}
/**
@ -2940,9 +2941,15 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
/*package-private*/
@ForceInline
final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v,
AbstractMask<$Boxtype$> m) {
return v.rearrange(this.toShuffle(), m);
final
<M extends VectorMask<$Boxtype$>>
$abstractvectortype$ selectFromTemplate($abstractvectortype$ v,
Class<M> masktype, M m) {
m.check(masktype, this);
return ($Type$Vector)VectorSupport.selectFromOp(getClass(), masktype, $type$.class,
length(), this, v, m,
(v1, v2, _m) ->
v2.rearrange(v1.toShuffle(), _m));
}
/// Ternary operations

@ -509,7 +509,7 @@ final class $vectortype$ extends $abstractvectortype$ {
VectorMask<$Boxtype$> m) {
return ($vectortype$)
super.selectFromTemplate(($vectortype$) v,
($masktype$) m); // specialize
$masktype$.class, ($masktype$) m); // specialize
}
@ -1118,6 +1118,13 @@ final class $vectortype$ extends $abstractvectortype$ {
return s.shuffleFromArray(shuffleArray, 0).check(s);
}
@Override
@ForceInline
public $shuffletype$ wrapIndexes() {
return VectorSupport.wrapShuffleIndexes(ETYPE, $shuffletype$.class, this, VLENGTH,
(s) -> (($shuffletype$)(((AbstractShuffle<$Boxtype$>)(s)).wrapIndexesTemplate())));
}
@ForceInline
@Override
public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) {