8340079: Modify rearrange/selectFrom Vector API methods to perform wrapIndexes instead of checkIndexes
Reviewed-by: jbhateja, psandoz
This commit is contained in:
parent
d2e77089aa
commit
83dcb02d77
src
hotspot/share
adlc
classfile
opto
java.base/share/classes/jdk/internal/vm/vector
jdk.incubator.vector/share/classes/jdk/incubator/vector
AbstractShuffle.javaByte128Vector.javaByte256Vector.javaByte512Vector.javaByte64Vector.javaByteMaxVector.javaByteVector.javaDouble128Vector.javaDouble256Vector.javaDouble512Vector.javaDouble64Vector.javaDoubleMaxVector.javaDoubleVector.javaFloat128Vector.javaFloat256Vector.javaFloat512Vector.javaFloat64Vector.javaFloatMaxVector.javaFloatVector.javaInt128Vector.javaInt256Vector.javaInt512Vector.javaInt64Vector.javaIntMaxVector.javaIntVector.javaLong128Vector.javaLong256Vector.javaLong512Vector.javaLong64Vector.javaLongMaxVector.javaLongVector.javaShort128Vector.javaShort256Vector.javaShort512Vector.javaShort64Vector.javaShortMaxVector.javaShortVector.javaVector.javaX-Vector.java.templateX-VectorBits.java.template
@ -4357,7 +4357,7 @@ bool MatchRule::is_vector() const {
|
||||
"RoundDoubleModeV","RotateLeftV" , "RotateRightV", "LoadVector","StoreVector",
|
||||
"LoadVectorGather", "StoreVectorScatter", "LoadVectorGatherMasked", "StoreVectorScatterMasked",
|
||||
"VectorTest", "VectorLoadMask", "VectorStoreMask", "VectorBlend", "VectorInsert",
|
||||
"VectorRearrange","VectorLoadShuffle", "VectorLoadConst",
|
||||
"VectorRearrange", "VectorLoadShuffle", "VectorLoadConst",
|
||||
"VectorCastB2X", "VectorCastS2X", "VectorCastI2X",
|
||||
"VectorCastL2X", "VectorCastF2X", "VectorCastD2X", "VectorCastF2HF", "VectorCastHF2F",
|
||||
"VectorUCastB2X", "VectorUCastS2X", "VectorUCastI2X",
|
||||
|
@ -1008,6 +1008,15 @@ class methodHandle;
|
||||
"Ljdk/internal/vm/vector/VectorSupport$Vector;") \
|
||||
do_name(vector_shuffle_to_vector_name, "shuffleToVector") \
|
||||
\
|
||||
do_intrinsic(_VectorWrapShuffleIndexes, jdk_internal_vm_vector_VectorSupport, vector_wrap_shuffle_indexes_name, \
|
||||
vector_wrap_shuffle_indexes_sig, F_S) \
|
||||
do_signature(vector_wrap_shuffle_indexes_sig, "(Ljava/lang/Class;" \
|
||||
"Ljava/lang/Class;" \
|
||||
"Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;" \
|
||||
"ILjdk/internal/vm/vector/VectorSupport$WrapShuffleIndexesOperation;)" \
|
||||
"Ljdk/internal/vm/vector/VectorSupport$VectorShuffle;") \
|
||||
do_name(vector_wrap_shuffle_indexes_name, "wrapShuffleIndexes") \
|
||||
\
|
||||
do_intrinsic(_VectorLoadOp, jdk_internal_vm_vector_VectorSupport, vector_load_op_name, vector_load_op_sig, F_S) \
|
||||
do_signature(vector_load_op_sig, "(Ljava/lang/Class;" \
|
||||
"Ljava/lang/Class;" \
|
||||
@ -1129,6 +1138,18 @@ class methodHandle;
|
||||
"Ljdk/internal/vm/vector/VectorSupport$Vector;") \
|
||||
do_name(vector_rearrange_name, "rearrangeOp") \
|
||||
\
|
||||
do_intrinsic(_VectorSelectFrom, jdk_internal_vm_vector_VectorSupport, vector_select_from_name, vector_select_from_sig, F_S) \
|
||||
do_signature(vector_select_from_sig, "(Ljava/lang/Class;" \
|
||||
"Ljava/lang/Class;" \
|
||||
"Ljava/lang/Class;" \
|
||||
"I" \
|
||||
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
|
||||
"Ljdk/internal/vm/vector/VectorSupport$Vector;" \
|
||||
"Ljdk/internal/vm/vector/VectorSupport$VectorMask;" \
|
||||
"Ljdk/internal/vm/vector/VectorSupport$VectorSelectFromOp;)" \
|
||||
"Ljdk/internal/vm/vector/VectorSupport$Vector;") \
|
||||
do_name(vector_select_from_name, "selectFromOp") \
|
||||
\
|
||||
do_intrinsic(_VectorExtract, jdk_internal_vm_vector_VectorSupport, vector_extract_name, vector_extract_sig, F_S) \
|
||||
do_signature(vector_extract_sig, "(Ljava/lang/Class;" \
|
||||
"Ljava/lang/Class;" \
|
||||
|
@ -811,6 +811,7 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
|
||||
case vmIntrinsics::_VectorFromBitsCoerced:
|
||||
case vmIntrinsics::_VectorShuffleIota:
|
||||
case vmIntrinsics::_VectorShuffleToVector:
|
||||
case vmIntrinsics::_VectorWrapShuffleIndexes:
|
||||
case vmIntrinsics::_VectorLoadOp:
|
||||
case vmIntrinsics::_VectorLoadMaskedOp:
|
||||
case vmIntrinsics::_VectorStoreOp:
|
||||
@ -821,6 +822,7 @@ bool C2Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
|
||||
case vmIntrinsics::_VectorTest:
|
||||
case vmIntrinsics::_VectorBlend:
|
||||
case vmIntrinsics::_VectorRearrange:
|
||||
case vmIntrinsics::_VectorSelectFrom:
|
||||
case vmIntrinsics::_VectorCompare:
|
||||
case vmIntrinsics::_VectorBroadcastInt:
|
||||
case vmIntrinsics::_VectorConvert:
|
||||
|
@ -717,6 +717,8 @@ bool LibraryCallKit::try_to_inline(int predicate) {
|
||||
return inline_vector_mask_operation();
|
||||
case vmIntrinsics::_VectorShuffleToVector:
|
||||
return inline_vector_shuffle_to_vector();
|
||||
case vmIntrinsics::_VectorWrapShuffleIndexes:
|
||||
return inline_vector_wrap_shuffle_indexes();
|
||||
case vmIntrinsics::_VectorLoadOp:
|
||||
return inline_vector_mem_operation(/*is_store=*/false);
|
||||
case vmIntrinsics::_VectorLoadMaskedOp:
|
||||
@ -737,6 +739,8 @@ bool LibraryCallKit::try_to_inline(int predicate) {
|
||||
return inline_vector_blend();
|
||||
case vmIntrinsics::_VectorRearrange:
|
||||
return inline_vector_rearrange();
|
||||
case vmIntrinsics::_VectorSelectFrom:
|
||||
return inline_vector_select_from();
|
||||
case vmIntrinsics::_VectorCompare:
|
||||
return inline_vector_compare();
|
||||
case vmIntrinsics::_VectorBroadcastInt:
|
||||
|
@ -353,6 +353,7 @@ class LibraryCallKit : public GraphKit {
|
||||
bool inline_vector_nary_operation(int n);
|
||||
bool inline_vector_frombits_coerced();
|
||||
bool inline_vector_shuffle_to_vector();
|
||||
bool inline_vector_wrap_shuffle_indexes();
|
||||
bool inline_vector_shuffle_iota();
|
||||
Node* partially_wrap_indexes(Node* index_vec, int num_elem, BasicType type_bt);
|
||||
bool inline_vector_mask_operation();
|
||||
@ -363,6 +364,7 @@ class LibraryCallKit : public GraphKit {
|
||||
bool inline_vector_test();
|
||||
bool inline_vector_blend();
|
||||
bool inline_vector_rearrange();
|
||||
bool inline_vector_select_from();
|
||||
bool inline_vector_compare();
|
||||
bool inline_vector_broadcast_int();
|
||||
bool inline_vector_convert();
|
||||
|
@ -757,6 +757,64 @@ bool LibraryCallKit::inline_vector_shuffle_to_vector() {
|
||||
return true;
|
||||
}
|
||||
|
||||
// public static
|
||||
// <E,
|
||||
// SH extends VectorShuffle<E>>
|
||||
// SH wrapShuffleIndexes(Class<E> eClass, Class<? extends SH> shClass, SH sh, int length,
|
||||
// ShuffleWrapIndexesOperation<SH> defaultImpl)
|
||||
bool LibraryCallKit::inline_vector_wrap_shuffle_indexes() {
|
||||
const TypeInstPtr* elem_klass = gvn().type(argument(0))->isa_instptr();
|
||||
const TypeInstPtr* shuffle_klass = gvn().type(argument(1))->isa_instptr();
|
||||
Node* shuffle = argument(2);
|
||||
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
|
||||
|
||||
if (elem_klass == nullptr || shuffle_klass == nullptr || shuffle->is_top() || vlen == nullptr ||
|
||||
!vlen->is_con() || shuffle_klass->const_oop() == nullptr) {
|
||||
// not enough info for intrinsification
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!is_klass_initialized(shuffle_klass)) {
|
||||
log_if_needed(" ** klass argument not initialized");
|
||||
return false;
|
||||
}
|
||||
|
||||
int num_elem = vlen->get_con();
|
||||
if ((num_elem < 4) || !is_power_of_2(num_elem)) {
|
||||
log_if_needed(" ** vlen < 4 or not power of two=%d", num_elem);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Shuffles use byte array based backing storage
|
||||
BasicType shuffle_bt = T_BYTE;
|
||||
if (!arch_supports_vector(Op_AndV, num_elem, shuffle_bt, VecMaskNotUsed) ||
|
||||
!arch_supports_vector(Op_Replicate, num_elem, shuffle_bt, VecMaskNotUsed)) {
|
||||
log_if_needed(" ** not supported: op=wrapShuffleIndexes vlen=%d etype=%s",
|
||||
num_elem, type2name(shuffle_bt));
|
||||
return false;
|
||||
}
|
||||
|
||||
ciKlass* sbox_klass = shuffle_klass->const_oop()->as_instance()->java_lang_Class_klass();
|
||||
const TypeInstPtr* shuffle_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, sbox_klass);
|
||||
|
||||
// Unbox shuffle with true flag to indicate its load shuffle to vector
|
||||
// shuffle is a byte array
|
||||
Node* shuffle_vec = unbox_vector(shuffle, shuffle_box_type, shuffle_bt, num_elem, true);
|
||||
|
||||
const TypeVect* vt = TypeVect::make(shuffle_bt, num_elem);
|
||||
const Type* shuffle_type_bt = Type::get_const_basic_type(shuffle_bt);
|
||||
Node* mod_mask = gvn().makecon(TypeInt::make(num_elem-1));
|
||||
Node* bcast_mod_mask = gvn().transform(VectorNode::scalar2vector(mod_mask, num_elem, shuffle_type_bt));
|
||||
// Wrap the indices greater than lane count.
|
||||
Node* res = gvn().transform(VectorNode::make(Op_AndV, shuffle_vec, bcast_mod_mask, vt));
|
||||
|
||||
// Wrap it up in VectorBox to keep object type information.
|
||||
res = box_vector(res, shuffle_box_type, shuffle_bt, num_elem);
|
||||
set_result(res);
|
||||
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(shuffle_bt))));
|
||||
return true;
|
||||
}
|
||||
|
||||
// public static
|
||||
// <M,
|
||||
// S extends VectorSpecies<E>,
|
||||
@ -2044,6 +2102,150 @@ static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr,
|
||||
return addr;
|
||||
}
|
||||
|
||||
// public static
|
||||
// <V extends Vector<E>,
|
||||
// M extends VectorMask<E>,
|
||||
// E>
|
||||
// V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
|
||||
// int length, V v1, V v2, M m,
|
||||
// VectorSelectFromOp<V, M> defaultImpl)
|
||||
bool LibraryCallKit::inline_vector_select_from() {
|
||||
const TypeInstPtr* vector_klass = gvn().type(argument(0))->isa_instptr();
|
||||
const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr();
|
||||
const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr();
|
||||
const TypeInt* vlen = gvn().type(argument(3))->isa_int();
|
||||
|
||||
if (vector_klass == nullptr || elem_klass == nullptr || vlen == nullptr ||
|
||||
vector_klass->const_oop() == nullptr ||
|
||||
elem_klass->const_oop() == nullptr ||
|
||||
!vlen->is_con()) {
|
||||
log_if_needed(" ** missing constant: vclass=%s etype=%s vlen=%s",
|
||||
NodeClassNames[argument(0)->Opcode()],
|
||||
NodeClassNames[argument(2)->Opcode()],
|
||||
NodeClassNames[argument(3)->Opcode()]);
|
||||
return false; // not enough info for intrinsification
|
||||
}
|
||||
if (!is_klass_initialized(vector_klass)) {
|
||||
log_if_needed(" ** klass argument not initialized");
|
||||
return false;
|
||||
}
|
||||
ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
|
||||
if (!elem_type->is_primitive_type()) {
|
||||
log_if_needed(" ** not a primitive bt=%d", elem_type->basic_type());
|
||||
return false; // should be primitive type
|
||||
}
|
||||
BasicType elem_bt = elem_type->basic_type();
|
||||
int num_elem = vlen->get_con();
|
||||
if (!is_power_of_2(num_elem)) {
|
||||
log_if_needed(" ** vlen not power of two=%d", num_elem);
|
||||
return false;
|
||||
}
|
||||
|
||||
int cast_vopc = VectorCastNode::opcode(-1, elem_bt); // from vector of type elem_bt
|
||||
if (!arch_supports_vector(Op_VectorLoadShuffle, num_elem, elem_bt, VecMaskNotUsed)||
|
||||
!arch_supports_vector(Op_AndV, num_elem, T_BYTE, VecMaskNotUsed) ||
|
||||
!arch_supports_vector(Op_Replicate, num_elem, T_BYTE, VecMaskNotUsed) ||
|
||||
!arch_supports_vector(cast_vopc, num_elem, T_BYTE, VecMaskNotUsed)) {
|
||||
log_if_needed(" ** not supported: arity=0 op=selectFrom vlen=%d etype=%s ismask=no",
|
||||
num_elem, type2name(elem_bt));
|
||||
return false; // not supported
|
||||
}
|
||||
|
||||
bool is_masked_op = argument(6)->bottom_type() != TypePtr::NULL_PTR;
|
||||
bool use_predicate = is_masked_op;
|
||||
if (is_masked_op &&
|
||||
(mask_klass == nullptr ||
|
||||
mask_klass->const_oop() == nullptr ||
|
||||
!is_klass_initialized(mask_klass))) {
|
||||
log_if_needed(" ** mask_klass argument not initialized");
|
||||
return false; // not supported
|
||||
}
|
||||
VectorMaskUseType checkFlags = (VectorMaskUseType)(is_masked_op ? (VecMaskUseLoad | VecMaskUsePred) : VecMaskNotUsed);
|
||||
if (!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, checkFlags)) {
|
||||
use_predicate = false;
|
||||
if(!is_masked_op ||
|
||||
(!arch_supports_vector(Op_VectorRearrange, num_elem, elem_bt, VecMaskNotUsed) ||
|
||||
!arch_supports_vector(Op_VectorBlend, num_elem, elem_bt, VecMaskUseLoad) ||
|
||||
!arch_supports_vector(Op_Replicate, num_elem, elem_bt, VecMaskNotUsed))) {
|
||||
log_if_needed(" ** not supported: op=selectFrom vlen=%d etype=%s is_masked_op=%d",
|
||||
num_elem, type2name(elem_bt), is_masked_op);
|
||||
return false; // not supported
|
||||
}
|
||||
}
|
||||
ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
|
||||
const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
|
||||
|
||||
// v1 is the index vector
|
||||
Node* v1 = unbox_vector(argument(4), vbox_type, elem_bt, num_elem);
|
||||
// v2 is the vector being rearranged
|
||||
Node* v2 = unbox_vector(argument(5), vbox_type, elem_bt, num_elem);
|
||||
|
||||
if (v1 == nullptr) {
|
||||
log_if_needed(" ** unbox failed v1=%s", NodeClassNames[argument(4)->Opcode()]);
|
||||
return false; // operand unboxing failed
|
||||
}
|
||||
|
||||
if (v2 == nullptr) {
|
||||
log_if_needed(" ** unbox failed v2=%s", NodeClassNames[argument(5)->Opcode()]);
|
||||
return false; // operand unboxing failed
|
||||
}
|
||||
|
||||
Node* mask = nullptr;
|
||||
if (is_masked_op) {
|
||||
ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass();
|
||||
const TypeInstPtr* mbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass);
|
||||
mask = unbox_vector(argument(6), mbox_type, elem_bt, num_elem);
|
||||
if (mask == nullptr) {
|
||||
log_if_needed(" ** unbox failed mask=%s", NodeClassNames[argument(6)->Opcode()]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// cast index vector from elem_bt vector to byte vector
|
||||
const Type * byte_bt = Type::get_const_basic_type(T_BYTE);
|
||||
const TypeVect * byte_vt = TypeVect::make(byte_bt, num_elem);
|
||||
Node* byte_shuffle = gvn().transform(VectorCastNode::make(cast_vopc, v1, T_BYTE, num_elem));
|
||||
|
||||
// wrap the byte vector lanes to (num_elem - 1) to form the shuffle vector where num_elem is vector length
|
||||
// this is a simple AND operation as we come here only for power of two vector length
|
||||
Node* mod_val = gvn().makecon(TypeInt::make(num_elem-1));
|
||||
Node* bcast_mod = gvn().transform(VectorNode::scalar2vector(mod_val, num_elem, byte_bt));
|
||||
byte_shuffle = gvn().transform(VectorNode::make(Op_AndV, byte_shuffle, bcast_mod, byte_vt));
|
||||
|
||||
// load the shuffle to use in rearrange
|
||||
const TypeVect * shuffle_vt = TypeVect::make(elem_bt, num_elem);
|
||||
Node* load_shuffle = gvn().transform(new VectorLoadShuffleNode(byte_shuffle, shuffle_vt));
|
||||
|
||||
// and finally rearrange
|
||||
Node* rearrange = new VectorRearrangeNode(v2, load_shuffle);
|
||||
if (is_masked_op) {
|
||||
if (use_predicate) {
|
||||
// masked rearrange is supported so use that directly
|
||||
rearrange->add_req(mask);
|
||||
rearrange->add_flag(Node::Flag_is_predicated_vector);
|
||||
} else {
|
||||
// masked rearrange is not supported so emulate usig blend
|
||||
const TypeVect* vt = v1->bottom_type()->is_vect();
|
||||
rearrange = gvn().transform(rearrange);
|
||||
|
||||
// create a zero vector with each lane element set as zero
|
||||
Node* zero = gvn().makecon(Type::get_zero_type(elem_bt));
|
||||
Node* zerovec = gvn().transform(VectorNode::scalar2vector(zero, num_elem, Type::get_const_basic_type(elem_bt)));
|
||||
|
||||
// For each lane for which mask is set, blend in the rearranged lane into zero vector
|
||||
rearrange = new VectorBlendNode(zerovec, rearrange, mask);
|
||||
}
|
||||
}
|
||||
rearrange = gvn().transform(rearrange);
|
||||
|
||||
// box the result
|
||||
Node* box = box_vector(rearrange, vbox_type, elem_bt, num_elem);
|
||||
set_result(box);
|
||||
|
||||
C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt))));
|
||||
return true;
|
||||
}
|
||||
|
||||
Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
|
||||
assert(UseVectorStubs, "sanity");
|
||||
assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
|
||||
|
@ -263,6 +263,20 @@ public class VectorSupport {
|
||||
return defaultImpl.apply(sh);
|
||||
}
|
||||
|
||||
public interface WrapShuffleIndexesOperation<SH extends VectorShuffle<?>> {
|
||||
SH apply(SH sh);
|
||||
}
|
||||
|
||||
@IntrinsicCandidate
|
||||
public static
|
||||
<E,
|
||||
SH extends VectorShuffle<E>>
|
||||
SH wrapShuffleIndexes(Class<E> eClass, Class<? extends SH> shClass, SH sh, int length,
|
||||
WrapShuffleIndexesOperation<SH> defaultImpl) {
|
||||
assert isNonCapturingLambda(defaultImpl) : defaultImpl;
|
||||
return defaultImpl.apply(sh);
|
||||
}
|
||||
|
||||
/* ============================================================================ */
|
||||
public interface IndexOperation<V extends Vector<?>,
|
||||
S extends VectorSpecies<?>> {
|
||||
@ -605,6 +619,23 @@ public class VectorSupport {
|
||||
return defaultImpl.apply(v, sh, m);
|
||||
}
|
||||
|
||||
public interface VectorSelectFromOp<V extends Vector<?>,
|
||||
M extends VectorMask<?>> {
|
||||
V apply(V v1, V v2, M m);
|
||||
}
|
||||
|
||||
@IntrinsicCandidate
|
||||
public static
|
||||
<V extends Vector<E>,
|
||||
M extends VectorMask<E>,
|
||||
E>
|
||||
V selectFromOp(Class<? extends V> vClass, Class<M> mClass, Class<E> eClass,
|
||||
int length, V v1, V v2, M m,
|
||||
VectorSelectFromOp<V, M> defaultImpl) {
|
||||
assert isNonCapturingLambda(defaultImpl) : defaultImpl;
|
||||
return defaultImpl.apply(v1, v2, m);
|
||||
}
|
||||
|
||||
/* ============================================================================ */
|
||||
|
||||
public interface VectorBlendOp<V extends Vector<?>,
|
||||
|
@ -133,7 +133,7 @@ abstract class AbstractShuffle<E> extends VectorShuffle<E> {
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
public final VectorShuffle<E> wrapIndexes() {
|
||||
public final VectorShuffle<E> wrapIndexesTemplate() {
|
||||
Vector<E> shufvec = this.toVector();
|
||||
VectorMask<E> vecmask = shufvec.compare(VectorOperators.LT, vspecies().zero());
|
||||
if (vecmask.anyTrue()) {
|
||||
|
@ -503,7 +503,7 @@ final class Byte128Vector extends ByteVector {
|
||||
VectorMask<Byte> m) {
|
||||
return (Byte128Vector)
|
||||
super.selectFromTemplate((Byte128Vector) v,
|
||||
(Byte128Mask) m); // specialize
|
||||
Byte128Mask.class, (Byte128Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -860,6 +860,13 @@ final class Byte128Vector extends ByteVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Byte128Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Byte128Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Byte128Shuffle)(((AbstractShuffle<Byte>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Byte128Shuffle rearrange(VectorShuffle<Byte> shuffle) {
|
||||
|
@ -503,7 +503,7 @@ final class Byte256Vector extends ByteVector {
|
||||
VectorMask<Byte> m) {
|
||||
return (Byte256Vector)
|
||||
super.selectFromTemplate((Byte256Vector) v,
|
||||
(Byte256Mask) m); // specialize
|
||||
Byte256Mask.class, (Byte256Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -892,6 +892,13 @@ final class Byte256Vector extends ByteVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Byte256Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Byte256Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Byte256Shuffle)(((AbstractShuffle<Byte>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Byte256Shuffle rearrange(VectorShuffle<Byte> shuffle) {
|
||||
|
@ -503,7 +503,7 @@ final class Byte512Vector extends ByteVector {
|
||||
VectorMask<Byte> m) {
|
||||
return (Byte512Vector)
|
||||
super.selectFromTemplate((Byte512Vector) v,
|
||||
(Byte512Mask) m); // specialize
|
||||
Byte512Mask.class, (Byte512Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -956,6 +956,13 @@ final class Byte512Vector extends ByteVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Byte512Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Byte512Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Byte512Shuffle)(((AbstractShuffle<Byte>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Byte512Shuffle rearrange(VectorShuffle<Byte> shuffle) {
|
||||
|
@ -503,7 +503,7 @@ final class Byte64Vector extends ByteVector {
|
||||
VectorMask<Byte> m) {
|
||||
return (Byte64Vector)
|
||||
super.selectFromTemplate((Byte64Vector) v,
|
||||
(Byte64Mask) m); // specialize
|
||||
Byte64Mask.class, (Byte64Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -844,6 +844,13 @@ final class Byte64Vector extends ByteVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Byte64Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Byte64Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Byte64Shuffle)(((AbstractShuffle<Byte>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Byte64Shuffle rearrange(VectorShuffle<Byte> shuffle) {
|
||||
|
@ -503,7 +503,7 @@ final class ByteMaxVector extends ByteVector {
|
||||
VectorMask<Byte> m) {
|
||||
return (ByteMaxVector)
|
||||
super.selectFromTemplate((ByteMaxVector) v,
|
||||
(ByteMaxMask) m); // specialize
|
||||
ByteMaxMask.class, (ByteMaxMask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -830,6 +830,13 @@ final class ByteMaxVector extends ByteVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public ByteMaxShuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, ByteMaxShuffle.class, this, VLENGTH,
|
||||
(s) -> ((ByteMaxShuffle)(((AbstractShuffle<Byte>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public ByteMaxShuffle rearrange(VectorShuffle<Byte> shuffle) {
|
||||
|
@ -2393,17 +2393,18 @@ public abstract class ByteVector extends AbstractVector<Byte> {
|
||||
*/
|
||||
@Override
|
||||
public abstract
|
||||
ByteVector rearrange(VectorShuffle<Byte> m);
|
||||
ByteVector rearrange(VectorShuffle<Byte> shuffle);
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final
|
||||
<S extends VectorShuffle<Byte>>
|
||||
ByteVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
|
||||
shuffle.checkIndexes();
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, null, byte.class, length(),
|
||||
this, shuffle, null,
|
||||
this, ws, null,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return v1.lane(ei);
|
||||
@ -2428,17 +2429,14 @@ public abstract class ByteVector extends AbstractVector<Byte> {
|
||||
M m) {
|
||||
|
||||
m.check(masktype, this);
|
||||
VectorMask<Byte> valid = shuffle.laneIsValid();
|
||||
if (m.andNot(valid).anyTrue()) {
|
||||
shuffle.checkIndexes();
|
||||
throw new AssertionError();
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, masktype, byte.class, length(),
|
||||
this, shuffle, m,
|
||||
this, ws, m,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
}));
|
||||
}
|
||||
|
||||
@ -2551,7 +2549,10 @@ public abstract class ByteVector extends AbstractVector<Byte> {
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final ByteVector selectFromTemplate(ByteVector v) {
|
||||
return v.rearrange(this.toShuffle());
|
||||
return (ByteVector)VectorSupport.selectFromOp(getClass(), null, byte.class,
|
||||
length(), this, v, null,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle()));
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2563,9 +2564,15 @@ public abstract class ByteVector extends AbstractVector<Byte> {
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final ByteVector selectFromTemplate(ByteVector v,
|
||||
AbstractMask<Byte> m) {
|
||||
return v.rearrange(this.toShuffle(), m);
|
||||
final
|
||||
<M extends VectorMask<Byte>>
|
||||
ByteVector selectFromTemplate(ByteVector v,
|
||||
Class<M> masktype, M m) {
|
||||
m.check(masktype, this);
|
||||
return (ByteVector)VectorSupport.selectFromOp(getClass(), masktype, byte.class,
|
||||
length(), this, v, m,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle(), _m));
|
||||
}
|
||||
|
||||
/// Ternary operations
|
||||
|
@ -490,7 +490,7 @@ final class Double128Vector extends DoubleVector {
|
||||
VectorMask<Double> m) {
|
||||
return (Double128Vector)
|
||||
super.selectFromTemplate((Double128Vector) v,
|
||||
(Double128Mask) m); // specialize
|
||||
Double128Mask.class, (Double128Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -821,6 +821,13 @@ final class Double128Vector extends DoubleVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Double128Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Double128Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Double128Shuffle)(((AbstractShuffle<Double>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Double128Shuffle rearrange(VectorShuffle<Double> shuffle) {
|
||||
|
@ -490,7 +490,7 @@ final class Double256Vector extends DoubleVector {
|
||||
VectorMask<Double> m) {
|
||||
return (Double256Vector)
|
||||
super.selectFromTemplate((Double256Vector) v,
|
||||
(Double256Mask) m); // specialize
|
||||
Double256Mask.class, (Double256Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -825,6 +825,13 @@ final class Double256Vector extends DoubleVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Double256Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Double256Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Double256Shuffle)(((AbstractShuffle<Double>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Double256Shuffle rearrange(VectorShuffle<Double> shuffle) {
|
||||
|
@ -490,7 +490,7 @@ final class Double512Vector extends DoubleVector {
|
||||
VectorMask<Double> m) {
|
||||
return (Double512Vector)
|
||||
super.selectFromTemplate((Double512Vector) v,
|
||||
(Double512Mask) m); // specialize
|
||||
Double512Mask.class, (Double512Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -833,6 +833,13 @@ final class Double512Vector extends DoubleVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Double512Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Double512Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Double512Shuffle)(((AbstractShuffle<Double>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Double512Shuffle rearrange(VectorShuffle<Double> shuffle) {
|
||||
|
@ -490,7 +490,7 @@ final class Double64Vector extends DoubleVector {
|
||||
VectorMask<Double> m) {
|
||||
return (Double64Vector)
|
||||
super.selectFromTemplate((Double64Vector) v,
|
||||
(Double64Mask) m); // specialize
|
||||
Double64Mask.class, (Double64Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -819,6 +819,13 @@ final class Double64Vector extends DoubleVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Double64Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Double64Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Double64Shuffle)(((AbstractShuffle<Double>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Double64Shuffle rearrange(VectorShuffle<Double> shuffle) {
|
||||
|
@ -490,7 +490,7 @@ final class DoubleMaxVector extends DoubleVector {
|
||||
VectorMask<Double> m) {
|
||||
return (DoubleMaxVector)
|
||||
super.selectFromTemplate((DoubleMaxVector) v,
|
||||
(DoubleMaxMask) m); // specialize
|
||||
DoubleMaxMask.class, (DoubleMaxMask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -818,6 +818,13 @@ final class DoubleMaxVector extends DoubleVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public DoubleMaxShuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, DoubleMaxShuffle.class, this, VLENGTH,
|
||||
(s) -> ((DoubleMaxShuffle)(((AbstractShuffle<Double>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public DoubleMaxShuffle rearrange(VectorShuffle<Double> shuffle) {
|
||||
|
@ -2235,17 +2235,18 @@ public abstract class DoubleVector extends AbstractVector<Double> {
|
||||
*/
|
||||
@Override
|
||||
public abstract
|
||||
DoubleVector rearrange(VectorShuffle<Double> m);
|
||||
DoubleVector rearrange(VectorShuffle<Double> shuffle);
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final
|
||||
<S extends VectorShuffle<Double>>
|
||||
DoubleVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
|
||||
shuffle.checkIndexes();
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, null, double.class, length(),
|
||||
this, shuffle, null,
|
||||
this, ws, null,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return v1.lane(ei);
|
||||
@ -2270,17 +2271,14 @@ public abstract class DoubleVector extends AbstractVector<Double> {
|
||||
M m) {
|
||||
|
||||
m.check(masktype, this);
|
||||
VectorMask<Double> valid = shuffle.laneIsValid();
|
||||
if (m.andNot(valid).anyTrue()) {
|
||||
shuffle.checkIndexes();
|
||||
throw new AssertionError();
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, masktype, double.class, length(),
|
||||
this, shuffle, m,
|
||||
this, ws, m,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
}));
|
||||
}
|
||||
|
||||
@ -2393,7 +2391,10 @@ public abstract class DoubleVector extends AbstractVector<Double> {
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final DoubleVector selectFromTemplate(DoubleVector v) {
|
||||
return v.rearrange(this.toShuffle());
|
||||
return (DoubleVector)VectorSupport.selectFromOp(getClass(), null, double.class,
|
||||
length(), this, v, null,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle()));
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2405,9 +2406,15 @@ public abstract class DoubleVector extends AbstractVector<Double> {
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final DoubleVector selectFromTemplate(DoubleVector v,
|
||||
AbstractMask<Double> m) {
|
||||
return v.rearrange(this.toShuffle(), m);
|
||||
final
|
||||
<M extends VectorMask<Double>>
|
||||
DoubleVector selectFromTemplate(DoubleVector v,
|
||||
Class<M> masktype, M m) {
|
||||
m.check(masktype, this);
|
||||
return (DoubleVector)VectorSupport.selectFromOp(getClass(), masktype, double.class,
|
||||
length(), this, v, m,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle(), _m));
|
||||
}
|
||||
|
||||
/// Ternary operations
|
||||
|
@ -490,7 +490,7 @@ final class Float128Vector extends FloatVector {
|
||||
VectorMask<Float> m) {
|
||||
return (Float128Vector)
|
||||
super.selectFromTemplate((Float128Vector) v,
|
||||
(Float128Mask) m); // specialize
|
||||
Float128Mask.class, (Float128Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -825,6 +825,13 @@ final class Float128Vector extends FloatVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Float128Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Float128Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Float128Shuffle)(((AbstractShuffle<Float>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Float128Shuffle rearrange(VectorShuffle<Float> shuffle) {
|
||||
|
@ -490,7 +490,7 @@ final class Float256Vector extends FloatVector {
|
||||
VectorMask<Float> m) {
|
||||
return (Float256Vector)
|
||||
super.selectFromTemplate((Float256Vector) v,
|
||||
(Float256Mask) m); // specialize
|
||||
Float256Mask.class, (Float256Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -833,6 +833,13 @@ final class Float256Vector extends FloatVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Float256Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Float256Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Float256Shuffle)(((AbstractShuffle<Float>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Float256Shuffle rearrange(VectorShuffle<Float> shuffle) {
|
||||
|
@ -490,7 +490,7 @@ final class Float512Vector extends FloatVector {
|
||||
VectorMask<Float> m) {
|
||||
return (Float512Vector)
|
||||
super.selectFromTemplate((Float512Vector) v,
|
||||
(Float512Mask) m); // specialize
|
||||
Float512Mask.class, (Float512Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -849,6 +849,13 @@ final class Float512Vector extends FloatVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Float512Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Float512Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Float512Shuffle)(((AbstractShuffle<Float>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Float512Shuffle rearrange(VectorShuffle<Float> shuffle) {
|
||||
|
@ -490,7 +490,7 @@ final class Float64Vector extends FloatVector {
|
||||
VectorMask<Float> m) {
|
||||
return (Float64Vector)
|
||||
super.selectFromTemplate((Float64Vector) v,
|
||||
(Float64Mask) m); // specialize
|
||||
Float64Mask.class, (Float64Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -821,6 +821,13 @@ final class Float64Vector extends FloatVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Float64Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Float64Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Float64Shuffle)(((AbstractShuffle<Float>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Float64Shuffle rearrange(VectorShuffle<Float> shuffle) {
|
||||
|
@ -490,7 +490,7 @@ final class FloatMaxVector extends FloatVector {
|
||||
VectorMask<Float> m) {
|
||||
return (FloatMaxVector)
|
||||
super.selectFromTemplate((FloatMaxVector) v,
|
||||
(FloatMaxMask) m); // specialize
|
||||
FloatMaxMask.class, (FloatMaxMask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -818,6 +818,13 @@ final class FloatMaxVector extends FloatVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public FloatMaxShuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, FloatMaxShuffle.class, this, VLENGTH,
|
||||
(s) -> ((FloatMaxShuffle)(((AbstractShuffle<Float>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public FloatMaxShuffle rearrange(VectorShuffle<Float> shuffle) {
|
||||
|
@ -2247,17 +2247,18 @@ public abstract class FloatVector extends AbstractVector<Float> {
|
||||
*/
|
||||
@Override
|
||||
public abstract
|
||||
FloatVector rearrange(VectorShuffle<Float> m);
|
||||
FloatVector rearrange(VectorShuffle<Float> shuffle);
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final
|
||||
<S extends VectorShuffle<Float>>
|
||||
FloatVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
|
||||
shuffle.checkIndexes();
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, null, float.class, length(),
|
||||
this, shuffle, null,
|
||||
this, ws, null,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return v1.lane(ei);
|
||||
@ -2282,17 +2283,14 @@ public abstract class FloatVector extends AbstractVector<Float> {
|
||||
M m) {
|
||||
|
||||
m.check(masktype, this);
|
||||
VectorMask<Float> valid = shuffle.laneIsValid();
|
||||
if (m.andNot(valid).anyTrue()) {
|
||||
shuffle.checkIndexes();
|
||||
throw new AssertionError();
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, masktype, float.class, length(),
|
||||
this, shuffle, m,
|
||||
this, ws, m,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
}));
|
||||
}
|
||||
|
||||
@ -2405,7 +2403,10 @@ public abstract class FloatVector extends AbstractVector<Float> {
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final FloatVector selectFromTemplate(FloatVector v) {
|
||||
return v.rearrange(this.toShuffle());
|
||||
return (FloatVector)VectorSupport.selectFromOp(getClass(), null, float.class,
|
||||
length(), this, v, null,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle()));
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2417,9 +2418,15 @@ public abstract class FloatVector extends AbstractVector<Float> {
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final FloatVector selectFromTemplate(FloatVector v,
|
||||
AbstractMask<Float> m) {
|
||||
return v.rearrange(this.toShuffle(), m);
|
||||
final
|
||||
<M extends VectorMask<Float>>
|
||||
FloatVector selectFromTemplate(FloatVector v,
|
||||
Class<M> masktype, M m) {
|
||||
m.check(masktype, this);
|
||||
return (FloatVector)VectorSupport.selectFromOp(getClass(), masktype, float.class,
|
||||
length(), this, v, m,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle(), _m));
|
||||
}
|
||||
|
||||
/// Ternary operations
|
||||
|
@ -503,7 +503,7 @@ final class Int128Vector extends IntVector {
|
||||
VectorMask<Integer> m) {
|
||||
return (Int128Vector)
|
||||
super.selectFromTemplate((Int128Vector) v,
|
||||
(Int128Mask) m); // specialize
|
||||
Int128Mask.class, (Int128Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -836,6 +836,13 @@ final class Int128Vector extends IntVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Int128Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Int128Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Int128Shuffle)(((AbstractShuffle<Integer>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Int128Shuffle rearrange(VectorShuffle<Integer> shuffle) {
|
||||
|
@ -503,7 +503,7 @@ final class Int256Vector extends IntVector {
|
||||
VectorMask<Integer> m) {
|
||||
return (Int256Vector)
|
||||
super.selectFromTemplate((Int256Vector) v,
|
||||
(Int256Mask) m); // specialize
|
||||
Int256Mask.class, (Int256Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -844,6 +844,13 @@ final class Int256Vector extends IntVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Int256Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Int256Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Int256Shuffle)(((AbstractShuffle<Integer>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Int256Shuffle rearrange(VectorShuffle<Integer> shuffle) {
|
||||
|
@ -503,7 +503,7 @@ final class Int512Vector extends IntVector {
|
||||
VectorMask<Integer> m) {
|
||||
return (Int512Vector)
|
||||
super.selectFromTemplate((Int512Vector) v,
|
||||
(Int512Mask) m); // specialize
|
||||
Int512Mask.class, (Int512Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -860,6 +860,13 @@ final class Int512Vector extends IntVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Int512Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Int512Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Int512Shuffle)(((AbstractShuffle<Integer>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Int512Shuffle rearrange(VectorShuffle<Integer> shuffle) {
|
||||
|
@ -503,7 +503,7 @@ final class Int64Vector extends IntVector {
|
||||
VectorMask<Integer> m) {
|
||||
return (Int64Vector)
|
||||
super.selectFromTemplate((Int64Vector) v,
|
||||
(Int64Mask) m); // specialize
|
||||
Int64Mask.class, (Int64Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -832,6 +832,13 @@ final class Int64Vector extends IntVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Int64Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Int64Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Int64Shuffle)(((AbstractShuffle<Integer>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Int64Shuffle rearrange(VectorShuffle<Integer> shuffle) {
|
||||
|
@ -503,7 +503,7 @@ final class IntMaxVector extends IntVector {
|
||||
VectorMask<Integer> m) {
|
||||
return (IntMaxVector)
|
||||
super.selectFromTemplate((IntMaxVector) v,
|
||||
(IntMaxMask) m); // specialize
|
||||
IntMaxMask.class, (IntMaxMask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -841,6 +841,13 @@ final class IntMaxVector extends IntVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public IntMaxShuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, IntMaxShuffle.class, this, VLENGTH,
|
||||
(s) -> ((IntMaxShuffle)(((AbstractShuffle<Integer>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public IntMaxShuffle rearrange(VectorShuffle<Integer> shuffle) {
|
||||
|
@ -2378,17 +2378,18 @@ public abstract class IntVector extends AbstractVector<Integer> {
|
||||
*/
|
||||
@Override
|
||||
public abstract
|
||||
IntVector rearrange(VectorShuffle<Integer> m);
|
||||
IntVector rearrange(VectorShuffle<Integer> shuffle);
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final
|
||||
<S extends VectorShuffle<Integer>>
|
||||
IntVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
|
||||
shuffle.checkIndexes();
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, null, int.class, length(),
|
||||
this, shuffle, null,
|
||||
this, ws, null,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return v1.lane(ei);
|
||||
@ -2413,17 +2414,14 @@ public abstract class IntVector extends AbstractVector<Integer> {
|
||||
M m) {
|
||||
|
||||
m.check(masktype, this);
|
||||
VectorMask<Integer> valid = shuffle.laneIsValid();
|
||||
if (m.andNot(valid).anyTrue()) {
|
||||
shuffle.checkIndexes();
|
||||
throw new AssertionError();
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, masktype, int.class, length(),
|
||||
this, shuffle, m,
|
||||
this, ws, m,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
}));
|
||||
}
|
||||
|
||||
@ -2536,7 +2534,10 @@ public abstract class IntVector extends AbstractVector<Integer> {
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final IntVector selectFromTemplate(IntVector v) {
|
||||
return v.rearrange(this.toShuffle());
|
||||
return (IntVector)VectorSupport.selectFromOp(getClass(), null, int.class,
|
||||
length(), this, v, null,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle()));
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2548,9 +2549,15 @@ public abstract class IntVector extends AbstractVector<Integer> {
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final IntVector selectFromTemplate(IntVector v,
|
||||
AbstractMask<Integer> m) {
|
||||
return v.rearrange(this.toShuffle(), m);
|
||||
final
|
||||
<M extends VectorMask<Integer>>
|
||||
IntVector selectFromTemplate(IntVector v,
|
||||
Class<M> masktype, M m) {
|
||||
m.check(masktype, this);
|
||||
return (IntVector)VectorSupport.selectFromOp(getClass(), masktype, int.class,
|
||||
length(), this, v, m,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle(), _m));
|
||||
}
|
||||
|
||||
/// Ternary operations
|
||||
|
@ -493,7 +493,7 @@ final class Long128Vector extends LongVector {
|
||||
VectorMask<Long> m) {
|
||||
return (Long128Vector)
|
||||
super.selectFromTemplate((Long128Vector) v,
|
||||
(Long128Mask) m); // specialize
|
||||
Long128Mask.class, (Long128Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -822,6 +822,13 @@ final class Long128Vector extends LongVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Long128Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Long128Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Long128Shuffle)(((AbstractShuffle<Long>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Long128Shuffle rearrange(VectorShuffle<Long> shuffle) {
|
||||
|
@ -493,7 +493,7 @@ final class Long256Vector extends LongVector {
|
||||
VectorMask<Long> m) {
|
||||
return (Long256Vector)
|
||||
super.selectFromTemplate((Long256Vector) v,
|
||||
(Long256Mask) m); // specialize
|
||||
Long256Mask.class, (Long256Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -826,6 +826,13 @@ final class Long256Vector extends LongVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Long256Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Long256Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Long256Shuffle)(((AbstractShuffle<Long>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Long256Shuffle rearrange(VectorShuffle<Long> shuffle) {
|
||||
|
@ -493,7 +493,7 @@ final class Long512Vector extends LongVector {
|
||||
VectorMask<Long> m) {
|
||||
return (Long512Vector)
|
||||
super.selectFromTemplate((Long512Vector) v,
|
||||
(Long512Mask) m); // specialize
|
||||
Long512Mask.class, (Long512Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -834,6 +834,13 @@ final class Long512Vector extends LongVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Long512Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Long512Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Long512Shuffle)(((AbstractShuffle<Long>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Long512Shuffle rearrange(VectorShuffle<Long> shuffle) {
|
||||
|
@ -493,7 +493,7 @@ final class Long64Vector extends LongVector {
|
||||
VectorMask<Long> m) {
|
||||
return (Long64Vector)
|
||||
super.selectFromTemplate((Long64Vector) v,
|
||||
(Long64Mask) m); // specialize
|
||||
Long64Mask.class, (Long64Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -820,6 +820,13 @@ final class Long64Vector extends LongVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Long64Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Long64Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Long64Shuffle)(((AbstractShuffle<Long>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Long64Shuffle rearrange(VectorShuffle<Long> shuffle) {
|
||||
|
@ -493,7 +493,7 @@ final class LongMaxVector extends LongVector {
|
||||
VectorMask<Long> m) {
|
||||
return (LongMaxVector)
|
||||
super.selectFromTemplate((LongMaxVector) v,
|
||||
(LongMaxMask) m); // specialize
|
||||
LongMaxMask.class, (LongMaxMask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -820,6 +820,13 @@ final class LongMaxVector extends LongVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public LongMaxShuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, LongMaxShuffle.class, this, VLENGTH,
|
||||
(s) -> ((LongMaxShuffle)(((AbstractShuffle<Long>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public LongMaxShuffle rearrange(VectorShuffle<Long> shuffle) {
|
||||
|
@ -2244,17 +2244,18 @@ public abstract class LongVector extends AbstractVector<Long> {
|
||||
*/
|
||||
@Override
|
||||
public abstract
|
||||
LongVector rearrange(VectorShuffle<Long> m);
|
||||
LongVector rearrange(VectorShuffle<Long> shuffle);
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final
|
||||
<S extends VectorShuffle<Long>>
|
||||
LongVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
|
||||
shuffle.checkIndexes();
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, null, long.class, length(),
|
||||
this, shuffle, null,
|
||||
this, ws, null,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return v1.lane(ei);
|
||||
@ -2279,17 +2280,14 @@ public abstract class LongVector extends AbstractVector<Long> {
|
||||
M m) {
|
||||
|
||||
m.check(masktype, this);
|
||||
VectorMask<Long> valid = shuffle.laneIsValid();
|
||||
if (m.andNot(valid).anyTrue()) {
|
||||
shuffle.checkIndexes();
|
||||
throw new AssertionError();
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, masktype, long.class, length(),
|
||||
this, shuffle, m,
|
||||
this, ws, m,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
}));
|
||||
}
|
||||
|
||||
@ -2402,7 +2400,10 @@ public abstract class LongVector extends AbstractVector<Long> {
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final LongVector selectFromTemplate(LongVector v) {
|
||||
return v.rearrange(this.toShuffle());
|
||||
return (LongVector)VectorSupport.selectFromOp(getClass(), null, long.class,
|
||||
length(), this, v, null,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle()));
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2414,9 +2415,15 @@ public abstract class LongVector extends AbstractVector<Long> {
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final LongVector selectFromTemplate(LongVector v,
|
||||
AbstractMask<Long> m) {
|
||||
return v.rearrange(this.toShuffle(), m);
|
||||
final
|
||||
<M extends VectorMask<Long>>
|
||||
LongVector selectFromTemplate(LongVector v,
|
||||
Class<M> masktype, M m) {
|
||||
m.check(masktype, this);
|
||||
return (LongVector)VectorSupport.selectFromOp(getClass(), masktype, long.class,
|
||||
length(), this, v, m,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle(), _m));
|
||||
}
|
||||
|
||||
/// Ternary operations
|
||||
|
@ -503,7 +503,7 @@ final class Short128Vector extends ShortVector {
|
||||
VectorMask<Short> m) {
|
||||
return (Short128Vector)
|
||||
super.selectFromTemplate((Short128Vector) v,
|
||||
(Short128Mask) m); // specialize
|
||||
Short128Mask.class, (Short128Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -844,6 +844,13 @@ final class Short128Vector extends ShortVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Short128Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Short128Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Short128Shuffle)(((AbstractShuffle<Short>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Short128Shuffle rearrange(VectorShuffle<Short> shuffle) {
|
||||
|
@ -503,7 +503,7 @@ final class Short256Vector extends ShortVector {
|
||||
VectorMask<Short> m) {
|
||||
return (Short256Vector)
|
||||
super.selectFromTemplate((Short256Vector) v,
|
||||
(Short256Mask) m); // specialize
|
||||
Short256Mask.class, (Short256Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -860,6 +860,13 @@ final class Short256Vector extends ShortVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Short256Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Short256Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Short256Shuffle)(((AbstractShuffle<Short>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Short256Shuffle rearrange(VectorShuffle<Short> shuffle) {
|
||||
|
@ -503,7 +503,7 @@ final class Short512Vector extends ShortVector {
|
||||
VectorMask<Short> m) {
|
||||
return (Short512Vector)
|
||||
super.selectFromTemplate((Short512Vector) v,
|
||||
(Short512Mask) m); // specialize
|
||||
Short512Mask.class, (Short512Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -892,6 +892,13 @@ final class Short512Vector extends ShortVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Short512Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Short512Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Short512Shuffle)(((AbstractShuffle<Short>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Short512Shuffle rearrange(VectorShuffle<Short> shuffle) {
|
||||
|
@ -503,7 +503,7 @@ final class Short64Vector extends ShortVector {
|
||||
VectorMask<Short> m) {
|
||||
return (Short64Vector)
|
||||
super.selectFromTemplate((Short64Vector) v,
|
||||
(Short64Mask) m); // specialize
|
||||
Short64Mask.class, (Short64Mask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -836,6 +836,13 @@ final class Short64Vector extends ShortVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public Short64Shuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, Short64Shuffle.class, this, VLENGTH,
|
||||
(s) -> ((Short64Shuffle)(((AbstractShuffle<Short>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public Short64Shuffle rearrange(VectorShuffle<Short> shuffle) {
|
||||
|
@ -503,7 +503,7 @@ final class ShortMaxVector extends ShortVector {
|
||||
VectorMask<Short> m) {
|
||||
return (ShortMaxVector)
|
||||
super.selectFromTemplate((ShortMaxVector) v,
|
||||
(ShortMaxMask) m); // specialize
|
||||
ShortMaxMask.class, (ShortMaxMask) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -830,6 +830,13 @@ final class ShortMaxVector extends ShortVector {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public ShortMaxShuffle wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, ShortMaxShuffle.class, this, VLENGTH,
|
||||
(s) -> ((ShortMaxShuffle)(((AbstractShuffle<Short>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public ShortMaxShuffle rearrange(VectorShuffle<Short> shuffle) {
|
||||
|
@ -2394,17 +2394,18 @@ public abstract class ShortVector extends AbstractVector<Short> {
|
||||
*/
|
||||
@Override
|
||||
public abstract
|
||||
ShortVector rearrange(VectorShuffle<Short> m);
|
||||
ShortVector rearrange(VectorShuffle<Short> shuffle);
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final
|
||||
<S extends VectorShuffle<Short>>
|
||||
ShortVector rearrangeTemplate(Class<S> shuffletype, S shuffle) {
|
||||
shuffle.checkIndexes();
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, null, short.class, length(),
|
||||
this, shuffle, null,
|
||||
this, ws, null,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return v1.lane(ei);
|
||||
@ -2429,17 +2430,14 @@ public abstract class ShortVector extends AbstractVector<Short> {
|
||||
M m) {
|
||||
|
||||
m.check(masktype, this);
|
||||
VectorMask<Short> valid = shuffle.laneIsValid();
|
||||
if (m.andNot(valid).anyTrue()) {
|
||||
shuffle.checkIndexes();
|
||||
throw new AssertionError();
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, masktype, short.class, length(),
|
||||
this, shuffle, m,
|
||||
this, ws, m,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
}));
|
||||
}
|
||||
|
||||
@ -2552,7 +2550,10 @@ public abstract class ShortVector extends AbstractVector<Short> {
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final ShortVector selectFromTemplate(ShortVector v) {
|
||||
return v.rearrange(this.toShuffle());
|
||||
return (ShortVector)VectorSupport.selectFromOp(getClass(), null, short.class,
|
||||
length(), this, v, null,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle()));
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2564,9 +2565,15 @@ public abstract class ShortVector extends AbstractVector<Short> {
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final ShortVector selectFromTemplate(ShortVector v,
|
||||
AbstractMask<Short> m) {
|
||||
return v.rearrange(this.toShuffle(), m);
|
||||
final
|
||||
<M extends VectorMask<Short>>
|
||||
ShortVector selectFromTemplate(ShortVector v,
|
||||
Class<M> masktype, M m) {
|
||||
m.check(masktype, this);
|
||||
return (ShortVector)VectorSupport.selectFromOp(getClass(), masktype, short.class,
|
||||
length(), this, v, m,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle(), _m));
|
||||
}
|
||||
|
||||
/// Ternary operations
|
||||
|
@ -2614,17 +2614,14 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
|
||||
* elements of this vector.
|
||||
*
|
||||
* For each lane {@code N} of the shuffle, and for each lane
|
||||
* source index {@code I=s.laneSource(N)} in the shuffle,
|
||||
* source index {@code I=s.wrapIndex(s.laneSource(N))} in the shuffle,
|
||||
* the output lane {@code N} obtains the value from
|
||||
* the input vector at lane {@code I}.
|
||||
*
|
||||
* @param s the shuffle controlling lane index selection
|
||||
* @return the rearrangement of the lane elements of this vector
|
||||
* @throws IndexOutOfBoundsException if there are any exceptional
|
||||
* source indexes in the shuffle
|
||||
* @see #rearrange(VectorShuffle,VectorMask)
|
||||
* @see #rearrange(VectorShuffle,Vector)
|
||||
* @see VectorShuffle#laneIsValid()
|
||||
*/
|
||||
public abstract Vector<E> rearrange(VectorShuffle<E> s);
|
||||
|
||||
@ -2636,27 +2633,22 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
|
||||
* elements of this vector.
|
||||
*
|
||||
* For each lane {@code N} of the shuffle, and for each lane
|
||||
* source index {@code I=s.laneSource(N)} in the shuffle,
|
||||
* source index {@code I=s.wrapIndex(s.laneSource(N))} in the shuffle,
|
||||
* the output lane {@code N} obtains the value from
|
||||
* the input vector at lane {@code I} if the mask is set.
|
||||
* Otherwise the output lane {@code N} is set to zero.
|
||||
*
|
||||
* <p> This method returns the value of this pseudocode:
|
||||
* <pre>{@code
|
||||
* Vector<E> r = this.rearrange(s.wrapIndexes());
|
||||
* VectorMask<E> valid = s.laneIsValid();
|
||||
* if (m.andNot(valid).anyTrue()) throw ...;
|
||||
* Vector<E> r = this.rearrange(s);
|
||||
* return broadcast(0).blend(r, m);
|
||||
* }</pre>
|
||||
*
|
||||
* @param s the shuffle controlling lane index selection
|
||||
* @param m the mask controlling application of the shuffle
|
||||
* @return the rearrangement of the lane elements of this vector
|
||||
* @throws IndexOutOfBoundsException if there are any exceptional
|
||||
* source indexes in the shuffle where the mask is set
|
||||
* @see #rearrange(VectorShuffle)
|
||||
* @see #rearrange(VectorShuffle,Vector)
|
||||
* @see VectorShuffle#laneIsValid()
|
||||
*/
|
||||
public abstract Vector<E> rearrange(VectorShuffle<E> s, VectorMask<E> m);
|
||||
|
||||
@ -2747,7 +2739,7 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
|
||||
* this vector.
|
||||
*
|
||||
* For each lane {@code N} of this vector, and for each lane
|
||||
* value {@code I=this.lane(N)} in this vector,
|
||||
* value {@code I=wrapIndex(this.lane(N))} in this vector,
|
||||
* the output lane {@code N} obtains the value from
|
||||
* the argument vector at lane {@code I}.
|
||||
*
|
||||
@ -2760,8 +2752,6 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
|
||||
*
|
||||
* @param v the vector supplying the result values
|
||||
* @return the rearrangement of the lane elements of {@code v}
|
||||
* @throws IndexOutOfBoundsException if any invalid
|
||||
* source indexes are found in {@code this}
|
||||
* @see #rearrange(VectorShuffle)
|
||||
*/
|
||||
public abstract Vector<E> selectFrom(Vector<E> v);
|
||||
@ -2787,9 +2777,6 @@ public abstract class Vector<E> extends jdk.internal.vm.vector.VectorSupport.Vec
|
||||
* @param v the vector supplying the result values
|
||||
* @param m the mask controlling selection from {@code v}
|
||||
* @return the rearrangement of the lane elements of {@code v}
|
||||
* @throws IndexOutOfBoundsException if any invalid
|
||||
* source indexes are found in {@code this},
|
||||
* in a lane which is set in the mask
|
||||
* @see #selectFrom(Vector)
|
||||
* @see #rearrange(VectorShuffle,VectorMask)
|
||||
*/
|
||||
|
@ -2770,17 +2770,18 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
|
||||
*/
|
||||
@Override
|
||||
public abstract
|
||||
$abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> m);
|
||||
$abstractvectortype$ rearrange(VectorShuffle<$Boxtype$> shuffle);
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final
|
||||
<S extends VectorShuffle<$Boxtype$>>
|
||||
$abstractvectortype$ rearrangeTemplate(Class<S> shuffletype, S shuffle) {
|
||||
shuffle.checkIndexes();
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, null, $type$.class, length(),
|
||||
this, shuffle, null,
|
||||
this, ws, null,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return v1.lane(ei);
|
||||
@ -2805,17 +2806,14 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
|
||||
M m) {
|
||||
|
||||
m.check(masktype, this);
|
||||
VectorMask<$Boxtype$> valid = shuffle.laneIsValid();
|
||||
if (m.andNot(valid).anyTrue()) {
|
||||
shuffle.checkIndexes();
|
||||
throw new AssertionError();
|
||||
}
|
||||
@SuppressWarnings("unchecked")
|
||||
S ws = (S) shuffle.wrapIndexes();
|
||||
return VectorSupport.rearrangeOp(
|
||||
getClass(), shuffletype, masktype, $type$.class, length(),
|
||||
this, shuffle, m,
|
||||
this, ws, m,
|
||||
(v1, s_, m_) -> v1.uOp((i, a) -> {
|
||||
int ei = s_.laneSource(i);
|
||||
return ei < 0 || !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
return !m_.laneIsSet(i) ? 0 : v1.lane(ei);
|
||||
}));
|
||||
}
|
||||
|
||||
@ -2928,7 +2926,10 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v) {
|
||||
return v.rearrange(this.toShuffle());
|
||||
return ($Type$Vector)VectorSupport.selectFromOp(getClass(), null, $type$.class,
|
||||
length(), this, v, null,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle()));
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2940,9 +2941,15 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
|
||||
|
||||
/*package-private*/
|
||||
@ForceInline
|
||||
final $abstractvectortype$ selectFromTemplate($abstractvectortype$ v,
|
||||
AbstractMask<$Boxtype$> m) {
|
||||
return v.rearrange(this.toShuffle(), m);
|
||||
final
|
||||
<M extends VectorMask<$Boxtype$>>
|
||||
$abstractvectortype$ selectFromTemplate($abstractvectortype$ v,
|
||||
Class<M> masktype, M m) {
|
||||
m.check(masktype, this);
|
||||
return ($Type$Vector)VectorSupport.selectFromOp(getClass(), masktype, $type$.class,
|
||||
length(), this, v, m,
|
||||
(v1, v2, _m) ->
|
||||
v2.rearrange(v1.toShuffle(), _m));
|
||||
}
|
||||
|
||||
/// Ternary operations
|
||||
|
@ -509,7 +509,7 @@ final class $vectortype$ extends $abstractvectortype$ {
|
||||
VectorMask<$Boxtype$> m) {
|
||||
return ($vectortype$)
|
||||
super.selectFromTemplate(($vectortype$) v,
|
||||
($masktype$) m); // specialize
|
||||
$masktype$.class, ($masktype$) m); // specialize
|
||||
}
|
||||
|
||||
|
||||
@ -1118,6 +1118,13 @@ final class $vectortype$ extends $abstractvectortype$ {
|
||||
return s.shuffleFromArray(shuffleArray, 0).check(s);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ForceInline
|
||||
public $shuffletype$ wrapIndexes() {
|
||||
return VectorSupport.wrapShuffleIndexes(ETYPE, $shuffletype$.class, this, VLENGTH,
|
||||
(s) -> (($shuffletype$)(((AbstractShuffle<$Boxtype$>)(s)).wrapIndexesTemplate())));
|
||||
}
|
||||
|
||||
@ForceInline
|
||||
@Override
|
||||
public $shuffletype$ rearrange(VectorShuffle<$Boxtype$> shuffle) {
|
||||
|
Loading…
x
Reference in New Issue
Block a user