From ab8c1361bc03a8afe016c82f1ad3da9204626d72 Mon Sep 17 00:00:00 2001 From: Xiaohong Gong Date: Wed, 12 Oct 2022 01:39:57 +0000 Subject: [PATCH] 8292898: [vectorapi] Unify vector mask cast operation Co-authored-by: Quan Anh Mai Reviewed-by: jbhateja, eliu --- src/hotspot/cpu/aarch64/aarch64_vector.ad | 50 +- src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 | 50 +- src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp | 55 ++ src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp | 2 + src/hotspot/cpu/x86/x86.ad | 19 +- src/hotspot/share/opto/vectorIntrinsics.cpp | 34 +- src/hotspot/share/opto/vectornode.cpp | 56 +- src/hotspot/share/opto/vectornode.hpp | 1 - .../compiler/lib/ir_framework/IRNode.java | 1 + .../vectorapi/VectorMaskCastTest.java | 524 ++++++++++-------- 10 files changed, 470 insertions(+), 322 deletions(-) diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad index 9e0bfcad085..caf769b4df9 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector.ad +++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad @@ -5087,6 +5087,48 @@ instruct vmaskcast_same_esize_neon(vReg dst_src) %{ ins_pipe(pipe_class_empty); %} +instruct vmaskcast_extend_neon(vReg dst, vReg src) %{ + predicate(UseSVE == 0 && + Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1))); + match(Set dst (VectorMaskCast src)); + format %{ "vmaskcast_extend_neon $dst, $src" %} + ins_encode %{ + BasicType dst_bt = Matcher::vector_element_basic_type(this); + if (is_floating_point_type(dst_bt)) { + dst_bt = (dst_bt == T_FLOAT) ? T_INT : T_LONG; + } + uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this); + BasicType src_bt = Matcher::vector_element_basic_type(this, $src); + if (is_floating_point_type(src_bt)) { + src_bt = (src_bt == T_FLOAT) ? T_INT : T_LONG; + } + __ neon_vector_extend($dst$$FloatRegister, dst_bt, length_in_bytes_dst, + $src$$FloatRegister, src_bt); + %} + ins_pipe(pipe_slow); +%} + +instruct vmaskcast_narrow_neon(vReg dst, vReg src) %{ + predicate(UseSVE == 0 && + Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1))); + match(Set dst (VectorMaskCast src)); + format %{ "vmaskcast_narrow_neon $dst, $src" %} + ins_encode %{ + BasicType dst_bt = Matcher::vector_element_basic_type(this); + if (is_floating_point_type(dst_bt)) { + dst_bt = (dst_bt == T_FLOAT) ? T_INT : T_LONG; + } + BasicType src_bt = Matcher::vector_element_basic_type(this, $src); + if (is_floating_point_type(src_bt)) { + src_bt = (src_bt == T_FLOAT) ? T_INT : T_LONG; + } + uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src); + __ neon_vector_narrow($dst$$FloatRegister, dst_bt, + $src$$FloatRegister, src_bt, length_in_bytes_src); + %} + ins_pipe(pipe_slow); +%} + instruct vmaskcast_same_esize_sve(pReg dst_src) %{ predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); @@ -5097,11 +5139,11 @@ instruct vmaskcast_same_esize_sve(pReg dst_src) %{ ins_pipe(pipe_class_empty); %} -instruct vmaskcast_extend(pReg dst, pReg src) %{ +instruct vmaskcast_extend_sve(pReg dst, pReg src) %{ predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1))); match(Set dst (VectorMaskCast src)); - format %{ "vmaskcast_extend $dst, $src" %} + format %{ "vmaskcast_extend_sve $dst, $src" %} ins_encode %{ uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this); uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src); @@ -5114,11 +5156,11 @@ instruct vmaskcast_extend(pReg dst, pReg src) %{ ins_pipe(pipe_slow); %} -instruct vmaskcast_narrow(pReg dst, pReg src) %{ +instruct vmaskcast_narrow_sve(pReg dst, pReg src) %{ predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1))); match(Set dst (VectorMaskCast src)); - format %{ "vmaskcast_narrow $dst, $src" %} + format %{ "vmaskcast_narrow_sve $dst, $src" %} ins_encode %{ uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this); uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src); diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 index 24f91ed5564..9286a4cd8fe 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 +++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 @@ -3503,6 +3503,48 @@ instruct vmaskcast_same_esize_neon(vReg dst_src) %{ ins_pipe(pipe_class_empty); %} +instruct vmaskcast_extend_neon(vReg dst, vReg src) %{ + predicate(UseSVE == 0 && + Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1))); + match(Set dst (VectorMaskCast src)); + format %{ "vmaskcast_extend_neon $dst, $src" %} + ins_encode %{ + BasicType dst_bt = Matcher::vector_element_basic_type(this); + if (is_floating_point_type(dst_bt)) { + dst_bt = (dst_bt == T_FLOAT) ? T_INT : T_LONG; + } + uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this); + BasicType src_bt = Matcher::vector_element_basic_type(this, $src); + if (is_floating_point_type(src_bt)) { + src_bt = (src_bt == T_FLOAT) ? T_INT : T_LONG; + } + __ neon_vector_extend($dst$$FloatRegister, dst_bt, length_in_bytes_dst, + $src$$FloatRegister, src_bt); + %} + ins_pipe(pipe_slow); +%} + +instruct vmaskcast_narrow_neon(vReg dst, vReg src) %{ + predicate(UseSVE == 0 && + Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1))); + match(Set dst (VectorMaskCast src)); + format %{ "vmaskcast_narrow_neon $dst, $src" %} + ins_encode %{ + BasicType dst_bt = Matcher::vector_element_basic_type(this); + if (is_floating_point_type(dst_bt)) { + dst_bt = (dst_bt == T_FLOAT) ? T_INT : T_LONG; + } + BasicType src_bt = Matcher::vector_element_basic_type(this, $src); + if (is_floating_point_type(src_bt)) { + src_bt = (src_bt == T_FLOAT) ? T_INT : T_LONG; + } + uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src); + __ neon_vector_narrow($dst$$FloatRegister, dst_bt, + $src$$FloatRegister, src_bt, length_in_bytes_src); + %} + ins_pipe(pipe_slow); +%} + instruct vmaskcast_same_esize_sve(pReg dst_src) %{ predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); @@ -3513,11 +3555,11 @@ instruct vmaskcast_same_esize_sve(pReg dst_src) %{ ins_pipe(pipe_class_empty); %} -instruct vmaskcast_extend(pReg dst, pReg src) %{ +instruct vmaskcast_extend_sve(pReg dst, pReg src) %{ predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1))); match(Set dst (VectorMaskCast src)); - format %{ "vmaskcast_extend $dst, $src" %} + format %{ "vmaskcast_extend_sve $dst, $src" %} ins_encode %{ uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this); uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src); @@ -3530,11 +3572,11 @@ instruct vmaskcast_extend(pReg dst, pReg src) %{ ins_pipe(pipe_slow); %} -instruct vmaskcast_narrow(pReg dst, pReg src) %{ +instruct vmaskcast_narrow_sve(pReg dst, pReg src) %{ predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1))); match(Set dst (VectorMaskCast src)); - format %{ "vmaskcast_narrow $dst, $src" %} + format %{ "vmaskcast_narrow_sve $dst, $src" %} ins_encode %{ uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this); uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src); diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp index 9027bafeb17..9a806641ce2 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp @@ -4750,6 +4750,61 @@ void C2_MacroAssembler::vector_unsigned_cast(XMMRegister dst, XMMRegister src, i } } +void C2_MacroAssembler::vector_mask_cast(XMMRegister dst, XMMRegister src, + BasicType dst_bt, BasicType src_bt, int vlen) { + int vlen_enc = vector_length_encoding(MAX2(type2aelembytes(src_bt), type2aelembytes(dst_bt)) * vlen); + assert(vlen_enc != AVX_512bit, ""); + + int dst_bt_size = type2aelembytes(dst_bt); + int src_bt_size = type2aelembytes(src_bt); + if (dst_bt_size > src_bt_size) { + switch (dst_bt_size / src_bt_size) { + case 2: vpmovsxbw(dst, src, vlen_enc); break; + case 4: vpmovsxbd(dst, src, vlen_enc); break; + case 8: vpmovsxbq(dst, src, vlen_enc); break; + default: ShouldNotReachHere(); + } + } else { + assert(dst_bt_size < src_bt_size, ""); + switch (src_bt_size / dst_bt_size) { + case 2: { + if (vlen_enc == AVX_128bit) { + vpacksswb(dst, src, src, vlen_enc); + } else { + vpacksswb(dst, src, src, vlen_enc); + vpermq(dst, dst, 0x08, vlen_enc); + } + break; + } + case 4: { + if (vlen_enc == AVX_128bit) { + vpackssdw(dst, src, src, vlen_enc); + vpacksswb(dst, dst, dst, vlen_enc); + } else { + vpackssdw(dst, src, src, vlen_enc); + vpermq(dst, dst, 0x08, vlen_enc); + vpacksswb(dst, dst, dst, AVX_128bit); + } + break; + } + case 8: { + if (vlen_enc == AVX_128bit) { + vpshufd(dst, src, 0x08, vlen_enc); + vpackssdw(dst, dst, dst, vlen_enc); + vpacksswb(dst, dst, dst, vlen_enc); + } else { + vpshufd(dst, src, 0x08, vlen_enc); + vpermq(dst, dst, 0x08, vlen_enc); + vpackssdw(dst, dst, dst, AVX_128bit); + vpacksswb(dst, dst, dst, AVX_128bit); + } + break; + } + default: ShouldNotReachHere(); + } + } +} + void C2_MacroAssembler::evpternlog(XMMRegister dst, int func, KRegister mask, XMMRegister src2, XMMRegister src3, bool merge, BasicType bt, int vlen_enc) { if (bt == T_INT) { diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp index 4d063f0ae3c..1e669ccaaab 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp @@ -362,6 +362,8 @@ public: void vector_crosslane_doubleword_pack_avx(XMMRegister dst, XMMRegister src, XMMRegister zero, XMMRegister xtmp, int index, int vec_enc); + void vector_mask_cast(XMMRegister dst, XMMRegister src, BasicType dst_bt, BasicType src_bt, int vlen); + #ifdef _LP64 void vector_round_double_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, AddressLiteral new_mxcsr, int vec_enc, Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2); diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 7637b3d4981..dca42b5122a 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -1480,6 +1480,7 @@ const bool Matcher::match_rule_supported(int opcode) { case Op_VectorUCastB2X: case Op_VectorUCastS2X: case Op_VectorUCastI2X: + case Op_VectorMaskCast: if (UseAVX < 1) { // enabled for AVX only return false; } @@ -1857,6 +1858,7 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType } break; case Op_VectorLoadMask: + case Op_VectorMaskCast: if (size_in_bits == 256 && UseAVX < 2) { return false; // Implementation limitation } @@ -8413,7 +8415,6 @@ instruct vstoreMask_evex(vec dst, kReg mask, immI size) %{ %} instruct vmaskcast_evex(kReg dst) %{ - predicate(Matcher::vector_length(n) == Matcher::vector_length(n->in(1))); match(Set dst (VectorMaskCast dst)); ins_cost(0); format %{ "vector_mask_cast $dst" %} @@ -8424,8 +8425,7 @@ instruct vmaskcast_evex(kReg dst) %{ %} instruct vmaskcast(vec dst) %{ - predicate((Matcher::vector_length(n) == Matcher::vector_length(n->in(1))) && - (Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)))); + predicate(Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1))); match(Set dst (VectorMaskCast dst)); ins_cost(0); format %{ "vector_mask_cast $dst" %} @@ -8435,6 +8435,19 @@ instruct vmaskcast(vec dst) %{ ins_pipe(empty); %} +instruct vmaskcast_avx(vec dst, vec src) %{ + predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1))); + match(Set dst (VectorMaskCast src)); + format %{ "vector_mask_cast $dst, $src" %} + ins_encode %{ + int vlen = Matcher::vector_length(this); + BasicType src_bt = Matcher::vector_element_basic_type(this, $src); + BasicType dst_bt = Matcher::vector_element_basic_type(this); + __ vector_mask_cast($dst$$XMMRegister, $src$$XMMRegister, dst_bt, src_bt, vlen); + %} + ins_pipe(pipe_slow); +%} + //-------------------------------- Load Iota Indices ---------------------------------- instruct loadIotaIndices(vec dst, immI_0 src) %{ diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp index 6a3e9c62535..d2796316bd1 100644 --- a/src/hotspot/share/opto/vectorIntrinsics.cpp +++ b/src/hotspot/share/opto/vectorIntrinsics.cpp @@ -2488,24 +2488,15 @@ bool LibraryCallKit::inline_vector_convert() { Node* op = opd1; if (is_cast) { - BasicType new_elem_bt_to = elem_bt_to; - BasicType new_elem_bt_from = elem_bt_from; - if (is_mask && is_floating_point_type(elem_bt_from)) { - new_elem_bt_from = elem_bt_from == T_FLOAT ? T_INT : T_LONG; - } - if (is_mask && is_floating_point_type(elem_bt_to)) { - new_elem_bt_to = elem_bt_to == T_FLOAT ? T_INT : T_LONG; - } - int cast_vopc = VectorCastNode::opcode(new_elem_bt_from, !is_ucast); + assert(!is_mask || num_elem_from == num_elem_to, "vector mask cast needs the same elem num"); + int cast_vopc = VectorCastNode::opcode(elem_bt_from, !is_ucast); - // Make sure that vector cast is implemented to particular type/size combination. - bool no_vec_cast_check = is_mask && - ((src_type->isa_vectmask() && dst_type->isa_vectmask()) || - type2aelembytes(elem_bt_from) == type2aelembytes(elem_bt_to)); - if (!no_vec_cast_check && !arch_supports_vector(cast_vopc, num_elem_to, new_elem_bt_to, VecMaskNotUsed)) { + // Make sure that vector cast is implemented to particular type/size combination if it is + // not a mask casting. + if (!is_mask && !arch_supports_vector(cast_vopc, num_elem_to, elem_bt_to, VecMaskNotUsed)) { if (C->print_intrinsics()) { tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s ismask=%d", - cast_vopc, num_elem_to, type2name(new_elem_bt_to), is_mask); + cast_vopc, num_elem_to, type2name(elem_bt_to), is_mask); } return false; } @@ -2552,12 +2543,15 @@ bool LibraryCallKit::inline_vector_convert() { op = gvn().transform(VectorCastNode::make(cast_vopc, op, elem_bt_to, num_elem_to)); } else { // num_elem_from == num_elem_to if (is_mask) { - if ((dst_type->isa_vectmask() && src_type->isa_vectmask()) || - (type2aelembytes(elem_bt_from) == type2aelembytes(elem_bt_to))) { - op = gvn().transform(new VectorMaskCastNode(op, dst_type)); - } else { - op = VectorMaskCastNode::makeCastNode(&gvn(), op, dst_type); + // Make sure that cast for vector mask is implemented to particular type/size combination. + if (!arch_supports_vector(Op_VectorMaskCast, num_elem_to, elem_bt_to, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=1 op=maskcast vlen2=%d etype2=%s ismask=%d", + num_elem_to, type2name(elem_bt_to), is_mask); + } + return false; } + op = gvn().transform(new VectorMaskCastNode(op, dst_type)); } else { // Since input and output number of elements match, and since we know this vector size is // supported, simply do a cast with no resize needed. diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 5231893339d..2a8df6cb6cd 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -1619,17 +1619,9 @@ Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) { bool is_vector_mask = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); if (is_vector_mask) { + // VectorUnbox (VectorBox vmask) ==> VectorMaskCast vmask const TypeVect* vmask_type = TypeVect::makemask(out_vt->element_basic_type(), out_vt->length()); - if (in_vt->length_in_bytes() == out_vt->length_in_bytes() && - Matcher::match_rule_supported_vector(Op_VectorMaskCast, out_vt->length(), out_vt->element_basic_type())) { - // Apply "VectorUnbox (VectorBox vmask) ==> VectorMaskCast (vmask)" - // directly. This could avoid the transformation ordering issue from - // "VectorStoreMask (VectorLoadMask vmask) => vmask". - return new VectorMaskCastNode(value, vmask_type); - } - // VectorUnbox (VectorBox vmask) ==> VectorLoadMask (VectorStoreMask vmask) - value = phase->transform(VectorStoreMaskNode::make(*phase, value, in_vt->element_basic_type(), in_vt->length())); - return new VectorLoadMaskNode(value, vmask_type); + return new VectorMaskCastNode(value, vmask_type); } else if (is_vector_shuffle) { if (!is_shuffle_to_vector()) { // VectorUnbox (VectorBox vshuffle) ==> VectorLoadShuffle vshuffle @@ -1720,48 +1712,6 @@ Node* VectorMaskToLongNode::Identity(PhaseGVN* phase) { return this; } -Node* VectorMaskCastNode::makeCastNode(PhaseGVN* phase, Node* src, const TypeVect* dst_type) { - const TypeVect* src_type = src->bottom_type()->is_vect(); - assert(src_type->length() == dst_type->length(), ""); - - int num_elem = src_type->length(); - BasicType elem_bt_from = src_type->element_basic_type(); - BasicType elem_bt_to = dst_type->element_basic_type(); - - if (dst_type->isa_vectmask() == NULL && src_type->isa_vectmask() == NULL && - type2aelembytes(elem_bt_from) != type2aelembytes(elem_bt_to)) { - - Node* op = src; - BasicType new_elem_bt_from = elem_bt_from; - BasicType new_elem_bt_to = elem_bt_to; - if (is_floating_point_type(elem_bt_from)) { - new_elem_bt_from = elem_bt_from == T_FLOAT ? T_INT : T_LONG; - } - if (is_floating_point_type(elem_bt_to)) { - new_elem_bt_to = elem_bt_to == T_FLOAT ? T_INT : T_LONG; - } - - // Special handling for casting operation involving floating point types. - // Case A) F -> X := F -> VectorMaskCast (F->I/L [NOP]) -> VectorCast[I/L]2X - // Case B) X -> F := X -> VectorCastX2[I/L] -> VectorMaskCast ([I/L]->F [NOP]) - // Case C) F -> F := VectorMaskCast (F->I/L [NOP]) -> VectorCast[I/L]2[L/I] -> VectotMaskCast (L/I->F [NOP]) - - if (new_elem_bt_from != elem_bt_from) { - const TypeVect* new_src_type = TypeVect::makemask(new_elem_bt_from, num_elem); - op = phase->transform(new VectorMaskCastNode(op, new_src_type)); - } - - op = phase->transform(VectorCastNode::make(VectorCastNode::opcode(new_elem_bt_from), op, new_elem_bt_to, num_elem)); - - if (new_elem_bt_to != elem_bt_to) { - op = phase->transform(new VectorMaskCastNode(op, dst_type)); - } - return op; - } else { - return new VectorMaskCastNode(src, dst_type); - } -} - Node* VectorLongToMaskNode::Ideal(PhaseGVN* phase, bool can_reshape) { const TypeVect* dst_type = bottom_type()->is_vect(); if (in(1)->Opcode() == Op_AndL && @@ -1782,7 +1732,7 @@ Node* VectorLongToMaskNode::Ideal(PhaseGVN* phase, bool can_reshape) { if (src_type->length() == dst_type->length() && ((src_type->isa_vectmask() == NULL && dst_type->isa_vectmask() == NULL) || (src_type->isa_vectmask() && dst_type->isa_vectmask()))) { - return VectorMaskCastNode::makeCastNode(phase, src, dst_type); + return new VectorMaskCastNode(src, dst_type); } } return NULL; diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index 8cc63015109..4418a074d28 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -1505,7 +1505,6 @@ class VectorMaskCastNode : public VectorNode { const TypeVect* in_vt = in->bottom_type()->is_vect(); assert(in_vt->length() == vt->length(), "vector length must match"); } - static Node* makeCastNode(PhaseGVN* phase, Node* in1, const TypeVect * vt); virtual int Opcode() const; }; diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java index 6443ed18e21..bfb6db96f82 100644 --- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java +++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java @@ -202,6 +202,7 @@ public class IRNode { public static final String OR_V_MASK = START + "OrVMask" + MID + END; public static final String XOR_V_MASK = START + "XorVMask" + MID + END; + public static final String VECTOR_MASK_CAST = START + "VectorMaskCast" + MID + END; public static final String VECTOR_CAST_B2X = START + "VectorCastB2X" + MID + END; public static final String VECTOR_CAST_S2X = START + "VectorCastS2X" + MID + END; public static final String VECTOR_CAST_I2X = START + "VectorCastI2X" + MID + END; diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastTest.java index e60d1ebf7f3..fa4da60c40b 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCastTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Arm Limited. All rights reserved. + * Copyright (c) 2021, 2022, Arm Limited. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,6 +23,8 @@ package compiler.vectorapi; +import compiler.lib.ir_framework.*; + import java.util.Random; import jdk.incubator.vector.ByteVector; @@ -32,23 +34,21 @@ import jdk.incubator.vector.IntVector; import jdk.incubator.vector.LongVector; import jdk.incubator.vector.ShortVector; import jdk.incubator.vector.VectorMask; -import jdk.test.lib.Utils; -import org.testng.Assert; -import org.testng.annotations.Test; +import jdk.test.lib.Asserts; +import jdk.test.lib.Utils; /** * @test - * @bug 8273264 + * @bug 8273264 8292898 * @key randomness - * @library /test/lib - * @summary AArch64: [vector] Add missing rules for VectorMaskCast + * @library /test/lib / + * @summary Unify vector mask cast and add missing rules for VectorMaskCast * @modules jdk.incubator.vector * - * @run testng/othervm -XX:-TieredCompilation -XX:CompileThreshold=100 compiler.vectorapi.VectorMaskCastTest + * @run driver compiler.vectorapi.VectorMaskCastTest */ - // Current vector mask cast test cases at test/jdk/jdk/incubator/vector/*ConversionTests.java // could not be intrinsfied, hence not able to verify compiler codegen, see [1]. As a // supplement, we add more tests for vector mask cast operations, which could be intrinsified @@ -56,413 +56,463 @@ import org.testng.annotations.Test; // // [1] https://bugs.openjdk.java.net/browse/JDK-8259610 -public class VectorMaskCastTest{ +public class VectorMaskCastTest { - private static final int NUM_ITER = 5000; private static final Random rd = Utils.getRandomInstance(); - public static boolean[] genMask() { - boolean[] mask = new boolean[64]; - for (int i = 0; i < 64; i ++) { - mask[i] = rd.nextBoolean(); + private static final boolean[] mask_arr; + + static { + mask_arr = new boolean[64]; + for (int i = 0; i < 64; i++) { + mask_arr[i] = rd.nextBoolean(); } - return mask; } // Byte - private static void testByte64ToShort128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true"}) + public static void testByte64ToShort128() { VectorMask mByte64 = VectorMask.fromArray(ByteVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mByte64.cast(ShortVector.SPECIES_128).toString(), mByte64.toString()); + Asserts.assertEquals(mByte64.cast(ShortVector.SPECIES_128).toString(), mByte64.toString()); } - private static void testByte64ToInt256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testByte64ToInt256() { VectorMask mByte64 = VectorMask.fromArray(ByteVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mByte64.cast(IntVector.SPECIES_256).toString(), mByte64.toString()); + Asserts.assertEquals(mByte64.cast(IntVector.SPECIES_256).toString(), mByte64.toString()); } - private static void testByte64ToFloat256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testByte64ToFloat256() { VectorMask mByte64 = VectorMask.fromArray(ByteVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mByte64.cast(FloatVector.SPECIES_256).toString(), mByte64.toString()); + Asserts.assertEquals(mByte64.cast(FloatVector.SPECIES_256).toString(), mByte64.toString()); } - private static void testByte64ToLong512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testByte64ToLong512() { VectorMask mByte64 = VectorMask.fromArray(ByteVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mByte64.cast(LongVector.SPECIES_512).toString(), mByte64.toString()); + Asserts.assertEquals(mByte64.cast(LongVector.SPECIES_512).toString(), mByte64.toString()); } - private static void testByte64ToDouble512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testByte64ToDouble512() { VectorMask mByte64 = VectorMask.fromArray(ByteVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mByte64.cast(DoubleVector.SPECIES_512).toString(), mByte64.toString()); + Asserts.assertEquals(mByte64.cast(DoubleVector.SPECIES_512).toString(), mByte64.toString()); } - private static void testByte128ToShort256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testByte128ToShort256() { VectorMask mByte128 = VectorMask.fromArray(ByteVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mByte128.cast(ShortVector.SPECIES_256).toString(), mByte128.toString()); + Asserts.assertEquals(mByte128.cast(ShortVector.SPECIES_256).toString(), mByte128.toString()); } - private static void testByte128ToInt512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testByte128ToInt512() { VectorMask mByte128 = VectorMask.fromArray(ByteVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mByte128.cast(IntVector.SPECIES_512).toString(), mByte128.toString()); + Asserts.assertEquals(mByte128.cast(IntVector.SPECIES_512).toString(), mByte128.toString()); } - private static void testByte128ToFloat512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testByte128ToFloat512() { VectorMask mByte128 = VectorMask.fromArray(ByteVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mByte128.cast(FloatVector.SPECIES_512).toString(), mByte128.toString()); + Asserts.assertEquals(mByte128.cast(FloatVector.SPECIES_512).toString(), mByte128.toString()); } - private static void testByte256ToShort512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testByte256ToShort512() { VectorMask mByte256 = VectorMask.fromArray(ByteVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mByte256.cast(ShortVector.SPECIES_512).toString(), mByte256.toString()); + Asserts.assertEquals(mByte256.cast(ShortVector.SPECIES_512).toString(), mByte256.toString()); } // Short - private static void testShort64ToInt128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true"}) + public static void testShort64ToInt128() { VectorMask mShort64 = VectorMask.fromArray(ShortVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mShort64.cast(IntVector.SPECIES_128).toString(), mShort64.toString()); + Asserts.assertEquals(mShort64.cast(IntVector.SPECIES_128).toString(), mShort64.toString()); } - private static void testShort64ToFloat128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true"}) + public static void testShort64ToFloat128() { VectorMask mShort64 = VectorMask.fromArray(ShortVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mShort64.cast(FloatVector.SPECIES_128).toString(), mShort64.toString()); + Asserts.assertEquals(mShort64.cast(FloatVector.SPECIES_128).toString(), mShort64.toString()); } - private static void testShort64ToLong256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testShort64ToLong256() { VectorMask mShort64 = VectorMask.fromArray(ShortVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mShort64.cast(LongVector.SPECIES_256).toString(), mShort64.toString()); + Asserts.assertEquals(mShort64.cast(LongVector.SPECIES_256).toString(), mShort64.toString()); } - private static void testShort64ToDouble256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testShort64ToDouble256() { VectorMask mShort64 = VectorMask.fromArray(ShortVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mShort64.cast(DoubleVector.SPECIES_256).toString(), mShort64.toString()); + Asserts.assertEquals(mShort64.cast(DoubleVector.SPECIES_256).toString(), mShort64.toString()); } - private static void testShort128ToByte64(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true"}) + public static void testShort128ToByte64() { VectorMask mShort128 = VectorMask.fromArray(ShortVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mShort128.cast(ByteVector.SPECIES_64).toString(), mShort128.toString()); + Asserts.assertEquals(mShort128.cast(ByteVector.SPECIES_64).toString(), mShort128.toString()); } - private static void testShort128ToInt256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testShort128ToInt256() { VectorMask mShort128 = VectorMask.fromArray(ShortVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mShort128.cast(IntVector.SPECIES_256).toString(), mShort128.toString()); + Asserts.assertEquals(mShort128.cast(IntVector.SPECIES_256).toString(), mShort128.toString()); } - private static void testShort128ToFloat256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testShort128ToFloat256() { VectorMask mShort128 = VectorMask.fromArray(ShortVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mShort128.cast(FloatVector.SPECIES_256).toString(), mShort128.toString()); + Asserts.assertEquals(mShort128.cast(FloatVector.SPECIES_256).toString(), mShort128.toString()); } - private static void testShort128ToLong512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testShort128ToLong512() { VectorMask mShort128 = VectorMask.fromArray(ShortVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mShort128.cast(LongVector.SPECIES_512).toString(), mShort128.toString()); + Asserts.assertEquals(mShort128.cast(LongVector.SPECIES_512).toString(), mShort128.toString()); } - private static void testShort128ToDouble512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testShort128ToDouble512() { VectorMask mShort128 = VectorMask.fromArray(ShortVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mShort128.cast(DoubleVector.SPECIES_512).toString(), mShort128.toString()); + Asserts.assertEquals(mShort128.cast(DoubleVector.SPECIES_512).toString(), mShort128.toString()); } - private static void testShort256ToByte128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testShort256ToByte128() { VectorMask mShort256 = VectorMask.fromArray(ShortVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mShort256.cast(ByteVector.SPECIES_128).toString(), mShort256.toString()); + Asserts.assertEquals(mShort256.cast(ByteVector.SPECIES_128).toString(), mShort256.toString()); } - private static void testShort256ToInt512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testShort256ToInt512() { VectorMask mShort256 = VectorMask.fromArray(ShortVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mShort256.cast(IntVector.SPECIES_512).toString(), mShort256.toString()); + Asserts.assertEquals(mShort256.cast(IntVector.SPECIES_512).toString(), mShort256.toString()); } - private static void testShort256ToFloat512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testShort256ToFloat512() { VectorMask mShort256 = VectorMask.fromArray(ShortVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mShort256.cast(FloatVector.SPECIES_512).toString(), mShort256.toString()); + Asserts.assertEquals(mShort256.cast(FloatVector.SPECIES_512).toString(), mShort256.toString()); } - private static void testShort512ToByte256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testShort512ToByte256() { VectorMask mShort512 = VectorMask.fromArray(ShortVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mShort512.cast(ByteVector.SPECIES_256).toString(), mShort512.toString()); + Asserts.assertEquals(mShort512.cast(ByteVector.SPECIES_256).toString(), mShort512.toString()); } // Int - private static void testInt64ToLong128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"asimd", "true"}) + public static void testInt64ToLong128() { VectorMask mInt64 = VectorMask.fromArray(IntVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mInt64.cast(LongVector.SPECIES_128).toString(), mInt64.toString()); + Asserts.assertEquals(mInt64.cast(LongVector.SPECIES_128).toString(), mInt64.toString()); } - private static void testInt64ToDouble128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"asimd", "true"}) + public static void testInt64ToDouble128() { VectorMask mInt64 = VectorMask.fromArray(IntVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mInt64.cast(DoubleVector.SPECIES_128).toString(), mInt64.toString()); + Asserts.assertEquals(mInt64.cast(DoubleVector.SPECIES_128).toString(), mInt64.toString()); } - private static void testInt128ToShort64(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true"}) + public static void testInt128ToShort64() { VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mInt128.cast(ShortVector.SPECIES_64).toString(), mInt128.toString()); + Asserts.assertEquals(mInt128.cast(ShortVector.SPECIES_64).toString(), mInt128.toString()); } - private static void testInt128ToLong256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testInt128ToLong256() { VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mInt128.cast(LongVector.SPECIES_256).toString(), mInt128.toString()); + Asserts.assertEquals(mInt128.cast(LongVector.SPECIES_256).toString(), mInt128.toString()); } - private static void testInt128ToDouble256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testInt128ToDouble256() { VectorMask mInt128 = VectorMask.fromArray(IntVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mInt128.cast(DoubleVector.SPECIES_256).toString(), mInt128.toString()); + Asserts.assertEquals(mInt128.cast(DoubleVector.SPECIES_256).toString(), mInt128.toString()); } - private static void testInt256ToShort128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testInt256ToShort128() { VectorMask mInt256 = VectorMask.fromArray(IntVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mInt256.cast(ShortVector.SPECIES_128).toString(), mInt256.toString()); + Asserts.assertEquals(mInt256.cast(ShortVector.SPECIES_128).toString(), mInt256.toString()); } - private static void testInt256ToByte64(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testInt256ToByte64() { VectorMask mInt256 = VectorMask.fromArray(IntVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mInt256.cast(ByteVector.SPECIES_64).toString(), mInt256.toString()); + Asserts.assertEquals(mInt256.cast(ByteVector.SPECIES_64).toString(), mInt256.toString()); } - private static void testInt256ToLong512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testInt256ToLong512() { VectorMask mInt256 = VectorMask.fromArray(IntVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mInt256.cast(LongVector.SPECIES_512).toString(), mInt256.toString()); + Asserts.assertEquals(mInt256.cast(LongVector.SPECIES_512).toString(), mInt256.toString()); } - private static void testInt256ToDouble512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testInt256ToDouble512() { VectorMask mInt256 = VectorMask.fromArray(IntVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mInt256.cast(DoubleVector.SPECIES_512).toString(), mInt256.toString()); + Asserts.assertEquals(mInt256.cast(DoubleVector.SPECIES_512).toString(), mInt256.toString()); } - private static void testInt512ToShort256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testInt512ToShort256() { VectorMask mInt512 = VectorMask.fromArray(IntVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mInt512.cast(ShortVector.SPECIES_256).toString(), mInt512.toString()); + Asserts.assertEquals(mInt512.cast(ShortVector.SPECIES_256).toString(), mInt512.toString()); } - private static void testInt512ToByte128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testInt512ToByte128() { VectorMask mInt512 = VectorMask.fromArray(IntVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mInt512.cast(ByteVector.SPECIES_128).toString(), mInt512.toString()); + Asserts.assertEquals(mInt512.cast(ByteVector.SPECIES_128).toString(), mInt512.toString()); } // Float - private static void testFloat64ToLong128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"asimd", "true"}) + public static void testFloat64ToLong128() { VectorMask mFloat64 = VectorMask.fromArray(FloatVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mFloat64.cast(LongVector.SPECIES_128).toString(), mFloat64.toString()); + Asserts.assertEquals(mFloat64.cast(LongVector.SPECIES_128).toString(), mFloat64.toString()); } - private static void testFloat64ToDouble128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"asimd", "true"}) + public static void testFloat64ToDouble128() { VectorMask mFloat64 = VectorMask.fromArray(FloatVector.SPECIES_64, mask_arr, 0); - Assert.assertEquals(mFloat64.cast(DoubleVector.SPECIES_128).toString(), mFloat64.toString()); + Asserts.assertEquals(mFloat64.cast(DoubleVector.SPECIES_128).toString(), mFloat64.toString()); } - private static void testFloat128ToShort64(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeatureOr = {"avx2", "true", "asimd", "true"}) + public static void testFloat128ToShort64() { VectorMask mFloat128 = VectorMask.fromArray(FloatVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mFloat128.cast(ShortVector.SPECIES_64).toString(), mFloat128.toString()); + Asserts.assertEquals(mFloat128.cast(ShortVector.SPECIES_64).toString(), mFloat128.toString()); } - private static void testFloat128ToLong256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testFloat128ToLong256() { VectorMask mFloat128 = VectorMask.fromArray(FloatVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mFloat128.cast(LongVector.SPECIES_256).toString(), mFloat128.toString()); + Asserts.assertEquals(mFloat128.cast(LongVector.SPECIES_256).toString(), mFloat128.toString()); } - private static void testFloat128ToDouble256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testFloat128ToDouble256() { VectorMask mFloat128 = VectorMask.fromArray(FloatVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mFloat128.cast(DoubleVector.SPECIES_256).toString(), mFloat128.toString()); + Asserts.assertEquals(mFloat128.cast(DoubleVector.SPECIES_256).toString(), mFloat128.toString()); } - private static void testFloat256ToShort128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testFloat256ToShort128() { VectorMask mFloat256 = VectorMask.fromArray(FloatVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mFloat256.cast(ShortVector.SPECIES_128).toString(), mFloat256.toString()); + Asserts.assertEquals(mFloat256.cast(ShortVector.SPECIES_128).toString(), mFloat256.toString()); } - private static void testFloat256ToByte64(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testFloat256ToByte64() { VectorMask mFloat256 = VectorMask.fromArray(FloatVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mFloat256.cast(ByteVector.SPECIES_64).toString(), mFloat256.toString()); + Asserts.assertEquals(mFloat256.cast(ByteVector.SPECIES_64).toString(), mFloat256.toString()); } - private static void testFloat256ToLong512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testFloat256ToLong512() { VectorMask mFloat256 = VectorMask.fromArray(FloatVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mFloat256.cast(LongVector.SPECIES_512).toString(), mFloat256.toString()); + Asserts.assertEquals(mFloat256.cast(LongVector.SPECIES_512).toString(), mFloat256.toString()); } - private static void testFloat256ToDouble512(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testFloat256ToDouble512() { VectorMask mFloat256 = VectorMask.fromArray(FloatVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mFloat256.cast(DoubleVector.SPECIES_512).toString(), mFloat256.toString()); + Asserts.assertEquals(mFloat256.cast(DoubleVector.SPECIES_512).toString(), mFloat256.toString()); } - private static void testFloat512ToShort256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testFloat512ToShort256() { VectorMask mFloat512 = VectorMask.fromArray(FloatVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mFloat512.cast(ShortVector.SPECIES_256).toString(), mFloat512.toString()); + Asserts.assertEquals(mFloat512.cast(ShortVector.SPECIES_256).toString(), mFloat512.toString()); } - private static void testFloat512ToByte128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testFloat512ToByte128() { VectorMask mFloat512 = VectorMask.fromArray(FloatVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mFloat512.cast(ByteVector.SPECIES_128).toString(), mFloat512.toString()); + Asserts.assertEquals(mFloat512.cast(ByteVector.SPECIES_128).toString(), mFloat512.toString()); } // Long - private static void testLong128ToInt64(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"asimd", "true"}) + public static void testLong128ToInt64() { VectorMask mLong128 = VectorMask.fromArray(LongVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mLong128.cast(IntVector.SPECIES_64).toString(), mLong128.toString()); + Asserts.assertEquals(mLong128.cast(IntVector.SPECIES_64).toString(), mLong128.toString()); } - private static void testLong128ToFloat64(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"asimd", "true"}) + public static void testLong128ToFloat64() { VectorMask mLong128 = VectorMask.fromArray(LongVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mLong128.cast(FloatVector.SPECIES_64).toString(), mLong128.toString()); + Asserts.assertEquals(mLong128.cast(FloatVector.SPECIES_64).toString(), mLong128.toString()); } - private static void testLong256ToInt128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testLong256ToInt128() { VectorMask mLong256 = VectorMask.fromArray(LongVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mLong256.cast(IntVector.SPECIES_128).toString(), mLong256.toString()); + Asserts.assertEquals(mLong256.cast(IntVector.SPECIES_128).toString(), mLong256.toString()); } - private static void testLong256ToFloat128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testLong256ToFloat128() { VectorMask mLong256 = VectorMask.fromArray(LongVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mLong256.cast(FloatVector.SPECIES_128).toString(), mLong256.toString()); + Asserts.assertEquals(mLong256.cast(FloatVector.SPECIES_128).toString(), mLong256.toString()); } - private static void testLong256ToShort64(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testLong256ToShort64() { VectorMask mLong256 = VectorMask.fromArray(LongVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mLong256.cast(ShortVector.SPECIES_64).toString(), mLong256.toString()); + Asserts.assertEquals(mLong256.cast(ShortVector.SPECIES_64).toString(), mLong256.toString()); } - private static void testLong512ToInt256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testLong512ToInt256() { VectorMask mLong512 = VectorMask.fromArray(LongVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mLong512.cast(IntVector.SPECIES_256).toString(), mLong512.toString()); + Asserts.assertEquals(mLong512.cast(IntVector.SPECIES_256).toString(), mLong512.toString()); } - private static void testLong512ToFloat256(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testLong512ToFloat256() { VectorMask mLong512 = VectorMask.fromArray(LongVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mLong512.cast(FloatVector.SPECIES_256).toString(), mLong512.toString()); + Asserts.assertEquals(mLong512.cast(FloatVector.SPECIES_256).toString(), mLong512.toString()); } - private static void testLong512ToShort128(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testLong512ToShort128() { VectorMask mLong512 = VectorMask.fromArray(LongVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mLong512.cast(ShortVector.SPECIES_128).toString(), mLong512.toString()); + Asserts.assertEquals(mLong512.cast(ShortVector.SPECIES_128).toString(), mLong512.toString()); } - private static void testLong512ToByte64(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testLong512ToByte64() { VectorMask mLong512 = VectorMask.fromArray(LongVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mLong512.cast(ByteVector.SPECIES_64).toString(), mLong512.toString()); + Asserts.assertEquals(mLong512.cast(ByteVector.SPECIES_64).toString(), mLong512.toString()); } // Double - private static void testDouble128ToInt64(boolean[] mask_arr) { + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"asimd", "true"}) + public static void testDouble128ToInt64() { VectorMask mDouble128 = VectorMask.fromArray(DoubleVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mDouble128.cast(IntVector.SPECIES_64).toString(), mDouble128.toString()); + Asserts.assertEquals(mDouble128.cast(IntVector.SPECIES_64).toString(), mDouble128.toString()); } - private static void testDouble128ToFloat64(boolean[] mask_arr) { - VectorMask mDouble128 = VectorMask.fromArray(DoubleVector.SPECIES_128, mask_arr, 0); - Assert.assertEquals(mDouble128.cast(FloatVector.SPECIES_64).toString(), mDouble128.toString()); - } - - private static void testDouble256ToInt128(boolean[] mask_arr) { - VectorMask mDouble256 = VectorMask.fromArray(DoubleVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mDouble256.cast(IntVector.SPECIES_128).toString(), mDouble256.toString()); - } - - private static void testDouble256ToFloat128(boolean[] mask_arr) { - VectorMask mDouble256 = VectorMask.fromArray(DoubleVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mDouble256.cast(FloatVector.SPECIES_128).toString(), mDouble256.toString()); - } - - private static void testDouble256ToShort64(boolean[] mask_arr) { - VectorMask mDouble256 = VectorMask.fromArray(DoubleVector.SPECIES_256, mask_arr, 0); - Assert.assertEquals(mDouble256.cast(ShortVector.SPECIES_64).toString(), mDouble256.toString()); - }; - - private static void testDouble512ToInt256(boolean[] mask_arr) { - VectorMask mDouble512 = VectorMask.fromArray(DoubleVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mDouble512.cast(IntVector.SPECIES_256).toString(), mDouble512.toString()); - } - - private static void testDouble512ToFloat256(boolean[] mask_arr) { - VectorMask mDouble512 = VectorMask.fromArray(DoubleVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mDouble512.cast(FloatVector.SPECIES_256).toString(), mDouble512.toString()); - } - - private static void testDouble512ToShort128(boolean[] mask_arr) { - VectorMask mDouble512 = VectorMask.fromArray(DoubleVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mDouble512.cast(ShortVector.SPECIES_128).toString(), mDouble512.toString()); - } - - private static void testDouble512ToByte64(boolean[] mask_arr) { - VectorMask mDouble512 = VectorMask.fromArray(DoubleVector.SPECIES_512, mask_arr, 0); - Assert.assertEquals(mDouble512.cast(ByteVector.SPECIES_64).toString(), mDouble512.toString()); - } - - @Test - public static void testMaskCast() { - for (int i = 0; i < NUM_ITER; i++) { - boolean[] mask = genMask(); - // Byte - testByte64ToShort128(mask); - testByte64ToInt256(mask); - testByte64ToFloat256(mask); - testByte64ToLong512(mask); - testByte64ToDouble512(mask); - testByte128ToShort256(mask); - testByte128ToInt512(mask); - testByte128ToFloat512(mask); - testByte256ToShort512(mask); + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"asimd", "true"}) + public static void testDouble128ToFloat64() { + VectorMask mDouble128 = VectorMask.fromArray(DoubleVector.SPECIES_128, mask_arr, 0); + Asserts.assertEquals(mDouble128.cast(FloatVector.SPECIES_64).toString(), mDouble128.toString()); + } - // Short - testShort64ToInt128(mask); - testShort64ToFloat128(mask); - testShort64ToLong256(mask); - testShort64ToDouble256(mask); - testShort128ToByte64(mask); - testShort128ToInt256(mask); - testShort128ToFloat256(mask); - testShort128ToLong512(mask); - testShort128ToDouble512(mask); - testShort256ToByte128(mask); - testShort256ToInt512(mask); - testShort256ToFloat512(mask); - testShort512ToByte256(mask); + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testDouble256ToInt128() { + VectorMask mDouble256 = VectorMask.fromArray(DoubleVector.SPECIES_256, mask_arr, 0); + Asserts.assertEquals(mDouble256.cast(IntVector.SPECIES_128).toString(), mDouble256.toString()); + } - // Int - testInt64ToLong128(mask); - testInt64ToDouble128(mask); - testInt128ToShort64(mask); - testInt128ToLong256(mask); - testInt128ToDouble256(mask); - testInt256ToShort128(mask); - testInt256ToByte64(mask); - testInt256ToLong512(mask); - testInt256ToDouble512(mask); - testInt512ToShort256(mask); - testInt512ToByte128(mask); + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testDouble256ToFloat128() { + VectorMask mDouble256 = VectorMask.fromArray(DoubleVector.SPECIES_256, mask_arr, 0); + Asserts.assertEquals(mDouble256.cast(FloatVector.SPECIES_128).toString(), mDouble256.toString()); + } - // Float - testFloat64ToLong128(mask); - testFloat64ToDouble128(mask); - testFloat128ToShort64(mask); - testFloat128ToLong256(mask); - testFloat128ToDouble256(mask); - testFloat256ToShort128(mask); - testFloat256ToByte64(mask); - testFloat256ToLong512(mask); - testFloat256ToDouble512(mask); - testFloat512ToShort256(mask); - testFloat512ToByte128(mask); + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx2", "true"}) + public static void testDouble256ToShort64() { + VectorMask mDouble256 = VectorMask.fromArray(DoubleVector.SPECIES_256, mask_arr, 0); + Asserts.assertEquals(mDouble256.cast(ShortVector.SPECIES_64).toString(), mDouble256.toString()); + } - // Long - testLong128ToInt64(mask); - testLong128ToFloat64(mask); - testLong256ToInt128(mask); - testLong256ToFloat128(mask); - testLong256ToShort64(mask); - testLong512ToInt256(mask); - testLong512ToFloat256(mask); - testLong512ToShort128(mask); - testLong512ToByte64(mask); + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testDouble512ToInt256() { + VectorMask mDouble512 = VectorMask.fromArray(DoubleVector.SPECIES_512, mask_arr, 0); + Asserts.assertEquals(mDouble512.cast(IntVector.SPECIES_256).toString(), mDouble512.toString()); + } - // Double - testDouble128ToInt64(mask); - testDouble128ToFloat64(mask); - testDouble256ToInt128(mask); - testDouble256ToFloat128(mask); - testDouble256ToShort64(mask); - testDouble512ToInt256(mask); - testDouble512ToFloat256(mask); - testDouble512ToShort128(mask); - testDouble512ToByte64(mask); - } + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testDouble512ToFloat256() { + VectorMask mDouble512 = VectorMask.fromArray(DoubleVector.SPECIES_512, mask_arr, 0); + Asserts.assertEquals(mDouble512.cast(FloatVector.SPECIES_256).toString(), mDouble512.toString()); + } + + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testDouble512ToShort128() { + VectorMask mDouble512 = VectorMask.fromArray(DoubleVector.SPECIES_512, mask_arr, 0); + Asserts.assertEquals(mDouble512.cast(ShortVector.SPECIES_128).toString(), mDouble512.toString()); + } + + @Test + @IR(counts = { IRNode.VECTOR_MASK_CAST, "> 0" }, applyIfCPUFeature = {"avx512vl", "true"}) + public static void testDouble512ToByte64() { + VectorMask mDouble512 = VectorMask.fromArray(DoubleVector.SPECIES_512, mask_arr, 0); + Asserts.assertEquals(mDouble512.cast(ByteVector.SPECIES_64).toString(), mDouble512.toString()); + } + + public static void main(String[] args) { + TestFramework testFramework = new TestFramework(); + testFramework.setDefaultWarmup(5000) + .addFlags("--add-modules=jdk.incubator.vector") + .start(); } }