8265783: Create a separate library for x86 Intel SVML assembly intrinsics

Co-authored-by: Sandhya Viswanathan <sviswanathan@openjdk.org>
Co-authored-by: Rahul Kandu <rkandu@openjdk.org>
Co-authored-by: Razvan Lupusoru <rlupusoru@openjdk.org>
Co-authored-by: Magnus Ihse Bursie <ihse@openjdk.org>
Co-authored-by: Jie Fu <jiefu@openjdk.org>
Co-authored-by: Ahmet Akkas <ahmet.akkas@intel.com>
Co-authored-by: Marius Cornea <marius.cornea@intel.com>
Reviewed-by: erikj, kvn, psandoz

make/modules/jdk.incubator.vector/Lib.gmk

@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+include LibCommon.gmk
+
+################################################################################
+
+ifeq ($(call isTargetOs, linux windows)+$(call isTargetCpu, x86_64), true+true)
+  $(eval $(call SetupJdkLibrary, BUILD_LIBSVML, \
+      NAME := svml, \
+      CFLAGS := $(CFLAGS_JDKLIB), \
+      LDFLAGS := $(LDFLAGS_JDKLIB) \
+          $(call SET_SHARED_LIBRARY_ORIGIN), \
+      LDFLAGS_windows := -defaultlib:msvcrt, \
+  ))
+
+  TARGETS += $(BUILD_LIBSVML)
+endif
+
+################################################################################

src/hotspot/cpu/aarch64/aarch64.ad

@@ -2428,6 +2428,16 @@ const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
   return new TypeVectMask(elemTy, length);
 }
 
+// Vector calling convention not yet implemented.
+const bool Matcher::supports_vector_calling_convention(void) {
+  return false;
+}
+
+OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
+  Unimplemented();
+  return OptoRegPair(0, 0);
+}
+
 const int Matcher::float_pressure(int default_pressure_threshold) {
   return default_pressure_threshold;
 }

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

@@ -864,6 +864,13 @@ static int c_calling_convention_priv(const BasicType *sig_bt,
   return stk_args;
 }
 
+int SharedRuntime::vector_calling_convention(VMRegPair *regs,
+                                             uint num_bits,
+                                             uint total_args_passed) {
+  Unimplemented();
+  return 0;
+}
+
 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
                                          VMRegPair *regs,
                                          VMRegPair *regs2,

src/hotspot/cpu/arm/arm.ad

@@ -991,6 +991,16 @@ const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
   return NULL;
 }
 
+// Vector calling convention not yet implemented.
+const bool Matcher::supports_vector_calling_convention(void) {
+  return false;
+}
+
+OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
+  Unimplemented();
+  return OptoRegPair(0, 0);
+}
+
 const int Matcher::float_pressure(int default_pressure_threshold) {
   return default_pressure_threshold;
 }

src/hotspot/cpu/arm/sharedRuntime_arm.cpp

@@ -354,6 +354,13 @@ int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
   return slot;
 }
 
+int SharedRuntime::vector_calling_convention(VMRegPair *regs,
+                                             uint num_bits,
+                                             uint total_args_passed) {
+  Unimplemented();
+  return 0;
+}
+
 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
                                            VMRegPair *regs,
                                            int total_args_passed) {

src/hotspot/cpu/ppc/ppc.ad

@@ -2183,6 +2183,16 @@ const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
   return NULL;
 }
 
+// Vector calling convention not yet implemented.
+const bool Matcher::supports_vector_calling_convention(void) {
+  return false;
+}
+
+OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
+  Unimplemented();
+  return OptoRegPair(0, 0);
+}
+
 const int Matcher::float_pressure(int default_pressure_threshold) {
   return default_pressure_threshold;
 }

src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp

@@ -917,6 +917,13 @@ int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
 }
 #endif // COMPILER2
 
+int SharedRuntime::vector_calling_convention(VMRegPair *regs,
+                                             uint num_bits,
+                                             uint total_args_passed) {
+  Unimplemented();
+  return 0;
+}
+
 static address gen_c2i_adapter(MacroAssembler *masm,
                             int total_args_passed,
                             int comp_args_on_stack,

src/hotspot/cpu/s390/s390.ad

@@ -1544,6 +1544,16 @@ const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
   return NULL;
 }
 
+// Vector calling convention not yet implemented.
+const bool Matcher::supports_vector_calling_convention(void) {
+  return false;
+}
+
+OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
+  Unimplemented();
+  return OptoRegPair(0, 0);
+}
+
 const int Matcher::float_pressure(int default_pressure_threshold) {
   return default_pressure_threshold;
 }

src/hotspot/cpu/s390/sharedRuntime_s390.cpp

@@ -852,6 +852,13 @@ int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
   return align_up(stk, 2);
 }
 
+int SharedRuntime::vector_calling_convention(VMRegPair *regs,
+                                             uint num_bits,
+                                             uint total_args_passed) {
+  Unimplemented();
+  return 0;
+}
+
 ////////////////////////////////////////////////////////////////////////
 //
 //  Argument shufflers

src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp

@@ -1044,6 +1044,13 @@ int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
   return stack;
 }
 
+int SharedRuntime::vector_calling_convention(VMRegPair *regs,
+                                             uint num_bits,
+                                             uint total_args_passed) {
+  Unimplemented();
+  return 0;
+}
+
 // A simple move of integer like type
 static void simple_move32(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
   if (src.first()->is_stack()) {

src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp

@@ -1149,6 +1149,31 @@ int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
   return stk_args;
 }
 
+int SharedRuntime::vector_calling_convention(VMRegPair *regs,
+                                             uint num_bits,
+                                             uint total_args_passed) {
+  assert(num_bits == 64 || num_bits == 128 || num_bits == 256 || num_bits == 512,
+         "only certain vector sizes are supported for now");
+
+  static const XMMRegister VEC_ArgReg[32] = {
+     xmm0,  xmm1,  xmm2,  xmm3,  xmm4,  xmm5,  xmm6,  xmm7,
+     xmm8,  xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,
+    xmm16, xmm17, xmm18, xmm19, xmm20, xmm21, xmm22, xmm23,
+    xmm24, xmm25, xmm26, xmm27, xmm28, xmm29, xmm30, xmm31
+  };
+
+  uint stk_args = 0;
+  uint fp_args = 0;
+
+  for (uint i = 0; i < total_args_passed; i++) {
+    VMReg vmreg = VEC_ArgReg[fp_args++]->as_VMReg();
+    int next_val = num_bits == 64 ? 1 : (num_bits == 128 ? 3 : (num_bits  == 256 ? 7 : 15));
+    regs[i].set_pair(vmreg->next(next_val), vmreg);
+  }
+
+  return stk_args;
+}
+
 void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
   // We always ignore the frame_slots arg and just use the space just below frame pointer
   // which by this time is free to use

src/hotspot/cpu/x86/stubGenerator_x86_64.cpp

@@ -7002,6 +7002,67 @@ address generate_avx_ghash_processBlocks() {
       StubRoutines::_montgomerySquare
         = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square);
     }
+
+    // Get svml stub routine addresses
+    void *libsvml = NULL;
+    char ebuf[1024];
+    libsvml = os::dll_load(JNI_LIB_PREFIX "svml" JNI_LIB_SUFFIX, ebuf, sizeof ebuf);
+    if (libsvml != NULL) {
+      // SVML method naming convention
+      //   All the methods are named as __svml_op<T><N>_ha_<VV>
+      //   Where:
+      //      ha stands for high accuracy
+      //      <T> is optional to indicate float/double
+      //              Set to f for vector float operation
+      //              Omitted for vector double operation
+      //      <N> is the number of elements in the vector
+      //              1, 2, 4, 8, 16
+      //              e.g. 128 bit float vector has 4 float elements
+      //      <VV> indicates the avx/sse level:
+      //              z0 is AVX512, l9 is AVX2, e9 is AVX1 and ex is for SSE2
+      //      e.g. __svml_expf16_ha_z0 is the method for computing 16 element vector float exp using AVX 512 insns
+      //           __svml_exp8_ha_z0 is the method for computing 8 element vector double exp using AVX 512 insns
+
+      log_info(library)("Loaded library %s, handle " INTPTR_FORMAT, JNI_LIB_PREFIX "svml" JNI_LIB_SUFFIX, p2i(libsvml));
+      if (UseAVX > 2) {
+        for (int op = 0; op < VectorSupport::NUM_SVML_OP; op++) {
+          int vop = VectorSupport::VECTOR_OP_SVML_START + op;
+          if ((!VM_Version::supports_avx512dq()) &&
+              (vop == VectorSupport::VECTOR_OP_LOG || vop == VectorSupport::VECTOR_OP_LOG10 || vop == VectorSupport::VECTOR_OP_POW)) {
+            continue;
+          }
+          snprintf(ebuf, sizeof(ebuf), "__svml_%sf16_ha_z0", VectorSupport::svmlname[op]);
+          StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libsvml, ebuf);
+
+          snprintf(ebuf, sizeof(ebuf), "__svml_%s8_ha_z0", VectorSupport::svmlname[op]);
+          StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_512][op] = (address)os::dll_lookup(libsvml, ebuf);
+        }
+      }
+      const char* avx_sse_str = (UseAVX >= 2) ? "l9" : ((UseAVX == 1) ? "e9" : "ex");
+      for (int op = 0; op < VectorSupport::NUM_SVML_OP; op++) {
+        int vop = VectorSupport::VECTOR_OP_SVML_START + op;
+        if (vop == VectorSupport::VECTOR_OP_POW) {
+          continue;
+        }
+        snprintf(ebuf, sizeof(ebuf), "__svml_%sf4_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
+        StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libsvml, ebuf);
+
+        snprintf(ebuf, sizeof(ebuf), "__svml_%sf4_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
+        StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libsvml, ebuf);
+
+        snprintf(ebuf, sizeof(ebuf), "__svml_%sf8_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
+        StubRoutines::_vector_f_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libsvml, ebuf);
+
+        snprintf(ebuf, sizeof(ebuf), "__svml_%s1_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
+        StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_64][op] = (address)os::dll_lookup(libsvml, ebuf);
+
+        snprintf(ebuf, sizeof(ebuf), "__svml_%s2_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
+        StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_128][op] = (address)os::dll_lookup(libsvml, ebuf);
+
+        snprintf(ebuf, sizeof(ebuf), "__svml_%s4_ha_%s", VectorSupport::svmlname[op], avx_sse_str);
+        StubRoutines::_vector_d_math[VectorSupport::VEC_SIZE_256][op] = (address)os::dll_lookup(libsvml, ebuf);
+      }
+    }
 #endif // COMPILER2
 
     if (UseVectorizedMismatchIntrinsic) {

src/hotspot/cpu/x86/x86.ad

@@ -1707,6 +1707,11 @@ const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType
           }
       }
       break;
+    case Op_CallLeafVector:
+      if (size_in_bits == 512 && !VM_Version::supports_avx512vlbwdq()) {
+        return false;
+      }
+      break;
     case Op_AddReductionVI:
       if (bt == T_INT && (UseSSE < 3 || !VM_Version::supports_ssse3())) {
         return false;
@@ -1954,6 +1959,10 @@ const int Matcher::min_vector_size(const BasicType bt) {
   int max_size = max_vector_size(bt);
   // Min size which can be loaded into vector is 4 bytes.
   int size = (type2aelembytes(bt) == 1) ? 4 : 2;
+  // Support for calling svml double64 vectors
+  if (bt == T_DOUBLE) {
+    size = 1;
+  }
   return MIN2(size,max_size);
 }
 

src/hotspot/cpu/x86/x86_32.ad

@@ -1398,6 +1398,16 @@ uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
 
 //=============================================================================
 
+// Vector calling convention not supported.
+const bool Matcher::supports_vector_calling_convention() {
+  return false;
+}
+
+OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
+  Unimplemented();
+  return OptoRegPair(0, 0);
+}
+
 // Is this branch offset short enough that a short branch can be used?
 //
 // NOTE: If the platform does not provide any short branch variants, then

src/hotspot/cpu/x86/x86_64.ad

@@ -463,7 +463,9 @@ int MachCallDynamicJavaNode::ret_addr_offset()
 
 int MachCallRuntimeNode::ret_addr_offset() {
   int offset = 13; // movq r10,#addr; callq (r10)
-  offset += clear_avx_size();
+  if (this->ideal_Opcode() != Op_CallLeafVector) {
+    offset += clear_avx_size();
+  }
   return offset;
 }
 
@@ -1692,6 +1694,23 @@ uint MachUEPNode::size(PhaseRegAlloc* ra_) const
 
 //=============================================================================
 
+const bool Matcher::supports_vector_calling_convention(void) {
+  if (EnableVectorSupport && UseVectorStubs) {
+    return true;
+  }
+  return false;
+}
+
+OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
+  assert(EnableVectorSupport && UseVectorStubs, "sanity");
+  int lo = XMM0_num;
+  int hi = XMM0b_num;
+  if (ideal_reg == Op_VecX) hi = XMM0d_num;
+  else if (ideal_reg == Op_VecY) hi = XMM0h_num;
+  else if (ideal_reg == Op_VecZ) hi = XMM0p_num;
+  return OptoRegPair(hi, lo);
+}
+
 // Is this branch offset short enough that a short branch can be used?
 //
 // NOTE: If the platform does not provide any short branch variants, then
@@ -12997,6 +13016,18 @@ instruct CallLeafDirect(method meth)
   ins_pipe(pipe_slow);
 %}
 
+// Call runtime without safepoint and with vector arguments
+instruct CallLeafDirectVector(method meth)
+%{
+  match(CallLeafVector);
+  effect(USE meth);
+
+  ins_cost(300);
+  format %{ "call_leaf,vector " %}
+  ins_encode(Java_To_Runtime(meth));
+  ins_pipe(pipe_slow);
+%}
+
 //
 instruct CallNativeDirect(method meth)
 %{

src/hotspot/cpu/zero/sharedRuntime_zero.cpp

@@ -125,3 +125,10 @@ int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
   ShouldNotCallThis();
   return 0;
 }
+
+int SharedRuntime::vector_calling_convention(VMRegPair *regs,
+                                             uint num_bits,
+                                             uint total_args_passed) {
+  ShouldNotCallThis();
+  return 0;
+}

src/hotspot/share/adlc/formssel.cpp

@@ -419,6 +419,8 @@ Form::CallType InstructForm::is_ideal_call() const {
   idx = 0;
   if(_matrule->find_type("CallLeafNoFP",idx))     return Form::JAVA_LEAF;
   idx = 0;
+  if(_matrule->find_type("CallLeafVector",idx))   return Form::JAVA_LEAF;
+  idx = 0;
   if(_matrule->find_type("CallNative",idx))       return Form::JAVA_NATIVE;
   idx = 0;
 

src/hotspot/share/classfile/modules.cpp

@@ -464,9 +464,13 @@ void Modules::define_module(Handle module, jboolean is_open, jstring version,
     if (EnableVectorSupport && EnableVectorReboxing && FLAG_IS_DEFAULT(EnableVectorAggressiveReboxing)) {
       FLAG_SET_DEFAULT(EnableVectorAggressiveReboxing, true);
     }
+    if (EnableVectorSupport && FLAG_IS_DEFAULT(UseVectorStubs)) {
+      FLAG_SET_DEFAULT(UseVectorStubs, true);
+    }
     log_info(compilation)("EnableVectorSupport=%s",            (EnableVectorSupport            ? "true" : "false"));
     log_info(compilation)("EnableVectorReboxing=%s",           (EnableVectorReboxing           ? "true" : "false"));
     log_info(compilation)("EnableVectorAggressiveReboxing=%s", (EnableVectorAggressiveReboxing ? "true" : "false"));
+    log_info(compilation)("UseVectorStubs=%s",                 (UseVectorStubs                 ? "true" : "false"));
   }
 #endif // COMPILER2
 }

src/hotspot/share/opto/c2_globals.hpp

@@ -721,6 +721,9 @@
   product(bool, EnableVectorAggressiveReboxing, false, EXPERIMENTAL,        \
           "Enables aggressive reboxing of vectors")                         \
                                                                             \
+  product(bool, UseVectorStubs, false, EXPERIMENTAL,                        \
+          "Use stubs for vector transcendental operations")                 \
+                                                                            \
   product(bool, UseTypeSpeculation, true,                                   \
           "Speculatively propagate types from profiles")                    \
                                                                             \

src/hotspot/share/opto/callnode.cpp

@@ -736,10 +736,24 @@ Node *CallNode::match( const ProjNode *proj, const Matcher *match ) {
 
   case TypeFunc::Parms: {       // Normal returns
     uint ideal_reg = tf()->range()->field_at(TypeFunc::Parms)->ideal_reg();
-    OptoRegPair regs = is_CallRuntime()
-      ? match->c_return_value(ideal_reg)  // Calls into C runtime
-      : match->  return_value(ideal_reg); // Calls into compiled Java code
+    OptoRegPair regs = Opcode() == Op_CallLeafVector
+      ? match->vector_return_value(ideal_reg)      // Calls into assembly vector routine
+      : is_CallRuntime()
+        ? match->c_return_value(ideal_reg)  // Calls into C runtime
+        : match->  return_value(ideal_reg); // Calls into compiled Java code
     RegMask rm = RegMask(regs.first());
+
+    if (Opcode() == Op_CallLeafVector) {
+      // If the return is in vector, compute appropriate regmask taking into account the whole range
+      if(ideal_reg >= Op_VecS && ideal_reg <= Op_VecZ) {
+        if(OptoReg::is_valid(regs.second())) {
+          for (OptoReg::Name r = regs.first(); r <= regs.second(); r = OptoReg::add(r, 1)) {
+            rm.Insert(r);
+          }
+        }
+      }
+    }
+
     if( OptoReg::is_valid(regs.second()) )
       rm.Insert( regs.second() );
     return new MachProjNode(this,proj->_con,rm,ideal_reg);
@@ -1201,6 +1215,11 @@ void CallRuntimeNode::dump_spec(outputStream *st) const {
   CallNode::dump_spec(st);
 }
 #endif
+uint CallLeafVectorNode::size_of() const { return sizeof(*this); }
+bool CallLeafVectorNode::cmp( const Node &n ) const {
+  CallLeafVectorNode &call = (CallLeafVectorNode&)n;
+  return CallLeafNode::cmp(call) && _num_bits == call._num_bits;
+}
 
 //=============================================================================
 uint CallNativeNode::size_of() const { return sizeof(*this); }
@@ -1272,6 +1291,21 @@ void CallRuntimeNode::calling_convention(BasicType* sig_bt, VMRegPair *parm_regs
   SharedRuntime::c_calling_convention(sig_bt, parm_regs, /*regs2=*/nullptr, argcnt);
 }
 
+void CallLeafVectorNode::calling_convention( BasicType* sig_bt, VMRegPair *parm_regs, uint argcnt ) const {
+#ifdef ASSERT
+  assert(tf()->range()->field_at(TypeFunc::Parms)->is_vect()->length_in_bytes() * BitsPerByte == _num_bits,
+         "return vector size must match");
+  const TypeTuple* d = tf()->domain();
+  for (uint i = TypeFunc::Parms; i < d->cnt(); i++) {
+    Node* arg = in(i);
+    assert(arg->bottom_type()->is_vect()->length_in_bytes() * BitsPerByte == _num_bits,
+           "vector argument size must match");
+  }
+#endif
+
+  SharedRuntime::vector_calling_convention(parm_regs, _num_bits, argcnt);
+}
+
 void CallNativeNode::calling_convention( BasicType* sig_bt, VMRegPair *parm_regs, uint argcnt ) const {
   assert((tf()->domain()->cnt() - TypeFunc::Parms) == argcnt, "arg counts must match!");
 #ifdef ASSERT

src/hotspot/share/opto/callnode.hpp

@@ -48,6 +48,7 @@ class       CallDynamicJavaNode;
 class     CallRuntimeNode;
 class       CallLeafNode;
 class         CallLeafNoFPNode;
+class         CallLeafVectorNode;
 class     CallNativeNode;
 class     AllocateNode;
 class       AllocateArrayNode;
@@ -785,6 +786,7 @@ public:
 //------------------------------CallRuntimeNode--------------------------------
 // Make a direct subroutine call node into compiled C++ code.
 class CallRuntimeNode : public CallNode {
+protected:
   virtual bool cmp( const Node &n ) const;
   virtual uint size_of() const; // Size is bigger
 public:
@@ -872,6 +874,24 @@ public:
   virtual int   Opcode() const;
 };
 
+//------------------------------CallLeafVectorNode-------------------------------
+// CallLeafNode but calling with vector calling convention instead.
+class CallLeafVectorNode : public CallLeafNode {
+private:
+  uint _num_bits;
+protected:
+  virtual bool cmp( const Node &n ) const;
+  virtual uint size_of() const; // Size is bigger
+public:
+  CallLeafVectorNode(const TypeFunc* tf, address addr, const char* name,
+                   const TypePtr* adr_type, uint num_bits)
+    : CallLeafNode(tf, addr, name, adr_type), _num_bits(num_bits)
+  {
+  }
+  virtual int   Opcode() const;
+  virtual void  calling_convention( BasicType* sig_bt, VMRegPair *parm_regs, uint argcnt ) const;
+};
+
 
 //------------------------------Allocate---------------------------------------
 // High-level memory allocation

src/hotspot/share/opto/classes.hpp

@@ -59,6 +59,7 @@ macro(CallDynamicJava)
 macro(CallJava)
 macro(CallLeaf)
 macro(CallLeafNoFP)
+macro(CallLeafVector)
 macro(CallRuntime)
 macro(CallNative)
 macro(CallStaticJava)

src/hotspot/share/opto/compile.cpp

@@ -2967,6 +2967,7 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f
     frc.inc_java_call_count(); // Count java call site;
   case Op_CallRuntime:
   case Op_CallLeaf:
+  case Op_CallLeafVector:
   case Op_CallNative:
   case Op_CallLeafNoFP: {
     assert (n->is_Call(), "");

src/hotspot/share/opto/escape.cpp

@@ -1024,6 +1024,7 @@ void ConnectionGraph::process_call_arguments(CallNode *call) {
       is_arraycopy = (call->Opcode() == Op_ArrayCopy) ||
         call->as_CallLeaf()->is_call_to_arraycopystub();
       // fall through
+    case Op_CallLeafVector:
     case Op_CallLeaf: {
       // Stub calls, objects do not escape but they are not scale replaceable.
       // Adjust escape state for outgoing arguments.

src/hotspot/share/opto/graphKit.cpp

@@ -2487,6 +2487,9 @@ Node* GraphKit::make_runtime_call(int flags,
     call = new CallStaticJavaNode(call_type, call_addr, call_name, adr_type);
   } else if (flags & RC_NO_FP) {
     call = new CallLeafNoFPNode(call_type, call_addr, call_name, adr_type);
+  } else  if (flags & RC_VECTOR){
+    uint num_bits = call_type->range()->field_at(TypeFunc::Parms)->is_vect()->length_in_bytes() * BitsPerByte;
+    call = new CallLeafVectorNode(call_type, call_addr, call_name, adr_type, num_bits);
   } else {
     call = new CallLeafNode(call_type, call_addr, call_name, adr_type);
   }

src/hotspot/share/opto/graphKit.hpp

@@ -810,6 +810,7 @@ class GraphKit : public Phase {
     RC_MUST_THROW = 8,          // flag passed to add_safepoint_edges
     RC_NARROW_MEM = 16,         // input memory is same as output
     RC_UNCOMMON = 32,           // freq. expected to be like uncommon trap
+    RC_VECTOR = 64,             // CallLeafVectorNode
     RC_LEAF = 0                 // null value:  no flags set
   };
 

src/hotspot/share/opto/lcm.cpp

@@ -870,6 +870,7 @@ uint PhaseCFG::sched_call(Block* block, uint node_cnt, Node_List& worklist, Grow
     case Op_CallRuntime:
     case Op_CallLeaf:
     case Op_CallLeafNoFP:
+    case Op_CallLeafVector:
       // Calling C code so use C calling convention
       save_policy = _matcher._c_reg_save_policy;
       break;

src/hotspot/share/opto/library_call.hpp

@@ -326,6 +326,7 @@ class LibraryCallKit : public GraphKit {
   bool inline_vector_convert();
   bool inline_vector_extract();
   bool inline_vector_insert();
+  Node* gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2);
 
   enum VectorMaskUseType {
     VecMaskUseLoad,

src/hotspot/share/opto/matcher.cpp

@@ -1365,16 +1365,27 @@ MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) {
     for( i = 0; i < argcnt; i++ ) {
       // Address of incoming argument mask to fill in
       RegMask *rm = &mcall->_in_rms[i+TypeFunc::Parms];
-      if( !parm_regs[i].first()->is_valid() &&
-          !parm_regs[i].second()->is_valid() ) {
+      VMReg first = parm_regs[i].first();
+      VMReg second = parm_regs[i].second();
+      if(!first->is_valid() &&
+         !second->is_valid()) {
         continue;               // Avoid Halves
       }
+      // Handle case where arguments are in vector registers.
+      if(call->in(TypeFunc::Parms + i)->bottom_type()->isa_vect()) {
+        OptoReg::Name reg_fst = OptoReg::as_OptoReg(first);
+        OptoReg::Name reg_snd = OptoReg::as_OptoReg(second);
+        assert (reg_fst <= reg_snd, "fst=%d snd=%d", reg_fst, reg_snd);
+        for (OptoReg::Name r = reg_fst; r <= reg_snd; r++) {
+          rm->Insert(r);
+        }
+      }
       // Grab first register, adjust stack slots and insert in mask.
-      OptoReg::Name reg1 = warp_outgoing_stk_arg(parm_regs[i].first(), begin_out_arg_area, out_arg_limit_per_call );
+      OptoReg::Name reg1 = warp_outgoing_stk_arg(first, begin_out_arg_area, out_arg_limit_per_call );
       if (OptoReg::is_valid(reg1))
         rm->Insert( reg1 );
       // Grab second register (if any), adjust stack slots and insert in mask.
-      OptoReg::Name reg2 = warp_outgoing_stk_arg(parm_regs[i].second(), begin_out_arg_area, out_arg_limit_per_call );
+      OptoReg::Name reg2 = warp_outgoing_stk_arg(second, begin_out_arg_area, out_arg_limit_per_call );
       if (OptoReg::is_valid(reg2))
         rm->Insert( reg2 );
     } // End of for all arguments

src/hotspot/share/opto/matcher.hpp

@@ -415,6 +415,10 @@ public:
   OptoReg::Name  c_frame_pointer() const;
   static RegMask c_frame_ptr_mask;
 
+  // Java-Native vector calling convention
+  static const bool supports_vector_calling_convention();
+  static OptoRegPair vector_return_value(uint ideal_reg);
+
   // Is this branch offset small enough to be addressed by a short branch?
   bool is_short_branch_offset(int rule, int br_size, int offset);
 

src/hotspot/share/opto/runtime.cpp

@@ -663,6 +663,25 @@ const TypeFunc *OptoRuntime::Math_D_D_Type() {
   return TypeFunc::make(domain, range);
 }
 
+const TypeFunc *OptoRuntime::Math_Vector_Vector_Type(uint num_arg, const TypeVect* in_type, const TypeVect* out_type) {
+  // create input type (domain)
+  const Type **fields = TypeTuple::fields(num_arg);
+  // Symbol* name of class to be loaded
+  assert(num_arg > 0, "must have at least 1 input");
+  for (uint i = 0; i < num_arg; i++) {
+    fields[TypeFunc::Parms+i] = in_type;
+  }
+  const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+num_arg, fields);
+
+  // create result type (range)
+  const uint num_ret = 1;
+  fields = TypeTuple::fields(num_ret);
+  fields[TypeFunc::Parms+0] = out_type;
+  const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+num_ret, fields);
+
+  return TypeFunc::make(domain, range);
+}
+
 const TypeFunc* OptoRuntime::Math_DD_D_Type() {
   const Type **fields = TypeTuple::fields(4);
   fields[TypeFunc::Parms+0] = Type::DOUBLE;

src/hotspot/share/opto/runtime.hpp

@@ -256,6 +256,7 @@ private:
   static const TypeFunc* rethrow_Type();
   static const TypeFunc* Math_D_D_Type();  // sin,cos & friends
   static const TypeFunc* Math_DD_D_Type(); // mod,pow & friends
+  static const TypeFunc* Math_Vector_Vector_Type(uint num_arg, const TypeVect* in_type, const TypeVect* out_type);
   static const TypeFunc* modf_Type();
   static const TypeFunc* l2f_Type();
   static const TypeFunc* void_long_Type();

src/hotspot/share/opto/vectorIntrinsics.cpp

@@ -29,6 +29,7 @@
 #include "opto/runtime.hpp"
 #include "opto/vectornode.hpp"
 #include "prims/vectorSupport.hpp"
+#include "runtime/stubRoutines.hpp"
 
 #ifdef ASSERT
 static bool is_vector(ciKlass* klass) {
@@ -124,6 +125,30 @@ bool LibraryCallKit::arch_supports_vector(int sopc, int num_elem, BasicType type
     assert(Matcher::match_rule_supported(sopc), "must be supported");
   }
 
+  if (num_elem == 1) {
+    if (mask_use_type != VecMaskNotUsed) {
+#ifndef PRODUCT
+      if (C->print_intrinsics()) {
+        tty->print_cr("  ** Rejected vector mask op (%s,%s,%d) because architecture does not support it",
+                      NodeClassNames[sopc], type2name(type), num_elem);
+      }
+#endif
+      return false;
+    }
+
+    if (sopc != 0) {
+      if (sopc != Op_LoadVector && sopc != Op_StoreVector) {
+#ifndef PRODUCT
+        if (C->print_intrinsics()) {
+          tty->print_cr("  ** Not a svml call or load/store vector op (%s,%s,%d)",
+                        NodeClassNames[sopc], type2name(type), num_elem);
+        }
+#endif
+        return false;
+      }
+    }
+  }
+
   if (!has_scalar_args && VectorNode::is_vector_shift(sopc) &&
       Matcher::supports_vector_variable_shifts() == false) {
     if (C->print_intrinsics()) {
@@ -231,17 +256,49 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) {
   int num_elem = vlen->get_con();
   int opc = VectorSupport::vop2ideal(opr->get_con(), elem_bt);
   int sopc = VectorNode::opcode(opc, elem_bt);
-  if (sopc == 0) {
+  if ((opc != Op_CallLeafVector) && (sopc == 0)) {
     if (C->print_intrinsics()) {
       tty->print_cr("  ** operation not supported: opc=%s bt=%s", NodeClassNames[opc], type2name(elem_bt));
     }
     return false; // operation not supported
   }
+  if (num_elem == 1) {
+    if (opc != Op_CallLeafVector || elem_bt != T_DOUBLE) {
+      if (C->print_intrinsics()) {
+        tty->print_cr("  ** not a svml call: arity=%d opc=%d vlen=%d etype=%s",
+                      n, opc, num_elem, type2name(elem_bt));
+      }
+      return false;
+    }
+  }
   ciKlass* vbox_klass = vector_klass->const_oop()->as_instance()->java_lang_Class_klass();
   const TypeInstPtr* vbox_type = TypeInstPtr::make_exact(TypePtr::NotNull, vbox_klass);
 
+  if (opc == Op_CallLeafVector) {
+    if (!UseVectorStubs) {
+      if (C->print_intrinsics()) {
+        tty->print_cr("  ** vector stubs support is disabled");
+      }
+      return false;
+    }
+    if (!Matcher::supports_vector_calling_convention()) {
+      if (C->print_intrinsics()) {
+        tty->print_cr("  ** no vector calling conventions supported");
+      }
+      return false;
+    }
+    if (!Matcher::vector_size_supported(elem_bt, num_elem)) {
+      if (C->print_intrinsics()) {
+        tty->print_cr("  ** vector size (vlen=%d, etype=%s) is not supported",
+                      num_elem, type2name(elem_bt));
+      }
+      return false;
+    }
+  }
+
   // TODO When mask usage is supported, VecMaskNotUsed needs to be VecMaskUseLoad.
-  if (!arch_supports_vector(sopc, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseAll : VecMaskNotUsed)) {
+  if ((sopc != 0) &&
+      !arch_supports_vector(sopc, num_elem, elem_bt, is_vector_mask(vbox_klass) ? VecMaskUseAll : VecMaskNotUsed)) {
     if (C->print_intrinsics()) {
       tty->print_cr("  ** not supported: arity=%d opc=%d vlen=%d etype=%s ismask=%d",
                     n, sopc, num_elem, type2name(elem_bt),
@@ -289,18 +346,32 @@ bool LibraryCallKit::inline_vector_nary_operation(int n) {
   }
 
   Node* operation = NULL;
-  const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
-  switch (n) {
-    case 1:
-    case 2: {
-      operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, vt));
-      break;
+  if (opc == Op_CallLeafVector) {
+    assert(UseVectorStubs, "sanity");
+    operation = gen_call_to_svml(opr->get_con(), elem_bt, num_elem, opd1, opd2);
+    if (operation == NULL) {
+      if (C->print_intrinsics()) {
+        tty->print_cr("  ** svml call failed for %s_%s_%d",
+                         (elem_bt == T_FLOAT)?"float":"double",
+                         VectorSupport::svmlname[opr->get_con() - VectorSupport::VECTOR_OP_SVML_START],
+                         num_elem * type2aelembytes(elem_bt));
+      }
+      return false;
+     }
+  } else {
+    const TypeVect* vt = TypeVect::make(elem_bt, num_elem);
+    switch (n) {
+      case 1:
+      case 2: {
+        operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, vt));
+        break;
+      }
+      case 3: {
+        operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, opd3, vt));
+        break;
+      }
+      default: fatal("unsupported arity: %d", n);
     }
-    case 3: {
-      operation = gvn().transform(VectorNode::make(sopc, opd1, opd2, opd3, vt));
-      break;
-    }
-    default: fatal("unsupported arity: %d", n);
   }
   // Wrap it up in VectorBox to keep object type information.
   Node* vbox = box_vector(operation, vbox_type, elem_bt, num_elem);
@@ -1330,6 +1401,63 @@ bool LibraryCallKit::inline_vector_rearrange() {
   return true;
 }
 
+static address get_svml_address(int vop, int bits, BasicType bt, char* name_ptr, int name_len) {
+  address addr = NULL;
+  assert(UseVectorStubs, "sanity");
+  assert(name_ptr != NULL, "unexpected");
+  assert((vop >= VectorSupport::VECTOR_OP_SVML_START) && (vop <= VectorSupport::VECTOR_OP_SVML_END), "unexpected");
+  int op = vop - VectorSupport::VECTOR_OP_SVML_START;
+
+  switch(bits) {
+    case 64:  //fallthough
+    case 128: //fallthough
+    case 256: //fallthough
+    case 512:
+      if (bt == T_FLOAT) {
+        snprintf(name_ptr, name_len, "vector_%s_float%d", VectorSupport::svmlname[op], bits);
+        addr = StubRoutines::_vector_f_math[exact_log2(bits/64)][op];
+      } else {
+        assert(bt == T_DOUBLE, "must be FP type only");
+        snprintf(name_ptr, name_len, "vector_%s_double%d", VectorSupport::svmlname[op], bits);
+        addr = StubRoutines::_vector_d_math[exact_log2(bits/64)][op];
+      }
+      break;
+    default:
+      snprintf(name_ptr, name_len, "invalid");
+      addr = NULL;
+      Unimplemented();
+      break;
+  }
+
+  return addr;
+}
+
+Node* LibraryCallKit::gen_call_to_svml(int vector_api_op_id, BasicType bt, int num_elem, Node* opd1, Node* opd2) {
+  assert(UseVectorStubs, "sanity");
+  assert(vector_api_op_id >= VectorSupport::VECTOR_OP_SVML_START && vector_api_op_id <= VectorSupport::VECTOR_OP_SVML_END, "need valid op id");
+  assert(opd1 != NULL, "must not be null");
+  const TypeVect* vt = TypeVect::make(bt, num_elem);
+  const TypeFunc* call_type = OptoRuntime::Math_Vector_Vector_Type(opd2 != NULL ? 2 : 1, vt, vt);
+  char name[100] = "";
+
+  // Get address for svml method.
+  address addr = get_svml_address(vector_api_op_id, vt->length_in_bytes() * BitsPerByte, bt, name, 100);
+
+  if (addr == NULL) {
+    return NULL;
+  }
+
+  assert(name != NULL, "name must not be null");
+  Node* operation = make_runtime_call(RC_VECTOR,
+                                      call_type,
+                                      addr,
+                                      name,
+                                      TypePtr::BOTTOM,
+                                      opd1,
+                                      opd2);
+  return gvn().transform(new ProjNode(gvn().transform(operation), TypeFunc::Parms));
+}
+
 //  public static
 //  <V extends Vector<?,?>>
 //  V broadcastInt(int opr, Class<V> vectorClass, Class<?> elementType, int vlen,

src/hotspot/share/prims/vectorSupport.cpp

@@ -32,6 +32,7 @@
 #include "oops/klass.inline.hpp"
 #include "prims/vectorSupport.hpp"
 #include "runtime/fieldDescriptor.inline.hpp"
+#include "runtime/frame.inline.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/jniHandles.inline.hpp"
@@ -41,6 +42,29 @@
 #include "opto/matcher.hpp" // Matcher::max_vector_size(BasicType)
 #endif // COMPILER2
 
+#ifdef COMPILER2
+const char* VectorSupport::svmlname[VectorSupport::NUM_SVML_OP] = {
+    "tan",
+    "tanh",
+    "sin",
+    "sinh",
+    "cos",
+    "cosh",
+    "asin",
+    "acos",
+    "atan",
+    "atan2",
+    "cbrt",
+    "log",
+    "log10",
+    "log1p",
+    "pow",
+    "exp",
+    "expm1",
+    "hypot",
+};
+#endif
+
 bool VectorSupport::is_vector(Klass* klass) {
   return klass->is_subclass_of(vmClasses::vector_VectorPayload_klass());
 }
@@ -385,6 +409,25 @@ int VectorSupport::vop2ideal(jint id, BasicType bt) {
       }
       break;
     }
+    case VECTOR_OP_TAN:
+    case VECTOR_OP_TANH:
+    case VECTOR_OP_SIN:
+    case VECTOR_OP_SINH:
+    case VECTOR_OP_COS:
+    case VECTOR_OP_COSH:
+    case VECTOR_OP_ASIN:
+    case VECTOR_OP_ACOS:
+    case VECTOR_OP_ATAN:
+    case VECTOR_OP_ATAN2:
+    case VECTOR_OP_CBRT:
+    case VECTOR_OP_LOG:
+    case VECTOR_OP_LOG10:
+    case VECTOR_OP_LOG1P:
+    case VECTOR_OP_POW:
+    case VECTOR_OP_EXP:
+    case VECTOR_OP_EXPM1:
+    case VECTOR_OP_HYPOT:
+      return Op_CallLeafVector;
     default: fatal("unknown op: %d", vop);
   }
   return 0; // Unimplemented

src/hotspot/share/prims/vectorSupport.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,7 +29,6 @@
 #include "code/debugInfo.hpp"
 #include "memory/allocation.hpp"
 #include "oops/typeArrayOop.inline.hpp"
-#include "runtime/frame.inline.hpp"
 #include "runtime/registerMap.hpp"
 #include "utilities/exceptions.hpp"
 
@@ -82,9 +81,43 @@ class VectorSupport : AllStatic {
     // Mask manipulation operations
     VECTOR_OP_MASK_TRUECOUNT = 19,
     VECTOR_OP_MASK_FIRSTTRUE = 20,
-    VECTOR_OP_MASK_LASTTRUE  = 21
+    VECTOR_OP_MASK_LASTTRUE  = 21,
+
+    // Vector Math Library
+    VECTOR_OP_TAN   = 101,
+    VECTOR_OP_TANH  = 102,
+    VECTOR_OP_SIN   = 103,
+    VECTOR_OP_SINH  = 104,
+    VECTOR_OP_COS   = 105,
+    VECTOR_OP_COSH  = 106,
+    VECTOR_OP_ASIN  = 107,
+    VECTOR_OP_ACOS  = 108,
+    VECTOR_OP_ATAN  = 109,
+    VECTOR_OP_ATAN2 = 110,
+    VECTOR_OP_CBRT  = 111,
+    VECTOR_OP_LOG   = 112,
+    VECTOR_OP_LOG10 = 113,
+    VECTOR_OP_LOG1P = 114,
+    VECTOR_OP_POW   = 115,
+    VECTOR_OP_EXP   = 116,
+    VECTOR_OP_EXPM1 = 117,
+    VECTOR_OP_HYPOT = 118,
+
+    VECTOR_OP_SVML_START = VECTOR_OP_TAN,
+    VECTOR_OP_SVML_END   = VECTOR_OP_HYPOT,
+    NUM_SVML_OP = VECTOR_OP_SVML_END - VECTOR_OP_SVML_START + 1
   };
 
+  enum {
+    VEC_SIZE_64 = 0,
+    VEC_SIZE_128 = 1,
+    VEC_SIZE_256 = 2,
+    VEC_SIZE_512 = 3,
+    NUM_VEC_SIZES = 4
+  };
+
+  static const char* svmlname[VectorSupport::NUM_SVML_OP];
+
   static int vop2ideal(jint vop, BasicType bt);
 
   static instanceOop allocate_vector(InstanceKlass* holder, frame* fr, RegisterMap* reg_map, ObjectValue* sv, TRAPS);

src/hotspot/share/runtime/arguments.cpp

@@ -4104,6 +4104,11 @@ jint Arguments::apply_ergo() {
       }
     }
     FLAG_SET_DEFAULT(EnableVectorAggressiveReboxing, false);
+
+    if (!FLAG_IS_DEFAULT(UseVectorStubs) && UseVectorStubs) {
+      warning("Disabling UseVectorStubs since EnableVectorSupport is turned off.");
+    }
+    FLAG_SET_DEFAULT(UseVectorStubs, false);
   }
 #endif // COMPILER2
 

src/hotspot/share/runtime/sharedRuntime.hpp

@@ -379,6 +379,10 @@ class SharedRuntime: AllStatic {
   static int c_calling_convention(const BasicType *sig_bt, VMRegPair *regs, VMRegPair *regs2,
                                   int total_args_passed);
 
+  static int vector_calling_convention(VMRegPair *regs,
+                                       uint num_bits,
+                                       uint total_args_passed);
+
   static size_t trampoline_size();
 
   // Generate I2C and C2I adapters. These adapters are simple argument marshalling

src/hotspot/share/runtime/stubRoutines.cpp

@@ -28,6 +28,7 @@
 #include "memory/resourceArea.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/oop.inline.hpp"
+#include "prims/vectorSupport.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/timerTrace.hpp"
 #include "runtime/safefetch.inline.hpp"
@@ -180,6 +181,9 @@ address StubRoutines::_safefetchN_entry                  = NULL;
 address StubRoutines::_safefetchN_fault_pc               = NULL;
 address StubRoutines::_safefetchN_continuation_pc        = NULL;
 
+address StubRoutines::_vector_f_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP] = {{NULL}, {NULL}};
+address StubRoutines::_vector_d_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP] = {{NULL}, {NULL}};
+
 // Initialization
 //
 // Note: to break cycle with universe initialization, stubs are generated in two phases.

src/hotspot/share/runtime/stubRoutines.hpp

@@ -27,6 +27,7 @@
 
 #include "code/codeBlob.hpp"
 #include "memory/allocation.hpp"
+#include "prims/vectorSupport.hpp"
 #include "runtime/frame.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/stubCodeGenerator.hpp"
@@ -263,6 +264,10 @@ class StubRoutines: AllStatic {
   static address _safefetchN_fault_pc;
   static address _safefetchN_continuation_pc;
 
+  // Vector Math Routines
+  static address _vector_f_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP];
+  static address _vector_d_math[VectorSupport::NUM_VEC_SIZES][VectorSupport::NUM_SVML_OP];
+
  public:
   // Initialization/Testing
   static void    initialize1();                            // must happen before universe::genesis

src/hotspot/share/runtime/vmStructs.cpp

@@ -1527,6 +1527,7 @@ typedef HashtableEntry<InstanceKlass*, mtClass>  KlassHashtableEntry;
   declare_c2_type(CallLeafNode, CallRuntimeNode)                          \
   declare_c2_type(CallNativeNode, CallNode)                               \
   declare_c2_type(CallLeafNoFPNode, CallLeafNode)                         \
+  declare_c2_type(CallLeafVectorNode, CallLeafNode)                       \
   declare_c2_type(AllocateNode, CallNode)                                 \
   declare_c2_type(AllocateArrayNode, AllocateNode)                        \
   declare_c2_type(LockNode, AbstractLockNode)                             \

src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java

@@ -70,6 +70,26 @@ public class VectorSupport {
     public static final int VECTOR_OP_MASK_FIRSTTRUE = 20;
     public static final int VECTOR_OP_MASK_LASTTRUE  = 21;
 
+    // Math routines
+    public static final int VECTOR_OP_TAN = 101;
+    public static final int VECTOR_OP_TANH = 102;
+    public static final int VECTOR_OP_SIN = 103;
+    public static final int VECTOR_OP_SINH = 104;
+    public static final int VECTOR_OP_COS = 105;
+    public static final int VECTOR_OP_COSH = 106;
+    public static final int VECTOR_OP_ASIN = 107;
+    public static final int VECTOR_OP_ACOS = 108;
+    public static final int VECTOR_OP_ATAN = 109;
+    public static final int VECTOR_OP_ATAN2 = 110;
+    public static final int VECTOR_OP_CBRT = 111;
+    public static final int VECTOR_OP_LOG = 112;
+    public static final int VECTOR_OP_LOG10 = 113;
+    public static final int VECTOR_OP_LOG1P = 114;
+    public static final int VECTOR_OP_POW = 115;
+    public static final int VECTOR_OP_EXP = 116;
+    public static final int VECTOR_OP_EXPM1 = 117;
+    public static final int VECTOR_OP_HYPOT = 118;
+
     // See src/hotspot/share/opto/subnode.hpp
     //     struct BoolTest, and enclosed enum mask
     public static final int BT_eq = 0;  // 0000

src/jdk.incubator.vector/linux/native/libsvml/globals_vectorApiSupport_linux.S.inc

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+// This file is used to provide some global declarations related to building
+// VM with Vector API support. Also, the reason the file is separated is because
+// the intent of this file to provide a header that can be included in .s files.
+
+#ifndef OS_CPU_LINUX_X86_GLOBALS_VECTORAPISUPPORT_LINUX_HPP
+#define OS_CPU_LINUX_X86_GLOBALS_VECTORAPISUPPORT_LINUX_HPP
+
+// GCC 4.9+ can build all .s files for Linux
+#if defined(_LP64) && (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 9))))
+#define __VECTOR_API_MATH_INTRINSICS_LINUX
+#endif
+
+#endif //OS_CPU_LINUX_X86_GLOBALS_VECTORAPISUPPORT_LINUX_HPP
+

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleVector.java

@@ -535,37 +535,6 @@ public abstract class DoubleVector extends AbstractVector<Double> {
             if (op == ZOMO) {
                 return blend(broadcast(-1), compare(NE, 0));
             }
-            if (op == SIN) {
-                return uOp((i, a) -> (double) Math.sin(a));
-            } else if (op == COS) {
-                return uOp((i, a) -> (double) Math.cos(a));
-            } else if (op == TAN) {
-                return uOp((i, a) -> (double) Math.tan(a));
-            } else if (op == ASIN) {
-                return uOp((i, a) -> (double) Math.asin(a));
-            } else if (op == ACOS) {
-                return uOp((i, a) -> (double) Math.acos(a));
-            } else if (op == ATAN) {
-                return uOp((i, a) -> (double) Math.atan(a));
-            } else if (op == EXP) {
-                return uOp((i, a) -> (double) Math.exp(a));
-            } else if (op == LOG) {
-                return uOp((i, a) -> (double) Math.log(a));
-            } else if (op == LOG10) {
-                return uOp((i, a) -> (double) Math.log10(a));
-            } else if (op == CBRT) {
-                return uOp((i, a) -> (double) Math.cbrt(a));
-            } else if (op == SINH) {
-                return uOp((i, a) -> (double) Math.sinh(a));
-            } else if (op == COSH) {
-                return uOp((i, a) -> (double) Math.cosh(a));
-            } else if (op == TANH) {
-                return uOp((i, a) -> (double) Math.tanh(a));
-            } else if (op == EXPM1) {
-                return uOp((i, a) -> (double) Math.expm1(a));
-            } else if (op == LOG1P) {
-                return uOp((i, a) -> (double) Math.log1p(a));
-            }
         }
         int opc = opCode(op);
         return VectorSupport.unaryOp(
@@ -577,8 +546,38 @@ public abstract class DoubleVector extends AbstractVector<Double> {
                         v0.uOp((i, a) -> (double) -a);
                 case VECTOR_OP_ABS: return v0 ->
                         v0.uOp((i, a) -> (double) Math.abs(a));
+                case VECTOR_OP_SIN: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.sin(a));
+                case VECTOR_OP_COS: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.cos(a));
+                case VECTOR_OP_TAN: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.tan(a));
+                case VECTOR_OP_ASIN: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.asin(a));
+                case VECTOR_OP_ACOS: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.acos(a));
+                case VECTOR_OP_ATAN: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.atan(a));
+                case VECTOR_OP_EXP: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.exp(a));
+                case VECTOR_OP_LOG: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.log(a));
+                case VECTOR_OP_LOG10: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.log10(a));
                 case VECTOR_OP_SQRT: return v0 ->
                         v0.uOp((i, a) -> (double) Math.sqrt(a));
+                case VECTOR_OP_CBRT: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.cbrt(a));
+                case VECTOR_OP_SINH: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.sinh(a));
+                case VECTOR_OP_COSH: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.cosh(a));
+                case VECTOR_OP_TANH: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.tanh(a));
+                case VECTOR_OP_EXPM1: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.expm1(a));
+                case VECTOR_OP_LOG1P: return v0 ->
+                        v0.uOp((i, a) -> (double) Math.log1p(a));
                 default: return null;
               }}));
     }
@@ -625,13 +624,6 @@ public abstract class DoubleVector extends AbstractVector<Double> {
                     .lanewise(op, that.viewAsIntegralLanes())
                     .viewAsFloatingLanes();
             }
-            if (op == ATAN2) {
-                return bOp(that, (i, a, b) -> (double) Math.atan2(a, b));
-            } else if (op == POW) {
-                return bOp(that, (i, a, b) -> (double) Math.pow(a, b));
-            } else if (op == HYPOT) {
-                return bOp(that, (i, a, b) -> (double) Math.hypot(a, b));
-            }
         }
         int opc = opCode(op);
         return VectorSupport.binaryOp(
@@ -651,6 +643,12 @@ public abstract class DoubleVector extends AbstractVector<Double> {
                         v0.bOp(v1, (i, a, b) -> (double)Math.max(a, b));
                 case VECTOR_OP_MIN: return (v0, v1) ->
                         v0.bOp(v1, (i, a, b) -> (double)Math.min(a, b));
+                case VECTOR_OP_ATAN2: return (v0, v1) ->
+                        v0.bOp(v1, (i, a, b) -> (double) Math.atan2(a, b));
+                case VECTOR_OP_POW: return (v0, v1) ->
+                        v0.bOp(v1, (i, a, b) -> (double) Math.pow(a, b));
+                case VECTOR_OP_HYPOT: return (v0, v1) ->
+                        v0.bOp(v1, (i, a, b) -> (double) Math.hypot(a, b));
                 default: return null;
                 }}));
     }

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatVector.java

@@ -535,37 +535,6 @@ public abstract class FloatVector extends AbstractVector<Float> {
             if (op == ZOMO) {
                 return blend(broadcast(-1), compare(NE, 0));
             }
-            if (op == SIN) {
-                return uOp((i, a) -> (float) Math.sin(a));
-            } else if (op == COS) {
-                return uOp((i, a) -> (float) Math.cos(a));
-            } else if (op == TAN) {
-                return uOp((i, a) -> (float) Math.tan(a));
-            } else if (op == ASIN) {
-                return uOp((i, a) -> (float) Math.asin(a));
-            } else if (op == ACOS) {
-                return uOp((i, a) -> (float) Math.acos(a));
-            } else if (op == ATAN) {
-                return uOp((i, a) -> (float) Math.atan(a));
-            } else if (op == EXP) {
-                return uOp((i, a) -> (float) Math.exp(a));
-            } else if (op == LOG) {
-                return uOp((i, a) -> (float) Math.log(a));
-            } else if (op == LOG10) {
-                return uOp((i, a) -> (float) Math.log10(a));
-            } else if (op == CBRT) {
-                return uOp((i, a) -> (float) Math.cbrt(a));
-            } else if (op == SINH) {
-                return uOp((i, a) -> (float) Math.sinh(a));
-            } else if (op == COSH) {
-                return uOp((i, a) -> (float) Math.cosh(a));
-            } else if (op == TANH) {
-                return uOp((i, a) -> (float) Math.tanh(a));
-            } else if (op == EXPM1) {
-                return uOp((i, a) -> (float) Math.expm1(a));
-            } else if (op == LOG1P) {
-                return uOp((i, a) -> (float) Math.log1p(a));
-            }
         }
         int opc = opCode(op);
         return VectorSupport.unaryOp(
@@ -577,8 +546,38 @@ public abstract class FloatVector extends AbstractVector<Float> {
                         v0.uOp((i, a) -> (float) -a);
                 case VECTOR_OP_ABS: return v0 ->
                         v0.uOp((i, a) -> (float) Math.abs(a));
+                case VECTOR_OP_SIN: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.sin(a));
+                case VECTOR_OP_COS: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.cos(a));
+                case VECTOR_OP_TAN: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.tan(a));
+                case VECTOR_OP_ASIN: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.asin(a));
+                case VECTOR_OP_ACOS: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.acos(a));
+                case VECTOR_OP_ATAN: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.atan(a));
+                case VECTOR_OP_EXP: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.exp(a));
+                case VECTOR_OP_LOG: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.log(a));
+                case VECTOR_OP_LOG10: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.log10(a));
                 case VECTOR_OP_SQRT: return v0 ->
                         v0.uOp((i, a) -> (float) Math.sqrt(a));
+                case VECTOR_OP_CBRT: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.cbrt(a));
+                case VECTOR_OP_SINH: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.sinh(a));
+                case VECTOR_OP_COSH: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.cosh(a));
+                case VECTOR_OP_TANH: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.tanh(a));
+                case VECTOR_OP_EXPM1: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.expm1(a));
+                case VECTOR_OP_LOG1P: return v0 ->
+                        v0.uOp((i, a) -> (float) Math.log1p(a));
                 default: return null;
               }}));
     }
@@ -625,13 +624,6 @@ public abstract class FloatVector extends AbstractVector<Float> {
                     .lanewise(op, that.viewAsIntegralLanes())
                     .viewAsFloatingLanes();
             }
-            if (op == ATAN2) {
-                return bOp(that, (i, a, b) -> (float) Math.atan2(a, b));
-            } else if (op == POW) {
-                return bOp(that, (i, a, b) -> (float) Math.pow(a, b));
-            } else if (op == HYPOT) {
-                return bOp(that, (i, a, b) -> (float) Math.hypot(a, b));
-            }
         }
         int opc = opCode(op);
         return VectorSupport.binaryOp(
@@ -651,6 +643,12 @@ public abstract class FloatVector extends AbstractVector<Float> {
                         v0.bOp(v1, (i, a, b) -> (float)Math.max(a, b));
                 case VECTOR_OP_MIN: return (v0, v1) ->
                         v0.bOp(v1, (i, a, b) -> (float)Math.min(a, b));
+                case VECTOR_OP_ATAN2: return (v0, v1) ->
+                        v0.bOp(v1, (i, a, b) -> (float) Math.atan2(a, b));
+                case VECTOR_OP_POW: return (v0, v1) ->
+                        v0.bOp(v1, (i, a, b) -> (float) Math.pow(a, b));
+                case VECTOR_OP_HYPOT: return (v0, v1) ->
+                        v0.bOp(v1, (i, a, b) -> (float) Math.hypot(a, b));
                 default: return null;
                 }}));
     }

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/VectorOperators.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -76,7 +76,7 @@ import jdk.internal.vm.vector.VectorSupport;
  * <ul>
  * <li>Lane-wise vector operations that apply to floating point vectors
  * follow the accuracy and monotonicity specifications of the equivalent
- * Java operation or method mentioned in its documentation.
+ * Java operation or method mentioned in its documentation unless specified otherwise.
  * If the vector element type is {@code float} and the Java operation or
  * method only accepts and returns {@code double} values, then the scalar operation
  * on each lane is adapted to cast operands and the result, specifically widening
@@ -451,40 +451,70 @@ public abstract class VectorOperators {
     /** Produce {@code -a}. */
     public static final Unary NEG = unary("NEG", "-a", VectorSupport.VECTOR_OP_NEG, VO_ALL|VO_SPECIAL);
 
-    /** Produce {@code sin(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary SIN = unary("SIN", "sin", -1 /*VectorSupport.VECTOR_OP_SIN*/, VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code cos(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary COS = unary("COS", "cos", -1 /*VectorSupport.VECTOR_OP_COS*/, VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code tan(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary TAN = unary("TAN", "tan", -1 /*VectorSupport.VECTOR_OP_TAN*/, VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code asin(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary ASIN = unary("ASIN", "asin", -1 /*VectorSupport.VECTOR_OP_ASIN*/, VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code acos(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary ACOS = unary("ACOS", "acos", -1 /*VectorSupport.VECTOR_OP_ACOS*/, VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code atan(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary ATAN = unary("ATAN", "atan", -1 /*VectorSupport.VECTOR_OP_ATAN*/, VO_ONLYFP | VO_SPECIAL);
+    /** Produce {@code sin(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary SIN = unary("SIN", "sin", VectorSupport.VECTOR_OP_SIN, VO_ONLYFP);
+    /** Produce {@code cos(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary COS = unary("COS", "cos", VectorSupport.VECTOR_OP_COS, VO_ONLYFP);
+    /** Produce {@code tan(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary TAN = unary("TAN", "tan", VectorSupport.VECTOR_OP_TAN, VO_ONLYFP);
+    /** Produce {@code asin(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary ASIN = unary("ASIN", "asin", VectorSupport.VECTOR_OP_ASIN, VO_ONLYFP);
+    /** Produce {@code acos(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary ACOS = unary("ACOS", "acos", VectorSupport.VECTOR_OP_ACOS, VO_ONLYFP);
+    /** Produce {@code atan(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary ATAN = unary("ATAN", "atan", VectorSupport.VECTOR_OP_ATAN, VO_ONLYFP);
 
-    /** Produce {@code exp(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary EXP = unary("EXP", "exp", -1 /*VectorSupport.VECTOR_OP_EXP*/, VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code log(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary LOG = unary("LOG", "log", -1 /*VectorSupport.VECTOR_OP_LOG*/, VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code log10(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary LOG10 = unary("LOG10", "log10", -1 /*VectorSupport.VECTOR_OP_LOG10*/, VO_ONLYFP | VO_SPECIAL);
+    /** Produce {@code exp(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary EXP = unary("EXP", "exp", VectorSupport.VECTOR_OP_EXP, VO_ONLYFP);
+    /** Produce {@code log(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary LOG = unary("LOG", "log", VectorSupport.VECTOR_OP_LOG, VO_ONLYFP);
+    /** Produce {@code log10(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary LOG10 = unary("LOG10", "log10", VectorSupport.VECTOR_OP_LOG10, VO_ONLYFP);
     /** Produce {@code sqrt(a)}.  Floating only.  See section "Operations on floating point vectors" above */
     public static final /*float*/ Unary SQRT = unary("SQRT", "sqrt", VectorSupport.VECTOR_OP_SQRT, VO_ONLYFP);
-    /** Produce {@code cbrt(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary CBRT = unary("CBRT", "cbrt", -1 /*VectorSupport.VECTOR_OP_CBRT*/, VO_ONLYFP | VO_SPECIAL);
+    /** Produce {@code cbrt(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary CBRT = unary("CBRT", "cbrt", VectorSupport.VECTOR_OP_CBRT, VO_ONLYFP);
 
-    /** Produce {@code sinh(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary SINH = unary("SINH", "sinh", -1 /*VectorSupport.VECTOR_OP_SINH*/, VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code cosh(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary COSH = unary("COSH", "cosh", -1 /*VectorSupport.VECTOR_OP_COSH*/, VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code tanh(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary TANH = unary("TANH", "tanh", -1 /*VectorSupport.VECTOR_OP_TANH*/, VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code expm1(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary EXPM1 = unary("EXPM1", "expm1", -1 /*VectorSupport.VECTOR_OP_EXPM1*/, VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code log1p(a)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Unary LOG1P = unary("LOG1P", "log1p", -1 /*VectorSupport.VECTOR_OP_LOG1P*/, VO_ONLYFP | VO_SPECIAL);
+    /** Produce {@code sinh(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary SINH = unary("SINH", "sinh", VectorSupport.VECTOR_OP_SINH, VO_ONLYFP);
+    /** Produce {@code cosh(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary COSH = unary("COSH", "cosh", VectorSupport.VECTOR_OP_COSH, VO_ONLYFP);
+    /** Produce {@code tanh(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary TANH = unary("TANH", "tanh", VectorSupport.VECTOR_OP_TANH, VO_ONLYFP);
+    /** Produce {@code expm1(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary EXPM1 = unary("EXPM1", "expm1", VectorSupport.VECTOR_OP_EXPM1, VO_ONLYFP);
+    /** Produce {@code log1p(a)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Unary LOG1P = unary("LOG1P", "log1p", VectorSupport.VECTOR_OP_LOG1P, VO_ONLYFP);
 
     // Binary operators
 
@@ -525,12 +555,18 @@ public abstract class VectorOperators {
     /** Produce {@code rotateRight(a,n)}.  Integral only. */
     public static final /*bitwise*/ Binary ROR = binary("ROR", "rotateRight", -1 /*VectorSupport.VECTOR_OP_RROTATE*/, VO_SHIFT | VO_SPECIAL);
 
-    /** Produce {@code atan2(a,b)}. See  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Binary ATAN2 = binary("ATAN2", "atan2", -1 /*VectorSupport.VECTOR_OP_ATAN2*/ , VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code pow(a,b)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Binary POW = binary("POW", "pow", -1 /*VectorSupport.VECTOR_OP_POW*/, VO_ONLYFP | VO_SPECIAL);
-    /** Produce {@code hypot(a,b)}.  Floating only.  See section "Operations on floating point vectors" above */
-    public static final /*float*/ Binary HYPOT = binary("HYPOT", "hypot", -1 /*VectorSupport.VECTOR_OP_HYPOT*/, VO_ONLYFP | VO_SPECIAL);
+    /** Produce {@code atan2(a,b)}. See  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Binary ATAN2 = binary("ATAN2", "atan2", VectorSupport.VECTOR_OP_ATAN2, VO_ONLYFP);
+    /** Produce {@code pow(a,b)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Binary POW = binary("POW", "pow", VectorSupport.VECTOR_OP_POW, VO_ONLYFP);
+    /** Produce {@code hypot(a,b)}.  Floating only.
+     *  Not guaranteed to be semi-monotonic. See section "Operations on floating point vectors" above
+     */
+    public static final /*float*/ Binary HYPOT = binary("HYPOT", "hypot", VectorSupport.VECTOR_OP_HYPOT, VO_ONLYFP);
 
     // Ternary operators
 
@@ -1251,11 +1287,7 @@ public abstract class VectorOperators {
                        op == ROR ||
                        op == IS_DEFAULT || op == IS_NEGATIVE ||
                        op == IS_FINITE || op == IS_NAN || op == IS_INFINITE ||
-                       op == BITWISE_BLEND ||
-                       op == SIN   || op == COS   || op == TAN   || op == ASIN || op == ACOS || op == ATAN || op == EXP  ||
-                       op == LOG   || op == LOG10 || op == SQRT  || op == CBRT || op == SINH || op == COSH || op == TANH ||
-                       op == EXPM1 || op == LOG1P || op == ATAN2 || op == POW || op == HYPOT
-                ) : op;
+                       op == BITWISE_BLEND) : op;
             }
         }
         return true;

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-Vector.java.template

@@ -555,39 +555,6 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
                 return broadcast(0).lanewiseTemplate(SUB, this);
             }
 #end[BITWISE]
-#if[FP]
-            if (op == SIN) {
-                return uOp((i, a) -> ($type$) Math.sin(a));
-            } else if (op == COS) {
-                return uOp((i, a) -> ($type$) Math.cos(a));
-            } else if (op == TAN) {
-                return uOp((i, a) -> ($type$) Math.tan(a));
-            } else if (op == ASIN) {
-                return uOp((i, a) -> ($type$) Math.asin(a));
-            } else if (op == ACOS) {
-                return uOp((i, a) -> ($type$) Math.acos(a));
-            } else if (op == ATAN) {
-                return uOp((i, a) -> ($type$) Math.atan(a));
-            } else if (op == EXP) {
-                return uOp((i, a) -> ($type$) Math.exp(a));
-            } else if (op == LOG) {
-                return uOp((i, a) -> ($type$) Math.log(a));
-            } else if (op == LOG10) {
-                return uOp((i, a) -> ($type$) Math.log10(a));
-            } else if (op == CBRT) {
-                return uOp((i, a) -> ($type$) Math.cbrt(a));
-            } else if (op == SINH) {
-                return uOp((i, a) -> ($type$) Math.sinh(a));
-            } else if (op == COSH) {
-                return uOp((i, a) -> ($type$) Math.cosh(a));
-            } else if (op == TANH) {
-                return uOp((i, a) -> ($type$) Math.tanh(a));
-            } else if (op == EXPM1) {
-                return uOp((i, a) -> ($type$) Math.expm1(a));
-            } else if (op == LOG1P) {
-                return uOp((i, a) -> ($type$) Math.log1p(a));
-            }
-#end[FP]
         }
         int opc = opCode(op);
         return VectorSupport.unaryOp(
@@ -600,8 +567,38 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
                 case VECTOR_OP_ABS: return v0 ->
                         v0.uOp((i, a) -> ($type$) Math.abs(a));
 #if[FP]
+                case VECTOR_OP_SIN: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.sin(a));
+                case VECTOR_OP_COS: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.cos(a));
+                case VECTOR_OP_TAN: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.tan(a));
+                case VECTOR_OP_ASIN: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.asin(a));
+                case VECTOR_OP_ACOS: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.acos(a));
+                case VECTOR_OP_ATAN: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.atan(a));
+                case VECTOR_OP_EXP: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.exp(a));
+                case VECTOR_OP_LOG: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.log(a));
+                case VECTOR_OP_LOG10: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.log10(a));
                 case VECTOR_OP_SQRT: return v0 ->
                         v0.uOp((i, a) -> ($type$) Math.sqrt(a));
+                case VECTOR_OP_CBRT: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.cbrt(a));
+                case VECTOR_OP_SINH: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.sinh(a));
+                case VECTOR_OP_COSH: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.cosh(a));
+                case VECTOR_OP_TANH: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.tanh(a));
+                case VECTOR_OP_EXPM1: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.expm1(a));
+                case VECTOR_OP_LOG1P: return v0 ->
+                        v0.uOp((i, a) -> ($type$) Math.log1p(a));
 #end[FP]
                 default: return null;
               }}));
@@ -675,15 +672,6 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
                 }
             }
 #end[BITWISE]
-#if[FP]
-            if (op == ATAN2) {
-                return bOp(that, (i, a, b) -> ($type$) Math.atan2(a, b));
-            } else if (op == POW) {
-                return bOp(that, (i, a, b) -> ($type$) Math.pow(a, b));
-            } else if (op == HYPOT) {
-                return bOp(that, (i, a, b) -> ($type$) Math.hypot(a, b));
-            }
-#end[FP]
         }
         int opc = opCode(op);
         return VectorSupport.binaryOp(
@@ -717,6 +705,14 @@ public abstract class $abstractvectortype$ extends AbstractVector<$Boxtype$> {
                 case VECTOR_OP_URSHIFT: return (v0, v1) ->
                         v0.bOp(v1, (i, a, n) -> ($type$)((a & LSHR_SETUP_MASK) >>> n));
 #end[BITWISE]
+#if[FP]
+                case VECTOR_OP_ATAN2: return (v0, v1) ->
+                        v0.bOp(v1, (i, a, b) -> ($type$) Math.atan2(a, b));
+                case VECTOR_OP_POW: return (v0, v1) ->
+                        v0.bOp(v1, (i, a, b) -> ($type$) Math.pow(a, b));
+                case VECTOR_OP_HYPOT: return (v0, v1) ->
+                        v0.bOp(v1, (i, a, b) -> ($type$) Math.hypot(a, b));
+#end[FP]
                 default: return null;
                 }}));
     }

src/jdk.incubator.vector/windows/native/libsvml/globals_vectorApiSupport_windows.S.inc

@@ -0,0 +1,37 @@
+; Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
+; DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+;
+; This code is free software; you can redistribute it and/or modify it
+; under the terms of the GNU General Public License version 2 only, as
+; published by the Free Software Foundation.
+;
+; This code is distributed in the hope that it will be useful, but WITHOUT
+; ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+; FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+; version 2 for more details (a copy is included in the LICENSE file that
+; accompanied this code).
+;
+; You should have received a copy of the GNU General Public License version
+; 2 along with this work; if not, write to the Free Software Foundation,
+; Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+;
+; Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+; or visit www.oracle.com if you need additional information or have any
+; questions.
+
+; This file contains duplicate entries as globalDefinitions_vecApi.hpp
+; It is intended for inclusion in .s files compiled with masm
+
+; Used to check whether building on x86_64 architecture. Equivalent to checking in regular hpp file for #ifdef _WIN64
+IFDEF RAX
+
+; @Version is defined by MASM to determine the Visual Studio version. 1410 is the version for VS17
+IF @Version GE 1410
+__VECTOR_API_MATH_INTRINSICS_WINDOWS TEXTEQU <"vector_api">
+ELSE
+__VECTOR_API_MATH_INTRINSICS_WINDOWS TEXTEQU <>
+ENDIF
+
+ELSE
+__VECTOR_API_MATH_INTRINSICS_WINDOWS TEXTEQU <>
+ENDIF