stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8292878: x86: Make scratch register usage explicit in assembler code

Browse Source 
Reviewed-by: kvn, shade
This commit is contained in:
Vladimir Ivanov 2022-08-30 18:45:24 +00:00
parent f5ebbf3225
commit 6e248279cf
32 changed files with 1068 additions and 981 deletions

4
src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@ -2453,7 +2453,7 @@ void MacroAssembler::verify_heapbase(const char* msg) {
if (CheckCompressedOops) {
Label ok;
push(1 << rscratch1->encoding(), sp); // cmpptr trashes rscratch1
cmpptr(rheapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
cmpptr(rheapbase, ExternalAddress(CompressedOops::ptrs_base_addr()));
br(Assembler::EQ, ok);
stop(msg);
bind(ok);
@ -2587,7 +2587,7 @@ void MacroAssembler::reinit_heapbase()
if (Universe::is_fully_initialized()) {
mov(rheapbase, CompressedOops::ptrs_base());
} else {
lea(rheapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
lea(rheapbase, ExternalAddress(CompressedOops::ptrs_base_addr()));
ldr(rheapbase, Address(rheapbase));
}
}

2
src/hotspot/cpu/riscv/macroAssembler_riscv.cpp

@ -1355,7 +1355,7 @@ void MacroAssembler::reinit_heapbase() {
mv(xheapbase, CompressedOops::ptrs_base());
} else {
int32_t offset = 0;
la_patchable(xheapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()), offset);
la_patchable(xheapbase, ExternalAddress(CompressedOops::ptrs_base_addr()), offset);
ld(xheapbase, Address(xheapbase, offset));
}
}

18
src/hotspot/cpu/x86/assembler_x86.cpp

@ -1255,12 +1255,12 @@ void Assembler::emit_operand(Register reg, Address adr,
}
void Assembler::emit_operand(XMMRegister reg, Address adr) {
if (adr.isxmmindex()) {
emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
} else {
emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
adr._rspec);
}
if (adr.isxmmindex()) {
emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
} else {
emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
adr._rspec);
}
}
// Now the Assembler instructions (identical for 32/64 bits)
@ -12246,11 +12246,17 @@ static bool is_reachable(address target, relocInfo::relocType reloc_type) {
bool Assembler::reachable(AddressLiteral adr) {
assert(CodeCache::contains(pc()), "required");
if (adr.is_lval()) {
return false;
}
return is_reachable(adr.target(), adr.reloc());
}
bool Assembler::always_reachable(AddressLiteral adr) {
assert(CodeCache::contains(pc()), "required");
if (adr.is_lval()) {
return false;
}
return is_always_reachable(adr.target(), adr.reloc());
}

2
src/hotspot/cpu/x86/c1_CodeStubs_x86.cpp

@ -544,7 +544,7 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
#ifndef PRODUCT
if (PrintC1Statistics) {
__ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_slowcase_cnt));
__ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_slowcase_cnt), rscratch1);
}
#endif

49
src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

@ -196,7 +196,7 @@ void LIR_Assembler::push(LIR_Opr opr) {
} else if (opr->is_constant()) {
LIR_Const* const_opr = opr->as_constant_ptr();
if (const_opr->type() == T_OBJECT) {
__ push_oop(const_opr->as_jobject());
__ push_oop(const_opr->as_jobject(), rscratch1);
} else if (const_opr->type() == T_INT) {
__ push_jint(const_opr->as_jint());
} else {
@ -469,7 +469,7 @@ int LIR_Assembler::emit_unwind_handler() {
#else
__ get_thread(rax);
__ movptr(Address(rsp, 0), rax);
__ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
__ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
#endif
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
}
@ -503,7 +503,7 @@ int LIR_Assembler::emit_deopt_handler() {
int offset = code_offset();
InternalAddress here(__ pc());
__ pushptr(here.addr());
__ pushptr(here.addr(), rscratch1);
__ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
__ end_a_stub();
@ -691,14 +691,16 @@ void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
break;
case T_OBJECT:
__ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject());
__ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
break;
case T_LONG: // fall through
case T_DOUBLE:
#ifdef _LP64
__ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
lo_word_offset_in_bytes), (intptr_t)c->as_jlong_bits());
lo_word_offset_in_bytes),
(intptr_t)c->as_jlong_bits(),
rscratch1);
#else
__ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
lo_word_offset_in_bytes), c->as_jint_lo_bits());
@ -746,7 +748,7 @@ void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmi
} else {
if (is_literal_address(addr)) {
ShouldNotReachHere();
__ movoop(as_Address(addr, noreg), c->as_jobject());
__ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
} else {
#ifdef _LP64
__ movoop(rscratch1, c->as_jobject());
@ -759,7 +761,7 @@ void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmi
__ movptr(as_Address_lo(addr), rscratch1);
}
#else
__ movoop(as_Address(addr), c->as_jobject());
__ movoop(as_Address(addr), c->as_jobject(), noreg);
#endif
}
}
@ -1784,7 +1786,7 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
#ifdef _LP64
__ push(k_RInfo);
#else
__ pushklass(k->constant_encoding());
__ pushklass(k->constant_encoding(), noreg);
#endif // _LP64
__ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
__ pop(klass_RInfo);
@ -2432,7 +2434,8 @@ void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_
}
assert(!tmp->is_valid(), "do not need temporary");
__ andpd(dest->as_xmm_double_reg(),
ExternalAddress((address)double_signmask_pool));
ExternalAddress((address)double_signmask_pool),
rscratch1);
}
}
break;
@ -2673,7 +2676,7 @@ void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2,
if (o == NULL) {
__ cmpptr(reg1, NULL_WORD);
} else {
__ cmpoop(reg1, o);
__ cmpoop(reg1, o, rscratch1);
}
} else {
fatal("unexpected type: %s", basictype_to_str(c->type()));
@ -3035,19 +3038,19 @@ void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
}
void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
__ movoop (Address(rsp, offset_from_rsp_in_bytes), o);
__ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
}
void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
__ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m);
__ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
}
@ -3109,7 +3112,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
store_parameter(j_rarg4, 4);
#ifndef PRODUCT
if (PrintC1Statistics) {
__ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
__ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
}
#endif
__ call(RuntimeAddress(copyfunc_addr));
@ -3118,7 +3121,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
__ mov(c_rarg4, j_rarg4);
#ifndef PRODUCT
if (PrintC1Statistics) {
__ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
__ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
}
#endif
__ call(RuntimeAddress(copyfunc_addr));
@ -3132,7 +3135,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
#ifndef PRODUCT
if (PrintC1Statistics) {
__ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
__ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
}
#endif
__ call_VM_leaf(copyfunc_addr, 5); // removes pushed parameter from the stack
@ -3365,7 +3368,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
Label failed;
__ testl(rax, rax);
__ jcc(Assembler::notZero, failed);
__ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
__ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
__ bind(failed);
}
#endif
@ -3375,7 +3378,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
#ifndef PRODUCT
if (PrintC1Statistics) {
__ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
__ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
}
#endif
@ -3584,7 +3587,7 @@ void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
ciKlass* receiver = vc_data->receiver(i);
if (receiver == NULL) {
Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
__ mov_metadata(recv_addr, known_klass->constant_encoding());
__ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
__ addptr(data_addr, DataLayout::counter_increment);
return;
@ -3816,7 +3819,8 @@ void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
__ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
}
__ xorps(dest->as_xmm_float_reg(),
ExternalAddress((address)float_signflip_pool));
ExternalAddress((address)float_signflip_pool),
rscratch1);
}
} else if (dest->is_double_xmm()) {
#ifdef _LP64
@ -3833,7 +3837,8 @@ void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
__ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
}
__ xorpd(dest->as_xmm_double_reg(),
ExternalAddress((address)double_signflip_pool));
ExternalAddress((address)double_signflip_pool),
rscratch1);
}
#ifndef _LP64
} else if (left->is_single_fpu() || left->is_double_fpu()) {

3
src/hotspot/cpu/x86/c1_LIRAssembler_x86.hpp

@ -37,8 +37,7 @@
bool is_literal_address(LIR_Address* addr);
// When we need to use something other than rscratch1 use this
// method.
// When we need to use something other than rscratch1 use this method.
Address as_Address(LIR_Address* addr, Register tmp);
// Record the type of the receiver in ReceiverTypeData

9
src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp

@ -39,7 +39,6 @@
#include "runtime/stubRoutines.hpp"
int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
const Register rklass_decode_tmp = LP64_ONLY(rscratch1) NOT_LP64(noreg);
const int aligned_mask = BytesPerWord -1;
const int hdr_offset = oopDesc::mark_offset_in_bytes();
assert(hdr == rax, "hdr must be rax, for the cmpxchg instruction");
@ -55,7 +54,7 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
null_check_offset = offset();
if (DiagnoseSyncOnValueBasedClasses != 0) {
load_klass(hdr, obj, rklass_decode_tmp);
load_klass(hdr, obj, rscratch1);
movl(hdr, Address(hdr, Klass::access_flags_offset()));
testl(hdr, JVM_ACC_IS_VALUE_BASED_CLASS);
jcc(Assembler::notZero, slow_case);
@ -146,12 +145,11 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, i
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
assert_different_registers(obj, klass, len);
Register tmp_encode_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
#ifdef _LP64
if (UseCompressedClassPointers) { // Take care not to kill klass
movptr(t1, klass);
encode_klass_not_null(t1, tmp_encode_klass);
encode_klass_not_null(t1, rscratch1);
movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
} else
#endif
@ -286,10 +284,9 @@ void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
// check against inline cache
assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
int start_offset = offset();
Register tmp_load_klass = LP64_ONLY(rscratch2) NOT_LP64(noreg);
if (UseCompressedClassPointers) {
load_klass(rscratch1, receiver, tmp_load_klass);
load_klass(rscratch1, receiver, rscratch2);
cmpptr(rscratch1, iCache);
} else {
cmpptr(iCache, Address(receiver, oopDesc::klass_offset_in_bytes()));

3
src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp

@ -98,12 +98,13 @@
// This helps us to track the rsp changes compared to the entry rsp (->_rsp_offset)
void push_jint (jint i) { _rsp_offset++; push(i); }
void push_oop (jobject o) { _rsp_offset++; pushoop(o); }
// Seems to always be in wordSize
void push_addr (Address a) { _rsp_offset++; pushptr(a); }
void push_reg (Register r) { _rsp_offset++; push(r); }
void pop_reg (Register r) { _rsp_offset--; pop(r); assert(_rsp_offset >= 0, "stack offset underflow"); }
void push_oop (jobject o, Register rscratch) { _rsp_offset++; pushoop(o, rscratch); }
void dec_stack (int nof_words) {
_rsp_offset -= nof_words;
assert(_rsp_offset >= 0, "stack offset underflow");

9
src/hotspot/cpu/x86/c1_Runtime1_x86.cpp

@ -77,11 +77,11 @@ int StubAssembler::call_RT(Register oop_result1, Register metadata_result, addre
int call_offset = -1;
if (!align_stack) {
set_last_Java_frame(thread, noreg, rbp, NULL);
set_last_Java_frame(thread, noreg, rbp, NULL, rscratch1);
} else {
address the_pc = pc();
call_offset = offset();
set_last_Java_frame(thread, noreg, rbp, the_pc);
set_last_Java_frame(thread, noreg, rbp, the_pc, rscratch1);
andptr(rsp, -(StackAlignmentInBytes)); // Align stack
}
@ -886,7 +886,7 @@ OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
__ get_thread(thread);
__ push(thread);
#endif // _LP64
__ set_last_Java_frame(thread, noreg, rbp, NULL);
__ set_last_Java_frame(thread, noreg, rbp, NULL, rscratch1);
// do the call
__ call(RuntimeAddress(target));
OopMapSet* oop_maps = new OopMapSet();
@ -1147,9 +1147,8 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
// load the klass and check the has finalizer flag
Label register_finalizer;
Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
Register t = rsi;
__ load_klass(t, rax, tmp_load_klass);
__ load_klass(t, rax, rscratch1);
__ movl(t, Address(t, Klass::access_flags_offset()));
__ testl(t, JVM_ACC_HAS_FINALIZER);
__ jcc(Assembler::notZero, register_finalizer);

191
src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

@ -1649,6 +1649,8 @@ void C2_MacroAssembler::load_vector(XMMRegister dst, Address src, int vlen_in_by
}
void C2_MacroAssembler::load_vector(XMMRegister dst, AddressLiteral src, int vlen_in_bytes, Register rscratch) {
assert(rscratch != noreg || always_reachable(src), "missing");
if (reachable(src)) {
load_vector(dst, as_Address(src), vlen_in_bytes);
} else {
@ -1662,21 +1664,21 @@ void C2_MacroAssembler::load_constant_vector(BasicType bt, XMMRegister dst, Inte
if (VM_Version::supports_avx()) {
if (bt == T_LONG) {
if (VM_Version::supports_avx2()) {
vpbroadcastq(dst, src, vlen_enc, noreg);
vpbroadcastq(dst, src, vlen_enc);
} else {
vmovddup(dst, src, vlen_enc, noreg);
vmovddup(dst, src, vlen_enc);
}
} else if (bt == T_DOUBLE) {
if (vlen_enc != Assembler::AVX_128bit) {
vbroadcastsd(dst, src, vlen_enc, noreg);
} else {
vmovddup(dst, src, vlen_enc, noreg);
vmovddup(dst, src, vlen_enc);
}
} else {
if (VM_Version::supports_avx2() && is_integral_type(bt)) {
vpbroadcastd(dst, src, vlen_enc, noreg);
vpbroadcastd(dst, src, vlen_enc);
} else {
vbroadcastss(dst, src, vlen_enc, noreg);
vbroadcastss(dst, src, vlen_enc);
}
}
} else if (VM_Version::supports_sse3()) {
@ -2395,25 +2397,25 @@ void C2_MacroAssembler::evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask
}
}
void C2_MacroAssembler::evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral adr, int comparison, int vector_len, Register rscratch) {
assert(rscratch != noreg || always_reachable(adr), "missing");
void C2_MacroAssembler::evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral src2, int comparison, int vector_len, Register rscratch) {
assert(rscratch != noreg || always_reachable(src2), "missing");
switch(typ) {
case T_BOOLEAN:
case T_BYTE:
evpcmpb(kdmask, ksmask, src1, adr, comparison, /*signed*/ true, vector_len, rscratch);
evpcmpb(kdmask, ksmask, src1, src2, comparison, /*signed*/ true, vector_len, rscratch);
break;
case T_CHAR:
case T_SHORT:
evpcmpw(kdmask, ksmask, src1, adr, comparison, /*signed*/ true, vector_len, rscratch);
evpcmpw(kdmask, ksmask, src1, src2, comparison, /*signed*/ true, vector_len, rscratch);
break;
case T_INT:
case T_FLOAT:
evpcmpd(kdmask, ksmask, src1, adr, comparison, /*signed*/ true, vector_len, rscratch);
evpcmpd(kdmask, ksmask, src1, src2, comparison, /*signed*/ true, vector_len, rscratch);
break;
case T_LONG:
case T_DOUBLE:
evpcmpq(kdmask, ksmask, src1, adr, comparison, /*signed*/ true, vector_len, rscratch);
evpcmpq(kdmask, ksmask, src1, src2, comparison, /*signed*/ true, vector_len, rscratch);
break;
default:
assert(false,"Should not reach here.");
@ -4357,12 +4359,11 @@ void C2_MacroAssembler::masked_op(int ideal_opc, int mask_len, KRegister dst,
* If the src is positive infinity or any value greater than or equal to the value of Integer.MAX_VALUE,
* the result is equal to the value of Integer.MAX_VALUE.
*/
void C2_MacroAssembler::vector_cast_float_special_cases_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4,
Register scratch, AddressLiteral float_sign_flip,
int vec_enc) {
void C2_MacroAssembler::vector_cast_float_special_cases_avx(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4,
Register rscratch) {
Label done;
vmovdqu(xtmp1, float_sign_flip, vec_enc, scratch);
vmovdqu(xtmp1, float_sign_flip, vec_enc, rscratch);
vpcmpeqd(xtmp2, dst, xtmp1, vec_enc);
vptest(xtmp2, xtmp2, vec_enc);
jccb(Assembler::equal, done);
@ -4386,12 +4387,12 @@ void C2_MacroAssembler::vector_cast_float_special_cases_avx(XMMRegister dst, XMM
bind(done);
}
void C2_MacroAssembler::vector_cast_float_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2,
Register scratch, AddressLiteral float_sign_flip,
int vec_enc) {
void C2_MacroAssembler::vector_cast_float_special_cases_evex(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2,
Register rscratch) {
Label done;
evmovdqul(xtmp1, k0, float_sign_flip, false, vec_enc, scratch);
evmovdqul(xtmp1, k0, float_sign_flip, false, vec_enc, rscratch);
Assembler::evpcmpeqd(ktmp1, k0, xtmp1, dst, vec_enc);
kortestwl(ktmp1, ktmp1);
jccb(Assembler::equal, done);
@ -4407,13 +4408,15 @@ void C2_MacroAssembler::vector_cast_float_special_cases_evex(XMMRegister dst, XM
bind(done);
}
void C2_MacroAssembler::vector_cast_float_to_long_special_cases_evex(
XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2,
Register scratch, AddressLiteral double_sign_flip,
int vec_enc) {
void C2_MacroAssembler::vector_cast_float_to_long_special_cases_evex(XMMRegister dst, XMMRegister src,
AddressLiteral double_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2,
Register rscratch) {
assert(rscratch != noreg || always_reachable(double_sign_flip), "missing");
Label done;
evmovdquq(xtmp1, k0, double_sign_flip, false, vec_enc, scratch);
evmovdquq(xtmp1, k0, double_sign_flip, false, vec_enc, rscratch);
Assembler::evpcmpeqq(ktmp1, k0, xtmp1, dst, vec_enc);
kortestwl(ktmp1, ktmp1);
jccb(Assembler::equal, done);
@ -4437,12 +4440,14 @@ void C2_MacroAssembler::vector_cast_float_to_long_special_cases_evex(
* If the src is positive infinity or any value greater than or equal to the value of Long.MAX_VALUE,
* the result is equal to the value of Long.MAX_VALUE.
*/
void C2_MacroAssembler::vector_cast_double_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2,
Register scratch, AddressLiteral double_sign_flip,
int vec_enc) {
void C2_MacroAssembler::vector_cast_double_special_cases_evex(XMMRegister dst, XMMRegister src,
AddressLiteral double_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2,
Register rscratch) {
assert(rscratch != noreg || always_reachable(double_sign_flip), "missing");
Label done;
evmovdqul(xtmp1, k0, double_sign_flip, false, vec_enc, scratch);
evmovdqul(xtmp1, k0, double_sign_flip, false, vec_enc, rscratch);
evpcmpeqq(ktmp1, xtmp1, dst, vec_enc);
kortestwl(ktmp1, ktmp1);
jccb(Assembler::equal, done);
@ -4468,38 +4473,48 @@ void C2_MacroAssembler::vector_cast_double_special_cases_evex(XMMRegister dst, X
* d) Replace 0x80000000 with MaxInt if source lane contains a +ve value.
*/
void C2_MacroAssembler::vector_castD2L_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
Register scratch, int vec_enc) {
void C2_MacroAssembler::vector_castD2L_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, Register rscratch) {
assert(rscratch != noreg || always_reachable(double_sign_flip), "missing");
evcvttpd2qq(dst, src, vec_enc);
vector_cast_double_special_cases_evex(dst, src, xtmp1, xtmp2, ktmp1, ktmp2, scratch, double_sign_flip, vec_enc);
vector_cast_double_special_cases_evex(dst, src, double_sign_flip, vec_enc,
xtmp1, xtmp2, ktmp1, ktmp2, rscratch);
}
void C2_MacroAssembler::vector_castF2I_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4,
AddressLiteral float_sign_flip, Register scratch, int vec_enc) {
void C2_MacroAssembler::vector_castF2I_avx(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, Register rscratch) {
assert(rscratch != noreg || always_reachable(float_sign_flip), "missing");
vcvttps2dq(dst, src, vec_enc);
vector_cast_float_special_cases_avx(dst, src, xtmp1, xtmp2, xtmp3, xtmp4, scratch, float_sign_flip, vec_enc);
vector_cast_float_special_cases_avx(dst, src, float_sign_flip, vec_enc,
xtmp1, xtmp2, xtmp3, xtmp4, rscratch);
}
void C2_MacroAssembler::vector_castF2I_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2, AddressLiteral float_sign_flip,
Register scratch, int vec_enc) {
void C2_MacroAssembler::vector_castF2I_evex(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, Register rscratch) {
assert(rscratch != noreg || always_reachable(float_sign_flip), "missing");
vcvttps2dq(dst, src, vec_enc);
vector_cast_float_special_cases_evex(dst, src, xtmp1, xtmp2, ktmp1, ktmp2, scratch, float_sign_flip, vec_enc);
vector_cast_float_special_cases_evex(dst, src, float_sign_flip, vec_enc,
xtmp1, xtmp2, ktmp1, ktmp2, rscratch);
}
void C2_MacroAssembler::vector_castF2L_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
Register scratch, int vec_enc) {
void C2_MacroAssembler::vector_castF2L_evex(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, Register rscratch) {
assert(rscratch != noreg || always_reachable(float_sign_flip), "missing");
evcvttps2qq(dst, src, vec_enc);
vector_cast_float_to_long_special_cases_evex(dst, src, xtmp1, xtmp2, ktmp1, ktmp2, scratch, double_sign_flip, vec_enc);
vector_cast_float_to_long_special_cases_evex(dst, src, float_sign_flip, vec_enc,
xtmp1, xtmp2, ktmp1, ktmp2, rscratch);
}
void C2_MacroAssembler::vector_castD2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2,
AddressLiteral double_sign_flip, Register scratch, int vec_enc) {
vector_castD2L_evex(dst, src, xtmp1, xtmp2, ktmp1, ktmp2, double_sign_flip, scratch, vec_enc);
void C2_MacroAssembler::vector_castD2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, Register rscratch) {
assert(rscratch != noreg || always_reachable(double_sign_flip), "missing");
vector_castD2L_evex(dst, src, double_sign_flip, vec_enc,
xtmp1, xtmp2, ktmp1, ktmp2, rscratch);
if (to_elem_bt != T_LONG) {
switch(to_elem_bt) {
case T_INT:
@ -4519,53 +4534,59 @@ void C2_MacroAssembler::vector_castD2X_evex(BasicType to_elem_bt, XMMRegister ds
}
#ifdef _LP64
void C2_MacroAssembler::vector_round_double_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
AddressLiteral new_mxcsr, Register scratch, int vec_enc) {
void C2_MacroAssembler::vector_round_double_evex(XMMRegister dst, XMMRegister src,
AddressLiteral double_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2) {
// Perform floor(val+0.5) operation under the influence of MXCSR.RC mode roundTowards -inf.
// and re-instantiate original MXCSR.RC mode after that.
ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
ldmxcsr(new_mxcsr, scratch);
mov64(scratch, julong_cast(0.5L));
evpbroadcastq(xtmp1, scratch, vec_enc);
ldmxcsr(new_mxcsr, tmp /*rscratch*/);
mov64(tmp, julong_cast(0.5L));
evpbroadcastq(xtmp1, tmp, vec_enc);
vaddpd(xtmp1, src , xtmp1, vec_enc);
evcvtpd2qq(dst, xtmp1, vec_enc);
vector_cast_double_special_cases_evex(dst, src, xtmp1, xtmp2, ktmp1, ktmp2, scratch, double_sign_flip, vec_enc);
ldmxcsr(mxcsr_std, scratch);
vector_cast_double_special_cases_evex(dst, src, double_sign_flip, vec_enc,
xtmp1, xtmp2, ktmp1, ktmp2, tmp);
ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), tmp /*rscratch*/);
}
void C2_MacroAssembler::vector_round_float_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2, AddressLiteral float_sign_flip,
AddressLiteral new_mxcsr, Register scratch, int vec_enc) {
void C2_MacroAssembler::vector_round_float_evex(XMMRegister dst, XMMRegister src,
AddressLiteral float_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2) {
// Perform floor(val+0.5) operation under the influence of MXCSR.RC mode roundTowards -inf.
// and re-instantiate original MXCSR.RC mode after that.
ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
ldmxcsr(new_mxcsr, scratch);
movl(scratch, jint_cast(0.5));
movq(xtmp1, scratch);
ldmxcsr(new_mxcsr, tmp /*rscratch*/);
movl(tmp, jint_cast(0.5));
movq(xtmp1, tmp);
vbroadcastss(xtmp1, xtmp1, vec_enc);
vaddps(xtmp1, src , xtmp1, vec_enc);
vcvtps2dq(dst, xtmp1, vec_enc);
vector_cast_float_special_cases_evex(dst, src, xtmp1, xtmp2, ktmp1, ktmp2, scratch, float_sign_flip, vec_enc);
ldmxcsr(mxcsr_std, scratch);
vector_cast_float_special_cases_evex(dst, src, float_sign_flip, vec_enc,
xtmp1, xtmp2, ktmp1, ktmp2, tmp);
ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), tmp /*rscratch*/);
}
void C2_MacroAssembler::vector_round_float_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
XMMRegister xtmp3, XMMRegister xtmp4, AddressLiteral float_sign_flip,
AddressLiteral new_mxcsr, Register scratch, int vec_enc) {
void C2_MacroAssembler::vector_round_float_avx(XMMRegister dst, XMMRegister src,
AddressLiteral float_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4) {
// Perform floor(val+0.5) operation under the influence of MXCSR.RC mode roundTowards -inf.
// and re-instantiate original MXCSR.RC mode after that.
ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
ldmxcsr(new_mxcsr, scratch);
movl(scratch, jint_cast(0.5));
movq(xtmp1, scratch);
ldmxcsr(new_mxcsr, tmp /*rscratch*/);
movl(tmp, jint_cast(0.5));
movq(xtmp1, tmp);
vbroadcastss(xtmp1, xtmp1, vec_enc);
vaddps(xtmp1, src , xtmp1, vec_enc);
vcvtps2dq(dst, xtmp1, vec_enc);
vector_cast_float_special_cases_avx(dst, src, xtmp1, xtmp2, xtmp3, xtmp4, scratch, float_sign_flip, vec_enc);
ldmxcsr(mxcsr_std, scratch);
vector_cast_float_special_cases_avx(dst, src, float_sign_flip, vec_enc,
xtmp1, xtmp2, xtmp3, xtmp4, tmp);
ldmxcsr(ExternalAddress(StubRoutines::x86::addr_mxcsr_std()), tmp /*rscratch*/);
}
#endif
#endif // _LP64
void C2_MacroAssembler::vector_unsigned_cast(XMMRegister dst, XMMRegister src, int vlen_enc,
BasicType from_elem_bt, BasicType to_elem_bt) {
@ -5125,12 +5146,14 @@ void C2_MacroAssembler::vector_reverse_bit(BasicType bt, XMMRegister dst, XMMReg
}
}
void C2_MacroAssembler::vector_reverse_bit_gfni(BasicType bt, XMMRegister dst, XMMRegister src,
XMMRegister xtmp, AddressLiteral mask, Register rtmp, int vec_enc) {
void C2_MacroAssembler::vector_reverse_bit_gfni(BasicType bt, XMMRegister dst, XMMRegister src, AddressLiteral mask, int vec_enc,
XMMRegister xtmp, Register rscratch) {
assert(VM_Version::supports_gfni(), "");
assert(rscratch != noreg || always_reachable(mask), "missing");
// Galois field instruction based bit reversal based on following algorithm.
// http://0x80.pl/articles/avx512-galois-field-for-bit-shuffling.html
assert(VM_Version::supports_gfni(), "");
vpbroadcastq(xtmp, mask, vec_enc, rtmp);
vpbroadcastq(xtmp, mask, vec_enc, rscratch);
vgf2p8affineqb(xtmp, src, xtmp, 0, vec_enc);
vector_reverse_byte(bt, dst, xtmp, vec_enc);
}

86
src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp

@ -148,15 +148,16 @@ public:
#endif
// blend
void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral adr, int comparison, int vector_len, Register rscratch = rscratch1);
void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, XMMRegister src2, int comparison, int vector_len);
void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, XMMRegister src2, int comparison, int vector_len);
void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral src2, int comparison, int vector_len, Register rscratch = noreg);
void evpblend(BasicType typ, XMMRegister dst, KRegister kmask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len);
void load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt, bool is_legacy);
void load_vector_mask(KRegister dst, XMMRegister src, XMMRegister xtmp, bool novlbwdq, int vlen_enc);
void load_vector(XMMRegister dst, Address src, int vlen_in_bytes);
void load_vector(XMMRegister dst, AddressLiteral src, int vlen_in_bytes, Register rscratch = noreg);
void load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt, bool is_legacy);
void load_vector_mask(KRegister dst, XMMRegister src, XMMRegister xtmp, bool novlbwdq, int vlen_enc);
void load_vector(XMMRegister dst, Address src, int vlen_in_bytes);
void load_vector(XMMRegister dst, AddressLiteral src, int vlen_in_bytes, Register rscratch = rscratch1);
void load_constant_vector(BasicType bt, XMMRegister dst, InternalAddress src, int vlen);
void load_iota_indices(XMMRegister dst, int vlen_in_bytes);
@ -307,60 +308,47 @@ public:
void masked_op(int ideal_opc, int mask_len, KRegister dst,
KRegister src1, KRegister src2);
void vector_castF2I_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4,
AddressLiteral float_sign_flip, Register scratch, int vec_enc);
void vector_castF2I_avx(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4, Register rscratch = noreg);
void vector_castF2I_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2, AddressLiteral float_sign_flip,
Register scratch, int vec_enc);
void vector_castF2I_evex(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, Register rscratch = noreg);
void vector_castF2L_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
Register scratch, int vec_enc);
void vector_castF2L_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, Register rscratch = noreg);
void vector_castD2L_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
Register scratch, int vec_enc);
void vector_castD2L_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, Register rscratch = noreg );
void vector_castD2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
Register scratch, int vec_enc);
void vector_castD2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, Register rscratch = noreg);
void vector_unsigned_cast(XMMRegister dst, XMMRegister src, int vlen_enc,
BasicType from_elem_bt, BasicType to_elem_bt);
void vector_unsigned_cast(XMMRegister dst, XMMRegister src, int vlen_enc, BasicType from_elem_bt, BasicType to_elem_bt);
void vector_cast_double_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2, Register scratch, AddressLiteral double_sign_flip,
int vec_enc);
void vector_cast_double_special_cases_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, Register rscratch = noreg );
void vector_cast_float_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2, Register scratch, AddressLiteral float_sign_flip,
int vec_enc);
void vector_cast_float_special_cases_evex(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, Register rscratch = noreg);
void vector_cast_float_to_long_special_cases_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2,
Register scratch, AddressLiteral double_sign_flip,
int vec_enc);
void vector_cast_float_to_long_special_cases_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2,
Register rscratch = noreg);
void vector_cast_float_special_cases_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4,
Register scratch, AddressLiteral float_sign_flip,
int vec_enc);
void vector_cast_float_special_cases_avx(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, int vec_enc,
XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4,
Register rscratch = noreg);
#ifdef _LP64
void vector_round_double_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
AddressLiteral new_mxcsr, Register scratch, int vec_enc);
void vector_round_double_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2);
void vector_round_float_evex(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
KRegister ktmp1, KRegister ktmp2, AddressLiteral double_sign_flip,
AddressLiteral new_mxcsr, Register scratch, int vec_enc);
void vector_round_float_evex(XMMRegister dst, XMMRegister src, AddressLiteral double_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2);
void vector_round_float_avx(XMMRegister dst, XMMRegister src, XMMRegister xtmp1, XMMRegister xtmp2,
XMMRegister xtmp3, XMMRegister xtmp4, AddressLiteral float_sign_flip,
AddressLiteral new_mxcsr, Register scratch, int vec_enc);
#endif
void vector_round_float_avx(XMMRegister dst, XMMRegister src, AddressLiteral float_sign_flip, AddressLiteral new_mxcsr, int vec_enc,
Register tmp, XMMRegister xtmp1, XMMRegister xtmp2, XMMRegister xtmp3, XMMRegister xtmp4);
#endif // _LP64
void udivI(Register rax, Register divisor, Register rdx);
void umodI(Register rax, Register divisor, Register rdx);
@ -385,8 +373,8 @@ public:
void vector_reverse_bit(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
XMMRegister xtmp2, Register rtmp, int vec_enc);
void vector_reverse_bit_gfni(BasicType bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp,
AddressLiteral mask, Register rtmp, int vec_enc);
void vector_reverse_bit_gfni(BasicType bt, XMMRegister dst, XMMRegister src, AddressLiteral mask, int vec_enc,
XMMRegister xtmp, Register rscratch = noreg);
void vector_reverse_byte(BasicType bt, XMMRegister dst, XMMRegister src, int vec_enc);

4
src/hotspot/cpu/x86/downcallLinker_x86_64.cpp

@ -190,7 +190,7 @@ void DowncallStubGenerator::generate() {
address the_pc = __ pc();
__ block_comment("{ thread java2native");
__ set_last_Java_frame(rsp, rbp, (address)the_pc);
__ set_last_Java_frame(rsp, rbp, (address)the_pc, rscratch1);
OopMap* map = new OopMap(_framesize, 0);
_oop_maps->add_gc_map(the_pc - start, map);
@ -246,7 +246,7 @@ void DowncallStubGenerator::generate() {
}
__ block_comment("{ thread native2java");
__ restore_cpu_control_state_after_jni();
__ restore_cpu_control_state_after_jni(rscratch1);
__ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);

11
src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp

@ -331,15 +331,12 @@ void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
__ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters.
__ jcc(Assembler::equal, bad_call);
#ifdef _LP64
Register tmp1 = rscratch1;
Register tmp2 = rscratch2;
#else
Register tmp1 = rax;
Register tmp2 = rcx;
Register tmp1 = LP64_ONLY( rscratch1 ) NOT_LP64( rax );
Register tmp2 = LP64_ONLY( rscratch2 ) NOT_LP64( rcx );
#ifndef _LP64
__ push(tmp1);
__ push(tmp2);
#endif // _LP64
#endif // !_LP64
// Pointer chase to the method holder to find out if the method is concurrently unloading.
Label method_live;

2
src/hotspot/cpu/x86/gc/shared/cardTableBarrierSetAssembler_x86.cpp

@ -112,7 +112,7 @@ void CardTableBarrierSetAssembler::store_check(MacroAssembler* masm, Register ob
// entry and that entry is not properly handled by the relocation code.
AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
Address index(noreg, obj, Address::times_1);
card_addr = __ as_Address(ArrayAddress(cardtable, index));
card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1);
}
int dirty = CardTable::dirty_card_val();

15
src/hotspot/cpu/x86/interp_masm_x86.cpp

@ -59,8 +59,7 @@ void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& md
jmpb(next);
bind(update);
Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
load_klass(obj, obj, tmp_load_klass);
load_klass(obj, obj, rscratch1);
xorptr(obj, mdo_addr);
testptr(obj, TypeEntries::type_klass_mask);
@ -880,13 +879,13 @@ void InterpreterMacroAssembler::dispatch_base(TosState state,
jccb(Assembler::zero, no_safepoint);
ArrayAddress dispatch_addr(ExternalAddress((address)safepoint_table), index);
jump(dispatch_addr);
jump(dispatch_addr, noreg);
bind(no_safepoint);
}
{
ArrayAddress dispatch_addr(ExternalAddress((address)table), index);
jump(dispatch_addr);
jump(dispatch_addr, noreg);
}
#endif // _LP64
}
@ -1003,7 +1002,7 @@ void InterpreterMacroAssembler::remove_activation(
jmp(fast_path);
bind(slow_path);
push(state);
set_last_Java_frame(rthread, noreg, rbp, (address)pc());
set_last_Java_frame(rthread, noreg, rbp, (address)pc(), rscratch1);
super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::at_unwind), rthread);
NOT_LP64(get_thread(rthread);) // call_VM clobbered it, restore
reset_last_Java_frame(rthread, true);
@ -1207,7 +1206,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) {
const Register swap_reg = rax; // Must use rax for cmpxchg instruction
const Register tmp_reg = rbx;
const Register obj_reg = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // Will contain the oop
const Register rklass_decode_tmp = LP64_ONLY(rscratch1) NOT_LP64(noreg);
const Register rklass_decode_tmp = rscratch1;
const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
@ -2012,7 +2011,7 @@ void InterpreterMacroAssembler::notify_method_entry() {
}
{
SkipIfEqual skip(this, &DTraceMethodProbes, false);
SkipIfEqual skip(this, &DTraceMethodProbes, false, rscratch1);
NOT_LP64(get_thread(rthread);)
get_method(rarg);
call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
@ -2057,7 +2056,7 @@ void InterpreterMacroAssembler::notify_method_exit(
}
{
SkipIfEqual skip(this, &DTraceMethodProbes, false);
SkipIfEqual skip(this, &DTraceMethodProbes, false, rscratch1);
push(state);
NOT_LP64(get_thread(rthread);)
get_method(rarg);

20
src/hotspot/cpu/x86/jniFastGetField_x86_64.cpp

@ -48,9 +48,6 @@ static const Register robj = r9;
static const Register roffset = r10;
static const Register rcounter = r11;
// Warning: do not use rip relative addressing after the first counter load
// since that may scratch r10!
address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
const char *name = NULL;
switch (type) {
@ -77,10 +74,9 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
__ jcc (Assembler::notZero, slow);
if (JvmtiExport::can_post_field_access()) {
// Check to see if a field access watch has been set before we
// take the fast path.
assert_different_registers(rscratch1, robj, rcounter); // cmp32 clobbers rscratch1!
__ cmp32(ExternalAddress((address) JvmtiExport::get_field_access_count_addr()), 0);
// Check to see if a field access watch has been set before we take the fast path.
assert_different_registers(rscratch1, robj, rcounter);
__ cmp32(ExternalAddress(JvmtiExport::get_field_access_count_addr()), 0, rscratch1);
__ jcc(Assembler::notZero, slow);
}
@ -104,7 +100,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
default: ShouldNotReachHere();
}
__ cmp32 (rcounter, counter);
__ cmp32 (rcounter, counter, rscratch1);
__ jcc (Assembler::notEqual, slow);
__ ret (0);
@ -122,7 +118,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
default: break;
}
// tail call
__ jump (ExternalAddress(slow_case_addr));
__ jump (ExternalAddress(slow_case_addr), rscratch1);
__ flush ();
@ -177,7 +173,7 @@ address JNI_FastGetField::generate_fast_get_float_field0(BasicType type) {
if (JvmtiExport::can_post_field_access()) {
// Check to see if a field access watch has been set before we
// take the fast path.
__ cmp32(ExternalAddress((address) JvmtiExport::get_field_access_count_addr()), 0);
__ cmp32(ExternalAddress(JvmtiExport::get_field_access_count_addr()), 0, rscratch1);
__ jcc(Assembler::notZero, slow);
}
@ -196,7 +192,7 @@ address JNI_FastGetField::generate_fast_get_float_field0(BasicType type) {
case T_DOUBLE: __ movdbl (xmm0, Address(robj, roffset, Address::times_1)); break;
default: ShouldNotReachHere();
}
__ cmp32 (rcounter, counter);
__ cmp32 (rcounter, counter, rscratch1);
__ jcc (Assembler::notEqual, slow);
__ ret (0);
@ -210,7 +206,7 @@ address JNI_FastGetField::generate_fast_get_float_field0(BasicType type) {
default: break;
}
// tail call
__ jump (ExternalAddress(slow_case_addr));
__ jump (ExternalAddress(slow_case_addr), rscratch1);
__ flush ();

803
src/hotspot/cpu/x86/macroAssembler_x86.cpp

File diff suppressed because it is too large Load Diff

453
src/hotspot/cpu/x86/macroAssembler_x86.hpp

@ -89,7 +89,7 @@ class MacroAssembler: public Assembler {
virtual void check_and_handle_earlyret(Register java_thread);
Address as_Address(AddressLiteral adr);
Address as_Address(ArrayAddress adr);
Address as_Address(ArrayAddress adr, Register rscratch);
// Support for NULL-checks
//
@ -169,7 +169,7 @@ class MacroAssembler: public Assembler {
else { movss (dst, src); return; }
}
void movflt(XMMRegister dst, Address src) { movss(dst, src); }
void movflt(XMMRegister dst, AddressLiteral src);
void movflt(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void movflt(Address dst, XMMRegister src) { movss(dst, src); }
// Move with zero extension
@ -181,7 +181,7 @@ class MacroAssembler: public Assembler {
else { movsd (dst, src); return; }
}
void movdbl(XMMRegister dst, AddressLiteral src);
void movdbl(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void movdbl(XMMRegister dst, Address src) {
if (UseXmmLoadAndClearUpper) { movsd (dst, src); return; }
@ -189,10 +189,10 @@ class MacroAssembler: public Assembler {
}
void movdbl(Address dst, XMMRegister src) { movsd(dst, src); }
void incrementl(AddressLiteral dst);
void incrementl(ArrayAddress dst);
void incrementl(AddressLiteral dst, Register rscratch = noreg);
void incrementl(ArrayAddress dst, Register rscratch);
void incrementq(AddressLiteral dst);
void incrementq(AddressLiteral dst, Register rscratch = noreg);
// Alignment
void align32();
@ -311,12 +311,14 @@ class MacroAssembler: public Assembler {
void set_last_Java_frame(Register thread,
Register last_java_sp,
Register last_java_fp,
address last_java_pc);
address last_java_pc,
Register rscratch);
// thread in the default location (r15_thread on 64bit)
void set_last_Java_frame(Register last_java_sp,
Register last_java_fp,
address last_java_pc);
address last_java_pc,
Register rscratch);
void reset_last_Java_frame(Register thread, bool clear_fp);
@ -337,9 +339,9 @@ class MacroAssembler: public Assembler {
void movbool(Address dst, Register src);
void testbool(Register dst);
void resolve_oop_handle(Register result, Register tmp = rscratch2);
void resolve_oop_handle(Register result, Register tmp);
void resolve_weak_handle(Register result, Register tmp);
void load_mirror(Register mirror, Register method, Register tmp = rscratch2);
void load_mirror(Register mirror, Register method, Register tmp);
void load_method_holder_cld(Register rresult, Register rmethod);
void load_method_holder(Register holder, Register method);
@ -660,7 +662,7 @@ public:
#define verify_klass_ptr(reg) _verify_klass_ptr(reg, "broken klass " #reg, __FILE__, __LINE__)
// Verify or restore cpu control state after JNI call
void restore_cpu_control_state_after_jni();
void restore_cpu_control_state_after_jni(Register rscratch);
// prints msg, dumps registers and stops execution
void stop(const char* msg);
@ -728,14 +730,14 @@ public:
void andptr(Register dst, int32_t src);
void andptr(Register src1, Register src2) { LP64_ONLY(andq(src1, src2)) NOT_LP64(andl(src1, src2)) ; }
void cmp8(AddressLiteral src1, int imm);
void cmp8(AddressLiteral src1, int imm, Register rscratch = noreg);
// renamed to drag out the casting of address to int32_t/intptr_t
void cmp32(Register src1, int32_t imm);
void cmp32(AddressLiteral src1, int32_t imm);
void cmp32(AddressLiteral src1, int32_t imm, Register rscratch = noreg);
// compare reg - mem, or reg - &mem
void cmp32(Register src1, AddressLiteral src2);
void cmp32(Register src1, AddressLiteral src2, Register rscratch = noreg);
void cmp32(Register src1, Address src2);
@ -747,12 +749,12 @@ public:
void cmpoop(Register src1, Register src2);
void cmpoop(Register src1, Address src2);
void cmpoop(Register dst, jobject obj);
void cmpoop(Register dst, jobject obj, Register rscratch);
// NOTE src2 must be the lval. This is NOT an mem-mem compare
void cmpptr(Address src1, AddressLiteral src2);
void cmpptr(Address src1, AddressLiteral src2, Register rscratch);
void cmpptr(Register src1, AddressLiteral src2);
void cmpptr(Register src1, AddressLiteral src2, Register rscratch = noreg);
void cmpptr(Register src1, Register src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
void cmpptr(Register src1, Address src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
@ -762,12 +764,11 @@ public:
void cmpptr(Address src1, int32_t src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
// cmp64 to avoild hiding cmpq
void cmp64(Register src1, AddressLiteral src, Register rscratch = rscratch1);
void cmp64(Register src1, AddressLiteral src, Register rscratch = noreg);
void cmpxchgptr(Register reg, Address adr);
void locked_cmpxchgptr(Register reg, AddressLiteral adr);
void locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch = noreg);
void imulptr(Register dst, Register src) { LP64_ONLY(imulq(dst, src)) NOT_LP64(imull(dst, src)); }
void imulptr(Register dst, Register src, int imm32) { LP64_ONLY(imulq(dst, src, imm32)) NOT_LP64(imull(dst, src, imm32)); }
@ -810,27 +811,27 @@ public:
// Helper functions for statistics gathering.
// Conditionally (atomically, on MPs) increments passed counter address, preserving condition codes.
void cond_inc32(Condition cond, AddressLiteral counter_addr);
void cond_inc32(Condition cond, AddressLiteral counter_addr, Register rscratch = noreg);
// Unconditional atomic increment.
void atomic_incl(Address counter_addr);
void atomic_incl(AddressLiteral counter_addr, Register scr = rscratch1);
void atomic_incl(AddressLiteral counter_addr, Register rscratch = noreg);
#ifdef _LP64
void atomic_incq(Address counter_addr);
void atomic_incq(AddressLiteral counter_addr, Register scr = rscratch1);
void atomic_incq(AddressLiteral counter_addr, Register rscratch = noreg);
#endif
void atomic_incptr(AddressLiteral counter_addr, Register scr = rscratch1) { LP64_ONLY(atomic_incq(counter_addr, scr)) NOT_LP64(atomic_incl(counter_addr, scr)) ; }
void atomic_incptr(AddressLiteral counter_addr, Register rscratch = noreg) { LP64_ONLY(atomic_incq(counter_addr, rscratch)) NOT_LP64(atomic_incl(counter_addr, rscratch)) ; }
void atomic_incptr(Address counter_addr) { LP64_ONLY(atomic_incq(counter_addr)) NOT_LP64(atomic_incl(counter_addr)) ; }
void lea(Register dst, Address adr) { Assembler::lea(dst, adr); }
void lea(Register dst, AddressLiteral adr);
void lea(Address dst, AddressLiteral adr);
void lea(Register dst, Address adr) { Assembler::lea(dst, adr); }
void lea(Address dst, AddressLiteral adr, Register rscratch);
void leal32(Register dst, Address src) { leal(dst, src); }
// Import other testl() methods from the parent class or else
// they will be hidden by the following overriding declaration.
using Assembler::testl;
void testl(Register dst, AddressLiteral src);
void testl(Register dst, AddressLiteral src); // requires reachable address
void orptr(Register dst, Address src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
void orptr(Register dst, Register src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
@ -853,7 +854,7 @@ public:
// NOTE: this call transfers to the effective address of entry NOT
// the address contained by entry. This is because this is more natural
// for jumps/calls.
void call(AddressLiteral entry);
void call(AddressLiteral entry, Register rscratch = rax);
// Emit the CompiledIC call idiom
void ic_call(address entry, jint method_index = 0);
@ -865,13 +866,14 @@ public:
// NOTE: these jumps transfer to the effective address of dst NOT
// the address contained by dst. This is because this is more natural
// for jumps/calls.
void jump(AddressLiteral dst);
void jump_cc(Condition cc, AddressLiteral dst);
void jump(AddressLiteral dst, Register rscratch = noreg);
void jump_cc(Condition cc, AddressLiteral dst, Register rscratch = noreg);
// 32bit can do a case table jump in one instruction but we no longer allow the base
// to be installed in the Address class. This jump will transfer to the address
// contained in the location described by entry (not the address of entry)
void jump(ArrayAddress entry);
void jump(ArrayAddress entry, Register rscratch);
// Floating
@ -880,45 +882,45 @@ public:
void push_d(XMMRegister r);
void pop_d(XMMRegister r);
void andpd(XMMRegister dst, Address src) { Assembler::andpd(dst, src); }
void andpd(XMMRegister dst, AddressLiteral src, Register scratch_reg = rscratch1);
void andpd(XMMRegister dst, XMMRegister src) { Assembler::andpd(dst, src); }
void andpd(XMMRegister dst, XMMRegister src) { Assembler::andpd(dst, src); }
void andpd(XMMRegister dst, Address src) { Assembler::andpd(dst, src); }
void andpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void andps(XMMRegister dst, XMMRegister src) { Assembler::andps(dst, src); }
void andps(XMMRegister dst, Address src) { Assembler::andps(dst, src); }
void andps(XMMRegister dst, AddressLiteral src, Register scratch_reg = rscratch1);
void andps(XMMRegister dst, XMMRegister src) { Assembler::andps(dst, src); }
void andps(XMMRegister dst, Address src) { Assembler::andps(dst, src); }
void andps(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void comiss(XMMRegister dst, XMMRegister src) { Assembler::comiss(dst, src); }
void comiss(XMMRegister dst, Address src) { Assembler::comiss(dst, src); }
void comiss(XMMRegister dst, AddressLiteral src);
void comiss(XMMRegister dst, XMMRegister src) { Assembler::comiss(dst, src); }
void comiss(XMMRegister dst, Address src) { Assembler::comiss(dst, src); }
void comiss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void comisd(XMMRegister dst, XMMRegister src) { Assembler::comisd(dst, src); }
void comisd(XMMRegister dst, Address src) { Assembler::comisd(dst, src); }
void comisd(XMMRegister dst, AddressLiteral src);
void comisd(XMMRegister dst, XMMRegister src) { Assembler::comisd(dst, src); }
void comisd(XMMRegister dst, Address src) { Assembler::comisd(dst, src); }
void comisd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
#ifndef _LP64
void fadd_s(Address src) { Assembler::fadd_s(src); }
void fadd_s(Address src) { Assembler::fadd_s(src); }
void fadd_s(AddressLiteral src) { Assembler::fadd_s(as_Address(src)); }
void fldcw(Address src) { Assembler::fldcw(src); }
void fldcw(Address src) { Assembler::fldcw(src); }
void fldcw(AddressLiteral src);
void fld_s(int index) { Assembler::fld_s(index); }
void fld_s(Address src) { Assembler::fld_s(src); }
void fld_s(int index) { Assembler::fld_s(index); }
void fld_s(Address src) { Assembler::fld_s(src); }
void fld_s(AddressLiteral src);
void fld_d(Address src) { Assembler::fld_d(src); }
void fld_d(Address src) { Assembler::fld_d(src); }
void fld_d(AddressLiteral src);
void fmul_s(Address src) { Assembler::fmul_s(src); }
void fmul_s(AddressLiteral src) { Assembler::fmul_s(as_Address(src)); }
#endif // _LP64
void fld_x(Address src) { Assembler::fld_x(src); }
void fld_x(AddressLiteral src) { Assembler::fld_x(as_Address(src)); }
void fld_x(Address src) { Assembler::fld_x(src); }
void fld_x(AddressLiteral src);
void fmul_s(Address src) { Assembler::fmul_s(src); }
void fmul_s(AddressLiteral src) { Assembler::fmul_s(as_Address(src)); }
#endif // !_LP64
void ldmxcsr(Address src) { Assembler::ldmxcsr(src); }
void ldmxcsr(AddressLiteral src, Register scratchReg = rscratch1);
void ldmxcsr(AddressLiteral src, Register rscratch = noreg);
#ifdef _LP64
private:
@ -1125,61 +1127,61 @@ private:
// these are private because users should be doing movflt/movdbl
void movss(XMMRegister dst, XMMRegister src) { Assembler::movss(dst, src); }
void movss(Address dst, XMMRegister src) { Assembler::movss(dst, src); }
void movss(XMMRegister dst, Address src) { Assembler::movss(dst, src); }
void movss(XMMRegister dst, AddressLiteral src);
void movss(Address dst, XMMRegister src) { Assembler::movss(dst, src); }
void movss(XMMRegister dst, XMMRegister src) { Assembler::movss(dst, src); }
void movss(XMMRegister dst, Address src) { Assembler::movss(dst, src); }
void movss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void movlpd(XMMRegister dst, Address src) {Assembler::movlpd(dst, src); }
void movlpd(XMMRegister dst, AddressLiteral src);
void movlpd(XMMRegister dst, Address src) {Assembler::movlpd(dst, src); }
void movlpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
public:
void addsd(XMMRegister dst, XMMRegister src) { Assembler::addsd(dst, src); }
void addsd(XMMRegister dst, Address src) { Assembler::addsd(dst, src); }
void addsd(XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void addsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void addss(XMMRegister dst, XMMRegister src) { Assembler::addss(dst, src); }
void addss(XMMRegister dst, Address src) { Assembler::addss(dst, src); }
void addss(XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void addss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void addpd(XMMRegister dst, XMMRegister src) { Assembler::addpd(dst, src); }
void addpd(XMMRegister dst, Address src) { Assembler::addpd(dst, src); }
void addpd(XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void addpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
using Assembler::vbroadcastsd;
void vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = rscratch1);
void vbroadcastsd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
using Assembler::vbroadcastss;
void vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = rscratch1);
void vbroadcastss(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
void divsd(XMMRegister dst, XMMRegister src) { Assembler::divsd(dst, src); }
void divsd(XMMRegister dst, Address src) { Assembler::divsd(dst, src); }
void divsd(XMMRegister dst, AddressLiteral src);
void divsd(XMMRegister dst, XMMRegister src) { Assembler::divsd(dst, src); }
void divsd(XMMRegister dst, Address src) { Assembler::divsd(dst, src); }
void divsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void divss(XMMRegister dst, XMMRegister src) { Assembler::divss(dst, src); }
void divss(XMMRegister dst, Address src) { Assembler::divss(dst, src); }
void divss(XMMRegister dst, AddressLiteral src);
void divss(XMMRegister dst, XMMRegister src) { Assembler::divss(dst, src); }
void divss(XMMRegister dst, Address src) { Assembler::divss(dst, src); }
void divss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
// Move Unaligned Double Quadword
void movdqu(Address dst, XMMRegister src);
void movdqu(XMMRegister dst, XMMRegister src);
void movdqu(XMMRegister dst, Address src);
void movdqu(XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void movdqu(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void kmovwl(Register dst, KRegister src) { Assembler::kmovwl(dst, src); }
void kmovwl(Address dst, KRegister src) { Assembler::kmovwl(dst, src); }
void kmovwl(KRegister dst, KRegister src) { Assembler::kmovwl(dst, src); }
void kmovwl(KRegister dst, Register src) { Assembler::kmovwl(dst, src); }
void kmovwl(KRegister dst, Address src) { Assembler::kmovwl(dst, src); }
void kmovwl(KRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void kmovwl(KRegister dst, AddressLiteral src, Register rscratch = noreg);
void kmovql(KRegister dst, KRegister src) { Assembler::kmovql(dst, src); }
void kmovql(KRegister dst, Register src) { Assembler::kmovql(dst, src); }
void kmovql(Register dst, KRegister src) { Assembler::kmovql(dst, src); }
void kmovql(KRegister dst, Address src) { Assembler::kmovql(dst, src); }
void kmovql(Address dst, KRegister src) { Assembler::kmovql(dst, src); }
void kmovql(KRegister dst, AddressLiteral src, Register scratch_reg = rscratch1);
void kmovql(KRegister dst, KRegister src) { Assembler::kmovql(dst, src); }
void kmovql(KRegister dst, Register src) { Assembler::kmovql(dst, src); }
void kmovql(Register dst, KRegister src) { Assembler::kmovql(dst, src); }
void kmovql(KRegister dst, Address src) { Assembler::kmovql(dst, src); }
void kmovql(Address dst, KRegister src) { Assembler::kmovql(dst, src); }
void kmovql(KRegister dst, AddressLiteral src, Register rscratch = noreg);
// Safe move operation, lowers down to 16bit moves for targets supporting
// AVX512F feature and 64bit moves for targets supporting AVX512BW feature.
@ -1190,16 +1192,17 @@ public:
void kmov(KRegister dst, Register src);
using Assembler::movddup;
void movddup(XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void movddup(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
using Assembler::vmovddup;
void vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = rscratch1);
void vmovddup(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
// AVX Unaligned forms
void vmovdqu(Address dst, XMMRegister src);
void vmovdqu(XMMRegister dst, Address src);
void vmovdqu(XMMRegister dst, XMMRegister src);
void vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = rscratch1);
void vmovdqu(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void vmovdqu(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
// AVX512 Unaligned
void evmovdqu(BasicType type, KRegister kmask, Address dst, XMMRegister src, bool merge, int vector_len);
@ -1213,28 +1216,29 @@ public:
Assembler::evmovdqub(dst, mask, src, merge, vector_len);
}
}
void evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
void evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
void evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register scratch_reg);
void evmovdqub(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
void evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
void evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
void evmovdquw(Address dst, XMMRegister src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
void evmovdquw(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
void evmovdquw(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
void evmovdquw(Address dst, XMMRegister src, int vector_len) { Assembler::evmovdquw(dst, src, vector_len); }
void evmovdquw(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
if (dst->encoding() != src->encoding() || mask != k0) {
Assembler::evmovdquw(dst, mask, src, merge, vector_len);
}
}
void evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdquw(dst, mask, src, merge, vector_len); }
void evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdquw(dst, mask, src, merge, vector_len); }
void evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register scratch_reg);
void evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdquw(dst, mask, src, merge, vector_len); }
void evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdquw(dst, mask, src, merge, vector_len); }
void evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
void evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
if (dst->encoding() != src->encoding()) {
Assembler::evmovdqul(dst, src, vector_len);
}
}
void evmovdqul(Address dst, XMMRegister src, int vector_len) { Assembler::evmovdqul(dst, src, vector_len); }
void evmovdqul(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdqul(dst, src, vector_len); }
void evmovdqul(Address dst, XMMRegister src, int vector_len) { Assembler::evmovdqul(dst, src, vector_len); }
void evmovdqul(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdqul(dst, src, vector_len); }
void evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
if (dst->encoding() != src->encoding() || mask != k0) {
@ -1243,7 +1247,7 @@ public:
}
void evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdqul(dst, mask, src, merge, vector_len); }
void evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdqul(dst, mask, src, merge, vector_len); }
void evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch);
void evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
void evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
if (dst->encoding() != src->encoding()) {
@ -1264,26 +1268,26 @@ public:
void evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
// Move Aligned Double Quadword
void movdqa(XMMRegister dst, Address src) { Assembler::movdqa(dst, src); }
void movdqa(XMMRegister dst, XMMRegister src) { Assembler::movdqa(dst, src); }
void movdqa(XMMRegister dst, AddressLiteral src);
void movdqa(XMMRegister dst, XMMRegister src) { Assembler::movdqa(dst, src); }
void movdqa(XMMRegister dst, Address src) { Assembler::movdqa(dst, src); }
void movdqa(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void movsd(Address dst, XMMRegister src) { Assembler::movsd(dst, src); }
void movsd(XMMRegister dst, XMMRegister src) { Assembler::movsd(dst, src); }
void movsd(XMMRegister dst, Address src) { Assembler::movsd(dst, src); }
void movsd(XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void movsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void mulpd(XMMRegister dst, XMMRegister src) { Assembler::mulpd(dst, src); }
void mulpd(XMMRegister dst, Address src) { Assembler::mulpd(dst, src); }
void mulpd(XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void mulpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void mulsd(XMMRegister dst, XMMRegister src) { Assembler::mulsd(dst, src); }
void mulsd(XMMRegister dst, Address src) { Assembler::mulsd(dst, src); }
void mulsd(XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void mulsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void mulss(XMMRegister dst, XMMRegister src) { Assembler::mulss(dst, src); }
void mulss(XMMRegister dst, Address src) { Assembler::mulss(dst, src); }
void mulss(XMMRegister dst, AddressLiteral src);
void mulss(XMMRegister dst, XMMRegister src) { Assembler::mulss(dst, src); }
void mulss(XMMRegister dst, Address src) { Assembler::mulss(dst, src); }
void mulss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
// Carry-Less Multiplication Quadword
void pclmulldq(XMMRegister dst, XMMRegister src) {
@ -1308,67 +1312,63 @@ public:
void ptest(XMMRegister dst, XMMRegister src);
void sqrtsd(XMMRegister dst, XMMRegister src) { Assembler::sqrtsd(dst, src); }
void sqrtsd(XMMRegister dst, Address src) { Assembler::sqrtsd(dst, src); }
void sqrtsd(XMMRegister dst, AddressLiteral src);
void roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) { Assembler::roundsd(dst, src, rmode); }
void roundsd(XMMRegister dst, Address src, int32_t rmode) { Assembler::roundsd(dst, src, rmode); }
void roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch);
void roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register rscratch = noreg);
void sqrtss(XMMRegister dst, XMMRegister src) { Assembler::sqrtss(dst, src); }
void sqrtss(XMMRegister dst, Address src) { Assembler::sqrtss(dst, src); }
void sqrtss(XMMRegister dst, AddressLiteral src);
void sqrtss(XMMRegister dst, XMMRegister src) { Assembler::sqrtss(dst, src); }
void sqrtss(XMMRegister dst, Address src) { Assembler::sqrtss(dst, src); }
void sqrtss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void subsd(XMMRegister dst, XMMRegister src) { Assembler::subsd(dst, src); }
void subsd(XMMRegister dst, Address src) { Assembler::subsd(dst, src); }
void subsd(XMMRegister dst, AddressLiteral src);
void subsd(XMMRegister dst, XMMRegister src) { Assembler::subsd(dst, src); }
void subsd(XMMRegister dst, Address src) { Assembler::subsd(dst, src); }
void subsd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void subss(XMMRegister dst, XMMRegister src) { Assembler::subss(dst, src); }
void subss(XMMRegister dst, Address src) { Assembler::subss(dst, src); }
void subss(XMMRegister dst, AddressLiteral src);
void subss(XMMRegister dst, XMMRegister src) { Assembler::subss(dst, src); }
void subss(XMMRegister dst, Address src) { Assembler::subss(dst, src); }
void subss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void ucomiss(XMMRegister dst, XMMRegister src) { Assembler::ucomiss(dst, src); }
void ucomiss(XMMRegister dst, Address src) { Assembler::ucomiss(dst, src); }
void ucomiss(XMMRegister dst, AddressLiteral src);
void ucomiss(XMMRegister dst, XMMRegister src) { Assembler::ucomiss(dst, src); }
void ucomiss(XMMRegister dst, Address src) { Assembler::ucomiss(dst, src); }
void ucomiss(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
void ucomisd(XMMRegister dst, XMMRegister src) { Assembler::ucomisd(dst, src); }
void ucomisd(XMMRegister dst, Address src) { Assembler::ucomisd(dst, src); }
void ucomisd(XMMRegister dst, AddressLiteral src);
void ucomisd(XMMRegister dst, XMMRegister src) { Assembler::ucomisd(dst, src); }
void ucomisd(XMMRegister dst, Address src) { Assembler::ucomisd(dst, src); }
void ucomisd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
// Bitwise Logical XOR of Packed Double-Precision Floating-Point Values
void xorpd(XMMRegister dst, XMMRegister src);
void xorpd(XMMRegister dst, Address src) { Assembler::xorpd(dst, src); }
void xorpd(XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void xorpd(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
// Bitwise Logical XOR of Packed Single-Precision Floating-Point Values
void xorps(XMMRegister dst, XMMRegister src);
void xorps(XMMRegister dst, Address src) { Assembler::xorps(dst, src); }
void xorps(XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void xorps(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
// Shuffle Bytes
void pshufb(XMMRegister dst, XMMRegister src) { Assembler::pshufb(dst, src); }
void pshufb(XMMRegister dst, Address src) { Assembler::pshufb(dst, src); }
void pshufb(XMMRegister dst, AddressLiteral src);
void pshufb(XMMRegister dst, XMMRegister src) { Assembler::pshufb(dst, src); }
void pshufb(XMMRegister dst, Address src) { Assembler::pshufb(dst, src); }
void pshufb(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
// AVX 3-operands instructions
void vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vaddsd(dst, nds, src); }
void vaddsd(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vaddsd(dst, nds, src); }
void vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
void vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vaddsd(dst, nds, src); }
void vaddsd(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vaddsd(dst, nds, src); }
void vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
void vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vaddss(dst, nds, src); }
void vaddss(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vaddss(dst, nds, src); }
void vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
void vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vaddss(dst, nds, src); }
void vaddss(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vaddss(dst, nds, src); }
void vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
void vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len);
void vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len);
void vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch = noreg);
void vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len, Register rscratch = noreg);
void vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch);
void vpaddb(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
void vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpaddd(dst, nds, src, vector_len); }
void vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vpaddd(dst, nds, src, vector_len); }
@ -1376,36 +1376,39 @@ public:
void vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpand(dst, nds, src, vector_len); }
void vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vpand(dst, nds, src, vector_len); }
void vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = rscratch1);
void vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
using Assembler::vpbroadcastd;
void vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = rscratch1);
void vpbroadcastd(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
using Assembler::vpbroadcastq;
void vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = rscratch1);
void vpbroadcastq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch = noreg);
void vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg);
void evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
// Vector compares
void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
int comparison, bool is_signed, int vector_len) { Assembler::evpcmpd(kdst, mask, nds, src, comparison, is_signed, vector_len); }
void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
int comparison, bool is_signed, int vector_len, Register scratch_reg);
void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
int comparison, bool is_signed, int vector_len) { Assembler::evpcmpq(kdst, mask, nds, src, comparison, is_signed, vector_len); }
void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
int comparison, bool is_signed, int vector_len, Register scratch_reg);
void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
int comparison, bool is_signed, int vector_len) { Assembler::evpcmpb(kdst, mask, nds, src, comparison, is_signed, vector_len); }
void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
int comparison, bool is_signed, int vector_len, Register scratch_reg);
void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
int comparison, bool is_signed, int vector_len) { Assembler::evpcmpw(kdst, mask, nds, src, comparison, is_signed, vector_len); }
void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
int comparison, bool is_signed, int vector_len, Register scratch_reg);
void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int comparison, bool is_signed, int vector_len) {
Assembler::evpcmpd(kdst, mask, nds, src, comparison, is_signed, vector_len);
}
void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int comparison, bool is_signed, int vector_len) {
Assembler::evpcmpq(kdst, mask, nds, src, comparison, is_signed, vector_len);
}
void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int comparison, bool is_signed, int vector_len) {
Assembler::evpcmpb(kdst, mask, nds, src, comparison, is_signed, vector_len);
}
void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, int comparison, bool is_signed, int vector_len) {
Assembler::evpcmpw(kdst, mask, nds, src, comparison, is_signed, vector_len);
}
void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int comparison, bool is_signed, int vector_len, Register rscratch = noreg);
void evpbroadcast(BasicType type, XMMRegister dst, Register src, int vector_len);
@ -1413,32 +1416,29 @@ public:
void vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, XMMRegister xtmp, ComparisonPredicate cond, Width width, int vector_len);
void vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len);
void vpmovzxbw(XMMRegister dst, Address src, int vector_len);
void vpmovzxbw(XMMRegister dst, Address src, int vector_len);
void vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::vpmovzxbw(dst, src, vector_len); }
void vpmovmskb(Register dst, XMMRegister src, int vector_len = Assembler::AVX_256bit);
void vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
Assembler::vpmulld(dst, nds, src, vector_len);
};
void vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
Assembler::vpmulld(dst, nds, src, vector_len);
}
void vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg);
void vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpmulld(dst, nds, src, vector_len); }
void vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vpmulld(dst, nds, src, vector_len); }
void vpmulld(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
void vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
void vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
void vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
void vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
void vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
void evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
void evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
void evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
void evpsllw(XMMRegister dst, KRegister mask, XMMRegister nds, XMMRegister src, bool merge, int vector_len, bool is_varshift) {
if (!is_varshift) {
@ -1527,52 +1527,52 @@ public:
void pshuflw(XMMRegister dst, XMMRegister src, int mode);
void pshuflw(XMMRegister dst, Address src, int mode) { Assembler::pshuflw(dst, src, mode); }
void vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vandpd(dst, nds, src, vector_len); }
void vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vandpd(dst, nds, src, vector_len); }
void vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg = rscratch1);
void vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vandpd(dst, nds, src, vector_len); }
void vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vandpd(dst, nds, src, vector_len); }
void vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
void vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vandps(dst, nds, src, vector_len); }
void vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vandps(dst, nds, src, vector_len); }
void vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg = rscratch1);
void vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vandps(dst, nds, src, vector_len); }
void vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vandps(dst, nds, src, vector_len); }
void vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
void evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register scratch_reg);
void evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register rscratch = noreg);
void vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vdivsd(dst, nds, src); }
void vdivsd(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vdivsd(dst, nds, src); }
void vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
void vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vdivsd(dst, nds, src); }
void vdivsd(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vdivsd(dst, nds, src); }
void vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
void vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vdivss(dst, nds, src); }
void vdivss(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vdivss(dst, nds, src); }
void vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
void vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vdivss(dst, nds, src); }
void vdivss(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vdivss(dst, nds, src); }
void vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
void vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vmulsd(dst, nds, src); }
void vmulsd(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vmulsd(dst, nds, src); }
void vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
void vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vmulsd(dst, nds, src); }
void vmulsd(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vmulsd(dst, nds, src); }
void vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
void vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vmulss(dst, nds, src); }
void vmulss(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vmulss(dst, nds, src); }
void vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
void vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vmulss(dst, nds, src); }
void vmulss(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vmulss(dst, nds, src); }
void vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
void vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vsubsd(dst, nds, src); }
void vsubsd(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vsubsd(dst, nds, src); }
void vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
void vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vsubsd(dst, nds, src); }
void vsubsd(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vsubsd(dst, nds, src); }
void vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vsubss(dst, nds, src); }
void vsubss(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vsubss(dst, nds, src); }
void vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vsubss(dst, nds, src); }
void vsubss(XMMRegister dst, XMMRegister nds, Address src) { Assembler::vsubss(dst, nds, src); }
void vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
void vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src);
void vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
void vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
void vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src, Register rscratch = noreg);
// AVX Vector instructions
void vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
void vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
void vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg = rscratch1);
void vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
void vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
void vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
void vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
void vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
void vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg = rscratch1);
void vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
void vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
void vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
void vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
if (UseAVX > 1 || (vector_len < 1)) // vpxor 256 bit is available only in AVX2
@ -1586,7 +1586,7 @@ public:
else
Assembler::vxorpd(dst, nds, src, vector_len);
}
void vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg = rscratch1);
void vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
// Simple version for AVX2 256bit vectors
void vpxor(XMMRegister dst, XMMRegister src) {
@ -1598,8 +1598,8 @@ public:
Assembler::vpxor(dst, dst, src, AVX_256bit);
}
void vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpermd(dst, nds, src, vector_len); }
void vpermd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg);
void vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpermd(dst, nds, src, vector_len); }
void vpermd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch = noreg);
void vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
@ -1841,10 +1841,10 @@ public:
void cmovptr(Condition cc, Register dst, Register src) { LP64_ONLY(cmovq(cc, dst, src)) NOT_LP64(cmov32(cc, dst, src)); }
void movoop(Register dst, jobject obj);
void movoop(Address dst, jobject obj);
void movoop(Address dst, jobject obj, Register rscratch);
void mov_metadata(Register dst, Metadata* obj);
void mov_metadata(Address dst, Metadata* obj);
void mov_metadata(Address dst, Metadata* obj, Register rscratch);
void movptr(Register dst, Register src);
void movptr(Register dst, Address src);
@ -1852,47 +1852,36 @@ public:
void movptr(Register dst, ArrayAddress src);
void movptr(Register dst, intptr_t src);
void movptr(Address dst, Register src);
void movptr(Address dst, intptr_t src);
void movptr(ArrayAddress dst, Register src);
void movptr(Address dst, int32_t imm);
void movptr(Address dst, intptr_t src, Register rscratch);
void movptr(ArrayAddress dst, Register src, Register rscratch);
void movptr(Register dst, RegisterOrConstant src) {
if (src.is_constant()) movptr(dst, src.as_constant());
else movptr(dst, src.as_register());
}
#ifdef _LP64
// Generally the next two are only used for moving NULL
// Although there are situations in initializing the mark word where
// they could be used. They are dangerous.
// They only exist on LP64 so that int32_t and intptr_t are not the same
// and we have ambiguous declarations.
void movptr(Address dst, int32_t imm32);
#endif // _LP64
// to avoid hiding movl
void mov32(AddressLiteral dst, Register src);
void mov32(Register dst, AddressLiteral src);
// to avoid hiding movb
void movbyte(ArrayAddress dst, int src);
void mov32(Register dst, AddressLiteral src);
void mov32(AddressLiteral dst, Register src, Register rscratch = noreg);
// Import other mov() methods from the parent class or else
// they will be hidden by the following overriding declaration.
using Assembler::movdl;
void movdl(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
using Assembler::movq;
void movdl(XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void movq (XMMRegister dst, AddressLiteral src, Register rscratch = rscratch1);
void movq(XMMRegister dst, AddressLiteral src, Register rscratch = noreg);
// Can push value or effective address
void pushptr(AddressLiteral src);
void pushptr(AddressLiteral src, Register rscratch);
void pushptr(Address src) { LP64_ONLY(pushq(src)) NOT_LP64(pushl(src)); }
void popptr(Address src) { LP64_ONLY(popq(src)) NOT_LP64(popl(src)); }
void pushoop(jobject obj);
void pushklass(Metadata* obj);
void pushoop(jobject obj, Register rscratch);
void pushklass(Metadata* obj, Register rscratch);
// sign extend as need a l to ptr sized element
void movl2ptr(Register dst, Address src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src)); }
@ -2117,7 +2106,7 @@ class SkipIfEqual {
Label _label;
public:
SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value);
SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value, Register rscratch);
~SkipIfEqual();
};

21
src/hotspot/cpu/x86/methodHandles_x86.cpp

@ -78,28 +78,29 @@ void MethodHandles::verify_klass(MacroAssembler* _masm,
InstanceKlass** klass_addr = vmClasses::klass_addr_at(klass_id);
Klass* klass = vmClasses::klass_at(klass_id);
Register temp = rdi;
Register temp2 = noreg;
LP64_ONLY(temp2 = rscratch1); // used by MacroAssembler::cmpptr and load_klass
Label L_ok, L_bad;
BLOCK_COMMENT("verify_klass {");
__ verify_oop(obj);
__ testptr(obj, obj);
__ jcc(Assembler::zero, L_bad);
__ push(temp); if (temp2 != noreg) __ push(temp2);
#define UNPUSH { if (temp2 != noreg) __ pop(temp2); __ pop(temp); }
__ load_klass(temp, obj, temp2);
__ cmpptr(temp, ExternalAddress((address) klass_addr));
#define PUSH { __ push(temp); LP64_ONLY( __ push(rscratch1); ) }
#define POP { LP64_ONLY( __ pop(rscratch1); ) __ pop(temp); }
PUSH;
__ load_klass(temp, obj, rscratch1);
__ cmpptr(temp, ExternalAddress((address) klass_addr), rscratch1);
__ jcc(Assembler::equal, L_ok);
intptr_t super_check_offset = klass->super_check_offset();
__ movptr(temp, Address(temp, super_check_offset));
__ cmpptr(temp, ExternalAddress((address) klass_addr));
__ cmpptr(temp, ExternalAddress((address) klass_addr), rscratch1);
__ jcc(Assembler::equal, L_ok);
UNPUSH;
POP;
__ bind(L_bad);
__ STOP(error_message);
__ BIND(L_ok);
UNPUSH;
POP;
BLOCK_COMMENT("} verify_klass");
#undef POP
#undef PUSH
}
void MethodHandles::verify_ref_kind(MacroAssembler* _masm, int ref_kind, Register member_reg, Register temp) {
@ -672,7 +673,7 @@ void MethodHandles::trace_method_handle(MacroAssembler* _masm, const char* adapt
__ push(rbx); // pusha saved_regs
__ push(rcx); // mh
__ push(rcx); // slot for adaptername
__ movptr(Address(rsp, 0), (intptr_t) adaptername);
__ movptr(Address(rsp, 0), (intptr_t) adaptername, rscratch1);
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, trace_method_handle_stub_wrapper), rsp);
__ increment(rsp, sizeof(MethodHandleStubArguments));

2
src/hotspot/cpu/x86/runtime_x86_32.cpp

@ -109,7 +109,7 @@ void OptoRuntime::generate_exception_blob() {
// registers of the frame being removed.
//
__ movptr(Address(rsp, thread_off * wordSize), rcx); // Thread is first argument
__ set_last_Java_frame(rcx, noreg, noreg, NULL);
__ set_last_Java_frame(rcx, noreg, noreg, NULL, noreg);
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, OptoRuntime::handle_exception_C)));

20
src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp

@ -1660,14 +1660,14 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
intptr_t the_pc = (intptr_t) __ pc();
oop_maps->add_gc_map(the_pc - start, map);
__ set_last_Java_frame(thread, rsp, noreg, (address)the_pc);
__ set_last_Java_frame(thread, rsp, noreg, (address)the_pc, noreg);
// We have all of the arguments setup at this point. We must not touch any register
// argument registers at this point (what if we save/restore them there are no oop?
{
SkipIfEqual skip_if(masm, &DTraceMethodProbes, 0);
SkipIfEqual skip_if(masm, &DTraceMethodProbes, 0, noreg);
__ mov_metadata(rax, method());
__ call_VM_leaf(
CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
@ -1760,7 +1760,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ call(RuntimeAddress(native_func));
// Verify or restore cpu control state after JNI call
__ restore_cpu_control_state_after_jni();
__ restore_cpu_control_state_after_jni(noreg);
// WARNING - on Windows Java Natives use pascal calling convention and pop the
// arguments off of the stack. We could just re-adjust the stack pointer here
@ -1903,7 +1903,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
}
{
SkipIfEqual skip_if(masm, &DTraceMethodProbes, 0);
SkipIfEqual skip_if(masm, &DTraceMethodProbes, 0, noreg);
// Tell dtrace about this method exit
save_native_result(masm, ret_type, stack_slots);
__ mov_metadata(rax, method());
@ -2233,7 +2233,7 @@ void SharedRuntime::generate_deopt_blob() {
__ get_thread(rcx);
__ push(rcx);
// fetch_unroll_info needs to call last_java_frame()
__ set_last_Java_frame(rcx, noreg, noreg, NULL);
__ set_last_Java_frame(rcx, noreg, noreg, NULL, noreg);
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info)));
@ -2381,7 +2381,7 @@ void SharedRuntime::generate_deopt_blob() {
__ push(rcx);
// set last_Java_sp, last_Java_fp
__ set_last_Java_frame(rcx, noreg, rbp, NULL);
__ set_last_Java_frame(rcx, noreg, rbp, NULL, noreg);
// Call C code. Need thread but NOT official VM entry
// crud. We cannot block on this call, no GC can happen. Call should
@ -2478,7 +2478,7 @@ void SharedRuntime::generate_uncommon_trap_blob() {
// set last_Java_sp
__ get_thread(rdx);
__ set_last_Java_frame(rdx, noreg, noreg, NULL);
__ set_last_Java_frame(rdx, noreg, noreg, NULL, noreg);
// Call C code. Need thread but NOT official VM entry
// crud. We cannot block on this call, no GC can happen. Call should
@ -2590,7 +2590,7 @@ void SharedRuntime::generate_uncommon_trap_blob() {
// set last_Java_sp, last_Java_fp
__ get_thread(rdi);
__ set_last_Java_frame(rdi, noreg, rbp, NULL);
__ set_last_Java_frame(rdi, noreg, rbp, NULL, noreg);
// Call C code. Need thread but NOT official VM entry
// crud. We cannot block on this call, no GC can happen. Call should
@ -2672,7 +2672,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_t
// Push thread argument and setup last_Java_sp
__ get_thread(java_thread);
__ push(java_thread);
__ set_last_Java_frame(java_thread, noreg, noreg, NULL);
__ set_last_Java_frame(java_thread, noreg, noreg, NULL, noreg);
// if this was not a poll_return then we need to correct the return address now.
if (!cause_return) {
@ -2811,7 +2811,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const cha
__ get_thread(rdi);
__ push(thread);
__ set_last_Java_frame(thread, noreg, rbp, NULL);
__ set_last_Java_frame(thread, noreg, rbp, NULL, noreg);
__ call(RuntimeAddress(destination));

30
src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp

@ -1019,7 +1019,7 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
Register method = rbx;
{ // Bypass the barrier for non-static methods
Register flags = rscratch1;
Register flags = rscratch1;
__ movl(flags, Address(method, Method::access_flags_offset()));
__ testl(flags, JVM_ACC_STATIC);
__ jcc(Assembler::zero, L_skip_barrier); // non-static
@ -1697,7 +1697,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
Label hit;
Label exception_pending;
assert_different_registers(ic_reg, receiver, rscratch1);
assert_different_registers(ic_reg, receiver, rscratch1, rscratch2);
__ verify_oop(receiver);
__ load_klass(rscratch1, receiver, rscratch2);
__ cmpq(ic_reg, rscratch1);
@ -1906,14 +1906,14 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
intptr_t the_pc = (intptr_t) __ pc();
oop_maps->add_gc_map(the_pc - start, map);
__ set_last_Java_frame(rsp, noreg, (address)the_pc);
__ set_last_Java_frame(rsp, noreg, (address)the_pc, rscratch1);
// We have all of the arguments setup at this point. We must not touch any register
// argument registers at this point (what if we save/restore them there are no oop?
{
SkipIfEqual skip(masm, &DTraceMethodProbes, false);
SkipIfEqual skip(masm, &DTraceMethodProbes, false, rscratch1);
// protect the args we've loaded
save_args(masm, total_c_args, c_arg, out_regs);
__ mov_metadata(c_rarg1, method());
@ -2015,7 +2015,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ call(RuntimeAddress(native_func));
// Verify or restore cpu control state after JNI call
__ restore_cpu_control_state_after_jni();
__ restore_cpu_control_state_after_jni(rscratch1);
// Unpack native results.
switch (ret_type) {
@ -2144,7 +2144,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ bind(fast_done);
}
{
SkipIfEqual skip(masm, &DTraceMethodProbes, false);
SkipIfEqual skip(masm, &DTraceMethodProbes, false, rscratch1);
save_native_result(masm, ret_type, stack_slots);
__ mov_metadata(c_rarg1, method());
__ call_VM_leaf(
@ -2418,7 +2418,7 @@ void SharedRuntime::generate_deopt_blob() {
// Save everything in sight.
RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_wide_vectors*/ true);
// fetch_unroll_info needs to call last_java_frame()
__ set_last_Java_frame(noreg, noreg, NULL);
__ set_last_Java_frame(noreg, noreg, NULL, rscratch1);
__ movl(c_rarg1, Address(r15_thread, in_bytes(JavaThread::pending_deoptimization_offset())));
__ movl(Address(r15_thread, in_bytes(JavaThread::pending_deoptimization_offset())), -1);
@ -2500,7 +2500,7 @@ void SharedRuntime::generate_deopt_blob() {
// fetch_unroll_info needs to call last_java_frame().
__ set_last_Java_frame(noreg, noreg, NULL);
__ set_last_Java_frame(noreg, noreg, NULL, rscratch1);
#ifdef ASSERT
{ Label L;
__ cmpptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
@ -2648,7 +2648,7 @@ void SharedRuntime::generate_deopt_blob() {
// Save "the_pc" since it cannot easily be retrieved using the last_java_SP after we aligned SP.
// Don't need the precise return PC here, just precise enough to point into this code blob.
address the_pc = __ pc();
__ set_last_Java_frame(noreg, rbp, the_pc);
__ set_last_Java_frame(noreg, rbp, the_pc, rscratch1);
__ andptr(rsp, -(StackAlignmentInBytes)); // Fix stack alignment as required by ABI
__ mov(c_rarg0, r15_thread);
@ -2719,7 +2719,7 @@ void SharedRuntime::generate_uncommon_trap_blob() {
// runtime expects it.
__ movl(c_rarg1, j_rarg0);
__ set_last_Java_frame(noreg, noreg, NULL);
__ set_last_Java_frame(noreg, noreg, NULL, rscratch1);
// Call C code. Need thread but NOT official VM entry
// crud. We cannot block on this call, no GC can happen. Call should
@ -2836,7 +2836,7 @@ void SharedRuntime::generate_uncommon_trap_blob() {
// Save "the_pc" since it cannot easily be retrieved using the last_java_SP after we aligned SP.
// Don't need the precise return PC here, just precise enough to point into this code blob.
address the_pc = __ pc();
__ set_last_Java_frame(noreg, rbp, the_pc);
__ set_last_Java_frame(noreg, rbp, the_pc, rscratch1);
// Call C code. Need thread but NOT official VM entry
// crud. We cannot block on this call, no GC can happen. Call should
@ -2913,7 +2913,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_t
// address of the call in order to generate an oopmap. Hence, we do all the
// work ourselves.
__ set_last_Java_frame(noreg, noreg, NULL); // JavaFrameAnchor::capture_last_Java_pc() will get the pc from the return address, which we store next:
__ set_last_Java_frame(noreg, noreg, NULL, rscratch1); // JavaFrameAnchor::capture_last_Java_pc() will get the pc from the return address, which we store next:
// The return address must always be correct so that frame constructor never
// sees an invalid pc.
@ -3049,7 +3049,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const cha
ResourceMark rm;
CodeBuffer buffer(name, 1200, 512);
MacroAssembler* masm = new MacroAssembler(&buffer);
MacroAssembler* masm = new MacroAssembler(&buffer);
int frame_size_in_words;
@ -3063,7 +3063,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const cha
int frame_complete = __ offset();
__ set_last_Java_frame(noreg, noreg, NULL);
__ set_last_Java_frame(noreg, noreg, NULL, rscratch1);
__ mov(c_rarg0, r15_thread);
@ -3459,7 +3459,7 @@ void OptoRuntime::generate_exception_blob() {
// At a method handle call, the stack may not be properly aligned
// when returning with an exception.
address the_pc = __ pc();
__ set_last_Java_frame(noreg, noreg, the_pc);
__ set_last_Java_frame(noreg, noreg, the_pc, rscratch1);
__ mov(c_rarg0, r15_thread);
__ andptr(rsp, -(StackAlignmentInBytes)); // Align stack
__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, OptoRuntime::handle_exception_C)));

12
src/hotspot/cpu/x86/stubGenerator_x86_32.cpp

@ -343,10 +343,10 @@ class StubGenerator: public StubCodeGenerator {
#endif
// set pending exception
__ verify_oop(rax);
__ movptr(Address(rcx, Thread::pending_exception_offset()), rax );
__ lea(Address(rcx, Thread::exception_file_offset ()),
ExternalAddress((address)__FILE__));
__ movl(Address(rcx, Thread::exception_line_offset ()), __LINE__ );
__ movptr(Address(rcx, Thread::pending_exception_offset()), rax);
__ lea(Address(rcx, Thread::exception_file_offset()),
ExternalAddress((address)__FILE__), noreg);
__ movl(Address(rcx, Thread::exception_line_offset()), __LINE__ );
// complete return to VM
assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");
__ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
@ -3846,7 +3846,7 @@ class StubGenerator: public StubCodeGenerator {
}
// Set up last_Java_sp and last_Java_fp
__ set_last_Java_frame(java_thread, rsp, rbp, NULL);
__ set_last_Java_frame(java_thread, rsp, rbp, NULL, noreg);
// Call runtime
BLOCK_COMMENT("call runtime_entry");
@ -3930,7 +3930,7 @@ class StubGenerator: public StubCodeGenerator {
static void jfr_prologue(address the_pc, MacroAssembler* masm) {
Register java_thread = rdi;
__ get_thread(java_thread);
__ set_last_Java_frame(java_thread, rsp, rbp, the_pc);
__ set_last_Java_frame(java_thread, rsp, rbp, the_pc, noreg);
__ movptr(Address(rsp, 0), java_thread);
}

75
src/hotspot/cpu/x86/stubGenerator_x86_64.cpp

@ -85,15 +85,14 @@ class StubGenerator: public StubCodeGenerator {
private:
#ifdef PRODUCT
#define inc_counter_np(counter) ((void)0)
#define INC_COUNTER_NP(counter, rscratch) ((void)0)
#else
void inc_counter_np_(int& counter) {
// This can destroy rscratch1 if counter is far from the code cache
__ incrementl(ExternalAddress((address)&counter));
void inc_counter_np(int& counter, Register rscratch) {
__ incrementl(ExternalAddress((address)&counter), rscratch);
}
#define inc_counter_np(counter) \
#define INC_COUNTER_NP(counter, rscratch) \
BLOCK_COMMENT("inc_counter " #counter); \
inc_counter_np_(counter);
inc_counter_np(counter, rscratch);
#endif
// Call stubs are used to call Java from C
@ -297,9 +296,9 @@ class StubGenerator: public StubCodeGenerator {
__ movl(rax, mxcsr_save);
__ andl(rax, MXCSR_MASK); // Only check control and mask bits
ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
__ cmp32(rax, mxcsr_std);
__ cmp32(rax, mxcsr_std, rscratch1);
__ jcc(Assembler::equal, skip_ldmx);
__ ldmxcsr(mxcsr_std);
__ ldmxcsr(mxcsr_std, rscratch1);
__ bind(skip_ldmx);
}
#endif
@ -617,12 +616,12 @@ class StubGenerator: public StubCodeGenerator {
__ stmxcsr(mxcsr_save);
__ movl(rax, mxcsr_save);
__ andl(rax, MXCSR_MASK); // Only check control and mask bits
__ cmp32(rax, mxcsr_std);
__ cmp32(rax, mxcsr_std, rscratch1);
__ jcc(Assembler::equal, ok_ret);
__ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
__ ldmxcsr(mxcsr_std);
__ ldmxcsr(mxcsr_std, rscratch1);
__ bind(ok_ret);
__ addptr(rsp, wordSize);
@ -1038,7 +1037,7 @@ class StubGenerator: public StubCodeGenerator {
Label exit, error;
__ pushf();
__ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
__ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()), rscratch1);
__ push(r12);
@ -1670,7 +1669,7 @@ class StubGenerator: public StubCodeGenerator {
}
bs->arraycopy_epilogue(_masm, decorators, type, from, to, count);
restore_argument_regs(type);
inc_counter_np(get_profile_ctr(shift)); // Update counter after rscratch1 is free
INC_COUNTER_NP(get_profile_ctr(shift), rscratch1); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -1845,7 +1844,7 @@ class StubGenerator: public StubCodeGenerator {
}
bs->arraycopy_epilogue(_masm, decorators, type, from, to, count);
restore_argument_regs(type);
inc_counter_np(get_profile_ctr(shift)); // Update counter after rscratch1 is free
INC_COUNTER_NP(get_profile_ctr(shift), rscratch1); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -1959,7 +1958,7 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(L_exit);
address ucme_exit_pc = __ pc();
restore_arg_regs();
inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
INC_COUNTER_NP(SharedRuntime::_jbyte_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2060,7 +2059,7 @@ class StubGenerator: public StubCodeGenerator {
__ jcc(Assembler::notZero, L_copy_8_bytes);
}
restore_arg_regs();
inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
INC_COUNTER_NP(SharedRuntime::_jbyte_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2073,7 +2072,7 @@ class StubGenerator: public StubCodeGenerator {
copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
}
restore_arg_regs();
inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
INC_COUNTER_NP(SharedRuntime::_jbyte_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2179,7 +2178,7 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(L_exit);
address ucme_exit_pc = __ pc();
restore_arg_regs();
inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
INC_COUNTER_NP(SharedRuntime::_jshort_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2295,7 +2294,7 @@ class StubGenerator: public StubCodeGenerator {
__ jcc(Assembler::notZero, L_copy_8_bytes);
}
restore_arg_regs();
inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
INC_COUNTER_NP(SharedRuntime::_jshort_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2308,7 +2307,7 @@ class StubGenerator: public StubCodeGenerator {
copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
}
restore_arg_regs();
inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
INC_COUNTER_NP(SharedRuntime::_jshort_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2415,7 +2414,7 @@ class StubGenerator: public StubCodeGenerator {
address ucme_exit_pc = __ pc();
bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
restore_arg_regs_using_thread();
inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
__ vzeroupper();
__ xorptr(rax, rax); // return 0
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2520,7 +2519,7 @@ class StubGenerator: public StubCodeGenerator {
__ jmp(L_exit);
}
restore_arg_regs_using_thread();
inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2536,7 +2535,7 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(L_exit);
bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
restore_arg_regs_using_thread();
inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
INC_COUNTER_NP(SharedRuntime::_jint_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2628,7 +2627,7 @@ class StubGenerator: public StubCodeGenerator {
__ jmp(L_exit);
} else {
restore_arg_regs_using_thread();
inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
INC_COUNTER_NP(SharedRuntime::_jlong_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2645,11 +2644,9 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(L_exit);
bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
restore_arg_regs_using_thread();
if (is_oop) {
inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
} else {
inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
}
INC_COUNTER_NP(is_oop ? SharedRuntime::_oop_array_copy_ctr :
SharedRuntime::_jlong_array_copy_ctr,
rscratch1); // Update counter after rscratch1 is free
__ vzeroupper();
__ xorptr(rax, rax); // return 0
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2730,7 +2727,7 @@ class StubGenerator: public StubCodeGenerator {
__ jmp(L_exit);
} else {
restore_arg_regs_using_thread();
inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
INC_COUNTER_NP(SharedRuntime::_jlong_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
__ xorptr(rax, rax); // return 0
__ vzeroupper();
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2746,11 +2743,9 @@ class StubGenerator: public StubCodeGenerator {
__ BIND(L_exit);
bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
restore_arg_regs_using_thread();
if (is_oop) {
inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
} else {
inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
}
INC_COUNTER_NP(is_oop ? SharedRuntime::_oop_array_copy_ctr :
SharedRuntime::_jlong_array_copy_ctr,
rscratch1); // Update counter after rscratch1 is free
__ vzeroupper();
__ xorptr(rax, rax); // return 0
__ leave(); // required for proper stackwalking of RuntimeStub frame
@ -2970,7 +2965,7 @@ class StubGenerator: public StubCodeGenerator {
__ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
__ movptr(r10, Address(rsp, saved_r10_offset * wordSize));
restore_arg_regs();
inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); // Update counter after rscratch1 is free
INC_COUNTER_NP(SharedRuntime::_checkcast_array_copy_ctr, rscratch1); // Update counter after rscratch1 is free
__ leave(); // required for proper stackwalking of RuntimeStub frame
__ ret(0);
@ -3011,7 +3006,7 @@ class StubGenerator: public StubCodeGenerator {
__ enter(); // required for proper stackwalking of RuntimeStub frame
// bump this on entry, not on exit:
inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
INC_COUNTER_NP(SharedRuntime::_unsafe_array_copy_ctr, rscratch1);
__ mov(bits, from);
__ orptr(bits, to);
@ -3134,7 +3129,7 @@ class StubGenerator: public StubCodeGenerator {
#endif
// bump this on entry, not on exit:
inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
INC_COUNTER_NP(SharedRuntime::_generic_array_copy_ctr, rscratch1);
//-----------------------------------------------------------------------
// Assembler stub will be used for this call to arraycopy
@ -7434,7 +7429,7 @@ address generate_avx_ghash_processBlocks() {
OopMap* map = new OopMap(framesize, 1);
oop_maps->add_gc_map(frame_complete, map);
__ set_last_Java_frame(rsp, rbp, the_pc);
__ set_last_Java_frame(rsp, rbp, the_pc, rscratch1);
__ movptr(c_rarg0, r15_thread);
__ movptr(c_rarg1, rsp);
__ call_VM_leaf(Continuation::freeze_entry(), 2);
@ -7626,7 +7621,7 @@ address generate_avx_ghash_processBlocks() {
int frame_complete = the_pc - start;
__ set_last_Java_frame(rsp, rbp, the_pc);
__ set_last_Java_frame(rsp, rbp, the_pc, rscratch1);
__ movptr(c_rarg0, r15_thread);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::write_checkpoint), 1);
__ reset_last_Java_frame(true);
@ -7719,7 +7714,7 @@ address generate_avx_ghash_processBlocks() {
// Set up last_Java_sp and last_Java_fp
address the_pc = __ pc();
__ set_last_Java_frame(rsp, rbp, the_pc);
__ set_last_Java_frame(rsp, rbp, the_pc, rscratch1);
__ andptr(rsp, -(StackAlignmentInBytes)); // Align stack
// Call runtime

29
src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp

@ -581,7 +581,7 @@ void TemplateInterpreterGenerator::lock_method() {
// get receiver (assume this is frequent case)
__ movptr(rax, Address(rlocals, Interpreter::local_offset_in_bytes(0)));
__ jcc(Assembler::zero, done);
__ load_mirror(rax, rbx);
__ load_mirror(rax, rbx, rscratch2);
#ifdef ASSERT
{
@ -625,7 +625,7 @@ void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
__ lea(rbcp, Address(rbcp, ConstMethod::codes_offset())); // get codebase
__ push(rbx); // save Method*
// Get mirror and store it in the frame as GC root for this Method*
__ load_mirror(rdx, rbx);
__ load_mirror(rdx, rbx, rscratch2);
__ push(rdx);
if (ProfileInterpreter) {
Label method_data_continue;
@ -1000,7 +1000,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
Label L;
__ movptr(rax, Address(method, Method::native_function_offset()));
ExternalAddress unsatisfied(SharedRuntime::native_method_throw_unsatisfied_link_error_entry());
__ cmpptr(rax, unsatisfied.addr());
__ cmpptr(rax, unsatisfied.addr(), rscratch1);
__ jcc(Assembler::notEqual, L);
__ call_VM(noreg,
CAST_FROM_FN_PTR(address,
@ -1020,13 +1020,13 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
// set_last_Java_frame_before_call
// It is enough that the pc()
// points into the right code segment. It does not have to be the correct return pc.
__ set_last_Java_frame(thread, noreg, rbp, __ pc());
__ set_last_Java_frame(thread, noreg, rbp, __ pc(), noreg);
#else
__ lea(c_rarg0, Address(r15_thread, JavaThread::jni_environment_offset()));
// It is enough that the pc() points into the right code
// segment. It does not have to be the correct return pc.
__ set_last_Java_frame(rsp, rbp, (address) __ pc());
__ set_last_Java_frame(rsp, rbp, (address) __ pc(), rscratch1);
#endif // _LP64
// change thread state
@ -1052,7 +1052,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
// 64: result potentially in rax or xmm0
// Verify or restore cpu control state after JNI call
__ restore_cpu_control_state_after_jni();
__ restore_cpu_control_state_after_jni(rscratch1);
// NOTE: The order of these pushes is known to frame::interpreter_frame_result
// in order to extract the result of a method call. If the order of these
@ -1075,10 +1075,10 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
ExternalAddress float_handler(AbstractInterpreter::result_handler(T_FLOAT));
ExternalAddress double_handler(AbstractInterpreter::result_handler(T_DOUBLE));
__ cmpptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset + 1)*wordSize),
float_handler.addr());
float_handler.addr(), noreg);
__ jcc(Assembler::equal, push_double);
__ cmpptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset + 1)*wordSize),
double_handler.addr());
double_handler.addr(), noreg);
__ jcc(Assembler::notEqual, L);
__ bind(push_double);
__ push_d(); // FP values are returned using the FPU, so push FPU contents (even if UseSSE > 0).
@ -1619,14 +1619,14 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
__ movptr(rbx, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
__ get_thread(thread);
// PC must point into interpreter here
__ set_last_Java_frame(thread, noreg, rbp, __ pc());
__ set_last_Java_frame(thread, noreg, rbp, __ pc(), noreg);
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), thread, rax, rbx);
__ get_thread(thread);
#else
__ mov(c_rarg1, rsp);
__ movptr(c_rarg2, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
// PC must point into interpreter here
__ set_last_Java_frame(noreg, rbp, __ pc());
__ set_last_Java_frame(noreg, rbp, __ pc(), rscratch1);
__ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), r15_thread, c_rarg1, c_rarg2);
#endif
__ reset_last_Java_frame(thread, true);
@ -1822,11 +1822,11 @@ address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
}
void TemplateInterpreterGenerator::count_bytecode() {
__ incrementl(ExternalAddress((address) &BytecodeCounter::_counter_value));
__ incrementl(ExternalAddress((address) &BytecodeCounter::_counter_value), rscratch1);
}
void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
__ incrementl(ExternalAddress((address) &BytecodeHistogram::_counters[t->bytecode()]));
__ incrementl(ExternalAddress((address) &BytecodeHistogram::_counters[t->bytecode()]), rscratch1);
}
void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
@ -1835,7 +1835,7 @@ void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
__ orl(rbx,
((int) t->bytecode()) <<
BytecodePairHistogram::log2_number_of_codes);
__ mov32(ExternalAddress((address) &BytecodePairHistogram::_index), rbx);
__ mov32(ExternalAddress((address) &BytecodePairHistogram::_index), rbx, rscratch1);
__ lea(rscratch1, ExternalAddress((address) BytecodePairHistogram::_counters));
__ incrementl(Address(rscratch1, rbx, Address::times_4));
}
@ -1863,7 +1863,8 @@ void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
void TemplateInterpreterGenerator::stop_interpreter_at() {
Label L;
__ cmp32(ExternalAddress((address) &BytecodeCounter::_counter_value),
StopInterpreterAt);
StopInterpreterAt,
rscratch1);
__ jcc(Assembler::notEqual, L);
__ int3();
__ bind(L);

81
src/hotspot/cpu/x86/templateTable_x86.cpp

@ -289,10 +289,10 @@ void TemplateTable::fconst(int value) {
__ xorps(xmm0, xmm0);
break;
case 1:
__ movflt(xmm0, ExternalAddress((address) &one));
__ movflt(xmm0, ExternalAddress((address) &one), rscratch1);
break;
case 2:
__ movflt(xmm0, ExternalAddress((address) &two));
__ movflt(xmm0, ExternalAddress((address) &two), rscratch1);
break;
default:
ShouldNotReachHere();
@ -320,7 +320,7 @@ void TemplateTable::dconst(int value) {
__ xorpd(xmm0, xmm0);
break;
case 1:
__ movdbl(xmm0, ExternalAddress((address) &one));
__ movdbl(xmm0, ExternalAddress((address) &one), rscratch1);
break;
default:
ShouldNotReachHere();
@ -446,7 +446,7 @@ void TemplateTable::fast_aldc(bool wide) {
Label notNull;
ExternalAddress null_sentinel((address)Universe::the_null_sentinel_addr());
__ movptr(tmp, null_sentinel);
__ resolve_oop_handle(tmp);
__ resolve_oop_handle(tmp, rscratch2);
__ cmpoop(tmp, result);
__ jccb(Assembler::notEqual, notNull);
__ xorptr(result, result); // NULL object reference
@ -1127,11 +1127,10 @@ void TemplateTable::aastore() {
__ testptr(rax, rax);
__ jcc(Assembler::zero, is_null);
Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
// Move subklass into rbx
__ load_klass(rbx, rax, tmp_load_klass);
__ load_klass(rbx, rax, rscratch1);
// Move superklass into rax
__ load_klass(rax, rdx, tmp_load_klass);
__ load_klass(rax, rdx, rscratch1);
__ movptr(rax, Address(rax,
ObjArrayKlass::element_klass_offset()));
@ -1174,8 +1173,7 @@ void TemplateTable::bastore() {
index_check(rdx, rbx); // prefer index in rbx
// Need to check whether array is boolean or byte
// since both types share the bastore bytecode.
Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
__ load_klass(rcx, rdx, tmp_load_klass);
__ load_klass(rcx, rdx, rscratch1);
__ movl(rcx, Address(rcx, Klass::layout_helper_offset()));
int diffbit = Klass::layout_helper_boolean_diffbit();
__ testl(rcx, diffbit);
@ -1546,7 +1544,7 @@ void TemplateTable::fop2(Operation op) {
__ movflt(xmm1, xmm0);
__ pop_f(xmm0);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::frem), 2);
#else
#else // !_LP64
__ push_f(xmm0);
__ pop_f();
__ fld_s(at_rsp());
@ -1555,7 +1553,7 @@ void TemplateTable::fop2(Operation op) {
__ pop(rax); // pop second operand off the stack
__ push_f();
__ pop_f(xmm0);
#endif
#endif // _LP64
break;
default:
ShouldNotReachHere();
@ -1564,7 +1562,7 @@ void TemplateTable::fop2(Operation op) {
} else {
#ifdef _LP64
ShouldNotReachHere();
#else
#else // !_LP64
switch (op) {
case add: __ fadd_s (at_rsp()); break;
case sub: __ fsubr_s(at_rsp()); break;
@ -1609,7 +1607,7 @@ void TemplateTable::dop2(Operation op) {
__ movdbl(xmm1, xmm0);
__ pop_d(xmm0);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::drem), 2);
#else
#else // !_LP64
__ push_d(xmm0);
__ pop_d();
__ fld_d(at_rsp());
@ -1619,7 +1617,7 @@ void TemplateTable::dop2(Operation op) {
__ pop(rdx);
__ push_d();
__ pop_d(xmm0);
#endif
#endif // _LP64
break;
default:
ShouldNotReachHere();
@ -1628,7 +1626,7 @@ void TemplateTable::dop2(Operation op) {
} else {
#ifdef _LP64
ShouldNotReachHere();
#else
#else // !_LP64
switch (op) {
case add: __ fadd_d (at_rsp()); break;
case sub: __ fsubr_d(at_rsp()); break;
@ -1657,7 +1655,7 @@ void TemplateTable::dop2(Operation op) {
// Pop double precision number from rsp.
__ pop(rax);
__ pop(rdx);
#endif
#endif // _LP64
}
}
@ -1691,7 +1689,7 @@ void TemplateTable::fneg() {
transition(ftos, ftos);
if (UseSSE >= 1) {
static jlong *float_signflip = double_quadword(&float_signflip_pool[1], CONST64(0x8000000080000000), CONST64(0x8000000080000000));
__ xorps(xmm0, ExternalAddress((address) float_signflip));
__ xorps(xmm0, ExternalAddress((address) float_signflip), rscratch1);
} else {
LP64_ONLY(ShouldNotReachHere());
NOT_LP64(__ fchs());
@ -1703,7 +1701,7 @@ void TemplateTable::dneg() {
if (UseSSE >= 2) {
static jlong *double_signflip =
double_quadword(&double_signflip_pool[1], CONST64(0x8000000000000000), CONST64(0x8000000000000000));
__ xorpd(xmm0, ExternalAddress((address) double_signflip));
__ xorpd(xmm0, ExternalAddress((address) double_signflip), rscratch1);
} else {
#ifdef _LP64
ShouldNotReachHere();
@ -1824,7 +1822,7 @@ void TemplateTable::convert() {
Label L;
__ cvttss2siq(rax, xmm0);
// NaN or overflow/underflow?
__ cmp64(rax, ExternalAddress((address) &is_nan));
__ cmp64(rax, ExternalAddress((address) &is_nan), rscratch1);
__ jcc(Assembler::notEqual, L);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::f2l), 1);
__ bind(L);
@ -1848,7 +1846,7 @@ void TemplateTable::convert() {
Label L;
__ cvttsd2siq(rax, xmm0);
// NaN or overflow/underflow?
__ cmp64(rax, ExternalAddress((address) &is_nan));
__ cmp64(rax, ExternalAddress((address) &is_nan), rscratch1);
__ jcc(Assembler::notEqual, L);
__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::d2l), 1);
__ bind(L);
@ -1860,7 +1858,7 @@ void TemplateTable::convert() {
default:
ShouldNotReachHere();
}
#else
#else // !_LP64
// Checking
#ifdef ASSERT
{ TosState tos_in = ilgl;
@ -2051,7 +2049,7 @@ void TemplateTable::convert() {
default :
ShouldNotReachHere();
}
#endif
#endif // _LP64
}
void TemplateTable::lcmp() {
@ -2105,7 +2103,7 @@ void TemplateTable::float_cmp(bool is_float, int unordered_result) {
} else {
#ifdef _LP64
ShouldNotReachHere();
#else
#else // !_LP64
if (is_float) {
__ fld_s(at_rsp());
} else {
@ -2568,8 +2566,7 @@ void TemplateTable::_return(TosState state) {
assert(state == vtos, "only valid state");
Register robj = LP64_ONLY(c_rarg1) NOT_LP64(rax);
__ movptr(robj, aaddress(0));
Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
__ load_klass(rdi, robj, tmp_load_klass);
__ load_klass(rdi, robj, rscratch1);
__ movl(rdi, Address(rdi, Klass::access_flags_offset()));
__ testl(rdi, JVM_ACC_HAS_FINALIZER);
Label skip_register_finalizer;
@ -2717,7 +2714,7 @@ void TemplateTable::load_field_cp_cache_entry(Register obj,
ConstantPoolCacheEntry::f1_offset())));
const int mirror_offset = in_bytes(Klass::java_mirror_offset());
__ movptr(obj, Address(obj, mirror_offset));
__ resolve_oop_handle(obj);
__ resolve_oop_handle(obj, rscratch2);
}
}
@ -3609,9 +3606,12 @@ void TemplateTable::prepare_invoke(int byte_no,
{
const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
ExternalAddress table(table_addr);
LP64_ONLY(__ lea(rscratch1, table));
LP64_ONLY(__ movptr(flags, Address(rscratch1, flags, Address::times_ptr)));
NOT_LP64(__ movptr(flags, ArrayAddress(table, Address(noreg, flags, Address::times_ptr))));
#ifdef _LP64
__ lea(rscratch1, table);
__ movptr(flags, Address(rscratch1, flags, Address::times_ptr));
#else
__ movptr(flags, ArrayAddress(table, Address(noreg, flags, Address::times_ptr)));
#endif // _LP64
}
// push return address
@ -3659,8 +3659,7 @@ void TemplateTable::invokevirtual_helper(Register index,
// get receiver klass
__ null_check(recv, oopDesc::klass_offset_in_bytes());
Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
__ load_klass(rax, recv, tmp_load_klass);
__ load_klass(rax, recv, rscratch1);
// profile this call
__ profile_virtual_call(rax, rlocals, rdx);
@ -3752,8 +3751,7 @@ void TemplateTable::invokeinterface(int byte_no) {
// Get receiver klass into rlocals - also a null check
__ null_check(rcx, oopDesc::klass_offset_in_bytes());
Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
__ load_klass(rlocals, rcx, tmp_load_klass);
__ load_klass(rlocals, rcx, rscratch1);
Label subtype;
__ check_klass_subtype(rlocals, rax, rbcp, subtype);
@ -3776,7 +3774,7 @@ void TemplateTable::invokeinterface(int byte_no) {
// Get receiver klass into rdx - also a null check
__ restore_locals(); // restore r14
__ null_check(rcx, oopDesc::klass_offset_in_bytes());
__ load_klass(rdx, rcx, tmp_load_klass);
__ load_klass(rdx, rcx, rscratch1);
Label no_such_method;
@ -4005,11 +4003,10 @@ void TemplateTable::_new() {
__ xorl(rsi, rsi); // use zero reg to clear memory (shorter code)
__ store_klass_gap(rax, rsi); // zero klass gap for compressed oops
#endif
Register tmp_store_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
__ store_klass(rax, rcx, tmp_store_klass); // klass
__ store_klass(rax, rcx, rscratch1); // klass
{
SkipIfEqual skip_if(_masm, &DTraceAllocProbes, 0);
SkipIfEqual skip_if(_masm, &DTraceAllocProbes, 0, rscratch1);
// Trigger dtrace event for fastpath
__ push(atos);
__ call_VM_leaf(
@ -4100,8 +4097,7 @@ void TemplateTable::checkcast() {
__ load_resolved_klass_at_index(rax, rcx, rbx);
__ bind(resolved);
Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
__ load_klass(rbx, rdx, tmp_load_klass);
__ load_klass(rbx, rdx, rscratch1);
// Generate subtype check. Blows rcx, rdi. Object in rdx.
// Superklass in rax. Subklass in rbx.
@ -4158,13 +4154,12 @@ void TemplateTable::instanceof() {
__ pop_ptr(rdx); // restore receiver
__ verify_oop(rdx);
Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
__ load_klass(rdx, rdx, tmp_load_klass);
__ load_klass(rdx, rdx, rscratch1);
__ jmpb(resolved);
// Get superklass in rax and subklass in rdx
__ bind(quicked);
__ load_klass(rdx, rax, tmp_load_klass);
__ load_klass(rdx, rax, rscratch1);
__ load_resolved_klass_at_index(rax, rcx, rbx);
__ bind(resolved);
@ -4403,7 +4398,7 @@ void TemplateTable::wide() {
transition(vtos, vtos);
__ load_unsigned_byte(rbx, at_bcp(1));
ExternalAddress wtable((address)Interpreter::_wentry_point);
__ jump(ArrayAddress(wtable, Address(noreg, rbx, Address::times_ptr)));
__ jump(ArrayAddress(wtable, Address(noreg, rbx, Address::times_ptr)), rscratch1);
// Note: the rbcp increment step is part of the individual wide bytecode implementations
}

4
src/hotspot/cpu/x86/upcallLinker_x86_64.cpp

@ -115,9 +115,9 @@ static void preserve_callee_saved_registers(MacroAssembler* _masm, const ABIDesc
__ movl(rax, mxcsr_save);
__ andl(rax, MXCSR_MASK); // Only check control and mask bits
ExternalAddress mxcsr_std(StubRoutines::x86::addr_mxcsr_std());
__ cmp32(rax, mxcsr_std);
__ cmp32(rax, mxcsr_std, rscratch1);
__ jcc(Assembler::equal, skip_ldmx);
__ ldmxcsr(mxcsr_std);
__ ldmxcsr(mxcsr_std, rscratch1);
__ bind(skip_ldmx);
}
#endif

7
src/hotspot/cpu/x86/vtableStubs_x86_64.cpp

@ -48,7 +48,6 @@ extern "C" void bad_compiled_vtable_index(JavaThread* thread, oop receiver, int
VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
// Read "A word on VtableStub sizing" in share/code/vtableStubs.hpp for details on stub sizing.
const int stub_code_length = code_size_limit(true);
Register tmp_load_klass = rscratch1;
VtableStub* s = new(stub_code_length) VtableStub(true, vtable_index);
// Can be NULL if there is no free space in the code cache.
if (s == NULL) {
@ -70,7 +69,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
#if (!defined(PRODUCT) && defined(COMPILER2))
if (CountCompiledCalls) {
__ incrementq(ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
__ incrementq(ExternalAddress(SharedRuntime::nof_megamorphic_calls_addr()), rscratch1);
}
#endif
@ -81,7 +80,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index) {
// get receiver klass
address npe_addr = __ pc();
__ load_klass(rax, j_rarg0, tmp_load_klass);
__ load_klass(rax, j_rarg0, rscratch1);
#ifndef PRODUCT
if (DebugVtables) {
@ -164,7 +163,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
#if (!defined(PRODUCT) && defined(COMPILER2))
if (CountCompiledCalls) {
__ incrementq(ExternalAddress((address) SharedRuntime::nof_megamorphic_calls_addr()));
__ incrementq(ExternalAddress(SharedRuntime::nof_megamorphic_calls_addr()), rscratch1);
}
#endif // PRODUCT

78
src/hotspot/cpu/x86/x86.ad

@ -1343,7 +1343,7 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
__ subptr(Address(rsp, 0), __ offset() - offset);
#else
InternalAddress here(__ pc());
__ pushptr(here.addr());
__ pushptr(here.addr(), noreg);
#endif
__ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
@ -1392,8 +1392,6 @@ Assembler::Width widthForType(BasicType bt) {
static address vector_long_shufflemask() { return StubRoutines::x86::vector_long_shuffle_mask(); }
static address vector_32_bit_mask() { return StubRoutines::x86::vector_32_bit_mask(); }
static address vector_64_bit_mask() { return StubRoutines::x86::vector_64_bit_mask(); }
static address vector_float_signflip() { return StubRoutines::x86::vector_float_sign_flip();}
static address vector_double_signflip() { return StubRoutines::x86::vector_double_sign_flip();}
//=============================================================================
const bool Matcher::match_rule_supported(int opcode) {
@ -7352,9 +7350,9 @@ instruct castFtoI_reg_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec
format %{ "vector_cast_f2i $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3, $xtmp4 as TEMP" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
__ vector_castF2I_avx($dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
$xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister,
ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
__ vector_castF2I_avx($dst$$XMMRegister, $src$$XMMRegister,
ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), vlen_enc,
$xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister);
%}
ins_pipe( pipe_slow );
%}
@ -7368,9 +7366,9 @@ instruct castFtoI_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, k
format %{ "vector_cast_f2i $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
__ vector_castF2I_evex($dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
$xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
__ vector_castF2I_evex($dst$$XMMRegister, $src$$XMMRegister,
ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), vlen_enc,
$xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
%}
ins_pipe( pipe_slow );
%}
@ -7387,14 +7385,14 @@ instruct castFtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, k
BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
if (to_elem_bt == T_LONG) {
int vlen_enc = vector_length_encoding(this);
__ vector_castF2L_evex($dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
$xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
ExternalAddress(vector_double_signflip()), noreg, vlen_enc);
__ vector_castF2L_evex($dst$$XMMRegister, $src$$XMMRegister,
ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), vlen_enc,
$xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
} else {
int vlen_enc = vector_length_encoding(this, $src);
__ vector_castF2I_evex($dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
$xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
ExternalAddress(vector_float_signflip()), noreg, vlen_enc);
__ vector_castF2I_evex($dst$$XMMRegister, $src$$XMMRegister,
ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), vlen_enc,
$xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
if (to_elem_bt == T_SHORT) {
__ evpmovdw($dst$$XMMRegister, $dst$$XMMRegister, vlen_enc);
} else {
@ -7425,9 +7423,9 @@ instruct castDtoX_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, k
ins_encode %{
int vlen_enc = vector_length_encoding(this, $src);
BasicType to_elem_bt = Matcher::vector_element_basic_type(this);
__ vector_castD2X_evex(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
$xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
ExternalAddress(vector_double_signflip()), noreg, vlen_enc);
__ vector_castD2X_evex(to_elem_bt, $dst$$XMMRegister, $src$$XMMRegister,
ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), vlen_enc,
$xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
%}
ins_pipe( pipe_slow );
%}
@ -7449,55 +7447,57 @@ instruct vucast(vec dst, vec src) %{
%}
#ifdef _LP64
instruct vround_float_avx(vec dst, vec src, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rRegP scratch, rFlagsReg cr) %{
instruct vround_float_avx(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, vec xtmp3, vec xtmp4, rFlagsReg cr) %{
predicate(!VM_Version::supports_avx512vl() &&
Matcher::vector_length_in_bytes(n) < 64 &&
Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst (RoundVF src));
effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, TEMP scratch, KILL cr);
format %{ "vector_round_float $dst,$src\t! using $xtmp1, $xtmp2, $xtmp3, $xtmp4 and $scratch as TEMP" %}
effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP xtmp3, TEMP xtmp4, KILL cr);
format %{ "vector_round_float $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $xtmp3, $xtmp4 as TEMP" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
InternalAddress new_mxcsr = $constantaddress((jint)0x3F80);
__ vector_round_float_avx($dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
$xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister,
ExternalAddress(vector_float_signflip()), new_mxcsr, $scratch$$Register, vlen_enc);
__ vector_round_float_avx($dst$$XMMRegister, $src$$XMMRegister,
ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), new_mxcsr, vlen_enc,
$tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $xtmp3$$XMMRegister, $xtmp4$$XMMRegister);
%}
ins_pipe( pipe_slow );
%}
instruct vround_float_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rRegP scratch, rFlagsReg cr) %{
instruct vround_float_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
predicate((VM_Version::supports_avx512vl() ||
Matcher::vector_length_in_bytes(n) == 64) &&
Matcher::vector_element_basic_type(n) == T_INT);
match(Set dst (RoundVF src));
effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, TEMP scratch, KILL cr);
format %{ "vector_round_float $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1, $ktmp2 and $scratch as TEMP" %}
effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
format %{ "vector_round_float $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
InternalAddress new_mxcsr = $constantaddress((jint)0x3F80);
__ vector_round_float_evex($dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
$xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
ExternalAddress(vector_float_signflip()), new_mxcsr, $scratch$$Register, vlen_enc);
__ vector_round_float_evex($dst$$XMMRegister, $src$$XMMRegister,
ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), new_mxcsr, vlen_enc,
$tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
%}
ins_pipe( pipe_slow );
%}
instruct vround_reg_evex(vec dst, vec src, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rRegP scratch, rFlagsReg cr) %{
instruct vround_reg_evex(vec dst, vec src, rRegP tmp, vec xtmp1, vec xtmp2, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
predicate(Matcher::vector_element_basic_type(n) == T_LONG);
match(Set dst (RoundVD src));
effect(TEMP dst, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, TEMP scratch, KILL cr);
format %{ "vector_round_long $dst,$src\t! using $xtmp1, $xtmp2, $ktmp1, $ktmp2 and $scratch as TEMP" %}
effect(TEMP dst, TEMP tmp, TEMP xtmp1, TEMP xtmp2, TEMP ktmp1, TEMP ktmp2, KILL cr);
format %{ "vector_round_long $dst,$src\t! using $tmp, $xtmp1, $xtmp2, $ktmp1, $ktmp2 as TEMP" %}
ins_encode %{
int vlen_enc = vector_length_encoding(this);
InternalAddress new_mxcsr = $constantaddress((jint)0x3F80);
__ vector_round_double_evex($dst$$XMMRegister, $src$$XMMRegister, $xtmp1$$XMMRegister,
$xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister,
ExternalAddress(vector_double_signflip()), new_mxcsr, $scratch$$Register, vlen_enc);
__ vector_round_double_evex($dst$$XMMRegister, $src$$XMMRegister,
ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), new_mxcsr, vlen_enc,
$tmp$$Register, $xtmp1$$XMMRegister, $xtmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
%}
ins_pipe( pipe_slow );
%}
#endif
#endif // _LP64
// --------------------------------- VectorMaskCmp --------------------------------------
instruct vcmpFD(legVec dst, legVec src1, legVec src2, immI8 cond) %{
@ -9390,8 +9390,8 @@ instruct vreverse_reg_gfni(vec dst, vec src, vec xtmp) %{
int vec_enc = vector_length_encoding(this);
BasicType bt = Matcher::vector_element_basic_type(this);
InternalAddress addr = $constantaddress(T_LONG, vreplicate_imm(T_LONG, 0x8040201008040201L, 1));
__ vector_reverse_bit_gfni(bt, $dst$$XMMRegister, $src$$XMMRegister, $xtmp$$XMMRegister,
addr, noreg, vec_enc);
__ vector_reverse_bit_gfni(bt, $dst$$XMMRegister, $src$$XMMRegister, addr, vec_enc,
$xtmp$$XMMRegister);
%}
ins_pipe( pipe_slow );
%}

2
src/hotspot/cpu/x86/x86_32.ad

@ -12629,7 +12629,7 @@ instruct jumpXtnd(rRegI switch_val) %{
ins_encode %{
// Jump to Address(table_base + switch_reg)
Address index(noreg, $switch_val$$Register, Address::times_1);
__ jump(ArrayAddress($constantaddress, index));
__ jump(ArrayAddress($constantaddress, index), noreg);
%}
ins_pipe(pipe_jmp);
%}

2
src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp

@ -181,7 +181,7 @@ bool os::win32::register_code_area(char *low, char *high) {
MacroAssembler* masm = new MacroAssembler(&cb);
pDCD = (pDynamicCodeData) masm->pc();
masm->jump(ExternalAddress((address)&HandleExceptionFromCodeCache));
masm->jump(ExternalAddress((address)&HandleExceptionFromCodeCache), rscratch1);
masm->flush();
// Create an Unwind Structure specifying no unwind info

2
src/hotspot/share/oops/compressedOops.hpp

@ -94,7 +94,7 @@ public:
static int shift() { return _narrow_oop._shift; }
static bool use_implicit_null_checks() { return _narrow_oop._use_implicit_null_checks; }
static address* ptrs_base_addr() { return &_narrow_oop._base; }
static address ptrs_base_addr() { return (address)&_narrow_oop._base; }
static address ptrs_base() { return _narrow_oop._base; }
static bool is_in(void* addr);