stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8305895: Implement JEP 450: Compact Object Headers (Experimental)

Browse Source 
Co-authored-by: Sandhya Viswanathan <sviswanathan@openjdk.org>
Co-authored-by: Martin Doerr <mdoerr@openjdk.org>
Co-authored-by: Hamlin Li <mli@openjdk.org>
Co-authored-by: Thomas Stuefe <stuefe@openjdk.org>
Co-authored-by: Amit Kumar <amitkumar@openjdk.org>
Co-authored-by: Stefan Karlsson <stefank@openjdk.org>
Co-authored-by: Coleen Phillimore <coleenp@openjdk.org>
Co-authored-by: Axel Boldt-Christmas <aboldtch@openjdk.org>
Reviewed-by: coleenp, stefank, stuefe, phh, ihse, lmesnik, tschatzl, matsaave, rcastanedalo, vpaprotski, yzheng, egahlin
This commit is contained in:
Roman Kennke 2024-11-08 17:21:39 +00:00
parent 605396280d
commit 44ec501a41
218 changed files with 4353 additions and 1632 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
make/Images.gmk
View File

@ -132,10 +132,16 @@ CDS_DUMP_FLAGS = -Xmx128M -Xms128M
# Helper function for creating the CDS archives for the JDK and JRE
#
# Param1 - VM variant (e.g., server, client, zero, ...)
# Param2 - _nocoops, or empty
# Param2 - _nocoops, _coh, _nocoops_coh, or empty
define CreateCDSArchive
$1_$2_DUMP_EXTRA_ARG := $(if $(filter _nocoops, $2), -XX:-UseCompressedOops, )
$1_$2_DUMP_TYPE := $(if $(filter _nocoops, $2), -NOCOOPS, )
$1_$2_COOPS_OPTION := $(if $(findstring _nocoops, $2),-XX:-UseCompressedOops)
# enable and also explicitly disable coh as needed.
ifeq ($(call isTargetCpuBits, 64), true)
$1_$2_COH_OPTION := -XX:+UnlockExperimentalVMOptions \
$(if $(findstring _coh, $2),-XX:+UseCompactObjectHeaders,-XX:-UseCompactObjectHeaders)
endif
$1_$2_DUMP_EXTRA_ARG := $$($1_$2_COOPS_OPTION) $$($1_$2_COH_OPTION)
$1_$2_DUMP_TYPE := $(if $(findstring _nocoops, $2),-NOCOOPS,)$(if $(findstring _coh, $2),-COH,)
# Only G1 supports dumping the shared heap, so explicitly use G1 if the JVM supports it.
$1_$2_CDS_DUMP_FLAGS := $(CDS_DUMP_FLAGS) $(if $(filter g1gc, $(JVM_FEATURES_$1)), -XX:+UseG1GC)
@ -190,6 +196,14 @@ ifeq ($(BUILD_CDS_ARCHIVE), true)
$(foreach v, $(JVM_VARIANTS), \
$(eval $(call CreateCDSArchive,$v,_nocoops)) \
)
ifeq ($(BUILD_CDS_ARCHIVE_COH), true)
$(foreach v, $(JVM_VARIANTS), \
$(eval $(call CreateCDSArchive,$v,_coh)) \
)
$(foreach v, $(JVM_VARIANTS), \
$(eval $(call CreateCDSArchive,$v,_nocoops_coh)) \
)
endif
endif
endif

1
make/autoconf/configure.ac
View File

@ -261,6 +261,7 @@ JDKOPT_ENABLE_DISABLE_GENERATE_CLASSLIST
JDKOPT_EXCLUDE_TRANSLATIONS
JDKOPT_ENABLE_DISABLE_MANPAGES
JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE
JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE_COH
JDKOPT_ENABLE_DISABLE_COMPATIBLE_CDS_ALIGNMENT
JDKOPT_SETUP_MACOSX_SIGNING

31
make/autoconf/jdk-options.m4
View File

@ -673,6 +673,37 @@ AC_DEFUN([JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE],
AC_SUBST(BUILD_CDS_ARCHIVE)
])
################################################################################
#
# Enable or disable the default CDS archive generation for Compact Object Headers
#
AC_DEFUN([JDKOPT_ENABLE_DISABLE_CDS_ARCHIVE_COH],
[
UTIL_ARG_ENABLE(NAME: cds-archive-coh, DEFAULT: auto, RESULT: BUILD_CDS_ARCHIVE_COH,
DESC: [enable generation of default CDS archives for compact object headers (requires --enable-cds-archive)],
DEFAULT_DESC: [auto],
CHECKING_MSG: [if default CDS archives for compact object headers should be generated],
CHECK_AVAILABLE: [
AC_MSG_CHECKING([if CDS archive with compact object headers is available])
if test "x$BUILD_CDS_ARCHIVE" = "xfalse"; then
AC_MSG_RESULT([no (CDS default archive generation is disabled)])
AVAILABLE=false
elif test "x$OPENJDK_TARGET_CPU" != "xx86_64" &&
test "x$OPENJDK_TARGET_CPU" != "xaarch64" &&
test "x$OPENJDK_TARGET_CPU" != "xppc64" &&
test "x$OPENJDK_TARGET_CPU" != "xppc64le" &&
test "x$OPENJDK_TARGET_CPU" != "xriscv64" &&
test "x$OPENJDK_TARGET_CPU" != "xs390x"; then
AC_MSG_RESULT([no (compact object headers not supported for this platform)])
AVAILABLE=false
else
AC_MSG_RESULT([yes])
AVAILABLE=true
fi
])
AC_SUBST(BUILD_CDS_ARCHIVE_COH)
])
################################################################################
#
# Enable the alternative CDS core region alignment

1
make/autoconf/spec.gmk.template
View File

@ -370,6 +370,7 @@ EXCLUDE_TRANSLATIONS := @EXCLUDE_TRANSLATIONS@
BUILD_MANPAGES := @BUILD_MANPAGES@
BUILD_CDS_ARCHIVE := @BUILD_CDS_ARCHIVE@
BUILD_CDS_ARCHIVE_COH := @BUILD_CDS_ARCHIVE_COH@
ENABLE_COMPATIBLE_CDS_ALIGNMENT := @ENABLE_COMPATIBLE_CDS_ALIGNMENT@

20
src/hotspot/cpu/aarch64/aarch64.ad
View File

@ -5756,6 +5756,10 @@ opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indInde
indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
opclass memory_noindex(indirect,
indOffI1, indOffL1,indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
indirectN, indOffIN, indOffLN, indirectX2P, indOffX2P);
// iRegIorL2I is used for src inputs in rules for 32 bit int (I)
// operations. it allows the src to be either an iRegI or a (ConvL2I
// iRegL). in the latter case the l2i normally planted for a ConvL2I
@ -6682,7 +6686,7 @@ instruct loadKlass(iRegPNoSp dst, memory8 mem)
instruct loadNKlass(iRegNNoSp dst, memory4 mem)
%{
match(Set dst (LoadNKlass mem));
predicate(!needs_acquiring_load(n));
predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
ins_cost(4 * INSN_COST);
format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
@ -6692,6 +6696,20 @@ instruct loadNKlass(iRegNNoSp dst, memory4 mem)
ins_pipe(iload_reg_mem);
%}
instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory_noindex mem)
%{
match(Set dst (LoadNKlass mem));
predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
ins_cost(4 * INSN_COST);
format %{ "load_narrow_klass_compact $dst, $mem\t# compressed class ptr" %}
ins_encode %{
assert($mem$$index$$Register == noreg, "must not have indexed address");
__ load_narrow_klass_compact_c2($dst$$Register, $mem$$base$$Register, $mem$$disp);
%}
ins_pipe(iload_reg_mem);
%}
// Load Float
instruct loadF(vRegF dst, memory4 mem)
%{

47
src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
View File

@ -2243,8 +2243,6 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
// test for null
if (flags & LIR_OpArrayCopy::src_null_check) {
@ -2305,15 +2303,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// We don't know the array types are compatible
if (basic_type != T_OBJECT) {
// Simple test for basic type arrays
if (UseCompressedClassPointers) {
__ ldrw(tmp, src_klass_addr);
__ ldrw(rscratch1, dst_klass_addr);
__ cmpw(tmp, rscratch1);
} else {
__ ldr(tmp, src_klass_addr);
__ ldr(rscratch1, dst_klass_addr);
__ cmp(tmp, rscratch1);
}
__ cmp_klasses_from_objects(src, dst, tmp, rscratch1);
__ br(Assembler::NE, *stub->entry());
} else {
// For object arrays, if src is a sub class of dst then we can
@ -2435,36 +2425,14 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// but not necessarily exactly of type default_type.
Label known_ok, halt;
__ mov_metadata(tmp, default_type->constant_encoding());
if (UseCompressedClassPointers) {
__ encode_klass_not_null(tmp);
}
if (basic_type != T_OBJECT) {
if (UseCompressedClassPointers) {
__ ldrw(rscratch1, dst_klass_addr);
__ cmpw(tmp, rscratch1);
} else {
__ ldr(rscratch1, dst_klass_addr);
__ cmp(tmp, rscratch1);
}
__ cmp_klass(dst, tmp, rscratch1);
__ br(Assembler::NE, halt);
if (UseCompressedClassPointers) {
__ ldrw(rscratch1, src_klass_addr);
__ cmpw(tmp, rscratch1);
} else {
__ ldr(rscratch1, src_klass_addr);
__ cmp(tmp, rscratch1);
}
__ cmp_klass(src, tmp, rscratch1);
__ br(Assembler::EQ, known_ok);
} else {
if (UseCompressedClassPointers) {
__ ldrw(rscratch1, dst_klass_addr);
__ cmpw(tmp, rscratch1);
} else {
__ ldr(rscratch1, dst_klass_addr);
__ cmp(tmp, rscratch1);
}
__ cmp_klass(dst, tmp, rscratch1);
__ br(Assembler::EQ, known_ok);
__ cmp(src, dst);
__ br(Assembler::EQ, known_ok);
@ -2547,12 +2515,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
add_debug_info_for_null_check_here(info);
}
if (UseCompressedClassPointers) {
__ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
__ decode_klass_not_null(result);
} else {
__ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
}
__ load_klass(result, obj);
}
void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {

20
src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
View File

@ -175,15 +175,19 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, i
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
assert_different_registers(obj, klass, len);
// This assumes that all prototype bits fit in an int32_t
mov(t1, (int32_t)(intptr_t)markWord::prototype().value());
str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
if (UseCompressedClassPointers) { // Take care not to kill klass
encode_klass_not_null(t1, klass);
strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
if (UseCompactObjectHeaders) {
ldr(t1, Address(klass, Klass::prototype_header_offset()));
str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
} else {
str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
mov(t1, checked_cast<int32_t>(markWord::prototype().value()));
str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
if (UseCompressedClassPointers) { // Take care not to kill klass
encode_klass_not_null(t1, klass);
strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
} else {
str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
}
}
if (len->is_valid()) {
@ -194,7 +198,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
// Clear gap/first 4 bytes following the length field.
strw(zr, Address(obj, base_offset));
}
} else if (UseCompressedClassPointers) {
} else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
store_klass_gap(obj, zr);
}
}

9
src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
View File

@ -2689,3 +2689,12 @@ bool C2_MacroAssembler::in_scratch_emit_size() {
}
return MacroAssembler::in_scratch_emit_size();
}
void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Register obj, int disp) {
// Note: Don't clobber obj anywhere in that method!
// The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
// obj-start, so that we can load from the object's mark-word instead.
ldr(dst, Address(obj, disp - oopDesc::klass_offset_in_bytes()));
lsr(dst, dst, markWord::klass_shift);
}

2
src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
View File

@ -186,4 +186,6 @@
void vector_signum_sve(FloatRegister dst, FloatRegister src, FloatRegister zero,
FloatRegister one, FloatRegister vtmp, PRegister pgtmp, SIMD_RegVariant T);
void load_narrow_klass_compact_c2(Register dst, Register obj, int disp);
#endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP

16
src/hotspot/cpu/aarch64/compressedKlass_aarch64.cpp
View File

@ -117,19 +117,3 @@ char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size
return result;
}
void CompressedKlassPointers::initialize(address addr, size_t len) {
constexpr uintptr_t unscaled_max = nth_bit(32);
assert(len <= unscaled_max, "Klass range larger than 32 bits?");
// Shift is always 0 on aarch64.
_shift = 0;
// On aarch64, we don't bother with zero-based encoding (base=0 shift>0).
address const end = addr + len;
_base = (end <= (address)unscaled_max) ? nullptr : addr;
// Remember the Klass range:
_klass_range_start = addr;
_klass_range_end = addr + len;
}

77
src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
View File

@ -1002,10 +1002,11 @@ address MacroAssembler::ic_call(address entry, jint method_index) {
}
int MacroAssembler::ic_check_size() {
int extra_instructions = UseCompactObjectHeaders ? 1 : 0;
if (target_needs_far_branch(CAST_FROM_FN_PTR(address, SharedRuntime::get_ic_miss_stub()))) {
return NativeInstruction::instruction_size * 7;
return NativeInstruction::instruction_size * (7 + extra_instructions);
} else {
return NativeInstruction::instruction_size * 5;
return NativeInstruction::instruction_size * (5 + extra_instructions);
}
}
@ -1023,7 +1024,11 @@ int MacroAssembler::ic_check(int end_alignment) {
int uep_offset = offset();
if (UseCompressedClassPointers) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp1, receiver);
ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
cmpw(tmp1, tmp2);
} else if (UseCompressedClassPointers) {
ldrw(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
cmpw(tmp1, tmp2);
@ -5009,8 +5014,22 @@ void MacroAssembler::load_method_holder(Register holder, Register method) {
ldr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass*
}
// Loads the obj's Klass* into dst.
// Preserves all registers (incl src, rscratch1 and rscratch2).
// Input:
// src - the oop we want to load the klass from.
// dst - output narrow klass.
void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
ldr(dst, Address(src, oopDesc::mark_offset_in_bytes()));
lsr(dst, dst, markWord::klass_shift);
}
void MacroAssembler::load_klass(Register dst, Register src) {
if (UseCompressedClassPointers) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
decode_klass_not_null(dst);
} else if (UseCompressedClassPointers) {
ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
decode_klass_not_null(dst);
} else {
@ -5065,28 +5084,50 @@ void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, R
resolve_oop_handle(dst, tmp1, tmp2);
}
void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
void MacroAssembler::cmp_klass(Register obj, Register klass, Register tmp) {
assert_different_registers(obj, klass, tmp);
if (UseCompressedClassPointers) {
ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp, obj);
} else {
ldrw(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
}
if (CompressedKlassPointers::base() == nullptr) {
cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
cmp(klass, tmp, LSL, CompressedKlassPointers::shift());
return;
} else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
&& CompressedKlassPointers::shift() == 0) {
// Only the bottom 32 bits matter
cmpw(trial_klass, tmp);
cmpw(klass, tmp);
return;
}
decode_klass_not_null(tmp);
} else {
ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
ldr(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
}
cmp(klass, tmp);
}
void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
cmpw(tmp1, tmp2);
} else if (UseCompressedClassPointers) {
ldrw(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
ldrw(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes()));
cmpw(tmp1, tmp2);
} else {
ldr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
ldr(tmp2, Address(obj2, oopDesc::klass_offset_in_bytes()));
cmp(tmp1, tmp2);
}
cmp(trial_klass, tmp);
}
void MacroAssembler::store_klass(Register dst, Register src) {
// FIXME: Should this be a store release? concurrent gcs assumes
// klass length is valid if klass field is not null.
assert(!UseCompactObjectHeaders, "not with compact headers");
if (UseCompressedClassPointers) {
encode_klass_not_null(src);
strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
@ -5096,6 +5137,7 @@ void MacroAssembler::store_klass(Register dst, Register src) {
}
void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "not with compact headers");
if (UseCompressedClassPointers) {
// Store to klass gap in destination
strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
@ -5246,9 +5288,6 @@ MacroAssembler::KlassDecodeMode MacroAssembler::klass_decode_mode() {
return _klass_decode_mode;
}
assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift()
|| 0 == CompressedKlassPointers::shift(), "decode alg wrong");
if (CompressedKlassPointers::base() == nullptr) {
return (_klass_decode_mode = KlassDecodeZero);
}
@ -5274,7 +5313,7 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
switch (klass_decode_mode()) {
case KlassDecodeZero:
if (CompressedKlassPointers::shift() != 0) {
lsr(dst, src, LogKlassAlignmentInBytes);
lsr(dst, src, CompressedKlassPointers::shift());
} else {
if (dst != src) mov(dst, src);
}
@ -5283,7 +5322,7 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
case KlassDecodeXor:
if (CompressedKlassPointers::shift() != 0) {
eor(dst, src, (uint64_t)CompressedKlassPointers::base());
lsr(dst, dst, LogKlassAlignmentInBytes);
lsr(dst, dst, CompressedKlassPointers::shift());
} else {
eor(dst, src, (uint64_t)CompressedKlassPointers::base());
}
@ -5291,7 +5330,7 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
case KlassDecodeMovk:
if (CompressedKlassPointers::shift() != 0) {
ubfx(dst, src, LogKlassAlignmentInBytes, 32);
ubfx(dst, src, CompressedKlassPointers::shift(), 32);
} else {
movw(dst, src);
}
@ -5313,7 +5352,7 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
switch (klass_decode_mode()) {
case KlassDecodeZero:
if (CompressedKlassPointers::shift() != 0) {
lsl(dst, src, LogKlassAlignmentInBytes);
lsl(dst, src, CompressedKlassPointers::shift());
} else {
if (dst != src) mov(dst, src);
}
@ -5321,7 +5360,7 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
case KlassDecodeXor:
if (CompressedKlassPointers::shift() != 0) {
lsl(dst, src, LogKlassAlignmentInBytes);
lsl(dst, src, CompressedKlassPointers::shift());
eor(dst, dst, (uint64_t)CompressedKlassPointers::base());
} else {
eor(dst, src, (uint64_t)CompressedKlassPointers::base());
@ -5336,7 +5375,7 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
movk(dst, shifted_base >> 32, 32);
if (CompressedKlassPointers::shift() != 0) {
lsl(dst, dst, LogKlassAlignmentInBytes);
lsl(dst, dst, CompressedKlassPointers::shift());
}
break;

4
src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
View File

@ -875,9 +875,11 @@ public:
void load_method_holder(Register holder, Register method);
// oop manipulations
void load_narrow_klass_compact(Register dst, Register src);
void load_klass(Register dst, Register src);
void store_klass(Register dst, Register src);
void cmp_klass(Register oop, Register trial_klass, Register tmp);
void cmp_klass(Register obj, Register klass, Register tmp);
void cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2);
void resolve_weak_handle(Register result, Register tmp1, Register tmp2);
void resolve_oop_handle(Register result, Register tmp1, Register tmp2);

19
src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
View File

@ -3629,12 +3629,14 @@ void TemplateTable::_new() {
// The object is initialized before the header. If the object size is
// zero, go directly to the header initialization.
__ sub(r3, r3, sizeof(oopDesc));
int header_size = oopDesc::header_size() * HeapWordSize;
assert(is_aligned(header_size, BytesPerLong), "oop header size must be 8-byte-aligned");
__ sub(r3, r3, header_size);
__ cbz(r3, initialize_header);
// Initialize object fields
{
__ add(r2, r0, sizeof(oopDesc));
__ add(r2, r0, header_size);
Label loop;
__ bind(loop);
__ str(zr, Address(__ post(r2, BytesPerLong)));
@ -3644,10 +3646,15 @@ void TemplateTable::_new() {
// initialize object header only.
__ bind(initialize_header);
__ mov(rscratch1, (intptr_t)markWord::prototype().value());
__ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
__ store_klass_gap(r0, zr); // zero klass gap for compressed oops
__ store_klass(r0, r4); // store klass last
if (UseCompactObjectHeaders) {
__ ldr(rscratch1, Address(r4, Klass::prototype_header_offset()));
__ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
} else {
__ mov(rscratch1, (intptr_t)markWord::prototype().value());
__ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
__ store_klass_gap(r0, zr); // zero klass gap for compressed oops
__ store_klass(r0, r4); // store klass last
}
if (DTraceAllocProbes) {
// Trigger dtrace event for fastpath

58
src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
View File

@ -1996,16 +1996,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// We don't know the array types are compatible.
if (basic_type != T_OBJECT) {
// Simple test for basic type arrays.
if (UseCompressedClassPointers) {
// We don't need decode because we just need to compare.
__ lwz(tmp, oopDesc::klass_offset_in_bytes(), src);
__ lwz(tmp2, oopDesc::klass_offset_in_bytes(), dst);
__ cmpw(CCR0, tmp, tmp2);
} else {
__ ld(tmp, oopDesc::klass_offset_in_bytes(), src);
__ ld(tmp2, oopDesc::klass_offset_in_bytes(), dst);
__ cmpd(CCR0, tmp, tmp2);
}
__ cmp_klasses_from_objects(CCR0, src, dst, tmp, tmp2);
__ beq(CCR0, cont);
} else {
// For object arrays, if src is a sub class of dst then we can
@ -2128,39 +2119,15 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// but not necessarily exactly of type default_type.
Label known_ok, halt;
metadata2reg(default_type->constant_encoding(), tmp);
if (UseCompressedClassPointers) {
// Tmp holds the default type. It currently comes uncompressed after the
// load of a constant, so encode it.
__ encode_klass_not_null(tmp);
// Load the raw value of the dst klass, since we will be comparing
// uncompressed values directly.
__ lwz(tmp2, oopDesc::klass_offset_in_bytes(), dst);
__ cmpw(CCR0, tmp, tmp2);
if (basic_type != T_OBJECT) {
__ bne(CCR0, halt);
// Load the raw value of the src klass.
__ lwz(tmp2, oopDesc::klass_offset_in_bytes(), src);
__ cmpw(CCR0, tmp, tmp2);
__ beq(CCR0, known_ok);
} else {
__ beq(CCR0, known_ok);
__ cmpw(CCR0, src, dst);
__ beq(CCR0, known_ok);
}
__ cmp_klass(CCR0, dst, tmp, R11_scratch1, R12_scratch2);
if (basic_type != T_OBJECT) {
__ bne(CCR0, halt);
__ cmp_klass(CCR0, src, tmp, R11_scratch1, R12_scratch2);
__ beq(CCR0, known_ok);
} else {
__ ld(tmp2, oopDesc::klass_offset_in_bytes(), dst);
__ cmpd(CCR0, tmp, tmp2);
if (basic_type != T_OBJECT) {
__ bne(CCR0, halt);
// Load the raw value of the src klass.
__ ld(tmp2, oopDesc::klass_offset_in_bytes(), src);
__ cmpd(CCR0, tmp, tmp2);
__ beq(CCR0, known_ok);
} else {
__ beq(CCR0, known_ok);
__ cmpd(CCR0, src, dst);
__ beq(CCR0, known_ok);
}
__ beq(CCR0, known_ok);
__ cmpw(CCR0, src, dst);
__ beq(CCR0, known_ok);
}
__ bind(halt);
__ stop("incorrect type information in arraycopy");
@ -2738,12 +2705,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
}
}
if (UseCompressedClassPointers) {
__ lwz(result, oopDesc::klass_offset_in_bytes(), obj);
__ decode_klass_not_null(result);
} else {
__ ld(result, oopDesc::klass_offset_in_bytes(), obj);
}
__ load_klass(result, obj);
}
void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {

15
src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
View File

@ -201,12 +201,19 @@ void C1_MacroAssembler::try_allocate(
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
assert_different_registers(obj, klass, len, t1, t2);
load_const_optimized(t1, (intx)markWord::prototype().value());
std(t1, oopDesc::mark_offset_in_bytes(), obj);
store_klass(obj, klass);
if (UseCompactObjectHeaders) {
ld(t1, in_bytes(Klass::prototype_header_offset()), klass);
std(t1, oopDesc::mark_offset_in_bytes(), obj);
} else {
load_const_optimized(t1, (intx)markWord::prototype().value());
std(t1, oopDesc::mark_offset_in_bytes(), obj);
store_klass(obj, klass);
}
if (len->is_valid()) {
stw(len, arrayOopDesc::length_offset_in_bytes(), obj);
} else if (UseCompressedClassPointers) {
} else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
// Otherwise length is in the class gap.
store_klass_gap(obj);
}

9
src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.cpp
View File

@ -47,6 +47,15 @@ void C2_MacroAssembler::fast_unlock_lightweight(ConditionRegister flag, Register
compiler_fast_unlock_lightweight_object(flag, obj, box, tmp1, tmp2, tmp3);
}
void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Register obj, int disp) {
// Note: Don't clobber obj anywhere in that method!
// The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
// obj-start, so that we can load from the object's mark-word instead.
ld(dst, disp - oopDesc::klass_offset_in_bytes(), obj);
srdi(dst, dst, markWord::klass_shift);
}
// Intrinsics for CompactStrings
// Compress char[] to byte[] by compressing 16 bytes at once.

2
src/hotspot/cpu/ppc/c2_MacroAssembler_ppc.hpp
View File

@ -34,6 +34,8 @@
void fast_unlock_lightweight(ConditionRegister flag, Register obj, Register box,
Register tmp1, Register tmp2, Register tmp3);
void load_narrow_klass_compact_c2(Register dst, Register obj, int disp);
// Intrinsics for CompactStrings
// Compress char[] to byte[] by compressing 16 bytes at once.
void string_compress_16(Register src, Register dst, Register cnt,

62
src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
View File

@ -1218,6 +1218,9 @@ int MacroAssembler::ic_check_size() {
num_ins = 7;
if (!implicit_null_checks_available) num_ins += 2;
}
if (UseCompactObjectHeaders) num_ins++;
return num_ins * BytesPerInstWord;
}
@ -1245,7 +1248,9 @@ int MacroAssembler::ic_check(int end_alignment) {
if (use_trap_based_null_check) {
trap_null_check(receiver);
}
if (UseCompressedClassPointers) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp1, receiver);
} else if (UseCompressedClassPointers) {
lwz(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
} else {
ld(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
@ -3258,6 +3263,7 @@ Register MacroAssembler::encode_klass_not_null(Register dst, Register src) {
}
void MacroAssembler::store_klass(Register dst_oop, Register klass, Register ck) {
assert(!UseCompactObjectHeaders, "not with compact headers");
if (UseCompressedClassPointers) {
Register compressedKlass = encode_klass_not_null(ck, klass);
stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop);
@ -3267,12 +3273,13 @@ void MacroAssembler::store_klass(Register dst_oop, Register klass, Register ck)
}
void MacroAssembler::store_klass_gap(Register dst_oop, Register val) {
assert(!UseCompactObjectHeaders, "not with compact headers");
if (UseCompressedClassPointers) {
if (val == noreg) {
val = R0;
li(val, 0);
}
stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop); // klass gap if compressed
stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop);
}
}
@ -3313,15 +3320,60 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
}
void MacroAssembler::load_klass(Register dst, Register src) {
if (UseCompressedClassPointers) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
decode_klass_not_null(dst);
} else if (UseCompressedClassPointers) {
lwz(dst, oopDesc::klass_offset_in_bytes(), src);
// Attention: no null check here!
decode_klass_not_null(dst, dst);
decode_klass_not_null(dst);
} else {
ld(dst, oopDesc::klass_offset_in_bytes(), src);
}
}
// Loads the obj's Klass* into dst.
// Preserves all registers (incl src, rscratch1 and rscratch2).
// Input:
// src - the oop we want to load the klass from.
// dst - output nklass.
void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
ld(dst, oopDesc::mark_offset_in_bytes(), src);
srdi(dst, dst, markWord::klass_shift);
}
void MacroAssembler::cmp_klass(ConditionRegister dst, Register obj, Register klass, Register tmp, Register tmp2) {
assert_different_registers(obj, klass, tmp);
if (UseCompressedClassPointers) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp, obj);
} else {
lwz(tmp, oopDesc::klass_offset_in_bytes(), obj);
}
Register encoded_klass = encode_klass_not_null(tmp2, klass);
cmpw(dst, tmp, encoded_klass);
} else {
ld(tmp, oopDesc::klass_offset_in_bytes(), obj);
cmpd(dst, tmp, klass);
}
}
void MacroAssembler::cmp_klasses_from_objects(ConditionRegister dst, Register obj1, Register obj2, Register tmp1, Register tmp2) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
cmpw(dst, tmp1, tmp2);
} else if (UseCompressedClassPointers) {
lwz(tmp1, oopDesc::klass_offset_in_bytes(), obj1);
lwz(tmp2, oopDesc::klass_offset_in_bytes(), obj2);
cmpw(dst, tmp1, tmp2);
} else {
ld(tmp1, oopDesc::klass_offset_in_bytes(), obj1);
ld(tmp2, oopDesc::klass_offset_in_bytes(), obj2);
cmpd(dst, tmp1, tmp2);
}
}
void MacroAssembler::load_klass_check_null(Register dst, Register src, Label* is_null) {
null_check(src, oopDesc::klass_offset_in_bytes(), is_null);
load_klass(dst, src);

3
src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
View File

@ -757,6 +757,9 @@ class MacroAssembler: public Assembler {
// Load/Store klass oop from klass field. Compress.
void load_klass(Register dst, Register src);
void load_narrow_klass_compact(Register dst, Register src);
void cmp_klass(ConditionRegister dst, Register obj, Register klass, Register tmp, Register tmp2);
void cmp_klasses_from_objects(ConditionRegister dst, Register obj1, Register obj2, Register tmp1, Register tmp2);
void load_klass_check_null(Register dst, Register src, Label* is_null = nullptr);
void store_klass(Register dst_oop, Register klass, Register tmp = R0);
void store_klass_gap(Register dst_oop, Register val = noreg); // Will store 0 if val not specified.

15
src/hotspot/cpu/ppc/ppc.ad
View File

@ -5496,6 +5496,7 @@ instruct loadP2X(iRegLdst dst, memoryAlg4 mem) %{
// Load compressed klass pointer.
instruct loadNKlass(iRegNdst dst, memory mem) %{
match(Set dst (LoadNKlass mem));
predicate(!UseCompactObjectHeaders);
ins_cost(MEMORY_REF_COST);
format %{ "LWZ $dst, $mem \t// compressed klass ptr" %}
@ -5504,6 +5505,20 @@ instruct loadNKlass(iRegNdst dst, memory mem) %{
ins_pipe(pipe_class_memory);
%}
instruct loadNKlassCompactHeaders(iRegNdst dst, memory mem) %{
match(Set dst (LoadNKlass mem));
predicate(UseCompactObjectHeaders);
ins_cost(MEMORY_REF_COST);
format %{ "load_narrow_klass_compact $dst, $mem \t// compressed class ptr" %}
size(8);
ins_encode %{
assert($mem$$index$$Register == R0, "must not have indexed address: %s[%s]", $mem$$base$$Register.name(), $mem$$index$$Register.name());
__ load_narrow_klass_compact_c2($dst$$Register, $mem$$base$$Register, $mem$$disp);
%}
ins_pipe(pipe_class_memory);
%}
// Load Klass Pointer
instruct loadKlass(iRegPdst dst, memoryAlg4 mem) %{
match(Set dst (LoadKlass mem));

20
src/hotspot/cpu/ppc/templateTable_ppc_64.cpp
View File

@ -3840,8 +3840,9 @@ void TemplateTable::_new() {
// Init1: Zero out newly allocated memory.
// Initialize remaining object fields.
Register Rbase = Rtags;
__ addi(Rinstance_size, Rinstance_size, 7 - (int)sizeof(oopDesc));
__ addi(Rbase, RallocatedObject, sizeof(oopDesc));
int header_size = oopDesc::header_size() * HeapWordSize;
__ addi(Rinstance_size, Rinstance_size, 7 - header_size);
__ addi(Rbase, RallocatedObject, header_size);
__ srdi(Rinstance_size, Rinstance_size, 3);
// Clear out object skipping header. Takes also care of the zero length case.
@ -3851,12 +3852,15 @@ void TemplateTable::_new() {
// --------------------------------------------------------------------------
// Init2: Initialize the header: mark, klass
// Init mark.
__ load_const_optimized(Rscratch, markWord::prototype().value(), R0);
__ std(Rscratch, oopDesc::mark_offset_in_bytes(), RallocatedObject);
// Init klass.
__ store_klass_gap(RallocatedObject);
__ store_klass(RallocatedObject, RinstanceKlass, Rscratch); // klass (last for cms)
if (UseCompactObjectHeaders) {
__ ld(Rscratch, in_bytes(Klass::prototype_header_offset()), RinstanceKlass);
__ std(Rscratch, oopDesc::mark_offset_in_bytes(), RallocatedObject);
} else {
__ load_const_optimized(Rscratch, markWord::prototype().value(), R0);
__ std(Rscratch, oopDesc::mark_offset_in_bytes(), RallocatedObject);
__ store_klass_gap(RallocatedObject);
__ store_klass(RallocatedObject, RinstanceKlass, Rscratch);
}
// Check and trigger dtrace event.
if (DTraceAllocProbes) {

27
src/hotspot/cpu/riscv/c1_LIRAssembler_arraycopy_riscv.cpp
View File

@ -194,7 +194,10 @@ void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Registe
// We don't know the array types are compatible
if (basic_type != T_OBJECT) {
// Simple test for basic type arrays
if (UseCompressedClassPointers) {
if (UseCompactObjectHeaders) {
__ load_narrow_klass_compact(tmp, src);
__ load_narrow_klass_compact(t0, dst);
} else if (UseCompressedClassPointers) {
__ lwu(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
__ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
} else {
@ -244,7 +247,6 @@ void LIR_Assembler::arraycopy_type_check(Register src, Register src_pos, Registe
void LIR_Assembler::arraycopy_assert(Register src, Register dst, Register tmp, ciArrayKlass *default_type, int flags) {
assert(default_type != nullptr, "null default_type!");
BasicType basic_type = default_type->element_type()->basic_type();
if (basic_type == T_ARRAY) { basic_type = T_OBJECT; }
if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
// Sanity check the known type with the incoming class. For the
@ -261,25 +263,10 @@ void LIR_Assembler::arraycopy_assert(Register src, Register dst, Register tmp, c
}
if (basic_type != T_OBJECT) {
if (UseCompressedClassPointers) {
__ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
} else {
__ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
}
__ bne(tmp, t0, halt);
if (UseCompressedClassPointers) {
__ lwu(t0, Address(src, oopDesc::klass_offset_in_bytes()));
} else {
__ ld(t0, Address(src, oopDesc::klass_offset_in_bytes()));
}
__ beq(tmp, t0, known_ok);
__ cmp_klass_compressed(dst, tmp, t0, halt, false);
__ cmp_klass_compressed(src, tmp, t0, known_ok, true);
} else {
if (UseCompressedClassPointers) {
__ lwu(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
} else {
__ ld(t0, Address(dst, oopDesc::klass_offset_in_bytes()));
}
__ beq(tmp, t0, known_ok);
__ cmp_klass_compressed(dst, tmp, t0, known_ok, true);
__ beq(src, dst, known_ok);
}
__ bind(halt);

7
src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
View File

@ -1518,12 +1518,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
add_debug_info_for_null_check_here(info);
}
if (UseCompressedClassPointers) {
__ lwu(result, Address(obj, oopDesc::klass_offset_in_bytes()));
__ decode_klass_not_null(result);
} else {
__ ld(result, Address(obj, oopDesc::klass_offset_in_bytes()));
}
__ load_klass(result, obj);
}
void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {

22
src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
View File

@ -164,15 +164,19 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, i
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register tmp1, Register tmp2) {
assert_different_registers(obj, klass, len, tmp1, tmp2);
// This assumes that all prototype bits fitr in an int32_t
mv(tmp1, (int32_t)(intptr_t)markWord::prototype().value());
sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes()));
if (UseCompressedClassPointers) { // Take care not to kill klass
encode_klass_not_null(tmp1, klass, tmp2);
sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
if (UseCompactObjectHeaders) {
ld(tmp1, Address(klass, Klass::prototype_header_offset()));
sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes()));
} else {
sd(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
// This assumes that all prototype bits fitr in an int32_t
mv(tmp1, checked_cast<int32_t>(markWord::prototype().value()));
sd(tmp1, Address(obj, oopDesc::mark_offset_in_bytes()));
if (UseCompressedClassPointers) { // Take care not to kill klass
encode_klass_not_null(tmp1, klass, tmp2);
sw(tmp1, Address(obj, oopDesc::klass_offset_in_bytes()));
} else {
sd(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
}
}
if (len->is_valid()) {
@ -183,7 +187,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
// Clear gap/first 4 bytes following the length field.
sw(zr, Address(obj, base_offset));
}
} else if (UseCompressedClassPointers) {
} else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
store_klass_gap(obj, zr);
}
}

10
src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
View File

@ -3119,3 +3119,13 @@ void C2_MacroAssembler::extract_fp_v(FloatRegister dst, VectorRegister src, Basi
vfmv_f_s(dst, tmp);
}
}
void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Address src) {
// The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
// obj-start, so that we can load from the object's mark-word instead. Usually the address
// comes as obj-start in obj and klass_offset_in_bytes in disp.
assert(UseCompactObjectHeaders, "must");
int offset = oopDesc::mark_offset_in_bytes() - oopDesc::klass_offset_in_bytes();
ld(dst, Address(src.base(), src.offset() + offset));
srli(dst, dst, markWord::klass_shift);
}

2
src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
View File

@ -277,4 +277,6 @@
void extract_v(Register dst, VectorRegister src, BasicType bt, int idx, VectorRegister tmp);
void extract_fp_v(FloatRegister dst, VectorRegister src, BasicType bt, int idx, VectorRegister tmp);
void load_narrow_klass_compact_c2(Register dst, Address src);
#endif // CPU_RISCV_C2_MACROASSEMBLER_RISCV_HPP

4
src/hotspot/cpu/riscv/compressedKlass_riscv.cpp
View File

@ -56,9 +56,9 @@ char* CompressedKlassPointers::reserve_address_space_for_compressed_classes(size
result = reserve_address_space_for_zerobased_encoding(size, aslr);
}
// Failing that, optimize for case (3) - a base with only bits set between [33-44)
// Failing that, optimize for case (3) - a base with only bits set between [32-44)
if (result == nullptr) {
const uintptr_t from = nth_bit(32 + (optimize_for_zero_base ? LogKlassAlignmentInBytes : 0));
const uintptr_t from = nth_bit(32);
constexpr uintptr_t to = nth_bit(44);
constexpr size_t alignment = nth_bit(32);
result = reserve_address_space_X(from, to, size, alignment, aslr);

58
src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
View File

@ -2503,20 +2503,19 @@ void MacroAssembler::orptr(Address adr, RegisterOrConstant src, Register tmp1, R
sd(tmp1, adr);
}
void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp1, Register tmp2, Label &L) {
assert_different_registers(oop, trial_klass, tmp1, tmp2);
if (UseCompressedClassPointers) {
lwu(tmp1, Address(oop, oopDesc::klass_offset_in_bytes()));
if (CompressedKlassPointers::base() == nullptr) {
slli(tmp1, tmp1, CompressedKlassPointers::shift());
beq(trial_klass, tmp1, L);
return;
}
decode_klass_not_null(tmp1, tmp2);
void MacroAssembler::cmp_klass_compressed(Register oop, Register trial_klass, Register tmp, Label &L, bool equal) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp, oop);
} else if (UseCompressedClassPointers) {
lwu(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
} else {
ld(tmp1, Address(oop, oopDesc::klass_offset_in_bytes()));
ld(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
}
if (equal) {
beq(trial_klass, tmp, L);
} else {
bne(trial_klass, tmp, L);
}
beq(trial_klass, tmp1, L);
}
// Move an oop into a register.
@ -2722,10 +2721,19 @@ void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
}
}
void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
ld(dst, Address(src, oopDesc::mark_offset_in_bytes()));
srli(dst, dst, markWord::klass_shift);
}
void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
assert_different_registers(dst, tmp);
assert_different_registers(src, tmp);
if (UseCompressedClassPointers) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
decode_klass_not_null(dst, tmp);
} else if (UseCompressedClassPointers) {
lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
decode_klass_not_null(dst, tmp);
} else {
@ -2736,6 +2744,7 @@ void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
// FIXME: Should this be a store release? concurrent gcs assumes
// klass length is valid if klass field is not null.
assert(!UseCompactObjectHeaders, "not with compact headers");
if (UseCompressedClassPointers) {
encode_klass_not_null(src, tmp);
sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
@ -2745,6 +2754,7 @@ void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
}
void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "not with compact headers");
if (UseCompressedClassPointers) {
// Store to klass gap in destination
sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
@ -2761,8 +2771,7 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register
if (CompressedKlassPointers::base() == nullptr) {
if (CompressedKlassPointers::shift() != 0) {
assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
slli(dst, src, LogKlassAlignmentInBytes);
slli(dst, src, CompressedKlassPointers::shift());
} else {
mv(dst, src);
}
@ -2778,9 +2787,9 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register
mv(xbase, (uintptr_t)CompressedKlassPointers::base());
if (CompressedKlassPointers::shift() != 0) {
assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
assert_different_registers(t0, xbase);
shadd(dst, src, xbase, t0, LogKlassAlignmentInBytes);
Register t = src == dst ? dst : t0;
assert_different_registers(t, xbase);
shadd(dst, src, xbase, t, CompressedKlassPointers::shift());
} else {
add(dst, xbase, src);
}
@ -2796,8 +2805,7 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src, Register
if (CompressedKlassPointers::base() == nullptr) {
if (CompressedKlassPointers::shift() != 0) {
assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
srli(dst, src, LogKlassAlignmentInBytes);
srli(dst, src, CompressedKlassPointers::shift());
} else {
mv(dst, src);
}
@ -2819,8 +2827,7 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src, Register
mv(xbase, (uintptr_t)CompressedKlassPointers::base());
sub(dst, src, xbase);
if (CompressedKlassPointers::shift() != 0) {
assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
srli(dst, dst, LogKlassAlignmentInBytes);
srli(dst, dst, CompressedKlassPointers::shift());
}
}
@ -4315,7 +4322,7 @@ address MacroAssembler::ic_call(address entry, jint method_index) {
int MacroAssembler::ic_check_size() {
// No compressed
return (MacroAssembler::instruction_size * (2 /* 2 loads */ + 1 /* branch */)) +
far_branch_size();
far_branch_size() + (UseCompactObjectHeaders ? MacroAssembler::instruction_size * 1 : 0);
}
int MacroAssembler::ic_check(int end_alignment) {
@ -4335,7 +4342,10 @@ int MacroAssembler::ic_check(int end_alignment) {
align(end_alignment, ic_check_size());
int uep_offset = offset();
if (UseCompressedClassPointers) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(tmp1, receiver);
lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
} else if (UseCompressedClassPointers) {
lwu(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
} else {

3
src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
View File

@ -195,8 +195,9 @@ class MacroAssembler: public Assembler {
void access_store_at(BasicType type, DecoratorSet decorators, Address dst,
Register val, Register tmp1, Register tmp2, Register tmp3);
void load_klass(Register dst, Register src, Register tmp = t0);
void load_narrow_klass_compact(Register dst, Register src);
void store_klass(Register dst, Register src, Register tmp = t0);
void cmp_klass(Register oop, Register trial_klass, Register tmp1, Register tmp2, Label &L);
void cmp_klass_compressed(Register oop, Register trial_klass, Register tmp, Label &L, bool equal);
void encode_klass_not_null(Register r, Register tmp = t0);
void decode_klass_not_null(Register r, Register tmp = t0);

16
src/hotspot/cpu/riscv/riscv.ad
View File

@ -4817,6 +4817,7 @@ instruct loadKlass(iRegPNoSp dst, memory mem)
// Load Narrow Klass Pointer
instruct loadNKlass(iRegNNoSp dst, memory mem)
%{
predicate(!UseCompactObjectHeaders);
match(Set dst (LoadNKlass mem));
ins_cost(LOAD_COST);
@ -4829,6 +4830,21 @@ instruct loadNKlass(iRegNNoSp dst, memory mem)
ins_pipe(iload_reg_mem);
%}
instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory mem)
%{
predicate(UseCompactObjectHeaders);
match(Set dst (LoadNKlass mem));
ins_cost(LOAD_COST);
format %{ "lwu $dst, $mem\t# loadNKlass, compressed class ptr, #@loadNKlass" %}
ins_encode %{
__ load_narrow_klass_compact_c2(as_Register($dst$$reg), Address(as_Register($mem$$base), $mem$$disp));
%}
ins_pipe(iload_reg_mem);
%}
// Load Float
instruct loadF(fRegF dst, memory mem)
%{

27
src/hotspot/cpu/riscv/templateTable_riscv.cpp
View File

@ -3546,12 +3546,22 @@ void TemplateTable::_new() {
// The object is initialized before the header. If the object size is
// zero, go directly to the header initialization.
__ sub(x13, x13, sizeof(oopDesc));
if (UseCompactObjectHeaders) {
assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
__ sub(x13, x13, oopDesc::base_offset_in_bytes());
} else {
__ sub(x13, x13, sizeof(oopDesc));
}
__ beqz(x13, initialize_header);
// Initialize object fields
{
__ add(x12, x10, sizeof(oopDesc));
if (UseCompactObjectHeaders) {
assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
__ add(x12, x10, oopDesc::base_offset_in_bytes());
} else {
__ add(x12, x10, sizeof(oopDesc));
}
Label loop;
__ bind(loop);
__ sd(zr, Address(x12));
@ -3562,10 +3572,15 @@ void TemplateTable::_new() {
// initialize object hader only.
__ bind(initialize_header);
__ mv(t0, (intptr_t)markWord::prototype().value());
__ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes()));
__ store_klass_gap(x10, zr); // zero klass gap for compressed oops
__ store_klass(x10, x14); // store klass last
if (UseCompactObjectHeaders) {
__ ld(t0, Address(x14, Klass::prototype_header_offset()));
__ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes()));
} else {
__ mv(t0, (intptr_t)markWord::prototype().value());
__ sd(t0, Address(x10, oopDesc::mark_offset_in_bytes()));
__ store_klass_gap(x10, zr); // zero klass gap for compressed oops
__ store_klass(x10, x14); // store klass last
}
if (DTraceAllocProbes) {
// Trigger dtrace event for fastpath

27
src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
View File

@ -2047,8 +2047,6 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
// Length and pos's are all sign extended at this point on 64bit.
@ -2112,13 +2110,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// We don't know the array types are compatible.
if (basic_type != T_OBJECT) {
// Simple test for basic type arrays.
if (UseCompressedClassPointers) {
__ z_l(tmp, src_klass_addr);
__ z_c(tmp, dst_klass_addr);
} else {
__ z_lg(tmp, src_klass_addr);
__ z_cg(tmp, dst_klass_addr);
}
__ cmp_klasses_from_objects(src, dst, tmp, Z_R1_scratch);
__ branch_optimized(Assembler::bcondNotEqual, *stub->entry());
} else {
// For object arrays, if src is a sub class of dst then we can
@ -2252,15 +2244,13 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
}
if (basic_type != T_OBJECT) {
if (UseCompressedClassPointers) { __ z_c (tmp, dst_klass_addr); }
else { __ z_cg(tmp, dst_klass_addr); }
__ cmp_klass(tmp, dst, Z_R1_scratch);
__ branch_optimized(Assembler::bcondNotEqual, halt);
if (UseCompressedClassPointers) { __ z_c (tmp, src_klass_addr); }
else { __ z_cg(tmp, src_klass_addr); }
__ cmp_klass(tmp, src, Z_R1_scratch);
__ branch_optimized(Assembler::bcondEqual, known_ok);
} else {
if (UseCompressedClassPointers) { __ z_c (tmp, dst_klass_addr); }
else { __ z_cg(tmp, dst_klass_addr); }
__ cmp_klass(tmp, dst, Z_R1_scratch);
__ branch_optimized(Assembler::bcondEqual, known_ok);
__ compareU64_and_branch(src, dst, Assembler::bcondEqual, known_ok);
}
@ -2755,12 +2745,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
add_debug_info_for_null_check_here(info);
}
if (UseCompressedClassPointers) {
__ z_llgf(result, Address(obj, oopDesc::klass_offset_in_bytes()));
__ decode_klass_not_null(result);
} else {
__ z_lg(result, Address(obj, oopDesc::klass_offset_in_bytes()));
}
__ load_klass(result, obj);
}
void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
ciMethod* method = op->profiled_method();

14
src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
View File

@ -177,17 +177,21 @@ void C1_MacroAssembler::try_allocate(
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register Rzero, Register t1) {
assert_different_registers(obj, klass, len, t1, Rzero);
// This assumes that all prototype bits fit in an int32_t.
load_const_optimized(t1, (intx)markWord::prototype().value());
z_stg(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
if (UseCompactObjectHeaders) {
z_lg(t1, Address(klass, in_bytes(Klass::prototype_header_offset())));
z_stg(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
} else {
load_const_optimized(t1, (intx)markWord::prototype().value());
z_stg(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
store_klass(klass, obj, t1);
}
if (len->is_valid()) {
// Length will be in the klass gap, if one exists.
z_st(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
} else if (UseCompressedClassPointers) {
} else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
store_klass_gap(Rzero, obj); // Zero klass gap for compressed oops.
}
store_klass(klass, obj, t1);
}
void C1_MacroAssembler::initialize_body(Register objectFields, Register len_in_bytes, Register Rzero) {

7
src/hotspot/cpu/s390/c2_MacroAssembler_s390.cpp
View File

@ -42,6 +42,13 @@ void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Regi
compiler_fast_unlock_lightweight_object(obj, box, temp1, temp2);
}
void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Address src) {
// The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
// obj-start, so that we can load from the object's mark-word instead.
z_lg(dst, src.plus_disp(-oopDesc::klass_offset_in_bytes()));
z_srlg(dst, dst, markWord::klass_shift); // TODO: could be z_sra
}
//------------------------------------------------------
// Special String Intrinsics. Implementation
//------------------------------------------------------

2
src/hotspot/cpu/s390/c2_MacroAssembler_s390.hpp
View File

@ -33,6 +33,8 @@
void fast_lock_lightweight(Register obj, Register box, Register temp1, Register temp2);
void fast_unlock_lightweight(Register obj, Register box, Register temp1, Register temp2);
void load_narrow_klass_compact_c2(Register dst, Address src);
//-------------------------------------------
// Special String Intrinsics Implementation.
//-------------------------------------------

82
src/hotspot/cpu/s390/macroAssembler_s390.cpp
View File

@ -2160,7 +2160,16 @@ void MacroAssembler::call_VM_leaf_base(address entry_point) {
}
int MacroAssembler::ic_check_size() {
return 30 + (ImplicitNullChecks ? 0 : 6);
int ic_size = 24;
if (!ImplicitNullChecks) {
ic_size += 6;
}
if (UseCompactObjectHeaders) {
ic_size += 12;
} else {
ic_size += 6; // either z_llgf or z_lg
}
return ic_size;
}
int MacroAssembler::ic_check(int end_alignment) {
@ -2181,7 +2190,9 @@ int MacroAssembler::ic_check(int end_alignment) {
z_cgij(R2_receiver, 0, Assembler::bcondEqual, failure);
}
if (UseCompressedClassPointers) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(R1_scratch, R2_receiver);
} else if (UseCompressedClassPointers) {
z_llgf(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
} else {
z_lg(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
@ -3852,7 +3863,7 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
#ifdef ASSERT
Label ok;
z_tmll(current, KlassAlignmentInBytes-1); // Check alignment.
z_tmll(current, CompressedKlassPointers::klass_alignment_in_bytes() - 1); // Check alignment.
z_brc(Assembler::bcondAllZero, ok);
// The plain disassembler does not recognize illtrap. It instead displays
// a 32-bit value. Issuing two illtraps assures the disassembler finds
@ -3866,7 +3877,6 @@ void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
// We then can be sure we calculate an offset that fits into 32 bit.
// More generally speaking: all subsequent calculations are purely 32-bit.
if (shift != 0) {
assert (LogKlassAlignmentInBytes == shift, "decode alg wrong");
z_srlg(dst, current, shift);
current = dst;
}
@ -3996,7 +4006,7 @@ void MacroAssembler::decode_klass_not_null(Register dst) {
#ifdef ASSERT
Label ok;
z_tmll(dst, KlassAlignmentInBytes-1); // Check alignment.
z_tmll(dst, CompressedKlassPointers::klass_alignment_in_bytes() - 1); // Check alignment.
z_brc(Assembler::bcondAllZero, ok);
// The plain disassembler does not recognize illtrap. It instead displays
// a 32-bit value. Issuing two illtraps assures the disassembler finds
@ -4043,7 +4053,7 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
#ifdef ASSERT
Label ok;
z_tmll(dst, KlassAlignmentInBytes-1); // Check alignment.
z_tmll(dst, CompressedKlassPointers::klass_alignment_in_bytes() - 1); // Check alignment.
z_brc(Assembler::bcondAllZero, ok);
// The plain disassembler does not recognize illtrap. It instead displays
// a 32-bit value. Issuing two illtraps assures the disassembler finds
@ -4065,10 +4075,58 @@ void MacroAssembler::load_klass(Register klass, Address mem) {
}
}
// Loads the obj's Klass* into dst.
// Input:
// src - the oop we want to load the klass from.
// dst - output nklass.
void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
BLOCK_COMMENT("load_narrow_klass_compact {");
assert(UseCompactObjectHeaders, "expects UseCompactObjectHeaders");
z_lg(dst, Address(src, oopDesc::mark_offset_in_bytes()));
z_srlg(dst, dst, markWord::klass_shift);
BLOCK_COMMENT("} load_narrow_klass_compact");
}
void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
BLOCK_COMMENT("cmp_klass {");
assert_different_registers(obj, klass, tmp);
if (UseCompactObjectHeaders) {
assert(tmp != noreg, "required");
assert_different_registers(klass, obj, tmp);
load_narrow_klass_compact(tmp, obj);
z_cr(klass, tmp);
} else if (UseCompressedClassPointers) {
z_c(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
} else {
z_cg(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
}
BLOCK_COMMENT("} cmp_klass");
}
void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) {
BLOCK_COMMENT("cmp_klasses_from_objects {");
if (UseCompactObjectHeaders) {
assert(tmp1 != noreg && tmp2 != noreg, "required");
assert_different_registers(obj1, obj2, tmp1, tmp2);
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
z_cr(tmp1, tmp2);
} else if (UseCompressedClassPointers) {
z_l(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
z_c(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
} else {
z_lg(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
z_cg(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
}
BLOCK_COMMENT("} cmp_klasses_from_objects");
}
void MacroAssembler::load_klass(Register klass, Register src_oop) {
if (UseCompressedClassPointers) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(klass, src_oop);
decode_klass_not_null(klass);
} else if (UseCompressedClassPointers) {
z_llgf(klass, oopDesc::klass_offset_in_bytes(), src_oop);
// Attention: no null check here!
decode_klass_not_null(klass);
} else {
z_lg(klass, oopDesc::klass_offset_in_bytes(), src_oop);
@ -4076,6 +4134,7 @@ void MacroAssembler::load_klass(Register klass, Register src_oop) {
}
void MacroAssembler::store_klass(Register klass, Register dst_oop, Register ck) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
if (UseCompressedClassPointers) {
assert_different_registers(dst_oop, klass, Z_R0);
if (ck == noreg) ck = klass;
@ -4087,6 +4146,7 @@ void MacroAssembler::store_klass(Register klass, Register dst_oop, Register ck)
}
void MacroAssembler::store_klass_gap(Register s, Register d) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
if (UseCompressedClassPointers) {
assert(s != d, "not enough registers");
// Support s = noreg.
@ -4112,7 +4172,11 @@ void MacroAssembler::compare_klass_ptr(Register Rop1, int64_t disp, Register Rba
const int shift = CompressedKlassPointers::shift();
address base = CompressedKlassPointers::base();
assert((shift == 0) || (shift == LogKlassAlignmentInBytes), "cKlass encoder detected bad shift");
if (UseCompactObjectHeaders) {
assert(shift >= 3, "cKlass encoder detected bad shift");
} else {
assert((shift == 0) || (shift == 3), "cKlass encoder detected bad shift");
}
assert_different_registers(Rop1, Z_R0);
assert_different_registers(Rop1, Rbase, Z_R1);

7
src/hotspot/cpu/s390/macroAssembler_s390.hpp
View File

@ -803,6 +803,13 @@ class MacroAssembler: public Assembler {
void load_klass(Register klass, Register src_oop);
void store_klass(Register klass, Register dst_oop, Register ck = noreg); // Klass will get compressed if ck not provided.
void store_klass_gap(Register s, Register dst_oop);
void load_narrow_klass_compact(Register dst, Register src);
// Compares the Klass pointer of an object to a given Klass (which might be narrow,
// depending on UseCompressedClassPointers).
void cmp_klass(Register klass, Register obj, Register tmp);
// Compares the Klass pointer of two objects obj1 and obj2. Result is in the condition flags.
// Uses tmp1 and tmp2 as temporary registers.
void cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2);
// This function calculates the size of the code generated by
// decode_klass_not_null(register dst)

16
src/hotspot/cpu/s390/s390.ad
View File

@ -4410,6 +4410,7 @@ instruct loadN(iRegN dst, memory mem) %{
// Load narrow Klass Pointer
instruct loadNKlass(iRegN dst, memory mem) %{
predicate(!UseCompactObjectHeaders);
match(Set dst (LoadNKlass mem));
ins_cost(MEMORY_REF_COST);
size(Z_DISP3_SIZE);
@ -4419,6 +4420,21 @@ instruct loadNKlass(iRegN dst, memory mem) %{
ins_pipe(pipe_class_dummy);
%}
instruct loadNKlassCompactHeaders(iRegN dst, memory mem, flagsReg cr) %{
match(Set dst (LoadNKlass mem));
predicate(UseCompactObjectHeaders);
effect(KILL cr);
ins_cost(MEMORY_REF_COST);
format %{ "load_narrow_klass_compact $dst,$mem \t# compressed class ptr" %}
// TODO: size()
ins_encode %{
__ block_comment("load_narrow_klass_compact_c2 {");
__ load_narrow_klass_compact_c2($dst$$Register, $mem$$Address);
__ block_comment("} load_narrow_klass_compact");
%}
ins_pipe(pipe_class_dummy);
%}
// Load constant Compressed Pointer
instruct loadConN(iRegN dst, immN src) %{

27
src/hotspot/cpu/s390/templateTable_s390.cpp
View File

@ -3952,7 +3952,12 @@ void TemplateTable::_new() {
if (!ZeroTLAB) {
// The object is initialized before the header. If the object size is
// zero, go directly to the header initialization.
__ z_aghi(Rsize, (int)-sizeof(oopDesc)); // Subtract header size, set CC.
if (UseCompactObjectHeaders) {
assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
__ z_aghi(Rsize, (int)-oopDesc::base_offset_in_bytes());
} else {
__ z_aghi(Rsize, (int)-sizeof(oopDesc)); // Subtract header size, set CC.
}
__ z_bre(initialize_header); // Jump if size of fields is zero.
// Initialize object fields.
@ -3964,17 +3969,25 @@ void TemplateTable::_new() {
// Set Rzero to 0 and use it as src length, then mvcle will copy nothing
// and fill the object with the padding value 0.
__ add2reg(RobjectFields, sizeof(oopDesc), RallocatedObject);
if (UseCompactObjectHeaders) {
__ add2reg(RobjectFields, oopDesc::base_offset_in_bytes(), RallocatedObject);
} else {
__ add2reg(RobjectFields, sizeof(oopDesc), RallocatedObject);
}
__ move_long_ext(RobjectFields, as_Register(Rzero->encoding() - 1), 0);
}
// Initialize object header only.
__ bind(initialize_header);
__ store_const(Address(RallocatedObject, oopDesc::mark_offset_in_bytes()),
(long)markWord::prototype().value());
__ store_klass_gap(Rzero, RallocatedObject); // Zero klass gap for compressed oops.
__ store_klass(iklass, RallocatedObject); // Store klass last.
if (UseCompactObjectHeaders) {
__ z_lg(tmp, Address(iklass, in_bytes(Klass::prototype_header_offset())));
__ z_stg(tmp, Address(RallocatedObject, oopDesc::mark_offset_in_bytes()));
} else {
__ store_const(Address(RallocatedObject, oopDesc::mark_offset_in_bytes()),
(long) markWord::prototype().value());
__ store_klass_gap(Rzero, RallocatedObject); // Zero klass gap for compressed oops.
__ store_klass(iklass, RallocatedObject); // Store klass last.
}
if (DTraceAllocProbes) {
// Trigger dtrace event for fastpath.

30
src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
View File

@ -3046,6 +3046,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
Register length = op->length()->as_register();
Register tmp = op->tmp()->as_register();
Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
CodeStub* stub = op->stub();
int flags = op->flags();
@ -3170,8 +3171,6 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
// length and pos's are all sign extended at this point on 64bit
@ -3237,13 +3236,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
// We don't know the array types are compatible
if (basic_type != T_OBJECT) {
// Simple test for basic type arrays
if (UseCompressedClassPointers) {
__ movl(tmp, src_klass_addr);
__ cmpl(tmp, dst_klass_addr);
} else {
__ movptr(tmp, src_klass_addr);
__ cmpptr(tmp, dst_klass_addr);
}
__ cmp_klasses_from_objects(src, dst, tmp, tmp2);
__ jcc(Assembler::notEqual, *stub->entry());
} else {
// For object arrays, if src is a sub class of dst then we can
@ -3302,6 +3295,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
store_parameter(src, 4);
#ifndef _LP64
Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
__ movptr(tmp, dst_klass_addr);
__ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset()));
__ push(tmp);
@ -3405,16 +3399,12 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
#endif
if (basic_type != T_OBJECT) {
if (UseCompressedClassPointers) __ cmpl(tmp, dst_klass_addr);
else __ cmpptr(tmp, dst_klass_addr);
__ cmp_klass(tmp, dst, tmp2);
__ jcc(Assembler::notEqual, halt);
if (UseCompressedClassPointers) __ cmpl(tmp, src_klass_addr);
else __ cmpptr(tmp, src_klass_addr);
__ cmp_klass(tmp, src, tmp2);
__ jcc(Assembler::equal, known_ok);
} else {
if (UseCompressedClassPointers) __ cmpl(tmp, dst_klass_addr);
else __ cmpptr(tmp, dst_klass_addr);
__ cmp_klass(tmp, dst, tmp2);
__ jcc(Assembler::equal, known_ok);
__ cmpptr(src, dst);
__ jcc(Assembler::equal, known_ok);
@ -3511,13 +3501,7 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
add_debug_info_for_null_check_here(info);
}
#ifdef _LP64
if (UseCompressedClassPointers) {
__ movl(result, Address(obj, oopDesc::klass_offset_in_bytes()));
__ decode_klass_not_null(result, rscratch1);
} else
#endif
__ movptr(result, Address(obj, oopDesc::klass_offset_in_bytes()));
__ load_klass(result, obj, rscratch1);
}
void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {

16
src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
View File

@ -170,16 +170,20 @@ void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, i
void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
assert_different_registers(obj, klass, len);
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
assert_different_registers(obj, klass, len, t1, t2);
#ifdef _LP64
if (UseCompressedClassPointers) { // Take care not to kill klass
if (UseCompactObjectHeaders) {
movptr(t1, Address(klass, Klass::prototype_header_offset()));
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), t1);
} else if (UseCompressedClassPointers) { // Take care not to kill klass
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
movptr(t1, klass);
encode_klass_not_null(t1, rscratch1);
movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
} else
#endif
{
movptr(Address(obj, oopDesc::mark_offset_in_bytes()), checked_cast<int32_t>(markWord::prototype().value()));
movptr(Address(obj, oopDesc::klass_offset_in_bytes()), klass);
}
@ -196,7 +200,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
#endif
}
#ifdef _LP64
else if (UseCompressedClassPointers) {
else if (UseCompressedClassPointers && !UseCompactObjectHeaders) {
xorptr(t1, t1);
store_klass_gap(obj, t1);
}
@ -230,7 +234,9 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0,
"con_size_in_bytes is not multiple of alignment");
const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize;
if (UseCompactObjectHeaders) {
assert(hdr_size_in_bytes == 8, "check object headers size");
}
initialize_header(obj, klass, noreg, t1, t2);
if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) {

10
src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
View File

@ -7093,3 +7093,13 @@ void C2_MacroAssembler::vector_saturating_op(int ideal_opc, BasicType elem_bt, X
vector_saturating_op(ideal_opc, elem_bt, dst, src1, src2, vlen_enc);
}
}
#ifdef _LP64
void C2_MacroAssembler::load_narrow_klass_compact_c2(Register dst, Address src) {
// The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
// obj-start, so that we can load from the object's mark-word instead. Usually the address
// comes as obj-start in obj and klass_offset_in_bytes in disp.
movq(dst, src.plus_disp(-oopDesc::klass_offset_in_bytes()));
shrq(dst, markWord::klass_shift);
}
#endif

4
src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
View File

@ -583,4 +583,8 @@ public:
void select_from_two_vectors_evex(BasicType elem_bt, XMMRegister dst, XMMRegister src1, XMMRegister src2, int vlen_enc);
#ifdef _LP64
void load_narrow_klass_compact_c2(Register dst, Address src);
#endif
#endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP

178
src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp
View File

@ -26,6 +26,7 @@
#include "precompiled.hpp"
#include "macroAssembler_x86.hpp"
#include "stubGenerator_x86_64.hpp"
#include "oops/arrayOop.hpp"
#include "opto/c2_MacroAssembler.hpp"
#include "opto/intrinsicnode.hpp"
@ -160,6 +161,9 @@ static void highly_optimized_short_cases(StrIntrinsicNode::ArgEncoding ae, Regis
Register needle_len, XMMRegister XMM0, XMMRegister XMM1,
Register mask, Register tmp, MacroAssembler *_masm);
static void copy_to_stack(Register haystack, Register haystack_len, bool isU, Register tmp,
XMMRegister xtmp, MacroAssembler *_masm);
static void setup_jump_tables(StrIntrinsicNode::ArgEncoding ae, Label &L_error, Label &L_checkRange,
Label &L_fixup, address *big_jump_table, address *small_jump_table,
MacroAssembler *_masm);
@ -395,41 +399,21 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr
// Do "big switch" if haystack size > 32
__ cmpq(haystack_len, 0x20);
__ ja_b(L_bigSwitchTop);
__ ja(L_bigSwitchTop);
// Copy the small (< 32 byte) haystack to the stack. Allows for vector reads without page fault
// Only done for small haystacks
//
// NOTE: This code assumes that the haystack points to a java array type AND there are
// at least 16 bytes of header preceeding the haystack pointer.
// at least 8 bytes of header preceeding the haystack pointer.
//
// This means that we're copying up to 15 bytes of the header onto the stack along
// This means that we're copying up to 7 bytes of the header onto the stack along
// with the haystack bytes. After the copy completes, we adjust the haystack pointer
// to the valid haystack bytes on the stack.
{
Label L_moreThan16, L_adjustHaystack;
const Register index = rax;
const Register tmp = rax;
const Register haystack = rbx;
// Only a single vector load/store of either 16 or 32 bytes
__ cmpq(haystack_len, 0x10);
__ ja_b(L_moreThan16);
__ movq(index, COPIED_HAYSTACK_STACK_OFFSET + 0x10);
__ movdqu(XMM_TMP1, Address(haystack, haystack_len, Address::times_1, -0x10));
__ movdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), XMM_TMP1);
__ jmpb(L_adjustHaystack);
__ bind(L_moreThan16);
__ movq(index, COPIED_HAYSTACK_STACK_OFFSET + 0x20);
__ vmovdqu(XMM_TMP1, Address(haystack, haystack_len, Address::times_1, -0x20));
__ vmovdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), XMM_TMP1);
// Point the haystack at the correct location of the first byte of the "real" haystack on the stack
__ bind(L_adjustHaystack);
__ subq(index, haystack_len);
__ leaq(haystack, Address(rsp, index, Address::times_1));
copy_to_stack(haystack, haystack_len, false, tmp, XMM_TMP1, _masm);
}
// Dispatch to handlers for small needle and small haystack
@ -760,39 +744,39 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr
__ ja(L_wideNoExpand);
//
// Reads of existing needle are 16-byte chunks
// Writes to copied needle are 32-byte chunks
// Reads of existing needle are 8-byte chunks
// Writes to copied needle are 16-byte chunks
// Don't read past the end of the existing needle
//
// Start first read at [((ndlLen % 16) - 16) & 0xf]
// outndx += 32
// inndx += 16
// Start first read at [((ndlLen % 8) - 8) & 0x7]
// outndx += 16
// inndx += 8
// cmp nndx, ndlLen
// jae done
//
// Final index of start of needle at ((16 - (ndlLen %16)) & 0xf) << 1
// Final index of start of needle at ((8 - (ndlLen % 8)) & 0x7) << 1
//
// Starting read for needle at -(16 - (nLen % 16))
// Offset of needle in stack should be (16 - (nLen % 16)) * 2
// Starting read for needle at -(8 - (nLen % 8))
// Offset of needle in stack should be (8 - (nLen % 8)) * 2
__ movq(index, needle_len);
__ andq(index, 0xf); // nLen % 16
__ movq(offset, 0x10);
__ subq(offset, index); // 16 - (nLen % 16)
__ andq(index, 0x7); // nLen % 8
__ movq(offset, 0x8);
__ subq(offset, index); // 8 - (nLen % 8)
__ movq(index, offset);
__ shlq(offset, 1); // * 2
__ negq(index); // -(16 - (nLen % 16))
__ negq(index); // -(8 - (nLen % 8))
__ xorq(wr_index, wr_index);
__ bind(L_top);
// load needle and expand
__ vpmovzxbw(xmm0, Address(needle, index, Address::times_1), Assembler::AVX_256bit);
__ vpmovzxbw(xmm0, Address(needle, index, Address::times_1), Assembler::AVX_128bit);
// store expanded needle to stack
__ vmovdqu(Address(rsp, wr_index, Address::times_1, EXPANDED_NEEDLE_STACK_OFFSET), xmm0);
__ addq(index, 0x10);
__ movdqu(Address(rsp, wr_index, Address::times_1, EXPANDED_NEEDLE_STACK_OFFSET), xmm0);
__ addq(index, 0x8);
__ cmpq(index, needle_len);
__ jae(L_finished);
__ addq(wr_index, 32);
__ addq(wr_index, 16);
__ jmpb(L_top);
// adjust pointer and length of needle
@ -1582,35 +1566,9 @@ static void highly_optimized_short_cases(StrIntrinsicNode::ArgEncoding ae, Regis
assert((COPIED_HAYSTACK_STACK_OFFSET == 0), "Must be zero!");
assert((COPIED_HAYSTACK_STACK_SIZE == 64), "Must be 64!");
// Copy incoming haystack onto stack
{
Label L_adjustHaystack, L_moreThan16;
// Copy haystack to stack (haystack <= 32 bytes)
__ subptr(rsp, COPIED_HAYSTACK_STACK_SIZE);
__ cmpq(haystack_len, isU ? 0x8 : 0x10);
__ ja_b(L_moreThan16);
__ movq(tmp, COPIED_HAYSTACK_STACK_OFFSET + 0x10);
__ movdqu(XMM0, Address(haystack, haystack_len, isU ? Address::times_2 : Address::times_1, -0x10));
__ movdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), XMM0);
__ jmpb(L_adjustHaystack);
__ bind(L_moreThan16);
__ movq(tmp, COPIED_HAYSTACK_STACK_OFFSET + 0x20);
__ vmovdqu(XMM0, Address(haystack, haystack_len, isU ? Address::times_2 : Address::times_1, -0x20));
__ vmovdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), XMM0);
__ bind(L_adjustHaystack);
__ subptr(tmp, haystack_len);
if (isU) {
// For UTF-16, lengths are half
__ subptr(tmp, haystack_len);
}
// Point the haystack to the stack
__ leaq(haystack, Address(rsp, tmp, Address::times_1));
}
// Copy incoming haystack onto stack (haystack <= 32 bytes)
__ subptr(rsp, COPIED_HAYSTACK_STACK_SIZE);
copy_to_stack(haystack, haystack_len, isU, tmp, XMM0, _masm);
// Creates a mask of (n - k + 1) ones. This prevents recognizing any false-positives
// past the end of the valid haystack.
@ -1672,6 +1630,86 @@ static void highly_optimized_short_cases(StrIntrinsicNode::ArgEncoding ae, Regis
__ jmpb(L_out);
}
// Copy the small (<= 32 byte) haystack to the stack. Allows for vector reads without page fault
// Only done for small haystacks
// NOTE: This code assumes that the haystack points to a java array type AND there are
// at least 8 bytes of header preceeding the haystack pointer.
// We're copying up to 7 bytes of the header onto the stack along with the haystack bytes.
// After the copy completes, we adjust the haystack pointer
// to the valid haystack bytes on the stack.
//
// Copy haystack array elements to stack at region
// (COPIED_HAYSTACK_STACK_OFFSET - COPIED_HAYSTACK_STACK_OFFSET+63) with the following conditions:
// It may copy up to 7 bytes that precede the array
// It doesn't read beyond the end of the array
// There are atleast 31 bytes of stack region beyond the end of array
// Inputs:
// haystack - Address of haystack
// haystack_len - Number of elements in haystack
// isU - Boolean indicating if each element is Latin1 or UTF16
// tmp, xtmp - Scratch registers
// Output:
// haystack - Address of copied string on stack
static void copy_to_stack(Register haystack, Register haystack_len, bool isU,
Register tmp, XMMRegister xtmp, MacroAssembler *_masm) {
Label L_moreThan8, L_moreThan16, L_moreThan24, L_adjustHaystack;
assert(arrayOopDesc::base_offset_in_bytes(isU ? T_CHAR : T_BYTE) >= 8,
"Needs at least 8 bytes preceding the array body");
// Copy haystack to stack (haystack <= 32 bytes)
int scale = isU ? 2 : 1; // bytes per char
Address::ScaleFactor addrScale = isU ? Address::times_2 : Address::times_1;
__ cmpq(haystack_len, 16/scale);
__ ja_b(L_moreThan16);
__ cmpq(haystack_len, 8/scale);
__ ja_b(L_moreThan8);
// haystack length <= 8 bytes, copy 8 bytes upto haystack end reading at most 7 bytes into the header
__ movq(tmp, COPIED_HAYSTACK_STACK_OFFSET + 8);
__ movq(xtmp, Address(haystack, haystack_len, addrScale, -8));
__ movq(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), xtmp);
__ jmpb(L_adjustHaystack);
__ bind(L_moreThan8);
// haystack length > 8 and <=16 bytes, copy 16 bytes upto haystack end reading at most 7 bytes into the header
__ movq(tmp, COPIED_HAYSTACK_STACK_OFFSET + 16);
__ movdqu(xtmp, Address(haystack, haystack_len, addrScale, -16));
__ movdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), xtmp);
__ jmpb(L_adjustHaystack);
__ bind(L_moreThan16);
__ cmpq(haystack_len, 24/scale);
__ ja_b(L_moreThan24);
// haystack length > 16 and <=24 bytes, copy 24 bytes upto haystack end reading at most 7 bytes into the header
__ movq(tmp, COPIED_HAYSTACK_STACK_OFFSET + 24);
__ movdqu(xtmp, Address(haystack, haystack_len, addrScale, -24));
__ movdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), xtmp);
__ movq(xtmp, Address(haystack, haystack_len, addrScale, -8));
__ movq(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET + 16), xtmp);
__ jmpb(L_adjustHaystack);
__ bind(L_moreThan24);
// haystack length > 24 and < 32 bytes, copy 32 bytes upto haystack end reading at most 7 bytes into the header
__ movq(tmp, COPIED_HAYSTACK_STACK_OFFSET + 32);
__ vmovdqu(xtmp, Address(haystack, haystack_len, addrScale, -32));
__ vmovdqu(Address(rsp, COPIED_HAYSTACK_STACK_OFFSET), xtmp);
__ bind(L_adjustHaystack);
__ subptr(tmp, haystack_len);
if (isU) {
__ subptr(tmp, haystack_len);
}
// Point the haystack to the stack
__ leaq(haystack, Address(rsp, tmp, Address::times_1));
}
////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////

101
src/hotspot/cpu/x86/macroAssembler_x86.cpp
View File

@ -1350,7 +1350,8 @@ void MacroAssembler::ic_call(address entry, jint method_index) {
}
int MacroAssembler::ic_check_size() {
return LP64_ONLY(14) NOT_LP64(12);
return
LP64_ONLY(UseCompactObjectHeaders ? 17 : 14) NOT_LP64(12);
}
int MacroAssembler::ic_check(int end_alignment) {
@ -1366,6 +1367,12 @@ int MacroAssembler::ic_check(int end_alignment) {
int uep_offset = offset();
#ifdef _LP64
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(temp, receiver);
cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
} else
#endif
if (UseCompressedClassPointers) {
movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
@ -1376,7 +1383,7 @@ int MacroAssembler::ic_check(int end_alignment) {
// if inline cache check fails, then jump to runtime routine
jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
assert((offset() % end_alignment) == 0, "Misaligned verified entry point (%d, %d, %d)", uep_offset, offset(), end_alignment);
return uep_offset;
}
@ -5948,19 +5955,33 @@ void MacroAssembler::load_method_holder(Register holder, Register method) {
movptr(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass*
}
#ifdef _LP64
void MacroAssembler::load_narrow_klass_compact(Register dst, Register src) {
assert(UseCompactObjectHeaders, "expect compact object headers");
movq(dst, Address(src, oopDesc::mark_offset_in_bytes()));
shrq(dst, markWord::klass_shift);
}
#endif
void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
assert_different_registers(src, tmp);
assert_different_registers(dst, tmp);
#ifdef _LP64
if (UseCompressedClassPointers) {
if (UseCompactObjectHeaders) {
load_narrow_klass_compact(dst, src);
decode_klass_not_null(dst, tmp);
} else if (UseCompressedClassPointers) {
movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
decode_klass_not_null(dst, tmp);
} else
#endif
{
movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
}
}
void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
assert(!UseCompactObjectHeaders, "not with compact headers");
assert_different_registers(src, tmp);
assert_different_registers(dst, tmp);
#ifdef _LP64
@ -5972,6 +5993,41 @@ void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
}
void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
#ifdef _LP64
if (UseCompactObjectHeaders) {
assert(tmp != noreg, "need tmp");
assert_different_registers(klass, obj, tmp);
load_narrow_klass_compact(tmp, obj);
cmpl(klass, tmp);
} else if (UseCompressedClassPointers) {
cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
} else
#endif
{
cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
}
}
void MacroAssembler::cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2) {
#ifdef _LP64
if (UseCompactObjectHeaders) {
assert(tmp2 != noreg, "need tmp2");
assert_different_registers(obj1, obj2, tmp1, tmp2);
load_narrow_klass_compact(tmp1, obj1);
load_narrow_klass_compact(tmp2, obj2);
cmpl(tmp1, tmp2);
} else if (UseCompressedClassPointers) {
movl(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
cmpl(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
} else
#endif
{
movptr(tmp1, Address(obj1, oopDesc::klass_offset_in_bytes()));
cmpptr(tmp1, Address(obj2, oopDesc::klass_offset_in_bytes()));
}
}
void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
Register tmp1, Register thread_tmp) {
BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
@ -6019,6 +6075,7 @@ void MacroAssembler::store_heap_oop_null(Address dst) {
#ifdef _LP64
void MacroAssembler::store_klass_gap(Register dst, Register src) {
assert(!UseCompactObjectHeaders, "Don't use with compact headers");
if (UseCompressedClassPointers) {
// Store to klass gap in destination
movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
@ -6183,8 +6240,7 @@ void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
subq(r, tmp);
}
if (CompressedKlassPointers::shift() != 0) {
assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
shrq(r, LogKlassAlignmentInBytes);
shrq(r, CompressedKlassPointers::shift());
}
}
@ -6197,8 +6253,7 @@ void MacroAssembler::encode_and_move_klass_not_null(Register dst, Register src)
movptr(dst, src);
}
if (CompressedKlassPointers::shift() != 0) {
assert (LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
shrq(dst, LogKlassAlignmentInBytes);
shrq(dst, CompressedKlassPointers::shift());
}
}
@ -6210,8 +6265,7 @@ void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
// vtableStubs also counts instructions in pd_code_size_limit.
// Also do not verify_oop as this is called by verify_oop.
if (CompressedKlassPointers::shift() != 0) {
assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
shlq(r, LogKlassAlignmentInBytes);
shlq(r, CompressedKlassPointers::shift());
}
if (CompressedKlassPointers::base() != nullptr) {
mov64(tmp, (int64_t)CompressedKlassPointers::base());
@ -6233,17 +6287,28 @@ void MacroAssembler::decode_and_move_klass_not_null(Register dst, Register src)
// a pointer that needs nothing but a register rename.
movl(dst, src);
} else {
if (CompressedKlassPointers::base() != nullptr) {
mov64(dst, (int64_t)CompressedKlassPointers::base());
} else {
xorq(dst, dst);
}
if (CompressedKlassPointers::shift() != 0) {
assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
leaq(dst, Address(dst, src, Address::times_8, 0));
if (CompressedKlassPointers::shift() <= Address::times_8) {
if (CompressedKlassPointers::base() != nullptr) {
mov64(dst, (int64_t)CompressedKlassPointers::base());
} else {
xorq(dst, dst);
}
if (CompressedKlassPointers::shift() != 0) {
assert(CompressedKlassPointers::shift() == Address::times_8, "klass not aligned on 64bits?");
leaq(dst, Address(dst, src, Address::times_8, 0));
} else {
addq(dst, src);
}
} else {
if (CompressedKlassPointers::base() != nullptr) {
const uint64_t base_right_shifted =
(uint64_t)CompressedKlassPointers::base() >> CompressedKlassPointers::shift();
mov64(dst, base_right_shifted);
} else {
xorq(dst, dst);
}
addq(dst, src);
shlq(dst, CompressedKlassPointers::shift());
}
}
}

11
src/hotspot/cpu/x86/macroAssembler_x86.hpp
View File

@ -363,9 +363,20 @@ class MacroAssembler: public Assembler {
void load_method_holder(Register holder, Register method);
// oop manipulations
#ifdef _LP64
void load_narrow_klass_compact(Register dst, Register src);
#endif
void load_klass(Register dst, Register src, Register tmp);
void store_klass(Register dst, Register src, Register tmp);
// Compares the Klass pointer of an object to a given Klass (which might be narrow,
// depending on UseCompressedClassPointers).
void cmp_klass(Register klass, Register obj, Register tmp);
// Compares the Klass pointer of two objects obj1 and obj2. Result is in the condition flags.
// Uses tmp1 and tmp2 as temporary registers.
void cmp_klasses_from_objects(Register obj1, Register obj2, Register tmp1, Register tmp2);
void access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
Register tmp1, Register thread_tmp);
void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,

2
src/hotspot/cpu/x86/matcher_x86.hpp
View File

@ -93,7 +93,7 @@
static bool narrow_klass_use_complex_address() {
NOT_LP64(ShouldNotCallThis();)
assert(UseCompressedClassPointers, "only for compressed klass code");
return (LogKlassAlignmentInBytes <= 3);
return (CompressedKlassPointers::shift() <= 3);
}
// Prefer ConN+DecodeN over ConP.

2
src/hotspot/cpu/x86/stubRoutines_x86.hpp
View File

@ -37,7 +37,7 @@ enum platform_dependent_constants {
_continuation_stubs_code_size = 1000 LP64_ONLY(+1000),
// AVX512 intrinsics add more code in 64-bit VM,
// Windows have more code to save/restore registers
_compiler_stubs_code_size = 20000 LP64_ONLY(+46000) WINDOWS_ONLY(+2000),
_compiler_stubs_code_size = 20000 LP64_ONLY(+47000) WINDOWS_ONLY(+2000),
_final_stubs_code_size = 10000 LP64_ONLY(+20000) WINDOWS_ONLY(+2000) ZGC_ONLY(+20000)
};

31
src/hotspot/cpu/x86/templateTable_x86.cpp
View File

@ -4084,7 +4084,12 @@ void TemplateTable::_new() {
// The object is initialized before the header. If the object size is
// zero, go directly to the header initialization.
__ decrement(rdx, sizeof(oopDesc));
if (UseCompactObjectHeaders) {
assert(is_aligned(oopDesc::base_offset_in_bytes(), BytesPerLong), "oop base offset must be 8-byte-aligned");
__ decrement(rdx, oopDesc::base_offset_in_bytes());
} else {
__ decrement(rdx, sizeof(oopDesc));
}
__ jcc(Assembler::zero, initialize_header);
// Initialize topmost object field, divide rdx by 8, check if odd and
@ -4106,22 +4111,30 @@ void TemplateTable::_new() {
// initialize remaining object fields: rdx was a multiple of 8
{ Label loop;
__ bind(loop);
__ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 1*oopSize), rcx);
NOT_LP64(__ movptr(Address(rax, rdx, Address::times_8, sizeof(oopDesc) - 2*oopSize), rcx));
int header_size_bytes = oopDesc::header_size() * HeapWordSize;
assert(is_aligned(header_size_bytes, BytesPerLong), "oop header size must be 8-byte-aligned");
__ movptr(Address(rax, rdx, Address::times_8, header_size_bytes - 1*oopSize), rcx);
NOT_LP64(__ movptr(Address(rax, rdx, Address::times_8, header_size_bytes - 2*oopSize), rcx));
__ decrement(rdx);
__ jcc(Assembler::notZero, loop);
}
// initialize object header only.
__ bind(initialize_header);
__ movptr(Address(rax, oopDesc::mark_offset_in_bytes()),
(intptr_t)markWord::prototype().value()); // header
__ pop(rcx); // get saved klass back in the register.
if (UseCompactObjectHeaders) {
__ pop(rcx); // get saved klass back in the register.
__ movptr(rbx, Address(rcx, Klass::prototype_header_offset()));
__ movptr(Address(rax, oopDesc::mark_offset_in_bytes()), rbx);
} else {
__ movptr(Address(rax, oopDesc::mark_offset_in_bytes()),
(intptr_t)markWord::prototype().value()); // header
__ pop(rcx); // get saved klass back in the register.
#ifdef _LP64
__ xorl(rsi, rsi); // use zero reg to clear memory (shorter code)
__ store_klass_gap(rax, rsi); // zero klass gap for compressed oops
__ xorl(rsi, rsi); // use zero reg to clear memory (shorter code)
__ store_klass_gap(rax, rsi); // zero klass gap for compressed oops
#endif
__ store_klass(rax, rcx, rscratch1); // klass
__ store_klass(rax, rcx, rscratch1); // klass
}
if (DTraceAllocProbes) {
// Trigger dtrace event for fastpath

15
src/hotspot/cpu/x86/x86_64.ad
View File

@ -4351,6 +4351,7 @@ instruct loadKlass(rRegP dst, memory mem)
// Load narrow Klass Pointer
instruct loadNKlass(rRegN dst, memory mem)
%{
predicate(!UseCompactObjectHeaders);
match(Set dst (LoadNKlass mem));
ins_cost(125); // XXX
@ -4361,6 +4362,19 @@ instruct loadNKlass(rRegN dst, memory mem)
ins_pipe(ialu_reg_mem); // XXX
%}
instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
%{
predicate(UseCompactObjectHeaders);
match(Set dst (LoadNKlass mem));
effect(KILL cr);
ins_cost(125); // XXX
format %{ "load_narrow_klass_compact $dst, $mem\t# compressed klass ptr" %}
ins_encode %{
__ load_narrow_klass_compact_c2($dst$$Register, $mem$$Address);
%}
ins_pipe(pipe_slow); // XXX
%}
// Load Float
instruct loadF(regF dst, memory mem)
%{
@ -11665,6 +11679,7 @@ instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
%{
predicate(!UseCompactObjectHeaders);
match(Set cr (CmpN src (LoadNKlass mem)));
format %{ "cmpl $mem, $src\t# compressed klass ptr" %}

56
src/hotspot/share/cds/archiveBuilder.cpp
View File

@ -227,8 +227,10 @@ bool ArchiveBuilder::gather_klass_and_symbol(MetaspaceClosure::Ref* ref, bool re
if (!is_excluded(klass)) {
_klasses->append(klass);
}
// See RunTimeClassInfo::get_for()
_estimated_metaspaceobj_bytes += align_up(BytesPerWord, SharedSpaceObjectAlignment);
// See RunTimeClassInfo::get_for(): make sure we have enough space for both maximum
// Klass alignment as well as the RuntimeInfo* pointer we will embed in front of a Klass.
_estimated_metaspaceobj_bytes += align_up(BytesPerWord, CompressedKlassPointers::klass_alignment_in_bytes()) +
align_up(sizeof(void*), SharedSpaceObjectAlignment);
} else if (ref->msotype() == MetaspaceObj::SymbolType) {
// Make sure the symbol won't be GC'ed while we are dumping the archive.
Symbol* sym = (Symbol*)ref->obj();
@ -661,7 +663,7 @@ void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* s
oldtop = dump_region->top();
if (src_info->msotype() == MetaspaceObj::ClassType) {
// Save a pointer immediate in front of an InstanceKlass, so
// Allocate space for a pointer directly in front of the future InstanceKlass, so
// we can do a quick lookup from InstanceKlass* -> RunTimeClassInfo*
// without building another hashtable. See RunTimeClassInfo::get_for()
// in systemDictionaryShared.cpp.
@ -670,8 +672,19 @@ void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* s
SystemDictionaryShared::validate_before_archiving(InstanceKlass::cast(klass));
dump_region->allocate(sizeof(address));
}
// Allocate space for the future InstanceKlass with proper alignment
const size_t alignment =
#ifdef _LP64
UseCompressedClassPointers ?
nth_bit(ArchiveBuilder::precomputed_narrow_klass_shift()) :
SharedSpaceObjectAlignment;
#else
SharedSpaceObjectAlignment;
#endif
dest = dump_region->allocate(bytes, alignment);
} else {
dest = dump_region->allocate(bytes);
}
dest = dump_region->allocate(bytes);
newtop = dump_region->top();
memcpy(dest, src, bytes);
@ -702,6 +715,8 @@ void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* s
src_info->set_buffered_addr((address)dest);
_alloc_stats.record(src_info->msotype(), int(newtop - oldtop), src_info->read_only());
DEBUG_ONLY(_alloc_stats.verify((int)dump_region->used(), src_info->read_only()));
}
// This is used by code that hand-assembles data structures, such as the LambdaProxyClassKey, that are
@ -780,6 +795,15 @@ void ArchiveBuilder::make_klasses_shareable() {
const char* generated = "";
Klass* k = get_buffered_addr(klasses()->at(i));
k->remove_java_mirror();
#ifdef _LP64
if (UseCompactObjectHeaders) {
Klass* requested_k = to_requested(k);
address narrow_klass_base = _requested_static_archive_bottom; // runtime encoding base == runtime mapping start
const int narrow_klass_shift = precomputed_narrow_klass_shift();
narrowKlass nk = CompressedKlassPointers::encode_not_null_without_asserts(requested_k, narrow_klass_base, narrow_klass_shift);
k->set_prototype_header(markWord::prototype().set_narrow_klass(nk));
}
#endif //_LP64
if (k->is_objArray_klass()) {
// InstanceKlass and TypeArrayKlass will in turn call remove_unshareable_info
// on their array classes.
@ -884,9 +908,15 @@ narrowKlass ArchiveBuilder::get_requested_narrow_klass(Klass* k) {
assert(CDSConfig::is_dumping_heap(), "sanity");
k = get_buffered_klass(k);
Klass* requested_k = to_requested(k);
const int narrow_klass_shift = ArchiveBuilder::precomputed_narrow_klass_shift();
#ifdef ASSERT
const size_t klass_alignment = MAX2(SharedSpaceObjectAlignment, (size_t)nth_bit(narrow_klass_shift));
assert(is_aligned(k, klass_alignment), "Klass " PTR_FORMAT " misaligned.", p2i(k));
#endif
address narrow_klass_base = _requested_static_archive_bottom; // runtime encoding base == runtime mapping start
const int narrow_klass_shift = ArchiveHeapWriter::precomputed_narrow_klass_shift;
return CompressedKlassPointers::encode_not_null(requested_k, narrow_klass_base, narrow_klass_shift);
// Note: use the "raw" version of encode that takes explicit narrow klass base and shift. Don't use any
// of the variants that do sanity checks, nor any of those that use the current - dump - JVM's encoding setting.
return CompressedKlassPointers::encode_not_null_without_asserts(requested_k, narrow_klass_base, narrow_klass_shift);
}
#endif // INCLUDE_CDS_JAVA_HEAP
@ -966,6 +996,20 @@ class RelocateBufferToRequested : public BitMapClosure {
}
};
#ifdef _LP64
int ArchiveBuilder::precomputed_narrow_klass_shift() {
// Legacy Mode:
// We use 32 bits for narrowKlass, which should cover the full 4G Klass range. Shift can be 0.
// CompactObjectHeader Mode:
// narrowKlass is much smaller, and we use the highest possible shift value to later get the maximum
// Klass encoding range.
//
// Note that all of this may change in the future, if we decide to correct the pre-calculated
// narrow Klass IDs at archive load time.
assert(UseCompressedClassPointers, "Only needed for compressed class pointers");
return UseCompactObjectHeaders ? CompressedKlassPointers::max_shift() : 0;
}
#endif // _LP64
void ArchiveBuilder::relocate_to_requested() {
ro_region()->pack();

30
src/hotspot/share/cds/archiveBuilder.hpp
View File

@ -27,6 +27,7 @@
#include "cds/archiveUtils.hpp"
#include "cds/dumpAllocStats.hpp"
#include "memory/metaspace.hpp"
#include "memory/metaspaceClosure.hpp"
#include "oops/array.hpp"
#include "oops/klass.hpp"
@ -43,9 +44,9 @@ class Klass;
class MemRegion;
class Symbol;
// Metaspace::allocate() requires that all blocks must be aligned with KlassAlignmentInBytes.
// We enforce the same alignment rule in blocks allocated from the shared space.
const int SharedSpaceObjectAlignment = KlassAlignmentInBytes;
// The minimum alignment for non-Klass objects inside the CDS archive. Klass objects need
// to follow CompressedKlassPointers::klass_alignment_in_bytes().
constexpr size_t SharedSpaceObjectAlignment = Metaspace::min_allocation_alignment_bytes;
// Overview of CDS archive creation (for both static and dynamic dump):
//
@ -468,6 +469,29 @@ public:
void print_stats();
void report_out_of_space(const char* name, size_t needed_bytes);
#ifdef _LP64
// The CDS archive contains pre-computed narrow Klass IDs. It carries them in the headers of
// archived heap objects. With +UseCompactObjectHeaders, it also carries them in prototypes
// in Klass.
// When generating the archive, these narrow Klass IDs are computed using the following scheme:
// 1) The future encoding base is assumed to point to the first address of the generated mapping.
// That means that at runtime, the narrow Klass encoding must be set up with base pointing to
// the start address of the mapped CDS metadata archive (wherever that may be). This precludes
// zero-based encoding.
// 2) The shift must be large enough to result in an encoding range that covers the future assumed
// runtime Klass range. That future Klass range will contain both the CDS metadata archive and
// the future runtime class space. Since we do not know the size of the future class space, we
// need to chose an encoding base/shift combination that will result in a "large enough" size.
// The details depend on whether we use compact object headers or legacy object headers.
// In Legacy Mode, a narrow Klass ID is 32 bit. This gives us an encoding range size of 4G even
// with shift = 0, which is all we need. Therefore, we use a shift=0 for pre-calculating the
// narrow Klass IDs.
// TinyClassPointer Mode:
// We use the highest possible shift value to maximize the encoding range size.
static int precomputed_narrow_klass_shift();
#endif // _LP64
};
#endif // SHARE_CDS_ARCHIVEBUILDER_HPP

28
src/hotspot/share/cds/archiveHeapWriter.cpp
View File

@ -186,8 +186,12 @@ objArrayOop ArchiveHeapWriter::allocate_root_segment(size_t offset, int element_
memset(mem, 0, objArrayOopDesc::object_size(element_count));
// The initialization code is copied from MemAllocator::finish and ObjArrayAllocator::initialize.
oopDesc::set_mark(mem, markWord::prototype());
oopDesc::release_set_klass(mem, Universe::objectArrayKlass());
if (UseCompactObjectHeaders) {
oopDesc::release_set_mark(mem, Universe::objectArrayKlass()->prototype_header());
} else {
oopDesc::set_mark(mem, markWord::prototype());
oopDesc::release_set_klass(mem, Universe::objectArrayKlass());
}
arrayOopDesc::set_length(mem, element_count);
return objArrayOop(cast_to_oop(mem));
}
@ -350,9 +354,13 @@ HeapWord* ArchiveHeapWriter::init_filler_array_at_buffer_top(int array_length, s
Klass* oak = Universe::objectArrayKlass(); // already relocated to point to archived klass
HeapWord* mem = offset_to_buffered_address<HeapWord*>(_buffer_used);
memset(mem, 0, fill_bytes);
oopDesc::set_mark(mem, markWord::prototype());
narrowKlass nk = ArchiveBuilder::current()->get_requested_narrow_klass(oak);
cast_to_oop(mem)->set_narrow_klass(nk);
if (UseCompactObjectHeaders) {
oopDesc::release_set_mark(mem, markWord::prototype().set_narrow_klass(nk));
} else {
oopDesc::set_mark(mem, markWord::prototype());
cast_to_oop(mem)->set_narrow_klass(nk);
}
arrayOopDesc::set_length(mem, array_length);
return mem;
}
@ -556,7 +564,11 @@ void ArchiveHeapWriter::update_header_for_requested_obj(oop requested_obj, oop s
address buffered_addr = requested_addr_to_buffered_addr(cast_from_oop<address>(requested_obj));
oop fake_oop = cast_to_oop(buffered_addr);
fake_oop->set_narrow_klass(nk);
if (UseCompactObjectHeaders) {
fake_oop->set_mark(markWord::prototype().set_narrow_klass(nk));
} else {
fake_oop->set_narrow_klass(nk);
}
if (src_obj == nullptr) {
return;
@ -565,7 +577,11 @@ void ArchiveHeapWriter::update_header_for_requested_obj(oop requested_obj, oop s
// in the shared heap.
if (!src_obj->fast_no_hash_check()) {
intptr_t src_hash = src_obj->identity_hash();
fake_oop->set_mark(markWord::prototype().copy_set_hash(src_hash));
if (UseCompactObjectHeaders) {
fake_oop->set_mark(markWord::prototype().set_narrow_klass(nk).copy_set_hash(src_hash));
} else {
fake_oop->set_mark(markWord::prototype().copy_set_hash(src_hash));
}
assert(fake_oop->mark().is_unlocked(), "sanity");
DEBUG_ONLY(intptr_t archived_hash = fake_oop->identity_hash());

11
src/hotspot/share/cds/archiveHeapWriter.hpp
View File

@ -240,17 +240,6 @@ public:
static oop buffered_addr_to_source_obj(address buffered_addr);
static address buffered_addr_to_requested_addr(address buffered_addr);
// Archived heap object headers carry pre-computed narrow Klass ids calculated with the
// following scheme:
// 1) the encoding base must be the mapping start address.
// 2) shift must be large enough to result in an encoding range that covers the runtime Klass range.
// That Klass range is defined by CDS archive size and runtime class space size. Luckily, the maximum
// size can be predicted: archive size is assumed to be <1G, class space size capped at 3G, and at
// runtime we put both regions adjacent to each other. Therefore, runtime Klass range size < 4G.
// Since nKlass itself is 32 bit, our encoding range len is 4G, and since we set the base directly
// at mapping start, these 4G are enough. Therefore, we don't need to shift at all (shift=0).
static constexpr int precomputed_narrow_klass_shift = 0;
};
#endif // INCLUDE_CDS_JAVA_HEAP
#endif // SHARE_CDS_ARCHIVEHEAPWRITER_HPP

9
src/hotspot/share/cds/archiveUtils.cpp
View File

@ -241,9 +241,10 @@ void DumpRegion::commit_to(char* newtop) {
which, commit, _vs->actual_committed_size(), _vs->high());
}
char* DumpRegion::allocate(size_t num_bytes) {
char* p = (char*)align_up(_top, (size_t)SharedSpaceObjectAlignment);
char* DumpRegion::allocate(size_t num_bytes, size_t alignment) {
// Always align to at least minimum alignment
alignment = MAX2(SharedSpaceObjectAlignment, alignment);
char* p = (char*)align_up(_top, alignment);
char* newtop = p + align_up(num_bytes, (size_t)SharedSpaceObjectAlignment);
expand_top_to(newtop);
memset(p, 0, newtop - p);
@ -343,7 +344,7 @@ void ReadClosure::do_tag(int tag) {
int old_tag;
old_tag = (int)(intptr_t)nextPtr();
// do_int(&old_tag);
assert(tag == old_tag, "old tag doesn't match");
assert(tag == old_tag, "tag doesn't match (%d, expected %d)", old_tag, tag);
FileMapInfo::assert_mark(tag == old_tag);
}

5
src/hotspot/share/cds/archiveUtils.hpp
View File

@ -158,10 +158,11 @@ private:
public:
DumpRegion(const char* name, uintx max_delta = 0)
: _name(name), _base(nullptr), _top(nullptr), _end(nullptr),
_max_delta(max_delta), _is_packed(false) {}
_max_delta(max_delta), _is_packed(false),
_rs(NULL), _vs(NULL) {}
char* expand_top_to(char* newtop);
char* allocate(size_t num_bytes);
char* allocate(size_t num_bytes, size_t alignment = 0);
void append_intptr_t(intptr_t n, bool need_to_mark = false) NOT_CDS_RETURN;

21
src/hotspot/share/cds/cdsConfig.cpp
View File

@ -82,13 +82,20 @@ char* CDSConfig::default_archive_path() {
os::jvm_path(jvm_path, sizeof(jvm_path));
char *end = strrchr(jvm_path, *os::file_separator());
if (end != nullptr) *end = '\0';
size_t jvm_path_len = strlen(jvm_path);
size_t file_sep_len = strlen(os::file_separator());
const size_t len = jvm_path_len + file_sep_len + 20;
_default_archive_path = NEW_C_HEAP_ARRAY(char, len, mtArguments);
jio_snprintf(_default_archive_path, len,
LP64_ONLY(!UseCompressedOops ? "%s%sclasses_nocoops.jsa":) "%s%sclasses.jsa",
jvm_path, os::file_separator());
stringStream tmp;
tmp.print("%s%sclasses", jvm_path, os::file_separator());
#ifdef _LP64
if (!UseCompressedOops) {
tmp.print_raw("_nocoops");
}
if (UseCompactObjectHeaders) {
// Note that generation of xxx_coh.jsa variants require
// --enable-cds-archive-coh at build time
tmp.print_raw("_coh");
}
#endif
tmp.print_raw(".jsa");
_default_archive_path = os::strdup(tmp.base());
}
return _default_archive_path;
}

12
src/hotspot/share/cds/dumpAllocStats.cpp
View File

@ -115,3 +115,15 @@ void DumpAllocStats::print_stats(int ro_all, int rw_all) {
percent_of(_num_method_cp_entries_archived, _num_method_cp_entries),
_num_method_cp_entries_reverted);
}
#ifdef ASSERT
void DumpAllocStats::verify(int expected_byte_size, bool read_only) const {
int bytes = 0;
const int what = (int)(read_only ? RO : RW);
for (int type = 0; type < int(_number_of_types); type ++) {
bytes += _bytes[what][type];
}
assert(bytes == expected_byte_size, "counter mismatch (%s: %d vs %d)",
(read_only ? "RO" : "RW"), bytes, expected_byte_size);
}
#endif // ASSERT

3
src/hotspot/share/cds/dumpAllocStats.hpp
View File

@ -135,6 +135,9 @@ public:
}
void print_stats(int ro_all, int rw_all);
DEBUG_ONLY(void verify(int expected_byte_size, bool read_only) const);
};
#endif // SHARE_CDS_DUMPALLOCSTATS_HPP

73
src/hotspot/share/cds/filemap.cpp
View File

@ -55,6 +55,7 @@
#include "nmt/memTracker.hpp"
#include "oops/compressedOops.hpp"
#include "oops/compressedOops.inline.hpp"
#include "oops/compressedKlass.hpp"
#include "oops/objArrayOop.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
@ -204,6 +205,7 @@ void FileMapHeader::populate(FileMapInfo *info, size_t core_region_alignment,
_core_region_alignment = core_region_alignment;
_obj_alignment = ObjectAlignmentInBytes;
_compact_strings = CompactStrings;
_compact_headers = UseCompactObjectHeaders;
if (CDSConfig::is_dumping_heap()) {
_narrow_oop_mode = CompressedOops::mode();
_narrow_oop_base = CompressedOops::base();
@ -211,6 +213,14 @@ void FileMapHeader::populate(FileMapInfo *info, size_t core_region_alignment,
}
_compressed_oops = UseCompressedOops;
_compressed_class_ptrs = UseCompressedClassPointers;
if (UseCompressedClassPointers) {
#ifdef _LP64
_narrow_klass_pointer_bits = CompressedKlassPointers::narrow_klass_pointer_bits();
_narrow_klass_shift = ArchiveBuilder::precomputed_narrow_klass_shift();
#endif
} else {
_narrow_klass_pointer_bits = _narrow_klass_shift = -1;
}
_max_heap_size = MaxHeapSize;
_use_optimized_module_handling = CDSConfig::is_using_optimized_module_handling();
_has_full_module_graph = CDSConfig::is_dumping_full_module_graph();
@ -269,10 +279,13 @@ void FileMapHeader::print(outputStream* st) {
st->print_cr("- narrow_oop_base: " INTPTR_FORMAT, p2i(_narrow_oop_base));
st->print_cr("- narrow_oop_shift %d", _narrow_oop_shift);
st->print_cr("- compact_strings: %d", _compact_strings);
st->print_cr("- compact_headers: %d", _compact_headers);
st->print_cr("- max_heap_size: " UINTX_FORMAT, _max_heap_size);
st->print_cr("- narrow_oop_mode: %d", _narrow_oop_mode);
st->print_cr("- compressed_oops: %d", _compressed_oops);
st->print_cr("- compressed_class_ptrs: %d", _compressed_class_ptrs);
st->print_cr("- narrow_klass_pointer_bits: %d", _narrow_klass_pointer_bits);
st->print_cr("- narrow_klass_shift: %d", _narrow_klass_shift);
st->print_cr("- cloned_vtables_offset: " SIZE_FORMAT_X, _cloned_vtables_offset);
st->print_cr("- early_serialized_data_offset: " SIZE_FORMAT_X, _early_serialized_data_offset);
st->print_cr("- serialized_data_offset: " SIZE_FORMAT_X, _serialized_data_offset);
@ -2083,22 +2096,23 @@ bool FileMapInfo::can_use_heap_region() {
}
// We pre-compute narrow Klass IDs with the runtime mapping start intended to be the base, and a shift of
// ArchiveHeapWriter::precomputed_narrow_klass_shift. We enforce this encoding at runtime (see
// ArchiveBuilder::precomputed_narrow_klass_shift. We enforce this encoding at runtime (see
// CompressedKlassPointers::initialize_for_given_encoding()). Therefore, the following assertions must
// hold:
address archive_narrow_klass_base = (address)header()->mapped_base_address();
const int archive_narrow_klass_shift = ArchiveHeapWriter::precomputed_narrow_klass_shift;
const int archive_narrow_klass_pointer_bits = header()->narrow_klass_pointer_bits();
const int archive_narrow_klass_shift = header()->narrow_klass_shift();
log_info(cds)("CDS archive was created with max heap size = " SIZE_FORMAT "M, and the following configuration:",
max_heap_size()/M);
log_info(cds)(" narrow_klass_base at mapping start address, narrow_klass_shift = %d",
archive_narrow_klass_shift);
log_info(cds)(" narrow_klass_base at mapping start address, narrow_klass_pointer_bits = %d, narrow_klass_shift = %d",
archive_narrow_klass_pointer_bits, archive_narrow_klass_shift);
log_info(cds)(" narrow_oop_mode = %d, narrow_oop_base = " PTR_FORMAT ", narrow_oop_shift = %d",
narrow_oop_mode(), p2i(narrow_oop_base()), narrow_oop_shift());
log_info(cds)("The current max heap size = " SIZE_FORMAT "M, G1HeapRegion::GrainBytes = " SIZE_FORMAT,
MaxHeapSize/M, G1HeapRegion::GrainBytes);
log_info(cds)(" narrow_klass_base = " PTR_FORMAT ", narrow_klass_shift = %d",
p2i(CompressedKlassPointers::base()), CompressedKlassPointers::shift());
log_info(cds)(" narrow_klass_base = " PTR_FORMAT ", arrow_klass_pointer_bits = %d, narrow_klass_shift = %d",
p2i(CompressedKlassPointers::base()), CompressedKlassPointers::narrow_klass_pointer_bits(), CompressedKlassPointers::shift());
log_info(cds)(" narrow_oop_mode = %d, narrow_oop_base = " PTR_FORMAT ", narrow_oop_shift = %d",
CompressedOops::mode(), p2i(CompressedOops::base()), CompressedOops::shift());
log_info(cds)(" heap range = [" PTR_FORMAT " - " PTR_FORMAT "]",
@ -2107,10 +2121,35 @@ bool FileMapInfo::can_use_heap_region() {
UseCompressedOops ? p2i(CompressedOops::end()) :
UseG1GC ? p2i((address)G1CollectedHeap::heap()->reserved().end()) : 0L);
assert(archive_narrow_klass_base == CompressedKlassPointers::base(), "Unexpected encoding base encountered "
"(" PTR_FORMAT ", expected " PTR_FORMAT ")", p2i(CompressedKlassPointers::base()), p2i(archive_narrow_klass_base));
assert(archive_narrow_klass_shift == CompressedKlassPointers::shift(), "Unexpected encoding shift encountered "
"(%d, expected %d)", CompressedKlassPointers::shift(), archive_narrow_klass_shift);
int err = 0;
if ( archive_narrow_klass_base != CompressedKlassPointers::base() ||
(err = 1, archive_narrow_klass_pointer_bits != CompressedKlassPointers::narrow_klass_pointer_bits()) ||
(err = 2, archive_narrow_klass_shift != CompressedKlassPointers::shift()) ) {
stringStream ss;
switch (err) {
case 0:
ss.print("Unexpected encoding base encountered (" PTR_FORMAT ", expected " PTR_FORMAT ")",
p2i(CompressedKlassPointers::base()), p2i(archive_narrow_klass_base));
break;
case 1:
ss.print("Unexpected narrow Klass bit length encountered (%d, expected %d)",
CompressedKlassPointers::narrow_klass_pointer_bits(), archive_narrow_klass_pointer_bits);
break;
case 2:
ss.print("Unexpected narrow Klass shift encountered (%d, expected %d)",
CompressedKlassPointers::shift(), archive_narrow_klass_shift);
break;
default:
ShouldNotReachHere();
};
LogTarget(Info, cds) lt;
if (lt.is_enabled()) {
LogStream ls(lt);
ls.print_raw(ss.base());
header()->print(&ls);
}
assert(false, "%s", ss.base());
}
return true;
}
@ -2482,14 +2521,22 @@ bool FileMapHeader::validate() {
"for testing purposes only and should not be used in a production environment");
}
log_info(cds)("Archive was created with UseCompressedOops = %d, UseCompressedClassPointers = %d",
compressed_oops(), compressed_class_pointers());
log_info(cds)("Archive was created with UseCompressedOops = %d, UseCompressedClassPointers = %d, UseCompactObjectHeaders = %d",
compressed_oops(), compressed_class_pointers(), compact_headers());
if (compressed_oops() != UseCompressedOops || compressed_class_pointers() != UseCompressedClassPointers) {
log_info(cds)("Unable to use shared archive.\nThe saved state of UseCompressedOops and UseCompressedClassPointers is "
log_warning(cds)("Unable to use shared archive.\nThe saved state of UseCompressedOops and UseCompressedClassPointers is "
"different from runtime, CDS will be disabled.");
return false;
}
if (compact_headers() != UseCompactObjectHeaders) {
log_warning(cds)("Unable to use shared archive.\nThe shared archive file's UseCompactObjectHeaders setting (%s)"
" does not equal the current UseCompactObjectHeaders setting (%s).",
_compact_headers ? "enabled" : "disabled",
UseCompactObjectHeaders ? "enabled" : "disabled");
return false;
}
if (!_use_optimized_module_handling) {
CDSConfig::stop_using_optimized_module_handling();
log_info(cds)("optimized module handling: disabled because archive was created without optimized module handling");

6
src/hotspot/share/cds/filemap.hpp
View File

@ -188,10 +188,13 @@ private:
address _narrow_oop_base; // compressed oop encoding base
int _narrow_oop_shift; // compressed oop encoding shift
bool _compact_strings; // value of CompactStrings
bool _compact_headers; // value of UseCompactObjectHeaders
uintx _max_heap_size; // java max heap size during dumping
CompressedOops::Mode _narrow_oop_mode; // compressed oop encoding mode
bool _compressed_oops; // save the flag UseCompressedOops
bool _compressed_class_ptrs; // save the flag UseCompressedClassPointers
int _narrow_klass_pointer_bits; // save number of bits in narrowKlass
int _narrow_klass_shift; // save shift width used to pre-compute narrowKlass IDs in archived heap objects
size_t _cloned_vtables_offset; // The address of the first cloned vtable
size_t _early_serialized_data_offset; // Data accessed using {ReadClosure,WriteClosure}::serialize()
size_t _serialized_data_offset; // Data accessed using {ReadClosure,WriteClosure}::serialize()
@ -259,6 +262,7 @@ public:
address narrow_oop_base() const { return _narrow_oop_base; }
int narrow_oop_shift() const { return _narrow_oop_shift; }
bool compact_strings() const { return _compact_strings; }
bool compact_headers() const { return _compact_headers; }
uintx max_heap_size() const { return _max_heap_size; }
CompressedOops::Mode narrow_oop_mode() const { return _narrow_oop_mode; }
char* cloned_vtables() const { return from_mapped_offset(_cloned_vtables_offset); }
@ -271,6 +275,8 @@ public:
bool has_non_jar_in_classpath() const { return _has_non_jar_in_classpath; }
bool compressed_oops() const { return _compressed_oops; }
bool compressed_class_pointers() const { return _compressed_class_ptrs; }
int narrow_klass_pointer_bits() const { return _narrow_klass_pointer_bits; }
int narrow_klass_shift() const { return _narrow_klass_shift; }
HeapRootSegments heap_root_segments() const { return _heap_root_segments; }
bool has_full_module_graph() const { return _has_full_module_graph; }
size_t heap_oopmap_start_pos() const { return _heap_oopmap_start_pos; }

35
src/hotspot/share/cds/metaspaceShared.cpp
View File

@ -87,6 +87,7 @@
#include "utilities/align.hpp"
#include "utilities/bitMap.inline.hpp"
#include "utilities/defaultStream.hpp"
#include "utilities/macros.hpp"
#include "utilities/ostream.hpp"
#include "utilities/resourceHash.hpp"
@ -1247,19 +1248,25 @@ MapArchiveResult MetaspaceShared::map_archives(FileMapInfo* static_mapinfo, File
address cds_base = (address)static_mapinfo->mapped_base();
address ccs_end = (address)class_space_rs.end();
assert(ccs_end > cds_base, "Sanity check");
#if INCLUDE_CDS_JAVA_HEAP
// We archived objects with pre-computed narrow Klass id. Set up encoding such that these Ids stay valid.
address precomputed_narrow_klass_base = cds_base;
const int precomputed_narrow_klass_shift = ArchiveHeapWriter::precomputed_narrow_klass_shift;
CompressedKlassPointers::initialize_for_given_encoding(
cds_base, ccs_end - cds_base, // Klass range
precomputed_narrow_klass_base, precomputed_narrow_klass_shift // precomputed encoding, see ArchiveHeapWriter
if (INCLUDE_CDS_JAVA_HEAP || UseCompactObjectHeaders) {
// The CDS archive may contain narrow Klass IDs that were precomputed at archive generation time:
// - every archived java object header (only if INCLUDE_CDS_JAVA_HEAP)
// - every archived Klass' prototype (only if +UseCompactObjectHeaders)
//
// In order for those IDs to still be valid, we need to dictate base and shift: base should be the
// mapping start, shift the shift used at archive generation time.
address precomputed_narrow_klass_base = cds_base;
const int precomputed_narrow_klass_shift = ArchiveBuilder::precomputed_narrow_klass_shift();
CompressedKlassPointers::initialize_for_given_encoding(
cds_base, ccs_end - cds_base, // Klass range
precomputed_narrow_klass_base, precomputed_narrow_klass_shift // precomputed encoding, see ArchiveBuilder
);
#else
CompressedKlassPointers::initialize (
cds_base, ccs_end - cds_base // Klass range
);
#endif // INCLUDE_CDS_JAVA_HEAP
} else {
// Let JVM freely chose encoding base and shift
CompressedKlassPointers::initialize (
cds_base, ccs_end - cds_base // Klass range
);
}
// map_or_load_heap_region() compares the current narrow oop and klass encodings
// with the archived ones, so it must be done after all encodings are determined.
static_mapinfo->map_or_load_heap_region();
@ -1314,7 +1321,7 @@ MapArchiveResult MetaspaceShared::map_archives(FileMapInfo* static_mapinfo, File
//
// If UseCompressedClassPointers=1, the range encompassing both spaces will be
// suitable to en/decode narrow Klass pointers: the base will be valid for
// encoding, the range [Base, End) not surpass KlassEncodingMetaspaceMax.
// encoding, the range [Base, End) and not surpass the max. range for that encoding.
//
// Return:
//
@ -1435,7 +1442,7 @@ char* MetaspaceShared::reserve_address_space_for_archives(FileMapInfo* static_ma
} else {
// We did not manage to reserve at the preferred address, or were instructed to relocate. In that
// case we reserve wherever possible, but the start address needs to be encodable as narrow Klass
// encoding base since the archived heap objects contain nKlass IDs pre-calculated toward the start
// encoding base since the archived heap objects contain narrow Klass IDs pre-calculated toward the start
// of the shared Metaspace. That prevents us from using zero-based encoding and therefore we won't
// try allocating in low-address regions.
total_space_rs = Metaspace::reserve_address_space_for_compressed_classes(total_range_size, false /* optimize_for_zero_base */);

20
src/hotspot/share/ci/ciKlass.cpp
View File

@ -258,3 +258,23 @@ const char* ciKlass::external_name() const {
return get_Klass()->external_name();
)
}
// ------------------------------------------------------------------
// ciKlass::prototype_header_offset
juint ciKlass::prototype_header_offset() {
assert(is_loaded(), "must be loaded");
VM_ENTRY_MARK;
Klass* this_klass = get_Klass();
return in_bytes(this_klass->prototype_header_offset());
}
// ------------------------------------------------------------------
// ciKlass::prototype_header
uintptr_t ciKlass::prototype_header() {
assert(is_loaded(), "must be loaded");
VM_ENTRY_MARK;
Klass* this_klass = get_Klass();
return (uintptr_t)this_klass->prototype_header().to_pointer();
}

3
src/hotspot/share/ci/ciKlass.hpp
View File

@ -139,6 +139,9 @@ public:
void print_name_on(outputStream* st);
const char* external_name() const;
juint prototype_header_offset();
uintptr_t prototype_header();
};
#endif // SHARE_CI_CIKLASS_HPP

9
src/hotspot/share/classfile/classFileParser.cpp
View File

@ -5836,6 +5836,15 @@ bool ClassFileParser::is_java_lang_ref_Reference_subclass() const {
return _super_klass->reference_type() != REF_NONE;
}
// Returns true if the future Klass will need to be addressable with a narrow Klass ID.
bool ClassFileParser::klass_needs_narrow_id() const {
// Classes that are never instantiated need no narrow Klass Id, since the
// only point of having a narrow id is to put it into an object header. Keeping
// never instantiated classes out of class space lessens the class space pressure.
// For more details, see JDK-8338526.
return !is_interface() && !is_abstract();
}
// ----------------------------------------------------------------------------
// debugging

4
src/hotspot/share/classfile/classFileParser.hpp
View File

@ -511,6 +511,10 @@ class ClassFileParser {
bool is_interface() const { return _access_flags.is_interface(); }
bool is_abstract() const { return _access_flags.is_abstract(); }
// Returns true if the Klass to be generated will need to be addressable
// with a narrow Klass ID.
bool klass_needs_narrow_id() const;
ClassLoaderData* loader_data() const { return _loader_data; }
const Symbol* class_name() const { return _class_name; }
const InstanceKlass* super_klass() const { return _super_klass; }

16
src/hotspot/share/classfile/systemDictionaryShared.cpp
View File

@ -59,6 +59,7 @@
#include "memory/oopFactory.hpp"
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
#include "oops/compressedKlass.inline.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klass.inline.hpp"
#include "oops/objArrayKlass.hpp"
@ -83,6 +84,16 @@ DumpTimeLambdaProxyClassDictionary* SystemDictionaryShared::_dumptime_lambda_pro
// Used by NoClassLoadingMark
DEBUG_ONLY(bool SystemDictionaryShared::_class_loading_may_happen = true;)
#ifdef ASSERT
static void check_klass_after_loading(const Klass* k) {
#ifdef _LP64
if (k != nullptr && UseCompressedClassPointers && k->needs_narrow_id()) {
CompressedKlassPointers::check_encodable(k);
}
#endif
}
#endif
InstanceKlass* SystemDictionaryShared::load_shared_class_for_builtin_loader(
Symbol* class_name, Handle class_loader, TRAPS) {
assert(CDSConfig::is_using_archive(), "must be");
@ -425,6 +436,9 @@ InstanceKlass* SystemDictionaryShared::find_or_load_shared_class(
}
}
}
DEBUG_ONLY(check_klass_after_loading(k);)
return k;
}
@ -1345,7 +1359,7 @@ InstanceKlass* SystemDictionaryShared::find_builtin_class(Symbol* name) {
name);
if (record != nullptr) {
assert(!record->klass()->is_hidden(), "hidden class cannot be looked up by name");
assert(check_alignment(record->klass()), "Address not aligned");
DEBUG_ONLY(check_klass_after_loading(record->klass());)
// We did not save the classfile data of the generated LambdaForm invoker classes,
// so we cannot support CLFH for such classes.
if (record->klass()->is_generated_shared_class() && JvmtiExport::should_post_class_file_load_hook()) {

2
src/hotspot/share/gc/g1/g1Arguments.cpp
View File

@ -34,6 +34,7 @@
#include "gc/g1/g1HeapRegionRemSet.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
#include "gc/shared/cardTable.hpp"
#include "gc/shared/fullGCForwarding.hpp"
#include "gc/shared/gcArguments.hpp"
#include "gc/shared/workerPolicy.hpp"
#include "runtime/globals.hpp"
@ -247,6 +248,7 @@ void G1Arguments::initialize() {
void G1Arguments::initialize_heap_flags_and_sizes() {
GCArguments::initialize_heap_flags_and_sizes();
FullGCForwarding::initialize_flags(heap_reserved_size_bytes());
}
CollectedHeap* G1Arguments::create_heap() {

4
src/hotspot/share/gc/g1/g1CollectedHeap.cpp
View File

@ -77,6 +77,7 @@
#include "gc/g1/g1YoungGCAllocationFailureInjector.hpp"
#include "gc/shared/classUnloadingContext.hpp"
#include "gc/shared/concurrentGCBreakpoints.hpp"
#include "gc/shared/fullGCForwarding.hpp"
#include "gc/shared/gcBehaviours.hpp"
#include "gc/shared/gcHeapSummary.hpp"
#include "gc/shared/gcId.hpp"
@ -85,7 +86,6 @@
#include "gc/shared/isGCActiveMark.hpp"
#include "gc/shared/locationPrinter.inline.hpp"
#include "gc/shared/oopStorageParState.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/referenceProcessor.inline.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
#include "gc/shared/taskqueue.inline.hpp"
@ -1435,6 +1435,8 @@ jint G1CollectedHeap::initialize() {
G1InitLogger::print();
FullGCForwarding::initialize(heap_rs.region());
return JNI_OK;
}

13
src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -29,6 +29,7 @@
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCCompactTask.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/fullGCForwarding.inline.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "logging/log.hpp"
#include "oops/oop.inline.hpp"
@ -41,7 +42,7 @@ void G1FullGCCompactTask::G1CompactRegionClosure::clear_in_bitmap(oop obj) {
size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
size_t size = obj->size();
if (obj->is_forwarded()) {
if (FullGCForwarding::is_forwarded(obj)) {
G1FullGCCompactTask::copy_object_to_new_location(obj);
}
@ -52,13 +53,13 @@ size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
}
void G1FullGCCompactTask::copy_object_to_new_location(oop obj) {
assert(obj->is_forwarded(), "Sanity!");
assert(obj->forwardee() != obj, "Object must have a new location");
assert(FullGCForwarding::is_forwarded(obj), "Sanity!");
assert(FullGCForwarding::forwardee(obj) != obj, "Object must have a new location");
size_t size = obj->size();
// Copy object and reinit its mark.
HeapWord* obj_addr = cast_from_oop<HeapWord*>(obj);
HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
HeapWord* destination = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(obj));
Copy::aligned_conjoint_words(obj_addr, destination, size);
// There is no need to transform stack chunks - marking already did that.
@ -121,7 +122,7 @@ void G1FullGCCompactTask::compact_humongous_obj(G1HeapRegion* src_hr) {
size_t word_size = obj->size();
uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(word_size);
HeapWord* destination = cast_from_oop<HeapWord*>(obj->forwardee());
HeapWord* destination = cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(obj));
assert(collector()->mark_bitmap()->is_marked(obj), "Should only compact marked objects");
collector()->mark_bitmap()->clear(obj);

13
src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -26,6 +26,7 @@
#include "gc/g1/g1FullCollector.inline.hpp"
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1HeapRegion.hpp"
#include "gc/shared/fullGCForwarding.inline.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
#include "oops/oop.inline.hpp"
#include "utilities/debug.hpp"
@ -106,10 +107,10 @@ void G1FullGCCompactionPoint::forward(oop object, size_t size) {
if (!object->is_forwarded()) {
preserved_stack()->push_if_necessary(object, object->mark());
}
object->forward_to(cast_to_oop(_compaction_top));
assert(object->is_forwarded(), "must be forwarded");
FullGCForwarding::forward_to(object, cast_to_oop(_compaction_top));
assert(FullGCForwarding::is_forwarded(object), "must be forwarded");
} else {
assert(!object->is_forwarded(), "must not be forwarded");
assert(!FullGCForwarding::is_forwarded(object), "must not be forwarded");
}
// Update compaction values.
@ -172,8 +173,8 @@ void G1FullGCCompactionPoint::forward_humongous(G1HeapRegion* hr) {
preserved_stack()->push_if_necessary(obj, obj->mark());
G1HeapRegion* dest_hr = _compaction_regions->at(range_begin);
obj->forward_to(cast_to_oop(dest_hr->bottom()));
assert(obj->is_forwarded(), "Object must be forwarded!");
FullGCForwarding::forward_to(obj, cast_to_oop(dest_hr->bottom()));
assert(FullGCForwarding::is_forwarded(obj), "Object must be forwarded!");
// Add the humongous object regions to the compaction point.
add_humongous(hr);

7
src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -32,6 +32,7 @@
#include "gc/g1/g1FullCollector.inline.hpp"
#include "gc/g1/g1FullGCMarker.inline.hpp"
#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/shared/fullGCForwarding.inline.hpp"
#include "memory/iterator.inline.hpp"
#include "memory/universe.hpp"
#include "oops/access.inline.hpp"
@ -65,8 +66,8 @@ template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
return;
}
if (obj->is_forwarded()) {
oop forwardee = obj->forwardee();
if (FullGCForwarding::is_forwarded(obj)) {
oop forwardee = FullGCForwarding::forwardee(obj);
// Forwarded, just update.
assert(G1CollectedHeap::heap()->is_in_reserved(forwardee), "should be in object space");
RawAccess<IS_NOT_NULL>::oop_store(p, forwardee);

7
src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -32,6 +32,7 @@
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCScope.hpp"
#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/fullGCForwarding.inline.hpp"
void G1DetermineCompactionQueueClosure::free_empty_humongous_region(G1HeapRegion* hr) {
_g1h->free_humongous_region(hr, nullptr);
@ -114,10 +115,10 @@ inline bool G1DetermineCompactionQueueClosure::do_heap_region(G1HeapRegion* hr)
}
inline size_t G1SerialRePrepareClosure::apply(oop obj) {
if (obj->is_forwarded()) {
if (FullGCForwarding::is_forwarded(obj)) {
// We skip objects compiled into the first region or
// into regions not part of the serial compaction point.
if (cast_from_oop<HeapWord*>(obj->forwardee()) < _dense_prefix_top) {
if (cast_from_oop<HeapWord*>(FullGCForwarding::forwardee(obj)) < _dense_prefix_top) {
return obj->size();
}
}

4
src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -105,7 +105,6 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
_gc_par_phases[UpdateDerivedPointers] = new WorkerDataArray<double>("UpdateDerivedPointers", "Update Derived Pointers (ms):", max_gc_threads);
#endif
_gc_par_phases[EagerlyReclaimHumongousObjects] = new WorkerDataArray<double>("EagerlyReclaimHumongousObjects", "Eagerly Reclaim Humongous Objects (ms):", max_gc_threads);
_gc_par_phases[RestorePreservedMarks] = new WorkerDataArray<double>("RestorePreservedMarks", "Restore Preserved Marks (ms):", max_gc_threads);
_gc_par_phases[ProcessEvacuationFailedRegions] = new WorkerDataArray<double>("ProcessEvacuationFailedRegions", "Process Evacuation Failed Regions (ms):", max_gc_threads);
_gc_par_phases[ScanHR]->create_thread_work_items("Scanned Cards:", ScanHRScannedCards);
@ -512,7 +511,6 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set(bool evacuation_failed
debug_time("Post Evacuate Cleanup 2", _cur_post_evacuate_cleanup_2_time_ms);
if (evacuation_failed) {
debug_phase(_gc_par_phases[RecalculateUsed], 1);
debug_phase(_gc_par_phases[RestorePreservedMarks], 1);
debug_phase(_gc_par_phases[ProcessEvacuationFailedRegions], 1);
}
#if COMPILER2_OR_JVMCI

3
src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -87,7 +87,6 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
UpdateDerivedPointers,
#endif
EagerlyReclaimHumongousObjects,
RestorePreservedMarks,
ProcessEvacuationFailedRegions,
ResetMarkingState,
NoteStartOfMark,

2
src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
View File

@ -228,7 +228,7 @@ void G1ParCopyClosure<barrier, should_mark>::do_oop_work(T* p) {
oop forwardee;
markWord m = obj->mark();
if (m.is_forwarded()) {
forwardee = m.forwardee();
forwardee = obj->forwardee(m);
} else {
forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m);
}

41
src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
View File

@ -37,7 +37,6 @@
#include "gc/shared/continuationGCSupport.inline.hpp"
#include "gc/shared/partialArrayState.hpp"
#include "gc/shared/partialArrayTaskStepper.inline.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/stringdedup/stringDedup.hpp"
#include "gc/shared/taskqueue.inline.hpp"
#include "memory/allocation.inline.hpp"
@ -59,7 +58,6 @@
G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
G1RedirtyCardsQueueSet* rdcqs,
PreservedMarks* preserved_marks,
uint worker_id,
uint num_workers,
G1CollectionSet* collection_set,
@ -90,7 +88,6 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
_numa(g1h->numa()),
_obj_alloc_stat(nullptr),
ALLOCATION_FAILURE_INJECTOR_ONLY(_allocation_failure_inject_counter(0) COMMA)
_preserved_marks(preserved_marks),
_evacuation_failed_info(),
_evac_failure_regions(evac_failure_regions),
_evac_failure_enqueued_cards(0)
@ -216,7 +213,7 @@ void G1ParScanThreadState::do_oop_evac(T* p) {
markWord m = obj->mark();
if (m.is_forwarded()) {
obj = m.forwardee();
obj = obj->forwardee(m);
} else {
obj = do_copy_to_survivor_space(region_attr, obj, m);
}
@ -232,7 +229,6 @@ void G1ParScanThreadState::do_partial_array(PartialArrayState* state) {
#ifdef ASSERT
oop from_obj = state->source();
assert(_g1h->is_in_reserved(from_obj), "must be in heap.");
assert(from_obj->is_objArray(), "must be obj array");
assert(from_obj->is_forwarded(), "must be forwarded");
assert(from_obj != to_obj, "should not be chunking self-forwarded objects");
assert(to_obj->is_objArray(), "must be obj array");
@ -265,7 +261,6 @@ MAYBE_INLINE_EVACUATION
void G1ParScanThreadState::start_partial_objarray(G1HeapRegionAttr dest_attr,
oop from_obj,
oop to_obj) {
assert(from_obj->is_objArray(), "precondition");
assert(from_obj->is_forwarded(), "precondition");
assert(from_obj->forwardee() == to_obj, "precondition");
assert(to_obj->is_objArray(), "precondition");
@ -401,22 +396,22 @@ G1HeapRegionAttr G1ParScanThreadState::next_region_attr(G1HeapRegionAttr const r
}
void G1ParScanThreadState::report_promotion_event(G1HeapRegionAttr const dest_attr,
oop const old, size_t word_sz, uint age,
Klass* klass, size_t word_sz, uint age,
HeapWord * const obj_ptr, uint node_index) const {
PLAB* alloc_buf = _plab_allocator->alloc_buffer(dest_attr, node_index);
if (alloc_buf->contains(obj_ptr)) {
_g1h->gc_tracer_stw()->report_promotion_in_new_plab_event(old->klass(), word_sz * HeapWordSize, age,
_g1h->gc_tracer_stw()->report_promotion_in_new_plab_event(klass, word_sz * HeapWordSize, age,
dest_attr.type() == G1HeapRegionAttr::Old,
alloc_buf->word_sz() * HeapWordSize);
} else {
_g1h->gc_tracer_stw()->report_promotion_outside_plab_event(old->klass(), word_sz * HeapWordSize, age,
_g1h->gc_tracer_stw()->report_promotion_outside_plab_event(klass, word_sz * HeapWordSize, age,
dest_attr.type() == G1HeapRegionAttr::Old);
}
}
NOINLINE
HeapWord* G1ParScanThreadState::allocate_copy_slow(G1HeapRegionAttr* dest_attr,
oop old,
Klass* klass,
size_t word_sz,
uint age,
uint node_index) {
@ -439,7 +434,7 @@ HeapWord* G1ParScanThreadState::allocate_copy_slow(G1HeapRegionAttr* dest_attr,
update_numa_stats(node_index);
if (_g1h->gc_tracer_stw()->should_report_promotion_events()) {
// The events are checked individually as part of the actual commit
report_promotion_event(*dest_attr, old, word_sz, age, obj_ptr, node_index);
report_promotion_event(*dest_attr, klass, word_sz, age, obj_ptr, node_index);
}
}
return obj_ptr;
@ -474,9 +469,17 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
assert(region_attr.is_in_cset(),
"Unexpected region attr type: %s", region_attr.get_type_str());
// Get the klass once. We'll need it again later, and this avoids
// re-decoding when it's compressed.
Klass* klass = old->klass();
// NOTE: With compact headers, it is not safe to load the Klass* from old, because
// that would access the mark-word, that might change at any time by concurrent
// workers.
// This mark word would refer to a forwardee, which may not yet have completed
// copying. Therefore we must load the Klass* from the mark-word that we already
// loaded. This is safe, because we only enter here if not yet forwarded.
assert(!old_mark.is_forwarded(), "precondition");
Klass* klass = UseCompactObjectHeaders
? old_mark.klass()
: old->klass();
const size_t word_sz = old->size_given_klass(klass);
// JNI only allows pinning of typeArrays, so we only need to keep those in place.
@ -494,7 +497,7 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
// PLAB allocations should succeed most of the time, so we'll
// normally check against null once and that's it.
if (obj_ptr == nullptr) {
obj_ptr = allocate_copy_slow(&dest_attr, old, word_sz, age, node_index);
obj_ptr = allocate_copy_slow(&dest_attr, klass, word_sz, age, node_index);
if (obj_ptr == nullptr) {
// This will either forward-to-self, or detect that someone else has
// installed a forwarding pointer.
@ -595,7 +598,6 @@ G1ParScanThreadState* G1ParScanThreadStateSet::state_for_worker(uint worker_id)
if (_states[worker_id] == nullptr) {
_states[worker_id] =
new G1ParScanThreadState(_g1h, rdcqs(),
_preserved_marks_set.get(worker_id),
worker_id,
_num_workers,
_collection_set,
@ -655,7 +657,7 @@ NOINLINE
oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, size_t word_sz, bool cause_pinned) {
assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old));
oop forward_ptr = old->forward_to_atomic(old, m, memory_order_relaxed);
oop forward_ptr = old->forward_to_self_atomic(m, memory_order_relaxed);
if (forward_ptr == nullptr) {
// Forward-to-self succeeded. We are the "owner" of the object.
G1HeapRegion* r = _g1h->heap_region_containing(old);
@ -668,8 +670,6 @@ oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, siz
// evacuation failure recovery.
_g1h->mark_evac_failure_object(_worker_id, old, word_sz);
_preserved_marks->push_if_necessary(old, m);
ContinuationGCSupport::transform_stack_chunk(old);
_evacuation_failed_info.register_copy_failure(word_sz);
@ -727,7 +727,6 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
_g1h(g1h),
_collection_set(collection_set),
_rdcqs(G1BarrierSet::dirty_card_queue_set().allocator()),
_preserved_marks_set(true /* in_c_heap */),
_states(NEW_C_HEAP_ARRAY(G1ParScanThreadState*, num_workers, mtGC)),
_rdc_buffers(NEW_C_HEAP_ARRAY(BufferNodeList, num_workers, mtGC)),
_surviving_young_words_total(NEW_C_HEAP_ARRAY(size_t, collection_set->young_region_length() + 1, mtGC)),
@ -736,7 +735,6 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
_evac_failure_regions(evac_failure_regions),
_partial_array_state_allocator(num_workers)
{
_preserved_marks_set.init(num_workers);
for (uint i = 0; i < num_workers; ++i) {
_states[i] = nullptr;
_rdc_buffers[i] = BufferNodeList();
@ -749,5 +747,4 @@ G1ParScanThreadStateSet::~G1ParScanThreadStateSet() {
FREE_C_HEAP_ARRAY(G1ParScanThreadState*, _states);
FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_total);
FREE_C_HEAP_ARRAY(BufferNodeList, _rdc_buffers);
_preserved_marks_set.reclaim();
}

11
src/hotspot/share/gc/g1/g1ParScanThreadState.hpp
View File

@ -34,7 +34,6 @@
#include "gc/shared/gc_globals.hpp"
#include "gc/shared/partialArrayState.hpp"
#include "gc/shared/partialArrayTaskStepper.hpp"
#include "gc/shared/preservedMarks.hpp"
#include "gc/shared/stringdedup/stringDedup.hpp"
#include "gc/shared/taskqueue.hpp"
#include "memory/allocation.hpp"
@ -48,8 +47,6 @@ class G1EvacuationRootClosures;
class G1OopStarChunkedList;
class G1PLABAllocator;
class G1HeapRegion;
class PreservedMarks;
class PreservedMarksSet;
class outputStream;
class G1ParScanThreadState : public CHeapObj<mtGC> {
@ -106,7 +103,6 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
// Per-thread evacuation failure data structures.
ALLOCATION_FAILURE_INJECTOR_ONLY(size_t _allocation_failure_inject_counter;)
PreservedMarks* _preserved_marks;
EvacuationFailedInfo _evacuation_failed_info;
G1EvacFailureRegions* _evac_failure_regions;
// Number of additional cards into evacuation failed regions enqueued into
@ -125,7 +121,6 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
public:
G1ParScanThreadState(G1CollectedHeap* g1h,
G1RedirtyCardsQueueSet* rdcqs,
PreservedMarks* preserved_marks,
uint worker_id,
uint num_workers,
G1CollectionSet* collection_set,
@ -174,7 +169,7 @@ private:
void start_partial_objarray(G1HeapRegionAttr dest_dir, oop from, oop to);
HeapWord* allocate_copy_slow(G1HeapRegionAttr* dest_attr,
oop old,
Klass* klass,
size_t word_sz,
uint age,
uint node_index);
@ -209,7 +204,7 @@ private:
inline G1HeapRegionAttr next_region_attr(G1HeapRegionAttr const region_attr, markWord const m, uint& age);
void report_promotion_event(G1HeapRegionAttr const dest_attr,
oop const old, size_t word_sz, uint age,
Klass* klass, size_t word_sz, uint age,
HeapWord * const obj_ptr, uint node_index) const;
void trim_queue_to_threshold(uint threshold);
@ -246,7 +241,6 @@ class G1ParScanThreadStateSet : public StackObj {
G1CollectedHeap* _g1h;
G1CollectionSet* _collection_set;
G1RedirtyCardsQueueSet _rdcqs;
PreservedMarksSet _preserved_marks_set;
G1ParScanThreadState** _states;
BufferNodeList* _rdc_buffers;
size_t* _surviving_young_words_total;
@ -264,7 +258,6 @@ class G1ParScanThreadStateSet : public StackObj {
G1RedirtyCardsQueueSet* rdcqs() { return &_rdcqs; }
BufferNodeList* rdc_buffers() { return _rdc_buffers; }
PreservedMarksSet* preserved_marks_set() { return &_preserved_marks_set; }
void flush_stats();
void record_unused_optional_region(G1HeapRegion* hr);

1
src/hotspot/share/gc/g1/g1YoungCollector.cpp
View File

@ -53,7 +53,6 @@
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/gc_globals.hpp"
#include "gc/shared/preservedMarks.hpp"
#include "gc/shared/referenceProcessor.hpp"
#include "gc/shared/weakProcessor.inline.hpp"
#include "gc/shared/workerPolicy.hpp"

27
src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
View File

@ -42,7 +42,6 @@
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
#include "gc/shared/bufferNode.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
#include "jfr/jfrEvents.hpp"
#include "oops/access.inline.hpp"
#include "oops/compressedOops.inline.hpp"
@ -251,8 +250,8 @@ class G1PostEvacuateCollectionSetCleanupTask1::RestoreEvacFailureRegionsTask : p
{
// Process marked object.
assert(obj->is_forwarded() && obj->forwardee() == obj, "must be self-forwarded");
obj->init_mark();
assert(obj->is_self_forwarded(), "must be self-forwarded");
obj->unset_self_forwarded();
hr->update_bot_for_block(obj_addr, obj_end_addr);
// Statistics
@ -477,27 +476,6 @@ public:
}
};
class G1PostEvacuateCollectionSetCleanupTask2::RestorePreservedMarksTask : public G1AbstractSubTask {
PreservedMarksSet* _preserved_marks;
WorkerTask* _task;
public:
RestorePreservedMarksTask(PreservedMarksSet* preserved_marks) :
G1AbstractSubTask(G1GCPhaseTimes::RestorePreservedMarks),
_preserved_marks(preserved_marks),
_task(preserved_marks->create_task()) { }
virtual ~RestorePreservedMarksTask() {
delete _task;
}
double worker_cost() const override {
return _preserved_marks->num();
}
void do_work(uint worker_id) override { _task->work(worker_id); }
};
class RedirtyLoggedCardTableEntryClosure : public G1CardTableEntryClosure {
size_t _num_dirtied;
G1CollectedHeap* _g1h;
@ -979,7 +957,6 @@ G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2
}
if (evac_failure_regions->has_regions_evac_failed()) {
add_parallel_task(new RestorePreservedMarksTask(per_thread_states->preserved_marks_set()));
add_parallel_task(new ProcessEvacuationFailedRegionsTask(evac_failure_regions));
}
add_parallel_task(new RedirtyLoggedCardsTask(evac_failure_regions,

4
src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -56,7 +56,6 @@ public:
// - Update Derived Pointers (s)
// - Clear Retained Region Data (on evacuation failure)
// - Redirty Logged Cards
// - Restore Preserved Marks (on evacuation failure)
// - Free Collection Set
// - Resize TLABs
class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedTask {
@ -67,7 +66,6 @@ class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedTask {
class ProcessEvacuationFailedRegionsTask;
class RedirtyLoggedCardsTask;
class RestorePreservedMarksTask;
class FreeCollectionSetTask;
class ResizeTLABsTask;

15
src/hotspot/share/gc/parallel/mutableSpace.cpp
View File

@ -218,15 +218,16 @@ void MutableSpace::object_iterate(ObjectClosure* cl) {
// When promotion-failure occurs during Young GC, eden/from space is not cleared,
// so we can encounter objects with "forwarded" markword.
// They are essentially dead, so skipping them
if (!obj->is_forwarded()) {
if (obj->is_forwarded()) {
assert(!obj->is_self_forwarded(), "must not be self-forwarded");
// It is safe to use the forwardee here. Parallel GC only uses
// header-based forwarding during promotion. Full GC doesn't
// use the object header for forwarding at all.
p += obj->forwardee()->size();
} else {
cl->do_object(obj);
p += obj->size();
}
#ifdef ASSERT
else {
assert(obj->forwardee() != obj, "must not be self-forwarded");
}
#endif
p += obj->size();
}
}

4
src/hotspot/share/gc/parallel/parallelArguments.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, Red Hat, Inc. and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@ -27,6 +27,7 @@
#include "gc/parallel/parallelArguments.hpp"
#include "gc/parallel/parallelScavengeHeap.hpp"
#include "gc/shared/adaptiveSizePolicy.hpp"
#include "gc/shared/fullGCForwarding.hpp"
#include "gc/shared/gcArguments.hpp"
#include "gc/shared/genArguments.hpp"
#include "gc/shared/workerPolicy.hpp"
@ -127,6 +128,7 @@ void ParallelArguments::initialize_heap_flags_and_sizes() {
// Redo everything from the start
initialize_heap_flags_and_sizes_one_pass();
}
FullGCForwarding::initialize_flags(heap_reserved_size_bytes());
}
size_t ParallelArguments::heap_reserved_size_bytes() {

3
src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
View File

@ -33,6 +33,7 @@
#include "gc/parallel/psPromotionManager.hpp"
#include "gc/parallel/psScavenge.hpp"
#include "gc/parallel/psVMOperations.hpp"
#include "gc/shared/fullGCForwarding.inline.hpp"
#include "gc/shared/gcHeapSummary.hpp"
#include "gc/shared/gcLocker.inline.hpp"
#include "gc/shared/gcWhen.hpp"
@ -129,6 +130,8 @@ jint ParallelScavengeHeap::initialize() {
ParallelInitLogger::print();
FullGCForwarding::initialize(heap_rs.region());
return JNI_OK;
}

22
src/hotspot/share/gc/parallel/psParallelCompact.cpp
View File

@ -44,6 +44,7 @@
#include "gc/parallel/psStringDedup.hpp"
#include "gc/parallel/psYoungGen.hpp"
#include "gc/shared/classUnloadingContext.hpp"
#include "gc/shared/fullGCForwarding.inline.hpp"
#include "gc/shared/gcCause.hpp"
#include "gc/shared/gcHeapSummary.hpp"
#include "gc/shared/gcId.hpp"
@ -773,6 +774,8 @@ void PSParallelCompact::fill_dense_prefix_end(SpaceId id) {
//
// The size of the filler (min-obj-size) is 2 heap words with the default
// MinObjAlignment, since both markword and klass take 1 heap word.
// With +UseCompactObjectHeaders, the minimum filler size is only one word,
// because the Klass* gets encoded in the mark-word.
//
// The size of the gap (if any) right before dense-prefix-end is
// MinObjAlignment.
@ -780,16 +783,11 @@ void PSParallelCompact::fill_dense_prefix_end(SpaceId id) {
// Need to fill in the gap only if it's smaller than min-obj-size, and the
// filler obj will extend to next region.
// Note: If min-fill-size decreases to 1, this whole method becomes redundant.
assert(CollectedHeap::min_fill_size() >= 2, "inv");
#ifndef _LP64
// In 32-bit system, each heap word is 4 bytes, so MinObjAlignment == 2.
// The gap is always equal to min-fill-size, so nothing to do.
return;
#endif
if (MinObjAlignment > 1) {
if (MinObjAlignment >= checked_cast<int>(CollectedHeap::min_fill_size())) {
return;
}
assert(!UseCompactObjectHeaders, "Compact headers can allocate small objects");
assert(CollectedHeap::min_fill_size() == 2, "inv");
HeapWord* const dense_prefix_end = dense_prefix(id);
assert(_summary_data.is_region_aligned(dense_prefix_end), "precondition");
@ -1593,7 +1591,7 @@ void PSParallelCompact::forward_to_new_addr() {
oop obj = cast_to_oop(cur_addr);
if (new_addr != cur_addr) {
cm->preserved_marks()->push_if_necessary(obj, obj->mark());
obj->forward_to(cast_to_oop(new_addr));
FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
}
size_t obj_size = obj->size();
live_words += obj_size;
@ -1636,7 +1634,7 @@ void PSParallelCompact::verify_forward() {
}
oop obj = cast_to_oop(cur_addr);
if (cur_addr != bump_ptr) {
assert(obj->forwardee() == cast_to_oop(bump_ptr), "inv");
assert(FullGCForwarding::forwardee(obj) == cast_to_oop(bump_ptr), "inv");
}
bump_ptr += obj->size();
cur_addr += obj->size();
@ -2395,8 +2393,8 @@ void MoveAndUpdateClosure::do_addr(HeapWord* addr, size_t words) {
if (copy_destination() != source()) {
DEBUG_ONLY(PSParallelCompact::check_new_location(source(), destination());)
assert(source() != destination(), "inv");
assert(cast_to_oop(source())->is_forwarded(), "inv");
assert(cast_to_oop(source())->forwardee() == cast_to_oop(destination()), "inv");
assert(FullGCForwarding::is_forwarded(cast_to_oop(source())), "inv");
assert(FullGCForwarding::forwardee(cast_to_oop(source())) == cast_to_oop(destination()), "inv");
Copy::aligned_conjoint_words(source(), copy_destination(), words);
cast_to_oop(copy_destination())->init_mark();
}

5
src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -31,6 +31,7 @@
#include "gc/parallel/parMarkBitMap.inline.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "gc/shared/continuationGCSupport.inline.hpp"
#include "gc/shared/fullGCForwarding.inline.hpp"
#include "oops/access.inline.hpp"
#include "oops/compressedOops.inline.hpp"
#include "oops/klass.hpp"
@ -79,7 +80,7 @@ inline void PSParallelCompact::adjust_pointer(T* p) {
if (!obj->is_forwarded()) {
return;
}
oop new_obj = obj->forwardee();
oop new_obj = FullGCForwarding::forwardee(obj);
assert(new_obj != nullptr, "non-null address for live objects");
assert(new_obj != obj, "inv");
assert(ParallelScavengeHeap::heap()->is_in_reserved(new_obj),

3
src/hotspot/share/gc/parallel/psPromotionManager.cpp
View File

@ -321,7 +321,6 @@ void PSPromotionManager::process_array_chunk(PartialArrayState* state) {
}
void PSPromotionManager::push_objArray(oop old_obj, oop new_obj) {
assert(old_obj->is_objArray(), "precondition");
assert(old_obj->is_forwarded(), "precondition");
assert(old_obj->forwardee() == new_obj, "precondition");
assert(new_obj->is_objArray(), "precondition");
@ -357,7 +356,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) {
// this started. If it is the same (i.e., no forwarding
// pointer has been installed), then this thread owns
// it.
if (obj->forward_to_atomic(obj, obj_mark) == nullptr) {
if (obj->forward_to_self_atomic(obj_mark) == nullptr) {
// We won any races, we "own" this object.
assert(obj == obj->forwardee(), "Sanity");

2
src/hotspot/share/gc/parallel/psPromotionManager.hpp
View File

@ -111,7 +111,7 @@ class PSPromotionManager {
void push_depth(ScannerTask task);
inline void promotion_trace_event(oop new_obj, oop old_obj, size_t obj_size,
inline void promotion_trace_event(oop new_obj, Klass* klass, size_t obj_size,
uint age, bool tenured,
const PSPromotionLAB* lab);

30
src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
View File

@ -66,7 +66,7 @@ inline void PSPromotionManager::claim_or_forward_depth(T* p) {
}
}
inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj,
inline void PSPromotionManager::promotion_trace_event(oop new_obj, Klass* klass,
size_t obj_size,
uint age, bool tenured,
const PSPromotionLAB* lab) {
@ -79,14 +79,14 @@ inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj,
if (gc_tracer->should_report_promotion_in_new_plab_event()) {
size_t obj_bytes = obj_size * HeapWordSize;
size_t lab_size = lab->capacity();
gc_tracer->report_promotion_in_new_plab_event(old_obj->klass(), obj_bytes,
gc_tracer->report_promotion_in_new_plab_event(klass, obj_bytes,
age, tenured, lab_size);
}
} else {
// Promotion of object directly to heap
if (gc_tracer->should_report_promotion_outside_plab_event()) {
size_t obj_bytes = obj_size * HeapWordSize;
gc_tracer->report_promotion_outside_plab_event(old_obj->klass(), obj_bytes,
gc_tracer->report_promotion_outside_plab_event(klass, obj_bytes,
age, tenured);
}
}
@ -149,7 +149,7 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
return copy_unmarked_to_survivor_space<promote_immediately>(o, m);
} else {
// Return the already installed forwardee.
return m.forwardee();
return o->forwardee(m);
}
}
@ -165,7 +165,19 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
oop new_obj = nullptr;
bool new_obj_is_tenured = false;
size_t new_obj_size = o->size();
// NOTE: With compact headers, it is not safe to load the Klass* from old, because
// that would access the mark-word, that might change at any time by concurrent
// workers.
// This mark word would refer to a forwardee, which may not yet have completed
// copying. Therefore we must load the Klass* from the mark-word that we already
// loaded. This is safe, because we only enter here if not yet forwarded.
assert(!test_mark.is_forwarded(), "precondition");
Klass* klass = UseCompactObjectHeaders
? test_mark.klass()
: o->klass();
size_t new_obj_size = o->size_given_klass(klass);
// Find the objects age, MT safe.
uint age = (test_mark.has_displaced_mark_helper() /* o->has_displaced_mark() */) ?
@ -180,7 +192,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
if (new_obj_size > (YoungPLABSize / 2)) {
// Allocate this object directly
new_obj = cast_to_oop(young_space()->cas_allocate(new_obj_size));
promotion_trace_event(new_obj, o, new_obj_size, age, false, nullptr);
promotion_trace_event(new_obj, klass, new_obj_size, age, false, nullptr);
} else {
// Flush and fill
_young_lab.flush();
@ -190,7 +202,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
_young_lab.initialize(MemRegion(lab_base, YoungPLABSize));
// Try the young lab allocation again.
new_obj = cast_to_oop(_young_lab.allocate(new_obj_size));
promotion_trace_event(new_obj, o, new_obj_size, age, false, &_young_lab);
promotion_trace_event(new_obj, klass, new_obj_size, age, false, &_young_lab);
} else {
_young_gen_is_full = true;
}
@ -216,7 +228,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
if (new_obj_size > (OldPLABSize / 2)) {
// Allocate this object directly
new_obj = cast_to_oop(old_gen()->allocate(new_obj_size));
promotion_trace_event(new_obj, o, new_obj_size, age, true, nullptr);
promotion_trace_event(new_obj, klass, new_obj_size, age, true, nullptr);
} else {
// Flush and fill
_old_lab.flush();
@ -226,7 +238,7 @@ inline oop PSPromotionManager::copy_unmarked_to_survivor_space(oop o,
_old_lab.initialize(MemRegion(lab_base, OldPLABSize));
// Try the old lab allocation again.
new_obj = cast_to_oop(_old_lab.allocate(new_obj_size));
promotion_trace_event(new_obj, o, new_obj_size, age, true, &_old_lab);
promotion_trace_event(new_obj, klass, new_obj_size, age, true, &_old_lab);
}
}
}

23
src/hotspot/share/gc/serial/defNewGeneration.cpp
View File

@ -39,7 +39,6 @@
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/gcTrace.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/referencePolicy.hpp"
#include "gc/shared/referenceProcessorPhaseTimes.hpp"
#include "gc/shared/space.hpp"
@ -227,7 +226,6 @@ DefNewGeneration::DefNewGeneration(ReservedSpace rs,
const char* policy)
: Generation(rs, initial_size),
_promotion_failed(false),
_preserved_marks_set(false /* in_c_heap */),
_promo_failure_drain_in_progress(false),
_string_dedup_requests()
{
@ -609,8 +607,6 @@ bool DefNewGeneration::collect(bool clear_all_soft_refs) {
age_table()->clear();
to()->clear(SpaceDecorator::Mangle);
// The preserved marks should be empty at the start of the GC.
_preserved_marks_set.init(1);
YoungGenScanClosure young_gen_cl(this);
OldGenScanClosure old_gen_cl(this);
@ -681,8 +677,6 @@ bool DefNewGeneration::collect(bool clear_all_soft_refs) {
// Reset the PromotionFailureALot counters.
NOT_PRODUCT(heap->reset_promotion_should_fail();)
}
// We should have processed and cleared all the preserved marks.
_preserved_marks_set.reclaim();
heap->trace_heap_after_gc(_gc_tracer);
@ -706,19 +700,17 @@ void DefNewGeneration::remove_forwarding_pointers() {
// starts. (The mark word is overloaded: `is_marked()` == `is_forwarded()`.)
struct ResetForwardedMarkWord : ObjectClosure {
void do_object(oop obj) override {
if (obj->is_forwarded()) {
obj->init_mark();
if (obj->is_self_forwarded()) {
obj->unset_self_forwarded();
} else if (obj->is_forwarded()) {
// To restore the klass-bits in the header.
// Needed for object iteration to work properly.
obj->set_mark(obj->forwardee()->prototype_mark());
}
}
} cl;
eden()->object_iterate(&cl);
from()->object_iterate(&cl);
restore_preserved_marks();
}
void DefNewGeneration::restore_preserved_marks() {
_preserved_marks_set.restore(nullptr);
}
void DefNewGeneration::handle_promotion_failure(oop old) {
@ -726,12 +718,11 @@ void DefNewGeneration::handle_promotion_failure(oop old) {
_promotion_failed = true;
_promotion_failed_info.register_copy_failure(old->size());
_preserved_marks_set.get()->push_if_necessary(old, old->mark());
ContinuationGCSupport::transform_stack_chunk(old);
// forward to self
old->forward_to(old);
old->forward_to_self();
_promo_failure_scan_stack.push(old);

6
src/hotspot/share/gc/serial/defNewGeneration.hpp
View File

@ -32,7 +32,6 @@
#include "gc/shared/copyFailedInfo.hpp"
#include "gc/shared/gc_globals.hpp"
#include "gc/shared/generationCounters.hpp"
#include "gc/shared/preservedMarks.hpp"
#include "gc/shared/stringdedup/stringDedup.hpp"
#include "gc/shared/tlab_globals.hpp"
#include "utilities/align.hpp"
@ -99,11 +98,6 @@ class DefNewGeneration: public Generation {
// therefore we must remove their forwarding pointers.
void remove_forwarding_pointers();
virtual void restore_preserved_marks();
// Preserved marks
PreservedMarksSet _preserved_marks_set;
Stack<oop, mtGC> _promo_failure_scan_stack;
void drain_promo_failure_scan_stack(void);
bool _promo_failure_drain_in_progress;

6
src/hotspot/share/gc/serial/serialArguments.cpp
View File

@ -23,10 +23,16 @@
*/
#include "precompiled.hpp"
#include "gc/shared/fullGCForwarding.hpp"
#include "gc/shared/genArguments.hpp"
#include "gc/serial/serialArguments.hpp"
#include "gc/serial/serialHeap.hpp"
void SerialArguments::initialize_heap_flags_and_sizes() {
GenArguments::initialize_heap_flags_and_sizes();
FullGCForwarding::initialize_flags(MaxHeapSize);
}
CollectedHeap* SerialArguments::create_heap() {
return new SerialHeap();
}

1
src/hotspot/share/gc/serial/serialArguments.hpp
View File

@ -32,6 +32,7 @@ class CollectedHeap;
class SerialArguments : public GenArguments {
private:
virtual CollectedHeap* create_heap();
virtual void initialize_heap_flags_and_sizes();
};
#endif // SHARE_GC_SERIAL_SERIALARGUMENTS_HPP

15
src/hotspot/share/gc/serial/serialFullGC.cpp
View File

@ -43,6 +43,7 @@
#include "gc/shared/classUnloadingContext.hpp"
#include "gc/shared/collectedHeap.inline.hpp"
#include "gc/shared/continuationGCSupport.inline.hpp"
#include "gc/shared/fullGCForwarding.inline.hpp"
#include "gc/shared/gcHeapSummary.hpp"
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/gcTrace.hpp"
@ -230,7 +231,7 @@ class Compacter {
static void forward_obj(oop obj, HeapWord* new_addr) {
prefetch_write_scan(obj);
if (cast_from_oop<HeapWord*>(obj) != new_addr) {
obj->forward_to(cast_to_oop(new_addr));
FullGCForwarding::forward_to(obj, cast_to_oop(new_addr));
} else {
assert(obj->is_gc_marked(), "inv");
// This obj will stay in-place. Fix the markword.
@ -255,7 +256,7 @@ class Compacter {
prefetch_read_scan(addr);
oop obj = cast_to_oop(addr);
oop new_obj = obj->forwardee();
oop new_obj = FullGCForwarding::forwardee(obj);
HeapWord* new_addr = cast_from_oop<HeapWord*>(new_obj);
assert(addr != new_addr, "inv");
prefetch_write_copy(new_addr);
@ -352,13 +353,13 @@ public:
HeapWord* top = space->top();
// Check if the first obj inside this space is forwarded.
if (!cast_to_oop(cur_addr)->is_forwarded()) {
if (!FullGCForwarding::is_forwarded(cast_to_oop(cur_addr))) {
// Jump over consecutive (in-place) live-objs-chunk
cur_addr = get_first_dead(i);
}
while (cur_addr < top) {
if (!cast_to_oop(cur_addr)->is_forwarded()) {
if (!FullGCForwarding::is_forwarded(cast_to_oop(cur_addr))) {
cur_addr = *(HeapWord**) cur_addr;
continue;
}
@ -593,7 +594,7 @@ void SerialFullGC::mark_object(oop obj) {
// some marks may contain information we need to preserve so we store them away
// and overwrite the mark. We'll restore it at the end of serial full GC.
markWord mark = obj->mark();
obj->set_mark(markWord::prototype().set_marked());
obj->set_mark(obj->prototype_mark().set_marked());
ContinuationGCSupport::transform_stack_chunk(obj);
@ -624,8 +625,8 @@ template <class T> void SerialFullGC::adjust_pointer(T* p) {
oop obj = CompressedOops::decode_not_null(heap_oop);
assert(Universe::heap()->is_in(obj), "should be in heap");
if (obj->is_forwarded()) {
oop new_obj = obj->forwardee();
if (FullGCForwarding::is_forwarded(obj)) {
oop new_obj = FullGCForwarding::forwardee(obj);
assert(is_object_aligned(new_obj), "oop must be aligned");
RawAccess<IS_NOT_NULL>::oop_store(p, new_obj);
}

3
src/hotspot/share/gc/serial/serialHeap.cpp
View File

@ -40,6 +40,7 @@
#include "gc/shared/collectedHeap.inline.hpp"
#include "gc/shared/collectorCounters.hpp"
#include "gc/shared/continuationGCSupport.inline.hpp"
#include "gc/shared/fullGCForwarding.hpp"
#include "gc/shared/gcId.hpp"
#include "gc/shared/gcInitLogger.hpp"
#include "gc/shared/gcLocker.inline.hpp"
@ -200,6 +201,8 @@ jint SerialHeap::initialize() {
GCInitLogger::print();
FullGCForwarding::initialize(_reserved);
return JNI_OK;
}

5
src/hotspot/share/gc/shared/c2/barrierSetC2.cpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -710,11 +710,12 @@ int BarrierSetC2::arraycopy_payload_base_offset(bool is_array) {
int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
instanceOopDesc::base_offset_in_bytes();
// base_off:
// 8 - 32-bit VM
// 8 - 32-bit VM or 64-bit VM, compact headers
// 12 - 64-bit VM, compressed klass
// 16 - 64-bit VM, normal klass
if (base_off % BytesPerLong != 0) {
assert(UseCompressedClassPointers, "");
assert(!UseCompactObjectHeaders, "");
if (is_array) {
// Exclude length to copy by 8 bytes words.
base_off += sizeof(int);

22
src/hotspot/share/gc/shared/collectedHeap.cpp
View File

@ -220,6 +220,26 @@ bool CollectedHeap::supports_concurrent_gc_breakpoints() const {
return false;
}
static bool klass_is_sane(oop object) {
if (UseCompactObjectHeaders) {
// With compact headers, we can't safely access the Klass* when
// the object has been forwarded, because non-full-GC-forwarding
// temporarily overwrites the mark-word, and thus the Klass*, with
// the forwarding pointer, and here we have no way to make a
// distinction between Full-GC and regular GC forwarding.
markWord mark = object->mark();
if (mark.is_forwarded()) {
// We can't access the Klass*. We optimistically assume that
// it is ok. This happens very rarely.
return true;
}
return Metaspace::contains(mark.klass_without_asserts());
}
return Metaspace::contains(object->klass_without_asserts());
}
bool CollectedHeap::is_oop(oop object) const {
if (!is_object_aligned(object)) {
return false;
@ -229,7 +249,7 @@ bool CollectedHeap::is_oop(oop object) const {
return false;
}
if (!Metaspace::contains(object->klass_without_asserts())) {
if (!klass_is_sane(object)) {
return false;
}

2
src/hotspot/share/gc/shared/collectedHeap.hpp
View File

@ -306,7 +306,7 @@ protected:
}
virtual void fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap);
static constexpr size_t min_dummy_object_size() {
static size_t min_dummy_object_size() {
return oopDesc::header_size();
}

57
src/hotspot/share/gc/shared/fullGCForwarding.cpp Normal file
View File

@ -0,0 +1,57 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "gc/shared/fullGCForwarding.hpp"
#include "memory/memRegion.hpp"
#include "runtime/globals_extension.hpp"
HeapWord* FullGCForwarding::_heap_base = nullptr;
int FullGCForwarding::_num_low_bits = 0;
void FullGCForwarding::initialize_flags(size_t max_heap_size) {
#ifdef _LP64
size_t max_narrow_heap_size = right_n_bits(NumLowBitsNarrow - Shift);
if (UseCompactObjectHeaders && max_heap_size > max_narrow_heap_size * HeapWordSize) {
warning("Compact object headers require a java heap size smaller than " SIZE_FORMAT
"%s (given: " SIZE_FORMAT "%s). Disabling compact object headers.",
byte_size_in_proper_unit(max_narrow_heap_size * HeapWordSize),
proper_unit_for_byte_size(max_narrow_heap_size * HeapWordSize),
byte_size_in_proper_unit(max_heap_size),
proper_unit_for_byte_size(max_heap_size));
FLAG_SET_ERGO(UseCompactObjectHeaders, false);
}
#endif
}
void FullGCForwarding::initialize(MemRegion heap) {
#ifdef _LP64
_heap_base = heap.start();
if (UseCompactObjectHeaders) {
_num_low_bits = NumLowBitsNarrow;
} else {
_num_low_bits = NumLowBitsWide;
}
#endif
}

59
src/hotspot/share/gc/shared/fullGCForwarding.hpp Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#ifndef SHARE_GC_SHARED_FULLGCFORWARDING_HPP
#define SHARE_GC_SHARED_FULLGCFORWARDING_HPP
#include "memory/allStatic.hpp"
#include "memory/memRegion.hpp"
#include "oops/markWord.hpp"
#include "oops/oopsHierarchy.hpp"
/*
* Implements forwarding for the Full GCs of Serial, Parallel, G1 and Shenandoah in
* a way that preserves upper N bits of object mark-words, which contain crucial
* Klass* information when running with compact headers. The encoding is similar to
* compressed-oops encoding: it basically subtracts the forwardee address from the
* heap-base, shifts that difference into the right place, and sets the lowest two
* bits (to indicate 'forwarded' state as usual).
* With compact-headers, we have 40 bits to encode forwarding pointers. This is
* enough to address 8TB of heap. If the heap size exceeds that limit, we turn off
* compact headers.
*/
class FullGCForwarding : public AllStatic {
static const int NumLowBitsNarrow = LP64_ONLY(markWord::klass_shift) NOT_LP64(0 /*unused*/);
static const int NumLowBitsWide = BitsPerWord;
static const int Shift = markWord::lock_bits + markWord::lock_shift;
static HeapWord* _heap_base;
static int _num_low_bits;
public:
static void initialize_flags(size_t max_heap_size);
static void initialize(MemRegion heap);
static inline void forward_to(oop from, oop to);
static inline oop forwardee(oop from);
static inline bool is_forwarded(oop obj);
};
#endif // SHARE_GC_SHARED_FULLGCFORWARDING_HPP

Some files were not shown because too many files have changed in this diff Show More