stan/jdk-24
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

8223567: Rename ShenandoahBrooksPointer to ShenandoahForwarding

Browse Source 
Reviewed-by: shade
This commit is contained in:
Roman Kennke 2019-05-08 20:45:30 +02:00
parent 0450eca4e7
commit 94e4724219
20 changed files with 121 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
View File

@ -23,6 +23,7 @@
#include "precompiled.hpp"
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahHeuristics.hpp"
@ -221,7 +222,7 @@ void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm
// IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2.
void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) {
assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
__ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
__ ldr(dst, Address(dst, ShenandoahForwarding::byte_offset()));
}
void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp) {
@ -355,9 +356,9 @@ void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register
__ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
if (var_size_in_bytes == noreg) {
__ lea(end, Address(obj, (int) (con_size_in_bytes + ShenandoahBrooksPointer::byte_size())));
__ lea(end, Address(obj, (int) (con_size_in_bytes + ShenandoahForwarding::byte_size())));
} else {
__ add(var_size_in_bytes, var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
__ add(var_size_in_bytes, var_size_in_bytes, ShenandoahForwarding::byte_size());
__ lea(end, Address(obj, var_size_in_bytes));
}
__ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
@ -367,8 +368,8 @@ void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register
// update the tlab top pointer
__ str(end, Address(rthread, JavaThread::tlab_top_offset()));
__ add(obj, obj, ShenandoahBrooksPointer::byte_size());
__ str(obj, Address(obj, ShenandoahBrooksPointer::byte_offset()));
__ add(obj, obj, ShenandoahForwarding::byte_size());
__ str(obj, Address(obj, ShenandoahForwarding::byte_offset()));
// recover var_size_in_bytes if necessary
if (var_size_in_bytes == end) {

13
src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
View File

@ -23,6 +23,7 @@
#include "precompiled.hpp"
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahHeuristics.hpp"
@ -327,7 +328,7 @@ void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm
void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) {
assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled");
__ movptr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
__ movptr(dst, Address(dst, ShenandoahForwarding::byte_offset()));
}
@ -501,9 +502,9 @@ void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
__ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
if (var_size_in_bytes == noreg) {
__ lea(end, Address(obj, con_size_in_bytes + ShenandoahBrooksPointer::byte_size()));
__ lea(end, Address(obj, con_size_in_bytes + ShenandoahForwarding::byte_size()));
} else {
__ addptr(var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
__ addptr(var_size_in_bytes, ShenandoahForwarding::byte_size());
__ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
}
__ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
@ -514,11 +515,11 @@ void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
// Initialize brooks pointer
#ifdef _LP64
__ incrementq(obj, ShenandoahBrooksPointer::byte_size());
__ incrementq(obj, ShenandoahForwarding::byte_size());
#else
__ incrementl(obj, ShenandoahBrooksPointer::byte_size());
__ incrementl(obj, ShenandoahForwarding::byte_size());
#endif
__ movptr(Address(obj, ShenandoahBrooksPointer::byte_offset()), obj);
__ movptr(Address(obj, ShenandoahForwarding::byte_offset()), obj);
// recover var_size_in_bytes if necessary
if (var_size_in_bytes == end) {

1
src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp
View File

@ -25,7 +25,6 @@
#include "c1/c1_IR.hpp"
#include "gc/shared/satbMarkQueue.hpp"
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"

7
src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
View File

@ -23,6 +23,7 @@
#include "precompiled.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahHeuristics.hpp"
#include "gc/shenandoah/shenandoahRuntime.hpp"
@ -716,7 +717,7 @@ Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl,
PhaseIterGVN& igvn = macro->igvn();
// Allocate several words more for the Shenandoah brooks pointer.
size_in_bytes = new AddXNode(size_in_bytes, igvn.MakeConX(ShenandoahBrooksPointer::byte_size()));
size_in_bytes = new AddXNode(size_in_bytes, igvn.MakeConX(ShenandoahForwarding::byte_size()));
macro->transform_later(size_in_bytes);
Node* fast_oop = BarrierSetC2::obj_allocate(macro, ctrl, mem, toobig_false, size_in_bytes,
@ -724,11 +725,11 @@ Node* ShenandoahBarrierSetC2::obj_allocate(PhaseMacroExpand* macro, Node* ctrl,
prefetch_lines);
// Bump up object for Shenandoah brooks pointer.
fast_oop = new AddPNode(macro->top(), fast_oop, igvn.MakeConX(ShenandoahBrooksPointer::byte_size()));
fast_oop = new AddPNode(macro->top(), fast_oop, igvn.MakeConX(ShenandoahForwarding::byte_size()));
macro->transform_later(fast_oop);
// Initialize Shenandoah brooks pointer to point to the object itself.
fast_oop_rawmem = macro->make_store(fast_oop_ctrl, fast_oop_rawmem, fast_oop, ShenandoahBrooksPointer::byte_offset(), fast_oop, T_OBJECT);
fast_oop_rawmem = macro->make_store(fast_oop_ctrl, fast_oop_rawmem, fast_oop, ShenandoahForwarding::byte_offset(), fast_oop, T_OBJECT);
return fast_oop;
}

4
src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
View File

@ -26,7 +26,7 @@
#include "gc/shenandoah/c2/shenandoahSupport.hpp"
#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahRuntime.hpp"
@ -1505,7 +1505,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
IfNode* iff = unc_ctrl->in(0)->as_If();
phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1));
}
Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahBrooksPointer::byte_offset()));
Node* addr = new AddPNode(new_val, uncasted_val, phase->igvn().MakeConX(ShenandoahForwarding::byte_offset()));
phase->register_new_node(addr, ctrl);
assert(val->bottom_type()->isa_oopptr(), "what else?");
const TypePtr* obj_type = val->bottom_type()->is_oopptr();

1
src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
View File

@ -24,7 +24,6 @@
#ifndef SHARE_GC_SHENANDOAH_C2_SHENANDOAHSUPPORT_HPP
#define SHARE_GC_SHENANDOAH_C2_SHENANDOAHSUPPORT_HPP
#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
#include "memory/allocation.hpp"
#include "opto/addnode.hpp"
#include "opto/graphKit.hpp"

18
src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp
View File

@ -24,7 +24,7 @@
#include "precompiled.hpp"
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
@ -136,7 +136,7 @@ void ShenandoahAsserts::print_failure(SafeLevel level, oop obj, void* interior_l
msg.append("\n");
if (level >= _safe_oop) {
oop fwd = (oop) ShenandoahBrooksPointer::get_raw_unchecked(obj);
oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
msg.append("Forwardee:\n");
if (!oopDesc::equals_raw(obj, fwd)) {
if (level >= _safe_oop_fwd) {
@ -151,8 +151,8 @@ void ShenandoahAsserts::print_failure(SafeLevel level, oop obj, void* interior_l
}
if (level >= _safe_oop_fwd) {
oop fwd = (oop) ShenandoahBrooksPointer::get_raw_unchecked(obj);
oop fwd2 = (oop) ShenandoahBrooksPointer::get_raw_unchecked(fwd);
oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
oop fwd2 = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
if (!oopDesc::equals_raw(fwd, fwd2)) {
msg.append("Second forwardee:\n");
print_obj_safe(msg, fwd2);
@ -197,7 +197,7 @@ void ShenandoahAsserts::assert_correct(void* interior_loc, oop obj, const char*
file,line);
}
oop fwd = oop(ShenandoahBrooksPointer::get_raw_unchecked(obj));
oop fwd = oop(ShenandoahForwarding::get_forwardee_raw_unchecked(obj));
if (!oopDesc::equals_raw(obj, fwd)) {
// When Full GC moves the objects, we cannot trust fwdptrs. If we got here, it means something
@ -230,7 +230,7 @@ void ShenandoahAsserts::assert_correct(void* interior_loc, oop obj, const char*
}
// Step 4. Check for multiple forwardings
oop fwd2 = oop(ShenandoahBrooksPointer::get_raw_unchecked(fwd));
oop fwd2 = oop(ShenandoahForwarding::get_forwardee_raw_unchecked(fwd));
if (!oopDesc::equals_raw(fwd, fwd2)) {
print_failure(_safe_all, obj, interior_loc, NULL, "Shenandoah assert_correct failed",
"Multiple forwardings",
@ -250,7 +250,7 @@ void ShenandoahAsserts::assert_in_correct_region(void* interior_loc, oop obj, co
file, line);
}
size_t alloc_size = obj->size() + ShenandoahBrooksPointer::word_size();
size_t alloc_size = obj->size() + ShenandoahForwarding::word_size();
if (alloc_size > ShenandoahHeapRegion::humongous_threshold_words()) {
size_t idx = r->region_number();
size_t num_regions = ShenandoahHeapRegion::required_regions(alloc_size * HeapWordSize);
@ -272,7 +272,7 @@ void ShenandoahAsserts::assert_in_correct_region(void* interior_loc, oop obj, co
void ShenandoahAsserts::assert_forwarded(void* interior_loc, oop obj, const char* file, int line) {
assert_correct(interior_loc, obj, file, line);
oop fwd = oop(ShenandoahBrooksPointer::get_raw_unchecked(obj));
oop fwd = oop(ShenandoahForwarding::get_forwardee_raw_unchecked(obj));
if (oopDesc::equals_raw(obj, fwd)) {
print_failure(_safe_all, obj, interior_loc, NULL, "Shenandoah assert_forwarded failed",
@ -283,7 +283,7 @@ void ShenandoahAsserts::assert_forwarded(void* interior_loc, oop obj, const char
void ShenandoahAsserts::assert_not_forwarded(void* interior_loc, oop obj, const char* file, int line) {
assert_correct(interior_loc, obj, file, line);
oop fwd = oop(ShenandoahBrooksPointer::get_raw_unchecked(obj));
oop fwd = oop(ShenandoahForwarding::get_forwardee_raw_unchecked(obj));
if (!oopDesc::equals_raw(obj, fwd)) {
print_failure(_safe_all, obj, interior_loc, NULL, "Shenandoah assert_not_forwarded failed",

5
src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp
View File

@ -26,6 +26,7 @@
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeuristics.hpp"
#include "gc/shenandoah/shenandoahTraversalGC.hpp"
@ -261,7 +262,7 @@ oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj) {
ShenandoahHeapRegion* r = _heap->heap_region_containing(obj);
assert(r->is_cset(), "sanity");
HeapWord* cur = (HeapWord*)obj + obj->size() + ShenandoahBrooksPointer::word_size();
HeapWord* cur = (HeapWord*)obj + obj->size() + ShenandoahForwarding::word_size();
size_t count = 0;
while ((cur < r->top()) && ctx->is_marked(oop(cur)) && (count++ < max)) {
@ -269,7 +270,7 @@ oop ShenandoahBarrierSet::load_reference_barrier_mutator(oop obj) {
if (oopDesc::equals_raw(cur_oop, resolve_forwarded_not_null(cur_oop))) {
_heap->evacuate_object(cur_oop, thread);
}
cur = cur + cur_oop->size() + ShenandoahBrooksPointer::word_size();
cur = cur + cur_oop->size() + ShenandoahForwarding::word_size();
}
}

4
src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
View File

@ -26,14 +26,14 @@
#include "gc/shared/barrierSet.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.inline.hpp"
#include "gc/shenandoah/shenandoahForwarding.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
inline oop ShenandoahBarrierSet::resolve_forwarded_not_null(oop p) {
return ShenandoahBrooksPointer::forwardee(p);
return ShenandoahForwarding::get_forwardee(p);
}
inline oop ShenandoahBarrierSet::resolve_forwarded(oop p) {

1
src/hotspot/share/gc/shenandoah/shenandoahBarrierSetAssembler.hpp
View File

@ -25,7 +25,6 @@
#define SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_HPP
#include "utilities/macros.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
#include CPU_HEADER(gc/shenandoah/shenandoahBarrierSetAssembler)

4
src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp
View File

@ -25,7 +25,7 @@
#define SHARE_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_INLINE_HPP
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
#include "gc/shenandoah/shenandoahConcurrentMark.hpp"
#include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
@ -69,7 +69,7 @@ void ShenandoahConcurrentMark::do_task(ShenandoahObjToScanQueue* q, T* cl, jusho
inline void ShenandoahConcurrentMark::count_liveness(jushort* live_data, oop obj) {
size_t region_idx = _heap->heap_region_index_containing(obj);
ShenandoahHeapRegion* region = _heap->get_region(region_idx);
size_t size = obj->size() + ShenandoahBrooksPointer::word_size();
size_t size = obj->size() + ShenandoahForwarding::word_size();
if (!region->is_humongous_start()) {
assert(!region->is_humongous(), "Cannot have continuations here");

20
src/hotspot/share/gc/shenandoah/shenandoahBrooksPointer.hpp → src/hotspot/share/gc/shenandoah/shenandoahForwarding.hpp
View File

@ -21,13 +21,13 @@
*
*/
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHBROOKSPOINTER_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHBROOKSPOINTER_HPP
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_HPP
#include "oops/oop.hpp"
#include "utilities/globalDefinitions.hpp"
class ShenandoahBrooksPointer {
class ShenandoahForwarding {
/*
* Notes:
*
@ -70,13 +70,13 @@ public:
guarantee (MinObjAlignmentInBytes > 0, "sanity, byte_size is correct");
}
/* Initializes Brooks pointer (to self).
/* Initializes forwarding pointer (to self).
*/
static inline void initialize(oop obj);
/* Gets forwardee from the given object.
*/
static inline oop forwardee(oop obj);
static inline oop get_forwardee(oop obj);
/* Tries to atomically update forwardee in $holder object to $update.
* Assumes $holder points at itself.
@ -88,19 +88,19 @@ public:
/* Sets raw value for forwardee slot.
* THIS IS DANGEROUS: USERS HAVE TO INITIALIZE/SET FORWARDEE BACK AFTER THEY ARE DONE.
*/
static inline void set_raw(oop obj, HeapWord* update);
static inline void set_forwardee_raw(oop obj, HeapWord* update);
/* Returns the raw value from forwardee slot.
*/
static inline HeapWord* get_raw(oop obj);
static inline HeapWord* get_forwardee_raw(oop obj);
/* Returns the raw value from forwardee slot without any checks.
* Used for quick verification.
*/
static inline HeapWord* get_raw_unchecked(oop obj);
static inline HeapWord* get_forwardee_raw_unchecked(oop obj);
private:
static inline HeapWord** brooks_ptr_addr(oop obj);
static inline HeapWord** forward_ptr_addr(oop obj);
};
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHBROOKSPOINTER_HPP
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_HPP

34
src/hotspot/share/gc/shenandoah/shenandoahBrooksPointer.inline.hpp → src/hotspot/share/gc/shenandoah/shenandoahForwarding.inline.hpp
View File

@ -21,45 +21,45 @@
*
*/
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHBROOKSPOINTER_INLINE_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHBROOKSPOINTER_INLINE_HPP
#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP
#define SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "runtime/atomic.hpp"
inline HeapWord** ShenandoahBrooksPointer::brooks_ptr_addr(oop obj) {
inline HeapWord** ShenandoahForwarding::forward_ptr_addr(oop obj) {
return (HeapWord**)((HeapWord*) obj + word_offset());
}
inline void ShenandoahBrooksPointer::initialize(oop obj) {
inline void ShenandoahForwarding::initialize(oop obj) {
shenandoah_assert_in_heap(NULL, obj);
*brooks_ptr_addr(obj) = (HeapWord*) obj;
*forward_ptr_addr(obj) = (HeapWord*) obj;
}
inline void ShenandoahBrooksPointer::set_raw(oop obj, HeapWord* update) {
inline void ShenandoahForwarding::set_forwardee_raw(oop obj, HeapWord* update) {
shenandoah_assert_in_heap(NULL, obj);
*brooks_ptr_addr(obj) = update;
*forward_ptr_addr(obj) = update;
}
inline HeapWord* ShenandoahBrooksPointer::get_raw(oop obj) {
inline HeapWord* ShenandoahForwarding::get_forwardee_raw(oop obj) {
shenandoah_assert_in_heap(NULL, obj);
return *brooks_ptr_addr(obj);
return *forward_ptr_addr(obj);
}
inline HeapWord* ShenandoahBrooksPointer::get_raw_unchecked(oop obj) {
return *brooks_ptr_addr(obj);
inline HeapWord* ShenandoahForwarding::get_forwardee_raw_unchecked(oop obj) {
return *forward_ptr_addr(obj);
}
inline oop ShenandoahBrooksPointer::forwardee(oop obj) {
inline oop ShenandoahForwarding::get_forwardee(oop obj) {
shenandoah_assert_correct(NULL, obj);
return oop(*brooks_ptr_addr(obj));
return oop(*forward_ptr_addr(obj));
}
inline oop ShenandoahBrooksPointer::try_update_forwardee(oop obj, oop update) {
oop result = (oop) Atomic::cmpxchg(update, (oop*)brooks_ptr_addr(obj), obj);
inline oop ShenandoahForwarding::try_update_forwardee(oop obj, oop update) {
oop result = (oop) Atomic::cmpxchg(update, (oop*)forward_ptr_addr(obj), obj);
shenandoah_assert_correct_except(NULL, obj, !oopDesc::equals_raw(result, obj));
return result;
}
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHBROOKSPOINTER_INLINE_HPP
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHFORWARDING_INLINE_HPP

24
src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
View File

@ -33,7 +33,7 @@
#include "gc/shenandoah/shenandoahAllocTracker.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
@ -133,7 +133,7 @@ public:
};
jint ShenandoahHeap::initialize() {
ShenandoahBrooksPointer::initial_checks();
ShenandoahForwarding::initial_checks();
initialize_heuristics();
@ -861,7 +861,7 @@ private:
MemAllocator& _initializer;
public:
ShenandoahMemAllocator(MemAllocator& initializer, Klass* klass, size_t word_size, Thread* thread) :
MemAllocator(klass, word_size + ShenandoahBrooksPointer::word_size(), thread),
MemAllocator(klass, word_size + ShenandoahForwarding::word_size(), thread),
_initializer(initializer) {}
protected:
@ -869,8 +869,8 @@ protected:
HeapWord* result = MemAllocator::mem_allocate(allocation);
// Initialize brooks-pointer
if (result != NULL) {
result += ShenandoahBrooksPointer::word_size();
ShenandoahBrooksPointer::initialize(oop(result));
result += ShenandoahForwarding::word_size();
ShenandoahForwarding::initialize(oop(result));
assert(! ShenandoahHeap::heap()->in_collection_set(result), "never allocate in targetted region");
}
return result;
@ -947,7 +947,7 @@ void ShenandoahHeap::fill_with_dummy_object(HeapWord* start, HeapWord* end, bool
}
size_t ShenandoahHeap::min_dummy_object_size() const {
return CollectedHeap::min_dummy_object_size() + ShenandoahBrooksPointer::word_size();
return CollectedHeap::min_dummy_object_size() + ShenandoahForwarding::word_size();
}
class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
@ -1040,8 +1040,8 @@ void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
assert(start->is_humongous_start(), "reclaim regions starting with the first one");
oop humongous_obj = oop(start->bottom() + ShenandoahBrooksPointer::word_size());
size_t size = humongous_obj->size() + ShenandoahBrooksPointer::word_size();
oop humongous_obj = oop(start->bottom() + ShenandoahForwarding::word_size());
size_t size = humongous_obj->size() + ShenandoahForwarding::word_size();
size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
size_t index = start->region_number() + required_regions - 1;
@ -1854,8 +1854,8 @@ void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj) {
// Initialize Brooks pointer for the next object
HeapWord* result = obj + ShenandoahBrooksPointer::word_size();
ShenandoahBrooksPointer::initialize(oop(result));
HeapWord* result = obj + ShenandoahForwarding::word_size();
ShenandoahForwarding::initialize(oop(result));
return result;
}
@ -2824,9 +2824,9 @@ void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
}
size_t ShenandoahHeap::obj_size(oop obj) const {
return CollectedHeap::obj_size(obj) + ShenandoahBrooksPointer::word_size();
return CollectedHeap::obj_size(obj) + ShenandoahForwarding::word_size();
}
ptrdiff_t ShenandoahHeap::cell_header_size() const {
return ShenandoahBrooksPointer::byte_size();
return ShenandoahForwarding::byte_size();
}

24
src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp
View File

@ -30,8 +30,8 @@
#include "gc/shared/suspendibleThreadSet.hpp"
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.inline.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
#include "gc/shenandoah/shenandoahForwarding.inline.hpp"
#include "gc/shenandoah/shenandoahWorkGroup.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
@ -235,7 +235,7 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
size_t size_no_fwdptr = (size_t) p->size();
size_t size_with_fwdptr = size_no_fwdptr + ShenandoahBrooksPointer::word_size();
size_t size_with_fwdptr = size_no_fwdptr + ShenandoahForwarding::word_size();
assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
@ -269,14 +269,14 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
}
// Copy the object and initialize its forwarding ptr:
HeapWord* copy = filler + ShenandoahBrooksPointer::word_size();
HeapWord* copy = filler + ShenandoahForwarding::word_size();
oop copy_val = oop(copy);
Copy::aligned_disjoint_words((HeapWord*) p, copy, size_no_fwdptr);
ShenandoahBrooksPointer::initialize(oop(copy));
ShenandoahForwarding::initialize(oop(copy));
// Try to install the new forwarding pointer.
oop result = ShenandoahBrooksPointer::try_update_forwardee(p, copy_val);
oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
if (oopDesc::equals_raw(result, p)) {
// Successfully evacuated. Our copy is now the public one!
@ -370,7 +370,7 @@ inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region,
template<class T>
inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit) {
assert(ShenandoahBrooksPointer::word_offset() < 0, "skip_delta calculation below assumes the forwarding ptr is before obj");
assert(ShenandoahForwarding::word_offset() < 0, "skip_delta calculation below assumes the forwarding ptr is before obj");
assert(! region->is_humongous_continuation(), "no humongous continuation regions here");
ShenandoahMarkingContext* const ctx = complete_marking_context();
@ -379,10 +379,10 @@ inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region,
MarkBitMap* mark_bit_map = ctx->mark_bit_map();
HeapWord* tams = ctx->top_at_mark_start(region);
size_t skip_bitmap_delta = ShenandoahBrooksPointer::word_size() + 1;
size_t skip_objsize_delta = ShenandoahBrooksPointer::word_size() /* + actual obj.size() below */;
HeapWord* start = region->bottom() + ShenandoahBrooksPointer::word_size();
HeapWord* end = MIN2(tams + ShenandoahBrooksPointer::word_size(), region->end());
size_t skip_bitmap_delta = ShenandoahForwarding::word_size() + 1;
size_t skip_objsize_delta = ShenandoahForwarding::word_size() /* + actual obj.size() below */;
HeapWord* start = region->bottom() + ShenandoahForwarding::word_size();
HeapWord* end = MIN2(tams + ShenandoahForwarding::word_size(), region->end());
// Step 1. Scan below the TAMS based on bitmap data.
HeapWord* limit_bitmap = MIN2(limit, tams);
@ -412,7 +412,7 @@ inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region,
do {
avail = 0;
for (int c = 0; (c < dist) && (cb < limit_bitmap); c++) {
Prefetch::read(cb, ShenandoahBrooksPointer::byte_offset());
Prefetch::read(cb, ShenandoahForwarding::byte_offset());
slots[avail++] = cb;
cb += skip_bitmap_delta;
if (cb < limit_bitmap) {
@ -447,7 +447,7 @@ inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region,
// Step 2. Accurate size-based traversal, happens past the TAMS.
// This restarts the scan at TAMS, which makes sure we traverse all objects,
// regardless of what happened at Step 1.
HeapWord* cs = tams + ShenandoahBrooksPointer::word_size();
HeapWord* cs = tams + ShenandoahForwarding::word_size();
while (cs < limit) {
assert (cs > tams, "only objects past TAMS here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams));
assert (cs < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(limit));

12
src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp
View File

@ -23,7 +23,7 @@
#include "precompiled.hpp"
#include "memory/allocation.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
@ -452,12 +452,12 @@ void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) {
void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) {
assert(! is_humongous(), "no humongous region here");
HeapWord* obj_addr = bottom() + ShenandoahBrooksPointer::word_size();
HeapWord* obj_addr = bottom() + ShenandoahForwarding::word_size();
HeapWord* t = top();
// Could call objects iterate, but this is easier.
while (obj_addr < t) {
oop obj = oop(obj_addr);
obj_addr += obj->oop_iterate_size(blk) + ShenandoahBrooksPointer::word_size();
obj_addr += obj->oop_iterate_size(blk) + ShenandoahForwarding::word_size();
}
}
@ -466,7 +466,7 @@ void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) {
// Find head.
ShenandoahHeapRegion* r = humongous_start_region();
assert(r->is_humongous_start(), "need humongous head here");
oop obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
oop obj = oop(r->bottom() + ShenandoahForwarding::word_size());
obj->oop_iterate(blk, MemRegion(bottom(), top()));
}
@ -505,11 +505,11 @@ HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const {
if (p >= top()) {
return top();
} else {
HeapWord* last = bottom() + ShenandoahBrooksPointer::word_size();
HeapWord* last = bottom() + ShenandoahForwarding::word_size();
HeapWord* cur = last;
while (cur <= p) {
last = cur;
cur += oop(cur)->size() + ShenandoahBrooksPointer::word_size();
cur += oop(cur)->size() + ShenandoahForwarding::word_size();
}
shenandoah_assert_correct(NULL, oop(last));
return last;

4
src/hotspot/share/gc/shenandoah/shenandoahHeuristics.cpp
View File

@ -24,7 +24,7 @@
#include "precompiled.hpp"
#include "gc/shared/gcCause.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
@ -164,7 +164,7 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec
// Reclaim humongous regions here, and count them as the immediate garbage
#ifdef ASSERT
bool reg_live = region->has_live();
bool bm_live = ctx->is_marked(oop(region->bottom() + ShenandoahBrooksPointer::word_size()));
bool bm_live = ctx->is_marked(oop(region->bottom() + ShenandoahForwarding::word_size()));
assert(reg_live == bm_live,
"Humongous liveness and marks should agree. Region live: %s; Bitmap live: %s; Region Live Words: " SIZE_FORMAT,
BOOL_TO_STR(reg_live), BOOL_TO_STR(bm_live), region->get_live_data_words());

32
src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp
View File

@ -25,7 +25,7 @@
#include "code/codeCache.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
#include "gc/shenandoah/shenandoahForwarding.inline.hpp"
#include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
@ -266,7 +266,7 @@ public:
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
size_t obj_size = p->size() + ShenandoahBrooksPointer::word_size();
size_t obj_size = p->size() + ShenandoahForwarding::word_size();
if (_compact_point + obj_size > _to_region->end()) {
finish_region();
@ -289,7 +289,7 @@ public:
// Object fits into current region, record new location:
assert(_compact_point + obj_size <= _to_region->end(), "must fit");
shenandoah_assert_not_forwarded(NULL, p);
ShenandoahBrooksPointer::set_raw(p, _compact_point + ShenandoahBrooksPointer::word_size());
ShenandoahForwarding::set_forwardee_raw(p, _compact_point + ShenandoahForwarding::word_size());
_compact_point += obj_size;
}
};
@ -385,15 +385,15 @@ void ShenandoahMarkCompact::calculate_target_humongous_objects() {
if (r->is_humongous_start() && r->is_move_allowed()) {
// From-region candidate: movable humongous region
oop old_obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
size_t words_size = old_obj->size() + ShenandoahBrooksPointer::word_size();
oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
size_t words_size = old_obj->size() + ShenandoahForwarding::word_size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t start = to_end - num_regions;
if (start >= to_begin && start != r->region_number()) {
// Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
ShenandoahBrooksPointer::set_raw(old_obj, heap->get_region(start)->bottom() + ShenandoahBrooksPointer::word_size());
ShenandoahForwarding::set_forwardee_raw(old_obj, heap->get_region(start)->bottom() + ShenandoahForwarding::word_size());
to_end = start;
continue;
}
@ -441,7 +441,7 @@ public:
void heap_region_do(ShenandoahHeapRegion* r) {
if (r->is_humongous_start()) {
oop humongous_obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
oop humongous_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
if (!_ctx->is_marked(humongous_obj)) {
assert(!r->has_live(),
"Region " SIZE_FORMAT " is not marked, should not have live", r->region_number());
@ -504,7 +504,7 @@ private:
if (!CompressedOops::is_null(o)) {
oop obj = CompressedOops::decode_not_null(o);
assert(_ctx->is_marked(obj), "must be marked");
oop forw = oop(ShenandoahBrooksPointer::get_raw(obj));
oop forw = oop(ShenandoahForwarding::get_forwardee_raw(obj));
RawAccess<IS_NOT_NULL>::oop_store(p, forw);
}
}
@ -529,7 +529,7 @@ public:
}
void do_object(oop p) {
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
HeapWord* forw = ShenandoahBrooksPointer::get_raw(p);
HeapWord* forw = ShenandoahForwarding::get_forwardee_raw(p);
p->oop_iterate(&_cl);
}
};
@ -614,13 +614,13 @@ public:
void do_object(oop p) {
assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
size_t size = (size_t)p->size();
HeapWord* compact_to = ShenandoahBrooksPointer::get_raw(p);
HeapWord* compact_to = ShenandoahForwarding::get_forwardee_raw(p);
HeapWord* compact_from = (HeapWord*) p;
if (compact_from != compact_to) {
Copy::aligned_conjoint_words(compact_from, compact_to, size);
}
oop new_obj = oop(compact_to);
ShenandoahBrooksPointer::initialize(new_obj);
ShenandoahForwarding::initialize(new_obj);
}
};
@ -714,13 +714,13 @@ void ShenandoahMarkCompact::compact_humongous_objects() {
for (size_t c = heap->num_regions() - 1; c > 0; c--) {
ShenandoahHeapRegion* r = heap->get_region(c);
if (r->is_humongous_start()) {
oop old_obj = oop(r->bottom() + ShenandoahBrooksPointer::word_size());
size_t words_size = old_obj->size() + ShenandoahBrooksPointer::word_size();
oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
size_t words_size = old_obj->size() + ShenandoahForwarding::word_size();
size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
size_t old_start = r->region_number();
size_t old_end = old_start + num_regions - 1;
size_t new_start = heap->heap_region_index_containing(ShenandoahBrooksPointer::get_raw(old_obj));
size_t new_start = heap->heap_region_index_containing(ShenandoahForwarding::get_forwardee_raw(old_obj));
size_t new_end = new_start + num_regions - 1;
if (old_start == new_start) {
@ -734,8 +734,8 @@ void ShenandoahMarkCompact::compact_humongous_objects() {
heap->get_region(new_start)->bottom(),
ShenandoahHeapRegion::region_size_words()*num_regions);
oop new_obj = oop(heap->get_region(new_start)->bottom() + ShenandoahBrooksPointer::word_size());
ShenandoahBrooksPointer::initialize(new_obj);
oop new_obj = oop(heap->get_region(new_start)->bottom() + ShenandoahForwarding::word_size());
ShenandoahForwarding::initialize(new_obj);
{
for (size_t c = old_start; c <= old_end; c++) {

3
src/hotspot/share/gc/shenandoah/shenandoahTraversalGC.cpp
View File

@ -35,6 +35,7 @@
#include "gc/shenandoah/shenandoahCollectionSet.hpp"
#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
#include "gc/shenandoah/shenandoahFreeSet.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
@ -640,7 +641,7 @@ void ShenandoahTraversalGC::final_traversal_collection() {
bool candidate = traversal_regions->is_in(r) && !r->has_live() && not_allocated;
if (r->is_humongous_start() && candidate) {
// Trash humongous.
HeapWord* humongous_obj = r->bottom() + ShenandoahBrooksPointer::word_size();
HeapWord* humongous_obj = r->bottom() + ShenandoahForwarding::word_size();
assert(!ctx->is_marked(oop(humongous_obj)), "must not be marked");
r->make_trash_immediate();
while (i + 1 < num_regions && _heap->get_region(i + 1)->is_humongous_continuation()) {

18
src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
View File

@ -24,7 +24,7 @@
#include "precompiled.hpp"
#include "gc/shenandoah/shenandoahAsserts.hpp"
#include "gc/shenandoah/shenandoahBrooksPointer.hpp"
#include "gc/shenandoah/shenandoahForwarding.inline.hpp"
#include "gc/shenandoah/shenandoahPhaseTimings.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahRootProcessor.hpp"
@ -137,7 +137,7 @@ private:
// skip
break;
case ShenandoahVerifier::_verify_liveness_complete:
Atomic::add(obj->size() + ShenandoahBrooksPointer::word_size(), &_ld[obj_reg->region_number()]);
Atomic::add(obj->size() + ShenandoahForwarding::word_size(), &_ld[obj_reg->region_number()]);
// fallthrough for fast failure for un-live regions:
case ShenandoahVerifier::_verify_liveness_conservative:
check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(),
@ -148,7 +148,7 @@ private:
}
}
oop fwd = (oop) ShenandoahBrooksPointer::get_raw_unchecked(obj);
oop fwd = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(obj);
ShenandoahHeapRegion* fwd_reg = NULL;
@ -181,7 +181,7 @@ private:
check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(),
"Forwardee end should be within the region");
oop fwd2 = (oop) ShenandoahBrooksPointer::get_raw_unchecked(fwd);
oop fwd2 = (oop) ShenandoahForwarding::get_forwardee_raw_unchecked(fwd);
check(ShenandoahAsserts::_safe_oop, obj, oopDesc::equals_raw(fwd, fwd2),
"Double forwarding");
} else {
@ -528,7 +528,7 @@ public:
virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) {
size_t processed = 0;
HeapWord* obj = r->bottom() + ShenandoahBrooksPointer::word_size();
HeapWord* obj = r->bottom() + ShenandoahForwarding::word_size();
if (_heap->complete_marking_context()->is_marked((oop)obj)) {
verify_and_follow(obj, stack, cl, &processed);
}
@ -542,12 +542,12 @@ public:
// Bitmaps, before TAMS
if (tams > r->bottom()) {
HeapWord* start = r->bottom() + ShenandoahBrooksPointer::word_size();
HeapWord* start = r->bottom() + ShenandoahForwarding::word_size();
HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams);
while (addr < tams) {
verify_and_follow(addr, stack, cl, &processed);
addr += ShenandoahBrooksPointer::word_size();
addr += ShenandoahForwarding::word_size();
if (addr < tams) {
addr = mark_bit_map->get_next_marked_addr(addr, tams);
}
@ -557,11 +557,11 @@ public:
// Size-based, after TAMS
{
HeapWord* limit = r->top();
HeapWord* addr = tams + ShenandoahBrooksPointer::word_size();
HeapWord* addr = tams + ShenandoahForwarding::word_size();
while (addr < limit) {
verify_and_follow(addr, stack, cl, &processed);
addr += oop(addr)->size() + ShenandoahBrooksPointer::word_size();
addr += oop(addr)->size() + ShenandoahForwarding::word_size();
}
}