8232896: ZGC: Enable C2 clone intrinsic
Reviewed-by: pliden, kvn
This commit is contained in:
parent
55f7c48828
commit
3ca7e3f6ff
src/hotspot/share
classfile
gc
shared/c2
z
opto
test/micro/org/openjdk/bench/java/lang
@ -786,9 +786,6 @@ bool vmIntrinsics::is_disabled_by_flags(vmIntrinsics::ID id) {
|
||||
#endif // COMPILER1
|
||||
#ifdef COMPILER2
|
||||
case vmIntrinsics::_clone:
|
||||
#if INCLUDE_ZGC
|
||||
if (UseZGC) return true;
|
||||
#endif
|
||||
case vmIntrinsics::_copyOf:
|
||||
case vmIntrinsics::_copyOfRange:
|
||||
// These intrinsics use both the objectcopy and the arraycopy
|
||||
|
@ -96,7 +96,6 @@ Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) cons
|
||||
|
||||
store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(),
|
||||
access.addr().type(), mo, requires_atomic_access, unaligned, mismatched, unsafe);
|
||||
access.set_raw_access(store);
|
||||
} else {
|
||||
assert(!requires_atomic_access, "not yet supported");
|
||||
assert(access.is_opt_access(), "either parse or opt access");
|
||||
@ -120,6 +119,8 @@ Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) cons
|
||||
mm->set_memory_at(alias, st);
|
||||
}
|
||||
}
|
||||
access.set_raw_access(store);
|
||||
|
||||
return store;
|
||||
}
|
||||
|
||||
@ -153,7 +154,6 @@ Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) con
|
||||
load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
|
||||
dep, requires_atomic_access, unaligned, mismatched, unsafe);
|
||||
}
|
||||
access.set_raw_access(load);
|
||||
} else {
|
||||
assert(!requires_atomic_access, "not yet supported");
|
||||
assert(access.is_opt_access(), "either parse or opt access");
|
||||
@ -165,6 +165,7 @@ Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) con
|
||||
load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, unaligned, mismatched);
|
||||
load = gvn.transform(load);
|
||||
}
|
||||
access.set_raw_access(load);
|
||||
|
||||
return load;
|
||||
}
|
||||
@ -806,7 +807,8 @@ void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac
|
||||
Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
|
||||
Node* length = ac->in(ArrayCopyNode::Length);
|
||||
|
||||
assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null");
|
||||
assert (src_offset == NULL, "for clone offsets should be null");
|
||||
assert (dest_offset == NULL, "for clone offsets should be null");
|
||||
|
||||
const char* copyfunc_name = "arraycopy";
|
||||
address copyfunc_addr =
|
||||
|
@ -27,14 +27,17 @@
|
||||
#include "gc/z/zBarrierSet.hpp"
|
||||
#include "gc/z/zBarrierSetAssembler.hpp"
|
||||
#include "gc/z/zBarrierSetRuntime.hpp"
|
||||
#include "opto/arraycopynode.hpp"
|
||||
#include "opto/block.hpp"
|
||||
#include "opto/compile.hpp"
|
||||
#include "opto/graphKit.hpp"
|
||||
#include "opto/machnode.hpp"
|
||||
#include "opto/macro.hpp"
|
||||
#include "opto/memnode.hpp"
|
||||
#include "opto/node.hpp"
|
||||
#include "opto/regalloc.hpp"
|
||||
#include "opto/rootnode.hpp"
|
||||
#include "opto/runtime.hpp"
|
||||
#include "utilities/growableArray.hpp"
|
||||
#include "utilities/macros.hpp"
|
||||
|
||||
@ -429,3 +432,50 @@ void ZBarrierSetC2::compute_liveness_at_stubs() const {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const TypeFunc *oop_clone_Type() {
|
||||
// create input type (domain)
|
||||
const Type **fields = TypeTuple::fields(3);
|
||||
fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // src Object
|
||||
fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL; // dst Object
|
||||
fields[TypeFunc::Parms+2] = TypeInt::INT; // Object size
|
||||
const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+3, fields);
|
||||
|
||||
// create result type (range)
|
||||
fields = TypeTuple::fields(0);
|
||||
|
||||
const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
|
||||
|
||||
return TypeFunc::make(domain, range);
|
||||
}
|
||||
|
||||
void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
|
||||
Node *ctrl = ac->in(TypeFunc::Control);
|
||||
Node *mem = ac->in(TypeFunc::Memory);
|
||||
Node *src = ac->in(ArrayCopyNode::Src);
|
||||
Node *src_offset = ac->in(ArrayCopyNode::SrcPos);
|
||||
Node *dest = ac->in(ArrayCopyNode::Dest);
|
||||
Node *dest_offset = ac->in(ArrayCopyNode::DestPos);
|
||||
Node *length = ac->in(ArrayCopyNode::Length);
|
||||
|
||||
assert (src_offset == NULL, "for clone offsets should be null");
|
||||
assert (dest_offset == NULL, "for clone offsets should be null");
|
||||
|
||||
if (src->bottom_type()->isa_instptr()) {
|
||||
// Instances must have all oop fiels healed before cloning - call runtime leaf
|
||||
const char *clonefunc_name = "clone_oop";
|
||||
address clonefunc_addr = ZBarrierSetRuntime::clone_oop_addr();
|
||||
const TypePtr *raw_adr_type = TypeRawPtr::BOTTOM;
|
||||
const TypeFunc *call_type = oop_clone_Type();
|
||||
|
||||
Node *call = phase->make_leaf_call(ctrl, mem, call_type, clonefunc_addr, clonefunc_name, raw_adr_type, src, dest,
|
||||
length);
|
||||
phase->transform_later(call);
|
||||
phase->igvn().replace_node(ac, call);
|
||||
} else {
|
||||
assert(src->bottom_type()->isa_aryptr() != NULL, "Only arrays");
|
||||
|
||||
// Clones of primitive arrays go here
|
||||
BarrierSetC2::clone_at_expansion(phase, ac);
|
||||
}
|
||||
}
|
||||
|
@ -86,6 +86,7 @@ public:
|
||||
virtual void late_barrier_analysis() const;
|
||||
virtual int estimate_stub_size() const;
|
||||
virtual void emit_stubs(CodeBuffer& cb) const;
|
||||
virtual void clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const;
|
||||
};
|
||||
|
||||
#endif // SHARE_GC_Z_C2_ZBARRIERSETC2_HPP
|
||||
|
@ -119,6 +119,7 @@ public:
|
||||
static oop load_barrier_on_oop_field(volatile narrowOop* p);
|
||||
static oop load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o);
|
||||
static void load_barrier_on_oop_array(volatile narrowOop* p, size_t length);
|
||||
static void clone_oop(volatile oop src, oop dst, size_t length);
|
||||
static oop load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o);
|
||||
static oop load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o);
|
||||
static oop weak_load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o);
|
||||
|
@ -175,6 +175,10 @@ inline void ZBarrier::load_barrier_on_oop_array(volatile oop* p, size_t length)
|
||||
}
|
||||
}
|
||||
|
||||
inline void ZBarrier::clone_oop(volatile oop src, oop dst, size_t length) {
|
||||
HeapAccess<>::clone(src, dst, length);
|
||||
}
|
||||
|
||||
// ON_WEAK barriers should only ever be applied to j.l.r.Reference.referents.
|
||||
inline void verify_on_weak(volatile oop* referent_addr) {
|
||||
#ifdef ASSERT
|
||||
|
@ -42,6 +42,10 @@ JRT_LEAF(void, ZBarrierSetRuntime::load_barrier_on_oop_array(oop* p, size_t leng
|
||||
ZBarrier::load_barrier_on_oop_array(p, length);
|
||||
JRT_END
|
||||
|
||||
JRT_LEAF(void, ZBarrierSetRuntime::clone_oop(oop src, oop dst, size_t length))
|
||||
ZBarrier::clone_oop(src, dst, length);
|
||||
JRT_END
|
||||
|
||||
address ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(DecoratorSet decorators) {
|
||||
if (decorators & ON_PHANTOM_OOP_REF) {
|
||||
return load_barrier_on_phantom_oop_field_preloaded_addr();
|
||||
@ -67,3 +71,7 @@ address ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr() {
|
||||
address ZBarrierSetRuntime::load_barrier_on_oop_array_addr() {
|
||||
return reinterpret_cast<address>(load_barrier_on_oop_array);
|
||||
}
|
||||
|
||||
address ZBarrierSetRuntime::clone_oop_addr() {
|
||||
return reinterpret_cast<address>(clone_oop);
|
||||
}
|
||||
|
@ -36,6 +36,7 @@ private:
|
||||
static oopDesc* load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p);
|
||||
static oopDesc* load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p);
|
||||
static void load_barrier_on_oop_array(oop* p, size_t length);
|
||||
static void clone_oop(oop src, oop dst, size_t length);
|
||||
|
||||
public:
|
||||
static address load_barrier_on_oop_field_preloaded_addr(DecoratorSet decorators);
|
||||
@ -43,6 +44,7 @@ public:
|
||||
static address load_barrier_on_weak_oop_field_preloaded_addr();
|
||||
static address load_barrier_on_phantom_oop_field_preloaded_addr();
|
||||
static address load_barrier_on_oop_array_addr();
|
||||
static address clone_oop_addr();
|
||||
};
|
||||
|
||||
#endif // SHARE_GC_Z_ZBARRIERSETRUNTIME_HPP
|
||||
|
@ -38,7 +38,7 @@ private:
|
||||
None, // not set yet
|
||||
ArrayCopy, // System.arraycopy()
|
||||
CloneBasic, // A clone that can be copied by 64 bit chunks
|
||||
CloneOop, // An oop array clone
|
||||
CloneOopArray, // An oop array clone
|
||||
CopyOf, // Arrays.copyOf()
|
||||
CopyOfRange // Arrays.copyOfRange()
|
||||
} _kind;
|
||||
@ -147,7 +147,7 @@ public:
|
||||
bool is_arraycopy() const { assert(_kind != None, "should bet set"); return _kind == ArrayCopy; }
|
||||
bool is_arraycopy_validated() const { assert(_kind != None, "should bet set"); return _kind == ArrayCopy && _arguments_validated; }
|
||||
bool is_clonebasic() const { assert(_kind != None, "should bet set"); return _kind == CloneBasic; }
|
||||
bool is_cloneoop() const { assert(_kind != None, "should bet set"); return _kind == CloneOop; }
|
||||
bool is_clone_oop_array() const { assert(_kind != None, "should bet set"); return _kind == CloneOopArray; }
|
||||
bool is_copyof() const { assert(_kind != None, "should bet set"); return _kind == CopyOf; }
|
||||
bool is_copyof_validated() const { assert(_kind != None, "should bet set"); return _kind == CopyOf && _arguments_validated; }
|
||||
bool is_copyofrange() const { assert(_kind != None, "should bet set"); return _kind == CopyOfRange; }
|
||||
@ -155,7 +155,7 @@ public:
|
||||
|
||||
void set_arraycopy(bool validated) { assert(_kind == None, "shouldn't bet set yet"); _kind = ArrayCopy; _arguments_validated = validated; }
|
||||
void set_clonebasic() { assert(_kind == None, "shouldn't bet set yet"); _kind = CloneBasic; }
|
||||
void set_cloneoop() { assert(_kind == None, "shouldn't bet set yet"); _kind = CloneOop; }
|
||||
void set_clone_oop_array() { assert(_kind == None, "shouldn't bet set yet"); _kind = CloneOopArray; }
|
||||
void set_copyof(bool validated) { assert(_kind == None, "shouldn't bet set yet"); _kind = CopyOf; _arguments_validated = validated; }
|
||||
void set_copyofrange(bool validated) { assert(_kind == None, "shouldn't bet set yet"); _kind = CopyOfRange; _arguments_validated = validated; }
|
||||
|
||||
|
@ -4234,10 +4234,7 @@ void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, b
|
||||
alloc->initialization()->set_complete_with_arraycopy();
|
||||
}
|
||||
|
||||
// Copy the fastest available way.
|
||||
// TODO: generate fields copies for small objects instead.
|
||||
Node* size = _gvn.transform(obj_size);
|
||||
|
||||
access_clone(obj, alloc_obj, size, is_array);
|
||||
|
||||
// Do not let reads from the cloned object float above the arraycopy.
|
||||
@ -4304,12 +4301,6 @@ bool LibraryCallKit::inline_native_clone(bool is_virtual) {
|
||||
}
|
||||
}
|
||||
|
||||
Node* obj_klass = load_object_klass(obj);
|
||||
const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
|
||||
const TypeOopPtr* toop = ((tklass != NULL)
|
||||
? tklass->as_instance_type()
|
||||
: TypeInstPtr::NOTNULL);
|
||||
|
||||
// Conservatively insert a memory barrier on all memory slices.
|
||||
// Do not let writes into the original float below the clone.
|
||||
insert_mem_bar(Op_MemBarCPUOrder);
|
||||
@ -4328,6 +4319,7 @@ bool LibraryCallKit::inline_native_clone(bool is_virtual) {
|
||||
PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
|
||||
record_for_igvn(result_reg);
|
||||
|
||||
Node* obj_klass = load_object_klass(obj);
|
||||
Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
|
||||
if (array_ctl != NULL) {
|
||||
// It's an array.
|
||||
@ -4349,7 +4341,7 @@ bool LibraryCallKit::inline_native_clone(bool is_virtual) {
|
||||
// Generate a direct call to the right arraycopy function(s).
|
||||
Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
|
||||
ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL, false);
|
||||
ac->set_cloneoop();
|
||||
ac->set_clone_oop_array();
|
||||
Node* n = _gvn.transform(ac);
|
||||
assert(n == ac, "cannot disappear");
|
||||
ac->connect_outputs(this);
|
||||
|
@ -505,7 +505,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode*
|
||||
|
||||
// We don't need a subtype check for validated copies and Object[].clone()
|
||||
bool skip_subtype_check = ac->is_arraycopy_validated() || ac->is_copyof_validated() ||
|
||||
ac->is_copyofrange_validated() || ac->is_cloneoop();
|
||||
ac->is_copyofrange_validated() || ac->is_clone_oop_array();
|
||||
if (!skip_subtype_check) {
|
||||
// Get the klass* for both src and dest
|
||||
Node* src_klass = ac->in(ArrayCopyNode::SrcKlass);
|
||||
@ -1096,7 +1096,7 @@ void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) {
|
||||
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
|
||||
bs->clone_at_expansion(this, ac);
|
||||
return;
|
||||
} else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_cloneoop()) {
|
||||
} else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_clone_oop_array()) {
|
||||
Node* mem = ac->in(TypeFunc::Memory);
|
||||
merge_mem = MergeMemNode::make(mem);
|
||||
transform_later(merge_mem);
|
||||
|
@ -290,6 +290,7 @@ public:
|
||||
const TypeF *isa_float_constant() const; // Returns NULL if not a FloatCon
|
||||
const TypeTuple *is_tuple() const; // Collection of fields, NOT a pointer
|
||||
const TypeAry *is_ary() const; // Array, NOT array pointer
|
||||
const TypeAry *isa_ary() const; // Returns NULL of not ary
|
||||
const TypeVect *is_vect() const; // Vector
|
||||
const TypeVect *isa_vect() const; // Returns NULL if not a Vector
|
||||
const TypePtr *is_ptr() const; // Asserts it is a ptr type
|
||||
@ -1615,6 +1616,10 @@ inline const TypeAry *Type::is_ary() const {
|
||||
return (TypeAry*)this;
|
||||
}
|
||||
|
||||
inline const TypeAry *Type::isa_ary() const {
|
||||
return ((_base == Array) ? (TypeAry*)this : NULL);
|
||||
}
|
||||
|
||||
inline const TypeVect *Type::is_vect() const {
|
||||
assert( _base >= VectorS && _base <= VectorZ, "Not a Vector" );
|
||||
return (TypeVect*)this;
|
||||
|
@ -43,12 +43,17 @@ public class Clone {
|
||||
private BitSet testObj1;
|
||||
private Date testObj2;
|
||||
private char[] testObj3;
|
||||
private char[] testObj4;
|
||||
private String[] testObj5;
|
||||
|
||||
@Setup
|
||||
public void setup() {
|
||||
testObj1 = new BitSet(10);
|
||||
testObj2 = new Date();
|
||||
testObj3 = new char[5];
|
||||
testObj4 = new char[311];
|
||||
String str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut";
|
||||
testObj5 = str.split(" ", -1);
|
||||
}
|
||||
|
||||
/** Calls clone on three different types. The types are java.util.BitSet, java.util.Date and char[]. */
|
||||
@ -59,5 +64,11 @@ public class Clone {
|
||||
bh.consume(testObj3.clone());
|
||||
}
|
||||
|
||||
@Benchmark
|
||||
public void cloneLarge(Blackhole bh) {
|
||||
bh.consume(testObj4.clone());
|
||||
bh.consume(testObj5.clone());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user