8232896: ZGC: Enable C2 clone intrinsic

Reviewed-by: pliden, kvn
This commit is contained in:
Nils Eliasson 2019-10-31 17:16:36 +01:00
parent 55f7c48828
commit 3ca7e3f6ff
13 changed files with 94 additions and 21 deletions

@ -786,9 +786,6 @@ bool vmIntrinsics::is_disabled_by_flags(vmIntrinsics::ID id) {
#endif // COMPILER1
#ifdef COMPILER2
case vmIntrinsics::_clone:
#if INCLUDE_ZGC
if (UseZGC) return true;
#endif
case vmIntrinsics::_copyOf:
case vmIntrinsics::_copyOfRange:
// These intrinsics use both the objectcopy and the arraycopy

@ -96,7 +96,6 @@ Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) cons
store = kit->store_to_memory(kit->control(), access.addr().node(), val.node(), access.type(),
access.addr().type(), mo, requires_atomic_access, unaligned, mismatched, unsafe);
access.set_raw_access(store);
} else {
assert(!requires_atomic_access, "not yet supported");
assert(access.is_opt_access(), "either parse or opt access");
@ -120,6 +119,8 @@ Node* BarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) cons
mm->set_memory_at(alias, st);
}
}
access.set_raw_access(store);
return store;
}
@ -153,7 +154,6 @@ Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) con
load = kit->make_load(control, adr, val_type, access.type(), adr_type, mo,
dep, requires_atomic_access, unaligned, mismatched, unsafe);
}
access.set_raw_access(load);
} else {
assert(!requires_atomic_access, "not yet supported");
assert(access.is_opt_access(), "either parse or opt access");
@ -165,6 +165,7 @@ Node* BarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) con
load = LoadNode::make(gvn, control, mem, adr, adr_type, val_type, access.type(), mo, dep, unaligned, mismatched);
load = gvn.transform(load);
}
access.set_raw_access(load);
return load;
}
@ -806,7 +807,8 @@ void BarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac
Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
Node* length = ac->in(ArrayCopyNode::Length);
assert (src_offset == NULL && dest_offset == NULL, "for clone offsets should be null");
assert (src_offset == NULL, "for clone offsets should be null");
assert (dest_offset == NULL, "for clone offsets should be null");
const char* copyfunc_name = "arraycopy";
address copyfunc_addr =

@ -27,14 +27,17 @@
#include "gc/z/zBarrierSet.hpp"
#include "gc/z/zBarrierSetAssembler.hpp"
#include "gc/z/zBarrierSetRuntime.hpp"
#include "opto/arraycopynode.hpp"
#include "opto/block.hpp"
#include "opto/compile.hpp"
#include "opto/graphKit.hpp"
#include "opto/machnode.hpp"
#include "opto/macro.hpp"
#include "opto/memnode.hpp"
#include "opto/node.hpp"
#include "opto/regalloc.hpp"
#include "opto/rootnode.hpp"
#include "opto/runtime.hpp"
#include "utilities/growableArray.hpp"
#include "utilities/macros.hpp"
@ -429,3 +432,50 @@ void ZBarrierSetC2::compute_liveness_at_stubs() const {
}
}
}
const TypeFunc *oop_clone_Type() {
// create input type (domain)
const Type **fields = TypeTuple::fields(3);
fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // src Object
fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL; // dst Object
fields[TypeFunc::Parms+2] = TypeInt::INT; // Object size
const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+3, fields);
// create result type (range)
fields = TypeTuple::fields(0);
const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
return TypeFunc::make(domain, range);
}
void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
Node *ctrl = ac->in(TypeFunc::Control);
Node *mem = ac->in(TypeFunc::Memory);
Node *src = ac->in(ArrayCopyNode::Src);
Node *src_offset = ac->in(ArrayCopyNode::SrcPos);
Node *dest = ac->in(ArrayCopyNode::Dest);
Node *dest_offset = ac->in(ArrayCopyNode::DestPos);
Node *length = ac->in(ArrayCopyNode::Length);
assert (src_offset == NULL, "for clone offsets should be null");
assert (dest_offset == NULL, "for clone offsets should be null");
if (src->bottom_type()->isa_instptr()) {
// Instances must have all oop fiels healed before cloning - call runtime leaf
const char *clonefunc_name = "clone_oop";
address clonefunc_addr = ZBarrierSetRuntime::clone_oop_addr();
const TypePtr *raw_adr_type = TypeRawPtr::BOTTOM;
const TypeFunc *call_type = oop_clone_Type();
Node *call = phase->make_leaf_call(ctrl, mem, call_type, clonefunc_addr, clonefunc_name, raw_adr_type, src, dest,
length);
phase->transform_later(call);
phase->igvn().replace_node(ac, call);
} else {
assert(src->bottom_type()->isa_aryptr() != NULL, "Only arrays");
// Clones of primitive arrays go here
BarrierSetC2::clone_at_expansion(phase, ac);
}
}

@ -86,6 +86,7 @@ public:
virtual void late_barrier_analysis() const;
virtual int estimate_stub_size() const;
virtual void emit_stubs(CodeBuffer& cb) const;
virtual void clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const;
};
#endif // SHARE_GC_Z_C2_ZBARRIERSETC2_HPP

@ -119,6 +119,7 @@ public:
static oop load_barrier_on_oop_field(volatile narrowOop* p);
static oop load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o);
static void load_barrier_on_oop_array(volatile narrowOop* p, size_t length);
static void clone_oop(volatile oop src, oop dst, size_t length);
static oop load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o);
static oop load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o);
static oop weak_load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o);

@ -175,6 +175,10 @@ inline void ZBarrier::load_barrier_on_oop_array(volatile oop* p, size_t length)
}
}
inline void ZBarrier::clone_oop(volatile oop src, oop dst, size_t length) {
HeapAccess<>::clone(src, dst, length);
}
// ON_WEAK barriers should only ever be applied to j.l.r.Reference.referents.
inline void verify_on_weak(volatile oop* referent_addr) {
#ifdef ASSERT

@ -42,6 +42,10 @@ JRT_LEAF(void, ZBarrierSetRuntime::load_barrier_on_oop_array(oop* p, size_t leng
ZBarrier::load_barrier_on_oop_array(p, length);
JRT_END
JRT_LEAF(void, ZBarrierSetRuntime::clone_oop(oop src, oop dst, size_t length))
ZBarrier::clone_oop(src, dst, length);
JRT_END
address ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(DecoratorSet decorators) {
if (decorators & ON_PHANTOM_OOP_REF) {
return load_barrier_on_phantom_oop_field_preloaded_addr();
@ -67,3 +71,7 @@ address ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr() {
address ZBarrierSetRuntime::load_barrier_on_oop_array_addr() {
return reinterpret_cast<address>(load_barrier_on_oop_array);
}
address ZBarrierSetRuntime::clone_oop_addr() {
return reinterpret_cast<address>(clone_oop);
}

@ -36,6 +36,7 @@ private:
static oopDesc* load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p);
static oopDesc* load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p);
static void load_barrier_on_oop_array(oop* p, size_t length);
static void clone_oop(oop src, oop dst, size_t length);
public:
static address load_barrier_on_oop_field_preloaded_addr(DecoratorSet decorators);
@ -43,6 +44,7 @@ public:
static address load_barrier_on_weak_oop_field_preloaded_addr();
static address load_barrier_on_phantom_oop_field_preloaded_addr();
static address load_barrier_on_oop_array_addr();
static address clone_oop_addr();
};
#endif // SHARE_GC_Z_ZBARRIERSETRUNTIME_HPP

@ -38,7 +38,7 @@ private:
None, // not set yet
ArrayCopy, // System.arraycopy()
CloneBasic, // A clone that can be copied by 64 bit chunks
CloneOop, // An oop array clone
CloneOopArray, // An oop array clone
CopyOf, // Arrays.copyOf()
CopyOfRange // Arrays.copyOfRange()
} _kind;
@ -147,7 +147,7 @@ public:
bool is_arraycopy() const { assert(_kind != None, "should bet set"); return _kind == ArrayCopy; }
bool is_arraycopy_validated() const { assert(_kind != None, "should bet set"); return _kind == ArrayCopy && _arguments_validated; }
bool is_clonebasic() const { assert(_kind != None, "should bet set"); return _kind == CloneBasic; }
bool is_cloneoop() const { assert(_kind != None, "should bet set"); return _kind == CloneOop; }
bool is_clone_oop_array() const { assert(_kind != None, "should bet set"); return _kind == CloneOopArray; }
bool is_copyof() const { assert(_kind != None, "should bet set"); return _kind == CopyOf; }
bool is_copyof_validated() const { assert(_kind != None, "should bet set"); return _kind == CopyOf && _arguments_validated; }
bool is_copyofrange() const { assert(_kind != None, "should bet set"); return _kind == CopyOfRange; }
@ -155,7 +155,7 @@ public:
void set_arraycopy(bool validated) { assert(_kind == None, "shouldn't bet set yet"); _kind = ArrayCopy; _arguments_validated = validated; }
void set_clonebasic() { assert(_kind == None, "shouldn't bet set yet"); _kind = CloneBasic; }
void set_cloneoop() { assert(_kind == None, "shouldn't bet set yet"); _kind = CloneOop; }
void set_clone_oop_array() { assert(_kind == None, "shouldn't bet set yet"); _kind = CloneOopArray; }
void set_copyof(bool validated) { assert(_kind == None, "shouldn't bet set yet"); _kind = CopyOf; _arguments_validated = validated; }
void set_copyofrange(bool validated) { assert(_kind == None, "shouldn't bet set yet"); _kind = CopyOfRange; _arguments_validated = validated; }

@ -4234,10 +4234,7 @@ void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, b
alloc->initialization()->set_complete_with_arraycopy();
}
// Copy the fastest available way.
// TODO: generate fields copies for small objects instead.
Node* size = _gvn.transform(obj_size);
access_clone(obj, alloc_obj, size, is_array);
// Do not let reads from the cloned object float above the arraycopy.
@ -4304,12 +4301,6 @@ bool LibraryCallKit::inline_native_clone(bool is_virtual) {
}
}
Node* obj_klass = load_object_klass(obj);
const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
const TypeOopPtr* toop = ((tklass != NULL)
? tklass->as_instance_type()
: TypeInstPtr::NOTNULL);
// Conservatively insert a memory barrier on all memory slices.
// Do not let writes into the original float below the clone.
insert_mem_bar(Op_MemBarCPUOrder);
@ -4328,6 +4319,7 @@ bool LibraryCallKit::inline_native_clone(bool is_virtual) {
PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM);
record_for_igvn(result_reg);
Node* obj_klass = load_object_klass(obj);
Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
if (array_ctl != NULL) {
// It's an array.
@ -4349,7 +4341,7 @@ bool LibraryCallKit::inline_native_clone(bool is_virtual) {
// Generate a direct call to the right arraycopy function(s).
Node* alloc = tightly_coupled_allocation(alloc_obj, NULL);
ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, alloc != NULL, false);
ac->set_cloneoop();
ac->set_clone_oop_array();
Node* n = _gvn.transform(ac);
assert(n == ac, "cannot disappear");
ac->connect_outputs(this);

@ -505,7 +505,7 @@ Node* PhaseMacroExpand::generate_arraycopy(ArrayCopyNode *ac, AllocateArrayNode*
// We don't need a subtype check for validated copies and Object[].clone()
bool skip_subtype_check = ac->is_arraycopy_validated() || ac->is_copyof_validated() ||
ac->is_copyofrange_validated() || ac->is_cloneoop();
ac->is_copyofrange_validated() || ac->is_clone_oop_array();
if (!skip_subtype_check) {
// Get the klass* for both src and dest
Node* src_klass = ac->in(ArrayCopyNode::SrcKlass);
@ -1096,7 +1096,7 @@ void PhaseMacroExpand::expand_arraycopy_node(ArrayCopyNode *ac) {
BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
bs->clone_at_expansion(this, ac);
return;
} else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_cloneoop()) {
} else if (ac->is_copyof() || ac->is_copyofrange() || ac->is_clone_oop_array()) {
Node* mem = ac->in(TypeFunc::Memory);
merge_mem = MergeMemNode::make(mem);
transform_later(merge_mem);

@ -290,6 +290,7 @@ public:
const TypeF *isa_float_constant() const; // Returns NULL if not a FloatCon
const TypeTuple *is_tuple() const; // Collection of fields, NOT a pointer
const TypeAry *is_ary() const; // Array, NOT array pointer
const TypeAry *isa_ary() const; // Returns NULL of not ary
const TypeVect *is_vect() const; // Vector
const TypeVect *isa_vect() const; // Returns NULL if not a Vector
const TypePtr *is_ptr() const; // Asserts it is a ptr type
@ -1615,6 +1616,10 @@ inline const TypeAry *Type::is_ary() const {
return (TypeAry*)this;
}
inline const TypeAry *Type::isa_ary() const {
return ((_base == Array) ? (TypeAry*)this : NULL);
}
inline const TypeVect *Type::is_vect() const {
assert( _base >= VectorS && _base <= VectorZ, "Not a Vector" );
return (TypeVect*)this;

@ -43,12 +43,17 @@ public class Clone {
private BitSet testObj1;
private Date testObj2;
private char[] testObj3;
private char[] testObj4;
private String[] testObj5;
@Setup
public void setup() {
testObj1 = new BitSet(10);
testObj2 = new Date();
testObj3 = new char[5];
testObj4 = new char[311];
String str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut";
testObj5 = str.split(" ", -1);
}
/** Calls clone on three different types. The types are java.util.BitSet, java.util.Date and char[]. */
@ -59,5 +64,11 @@ public class Clone {
bh.consume(testObj3.clone());
}
@Benchmark
public void cloneLarge(Blackhole bh) {
bh.consume(testObj4.clone());
bh.consume(testObj5.clone());
}
}