From f857f8a092d12cc11b2d48a6c9d47a70ee04c88e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20=C3=96sterlund?= Date: Wed, 11 Jan 2023 09:33:05 +0000 Subject: [PATCH] 8299327: Allow super late barrier expansion of store barriers in C2 Reviewed-by: kvn, rcastanedalo --- src/hotspot/cpu/aarch64/aarch64.ad | 6 ++++-- src/hotspot/cpu/ppc/ppc.ad | 3 ++- src/hotspot/cpu/riscv/riscv.ad | 4 +++- src/hotspot/cpu/x86/x86_64.ad | 6 ++++-- src/hotspot/share/gc/shared/c2/barrierSetC2.hpp | 3 ++- src/hotspot/share/opto/graphKit.cpp | 6 ++++-- src/hotspot/share/opto/graphKit.hpp | 11 +++++++---- src/hotspot/share/opto/memnode.cpp | 4 +++- 8 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 950112f55c6..e6c2cc79ff1 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -7558,7 +7558,7 @@ instruct storeimmL0(immL0 zero, memory8 mem) instruct storeP(iRegP src, memory8 mem) %{ match(Set mem (StoreP mem src)); - predicate(!needs_releasing_store(n)); + predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0); ins_cost(INSN_COST); format %{ "str $src, $mem\t# ptr" %} @@ -7572,7 +7572,7 @@ instruct storeP(iRegP src, memory8 mem) instruct storeimmP0(immP0 zero, memory8 mem) %{ match(Set mem (StoreP mem zero)); - predicate(!needs_releasing_store(n)); + predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0); ins_cost(INSN_COST); format %{ "str zr, $mem\t# ptr" %} @@ -7973,6 +7973,7 @@ instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem) instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem) %{ match(Set mem (StoreP mem src)); + predicate(n->as_Store()->barrier_data() == 0); ins_cost(VOLATILE_REF_COST); format %{ "stlr $src, $mem\t# ptr" %} @@ -7985,6 +7986,7 @@ instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem) instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem) %{ match(Set mem (StoreP mem zero)); + predicate(n->as_Store()->barrier_data() == 0); ins_cost(VOLATILE_REF_COST); format %{ "stlr zr, $mem\t# ptr" %} diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index 0bcfc91c085..17d0dfc4e34 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -1,5 +1,5 @@ // -// Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2012, 2022 SAP SE. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // @@ -6537,6 +6537,7 @@ instruct storeNKlass(memory dst, iRegN_P2N src) %{ // Store Pointer instruct storeP(memoryAlg4 dst, iRegPsrc src) %{ match(Set dst (StoreP dst src)); + predicate(n->as_Store()->barrier_data() == 0); ins_cost(MEMORY_REF_COST); format %{ "STD $src, $dst \t// ptr" %} diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 735fbaa5d86..e3c0242cfd7 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -1,5 +1,5 @@ // -// Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved. // Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. // Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -5118,6 +5118,7 @@ instruct storeimmL0(immL0 zero, memory mem) instruct storeP(iRegP src, memory mem) %{ match(Set mem (StoreP mem src)); + predicate(n->as_Store()->barrier_data() == 0); ins_cost(STORE_COST); format %{ "sd $src, $mem\t# ptr, #@storeP" %} @@ -5133,6 +5134,7 @@ instruct storeP(iRegP src, memory mem) instruct storeimmP0(immP0 zero, memory mem) %{ match(Set mem (StoreP mem zero)); + predicate(n->as_Store()->barrier_data() == 0); ins_cost(STORE_COST); format %{ "sd zr, $mem\t# ptr, #@storeimmP0" %} diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad index 38326f50a5e..16cd035ac79 100644 --- a/src/hotspot/cpu/x86/x86_64.ad +++ b/src/hotspot/cpu/x86/x86_64.ad @@ -1,5 +1,5 @@ // -// Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved. +// Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved. // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. // // This code is free software; you can redistribute it and/or modify it @@ -6080,6 +6080,7 @@ instruct storeL(memory mem, rRegL src) // Store Pointer instruct storeP(memory mem, any_RegP src) %{ + predicate(n->as_Store()->barrier_data() == 0); match(Set mem (StoreP mem src)); ins_cost(125); // XXX @@ -6092,7 +6093,7 @@ instruct storeP(memory mem, any_RegP src) instruct storeImmP0(memory mem, immP0 zero) %{ - predicate(UseCompressedOops && (CompressedOops::base() == NULL)); + predicate(UseCompressedOops && (CompressedOops::base() == NULL) && n->as_Store()->barrier_data() == 0); match(Set mem (StoreP mem zero)); ins_cost(125); // XXX @@ -6106,6 +6107,7 @@ instruct storeImmP0(memory mem, immP0 zero) // Store NULL Pointer, mark word, or other simple pointer constant. instruct storeImmP(memory mem, immP31 src) %{ + predicate(n->as_Store()->barrier_data() == 0); match(Set mem (StoreP mem src)); ins_cost(150); // XXX diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp index 6238ae3234c..5c86dcc0c82 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -264,6 +264,7 @@ public: virtual void register_potential_barrier_node(Node* node) const { } virtual void unregister_potential_barrier_node(Node* node) const { } virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { } + virtual void eliminate_gc_barrier_data(Node* node) const { } virtual void enqueue_useful_gc_barrier(PhaseIterGVN* igvn, Node* node) const {} virtual void eliminate_useless_gc_barriers(Unique_Node_List &useful, Compile* C) const {} diff --git a/src/hotspot/share/opto/graphKit.cpp b/src/hotspot/share/opto/graphKit.cpp index bdab4d0e5d3..c7df38045c8 100644 --- a/src/hotspot/share/opto/graphKit.cpp +++ b/src/hotspot/share/opto/graphKit.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1550,7 +1550,8 @@ Node* GraphKit::store_to_memory(Node* ctl, Node* adr, Node *val, BasicType bt, bool require_atomic_access, bool unaligned, bool mismatched, - bool unsafe) { + bool unsafe, + int barrier_data) { assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" ); const TypePtr* adr_type = NULL; debug_only(adr_type = C->get_adr_type(adr_idx)); @@ -1565,6 +1566,7 @@ Node* GraphKit::store_to_memory(Node* ctl, Node* adr, Node *val, BasicType bt, if (unsafe) { st->as_Store()->set_unsafe_access(); } + st->as_Store()->set_barrier_data(barrier_data); st = _gvn.transform(st); set_memory(st, adr_idx); // Back-to-back stores can only remove intermediate store with DU info diff --git a/src/hotspot/share/opto/graphKit.hpp b/src/hotspot/share/opto/graphKit.hpp index 7ebb8b04baf..1cdeac78134 100644 --- a/src/hotspot/share/opto/graphKit.hpp +++ b/src/hotspot/share/opto/graphKit.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -568,13 +568,15 @@ class GraphKit : public Phase { bool require_atomic_access = false, bool unaligned = false, bool mismatched = false, - bool unsafe = false) { + bool unsafe = false, + int barrier_data = 0) { // This version computes alias_index from an address type assert(adr_type != NULL, "use other store_to_memory factory"); return store_to_memory(ctl, adr, val, bt, C->get_alias_index(adr_type), mo, require_atomic_access, - unaligned, mismatched, unsafe); + unaligned, mismatched, unsafe, + barrier_data); } // This is the base version which is given alias index // Return the new StoreXNode @@ -584,7 +586,8 @@ class GraphKit : public Phase { bool require_atomic_access = false, bool unaligned = false, bool mismatched = false, - bool unsafe = false); + bool unsafe = false, + int barrier_data = 0); // Perform decorated accesses diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp index a263b04ea25..bcd81e38ae9 100644 --- a/src/hotspot/share/opto/memnode.cpp +++ b/src/hotspot/share/opto/memnode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -4004,9 +4004,11 @@ Node* InitializeNode::capture_store(StoreNode* st, intptr_t start, ins_req(i, C->top()); // build a new edge } Node* new_st = st->clone(); + BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); new_st->set_req(MemNode::Control, in(Control)); new_st->set_req(MemNode::Memory, prev_mem); new_st->set_req(MemNode::Address, make_raw_address(start, phase)); + bs->eliminate_gc_barrier_data(new_st); new_st = phase->transform(new_st); // At this point, new_st might have swallowed a pre-existing store