diff --git a/src/hotspot/share/gc/z/zForwarding.hpp b/src/hotspot/share/gc/z/zForwarding.hpp index a99473322d4..ee37508903f 100644 --- a/src/hotspot/share/gc/z/zForwarding.hpp +++ b/src/hotspot/share/gc/z/zForwarding.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -83,6 +83,14 @@ private: ZForwardingEntry first(uintptr_t from_index, ZForwardingCursor* cursor) const; ZForwardingEntry next(ZForwardingCursor* cursor) const; + uintptr_t index(zoffset from_offset); + + ZForwardingEntry find(uintptr_t from_index, ZForwardingCursor* cursor) const; + zaddress find(zoffset from_offset, ZForwardingCursor* cursor); + + zoffset insert(uintptr_t from_index, zoffset to_offset, ZForwardingCursor* cursor); + zaddress insert(zoffset from_offset, zaddress to_addr, ZForwardingCursor* cursor); + template void object_iterate_forwarded_via_livemap(Function function); @@ -140,10 +148,11 @@ public: void mark_done(); bool is_done() const; - zaddress find(zaddress_unsafe addr); + zaddress find(zaddress from_addr, ZForwardingCursor* cursor); + zaddress find(zaddress_unsafe from_addr, ZForwardingCursor* cursor); + zaddress find(zaddress_unsafe from_addr); - ZForwardingEntry find(uintptr_t from_index, ZForwardingCursor* cursor) const; - zoffset insert(uintptr_t from_index, zoffset to_offset, ZForwardingCursor* cursor); + zaddress insert(zaddress from_addr, zaddress to_addr, ZForwardingCursor* cursor); // Relocated remembered set fields support void relocated_remembered_fields_register(volatile zpointer* p); diff --git a/src/hotspot/share/gc/z/zForwarding.inline.hpp b/src/hotspot/share/gc/z/zForwarding.inline.hpp index b0cb67a70cd..eb5f4a36161 100644 --- a/src/hotspot/share/gc/z/zForwarding.inline.hpp +++ b/src/hotspot/share/gc/z/zForwarding.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -218,11 +218,8 @@ inline ZForwardingEntry ZForwarding::next(ZForwardingCursor* cursor) const { return at(cursor); } -inline zaddress ZForwarding::find(zaddress_unsafe addr) { - const uintptr_t from_index = (ZAddress::offset(addr) - start()) >> object_alignment_shift(); - ZForwardingCursor cursor; - const ZForwardingEntry entry = find(from_index, &cursor); - return entry.populated() ? ZOffset::address(to_zoffset(entry.to_offset())) : zaddress::null; +inline uintptr_t ZForwarding::index(zoffset from_offset) { + return (from_offset - start()) >> object_alignment_shift(); } inline ZForwardingEntry ZForwarding::find(uintptr_t from_index, ZForwardingCursor* cursor) const { @@ -243,6 +240,25 @@ inline ZForwardingEntry ZForwarding::find(uintptr_t from_index, ZForwardingCurso return entry; } +inline zaddress ZForwarding::find(zoffset from_offset, ZForwardingCursor* cursor) { + const uintptr_t from_index = index(from_offset); + const ZForwardingEntry entry = find(from_index, cursor); + return entry.populated() ? ZOffset::address(to_zoffset(entry.to_offset())) : zaddress::null; +} + +inline zaddress ZForwarding::find(zaddress from_addr, ZForwardingCursor* cursor) { + return find(ZAddress::offset(from_addr), cursor); +} + +inline zaddress ZForwarding::find(zaddress_unsafe from_addr, ZForwardingCursor* cursor) { + return find(ZAddress::offset(from_addr), cursor); +} + +inline zaddress ZForwarding::find(zaddress_unsafe from_addr) { + ZForwardingCursor cursor; + return find(from_addr, &cursor); +} + inline zoffset ZForwarding::insert(uintptr_t from_index, zoffset to_offset, ZForwardingCursor* cursor) { const ZForwardingEntry new_entry(from_index, untype(to_offset)); const ZForwardingEntry old_entry; // Empty @@ -271,6 +287,17 @@ inline zoffset ZForwarding::insert(uintptr_t from_index, zoffset to_offset, ZFor } } +inline zaddress ZForwarding::insert(zoffset from_offset, zaddress to_addr, ZForwardingCursor* cursor) { + const uintptr_t from_index = index(from_offset); + const zoffset to_offset = ZAddress::offset(to_addr); + const zoffset to_offset_final = insert(from_index, to_offset, cursor); + return ZOffset::address(to_offset_final); +} + +inline zaddress ZForwarding::insert(zaddress from_addr, zaddress to_addr, ZForwardingCursor* cursor) { + return insert(ZAddress::offset(from_addr), to_addr, cursor); +} + inline void ZForwarding::relocated_remembered_fields_register(volatile zpointer* p) { // Invariant: Page is being retained assert(ZGeneration::young()->is_phase_mark(), "Only called when"); diff --git a/src/hotspot/share/gc/z/zRelocate.cpp b/src/hotspot/share/gc/z/zRelocate.cpp index 78efa7cdb12..b55a1863bde 100644 --- a/src/hotspot/share/gc/z/zRelocate.cpp +++ b/src/hotspot/share/gc/z/zRelocate.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -52,35 +52,6 @@ static const ZStatCriticalPhase ZCriticalPhaseRelocationStall("Relocation Stall"); static const ZStatSubPhase ZSubPhaseConcurrentRelocateRememberedSetFlipPromotedYoung("Concurrent Relocate Remset FP", ZGenerationId::young); -static uintptr_t forwarding_index(ZForwarding* forwarding, zoffset from_offset) { - return (from_offset - forwarding->start()) >> forwarding->object_alignment_shift(); -} - -static zaddress forwarding_find(ZForwarding* forwarding, zoffset from_offset, ZForwardingCursor* cursor) { - const uintptr_t from_index = forwarding_index(forwarding, from_offset); - const ZForwardingEntry entry = forwarding->find(from_index, cursor); - return entry.populated() ? ZOffset::address(to_zoffset(entry.to_offset())) : zaddress::null; -} - -static zaddress forwarding_find(ZForwarding* forwarding, zaddress_unsafe from_addr, ZForwardingCursor* cursor) { - return forwarding_find(forwarding, ZAddress::offset(from_addr), cursor); -} - -static zaddress forwarding_find(ZForwarding* forwarding, zaddress from_addr, ZForwardingCursor* cursor) { - return forwarding_find(forwarding, ZAddress::offset(from_addr), cursor); -} - -static zaddress forwarding_insert(ZForwarding* forwarding, zoffset from_offset, zaddress to_addr, ZForwardingCursor* cursor) { - const uintptr_t from_index = forwarding_index(forwarding, from_offset); - const zoffset to_offset = ZAddress::offset(to_addr); - const zoffset to_offset_final = forwarding->insert(from_index, to_offset, cursor); - return ZOffset::address(to_offset_final); -} - -static zaddress forwarding_insert(ZForwarding* forwarding, zaddress from_addr, zaddress to_addr, ZForwardingCursor* cursor) { - return forwarding_insert(forwarding, ZAddress::offset(from_addr), to_addr, cursor); -} - ZRelocateQueue::ZRelocateQueue() : _lock(), _queue(), @@ -368,7 +339,7 @@ static zaddress relocate_object_inner(ZForwarding* forwarding, zaddress from_add ZUtils::object_copy_disjoint(from_addr, to_addr, size); // Insert forwarding - const zaddress to_addr_final = forwarding_insert(forwarding, from_addr, to_addr, cursor); + const zaddress to_addr_final = forwarding->insert(from_addr, to_addr, cursor); if (to_addr_final != to_addr) { // Already relocated, try undo allocation @@ -382,7 +353,7 @@ zaddress ZRelocate::relocate_object(ZForwarding* forwarding, zaddress_unsafe fro ZForwardingCursor cursor; // Lookup forwarding - zaddress to_addr = forwarding_find(forwarding, from_addr, &cursor); + zaddress to_addr = forwarding->find(from_addr, &cursor); if (!is_null(to_addr)) { // Already relocated return to_addr; @@ -409,8 +380,7 @@ zaddress ZRelocate::relocate_object(ZForwarding* forwarding, zaddress_unsafe fro } zaddress ZRelocate::forward_object(ZForwarding* forwarding, zaddress_unsafe from_addr) { - ZForwardingCursor cursor; - const zaddress to_addr = forwarding_find(forwarding, from_addr, &cursor); + const zaddress to_addr = forwarding->find(from_addr); assert(!is_null(to_addr), "Should be forwarded: " PTR_FORMAT, untype(from_addr)); return to_addr; } @@ -626,7 +596,7 @@ private: // Lookup forwarding { - const zaddress to_addr = forwarding_find(_forwarding, from_addr, &cursor); + const zaddress to_addr = _forwarding->find(from_addr, &cursor); if (!is_null(to_addr)) { // Already relocated increase_other_forwarded(size); @@ -650,7 +620,7 @@ private: } // Insert forwarding - const zaddress to_addr = forwarding_insert(_forwarding, from_addr, allocated_addr, &cursor); + const zaddress to_addr = _forwarding->insert(from_addr, allocated_addr, &cursor); if (to_addr != allocated_addr) { // Already relocated, undo allocation _allocator->undo_alloc_object(to_page, to_addr, size);