diff --git a/src/hotspot/share/gc/z/zForwarding.cpp b/src/hotspot/share/gc/z/zForwarding.cpp index d8a3913806c..4df6e9a2d81 100644 --- a/src/hotspot/share/gc/z/zForwarding.cpp +++ b/src/hotspot/share/gc/z/zForwarding.cpp @@ -73,7 +73,7 @@ void ZForwarding::in_place_relocation_finish() { if (_from_age == ZPageAge::old || _to_age != ZPageAge::old) { // Only do this for non-promoted pages, that still need to reset live map. // Done with iterating over the "from-page" view, so can now drop the _livemap. - _page->finalize_reset_for_in_place_relocation(); + _page->reset_livemap(); } // Disable relaxed ZHeap::is_in checks diff --git a/src/hotspot/share/gc/z/zPage.cpp b/src/hotspot/share/gc/z/zPage.cpp index bfeda56de37..dc40c5367c1 100644 --- a/src/hotspot/share/gc/z/zPage.cpp +++ b/src/hotspot/share/gc/z/zPage.cpp @@ -57,14 +57,9 @@ ZPage::ZPage(ZPageType type, const ZVirtualMemory& vmem, const ZPhysicalMemory& } ZPage* ZPage::clone_limited() const { - // Only copy type and memory layouts. Let the rest be lazily reconstructed when needed. - return new ZPage(_type, _virtual, _physical); -} - -ZPage* ZPage::clone_limited_promote_flipped() const { + // Only copy type and memory layouts, and also update _top. Let the rest be + // lazily reconstructed when needed. ZPage* const page = new ZPage(_type, _virtual, _physical); - - // The page is still filled with the same objects, need to retain the top pointer. page->_top = _top; return page; @@ -83,58 +78,30 @@ void ZPage::reset_seqnum() { Atomic::store(&_seqnum_other, ZGeneration::generation(_generation_id == ZGenerationId::young ? ZGenerationId::old : ZGenerationId::young)->seqnum()); } +void ZPage::remset_initialize() { + // Remsets should only be initialized once and only for old pages. + assert(!_remembered_set.is_initialized(), "Should not be initialized"); + assert(is_old(), "Only old pages need a remset"); + + _remembered_set.initialize(size()); +} + +void ZPage::remset_initialize_or_verify_cleared() { + assert(is_old(), "Only old pages need a remset"); + + if (_remembered_set.is_initialized()) { + verify_remset_cleared_current(); + verify_remset_cleared_previous(); + } else { + remset_initialize(); + } +} + void ZPage::remset_clear() { _remembered_set.clear_all(); } -void ZPage::verify_remset_after_reset(ZPageAge prev_age, ZPageResetType type) { - // Young-to-old reset - if (prev_age != ZPageAge::old) { - verify_remset_cleared_previous(); - verify_remset_cleared_current(); - return; - } - - // Old-to-old reset - switch (type) { - case ZPageResetType::InPlaceRelocation: - // Relocation failed and page is being compacted in-place. - // The remset bits are flipped each young mark start, so - // the verification code below needs to use the right remset. - if (ZGeneration::old()->active_remset_is_current()) { - verify_remset_cleared_previous(); - } else { - verify_remset_cleared_current(); - } - break; - - case ZPageResetType::FlipAging: - fatal("Should not have called this for old-to-old flipping"); - break; - - case ZPageResetType::Allocation: - verify_remset_cleared_previous(); - verify_remset_cleared_current(); - break; - }; -} - -void ZPage::reset_remembered_set() { - if (is_young()) { - // Remset not needed - return; - } - - // Clearing of remsets is done when freeing a page, so this code only - // needs to ensure the remset is initialized the first time a page - // becomes old. - if (!_remembered_set.is_initialized()) { - _remembered_set.initialize(size()); - } -} - -void ZPage::reset(ZPageAge age, ZPageResetType type) { - const ZPageAge prev_age = _age; +void ZPage::reset(ZPageAge age) { _age = age; _last_used = 0; @@ -143,27 +110,16 @@ void ZPage::reset(ZPageAge age, ZPageResetType type) { : ZGenerationId::young; reset_seqnum(); - - // Flip aged pages are still filled with the same objects, need to retain the top pointer. - if (type != ZPageResetType::FlipAging) { - _top = to_zoffset_end(start()); - } - - reset_remembered_set(); - verify_remset_after_reset(prev_age, type); - - if (type != ZPageResetType::InPlaceRelocation || (prev_age != ZPageAge::old && age == ZPageAge::old)) { - // Promoted in-place relocations reset the live map, - // because they clone the page. - _livemap.reset(); - } } -void ZPage::finalize_reset_for_in_place_relocation() { - // Now we're done iterating over the livemaps +void ZPage::reset_livemap() { _livemap.reset(); } +void ZPage::reset_top_for_allocation() { + _top = to_zoffset_end(start()); +} + void ZPage::reset_type_and_size(ZPageType type) { _type = type; _livemap.resize(object_max_count()); @@ -261,11 +217,11 @@ void ZPage::verify_remset_cleared_previous() const { } void ZPage::clear_remset_current() { - _remembered_set.clear_current(); + _remembered_set.clear_current(); } void ZPage::clear_remset_previous() { - _remembered_set.clear_previous(); + _remembered_set.clear_previous(); } void ZPage::swap_remset_bitmaps() { diff --git a/src/hotspot/share/gc/z/zPage.hpp b/src/hotspot/share/gc/z/zPage.hpp index 3c9b3deda9f..42e14f904bc 100644 --- a/src/hotspot/share/gc/z/zPage.hpp +++ b/src/hotspot/share/gc/z/zPage.hpp @@ -36,16 +36,6 @@ class ZGeneration; -enum class ZPageResetType { - // Normal allocation path - Allocation, - // Relocation failed and started to relocate in-place - InPlaceRelocation, - // Page was not selected for relocation, all objects - // stayed, but the page aged. - FlipAging, -}; - class ZPage : public CHeapObj { friend class VMStructs; friend class ZList; @@ -80,17 +70,13 @@ private: const ZGeneration* generation() const; void reset_seqnum(); - void reset_remembered_set(); ZPage* split_with_pmem(ZPageType type, const ZPhysicalMemory& pmem); - void verify_remset_after_reset(ZPageAge prev_age, ZPageResetType type); - public: ZPage(ZPageType type, const ZVirtualMemory& vmem, const ZPhysicalMemory& pmem); ZPage* clone_limited() const; - ZPage* clone_limited_promote_flipped() const; uint32_t object_max_count() const; size_t object_alignment_shift() const; @@ -126,10 +112,9 @@ public: uint64_t last_used() const; void set_last_used(); - void reset(ZPageAge age, ZPageResetType type); - - void finalize_reset_for_in_place_relocation(); - + void reset(ZPageAge age); + void reset_livemap(); + void reset_top_for_allocation(); void reset_type_and_size(ZPageType type); ZPage* retype(ZPageType type); @@ -170,6 +155,8 @@ public: void clear_remset_range_non_par_current(uintptr_t l_offset, size_t size); void swap_remset_bitmaps(); + void remset_initialize(); + void remset_initialize_or_verify_cleared(); void remset_clear(); ZBitMap::ReverseIterator remset_reverse_iterator_previous(); diff --git a/src/hotspot/share/gc/z/zPageAllocator.cpp b/src/hotspot/share/gc/z/zPageAllocator.cpp index b1fb1e48373..f5d8ae6e3d1 100644 --- a/src/hotspot/share/gc/z/zPageAllocator.cpp +++ b/src/hotspot/share/gc/z/zPageAllocator.cpp @@ -729,7 +729,12 @@ retry: // Reset page. This updates the page's sequence number and must // be done after we potentially blocked in a safepoint (stalled) // where the global sequence number was updated. - page->reset(age, ZPageResetType::Allocation); + page->reset(age); + page->reset_top_for_allocation(); + page->reset_livemap(); + if (age == ZPageAge::old) { + page->remset_initialize_or_verify_cleared(); + } // Update allocation statistics. Exclude gc relocations to avoid // artificial inflation of the allocation rate during relocation. diff --git a/src/hotspot/share/gc/z/zRelocate.cpp b/src/hotspot/share/gc/z/zRelocate.cpp index b55a1863bde..90209e4c622 100644 --- a/src/hotspot/share/gc/z/zRelocate.cpp +++ b/src/hotspot/share/gc/z/zRelocate.cpp @@ -843,7 +843,23 @@ private: // Promotions happen through a new cloned page ZPage* const to_page = promotion ? from_page->clone_limited() : from_page; - to_page->reset(to_age, ZPageResetType::InPlaceRelocation); + + // Reset page for in-place relocation + to_page->reset(to_age); + to_page->reset_top_for_allocation(); + if (promotion) { + to_page->remset_initialize(); + } + + // Verify that the inactive remset is clear when resetting the page for + // in-place relocation. + if (from_page->age() == ZPageAge::old) { + if (ZGeneration::old()->active_remset_is_current()) { + to_page->verify_remset_cleared_previous(); + } else { + to_page->verify_remset_cleared_current(); + } + } // Clear remset bits for all objects that were relocated // before this page became an in-place relocated page. @@ -1270,8 +1286,14 @@ public: prev_page->log_msg(promotion ? " (flip promoted)" : " (flip survived)"); // Setup to-space page - ZPage* const new_page = promotion ? prev_page->clone_limited_promote_flipped() : prev_page; - new_page->reset(to_age, ZPageResetType::FlipAging); + ZPage* const new_page = promotion ? prev_page->clone_limited() : prev_page; + + // Reset page for flip aging + new_page->reset(to_age); + new_page->reset_livemap(); + if (promotion) { + new_page->remset_initialize(); + } if (promotion) { ZGeneration::young()->flip_promote(prev_page, new_page); diff --git a/test/hotspot/gtest/gc/z/test_zForwarding.cpp b/test/hotspot/gtest/gc/z/test_zForwarding.cpp index 622c6d9d8f4..ff2b3ee01e1 100644 --- a/test/hotspot/gtest/gc/z/test_zForwarding.cpp +++ b/test/hotspot/gtest/gc/z/test_zForwarding.cpp @@ -225,7 +225,7 @@ public: const ZPhysicalMemory pmem(ZPhysicalMemorySegment(zoffset(0), ZPageSizeSmall, true)); ZPage page(ZPageType::small, vmem, pmem); - page.reset(ZPageAge::eden, ZPageResetType::Allocation); + page.reset(ZPageAge::eden); const size_t object_size = 16; const zaddress object = page.alloc_object(object_size);