8220600: ZGC: Delete ZPages using ZSafeDelete

Reviewed-by: stefank, eosterlund

src/hotspot/share/gc/z/zDriver.cpp

@@ -43,7 +43,6 @@ static const ZStatPhaseConcurrent ZPhaseConcurrentMarkContinue("Concurrent Mark
 static const ZStatPhasePause      ZPhasePauseMarkEnd("Pause Mark End");
 static const ZStatPhaseConcurrent ZPhaseConcurrentProcessNonStrongReferences("Concurrent Process Non-Strong References");
 static const ZStatPhaseConcurrent ZPhaseConcurrentResetRelocationSet("Concurrent Reset Relocation Set");
-static const ZStatPhaseConcurrent ZPhaseConcurrentDestroyDetachedPages("Concurrent Destroy Detached Pages");
 static const ZStatPhaseConcurrent ZPhaseConcurrentSelectRelocationSet("Concurrent Select Relocation Set");
 static const ZStatPhasePause      ZPhasePauseRelocateStart("Pause Relocate Start");
 static const ZStatPhaseConcurrent ZPhaseConcurrentRelocated("Concurrent Relocate");
@@ -299,11 +298,6 @@ void ZDriver::concurrent_reset_relocation_set() {
   ZHeap::heap()->reset_relocation_set();
 }
 
-void ZDriver::concurrent_destroy_detached_pages() {
-  ZStatTimer timer(ZPhaseConcurrentDestroyDetachedPages);
-  ZHeap::heap()->destroy_detached_pages();
-}
-
 void ZDriver::pause_verify() {
   if (VerifyBeforeGC || VerifyDuringGC || VerifyAfterGC) {
     VM_Verify op;
@@ -378,19 +372,16 @@ void ZDriver::gc(GCCause::Cause cause) {
   // Phase 5: Concurrent Reset Relocation Set
   concurrent_reset_relocation_set();
 
-  // Phase 6: Concurrent Destroy Detached Pages
-  concurrent_destroy_detached_pages();
-
-  // Phase 7: Pause Verify
+  // Phase 6: Pause Verify
   pause_verify();
 
-  // Phase 8: Concurrent Select Relocation Set
+  // Phase 7: Concurrent Select Relocation Set
   concurrent_select_relocation_set();
 
-  // Phase 9: Pause Relocate Start
+  // Phase 8: Pause Relocate Start
   pause_relocate_start();
 
-  // Phase 10: Concurrent Relocate
+  // Phase 9: Concurrent Relocate
   concurrent_relocate();
 }
 

src/hotspot/share/gc/z/zDriver.hpp

@@ -43,7 +43,6 @@ private:
   void concurrent_mark_continue();
   void concurrent_process_non_strong_references();
   void concurrent_reset_relocation_set();
-  void concurrent_destroy_detached_pages();
   void pause_verify();
   void concurrent_select_relocation_set();
   void pause_relocate_start();

src/hotspot/share/gc/z/zHeap.cpp

@@ -403,6 +403,9 @@ void ZHeap::process_non_strong_references() {
 }
 
 void ZHeap::select_relocation_set() {
+  // Do not allow pages to be deleted
+  _page_allocator.enable_deferred_delete();
+
   // Register relocatable pages with selector
   ZRelocationSetSelector selector;
   ZPageTableIterator pt_iter(&_page_table);
@@ -424,6 +427,9 @@ void ZHeap::select_relocation_set() {
     }
   }
 
+  // Allow pages to be deleted
+  _page_allocator.disable_deferred_delete();
+
   // Select pages to relocate
   selector.select(&_relocation_set);
 
@@ -451,10 +457,6 @@ void ZHeap::reset_relocation_set() {
   _relocation_set.reset();
 }
 
-void ZHeap::destroy_detached_pages() {
-  _page_allocator.destroy_detached_pages();
-}
-
 void ZHeap::relocate_start() {
   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 
@@ -521,11 +523,18 @@ void ZHeap::print_extended_on(outputStream* st) const {
   print_on(st);
   st->cr();
 
+  // Do not allow pages to be deleted
+  _page_allocator.enable_deferred_delete();
+
+  // Print all pages
   ZPageTableIterator iter(&_page_table);
   for (ZPage* page; iter.next(&page);) {
     page->print_on(st);
   }
 
+  // Allow pages to be deleted
+  _page_allocator.enable_deferred_delete();
+
   st->cr();
 }
 

src/hotspot/share/gc/z/zHeap.hpp

@@ -144,9 +144,6 @@ public:
   void mark_flush_and_free(Thread* thread);
   bool mark_end();
 
-  // Post-marking & Pre-relocation
-  void destroy_detached_pages();
-
   // Relocation set
   void select_relocation_set();
   void reset_relocation_set();

src/hotspot/share/gc/z/zPageAllocator.cpp

@@ -31,6 +31,7 @@
 #include "gc/z/zPageAllocator.hpp"
 #include "gc/z/zPageCache.inline.hpp"
 #include "gc/z/zPreMappedMemory.inline.hpp"
+#include "gc/z/zSafeDelete.inline.hpp"
 #include "gc/z/zStat.hpp"
 #include "gc/z/zTracer.inline.hpp"
 #include "runtime/init.hpp"
@@ -96,7 +97,7 @@ ZPageAllocator::ZPageAllocator(size_t min_capacity, size_t max_capacity, size_t
     _allocated(0),
     _reclaimed(0),
     _queue(),
-    _detached() {}
+    _safe_delete() {}
 
 bool ZPageAllocator::is_initialized() const {
   return _physical.is_initialized() &&
@@ -242,27 +243,12 @@ void ZPageAllocator::flush_pre_mapped() {
   _pre_mapped.clear();
 }
 
-void ZPageAllocator::detach_page(ZPage* page) {
-  // Detach the memory mapping.
+void ZPageAllocator::destroy_page(ZPage* page) {
+  // Detach virtual and physical memory
   detach_memory(page->virtual_memory(), page->physical_memory());
 
-  // Add to list of detached pages
-  _detached.insert_last(page);
-}
-
-void ZPageAllocator::destroy_detached_pages() {
-  ZList<ZPage> list;
-
-  // Get and reset list of detached pages
-  {
-    ZLocker<ZLock> locker(&_lock);
-    list.transfer(&_detached);
-  }
-
-  // Destroy pages
-  for (ZPage* page = list.remove_first(); page != NULL; page = list.remove_first()) {
-    delete page;
-  }
+  // Delete page safely
+  _safe_delete(page);
 }
 
 void ZPageAllocator::map_page(ZPage* page) {
@@ -286,7 +272,7 @@ void ZPageAllocator::flush_cache(size_t size) {
   _cache.flush(&list, size);
 
   for (ZPage* page = list.remove_first(); page != NULL; page = list.remove_first()) {
-    detach_page(page);
+    destroy_page(page);
   }
 }
 
@@ -477,6 +463,14 @@ void ZPageAllocator::free_page(ZPage* page, bool reclaimed) {
   satisfy_alloc_queue();
 }
 
+void ZPageAllocator::enable_deferred_delete() const {
+  _safe_delete.enable_deferred_delete();
+}
+
+void ZPageAllocator::disable_deferred_delete() const {
+  _safe_delete.disable_deferred_delete();
+}
+
 bool ZPageAllocator::is_alloc_stalled() const {
   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
   return !_queue.is_empty();

src/hotspot/share/gc/z/zPageAllocator.hpp

@@ -30,6 +30,7 @@
 #include "gc/z/zPageCache.hpp"
 #include "gc/z/zPhysicalMemory.hpp"
 #include "gc/z/zPreMappedMemory.hpp"
+#include "gc/z/zSafeDelete.hpp"
 #include "gc/z/zVirtualMemory.hpp"
 #include "memory/allocation.hpp"
 
@@ -39,19 +40,19 @@ class ZPageAllocator {
   friend class VMStructs;
 
 private:
-  ZLock                    _lock;
-  ZVirtualMemoryManager    _virtual;
-  ZPhysicalMemoryManager   _physical;
-  ZPageCache               _cache;
-  const size_t             _max_reserve;
-  ZPreMappedMemory         _pre_mapped;
-  size_t                   _used_high;
-  size_t                   _used_low;
-  size_t                   _used;
-  size_t                   _allocated;
-  ssize_t                  _reclaimed;
-  ZList<ZPageAllocRequest> _queue;
-  ZList<ZPage>             _detached;
+  ZLock                      _lock;
+  ZVirtualMemoryManager      _virtual;
+  ZPhysicalMemoryManager     _physical;
+  ZPageCache                 _cache;
+  const size_t               _max_reserve;
+  ZPreMappedMemory           _pre_mapped;
+  size_t                     _used_high;
+  size_t                     _used_low;
+  size_t                     _used;
+  size_t                     _allocated;
+  ssize_t                    _reclaimed;
+  ZList<ZPageAllocRequest>   _queue;
+  mutable ZSafeDelete<ZPage> _safe_delete;
 
   static ZPage* const      gc_marker;
 
@@ -63,7 +64,8 @@ private:
   size_t try_ensure_unused_for_pre_mapped(size_t size);
 
   ZPage* create_page(uint8_t type, size_t size);
-  void detach_page(ZPage* page);
+  void destroy_page(ZPage* page);
+
   void flush_pre_mapped();
   void flush_cache(size_t size);
 
@@ -97,7 +99,9 @@ public:
 
   ZPage* alloc_page(uint8_t type, size_t size, ZAllocationFlags flags);
   void free_page(ZPage* page, bool reclaimed);
-  void destroy_detached_pages();
+
+  void enable_deferred_delete() const;
+  void disable_deferred_delete() const;
 
   void map_page(ZPage* page);
   void unmap_all_pages();