8220588: ZGC: Convert ZRelocationSet to hold ZForwardings instead of ZPages

Reviewed-by: stefank, eosterlund

src/hotspot/share/gc/z/zDriver.cpp

@@ -45,7 +45,6 @@ static const ZStatPhaseConcurrent ZPhaseConcurrentProcessNonStrongReferences("Co
 static const ZStatPhaseConcurrent ZPhaseConcurrentResetRelocationSet("Concurrent Reset Relocation Set");
 static const ZStatPhaseConcurrent ZPhaseConcurrentDestroyDetachedPages("Concurrent Destroy Detached Pages");
 static const ZStatPhaseConcurrent ZPhaseConcurrentSelectRelocationSet("Concurrent Select Relocation Set");
-static const ZStatPhaseConcurrent ZPhaseConcurrentPrepareRelocationSet("Concurrent Prepare Relocation Set");
 static const ZStatPhasePause      ZPhasePauseRelocateStart("Pause Relocate Start");
 static const ZStatPhaseConcurrent ZPhaseConcurrentRelocated("Concurrent Relocate");
 static const ZStatCriticalPhase   ZCriticalPhaseGCLockerStall("GC Locker Stall", false /* verbose */);
@@ -317,11 +316,6 @@ void ZDriver::concurrent_select_relocation_set() {
   ZHeap::heap()->select_relocation_set();
 }
 
-void ZDriver::concurrent_prepare_relocation_set() {
-  ZStatTimer timer(ZPhaseConcurrentPrepareRelocationSet);
-  ZHeap::heap()->prepare_relocation_set();
-}
-
 void ZDriver::pause_relocate_start() {
   pause<VM_ZRelocateStart>();
 }
@@ -393,13 +387,10 @@ void ZDriver::gc(GCCause::Cause cause) {
   // Phase 8: Concurrent Select Relocation Set
   concurrent_select_relocation_set();
 
-  // Phase 9: Concurrent Prepare Relocation Set
+  // Phase 9: Pause Relocate Start
-  concurrent_prepare_relocation_set();
-
-  // Phase 10: Pause Relocate Start
   pause_relocate_start();
 
-  // Phase 11: Concurrent Relocate
+  // Phase 10: Concurrent Relocate
   concurrent_relocate();
 }
 

src/hotspot/share/gc/z/zDriver.hpp

@@ -46,7 +46,6 @@ private:
   void concurrent_destroy_detached_pages();
   void pause_verify();
   void concurrent_select_relocation_set();
-  void concurrent_prepare_relocation_set();
   void pause_relocate_start();
   void concurrent_relocate();
 

src/hotspot/share/gc/z/zForwarding.cpp

@@ -23,30 +23,32 @@
 
 #include "precompiled.hpp"
 #include "gc/z/zForwarding.inline.hpp"
+#include "gc/z/zPage.inline.hpp"
 #include "gc/z/zUtils.inline.hpp"
 #include "memory/allocation.hpp"
 #include "utilities/debug.hpp"
 
-ZForwarding::ZForwarding(uintptr_t start, size_t object_alignment_shift, uint32_t nentries) :
+ZForwarding::ZForwarding(ZPage* page, uint32_t nentries) :
-    _start(start),
+    _virtual(page->virtual_memory()),
-    _object_alignment_shift(object_alignment_shift),
+    _object_alignment_shift(page->object_alignment_shift()),
     _nentries(nentries),
+    _page(page),
     _refcount(1),
     _pinned(false) {}
 
-ZForwarding* ZForwarding::create(uintptr_t start, size_t object_alignment_shift, uint32_t live_objects) {
+ZForwarding* ZForwarding::create(ZPage* page) {
-  assert(live_objects > 0, "Invalid value");
+  assert(page->live_objects() > 0, "Invalid value");
 
   // Allocate table for linear probing. The size of the table must be
   // a power of two to allow for quick and inexpensive indexing/masking.
   // The table is sized to have a load factor of 50%, i.e. sized to have
   // double the number of entries actually inserted.
-  const uint32_t nentries = ZUtils::round_up_power_of_2(live_objects * 2);
+  const uint32_t nentries = ZUtils::round_up_power_of_2(page->live_objects() * 2);
-
   const size_t size = sizeof(ZForwarding) + (sizeof(ZForwardingEntry) * nentries);
   uint8_t* const addr = NEW_C_HEAP_ARRAY(uint8_t, size, mtGC);
   ZForwardingEntry* const entries = ::new (addr + sizeof(ZForwarding)) ZForwardingEntry[nentries];
-  ZForwarding* const forwarding = ::new (addr) ZForwarding(start, object_alignment_shift, nentries);
+  ZForwarding* const forwarding = ::new (addr) ZForwarding(page, nentries);
+
   return forwarding;
 }
 
@@ -54,8 +56,11 @@ void ZForwarding::destroy(ZForwarding* forwarding) {
   FREE_C_HEAP_ARRAY(uint8_t, forwarding);
 }
 
-void ZForwarding::verify(uint32_t object_max_count, uint32_t live_objects) const {
+void ZForwarding::verify() const {
-  uint32_t count = 0;
+  guarantee(_refcount > 0, "Invalid refcount");
+  guarantee(_page != NULL, "Invalid page");
+
+  uint32_t live_objects = 0;
 
   for (ZForwardingCursor i = 0; i < _nentries; i++) {
     const ZForwardingEntry entry = at(&i);
@@ -65,7 +70,7 @@ void ZForwarding::verify(uint32_t object_max_count, uint32_t live_objects) const
     }
 
     // Check from index
-    guarantee(entry.from_index() < object_max_count, "Invalid from index");
+    guarantee(entry.from_index() < _page->object_max_count(), "Invalid from index");
 
     // Check for duplicates
     for (ZForwardingCursor j = i + 1; j < _nentries; j++) {
@@ -74,9 +79,9 @@ void ZForwarding::verify(uint32_t object_max_count, uint32_t live_objects) const
       guarantee(entry.to_offset() != other.to_offset(), "Duplicate to");
     }
 
-    count++;
+    live_objects++;
   }
 
-  // Check number of non-null entries
+  // Check number of non-empty entries
-  guarantee(live_objects == count, "Count mismatch");
+  guarantee(live_objects == _page->live_objects(), "Invalid number of entries");
 }

src/hotspot/share/gc/z/zForwarding.hpp

@@ -25,6 +25,9 @@
 #define SHARE_GC_Z_ZFORWARDING_HPP
 
 #include "gc/z/zForwardingEntry.hpp"
+#include "gc/z/zVirtualMemory.hpp"
+
+class ZPage;
 
 typedef uint32_t ZForwardingCursor;
 
@@ -33,37 +36,43 @@ class ZForwarding {
   friend class ZForwardingTest;
 
 private:
-  const uintptr_t   _start;
+  const ZVirtualMemory _virtual;
-  const size_t      _object_alignment_shift;
+  const size_t         _object_alignment_shift;
-  const uint32_t    _nentries;
+  const uint32_t       _nentries;
-  volatile uint32_t _refcount;
+  ZPage*               _page;
-  volatile bool     _pinned;
+  volatile uint32_t    _refcount;
+  volatile bool        _pinned;
+
+  bool inc_refcount();
+  bool dec_refcount();
 
   ZForwardingEntry* entries() const;
   ZForwardingEntry at(ZForwardingCursor* cursor) const;
   ZForwardingEntry first(uintptr_t from_index, ZForwardingCursor* cursor) const;
   ZForwardingEntry next(ZForwardingCursor* cursor) const;
 
-  ZForwarding(uintptr_t start, size_t object_alignment_shift, uint32_t nentries);
+  ZForwarding(ZPage* page, uint32_t nentries);
 
 public:
-  static ZForwarding* create(uintptr_t start, size_t object_alignment_shift, uint32_t live_objects);
+  static ZForwarding* create(ZPage* page);
   static void destroy(ZForwarding* forwarding);
 
   uintptr_t start() const;
+  size_t size() const;
   size_t object_alignment_shift() const;
-
+  ZPage* page() const;
-  bool inc_refcount();
-  bool dec_refcount();
 
   bool is_pinned() const;
   void set_pinned();
 
+  bool retain_page();
+  void release_page();
+
   ZForwardingEntry find(uintptr_t from_index) const;
   ZForwardingEntry find(uintptr_t from_index, ZForwardingCursor* cursor) const;
   uintptr_t insert(uintptr_t from_index, uintptr_t to_offset, ZForwardingCursor* cursor);
 
-  void verify(uint32_t object_max_count, uint32_t live_objects) const;
+  void verify() const;
 };
 
 #endif // SHARE_GC_Z_ZFORWARDING_HPP

src/hotspot/share/gc/z/zForwarding.inline.hpp

@@ -27,17 +27,35 @@
 #include "gc/z/zForwarding.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zHash.inline.hpp"
+#include "gc/z/zHeap.hpp"
+#include "gc/z/zVirtualMemory.inline.hpp"
 #include "runtime/atomic.hpp"
 #include "utilities/debug.hpp"
 
 inline uintptr_t ZForwarding::start() const {
-  return _start;
+  return _virtual.start();
+}
+
+inline size_t ZForwarding::size() const {
+  return _virtual.size();
 }
 
 inline size_t ZForwarding::object_alignment_shift() const {
   return _object_alignment_shift;
 }
 
+inline ZPage* ZForwarding::page() const {
+  return _page;
+}
+
+inline bool ZForwarding::is_pinned() const {
+  return Atomic::load(&_pinned);
+}
+
+inline void ZForwarding::set_pinned() {
+  Atomic::store(true, &_pinned);
+}
+
 inline bool ZForwarding::inc_refcount() {
   uint32_t refcount = Atomic::load(&_refcount);
 
@@ -60,12 +78,15 @@ inline bool ZForwarding::dec_refcount() {
   return Atomic::sub(1u, &_refcount) == 0u;
 }
 
-inline bool ZForwarding::is_pinned() const {
+inline bool ZForwarding::retain_page() {
-  return Atomic::load(&_pinned);
+  return inc_refcount();
 }
 
-inline void ZForwarding::set_pinned() {
+inline void ZForwarding::release_page() {
-  Atomic::store(true, &_pinned);
+  if (dec_refcount()) {
+    ZHeap::heap()->free_page(_page, true /* reclaimed */);
+    _page = NULL;
+  }
 }
 
 inline ZForwardingEntry* ZForwarding::entries() const {

src/hotspot/share/gc/z/zForwardingTable.cpp

@@ -22,7 +22,8 @@
  */
 
 #include "precompiled.hpp"
-#include "gc/z/zForwarding.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zForwarding.inline.hpp"
 #include "gc/z/zForwardingTable.inline.hpp"
 #include "gc/z/zGranuleMap.inline.hpp"
 #include "utilities/debug.hpp"
@@ -30,41 +31,18 @@
 ZForwardingTable::ZForwardingTable() :
     _map() {}
 
-void ZForwardingTable::insert(uintptr_t start,
+void ZForwardingTable::insert(ZForwarding* forwarding) {
-                              size_t size,
+  const uintptr_t addr = ZAddress::good(forwarding->start());
-                              size_t object_alignment_shift,
+  const size_t size = forwarding->size();
-                              uint32_t live_objects) {
-  // Allocate forwarding
-  ZForwarding* const forwarding = ZForwarding::create(start,
-                                                      object_alignment_shift,
-                                                      live_objects);
 
-  // Insert into forwarding table
-  const uintptr_t addr = ZAddress::good(start);
   assert(get(addr) == NULL, "Invalid entry");
   _map.put(addr, size, forwarding);
 }
 
-void ZForwardingTable::clear() {
+void ZForwardingTable::remove(ZForwarding* forwarding) {
-  ZForwarding* prev_forwarding = NULL;
+  const uintptr_t addr = ZAddress::good(forwarding->start());
+  const size_t size = forwarding->size();
 
-  // Clear and destroy all non-NULL entries
+  assert(get(addr) == forwarding, "Invalid entry");
-  ZGranuleMapIterator<ZForwarding*> iter(&_map);
+  _map.put(addr, size, NULL);
-  for (ZForwarding** entry; iter.next(&entry);) {
-    ZForwarding* const forwarding = *entry;
-    if (forwarding == NULL) {
-      // Skip entry
-      continue;
-    }
-
-    // Clear entry
-    *entry = NULL;
-
-    // More than one entry can point to the same
-    // forwarding. Make sure we only destroy it once.
-    if (forwarding != prev_forwarding) {
-      ZForwarding::destroy(forwarding);
-      prev_forwarding = forwarding;
-    }
-  }
 }

src/hotspot/share/gc/z/zForwardingTable.hpp

@@ -37,11 +37,8 @@ public:
 
   ZForwarding* get(uintptr_t addr) const;
 
-  void insert(uintptr_t start,
+  void insert(ZForwarding* forwarding);
-              size_t size,
+  void remove(ZForwarding* forwarding);
-              size_t object_alignment_shift,
-              uint32_t live_objects);
-  void clear();
 };
 
 #endif // SHARE_GC_Z_ZFORWARDINGTABLE_HPP

src/hotspot/share/gc/z/zHeap.cpp

@@ -417,8 +417,8 @@ void ZHeap::destroy_detached_pages() {
 void ZHeap::select_relocation_set() {
   // Register relocatable pages with selector
   ZRelocationSetSelector selector;
-  ZPageTableIterator iter(&_pagetable);
+  ZPageTableIterator pt_iter(&_pagetable);
-  for (ZPage* page; iter.next(&page);) {
+  for (ZPage* page; pt_iter.next(&page);) {
     if (!page->is_relocatable()) {
       // Not relocatable, don't register
       continue;
@@ -439,6 +439,12 @@ void ZHeap::select_relocation_set() {
   // Select pages to relocate
   selector.select(&_relocation_set);
 
+  // Setup forwarding table
+  ZRelocationSetIterator rs_iter(&_relocation_set);
+  for (ZForwarding* forwarding; rs_iter.next(&forwarding);) {
+    _forwarding_table.insert(forwarding);
+  }
+
   // Update statistics
   ZStatRelocation::set_at_select_relocation_set(selector.relocating());
   ZStatHeap::set_at_select_relocation_set(selector.live(),
@@ -446,20 +452,15 @@ void ZHeap::select_relocation_set() {
                                           reclaimed());
 }
 
-void ZHeap::prepare_relocation_set() {
-  ZRelocationSetIterator iter(&_relocation_set);
-  for (ZPage* page; iter.next(&page);) {
-    // Setup forwarding for page
-    _forwarding_table.insert(page->start(),
-                             page->size(),
-                             page->object_alignment_shift(),
-                             page->live_objects());
-  }
-}
-
 void ZHeap::reset_relocation_set() {
-  // Clear forwarding table
+  // Reset forwarding table
-  _forwarding_table.clear();
+  ZRelocationSetIterator iter(&_relocation_set);
+  for (ZForwarding* forwarding; iter.next(&forwarding);) {
+    _forwarding_table.remove(forwarding);
+  }
+
+  // Reset relocation set
+  _relocation_set.reset();
 }
 
 void ZHeap::relocate_start() {

src/hotspot/share/gc/z/zHeap.hpp

@@ -149,11 +149,9 @@ public:
 
   // Relocation set
   void select_relocation_set();
-  void prepare_relocation_set();
   void reset_relocation_set();
 
   // Relocation
-  ZForwarding* forwarding(uintptr_t addr);
   void relocate_start();
   uintptr_t relocate_object(uintptr_t addr);
   uintptr_t remap_object(uintptr_t addr);

src/hotspot/share/gc/z/zHeap.inline.hpp

@@ -44,10 +44,6 @@ inline ReferenceDiscoverer* ZHeap::reference_discoverer() {
   return &_reference_processor;
 }
 
-inline ZForwarding* ZHeap::forwarding(uintptr_t addr) {
-  return _forwarding_table.get(addr);
-}
-
 inline bool ZHeap::is_object_live(uintptr_t addr) const {
   ZPage* page = _pagetable.get(addr);
   return page->is_object_live(addr);
@@ -101,11 +97,10 @@ inline uintptr_t ZHeap::relocate_object(uintptr_t addr) {
   }
 
   // Relocate object
-  const bool retained = forwarding->inc_refcount();
+  const bool retained = forwarding->retain_page();
   const uintptr_t new_addr = _relocate.relocate_object(forwarding, addr);
-  if (retained && forwarding->dec_refcount()) {
+  if (retained) {
-    ZPage* const page = _pagetable.get(addr);
+    forwarding->release_page();
-    free_page(page, true /* reclaimed */);
   }
 
   return new_addr;

src/hotspot/share/gc/z/zRelocate.cpp

@@ -83,11 +83,6 @@ void ZRelocate::start() {
   _workers->run_parallel(&task);
 }
 
-ZForwarding* ZRelocate::forwarding_for_page(ZPage* page) const {
-  const uintptr_t addr = ZAddress::good(page->start());
-  return ZHeap::heap()->forwarding(addr);
-}
-
 uintptr_t ZRelocate::relocate_object_inner(ZForwarding* forwarding, uintptr_t from_index, uintptr_t from_offset) const {
   ZForwardingCursor cursor;
 
@@ -178,14 +173,13 @@ bool ZRelocate::work(ZRelocationSetParallelIterator* iter) {
   bool success = true;
 
   // Relocate pages in the relocation set
-  for (ZPage* page; iter->next(&page);) {
+  for (ZForwarding* forwarding; iter->next(&forwarding);) {
     // Relocate objects in page
-    ZForwarding* const forwarding = forwarding_for_page(page);
     ZRelocateObjectClosure cl(this, forwarding);
-    page->object_iterate(&cl);
+    forwarding->page()->object_iterate(&cl);
 
     if (ZVerifyForwarding) {
-      forwarding->verify(page->object_max_count(), page->live_objects());
+      forwarding->verify();
     }
 
     if (forwarding->is_pinned()) {
@@ -193,9 +187,7 @@ bool ZRelocate::work(ZRelocationSetParallelIterator* iter) {
       success = false;
     } else {
       // Relocation succeeded, release page
-      if (forwarding->dec_refcount()) {
+      forwarding->release_page();
-        ZHeap::heap()->free_page(page, true /* reclaimed */);
-      }
     }
   }
 

src/hotspot/share/gc/z/zRelocationSet.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,24 +22,35 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/z/zForwarding.hpp"
 #include "gc/z/zRelocationSet.hpp"
-#include "memory/allocation.inline.hpp"
+#include "memory/allocation.hpp"
 
 ZRelocationSet::ZRelocationSet() :
-    _pages(NULL),
+    _forwardings(NULL),
-    _npages(0) {}
+    _nforwardings(0) {}
 
-void ZRelocationSet::populate(const ZPage* const* group0, size_t ngroup0,
+void ZRelocationSet::populate(ZPage* const* group0, size_t ngroup0,
-                              const ZPage* const* group1, size_t ngroup1) {
+                              ZPage* const* group1, size_t ngroup1) {
-  _npages = ngroup0 + ngroup1;
+  _nforwardings = ngroup0 + ngroup1;
-  _pages = REALLOC_C_HEAP_ARRAY(ZPage*, _pages, _npages, mtGC);
+  _forwardings = REALLOC_C_HEAP_ARRAY(ZForwarding*, _forwardings, _nforwardings, mtGC);
 
-  if (_pages != NULL) {
+  size_t j = 0;
-    if (group0 != NULL) {
+
-      memcpy(_pages, group0, ngroup0 * sizeof(ZPage*));
+  // Populate group 0
-    }
+  for (size_t i = 0; i < ngroup0; i++) {
-    if (group1 != NULL) {
+    _forwardings[j++] = ZForwarding::create(group0[i]);
-      memcpy(_pages + ngroup0, group1, ngroup1 * sizeof(ZPage*));
+  }
-    }
+
+  // Populate group 1
+  for (size_t i = 0; i < ngroup1; i++) {
+    _forwardings[j++] = ZForwarding::create(group1[i]);
+  }
+}
+
+void ZRelocationSet::reset() {
+  for (size_t i = 0; i < _nforwardings; i++) {
+    ZForwarding::destroy(_forwardings[i]);
+    _forwardings[i] = NULL;
   }
 }

src/hotspot/share/gc/z/zRelocationSet.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,20 +26,22 @@
 
 #include "memory/allocation.hpp"
 
+class ZForwarding;
 class ZPage;
 
 class ZRelocationSet {
   template <bool> friend class ZRelocationSetIteratorImpl;
 
 private:
-  ZPage** _pages;
+  ZForwarding** _forwardings;
-  size_t  _npages;
+  size_t        _nforwardings;
 
 public:
   ZRelocationSet();
 
-  void populate(const ZPage* const* group0, size_t ngroup0,
+  void populate(ZPage* const* group0, size_t ngroup0,
-                const ZPage* const* group1, size_t ngroup1);
+                ZPage* const* group1, size_t ngroup1);
+  void reset();
 };
 
 template <bool parallel>
@@ -51,7 +53,7 @@ private:
 public:
   ZRelocationSetIteratorImpl(ZRelocationSet* relocation_set);
 
-  bool next(ZPage** page);
+  bool next(ZForwarding** forwarding);
 };
 
 // Iterator types

src/hotspot/share/gc/z/zRelocationSet.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,20 +33,20 @@ inline ZRelocationSetIteratorImpl<parallel>::ZRelocationSetIteratorImpl(ZRelocat
     _next(0) {}
 
 template <bool parallel>
-inline bool ZRelocationSetIteratorImpl<parallel>::next(ZPage** page) {
+inline bool ZRelocationSetIteratorImpl<parallel>::next(ZForwarding** forwarding) {
-  const size_t npages = _relocation_set->_npages;
+  const size_t nforwardings = _relocation_set->_nforwardings;
 
   if (parallel) {
-    if (_next < npages) {
+    if (_next < nforwardings) {
       const size_t next = Atomic::add(1u, &_next) - 1u;
-      if (next < npages) {
+      if (next < nforwardings) {
-        *page = _relocation_set->_pages[next];
+        *forwarding = _relocation_set->_forwardings[next];
         return true;
       }
     }
   } else {
-    if (_next < npages) {
+    if (_next < nforwardings) {
-      *page = _relocation_set->_pages[_next++];
+      *forwarding = _relocation_set->_forwardings[_next++];
       return true;
     }
   }

src/hotspot/share/gc/z/zRelocationSetSelector.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -44,10 +44,10 @@ ZRelocationSetSelectorGroup::ZRelocationSetSelectorGroup(const char* name,
     _fragmentation(0) {}
 
 ZRelocationSetSelectorGroup::~ZRelocationSetSelectorGroup() {
-  FREE_C_HEAP_ARRAY(const ZPage*, _sorted_pages);
+  FREE_C_HEAP_ARRAY(ZPage*, _sorted_pages);
 }
 
-void ZRelocationSetSelectorGroup::register_live_page(const ZPage* page, size_t garbage) {
+void ZRelocationSetSelectorGroup::register_live_page(ZPage* page, size_t garbage) {
   if (garbage > _fragmentation_limit) {
     _registered_pages.add(page);
   } else {
@@ -67,13 +67,13 @@ void ZRelocationSetSelectorGroup::semi_sort() {
   size_t partitions[npartitions];
 
   // Allocate destination array
-  _sorted_pages = REALLOC_C_HEAP_ARRAY(const ZPage*, _sorted_pages, npages, mtGC);
+  _sorted_pages = REALLOC_C_HEAP_ARRAY(ZPage*, _sorted_pages, npages, mtGC);
   debug_only(memset(_sorted_pages, 0, npages * sizeof(ZPage*)));
 
   // Calculate partition slots
   memset(partitions, 0, sizeof(partitions));
-  ZArrayIterator<const ZPage*> iter1(&_registered_pages);
+  ZArrayIterator<ZPage*> iter1(&_registered_pages);
-  for (const ZPage* page; iter1.next(&page);) {
+  for (ZPage* page; iter1.next(&page);) {
     const size_t index = page->live_bytes() >> partition_size_shift;
     partitions[index]++;
   }
@@ -87,8 +87,8 @@ void ZRelocationSetSelectorGroup::semi_sort() {
   }
 
   // Sort pages into partitions
-  ZArrayIterator<const ZPage*> iter2(&_registered_pages);
+  ZArrayIterator<ZPage*> iter2(&_registered_pages);
-  for (const ZPage* page; iter2.next(&page);) {
+  for (ZPage* page; iter2.next(&page);) {
     const size_t index = page->live_bytes() >> partition_size_shift;
     const size_t finger = partitions[index]++;
     assert(_sorted_pages[finger] == NULL, "Invalid finger");
@@ -140,7 +140,7 @@ void ZRelocationSetSelectorGroup::select() {
   // Update statistics
   _relocating = from_size;
   for (size_t i = _nselected; i < npages; i++) {
-    const ZPage* const page = _sorted_pages[i];
+    ZPage* const page = _sorted_pages[i];
     _fragmentation += page->size() - page->live_bytes();
   }
 
@@ -148,7 +148,7 @@ void ZRelocationSetSelectorGroup::select() {
                        _name, selected_from, selected_to, npages - _nselected);
 }
 
-const ZPage* const* ZRelocationSetSelectorGroup::selected() const {
+ZPage* const* ZRelocationSetSelectorGroup::selected() const {
   return _sorted_pages;
 }
 
@@ -171,7 +171,7 @@ ZRelocationSetSelector::ZRelocationSetSelector() :
     _garbage(0),
     _fragmentation(0) {}
 
-void ZRelocationSetSelector::register_live_page(const ZPage* page) {
+void ZRelocationSetSelector::register_live_page(ZPage* page) {
   const uint8_t type = page->type();
   const size_t live = page->live_bytes();
   const size_t garbage = page->size() - live;
@@ -188,7 +188,7 @@ void ZRelocationSetSelector::register_live_page(const ZPage* page) {
   _garbage += garbage;
 }
 
-void ZRelocationSetSelector::register_garbage_page(const ZPage* page) {
+void ZRelocationSetSelector::register_garbage_page(ZPage* page) {
   _garbage += page->size();
 }
 

src/hotspot/share/gc/z/zRelocationSetSelector.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,16 +32,16 @@ class ZRelocationSet;
 
 class ZRelocationSetSelectorGroup {
 private:
-  const char* const    _name;
+  const char* const _name;
-  const size_t         _page_size;
+  const size_t      _page_size;
-  const size_t         _object_size_limit;
+  const size_t      _object_size_limit;
-  const size_t         _fragmentation_limit;
+  const size_t      _fragmentation_limit;
 
-  ZArray<const ZPage*> _registered_pages;
+  ZArray<ZPage*>    _registered_pages;
-  const ZPage**        _sorted_pages;
+  ZPage**           _sorted_pages;
-  size_t               _nselected;
+  size_t            _nselected;
-  size_t               _relocating;
+  size_t            _relocating;
-  size_t               _fragmentation;
+  size_t            _fragmentation;
 
   void semi_sort();
 
@@ -51,10 +51,10 @@ public:
                               size_t object_size_limit);
   ~ZRelocationSetSelectorGroup();
 
-  void register_live_page(const ZPage* page, size_t garbage);
+  void register_live_page(ZPage* page, size_t garbage);
   void select();
 
-  const ZPage* const* selected() const;
+  ZPage* const* selected() const;
   size_t nselected() const;
   size_t relocating() const;
   size_t fragmentation() const;
@@ -71,8 +71,8 @@ private:
 public:
   ZRelocationSetSelector();
 
-  void register_live_page(const ZPage* page);
+  void register_live_page(ZPage* page);
-  void register_garbage_page(const ZPage* page);
+  void register_garbage_page(ZPage* page);
   void select(ZRelocationSet* relocation_set);
 
   size_t live() const;

test/hotspot/gtest/gc/z/test_zForwarding.cpp

@@ -22,7 +22,10 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zForwarding.inline.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "gc/z/zPage.inline.hpp"
 #include "unittest.hpp"
 
 using namespace testing;
@@ -141,16 +144,37 @@ public:
   }
 
   static void test(void (*function)(ZForwarding*), uint32_t size) {
-    // Setup
+    // Create page
-    ZForwarding* forwarding = ZForwarding::create(0 /* start */,
+    const ZVirtualMemory vmem(0, ZPageSizeSmall);
-                                                  0 /* object_alignment_shift */,
+    const ZPhysicalMemory pmem(ZPhysicalMemorySegment(0, ZPageSizeSmall));
-                                                  size);
+    ZPage page(ZPageTypeSmall, vmem, pmem);
+
+    page.reset();
+
+    const size_t object_size = 16;
+    const uintptr_t object = page.alloc_object(object_size);
+
+    ZGlobalSeqNum++;
+
+    bool dummy = false;
+    page.mark_object(ZAddress::marked(object), dummy, dummy);
+
+    const uint32_t live_objects = size;
+    const uint32_t live_bytes = live_objects * object_size;
+    page.inc_live_atomic(live_objects, live_bytes);
+
+    // Setup forwarding
+    ZForwarding* const forwarding = ZForwarding::create(&page);
 
     // Actual test function
     (*function)(forwarding);
 
-    // Teardown
+    // Teardown forwarding
     ZForwarding::destroy(forwarding);
+
+    // Teardown page
+    page.physical_memory().clear();
+    page.set_inactive();
   }
 
   // Run the given function with a few different input values.