8220586: ZGC: Move relocation logic from ZPage to ZRelocate

Reviewed-by: stefank, eosterlund

src/hotspot/share/gc/z/zHeap.cpp

@@ -497,7 +497,7 @@ uintptr_t ZHeap::relocate_object(uintptr_t addr) {
   assert(ZGlobalPhase == ZPhaseRelocate, "Relocate not allowed");
   ZPage* const page = _pagetable.get(addr);
   const bool retained = retain_page(page);
-  const uintptr_t new_addr = page->relocate_object(addr);
+  const uintptr_t new_addr = _relocate.relocate_object(page, addr);
   if (retained) {
     release_page(page, true /* reclaimed */);
   }
@@ -509,7 +509,7 @@ uintptr_t ZHeap::forward_object(uintptr_t addr) {
   assert(ZGlobalPhase == ZPhaseMark ||
          ZGlobalPhase == ZPhaseMarkCompleted, "Forward not allowed");
   ZPage* const page = _pagetable.get(addr);
-  return page->forward_object(addr);
+  return _relocate.forward_object(page, addr);
 }
 
 void ZHeap::relocate() {

src/hotspot/share/gc/z/zPage.cpp

@@ -22,24 +22,12 @@
  */
 
 #include "precompiled.hpp"
-#include "gc/shared/collectedHeap.hpp"
-#include "gc/z/zAddress.inline.hpp"
-#include "gc/z/zForwardingTable.inline.hpp"
-#include "gc/z/zHeap.inline.hpp"
-#include "gc/z/zLiveMap.inline.hpp"
-#include "gc/z/zMark.hpp"
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zPhysicalMemory.inline.hpp"
-#include "gc/z/zStat.hpp"
-#include "gc/z/zThread.hpp"
-#include "gc/z/zUtils.inline.hpp"
-#include "logging/log.hpp"
+#include "gc/z/zVirtualMemory.inline.hpp"
 #include "runtime/orderAccess.hpp"
 #include "utilities/align.hpp"
 #include "utilities/debug.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-static const ZStatCounter ZCounterRelocationContention("Contention", "Relocation Contention", ZStatUnitOpsPerSecond);
 
 ZPage::ZPage(uint8_t type, ZVirtualMemory vmem, ZPhysicalMemory pmem) :
     _type(type),
@@ -81,82 +69,6 @@ void ZPage::reset() {
   _refcount = 1;
 }
 
-uintptr_t ZPage::relocate_object_inner(uintptr_t from_index, uintptr_t from_offset) {
-  ZForwardingTableCursor cursor;
-
-  // Lookup address in forwarding table
-  const ZForwardingTableEntry entry = _forwarding.find(from_index, &cursor);
-  if (entry.from_index() == from_index) {
-    // Already relocated, return new address
-    return entry.to_offset();
-  }
-
-  // Not found in forwarding table, relocate object
-  assert(is_object_marked(from_offset), "Should be marked");
-
-  if (is_pinned()) {
-    // In-place forward
-    return _forwarding.insert(from_index, from_offset, &cursor);
-  }
-
-  // Allocate object
-  const uintptr_t from_good = ZAddress::good(from_offset);
-  const size_t size = ZUtils::object_size(from_good);
-  const uintptr_t to_good = ZHeap::heap()->alloc_object_for_relocation(size);
-  if (to_good == 0) {
-    // Failed, in-place forward
-    return _forwarding.insert(from_index, from_offset, &cursor);
-  }
-
-  // Copy object
-  ZUtils::object_copy(from_good, to_good, size);
-
-  // Update forwarding table
-  const uintptr_t to_offset = ZAddress::offset(to_good);
-  const uintptr_t to_offset_final = _forwarding.insert(from_index, to_offset, &cursor);
-  if (to_offset_final == to_offset) {
-    // Relocation succeeded
-    return to_offset;
-  }
-
-  // Relocation contention
-  ZStatInc(ZCounterRelocationContention);
-  log_trace(gc)("Relocation contention, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT
-                ", entry: " SIZE_FORMAT ", oop: " PTR_FORMAT ", size: " SIZE_FORMAT,
-                ZThread::id(), ZThread::name(), p2i(this), cursor, from_good, size);
-
-  // Try undo allocation
-  ZHeap::heap()->undo_alloc_object_for_relocation(to_good, size);
-
-  return to_offset_final;
-}
-
-uintptr_t ZPage::relocate_object(uintptr_t from) {
-  assert(ZHeap::heap()->is_relocating(from), "Should be relocating");
-
-  const uintptr_t from_offset = ZAddress::offset(from);
-  const uintptr_t from_index = (from_offset - start()) >> object_alignment_shift();
-  const uintptr_t to_offset = relocate_object_inner(from_index, from_offset);
-  if (from_offset == to_offset) {
-    // In-place forwarding, pin page
-    set_pinned();
-  }
-
-  return ZAddress::good(to_offset);
-}
-
-uintptr_t ZPage::forward_object(uintptr_t from) {
-  assert(ZHeap::heap()->is_relocating(from), "Should be relocated");
-
-  // Lookup address in forwarding table
-  const uintptr_t from_offset = ZAddress::offset(from);
-  const uintptr_t from_index = (from_offset - start()) >> object_alignment_shift();
-  const ZForwardingTableEntry entry = _forwarding.find(from_index);
-  assert(entry.from_index() == from_index, "Should be forwarded");
-
-  return ZAddress::good(entry.to_offset());
-}
-
 void ZPage::print_on(outputStream* out) const {
   out->print_cr(" %-6s  " PTR_FORMAT " " PTR_FORMAT " " PTR_FORMAT " %s%s%s%s%s%s",
                 type_to_string(), start(), top(), end(),

src/hotspot/share/gc/z/zPage.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -53,7 +53,6 @@ private:
 
   const char* type_to_string() const;
   uint32_t object_max_count() const;
-  uintptr_t relocate_object_inner(uintptr_t from_index, uintptr_t from_offset);
 
   bool is_object_marked(uintptr_t addr) const;
   bool is_object_strongly_marked(uintptr_t addr) const;
@@ -101,6 +100,9 @@ public:
   bool is_forwarding() const;
   void set_forwarding();
   void reset_forwarding();
+  ZForwardingTableEntry find_forwarding(uintptr_t from_index);
+  ZForwardingTableEntry find_forwarding(uintptr_t from_index, ZForwardingTableCursor* cursor);
+  uintptr_t insert_forwarding(uintptr_t from_index, uintptr_t to_offset, ZForwardingTableCursor* cursor);
   void verify_forwarding() const;
 
   bool is_marked() const;
@@ -119,9 +121,6 @@ public:
   bool undo_alloc_object(uintptr_t addr, size_t size);
   bool undo_alloc_object_atomic(uintptr_t addr, size_t size);
 
-  uintptr_t relocate_object(uintptr_t from);
-  uintptr_t forward_object(uintptr_t from);
-
   void print_on(outputStream* out) const;
   void print() const;
 };

src/hotspot/share/gc/z/zPage.inline.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -212,6 +212,18 @@ inline void ZPage::reset_forwarding() {
   _pinned = 0;
 }
 
+inline ZForwardingTableEntry ZPage::find_forwarding(uintptr_t from_index) {
+  return _forwarding.find(from_index);
+}
+
+inline ZForwardingTableEntry ZPage::find_forwarding(uintptr_t from_index, ZForwardingTableCursor* cursor) {
+  return _forwarding.find(from_index, cursor);
+}
+
+inline uintptr_t ZPage::insert_forwarding(uintptr_t from_index, uintptr_t to_offset, ZForwardingTableCursor* cursor) {
+  return _forwarding.insert(from_index, to_offset, cursor);
+}
+
 inline void ZPage::verify_forwarding() const {
   _forwarding.verify(object_max_count(), _livemap.live_objects());
 }

src/hotspot/share/gc/z/zRelocate.cpp

@@ -30,9 +30,13 @@
 #include "gc/z/zRelocate.hpp"
 #include "gc/z/zRelocationSet.inline.hpp"
 #include "gc/z/zRootsIterator.hpp"
+#include "gc/z/zStat.hpp"
 #include "gc/z/zTask.hpp"
 #include "gc/z/zThreadLocalAllocBuffer.hpp"
 #include "gc/z/zWorkers.hpp"
+#include "logging/log.hpp"
+
+static const ZStatCounter ZCounterRelocationContention("Contention", "Relocation Contention", ZStatUnitOpsPerSecond);
 
 ZRelocate::ZRelocate(ZWorkers* workers) :
     _workers(workers) {}
@@ -78,16 +82,93 @@ void ZRelocate::start() {
   _workers->run_parallel(&task);
 }
 
+uintptr_t ZRelocate::relocate_object_inner(ZPage* page, uintptr_t from_index, uintptr_t from_offset) const {
+  ZForwardingTableCursor cursor;
+
+  // Lookup address in forwarding table
+  const ZForwardingTableEntry entry = page->find_forwarding(from_index, &cursor);
+  if (entry.from_index() == from_index) {
+    // Already relocated, return new address
+    return entry.to_offset();
+  }
+
+  assert(ZHeap::heap()->is_object_live(ZAddress::good(from_offset)), "Should be live");
+
+  if (page->is_pinned()) {
+    // In-place forward
+    return page->insert_forwarding(from_index, from_offset, &cursor);
+  }
+
+  // Allocate object
+  const uintptr_t from_good = ZAddress::good(from_offset);
+  const size_t size = ZUtils::object_size(from_good);
+  const uintptr_t to_good = ZHeap::heap()->alloc_object_for_relocation(size);
+  if (to_good == 0) {
+    // Failed, in-place forward
+    return page->insert_forwarding(from_index, from_offset, &cursor);
+  }
+
+  // Copy object
+  ZUtils::object_copy(from_good, to_good, size);
+
+  // Update forwarding table
+  const uintptr_t to_offset = ZAddress::offset(to_good);
+  const uintptr_t to_offset_final = page->insert_forwarding(from_index, to_offset, &cursor);
+  if (to_offset_final == to_offset) {
+    // Relocation succeeded
+    return to_offset;
+  }
+
+  // Relocation contention
+  ZStatInc(ZCounterRelocationContention);
+  log_trace(gc)("Relocation contention, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT
+                ", entry: " SIZE_FORMAT ", oop: " PTR_FORMAT ", size: " SIZE_FORMAT,
+                ZThread::id(), ZThread::name(), p2i(this), cursor, from_good, size);
+
+  // Try undo allocation
+  ZHeap::heap()->undo_alloc_object_for_relocation(to_good, size);
+
+  return to_offset_final;
+}
+
+uintptr_t ZRelocate::relocate_object(ZPage* page, uintptr_t from_addr) const {
+  assert(ZHeap::heap()->is_relocating(from_addr), "Should be relocating");
+
+  const uintptr_t from_offset = ZAddress::offset(from_addr);
+  const uintptr_t from_index = (from_offset - page->start()) >> page->object_alignment_shift();
+  const uintptr_t to_offset = relocate_object_inner(page, from_index, from_offset);
+  if (from_offset == to_offset) {
+    // In-place forwarding, pin page
+    page->set_pinned();
+  }
+
+  return ZAddress::good(to_offset);
+}
+
+uintptr_t ZRelocate::forward_object(ZPage* page, uintptr_t from_addr) const {
+  assert(ZHeap::heap()->is_relocating(from_addr), "Should be relocated");
+
+  // Lookup address in forwarding table
+  const uintptr_t from_offset = ZAddress::offset(from_addr);
+  const uintptr_t from_index = (from_offset - page->start()) >> page->object_alignment_shift();
+  const ZForwardingTableEntry entry = page->find_forwarding(from_index);
+  assert(entry.from_index() == from_index, "Should be forwarded");
+
+  return ZAddress::good(entry.to_offset());
+}
+
 class ZRelocateObjectClosure : public ObjectClosure {
 private:
-  ZPage* const _page;
+  ZRelocate* const _relocate;
+  ZPage* const     _page;
 
 public:
-  ZRelocateObjectClosure(ZPage* page) :
+  ZRelocateObjectClosure(ZRelocate* relocate, ZPage* page) :
+      _relocate(relocate),
       _page(page) {}
 
   virtual void do_object(oop o) {
-    _page->relocate_object(ZOop::to_address(o));
+    _relocate->relocate_object(_page, ZOop::to_address(o));
   }
 };
 
@@ -97,7 +178,7 @@ bool ZRelocate::work(ZRelocationSetParallelIterator* iter) {
   // Relocate pages in the relocation set
   for (ZPage* page; iter->next(&page);) {
     // Relocate objects in page
-    ZRelocateObjectClosure cl(page);
+    ZRelocateObjectClosure cl(this, page);
     page->object_iterate(&cl);
 
     if (ZVerifyForwarding) {

src/hotspot/share/gc/z/zRelocate.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,11 +34,15 @@ class ZRelocate {
 private:
   ZWorkers* const _workers;
 
+  uintptr_t relocate_object_inner(ZPage* from_page, uintptr_t from_index, uintptr_t from_offset) const;
   bool work(ZRelocationSetParallelIterator* iter);
 
 public:
   ZRelocate(ZWorkers* workers);
 
+  uintptr_t relocate_object(ZPage* from_page, uintptr_t from_addr) const;
+  uintptr_t forward_object(ZPage* from_page, uintptr_t from_addr) const;
+
   void start();
   bool relocate(ZRelocationSet* relocation_set);
 };