diff --git a/src/hotspot/share/cds/archiveHeapLoader.cpp b/src/hotspot/share/cds/archiveHeapLoader.cpp index aa84ce15407..ce5eb2b6e75 100644 --- a/src/hotspot/share/cds/archiveHeapLoader.cpp +++ b/src/hotspot/share/cds/archiveHeapLoader.cpp @@ -24,7 +24,6 @@ #include "precompiled.hpp" #include "cds/archiveHeapLoader.inline.hpp" -#include "cds/filemap.hpp" #include "cds/heapShared.hpp" #include "cds/metaspaceShared.hpp" #include "classfile/classLoaderDataShared.hpp" @@ -114,12 +113,35 @@ class PatchCompressedEmbeddedPointers: public BitMapClosure { narrowOop* p = _start + offset; narrowOop v = *p; assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time"); - oop o = ArchiveHeapLoader::decode_from_archive(v); + oop o = ArchiveHeapLoader::decode_from_mapped_archive(v); RawAccess::oop_store(p, o); return true; } }; +class PatchCompressedEmbeddedPointersQuick: public BitMapClosure { + narrowOop* _start; + uint32_t _delta; + + public: + PatchCompressedEmbeddedPointersQuick(narrowOop* start, uint32_t delta) : _start(start), _delta(delta) {} + + bool do_bit(size_t offset) { + narrowOop* p = _start + offset; + narrowOop v = *p; + assert(!CompressedOops::is_null(v), "null oops should have been filtered out at dump time"); + narrowOop new_v = CompressedOops::narrow_oop_cast(CompressedOops::narrow_oop_value(v) + _delta); + assert(!CompressedOops::is_null(new_v), "should never relocate to narrowOop(0)"); +#ifdef ASSERT + oop o1 = ArchiveHeapLoader::decode_from_mapped_archive(v); + oop o2 = CompressedOops::decode_not_null(new_v); + assert(o1 == o2, "quick delta must work"); +#endif + RawAccess::oop_store(p, new_v); + return true; + } +}; + class PatchUncompressedEmbeddedPointers: public BitMapClosure { oop* _start; @@ -136,9 +158,38 @@ class PatchUncompressedEmbeddedPointers: public BitMapClosure { } }; +void ArchiveHeapLoader::patch_compressed_embedded_pointers(BitMapView bm, + FileMapInfo* info, + FileMapRegion* map_region, + MemRegion region) { + narrowOop dt_encoded_bottom = info->encoded_heap_region_dumptime_address(map_region); + narrowOop rt_encoded_bottom = CompressedOops::encode_not_null(cast_to_oop(region.start())); + log_info(cds)("patching heap embedded pointers: narrowOop 0x%8x -> 0x%8x", + (uint)dt_encoded_bottom, (uint)rt_encoded_bottom); + + // Optimization: if dumptime shift is the same as runtime shift, we can perform a + // quick conversion from "dumptime narrowOop" -> "runtime narrowOop". + if (_narrow_oop_shift == CompressedOops::shift()) { + uint32_t quick_delta = (uint32_t)rt_encoded_bottom - (uint32_t)dt_encoded_bottom; + log_info(cds)("CDS heap data relocation quick delta = 0x%x", quick_delta); + if (quick_delta == 0) { + log_info(cds)("CDS heap data relocation unnecessary, quick_delta = 0"); + } else { + PatchCompressedEmbeddedPointersQuick patcher((narrowOop*)region.start(), quick_delta); + bm.iterate(&patcher); + } + } else { + log_info(cds)("CDS heap data quick relocation not possible"); + PatchCompressedEmbeddedPointers patcher((narrowOop*)region.start()); + bm.iterate(&patcher); + } +} + // Patch all the non-null pointers that are embedded in the archived heap objects // in this (mapped) region -void ArchiveHeapLoader::patch_embedded_pointers(MemRegion region, address oopmap, +void ArchiveHeapLoader::patch_embedded_pointers(FileMapInfo* info, + FileMapRegion* map_region, + MemRegion region, address oopmap, size_t oopmap_size_in_bits) { BitMapView bm((BitMap::bm_word_t*)oopmap, oopmap_size_in_bits); @@ -149,8 +200,7 @@ void ArchiveHeapLoader::patch_embedded_pointers(MemRegion region, address oopmap #endif if (UseCompressedOops) { - PatchCompressedEmbeddedPointers patcher((narrowOop*)region.start()); - bm.iterate(&patcher); + patch_compressed_embedded_pointers(bm, info, map_region, region); } else { PatchUncompressedEmbeddedPointers patcher((oop*)region.start()); bm.iterate(&patcher); diff --git a/src/hotspot/share/cds/archiveHeapLoader.hpp b/src/hotspot/share/cds/archiveHeapLoader.hpp index 97e0edf6314..754893cb4fa 100644 --- a/src/hotspot/share/cds/archiveHeapLoader.hpp +++ b/src/hotspot/share/cds/archiveHeapLoader.hpp @@ -25,12 +25,14 @@ #ifndef SHARE_CDS_ARCHIVEHEAPLOADER_HPP #define SHARE_CDS_ARCHIVEHEAPLOADER_HPP +#include "cds/filemap.hpp" #include "gc/shared/gc_globals.hpp" #include "memory/allocation.hpp" #include "memory/allStatic.hpp" #include "memory/memRegion.hpp" #include "oops/oopsHierarchy.hpp" #include "runtime/globals.hpp" +#include "utilities/bitMap.hpp" #include "utilities/macros.hpp" class FileMapInfo; @@ -103,8 +105,18 @@ public: // function instead. inline static oop decode_from_archive(narrowOop v) NOT_CDS_JAVA_HEAP_RETURN_(NULL); - static void patch_embedded_pointers(MemRegion region, address oopmap, - size_t oopmap_in_bits) NOT_CDS_JAVA_HEAP_RETURN; + // More efficient version, but works only when ArchiveHeap is mapped. + inline static oop decode_from_mapped_archive(narrowOop v) NOT_CDS_JAVA_HEAP_RETURN_(NULL); + + static void patch_compressed_embedded_pointers(BitMapView bm, + FileMapInfo* info, + FileMapRegion* map_region, + MemRegion region) NOT_CDS_JAVA_HEAP_RETURN; + + static void patch_embedded_pointers(FileMapInfo* info, + FileMapRegion* map_region, + MemRegion region, address oopmap, + size_t oopmap_size_in_bits) NOT_CDS_JAVA_HEAP_RETURN; static void fixup_regions() NOT_CDS_JAVA_HEAP_RETURN; @@ -159,6 +171,9 @@ private: return (_loaded_heap_bottom <= o && o < _loaded_heap_top); } + template + inline static oop decode_from_archive_impl(narrowOop v) NOT_CDS_JAVA_HEAP_RETURN_(NULL); + public: static bool load_heap_regions(FileMapInfo* mapinfo); diff --git a/src/hotspot/share/cds/archiveHeapLoader.inline.hpp b/src/hotspot/share/cds/archiveHeapLoader.inline.hpp index 34ac6e15a1a..6f344ddf526 100644 --- a/src/hotspot/share/cds/archiveHeapLoader.inline.hpp +++ b/src/hotspot/share/cds/archiveHeapLoader.inline.hpp @@ -32,11 +32,14 @@ #if INCLUDE_CDS_JAVA_HEAP -inline oop ArchiveHeapLoader::decode_from_archive(narrowOop v) { +template +inline oop ArchiveHeapLoader::decode_from_archive_impl(narrowOop v) { assert(!CompressedOops::is_null(v), "narrow oop value can never be zero"); assert(_narrow_oop_base_initialized, "relocation information must have been initialized"); uintptr_t p = ((uintptr_t)_narrow_oop_base) + ((uintptr_t)v << _narrow_oop_shift); - if (p >= _dumptime_base_0) { + if (IS_MAPPED) { + assert(_dumptime_base_0 == UINTPTR_MAX, "must be"); + } else if (p >= _dumptime_base_0) { assert(p < _dumptime_top, "must be"); if (p >= _dumptime_base_3) { p += _runtime_offset_3; @@ -54,6 +57,14 @@ inline oop ArchiveHeapLoader::decode_from_archive(narrowOop v) { return result; } +inline oop ArchiveHeapLoader::decode_from_archive(narrowOop v) { + return decode_from_archive_impl(v); +} + +inline oop ArchiveHeapLoader::decode_from_mapped_archive(narrowOop v) { + return decode_from_archive_impl(v); +} + #endif #endif // SHARE_CDS_ARCHIVEHEAPLOADER_INLINE_HPP diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp index afc45424f7a..af79648a836 100644 --- a/src/hotspot/share/cds/filemap.cpp +++ b/src/hotspot/share/cds/filemap.cpp @@ -2447,6 +2447,13 @@ void FileMapInfo::patch_heap_embedded_pointers() { MetaspaceShared::first_open_heap_region); } +narrowOop FileMapInfo::encoded_heap_region_dumptime_address(FileMapRegion* r) { + assert(UseSharedSpaces, "runtime only"); + assert(UseCompressedOops, "sanity"); + r->assert_is_heap_region(); + return CompressedOops::narrow_oop_cast(r->mapping_offset() >> narrow_oop_shift()); +} + void FileMapInfo::patch_heap_embedded_pointers(MemRegion* regions, int num_regions, int first_region_idx) { char* bitmap_base = map_bitmap_region(); @@ -2454,24 +2461,9 @@ void FileMapInfo::patch_heap_embedded_pointers(MemRegion* regions, int num_regio for (int i=0; imapping_offset() >> narrow_oop_shift()); - narrowOop rt_encoded_bottom = CompressedOops::encode_not_null(cast_to_oop(regions[i].start())); - log_info(cds)("patching heap embedded pointers for %s: narrowOop 0x%8x -> 0x%8x", - region_name(region_idx), (uint)dt_encoded_bottom, (uint)rt_encoded_bottom); - // TODO JDK-8269736: if we have the same narrow_oop_shift between dumptime and runtime, - // Each embedded pointer P can be updated by: - // P += (rt_encoded_bottom - dt_encoded_bottom) - // - // TODO: - // if (dt_encoded_bottom == rt_encoded_bottom && narrow_oop_shift() == CompressedOops::shift()) { - // //nothing to do - // return; - // } - } + ArchiveHeapLoader::patch_embedded_pointers( - regions[i], + this, r, regions[i], (address)(region_at(MetaspaceShared::bm)->mapped_base()) + r->oopmap_offset(), r->oopmap_size_in_bits()); } diff --git a/src/hotspot/share/cds/filemap.hpp b/src/hotspot/share/cds/filemap.hpp index c71184b82a6..541b8fd603a 100644 --- a/src/hotspot/share/cds/filemap.hpp +++ b/src/hotspot/share/cds/filemap.hpp @@ -593,6 +593,7 @@ public: address heap_region_dumptime_address(FileMapRegion* r) NOT_CDS_JAVA_HEAP_RETURN_(NULL); address heap_region_requested_address(FileMapRegion* r) NOT_CDS_JAVA_HEAP_RETURN_(NULL); address heap_region_mapped_address(FileMapRegion* r) NOT_CDS_JAVA_HEAP_RETURN_(NULL); + narrowOop encoded_heap_region_dumptime_address(FileMapRegion* r); private: diff --git a/test/hotspot/jtreg/runtime/cds/appcds/cacheObject/DifferentHeapSizes.java b/test/hotspot/jtreg/runtime/cds/appcds/cacheObject/DifferentHeapSizes.java index 16b46c505bd..4107a9508e4 100644 --- a/test/hotspot/jtreg/runtime/cds/appcds/cacheObject/DifferentHeapSizes.java +++ b/test/hotspot/jtreg/runtime/cds/appcds/cacheObject/DifferentHeapSizes.java @@ -96,6 +96,8 @@ public class DifferentHeapSizes { /* dump xmx */ /* run xmx */ /* dump base */ /* run base */ 128 * M, 128 * M, default_base, default_base + 256L * 1024 * 1024, 128 * M, 16376 * M, 0x0000000119200000L, -1, + 16 * M * 1024, 6 * M * 1024, default_base, 0x90000000, + 128 * M, 128 * M, 0xf0000000, 0xe0000000 }; for (int i = 0; i < bases.length; i += 4) {